@rocicorp/zero 1.12.0-canary.3 → 1.12.0-canary.4

package/out/zero-cache/src/server/shadow-syncer.js.map

@@ -1 +1 @@
-{"version":3,"file":"shadow-syncer.js","names":[],"sources":["../../../../../zero-cache/src/server/shadow-syncer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {must} from '../../../shared/src/must.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {ShadowSyncService} from '../services/shadow-sync/shadow-sync-service.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\nconst MS_PER_HOUR = 1000 * 60 * 60;\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  const config = getNormalizedZeroConfig({env, argv});\n\n  startOtelAuto(\n    createLogContext(config, 'shadow-syncer', 0, false),\n    'shadow-syncer',\n    0,\n  );\n  lc = createLogContext(config, 'shadow-syncer');\n  initEventSink(lc, config);\n\n  const {shadowSync, upstream, initialSync} = config;\n  const shard = getShardConfig(config);\n  const service = new ShadowSyncService(\n    lc,\n    shard,\n    upstream.db,\n    getServerContext(config),\n    {\n      intervalMs: shadowSync.intervalHours * MS_PER_HOUR,\n      sampleRate: shadowSync.sampleRate,\n      maxRowsPerTable: shadowSync.maxRowsPerTable,\n      textCopy: initialSync.textCopy,\n    },\n  );\n\n  parent.send(['ready', {ready: true}]);\n\n  return runUntilKilled(lc, parent, service);\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;AAgBA,IAAM,cAAc,MAAO,KAAK;AAGhC,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,SAAwB,UACtB,QACA,KACA,GAAG,MACY;CACf,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAElD,cACE,iBAAiB,QAAQ,iBAAiB,GAAG,KAAK,GAClD,iBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,eAAe;CAC7C,cAAc,IAAI,MAAM;CAExB,MAAM,EAAC,YAAY,UAAU,gBAAe;CAC5C,MAAM,QAAQ,eAAe,MAAM;CACnC,MAAM,UAAU,IAAI,kBAClB,IACA,OACA,SAAS,IACT,iBAAiB,MAAM,GACvB;EACE,YAAY,WAAW,gBAAgB;EACvC,YAAY,WAAW;EACvB,iBAAiB,WAAW;EAC5B,UAAU,YAAY;CACxB,CACF;CAEA,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAEpC,OAAO,eAAe,IAAI,QAAQ,OAAO;AAC3C;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}
+{"version":3,"file":"shadow-syncer.js","names":[],"sources":["../../../../../zero-cache/src/server/shadow-syncer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {must} from '../../../shared/src/must.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {ShadowSyncService} from '../services/shadow-sync/shadow-sync-service.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\nconst MS_PER_HOUR = 1000 * 60 * 60;\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  const config = getNormalizedZeroConfig({env, argv});\n\n  startOtelAuto(\n    createLogContext(config, 'shadow-syncer', 0, false),\n    'shadow-syncer',\n    0,\n  );\n  lc = createLogContext(config, 'shadow-syncer');\n  initEventSink(lc, config);\n\n  const {shadowSync, upstream, initialSync} = config;\n  const shard = getShardConfig(config);\n  const service = new ShadowSyncService(\n    lc,\n    shard,\n    upstream.db,\n    getServerContext(config),\n    {\n      intervalMs: shadowSync.intervalHours * MS_PER_HOUR,\n      sampleRate: shadowSync.sampleRate,\n      maxRowsPerTable: shadowSync.maxRowsPerTable,\n      textCopy: initialSync.textCopy,\n    },\n  );\n\n  parent.send(['ready', {ready: true}]);\n\n  return runUntilKilled(lc, parent, service);\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;AAgBA,IAAM,cAAc;AAGpB,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,SAAwB,UACtB,QACA,KACA,GAAG,MACY;CACf,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAElD,cACE,iBAAiB,QAAQ,iBAAiB,GAAG,KAAK,GAClD,iBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,eAAe;CAC7C,cAAc,IAAI,MAAM;CAExB,MAAM,EAAC,YAAY,UAAU,gBAAe;CAC5C,MAAM,QAAQ,eAAe,MAAM;CACnC,MAAM,UAAU,IAAI,kBAClB,IACA,OACA,SAAS,IACT,iBAAiB,MAAM,GACvB;EACE,YAAY,WAAW,gBAAgB;EACvC,YAAY,WAAW;EACvB,iBAAiB,WAAW;EAC5B,UAAU,YAAY;CACxB,CACF;CAEA,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAEpC,OAAO,eAAe,IAAI,QAAQ,OAAO;AAC3C;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}

package/out/zero-cache/src/server/slot-keeper.js.map

@@ -1 +1 @@
-{"version":3,"file":"slot-keeper.js","names":[],"sources":["../../../../../zero-cache/src/server/slot-keeper.ts"],"sourcesContent":["import {resolve} from 'node:path';\nimport type {Readable} from 'node:stream';\nimport {fileURLToPath} from 'node:url';\nimport {PG_ADMIN_SHUTDOWN} from '@drdgvhbh/postgres-error-codes';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport {getLogConfig} from '../config/zero-config.ts';\nimport {\n  DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE,\n  makeAck,\n  startReplicationStream,\n  type PgConnectionConfig,\n} from '../services/change-source/pg/logical-replication/stream.ts';\nimport {toBigInt} from '../services/change-source/pg/lsn.ts';\nimport {exitAfter} from '../services/life-cycle.ts';\nimport {inactivityTimeoutSocket, isPostgresError} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {createLogContext} from './logging.ts';\n\nexport type SlotKeeperConfig = {\n  db: PgConnectionConfig;\n  slot: string;\n  dummyPublication: string;\n  lsn: string;\n  ackIntervalMs?: number | undefined;\n};\n\nexport function parseLsn(lsn: string): bigint {\n  return lsn.includes('/') ? toBigInt(lsn) : BigInt(lsn);\n}\n\nexport function waitForStartConfig(parent: Worker): Promise<SlotKeeperConfig> {\n  return new Promise<SlotKeeperConfig>((resolve, reject) => {\n    const cleanup = () => {\n      parent.off('disconnect', onDisconnect);\n      parent.off('close', onClose);\n      if (process.send) {\n        process.off('disconnect', onDisconnect);\n      }\n    };\n    const onDisconnect = () => {\n      cleanup();\n      reject(new Error('Parent disconnected before sending start config'));\n    };\n    const onClose = () => {\n      cleanup();\n      reject(new Error('Parent closed before sending start config'));\n    };\n    parent.once('disconnect', onDisconnect);\n    parent.once('close', onClose);\n    if (process.send) {\n      process.once('disconnect', onDisconnect);\n    }\n    parent.onceMessageType<['start', SlotKeeperConfig]>('start', cfg => {\n      cleanup();\n      resolve(cfg);\n    });\n    parent.send(['init', {}]);\n  });\n}\n\nexport type SlotKeeperDependencies = {\n  createSession?: (\n    config: SlotKeeperConfig,\n    lc: LogContext,\n  ) => postgres.Sql | {end: (opts?: {timeout?: number}) => Promise<void>};\n  startStream?: (\n    lc: LogContext,\n    session: postgres.Sql,\n    slot: string,\n    publications: string[],\n    startLsn: bigint,\n    maxAttempts?: number,\n  ) => Promise<[Readable, NodeJS.WritableStream]>;\n  sleepFn?: (ms: number) => Promise<void>;\n  logContext?: LogContext;\n};\n\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  depsOrArg?: string | SlotKeeperDependencies,\n  ...argv: string[]\n): Promise<void> {\n  // Only logging config is needed; avoid requiring the full (normalized) config.\n  const logConfig = getLogConfig({env, argv});\n  const deps: SlotKeeperDependencies =\n    typeof depsOrArg === 'object' && depsOrArg !== null ? depsOrArg : {};\n  const createSession =\n    deps.createSession ??\n    ((cfg: SlotKeeperConfig, log: LogContext) =>\n      postgres(\n        defu(\n          {\n            max: 1,\n            fetch_types: false,\n            idle_timeout: null,\n            max_lifetime: null as unknown as number,\n            connection: {\n              application_name: `slot-keeper-${cfg.slot}`,\n              replication: 'database',\n            },\n            socket: inactivityTimeoutSocket(log, 0, 60_000),\n          },\n          cfg.db as Options<Record<string, PostgresType>>,\n        ),\n      ));\n  const startStream = deps.startStream ?? startReplicationStream;\n  const sleepFn = deps.sleepFn ?? sleep;\n\n  const config = await waitForStartConfig(parent);\n\n  lc = (\n    deps.logContext ?? createLogContext(logConfig, 'slot-keeper')\n  ).withContext('slot', config.slot);\n  lc.info?.(`starting slot-keeper for ${config.slot}`);\n\n  let stopped = false;\n  let stopResolve: (() => void) | null = null;\n  let currentReadable: Readable | null = null;\n  let currentSession:\n    | postgres.Sql\n    | {end: (opts?: {timeout?: number}) => Promise<void>}\n    | null = null;\n  const lsn = parseLsn(config.lsn);\n  const ackIntervalMs = config.ackIntervalMs ?? 5_000;\n\n  const closeCurrent = async () => {\n    if (currentReadable) {\n      try {\n        currentReadable.destroy();\n      } catch {}\n      currentReadable = null;\n    }\n    if (currentSession) {\n      try {\n        await currentSession.end({timeout: 0});\n        lc.info?.(`ended slot-keeper replication session for ${config.slot}`);\n      } catch (e) {\n        lc.warn?.(`error ending slot-keeper session`, e);\n      }\n      currentSession = null;\n    }\n  };\n\n  const stop = async () => {\n    if (stopped) {\n      return;\n    }\n    stopped = true;\n    lc.info?.(`stopping slot-keeper for ${config.slot}`);\n    stopResolve?.();\n    await closeCurrent();\n  };\n\n  parent.onceMessageType('stop', () => void stop());\n  parent.on('SIGTERM', () => void stop());\n  parent.on('SIGINT', () => void stop());\n  parent.once('disconnect', () => void stop());\n  parent.once('close', () => void stop());\n  process.once('SIGTERM', () => void stop());\n  process.once('SIGINT', () => void stop());\n  if (process.send) {\n    process.once('disconnect', () => void stop());\n  }\n\n  let firstAttempt = true;\n  let delayMs = 1000;\n\n  while (!stopped) {\n    try {\n      lc.info?.(`connecting replication session for slot ${config.slot}`);\n      const session = createSession(config, lc);\n      currentSession = session;\n\n      const [readable, writeable] = await startStream(\n        lc,\n        session as postgres.Sql,\n        config.slot,\n        [config.dummyPublication],\n        lsn,\n        DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE + 1,\n      );\n      currentReadable = readable;\n      // Discard the streamed messages. Otherwise the paused socket backs up the\n      // wal_sender (e.g. with keepalives and logical messages) for the duration\n      // of the initial sync / restore, and a wal_sender blocked on writing to the\n      // client does not exit (and release the slot) when it is terminated by the\n      // takeover; it instead blocks on sending the termination error to the\n      // client, until signaled again.\n      readable.resume();\n\n      if (stopped) {\n        await closeCurrent();\n        break;\n      }\n\n      lc.info?.(`slot-keeper active for ${config.slot} at lsn ${config.lsn}`);\n      if (firstAttempt) {\n        firstAttempt = false;\n        parent.send(['ready', {ready: true}]);\n      }\n\n      // Reset backoff delay on successful connection\n      delayMs = 1000;\n\n      const ackInterval = setInterval(() => {\n        if (stopped) {\n          clearInterval(ackInterval);\n          return;\n        }\n        try {\n          writeable.write(makeAck(lsn));\n        } catch (e) {\n          lc.warn?.(`error writing ACK to replication stream`, e);\n        }\n      }, ackIntervalMs);\n\n      // Wait until stream closes or errors, or worker is stopped\n      await new Promise<void>((resolve, reject) => {\n        const onStop = () => {\n          cleanup();\n          resolve();\n        };\n        const onClose = () => {\n          cleanup();\n          resolve();\n        };\n        const onError = (err: unknown) => {\n          cleanup();\n          reject(err);\n        };\n        function cleanup() {\n          clearInterval(ackInterval);\n          readable.off('close', onClose);\n          readable.off('error', onError);\n        }\n        stopResolve = onStop;\n        readable.once('close', onClose);\n        readable.once('error', onError);\n      });\n\n      await closeCurrent();\n\n      if (stopped) {\n        break;\n      }\n    } catch (e) {\n      await closeCurrent();\n\n      if (stopped) {\n        break;\n      }\n\n      if (\n        isPostgresError(e, PG_ADMIN_SHUTDOWN) ||\n        (e instanceof Error &&\n          (e.message.includes('57P01') ||\n            e.message.includes(\n              'terminating connection due to administrator command',\n            )))\n      ) {\n        lc.info?.(\n          `slot ${config.slot} was taken over (admin shutdown / pg_terminate_backend)`,\n        );\n        break;\n      }\n\n      lc.warn?.(\n        `slot-keeper stream error for ${config.slot}, will reconnect in ${delayMs}ms`,\n        e,\n      );\n\n      if (firstAttempt) {\n        throw e;\n      }\n\n      await Promise.race([\n        sleepFn(delayMs),\n        new Promise<void>(resolve => {\n          stopResolve = resolve;\n        }),\n      ]);\n      delayMs = Math.min(delayMs * 2, 10_000);\n    }\n  }\n\n  lc.info?.(`slot-keeper finished for ${config.slot}`);\n}\n\n// fork()\nconst isMain =\n  typeof process.argv[1] === 'string' &&\n  fileURLToPath(import.meta.url) === resolve(process.argv[1]);\n\nconst parent = parentWorker;\nif (isMain && parent !== null && !singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(parent, process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;AAiCA,SAAgB,SAAS,KAAqB;CAC5C,OAAO,IAAI,SAAS,GAAG,IAAI,SAAS,GAAG,IAAI,OAAO,GAAG;AACvD;AAEA,SAAgB,mBAAmB,QAA2C;CAC5E,OAAO,IAAI,SAA2B,SAAS,WAAW;EACxD,MAAM,gBAAgB;GACpB,OAAO,IAAI,cAAc,YAAY;GACrC,OAAO,IAAI,SAAS,OAAO;GAC3B,IAAI,QAAQ,MACV,QAAQ,IAAI,cAAc,YAAY;EAE1C;EACA,MAAM,qBAAqB;GACzB,QAAQ;GACR,uBAAO,IAAI,MAAM,iDAAiD,CAAC;EACrE;EACA,MAAM,gBAAgB;GACpB,QAAQ;GACR,uBAAO,IAAI,MAAM,2CAA2C,CAAC;EAC/D;EACA,OAAO,KAAK,cAAc,YAAY;EACtC,OAAO,KAAK,SAAS,OAAO;EAC5B,IAAI,QAAQ,MACV,QAAQ,KAAK,cAAc,YAAY;EAEzC,OAAO,gBAA6C,UAAS,QAAO;GAClE,QAAQ;GACR,QAAQ,GAAG;EACb,CAAC;EACD,OAAO,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;CAC1B,CAAC;AACH;AAmBA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,WACA,GAAG,MACY;CAEf,MAAM,YAAY,aAAa;EAAC;EAAK;CAAI,CAAC;CAC1C,MAAM,OACJ,OAAO,cAAc,YAAY,cAAc,OAAO,YAAY,CAAC;CACrE,MAAM,gBACJ,KAAK,mBACH,KAAuB,QACvB,SACE,KACE;EACE,KAAK;EACL,aAAa;EACb,cAAc;EACd,cAAc;EACd,YAAY;GACV,kBAAkB,eAAe,IAAI;GACrC,aAAa;EACf;EACA,QAAQ,wBAAwB,KAAK,GAAG,GAAM;CAChD,GACA,IAAI,EACN,CACF;CACJ,MAAM,cAAc,KAAK,eAAe;CACxC,MAAM,UAAU,KAAK,WAAW;CAEhC,MAAM,SAAS,MAAM,mBAAmB,MAAM;CAE9C,MACE,KAAK,cAAc,iBAAiB,WAAW,aAAa,GAC5D,YAAY,QAAQ,OAAO,IAAI;CACjC,GAAG,OAAO,4BAA4B,OAAO,MAAM;CAEnD,IAAI,UAAU;CACd,IAAI,cAAmC;CACvC,IAAI,kBAAmC;CACvC,IAAI,iBAGO;CACX,MAAM,MAAM,SAAS,OAAO,GAAG;CAC/B,MAAM,gBAAgB,OAAO,iBAAiB;CAE9C,MAAM,eAAe,YAAY;EAC/B,IAAI,iBAAiB;GACnB,IAAI;IACF,gBAAgB,QAAQ;GAC1B,QAAQ,CAAC;GACT,kBAAkB;EACpB;EACA,IAAI,gBAAgB;GAClB,IAAI;IACF,MAAM,eAAe,IAAI,EAAC,SAAS,EAAC,CAAC;IACrC,GAAG,OAAO,6CAA6C,OAAO,MAAM;GACtE,SAAS,GAAG;IACV,GAAG,OAAO,oCAAoC,CAAC;GACjD;GACA,iBAAiB;EACnB;CACF;CAEA,MAAM,OAAO,YAAY;EACvB,IAAI,SACF;EAEF,UAAU;EACV,GAAG,OAAO,4BAA4B,OAAO,MAAM;EACnD,cAAc;EACd,MAAM,aAAa;CACrB;CAEA,OAAO,gBAAgB,cAAc,KAAK,KAAK,CAAC;CAChD,OAAO,GAAG,iBAAiB,KAAK,KAAK,CAAC;CACtC,OAAO,GAAG,gBAAgB,KAAK,KAAK,CAAC;CACrC,OAAO,KAAK,oBAAoB,KAAK,KAAK,CAAC;CAC3C,OAAO,KAAK,eAAe,KAAK,KAAK,CAAC;CACtC,QAAQ,KAAK,iBAAiB,KAAK,KAAK,CAAC;CACzC,QAAQ,KAAK,gBAAgB,KAAK,KAAK,CAAC;CACxC,IAAI,QAAQ,MACV,QAAQ,KAAK,oBAAoB,KAAK,KAAK,CAAC;CAG9C,IAAI,eAAe;CACnB,IAAI,UAAU;CAEd,OAAO,CAAC,SACN,IAAI;EACF,GAAG,OAAO,2CAA2C,OAAO,MAAM;EAClE,MAAM,UAAU,cAAc,QAAQ,EAAE;EACxC,iBAAiB;EAEjB,MAAM,CAAC,UAAU,aAAa,MAAM,YAClC,IACA,SACA,OAAO,MACP,CAAC,OAAO,gBAAgB,GACxB,KAAA,CAEF;EACA,kBAAkB;EAOlB,SAAS,OAAO;EAEhB,IAAI,SAAS;GACX,MAAM,aAAa;GACnB;EACF;EAEA,GAAG,OAAO,0BAA0B,OAAO,KAAK,UAAU,OAAO,KAAK;EACtE,IAAI,cAAc;GAChB,eAAe;GACf,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;EACtC;EAGA,UAAU;EAEV,MAAM,cAAc,kBAAkB;GACpC,IAAI,SAAS;IACX,cAAc,WAAW;IACzB;GACF;GACA,IAAI;IACF,UAAU,MAAM,QAAQ,GAAG,CAAC;GAC9B,SAAS,GAAG;IACV,GAAG,OAAO,2CAA2C,CAAC;GACxD;EACF,GAAG,aAAa;EAGhB,MAAM,IAAI,SAAe,SAAS,WAAW;GAC3C,MAAM,eAAe;IACnB,QAAQ;IACR,QAAQ;GACV;GACA,MAAM,gBAAgB;IACpB,QAAQ;IACR,QAAQ;GACV;GACA,MAAM,WAAW,QAAiB;IAChC,QAAQ;IACR,OAAO,GAAG;GACZ;GACA,SAAS,UAAU;IACjB,cAAc,WAAW;IACzB,SAAS,IAAI,SAAS,OAAO;IAC7B,SAAS,IAAI,SAAS,OAAO;GAC/B;GACA,cAAc;GACd,SAAS,KAAK,SAAS,OAAO;GAC9B,SAAS,KAAK,SAAS,OAAO;EAChC,CAAC;EAED,MAAM,aAAa;EAEnB,IAAI,SACF;CAEJ,SAAS,GAAG;EACV,MAAM,aAAa;EAEnB,IAAI,SACF;EAGF,IACE,gBAAgB,GAAG,iBAAiB,KACnC,aAAa,UACX,EAAE,QAAQ,SAAS,OAAO,KACzB,EAAE,QAAQ,SACR,qDACF,IACJ;GACA,GAAG,OACD,QAAQ,OAAO,KAAK,wDACtB;GACA;EACF;EAEA,GAAG,OACD,gCAAgC,OAAO,KAAK,sBAAsB,QAAQ,KAC1E,CACF;EAEA,IAAI,cACF,MAAM;EAGR,MAAM,QAAQ,KAAK,CACjB,QAAQ,OAAO,GACf,IAAI,SAAc,YAAW;GAC3B,cAAc;EAChB,CAAC,CACH,CAAC;EACD,UAAU,KAAK,IAAI,UAAU,GAAG,GAAM;CACxC;CAGF,GAAG,OAAO,4BAA4B,OAAO,MAAM;AACrD;AAGA,IAAM,SACJ,OAAO,QAAQ,KAAK,OAAO,YAC3B,cAAc,OAAO,KAAK,GAAG,MAAM,QAAQ,QAAQ,KAAK,EAAE;AAE5D,IAAM,SAAS;AACf,IAAI,UAAU,WAAW,QAAQ,CAAC,kBAAkB,GAClD,gBACQ,UACA,UAAU,QAAQ,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC/D"}
+{"version":3,"file":"slot-keeper.js","names":[],"sources":["../../../../../zero-cache/src/server/slot-keeper.ts"],"sourcesContent":["import {resolve} from 'node:path';\nimport type {Readable} from 'node:stream';\nimport {fileURLToPath} from 'node:url';\nimport {PG_ADMIN_SHUTDOWN} from '@drdgvhbh/postgres-error-codes';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport {getLogConfig} from '../config/zero-config.ts';\nimport {\n  DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE,\n  makeAck,\n  startReplicationStream,\n  type PgConnectionConfig,\n} from '../services/change-source/pg/logical-replication/stream.ts';\nimport {toBigInt} from '../services/change-source/pg/lsn.ts';\nimport {exitAfter} from '../services/life-cycle.ts';\nimport {inactivityTimeoutSocket, isPostgresError} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {createLogContext} from './logging.ts';\n\nexport type SlotKeeperConfig = {\n  db: PgConnectionConfig;\n  slot: string;\n  dummyPublication: string;\n  lsn: string;\n  ackIntervalMs?: number | undefined;\n};\n\nexport function parseLsn(lsn: string): bigint {\n  return lsn.includes('/') ? toBigInt(lsn) : BigInt(lsn);\n}\n\nexport function waitForStartConfig(parent: Worker): Promise<SlotKeeperConfig> {\n  return new Promise<SlotKeeperConfig>((resolve, reject) => {\n    const cleanup = () => {\n      parent.off('disconnect', onDisconnect);\n      parent.off('close', onClose);\n      if (process.send) {\n        process.off('disconnect', onDisconnect);\n      }\n    };\n    const onDisconnect = () => {\n      cleanup();\n      reject(new Error('Parent disconnected before sending start config'));\n    };\n    const onClose = () => {\n      cleanup();\n      reject(new Error('Parent closed before sending start config'));\n    };\n    parent.once('disconnect', onDisconnect);\n    parent.once('close', onClose);\n    if (process.send) {\n      process.once('disconnect', onDisconnect);\n    }\n    parent.onceMessageType<['start', SlotKeeperConfig]>('start', cfg => {\n      cleanup();\n      resolve(cfg);\n    });\n    parent.send(['init', {}]);\n  });\n}\n\nexport type SlotKeeperDependencies = {\n  createSession?: (\n    config: SlotKeeperConfig,\n    lc: LogContext,\n  ) => postgres.Sql | {end: (opts?: {timeout?: number}) => Promise<void>};\n  startStream?: (\n    lc: LogContext,\n    session: postgres.Sql,\n    slot: string,\n    publications: string[],\n    startLsn: bigint,\n    maxAttempts?: number,\n  ) => Promise<[Readable, NodeJS.WritableStream]>;\n  sleepFn?: (ms: number) => Promise<void>;\n  logContext?: LogContext;\n};\n\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  depsOrArg?: string | SlotKeeperDependencies,\n  ...argv: string[]\n): Promise<void> {\n  // Only logging config is needed; avoid requiring the full (normalized) config.\n  const logConfig = getLogConfig({env, argv});\n  const deps: SlotKeeperDependencies =\n    typeof depsOrArg === 'object' && depsOrArg !== null ? depsOrArg : {};\n  const createSession =\n    deps.createSession ??\n    ((cfg: SlotKeeperConfig, log: LogContext) =>\n      postgres(\n        defu(\n          {\n            max: 1,\n            fetch_types: false,\n            idle_timeout: null,\n            max_lifetime: null as unknown as number,\n            connection: {\n              application_name: `slot-keeper-${cfg.slot}`,\n              replication: 'database',\n            },\n            socket: inactivityTimeoutSocket(log, 0, 60_000),\n          },\n          cfg.db as Options<Record<string, PostgresType>>,\n        ),\n      ));\n  const startStream = deps.startStream ?? startReplicationStream;\n  const sleepFn = deps.sleepFn ?? sleep;\n\n  const config = await waitForStartConfig(parent);\n\n  lc = (\n    deps.logContext ?? createLogContext(logConfig, 'slot-keeper')\n  ).withContext('slot', config.slot);\n  lc.info?.(`starting slot-keeper for ${config.slot}`);\n\n  let stopped = false;\n  let stopResolve: (() => void) | null = null;\n  let currentReadable: Readable | null = null;\n  let currentSession:\n    | postgres.Sql\n    | {end: (opts?: {timeout?: number}) => Promise<void>}\n    | null = null;\n  const lsn = parseLsn(config.lsn);\n  const ackIntervalMs = config.ackIntervalMs ?? 5_000;\n\n  const closeCurrent = async () => {\n    if (currentReadable) {\n      try {\n        currentReadable.destroy();\n      } catch {}\n      currentReadable = null;\n    }\n    if (currentSession) {\n      try {\n        await currentSession.end({timeout: 0});\n        lc.info?.(`ended slot-keeper replication session for ${config.slot}`);\n      } catch (e) {\n        lc.warn?.(`error ending slot-keeper session`, e);\n      }\n      currentSession = null;\n    }\n  };\n\n  const stop = async () => {\n    if (stopped) {\n      return;\n    }\n    stopped = true;\n    lc.info?.(`stopping slot-keeper for ${config.slot}`);\n    stopResolve?.();\n    await closeCurrent();\n  };\n\n  parent.onceMessageType('stop', () => void stop());\n  parent.on('SIGTERM', () => void stop());\n  parent.on('SIGINT', () => void stop());\n  parent.once('disconnect', () => void stop());\n  parent.once('close', () => void stop());\n  process.once('SIGTERM', () => void stop());\n  process.once('SIGINT', () => void stop());\n  if (process.send) {\n    process.once('disconnect', () => void stop());\n  }\n\n  let firstAttempt = true;\n  let delayMs = 1000;\n\n  while (!stopped) {\n    try {\n      lc.info?.(`connecting replication session for slot ${config.slot}`);\n      const session = createSession(config, lc);\n      currentSession = session;\n\n      const [readable, writeable] = await startStream(\n        lc,\n        session as postgres.Sql,\n        config.slot,\n        [config.dummyPublication],\n        lsn,\n        DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE + 1,\n      );\n      currentReadable = readable;\n      // Discard the streamed messages. Otherwise the paused socket backs up the\n      // wal_sender (e.g. with keepalives and logical messages) for the duration\n      // of the initial sync / restore, and a wal_sender blocked on writing to the\n      // client does not exit (and release the slot) when it is terminated by the\n      // takeover; it instead blocks on sending the termination error to the\n      // client, until signaled again.\n      readable.resume();\n\n      if (stopped) {\n        await closeCurrent();\n        break;\n      }\n\n      lc.info?.(`slot-keeper active for ${config.slot} at lsn ${config.lsn}`);\n      if (firstAttempt) {\n        firstAttempt = false;\n        parent.send(['ready', {ready: true}]);\n      }\n\n      // Reset backoff delay on successful connection\n      delayMs = 1000;\n\n      const ackInterval = setInterval(() => {\n        if (stopped) {\n          clearInterval(ackInterval);\n          return;\n        }\n        try {\n          writeable.write(makeAck(lsn));\n        } catch (e) {\n          lc.warn?.(`error writing ACK to replication stream`, e);\n        }\n      }, ackIntervalMs);\n\n      // Wait until stream closes or errors, or worker is stopped\n      await new Promise<void>((resolve, reject) => {\n        const onStop = () => {\n          cleanup();\n          resolve();\n        };\n        const onClose = () => {\n          cleanup();\n          resolve();\n        };\n        const onError = (err: unknown) => {\n          cleanup();\n          reject(err);\n        };\n        function cleanup() {\n          clearInterval(ackInterval);\n          readable.off('close', onClose);\n          readable.off('error', onError);\n        }\n        stopResolve = onStop;\n        readable.once('close', onClose);\n        readable.once('error', onError);\n      });\n\n      await closeCurrent();\n\n      if (stopped) {\n        break;\n      }\n    } catch (e) {\n      await closeCurrent();\n\n      if (stopped) {\n        break;\n      }\n\n      if (\n        isPostgresError(e, PG_ADMIN_SHUTDOWN) ||\n        (e instanceof Error &&\n          (e.message.includes('57P01') ||\n            e.message.includes(\n              'terminating connection due to administrator command',\n            )))\n      ) {\n        lc.info?.(\n          `slot ${config.slot} was taken over (admin shutdown / pg_terminate_backend)`,\n        );\n        break;\n      }\n\n      lc.warn?.(\n        `slot-keeper stream error for ${config.slot}, will reconnect in ${delayMs}ms`,\n        e,\n      );\n\n      if (firstAttempt) {\n        throw e;\n      }\n\n      await Promise.race([\n        sleepFn(delayMs),\n        new Promise<void>(resolve => {\n          stopResolve = resolve;\n        }),\n      ]);\n      delayMs = Math.min(delayMs * 2, 10_000);\n    }\n  }\n\n  lc.info?.(`slot-keeper finished for ${config.slot}`);\n}\n\n// fork()\nconst isMain =\n  typeof process.argv[1] === 'string' &&\n  fileURLToPath(import.meta.url) === resolve(process.argv[1]);\n\nconst parent = parentWorker;\nif (isMain && parent !== null && !singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(parent, process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;AAiCA,SAAgB,SAAS,KAAqB;CAC5C,OAAO,IAAI,SAAS,GAAG,IAAI,SAAS,GAAG,IAAI,OAAO,GAAG;AACvD;AAEA,SAAgB,mBAAmB,QAA2C;CAC5E,OAAO,IAAI,SAA2B,SAAS,WAAW;EACxD,MAAM,gBAAgB;GACpB,OAAO,IAAI,cAAc,YAAY;GACrC,OAAO,IAAI,SAAS,OAAO;GAC3B,IAAI,QAAQ,MACV,QAAQ,IAAI,cAAc,YAAY;EAE1C;EACA,MAAM,qBAAqB;GACzB,QAAQ;GACR,uBAAO,IAAI,MAAM,iDAAiD,CAAC;EACrE;EACA,MAAM,gBAAgB;GACpB,QAAQ;GACR,uBAAO,IAAI,MAAM,2CAA2C,CAAC;EAC/D;EACA,OAAO,KAAK,cAAc,YAAY;EACtC,OAAO,KAAK,SAAS,OAAO;EAC5B,IAAI,QAAQ,MACV,QAAQ,KAAK,cAAc,YAAY;EAEzC,OAAO,gBAA6C,UAAS,QAAO;GAClE,QAAQ;GACR,QAAQ,GAAG;EACb,CAAC;EACD,OAAO,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;CAC1B,CAAC;AACH;AAmBA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,WACA,GAAG,MACY;CAEf,MAAM,YAAY,aAAa;EAAC;EAAK;CAAI,CAAC;CAC1C,MAAM,OACJ,OAAO,cAAc,YAAY,cAAc,OAAO,YAAY,CAAC;CACrE,MAAM,gBACJ,KAAK,mBACH,KAAuB,QACvB,SACE,KACE;EACE,KAAK;EACL,aAAa;EACb,cAAc;EACd,cAAc;EACd,YAAY;GACV,kBAAkB,eAAe,IAAI;GACrC,aAAa;EACf;EACA,QAAQ,wBAAwB,KAAK,GAAG,GAAM;CAChD,GACA,IAAI,EACN,CACF;CACJ,MAAM,cAAc,KAAK,eAAe;CACxC,MAAM,UAAU,KAAK,WAAW;CAEhC,MAAM,SAAS,MAAM,mBAAmB,MAAM;CAE9C,MACE,KAAK,cAAc,iBAAiB,WAAW,aAAa,EAAA,CAC5D,YAAY,QAAQ,OAAO,IAAI;CACjC,GAAG,OAAO,4BAA4B,OAAO,MAAM;CAEnD,IAAI,UAAU;CACd,IAAI,cAAmC;CACvC,IAAI,kBAAmC;CACvC,IAAI,iBAGO;CACX,MAAM,MAAM,SAAS,OAAO,GAAG;CAC/B,MAAM,gBAAgB,OAAO,iBAAiB;CAE9C,MAAM,eAAe,YAAY;EAC/B,IAAI,iBAAiB;GACnB,IAAI;IACF,gBAAgB,QAAQ;GAC1B,QAAQ,CAAC;GACT,kBAAkB;EACpB;EACA,IAAI,gBAAgB;GAClB,IAAI;IACF,MAAM,eAAe,IAAI,EAAC,SAAS,EAAC,CAAC;IACrC,GAAG,OAAO,6CAA6C,OAAO,MAAM;GACtE,SAAS,GAAG;IACV,GAAG,OAAO,oCAAoC,CAAC;GACjD;GACA,iBAAiB;EACnB;CACF;CAEA,MAAM,OAAO,YAAY;EACvB,IAAI,SACF;EAEF,UAAU;EACV,GAAG,OAAO,4BAA4B,OAAO,MAAM;EACnD,cAAc;EACd,MAAM,aAAa;CACrB;CAEA,OAAO,gBAAgB,cAAc,KAAK,KAAK,CAAC;CAChD,OAAO,GAAG,iBAAiB,KAAK,KAAK,CAAC;CACtC,OAAO,GAAG,gBAAgB,KAAK,KAAK,CAAC;CACrC,OAAO,KAAK,oBAAoB,KAAK,KAAK,CAAC;CAC3C,OAAO,KAAK,eAAe,KAAK,KAAK,CAAC;CACtC,QAAQ,KAAK,iBAAiB,KAAK,KAAK,CAAC;CACzC,QAAQ,KAAK,gBAAgB,KAAK,KAAK,CAAC;CACxC,IAAI,QAAQ,MACV,QAAQ,KAAK,oBAAoB,KAAK,KAAK,CAAC;CAG9C,IAAI,eAAe;CACnB,IAAI,UAAU;CAEd,OAAO,CAAC,SACN,IAAI;EACF,GAAG,OAAO,2CAA2C,OAAO,MAAM;EAClE,MAAM,UAAU,cAAc,QAAQ,EAAE;EACxC,iBAAiB;EAEjB,MAAM,CAAC,UAAU,aAAa,MAAM,YAClC,IACA,SACA,OAAO,MACP,CAAC,OAAO,gBAAgB,GACxB,KAAA,CAEF;EACA,kBAAkB;EAOlB,SAAS,OAAO;EAEhB,IAAI,SAAS;GACX,MAAM,aAAa;GACnB;EACF;EAEA,GAAG,OAAO,0BAA0B,OAAO,KAAK,UAAU,OAAO,KAAK;EACtE,IAAI,cAAc;GAChB,eAAe;GACf,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;EACtC;EAGA,UAAU;EAEV,MAAM,cAAc,kBAAkB;GACpC,IAAI,SAAS;IACX,cAAc,WAAW;IACzB;GACF;GACA,IAAI;IACF,UAAU,MAAM,QAAQ,GAAG,CAAC;GAC9B,SAAS,GAAG;IACV,GAAG,OAAO,2CAA2C,CAAC;GACxD;EACF,GAAG,aAAa;EAGhB,MAAM,IAAI,SAAe,SAAS,WAAW;GAC3C,MAAM,eAAe;IACnB,QAAQ;IACR,QAAQ;GACV;GACA,MAAM,gBAAgB;IACpB,QAAQ;IACR,QAAQ;GACV;GACA,MAAM,WAAW,QAAiB;IAChC,QAAQ;IACR,OAAO,GAAG;GACZ;GACA,SAAS,UAAU;IACjB,cAAc,WAAW;IACzB,SAAS,IAAI,SAAS,OAAO;IAC7B,SAAS,IAAI,SAAS,OAAO;GAC/B;GACA,cAAc;GACd,SAAS,KAAK,SAAS,OAAO;GAC9B,SAAS,KAAK,SAAS,OAAO;EAChC,CAAC;EAED,MAAM,aAAa;EAEnB,IAAI,SACF;CAEJ,SAAS,GAAG;EACV,MAAM,aAAa;EAEnB,IAAI,SACF;EAGF,IACE,gBAAgB,GAAG,iBAAiB,KACnC,aAAa,UACX,EAAE,QAAQ,SAAS,OAAO,KACzB,EAAE,QAAQ,SACR,qDACF,IACJ;GACA,GAAG,OACD,QAAQ,OAAO,KAAK,wDACtB;GACA;EACF;EAEA,GAAG,OACD,gCAAgC,OAAO,KAAK,sBAAsB,QAAQ,KAC1E,CACF;EAEA,IAAI,cACF,MAAM;EAGR,MAAM,QAAQ,KAAK,CACjB,QAAQ,OAAO,GACf,IAAI,SAAc,YAAW;GAC3B,cAAc;EAChB,CAAC,CACH,CAAC;EACD,UAAU,KAAK,IAAI,UAAU,GAAG,GAAM;CACxC;CAGF,GAAG,OAAO,4BAA4B,OAAO,MAAM;AACrD;AAGA,IAAM,SACJ,OAAO,QAAQ,KAAK,OAAO,YAC3B,cAAc,YAAY,GAAG,MAAM,QAAQ,QAAQ,KAAK,EAAE;AAE5D,IAAM,SAAS;AACf,IAAI,UAAU,WAAW,QAAQ,CAAC,kBAAkB,GAClD,gBACQ,UACA,UAAU,QAAQ,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC/D"}

package/out/zero-cache/src/server/syncer-assigner.js.map

@@ -1 +1 @@
-{"version":3,"file":"syncer-assigner.js","names":["#taskID","#numWorkers","#assignments","#workerLoads","#nextGeneration","#decrementLoad"],"sources":["../../../../../zero-cache/src/server/syncer-assigner.ts"],"sourcesContent":["import {h32} from '../../../shared/src/hash.ts';\n\nexport type Assignment = {\n  readonly worker: number;\n  readonly generation: number;\n};\n\nexport class SyncerAssigner {\n  readonly #taskID: string;\n  readonly #numWorkers: number;\n  readonly #assignments = new Map<string, Assignment>();\n  readonly #workerLoads: number[];\n  #nextGeneration = 0;\n\n  constructor(taskID: string, numWorkers: number) {\n    this.#taskID = taskID;\n    this.#numWorkers = numWorkers;\n    this.#workerLoads = new Array<number>(numWorkers).fill(0);\n  }\n\n  assign(clientGroupID: string): Assignment {\n    if (this.#numWorkers <= 0) {\n      return {worker: 0, generation: 0};\n    }\n\n    const generation = ++this.#nextGeneration;\n    const existing = this.#assignments.get(clientGroupID);\n    if (existing !== undefined) {\n      const updated: Assignment = {worker: existing.worker, generation};\n      this.#assignments.set(clientGroupID, updated);\n      return updated;\n    }\n\n    // Find worker(s) with minimum active load\n    let minLoad = Infinity;\n    const tied: number[] = [];\n    for (let i = 0; i < this.#numWorkers; i++) {\n      const load = this.#workerLoads[i];\n      if (load < minLoad) {\n        minLoad = load;\n        tied.length = 0;\n        tied.push(i);\n      } else if (load === minLoad) {\n        tied.push(i);\n      }\n    }\n\n    const chosen =\n      tied.length === 1\n        ? tied[0]\n        : tied[h32(this.#taskID + '/' + clientGroupID) % tied.length];\n\n    this.#workerLoads[chosen]++;\n    const assignment: Assignment = {worker: chosen, generation};\n    this.#assignments.set(clientGroupID, assignment);\n    return assignment;\n  }\n\n  release(\n    clientGroupID: string,\n    workerIndex: number,\n    generation?: number,\n  ): void {\n    const existing = this.#assignments.get(clientGroupID);\n    if (existing === undefined || existing.worker !== workerIndex) {\n      return;\n    }\n    if (generation !== undefined && existing.generation !== generation) {\n      return;\n    }\n    this.#decrementLoad(workerIndex);\n    this.#assignments.delete(clientGroupID);\n  }\n\n  getWorkerLoad(workerIndex: number): number {\n    return this.#workerLoads[workerIndex] ?? 0;\n  }\n\n  getAssignment(clientGroupID: string): number | undefined {\n    return this.#assignments.get(clientGroupID)?.worker;\n  }\n\n  #decrementLoad(worker: number): void {\n    if (this.#workerLoads[worker] > 0) {\n      this.#workerLoads[worker]--;\n    }\n  }\n\n  destroy(): void {\n    this.#assignments.clear();\n    this.#workerLoads.fill(0);\n  }\n}\n"],"mappings":";;AAOA,IAAa,iBAAb,MAA4B;CAC1B;CACA;CACA,+BAAwB,IAAI,IAAwB;CACpD;CACA,kBAAkB;CAElB,YAAY,QAAgB,YAAoB;EAC9C,KAAKA,UAAU;EACf,KAAKC,cAAc;EACnB,KAAKE,eAAe,IAAI,MAAc,UAAU,EAAE,KAAK,CAAC;CAC1D;CAEA,OAAO,eAAmC;EACxC,IAAI,KAAKF,eAAe,GACtB,OAAO;GAAC,QAAQ;GAAG,YAAY;EAAC;EAGlC,MAAM,aAAa,EAAE,KAAKG;EAC1B,MAAM,WAAW,KAAKF,aAAa,IAAI,aAAa;EACpD,IAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,UAAsB;IAAC,QAAQ,SAAS;IAAQ;GAAU;GAChE,KAAKA,aAAa,IAAI,eAAe,OAAO;GAC5C,OAAO;EACT;EAGA,IAAI,UAAU;EACd,MAAM,OAAiB,CAAC;EACxB,KAAK,IAAI,IAAI,GAAG,IAAI,KAAKD,aAAa,KAAK;GACzC,MAAM,OAAO,KAAKE,aAAa;GAC/B,IAAI,OAAO,SAAS;IAClB,UAAU;IACV,KAAK,SAAS;IACd,KAAK,KAAK,CAAC;GACb,OAAO,IAAI,SAAS,SAClB,KAAK,KAAK,CAAC;EAEf;EAEA,MAAM,SACJ,KAAK,WAAW,IACZ,KAAK,KACL,KAAK,IAAI,KAAKH,UAAU,MAAM,aAAa,IAAI,KAAK;EAE1D,KAAKG,aAAa;EAClB,MAAM,aAAyB;GAAC,QAAQ;GAAQ;EAAU;EAC1D,KAAKD,aAAa,IAAI,eAAe,UAAU;EAC/C,OAAO;CACT;CAEA,QACE,eACA,aACA,YACM;EACN,MAAM,WAAW,KAAKA,aAAa,IAAI,aAAa;EACpD,IAAI,aAAa,KAAA,KAAa,SAAS,WAAW,aAChD;EAEF,IAAI,eAAe,KAAA,KAAa,SAAS,eAAe,YACtD;EAEF,KAAKG,eAAe,WAAW;EAC/B,KAAKH,aAAa,OAAO,aAAa;CACxC;CAEA,cAAc,aAA6B;EACzC,OAAO,KAAKC,aAAa,gBAAgB;CAC3C;CAEA,cAAc,eAA2C;EACvD,OAAO,KAAKD,aAAa,IAAI,aAAa,GAAG;CAC/C;CAEA,eAAe,QAAsB;EACnC,IAAI,KAAKC,aAAa,UAAU,GAC9B,KAAKA,aAAa;CAEtB;CAEA,UAAgB;EACd,KAAKD,aAAa,MAAM;EACxB,KAAKC,aAAa,KAAK,CAAC;CAC1B;AACF"}
+{"version":3,"file":"syncer-assigner.js","names":[],"sources":["../../../../../zero-cache/src/server/syncer-assigner.ts"],"sourcesContent":["import {h32} from '../../../shared/src/hash.ts';\n\nexport type Assignment = {\n  readonly worker: number;\n  readonly generation: number;\n};\n\nexport class SyncerAssigner {\n  readonly #taskID: string;\n  readonly #numWorkers: number;\n  readonly #assignments = new Map<string, Assignment>();\n  readonly #workerLoads: number[];\n  #nextGeneration = 0;\n\n  constructor(taskID: string, numWorkers: number) {\n    this.#taskID = taskID;\n    this.#numWorkers = numWorkers;\n    this.#workerLoads = new Array<number>(numWorkers).fill(0);\n  }\n\n  assign(clientGroupID: string): Assignment {\n    if (this.#numWorkers <= 0) {\n      return {worker: 0, generation: 0};\n    }\n\n    const generation = ++this.#nextGeneration;\n    const existing = this.#assignments.get(clientGroupID);\n    if (existing !== undefined) {\n      const updated: Assignment = {worker: existing.worker, generation};\n      this.#assignments.set(clientGroupID, updated);\n      return updated;\n    }\n\n    // Find worker(s) with minimum active load\n    let minLoad = Infinity;\n    const tied: number[] = [];\n    for (let i = 0; i < this.#numWorkers; i++) {\n      const load = this.#workerLoads[i];\n      if (load < minLoad) {\n        minLoad = load;\n        tied.length = 0;\n        tied.push(i);\n      } else if (load === minLoad) {\n        tied.push(i);\n      }\n    }\n\n    const chosen =\n      tied.length === 1\n        ? tied[0]\n        : tied[h32(this.#taskID + '/' + clientGroupID) % tied.length];\n\n    this.#workerLoads[chosen]++;\n    const assignment: Assignment = {worker: chosen, generation};\n    this.#assignments.set(clientGroupID, assignment);\n    return assignment;\n  }\n\n  release(\n    clientGroupID: string,\n    workerIndex: number,\n    generation?: number,\n  ): void {\n    const existing = this.#assignments.get(clientGroupID);\n    if (existing === undefined || existing.worker !== workerIndex) {\n      return;\n    }\n    if (generation !== undefined && existing.generation !== generation) {\n      return;\n    }\n    this.#decrementLoad(workerIndex);\n    this.#assignments.delete(clientGroupID);\n  }\n\n  getWorkerLoad(workerIndex: number): number {\n    return this.#workerLoads[workerIndex] ?? 0;\n  }\n\n  getAssignment(clientGroupID: string): number | undefined {\n    return this.#assignments.get(clientGroupID)?.worker;\n  }\n\n  #decrementLoad(worker: number): void {\n    if (this.#workerLoads[worker] > 0) {\n      this.#workerLoads[worker]--;\n    }\n  }\n\n  destroy(): void {\n    this.#assignments.clear();\n    this.#workerLoads.fill(0);\n  }\n}\n"],"mappings":";;AAOA,IAAa,iBAAb,MAA4B;CAC1B;CACA;CACA,+BAAwB,IAAI,IAAwB;CACpD;CACA,kBAAkB;CAElB,YAAY,QAAgB,YAAoB;EAC9C,KAAK,UAAU;EACf,KAAK,cAAc;EACnB,KAAK,eAAe,IAAI,MAAc,UAAU,CAAC,CAAC,KAAK,CAAC;CAC1D;CAEA,OAAO,eAAmC;EACxC,IAAI,KAAK,eAAe,GACtB,OAAO;GAAC,QAAQ;GAAG,YAAY;EAAC;EAGlC,MAAM,aAAa,EAAE,KAAK;EAC1B,MAAM,WAAW,KAAK,aAAa,IAAI,aAAa;EACpD,IAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,UAAsB;IAAC,QAAQ,SAAS;IAAQ;GAAU;GAChE,KAAK,aAAa,IAAI,eAAe,OAAO;GAC5C,OAAO;EACT;EAGA,IAAI,UAAU;EACd,MAAM,OAAiB,CAAC;EACxB,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,aAAa,KAAK;GACzC,MAAM,OAAO,KAAK,aAAa;GAC/B,IAAI,OAAO,SAAS;IAClB,UAAU;IACV,KAAK,SAAS;IACd,KAAK,KAAK,CAAC;GACb,OAAO,IAAI,SAAS,SAClB,KAAK,KAAK,CAAC;EAEf;EAEA,MAAM,SACJ,KAAK,WAAW,IACZ,KAAK,KACL,KAAK,IAAI,KAAK,UAAU,MAAM,aAAa,IAAI,KAAK;EAE1D,KAAK,aAAa,OAAO;EACzB,MAAM,aAAyB;GAAC,QAAQ;GAAQ;EAAU;EAC1D,KAAK,aAAa,IAAI,eAAe,UAAU;EAC/C,OAAO;CACT;CAEA,QACE,eACA,aACA,YACM;EACN,MAAM,WAAW,KAAK,aAAa,IAAI,aAAa;EACpD,IAAI,aAAa,KAAA,KAAa,SAAS,WAAW,aAChD;EAEF,IAAI,eAAe,KAAA,KAAa,SAAS,eAAe,YACtD;EAEF,KAAK,eAAe,WAAW;EAC/B,KAAK,aAAa,OAAO,aAAa;CACxC;CAEA,cAAc,aAA6B;EACzC,OAAO,KAAK,aAAa,gBAAgB;CAC3C;CAEA,cAAc,eAA2C;EACvD,OAAO,KAAK,aAAa,IAAI,aAAa,CAAC,EAAE;CAC/C;CAEA,eAAe,QAAsB;EACnC,IAAI,KAAK,aAAa,UAAU,GAC9B,KAAK,aAAa,OAAO;CAE7B;CAEA,UAAgB;EACd,KAAK,aAAa,MAAM;EACxB,KAAK,aAAa,KAAK,CAAC;CAC1B;AACF"}

package/out/zero-cache/src/server/syncer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"syncer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/syncer.ts"],"names":[],"mappings":"AAsCA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAqC/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CA0Sf"}
+{"version":3,"file":"syncer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/syncer.ts"],"names":[],"mappings":"AAuCA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAqC/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAsTf"}

package/out/zero-cache/src/server/syncer.js

@@ -39,6 +39,7 @@ import { pid } from "node:process";
 import { randomUUID } from "node:crypto";
 import { tmpdir } from "node:os";
 import path from "node:path";
+import { getHeapStatistics } from "node:v8";
 //#region ../zero-cache/src/server/syncer.ts
 function randomID() {
 	return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);
@@ -78,7 +79,11 @@ async function runWorker(parent, env, ...args) {
 	const upstreamDB = enableCrudMutations && upstream.type === "pg" ? await connectPgClient(lc, upstream.db, `sync-worker-${pid}-upstream`, { max: must(upstream.maxConnsPerWorker, "upstream.maxConnsPerWorker must be set") }) : void 0;
 	const dbWarmup = Promise.allSettled([warmupConnections(lc, cvrDB, "cvr"), upstreamDB ? warmupConnections(lc, upstreamDB, "upstream") : promiseVoid]);
 	const tmpDir = config.storageDBTmpDir ?? tmpdir();
-	const operatorStorage = DatabaseStorage.create(lc, path.join(tmpDir, `sync-worker-${randomUUID()}`));
+	const operatorStorage = DatabaseStorage.create(lc, path.join(tmpDir, `sync-worker-${randomUUID()}`), { maxMemoryBytes: Math.floor(getHeapStatistics().heap_size_limit * config.operatorStorageHeapProportion) });
+	getOrCreateGauge("sync", "ivm.operator-storage-memory-bytes", {
+		description: "Estimated bytes of IVM operator state that the client groups of a sync worker keep in memory rather than in SQLite",
+		unit: "By"
+	}).addCallback((o) => o.observe(operatorStorage.memoryBytes));
 	const writeAuthzStorage = DatabaseStorage.create(lc, path.join(tmpDir, `mutagen-${randomUUID()}`));
 	const shard = getShardID(config);
 	const customQueryConfig = getCustomQueryConfig(config);

package/out/zero-cache/src/server/syncer.js.map

@@ -1 +1 @@
-{"version":3,"file":"syncer.js","names":[],"sources":["../../../../../zero-cache/src/server/syncer.ts"],"sourcesContent":["import {randomUUID} from 'node:crypto';\nimport {tmpdir} from 'node:os';\nimport path from 'node:path';\nimport {pid} from 'node:process';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {DatabaseStorage} from '../../../zqlite/src/database-storage.ts';\nimport type {ValidateLegacyJWT} from '../auth/auth.ts';\nimport {tokenConfigOptions, verifyToken} from '../auth/jwt.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {CustomQueryTransformer} from '../custom-queries/transform-query.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {MutagenService} from '../services/mutagen/mutagen.ts';\nimport {PusherService} from '../services/mutagen/pusher.ts';\nimport type {ReplicaState} from '../services/replicator/replicator.ts';\nimport {BatchPacer} from '../services/view-syncer/batch-pacer.ts';\nimport {\n  type ConnectionContextManager,\n  ConnectionContextManagerImpl,\n} from '../services/view-syncer/connection-context-manager.ts';\nimport {DeferredWritesBudget} from '../services/view-syncer/deferred-writes-budget.ts';\nimport type {DrainCoordinator} from '../services/view-syncer/drain-coordinator.ts';\nimport {PipelineDriver} from '../services/view-syncer/pipeline-driver.ts';\nimport {QueryStats} from '../services/view-syncer/query-stats.ts';\nimport {SnapshotRowCache} from '../services/view-syncer/snapshot-row-cache.ts';\nimport {Snapshotter} from '../services/view-syncer/snapshotter.ts';\nimport {ViewSyncerService} from '../services/view-syncer/view-syncer.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardID} from '../types/shards.ts';\nimport type {Subscription} from '../types/subscription.ts';\nimport {replicaFileModeSchema, replicaFileName} from '../workers/replicator.ts';\nimport {Syncer} from '../workers/syncer.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {InspectorDelegate} from './inspector-delegate.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {isPriorityOpRunning, runPriorityOp} from './priority-op.ts';\n\nfunction randomID() {\n  return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);\n}\n\nfunction getCustomQueryConfig(\n  config: Pick<NormalizedZeroConfig, 'query' | 'getQueries'>,\n) {\n  const queryConfig = config.query?.url ? config.query : config.getQueries;\n\n  if (!queryConfig?.url) {\n    return undefined;\n  }\n\n  return {\n    url: queryConfig.url,\n    apiKey: queryConfig.apiKey,\n    allowedClientHeaders: queryConfig.allowedClientHeaders,\n    allowedRequestHeaders: queryConfig.allowedRequestHeaders,\n    forwardCookies: queryConfig.forwardCookies ?? false,\n  };\n}\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length >= 2, `expected [fileMode, workerIndex, ...flags]`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n  const workerIndex = Number(args[1]);\n  installProfileHandler(parent, `syncer-${workerIndex}`, workerIndex);\n  const config = getNormalizedZeroConfig({env, argv: args.slice(2)});\n\n  startOtelAuto(\n    createLogContext(config, 'syncer', workerIndex, false),\n    'syncer',\n    workerIndex,\n  );\n  lc = createLogContext(config, 'syncer', workerIndex);\n  initEventSink(lc, config);\n\n  const {cvr, upstream, enableCrudMutations} = config;\n\n  const replicaFile = replicaFileName(config.replica.file, fileMode);\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'syncer replica',\n      dbPath: replicaFile,\n    },\n    config.sqliteCorruptionChecks,\n  );\n  lc.debug?.(`running view-syncer on ${replicaFile}`);\n\n  const cvrDB = await connectPgClient(lc, cvr.db, `sync-worker-${pid}-cvr`, {\n    max: must(cvr.maxConnsPerWorker, 'cvr.maxConnsPerWorker must be set'),\n  });\n\n  const upstreamDB =\n    enableCrudMutations && upstream.type === 'pg'\n      ? await connectPgClient(lc, upstream.db, `sync-worker-${pid}-upstream`, {\n          max: must(\n            upstream.maxConnsPerWorker,\n            'upstream.maxConnsPerWorker must be set',\n          ),\n        })\n      : undefined;\n\n  const dbWarmup = Promise.allSettled([\n    warmupConnections(lc, cvrDB, 'cvr'),\n    upstreamDB ? warmupConnections(lc, upstreamDB, 'upstream') : promiseVoid,\n  ]);\n\n  const tmpDir = config.storageDBTmpDir ?? tmpdir();\n  const operatorStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `sync-worker-${randomUUID()}`),\n  );\n  const writeAuthzStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `mutagen-${randomUUID()}`),\n  );\n\n  const shard = getShardID(config);\n  const customQueryConfig = getCustomQueryConfig(config);\n  const pushConfig =\n    config.push.url === undefined && config.mutate.url === undefined\n      ? undefined\n      : {\n          ...config.push,\n          ...config.mutate,\n          url: must(\n            config.push.url ?? config.mutate.url,\n            'No push or mutate URL configured',\n          ),\n        };\n\n  /** @deprecated used in JWT validation */\n  let validateLegacyJWT: ValidateLegacyJWT | undefined = undefined;\n\n  const tokenOptions = tokenConfigOptions(config.auth ?? {});\n  if (tokenOptions.length === 1) {\n    validateLegacyJWT = async (token, {userID}) => {\n      if (!userID) {\n        throw new ProtocolErrorWithLevel(\n          {\n            kind: 'Unauthorized',\n            message: 'UserID is required for JWT validation.',\n            origin: 'zeroCache',\n          },\n          'warn',\n        );\n      }\n\n      const decoded = await verifyToken(config.auth, token, {\n        subject: userID,\n        ...(config.auth?.issuer && {issuer: config.auth.issuer}),\n        ...(config.auth?.audience && {\n          audience: config.auth.audience,\n        }),\n      });\n      return {\n        type: 'jwt',\n        raw: token,\n        decoded,\n      };\n    };\n  }\n\n  // Shared by all of the view-syncers on this worker so that the row reads\n  // performed when advancing their pipelines are done once per worker rather\n  // than once per client group.\n  const snapshotRowCache =\n    config.snapshotRowCacheSize > 0\n      ? new SnapshotRowCache(config.snapshotRowCacheSize)\n      : undefined;\n\n  // Shared by all of the view-syncers on this worker, which each hold their\n  // own copy of the changes they are advancing through.\n  const deferredWritesBudget = config.deferIvmWrites\n    ? DeferredWritesBudget.forHeapProportion(\n        config.deferIvmWritesHeapProportion,\n      )\n    : undefined;\n  if (deferredWritesBudget) {\n    lc.info?.(\n      `Deferred IVM writes may hold up to ${deferredWritesBudget.maxRows} ` +\n        `rows (~${(deferredWritesBudget.maxBytes / 1024 ** 2).toFixed(2)} MB) ` +\n        `across client groups`,\n    );\n    getOrCreateGauge(\n      'sync',\n      'ivm.deferred-writes-reserved-rows',\n      'Rows reserved by the client groups of a sync worker to hold IVM ' +\n        'changes in memory (deferIvmWrites)',\n    ).addCallback(o => o.observe(deferredWritesBudget.reservedRows));\n    getOrCreateGauge('sync', 'ivm.deferred-writes-held-bytes', {\n      description:\n        'Estimated bytes of the IVM changes that the client groups of a sync ' +\n        'worker hold in memory (deferIvmWrites)',\n      unit: 'By',\n    }).addCallback(o => o.observe(deferredWritesBudget.heldBytes));\n  }\n\n  // Shared by all of the view-syncers on this worker, whose batches of\n  // changes together are held to a share of the worker's time.\n  const batchPacer = config.costBasedSyncEnabled\n    ? new BatchPacer({\n        shareTarget: config.syncUpdateShareTarget,\n        maxIntervalMs: config.syncMaxBatchIntervalMs,\n        lc,\n      })\n    : undefined;\n  if (batchPacer) {\n    getOrCreateGauge('sync', 'worker-busy-share', {\n      description:\n        'Share of its time that a sync worker was busy over the last few ' +\n        'seconds, which the pacing of its client groups holds to a target',\n      unit: '1',\n    }).addCallback(o => o.observe(batchPacer.share));\n    getOrCreateGauge('sync', 'batch-wait-multiplier', {\n      description:\n        'How many times the processing time of a batch of changes the ' +\n        'client groups of a sync worker wait before their next batch',\n      unit: '1',\n    }).addCallback(o => o.observe(batchPacer.multiplier));\n  }\n\n  // Shared by all of the view-syncers on this worker, so that the work done\n  // for each query shape is logged once per worker per interval.\n  const queryStats =\n    config.log.queryStatsIntervalSeconds > 0 ? new QueryStats() : undefined;\n  const stopQueryStats = queryStats?.start(\n    lc\n      .withContext('taskID', config.taskID)\n      .withContext('component', 'view-syncer'),\n    config.log.queryStatsIntervalSeconds * 1000,\n  );\n\n  const viewSyncerFactory = (\n    id: string,\n    sub: Subscription<ReplicaState>,\n    drainCoordinator: DrainCoordinator,\n  ) => {\n    const logger = lc\n      .withContext('taskID', config.taskID)\n      .withContext('component', 'view-syncer')\n      .withContext('appID', shard.appID)\n      .withContext('shardNum', shard.shardNum)\n      .withContext('clientGroupID', id)\n      .withContext('instance', randomID());\n\n    const customQueryTransformer =\n      customQueryConfig && new CustomQueryTransformer(logger, shard);\n    const connContextManager = new ConnectionContextManagerImpl(\n      logger,\n      config.auth.revalidateIntervalSeconds,\n      config.auth.retransformIntervalSeconds,\n      customQueryConfig,\n      pushConfig,\n      validateLegacyJWT,\n    );\n\n    lc.debug?.(\n      `creating view syncer. Query Planner Enabled: ${config.enableQueryPlanner}`,\n    );\n\n    const inspectorDelegate = new InspectorDelegate(customQueryTransformer);\n\n    const priorityOpRunningYieldThresholdMs = Math.max(\n      config.yieldThresholdMs / 4,\n      2,\n    );\n    const normalYieldThresholdMs = Math.max(config.yieldThresholdMs, 2);\n\n    return new ViewSyncerService(\n      config,\n      logger,\n      shard,\n      config.taskID,\n      id,\n      cvrDB,\n      new PipelineDriver(\n        logger,\n        config.log,\n        new Snapshotter(\n          logger,\n          replicaFile,\n          shard,\n          undefined,\n          snapshotRowCache,\n        ),\n        shard,\n        operatorStorage.createClientGroupStorage(id),\n        id,\n        inspectorDelegate,\n        () =>\n          isPriorityOpRunning()\n            ? priorityOpRunningYieldThresholdMs\n            : normalYieldThresholdMs,\n        config.enableQueryPlanner,\n        config,\n        deferredWritesBudget,\n        queryStats,\n      ),\n      sub,\n      drainCoordinator,\n      config.log.slowHydrateThreshold,\n      inspectorDelegate,\n      connContextManager,\n      customQueryTransformer,\n      runPriorityOp,\n      undefined,\n      undefined,\n      undefined,\n      undefined,\n      batchPacer,\n    );\n  };\n\n  const mutagenFactory = upstreamDB\n    ? (id: string) =>\n        new MutagenService(\n          lc\n            .withContext('component', 'mutagen')\n            .withContext('clientGroupID', id),\n          shard,\n          id,\n          upstreamDB,\n          config,\n          writeAuthzStorage,\n        )\n    : undefined;\n\n  const pusherFactory =\n    pushConfig === undefined\n      ? undefined\n      : (id: string, connContextManager: ConnectionContextManager) =>\n          new PusherService(\n            config,\n            lc.withContext('clientGroupID', id),\n            id,\n            connContextManager,\n          );\n\n  const syncer = new Syncer(\n    lc,\n    config,\n    viewSyncerFactory,\n    mutagenFactory,\n    pusherFactory,\n    parent,\n    validateLegacyJWT,\n  );\n\n  startAnonymousTelemetry(lc, config);\n\n  void dbWarmup.then(() => parent.send(['ready', {ready: true}]));\n\n  try {\n    return await runUntilKilled(lc, parent, syncer);\n  } finally {\n    // Logs the work done since the last interval.\n    stopQueryStats?.();\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsDA,SAAS,WAAW;CAClB,OAAO,QAAQ,GAAG,OAAO,gBAAgB,EAAE,SAAS,EAAE;AACxD;AAEA,SAAS,qBACP,QACA;CACA,MAAM,cAAc,OAAO,OAAO,MAAM,OAAO,QAAQ,OAAO;CAE9D,IAAI,CAAC,aAAa,KAChB;CAGF,OAAO;EACL,KAAK,YAAY;EACjB,QAAQ,YAAY;EACpB,sBAAsB,YAAY;EAClC,uBAAuB,YAAY;EACnC,gBAAgB,YAAY,kBAAkB;CAChD;AACF;AAGA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,UAAU,GAAG,4CAA4C;CACrE,MAAM,WAAW,MAAQ,KAAK,IAAI,qBAAqB;CACvD,MAAM,cAAc,OAAO,KAAK,EAAE;CAClC,sBAAsB,QAAQ,UAAU,eAAe,WAAW;CAClE,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CAEjE,cACE,iBAAiB,QAAQ,UAAU,aAAa,KAAK,GACrD,UACA,WACF;CACA,KAAK,iBAAiB,QAAQ,UAAU,WAAW;CACnD,cAAc,IAAI,MAAM;CAExB,MAAM,EAAC,KAAK,UAAU,wBAAuB;CAE7C,MAAM,cAAc,gBAAgB,OAAO,QAAQ,MAAM,QAAQ;CACjE,yCACE;EACE,WAAW;EACX,QAAQ;CACV,GACA,OAAO,sBACT;CACA,GAAG,QAAQ,0BAA0B,aAAa;CAElD,MAAM,QAAQ,MAAM,gBAAgB,IAAI,IAAI,IAAI,eAAe,IAAI,OAAO,EACxE,KAAK,KAAK,IAAI,mBAAmB,mCAAmC,EACtE,CAAC;CAED,MAAM,aACJ,uBAAuB,SAAS,SAAS,OACrC,MAAM,gBAAgB,IAAI,SAAS,IAAI,eAAe,IAAI,YAAY,EACpE,KAAK,KACH,SAAS,mBACT,wCACF,EACF,CAAC,IACD,KAAA;CAEN,MAAM,WAAW,QAAQ,WAAW,CAClC,kBAAkB,IAAI,OAAO,KAAK,GAClC,aAAa,kBAAkB,IAAI,YAAY,UAAU,IAAI,WAC/D,CAAC;CAED,MAAM,SAAS,OAAO,mBAAmB,OAAO;CAChD,MAAM,kBAAkB,gBAAgB,OACtC,IACA,KAAK,KAAK,QAAQ,eAAe,WAAW,GAAG,CACjD;CACA,MAAM,oBAAoB,gBAAgB,OACxC,IACA,KAAK,KAAK,QAAQ,WAAW,WAAW,GAAG,CAC7C;CAEA,MAAM,QAAQ,WAAW,MAAM;CAC/B,MAAM,oBAAoB,qBAAqB,MAAM;CACrD,MAAM,aACJ,OAAO,KAAK,QAAQ,KAAA,KAAa,OAAO,OAAO,QAAQ,KAAA,IACnD,KAAA,IACA;EACE,GAAG,OAAO;EACV,GAAG,OAAO;EACV,KAAK,KACH,OAAO,KAAK,OAAO,OAAO,OAAO,KACjC,kCACF;CACF;;CAGN,IAAI,oBAAmD,KAAA;CAGvD,IADqB,mBAAmB,OAAO,QAAQ,CAAC,CACpD,EAAa,WAAW,GAC1B,oBAAoB,OAAO,OAAO,EAAC,aAAY;EAC7C,IAAI,CAAC,QACH,MAAM,IAAI,uBACR;GACE,MAAM;GACN,SAAS;GACT,QAAQ;EACV,GACA,MACF;EAUF,OAAO;GACL,MAAM;GACN,KAAK;GACL,SAAA,MAVoB,YAAY,OAAO,MAAM,OAAO;IACpD,SAAS;IACT,GAAI,OAAO,MAAM,UAAU,EAAC,QAAQ,OAAO,KAAK,OAAM;IACtD,GAAI,OAAO,MAAM,YAAY,EAC3B,UAAU,OAAO,KAAK,SACxB;GACF,CAAC;EAKD;CACF;CAMF,MAAM,mBACJ,OAAO,uBAAuB,IAC1B,IAAI,iBAAiB,OAAO,oBAAoB,IAChD,KAAA;CAIN,MAAM,uBAAuB,OAAO,iBAChC,qBAAqB,kBACnB,OAAO,4BACT,IACA,KAAA;CACJ,IAAI,sBAAsB;EACxB,GAAG,OACD,sCAAsC,qBAAqB,QAAQ,WACtD,qBAAqB,WAAW,QAAQ,GAAG,QAAQ,CAAC,EAAE,0BAErE;EACA,iBACE,QACA,qCACA,oGAEF,EAAE,aAAY,MAAK,EAAE,QAAQ,qBAAqB,YAAY,CAAC;EAC/D,iBAAiB,QAAQ,kCAAkC;GACzD,aACE;GAEF,MAAM;EACR,CAAC,EAAE,aAAY,MAAK,EAAE,QAAQ,qBAAqB,SAAS,CAAC;CAC/D;CAIA,MAAM,aAAa,OAAO,uBACtB,IAAI,WAAW;EACb,aAAa,OAAO;EACpB,eAAe,OAAO;EACtB;CACF,CAAC,IACD,KAAA;CACJ,IAAI,YAAY;EACd,iBAAiB,QAAQ,qBAAqB;GAC5C,aACE;GAEF,MAAM;EACR,CAAC,EAAE,aAAY,MAAK,EAAE,QAAQ,WAAW,KAAK,CAAC;EAC/C,iBAAiB,QAAQ,yBAAyB;GAChD,aACE;GAEF,MAAM;EACR,CAAC,EAAE,aAAY,MAAK,EAAE,QAAQ,WAAW,UAAU,CAAC;CACtD;CAIA,MAAM,aACJ,OAAO,IAAI,4BAA4B,IAAI,IAAI,WAAW,IAAI,KAAA;CAChE,MAAM,iBAAiB,YAAY,MACjC,GACG,YAAY,UAAU,OAAO,MAAM,EACnC,YAAY,aAAa,aAAa,GACzC,OAAO,IAAI,4BAA4B,GACzC;CAEA,MAAM,qBACJ,IACA,KACA,qBACG;EACH,MAAM,SAAS,GACZ,YAAY,UAAU,OAAO,MAAM,EACnC,YAAY,aAAa,aAAa,EACtC,YAAY,SAAS,MAAM,KAAK,EAChC,YAAY,YAAY,MAAM,QAAQ,EACtC,YAAY,iBAAiB,EAAE,EAC/B,YAAY,YAAY,SAAS,CAAC;EAErC,MAAM,yBACJ,qBAAqB,IAAI,uBAAuB,QAAQ,KAAK;EAC/D,MAAM,qBAAqB,IAAI,6BAC7B,QACA,OAAO,KAAK,2BACZ,OAAO,KAAK,4BACZ,mBACA,YACA,iBACF;EAEA,GAAG,QACD,gDAAgD,OAAO,oBACzD;EAEA,MAAM,oBAAoB,IAAI,kBAAkB,sBAAsB;EAEtE,MAAM,oCAAoC,KAAK,IAC7C,OAAO,mBAAmB,GAC1B,CACF;EACA,MAAM,yBAAyB,KAAK,IAAI,OAAO,kBAAkB,CAAC;EAElE,OAAO,IAAI,kBACT,QACA,QACA,OACA,OAAO,QACP,IACA,OACA,IAAI,eACF,QACA,OAAO,KACP,IAAI,YACF,QACA,aACA,OACA,KAAA,GACA,gBACF,GACA,OACA,gBAAgB,yBAAyB,EAAE,GAC3C,IACA,yBAEE,oBAAoB,IAChB,oCACA,wBACN,OAAO,oBACP,QACA,sBACA,UACF,GACA,KACA,kBACA,OAAO,IAAI,sBACX,mBACA,oBACA,wBACA,eACA,KAAA,GACA,KAAA,GACA,KAAA,GACA,KAAA,GACA,UACF;CACF;CA2BA,MAAM,SAAS,IAAI,OACjB,IACA,QACA,mBA5BqB,cAClB,OACC,IAAI,eACF,GACG,YAAY,aAAa,SAAS,EAClC,YAAY,iBAAiB,EAAE,GAClC,OACA,IACA,YACA,QACA,iBACF,IACF,KAAA,GAGF,eAAe,KAAA,IACX,KAAA,KACC,IAAY,uBACX,IAAI,cACF,QACA,GAAG,YAAY,iBAAiB,EAAE,GAClC,IACA,kBACF,GAQN,QACA,iBACF;CAEA,wBAAwB,IAAI,MAAM;CAElC,SAAc,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CAE9D,IAAI;EACF,OAAO,MAAM,eAAe,IAAI,QAAQ,MAAM;CAChD,UAAU;EAER,iBAAiB;CACnB;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}
+{"version":3,"file":"syncer.js","names":["v.parse"],"sources":["../../../../../zero-cache/src/server/syncer.ts"],"sourcesContent":["import {randomUUID} from 'node:crypto';\nimport {tmpdir} from 'node:os';\nimport path from 'node:path';\nimport {pid} from 'node:process';\nimport {getHeapStatistics} from 'node:v8';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {DatabaseStorage} from '../../../zqlite/src/database-storage.ts';\nimport type {ValidateLegacyJWT} from '../auth/auth.ts';\nimport {tokenConfigOptions, verifyToken} from '../auth/jwt.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {CustomQueryTransformer} from '../custom-queries/transform-query.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {MutagenService} from '../services/mutagen/mutagen.ts';\nimport {PusherService} from '../services/mutagen/pusher.ts';\nimport type {ReplicaState} from '../services/replicator/replicator.ts';\nimport {BatchPacer} from '../services/view-syncer/batch-pacer.ts';\nimport {\n  type ConnectionContextManager,\n  ConnectionContextManagerImpl,\n} from '../services/view-syncer/connection-context-manager.ts';\nimport {DeferredWritesBudget} from '../services/view-syncer/deferred-writes-budget.ts';\nimport type {DrainCoordinator} from '../services/view-syncer/drain-coordinator.ts';\nimport {PipelineDriver} from '../services/view-syncer/pipeline-driver.ts';\nimport {QueryStats} from '../services/view-syncer/query-stats.ts';\nimport {SnapshotRowCache} from '../services/view-syncer/snapshot-row-cache.ts';\nimport {Snapshotter} from '../services/view-syncer/snapshotter.ts';\nimport {ViewSyncerService} from '../services/view-syncer/view-syncer.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardID} from '../types/shards.ts';\nimport type {Subscription} from '../types/subscription.ts';\nimport {replicaFileModeSchema, replicaFileName} from '../workers/replicator.ts';\nimport {Syncer} from '../workers/syncer.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {InspectorDelegate} from './inspector-delegate.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {isPriorityOpRunning, runPriorityOp} from './priority-op.ts';\n\nfunction randomID() {\n  return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);\n}\n\nfunction getCustomQueryConfig(\n  config: Pick<NormalizedZeroConfig, 'query' | 'getQueries'>,\n) {\n  const queryConfig = config.query?.url ? config.query : config.getQueries;\n\n  if (!queryConfig?.url) {\n    return undefined;\n  }\n\n  return {\n    url: queryConfig.url,\n    apiKey: queryConfig.apiKey,\n    allowedClientHeaders: queryConfig.allowedClientHeaders,\n    allowedRequestHeaders: queryConfig.allowedRequestHeaders,\n    forwardCookies: queryConfig.forwardCookies ?? false,\n  };\n}\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length >= 2, `expected [fileMode, workerIndex, ...flags]`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n  const workerIndex = Number(args[1]);\n  installProfileHandler(parent, `syncer-${workerIndex}`, workerIndex);\n  const config = getNormalizedZeroConfig({env, argv: args.slice(2)});\n\n  startOtelAuto(\n    createLogContext(config, 'syncer', workerIndex, false),\n    'syncer',\n    workerIndex,\n  );\n  lc = createLogContext(config, 'syncer', workerIndex);\n  initEventSink(lc, config);\n\n  const {cvr, upstream, enableCrudMutations} = config;\n\n  const replicaFile = replicaFileName(config.replica.file, fileMode);\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'syncer replica',\n      dbPath: replicaFile,\n    },\n    config.sqliteCorruptionChecks,\n  );\n  lc.debug?.(`running view-syncer on ${replicaFile}`);\n\n  const cvrDB = await connectPgClient(lc, cvr.db, `sync-worker-${pid}-cvr`, {\n    max: must(cvr.maxConnsPerWorker, 'cvr.maxConnsPerWorker must be set'),\n  });\n\n  const upstreamDB =\n    enableCrudMutations && upstream.type === 'pg'\n      ? await connectPgClient(lc, upstream.db, `sync-worker-${pid}-upstream`, {\n          max: must(\n            upstream.maxConnsPerWorker,\n            'upstream.maxConnsPerWorker must be set',\n          ),\n        })\n      : undefined;\n\n  const dbWarmup = Promise.allSettled([\n    warmupConnections(lc, cvrDB, 'cvr'),\n    upstreamDB ? warmupConnections(lc, upstreamDB, 'upstream') : promiseVoid,\n  ]);\n\n  const tmpDir = config.storageDBTmpDir ?? tmpdir();\n  const operatorStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `sync-worker-${randomUUID()}`),\n    {\n      maxMemoryBytes: Math.floor(\n        getHeapStatistics().heap_size_limit *\n          config.operatorStorageHeapProportion,\n      ),\n    },\n  );\n  getOrCreateGauge('sync', 'ivm.operator-storage-memory-bytes', {\n    description:\n      'Estimated bytes of IVM operator state that the client groups of a ' +\n      'sync worker keep in memory rather than in SQLite',\n    unit: 'By',\n  }).addCallback(o => o.observe(operatorStorage.memoryBytes));\n  const writeAuthzStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `mutagen-${randomUUID()}`),\n  );\n\n  const shard = getShardID(config);\n  const customQueryConfig = getCustomQueryConfig(config);\n  const pushConfig =\n    config.push.url === undefined && config.mutate.url === undefined\n      ? undefined\n      : {\n          ...config.push,\n          ...config.mutate,\n          url: must(\n            config.push.url ?? config.mutate.url,\n            'No push or mutate URL configured',\n          ),\n        };\n\n  /** @deprecated used in JWT validation */\n  let validateLegacyJWT: ValidateLegacyJWT | undefined = undefined;\n\n  const tokenOptions = tokenConfigOptions(config.auth ?? {});\n  if (tokenOptions.length === 1) {\n    validateLegacyJWT = async (token, {userID}) => {\n      if (!userID) {\n        throw new ProtocolErrorWithLevel(\n          {\n            kind: 'Unauthorized',\n            message: 'UserID is required for JWT validation.',\n            origin: 'zeroCache',\n          },\n          'warn',\n        );\n      }\n\n      const decoded = await verifyToken(config.auth, token, {\n        subject: userID,\n        ...(config.auth?.issuer && {issuer: config.auth.issuer}),\n        ...(config.auth?.audience && {\n          audience: config.auth.audience,\n        }),\n      });\n      return {\n        type: 'jwt',\n        raw: token,\n        decoded,\n      };\n    };\n  }\n\n  // Shared by all of the view-syncers on this worker so that the row reads\n  // performed when advancing their pipelines are done once per worker rather\n  // than once per client group.\n  const snapshotRowCache =\n    config.snapshotRowCacheSize > 0\n      ? new SnapshotRowCache(config.snapshotRowCacheSize)\n      : undefined;\n\n  // Shared by all of the view-syncers on this worker, which each hold their\n  // own copy of the changes they are advancing through.\n  const deferredWritesBudget = config.deferIvmWrites\n    ? DeferredWritesBudget.forHeapProportion(\n        config.deferIvmWritesHeapProportion,\n      )\n    : undefined;\n  if (deferredWritesBudget) {\n    lc.info?.(\n      `Deferred IVM writes may hold up to ${deferredWritesBudget.maxRows} ` +\n        `rows (~${(deferredWritesBudget.maxBytes / 1024 ** 2).toFixed(2)} MB) ` +\n        `across client groups`,\n    );\n    getOrCreateGauge(\n      'sync',\n      'ivm.deferred-writes-reserved-rows',\n      'Rows reserved by the client groups of a sync worker to hold IVM ' +\n        'changes in memory (deferIvmWrites)',\n    ).addCallback(o => o.observe(deferredWritesBudget.reservedRows));\n    getOrCreateGauge('sync', 'ivm.deferred-writes-held-bytes', {\n      description:\n        'Estimated bytes of the IVM changes that the client groups of a sync ' +\n        'worker hold in memory (deferIvmWrites)',\n      unit: 'By',\n    }).addCallback(o => o.observe(deferredWritesBudget.heldBytes));\n  }\n\n  // Shared by all of the view-syncers on this worker, whose batches of\n  // changes together are held to a share of the worker's time.\n  const batchPacer = config.costBasedSyncEnabled\n    ? new BatchPacer({\n        shareTarget: config.syncUpdateShareTarget,\n        maxIntervalMs: config.syncMaxBatchIntervalMs,\n        lc,\n      })\n    : undefined;\n  if (batchPacer) {\n    getOrCreateGauge('sync', 'worker-busy-share', {\n      description:\n        'Share of its time that a sync worker was busy over the last few ' +\n        'seconds, which the pacing of its client groups holds to a target',\n      unit: '1',\n    }).addCallback(o => o.observe(batchPacer.share));\n    getOrCreateGauge('sync', 'batch-wait-multiplier', {\n      description:\n        'How many times the processing time of a batch of changes the ' +\n        'client groups of a sync worker wait before their next batch',\n      unit: '1',\n    }).addCallback(o => o.observe(batchPacer.multiplier));\n  }\n\n  // Shared by all of the view-syncers on this worker, so that the work done\n  // for each query shape is logged once per worker per interval.\n  const queryStats =\n    config.log.queryStatsIntervalSeconds > 0 ? new QueryStats() : undefined;\n  const stopQueryStats = queryStats?.start(\n    lc\n      .withContext('taskID', config.taskID)\n      .withContext('component', 'view-syncer'),\n    config.log.queryStatsIntervalSeconds * 1000,\n  );\n\n  const viewSyncerFactory = (\n    id: string,\n    sub: Subscription<ReplicaState>,\n    drainCoordinator: DrainCoordinator,\n  ) => {\n    const logger = lc\n      .withContext('taskID', config.taskID)\n      .withContext('component', 'view-syncer')\n      .withContext('appID', shard.appID)\n      .withContext('shardNum', shard.shardNum)\n      .withContext('clientGroupID', id)\n      .withContext('instance', randomID());\n\n    const customQueryTransformer =\n      customQueryConfig && new CustomQueryTransformer(logger, shard);\n    const connContextManager = new ConnectionContextManagerImpl(\n      logger,\n      config.auth.revalidateIntervalSeconds,\n      config.auth.retransformIntervalSeconds,\n      customQueryConfig,\n      pushConfig,\n      validateLegacyJWT,\n    );\n\n    lc.debug?.(\n      `creating view syncer. Query Planner Enabled: ${config.enableQueryPlanner}`,\n    );\n\n    const inspectorDelegate = new InspectorDelegate(customQueryTransformer);\n\n    const priorityOpRunningYieldThresholdMs = Math.max(\n      config.yieldThresholdMs / 4,\n      2,\n    );\n    const normalYieldThresholdMs = Math.max(config.yieldThresholdMs, 2);\n\n    return new ViewSyncerService(\n      config,\n      logger,\n      shard,\n      config.taskID,\n      id,\n      cvrDB,\n      new PipelineDriver(\n        logger,\n        config.log,\n        new Snapshotter(\n          logger,\n          replicaFile,\n          shard,\n          undefined,\n          snapshotRowCache,\n        ),\n        shard,\n        operatorStorage.createClientGroupStorage(id),\n        id,\n        inspectorDelegate,\n        () =>\n          isPriorityOpRunning()\n            ? priorityOpRunningYieldThresholdMs\n            : normalYieldThresholdMs,\n        config.enableQueryPlanner,\n        config,\n        deferredWritesBudget,\n        queryStats,\n      ),\n      sub,\n      drainCoordinator,\n      config.log.slowHydrateThreshold,\n      inspectorDelegate,\n      connContextManager,\n      customQueryTransformer,\n      runPriorityOp,\n      undefined,\n      undefined,\n      undefined,\n      undefined,\n      batchPacer,\n    );\n  };\n\n  const mutagenFactory = upstreamDB\n    ? (id: string) =>\n        new MutagenService(\n          lc\n            .withContext('component', 'mutagen')\n            .withContext('clientGroupID', id),\n          shard,\n          id,\n          upstreamDB,\n          config,\n          writeAuthzStorage,\n        )\n    : undefined;\n\n  const pusherFactory =\n    pushConfig === undefined\n      ? undefined\n      : (id: string, connContextManager: ConnectionContextManager) =>\n          new PusherService(\n            config,\n            lc.withContext('clientGroupID', id),\n            id,\n            connContextManager,\n          );\n\n  const syncer = new Syncer(\n    lc,\n    config,\n    viewSyncerFactory,\n    mutagenFactory,\n    pusherFactory,\n    parent,\n    validateLegacyJWT,\n  );\n\n  startAnonymousTelemetry(lc, config);\n\n  void dbWarmup.then(() => parent.send(['ready', {ready: true}]));\n\n  try {\n    return await runUntilKilled(lc, parent, syncer);\n  } finally {\n    // Logs the work done since the last interval.\n    stopQueryStats?.();\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuDA,SAAS,WAAW;CAClB,OAAO,QAAQ,GAAG,OAAO,gBAAgB,CAAC,CAAC,SAAS,EAAE;AACxD;AAEA,SAAS,qBACP,QACA;CACA,MAAM,cAAc,OAAO,OAAO,MAAM,OAAO,QAAQ,OAAO;CAE9D,IAAI,CAAC,aAAa,KAChB;CAGF,OAAO;EACL,KAAK,YAAY;EACjB,QAAQ,YAAY;EACpB,sBAAsB,YAAY;EAClC,uBAAuB,YAAY;EACnC,gBAAgB,YAAY,kBAAkB;CAChD;AACF;AAGA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,UAAU,GAAG,4CAA4C;CACrE,MAAM,WAAWA,MAAQ,KAAK,IAAI,qBAAqB;CACvD,MAAM,cAAc,OAAO,KAAK,EAAE;CAClC,sBAAsB,QAAQ,UAAU,eAAe,WAAW;CAClE,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CAEjE,cACE,iBAAiB,QAAQ,UAAU,aAAa,KAAK,GACrD,UACA,WACF;CACA,KAAK,iBAAiB,QAAQ,UAAU,WAAW;CACnD,cAAc,IAAI,MAAM;CAExB,MAAM,EAAC,KAAK,UAAU,wBAAuB;CAE7C,MAAM,cAAc,gBAAgB,OAAO,QAAQ,MAAM,QAAQ;CACjE,yCACE;EACE,WAAW;EACX,QAAQ;CACV,GACA,OAAO,sBACT;CACA,GAAG,QAAQ,0BAA0B,aAAa;CAElD,MAAM,QAAQ,MAAM,gBAAgB,IAAI,IAAI,IAAI,eAAe,IAAI,OAAO,EACxE,KAAK,KAAK,IAAI,mBAAmB,mCAAmC,EACtE,CAAC;CAED,MAAM,aACJ,uBAAuB,SAAS,SAAS,OACrC,MAAM,gBAAgB,IAAI,SAAS,IAAI,eAAe,IAAI,YAAY,EACpE,KAAK,KACH,SAAS,mBACT,wCACF,EACF,CAAC,IACD,KAAA;CAEN,MAAM,WAAW,QAAQ,WAAW,CAClC,kBAAkB,IAAI,OAAO,KAAK,GAClC,aAAa,kBAAkB,IAAI,YAAY,UAAU,IAAI,WAC/D,CAAC;CAED,MAAM,SAAS,OAAO,mBAAmB,OAAO;CAChD,MAAM,kBAAkB,gBAAgB,OACtC,IACA,KAAK,KAAK,QAAQ,eAAe,WAAW,GAAG,GAC/C,EACE,gBAAgB,KAAK,MACnB,kBAAkB,CAAC,CAAC,kBAClB,OAAO,6BACX,EACF,CACF;CACA,iBAAiB,QAAQ,qCAAqC;EAC5D,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,aAAY,MAAK,EAAE,QAAQ,gBAAgB,WAAW,CAAC;CAC1D,MAAM,oBAAoB,gBAAgB,OACxC,IACA,KAAK,KAAK,QAAQ,WAAW,WAAW,GAAG,CAC7C;CAEA,MAAM,QAAQ,WAAW,MAAM;CAC/B,MAAM,oBAAoB,qBAAqB,MAAM;CACrD,MAAM,aACJ,OAAO,KAAK,QAAQ,KAAA,KAAa,OAAO,OAAO,QAAQ,KAAA,IACnD,KAAA,IACA;EACE,GAAG,OAAO;EACV,GAAG,OAAO;EACV,KAAK,KACH,OAAO,KAAK,OAAO,OAAO,OAAO,KACjC,kCACF;CACF;;CAGN,IAAI,oBAAmD,KAAA;CAGvD,IADqB,mBAAmB,OAAO,QAAQ,CAAC,CACpD,CAAA,CAAa,WAAW,GAC1B,oBAAoB,OAAO,OAAO,EAAC,aAAY;EAC7C,IAAI,CAAC,QACH,MAAM,IAAI,uBACR;GACE,MAAM;GACN,SAAS;GACT,QAAQ;EACV,GACA,MACF;EAUF,OAAO;GACL,MAAM;GACN,KAAK;GACL,SAAA,MAVoB,YAAY,OAAO,MAAM,OAAO;IACpD,SAAS;IACT,GAAI,OAAO,MAAM,UAAU,EAAC,QAAQ,OAAO,KAAK,OAAM;IACtD,GAAI,OAAO,MAAM,YAAY,EAC3B,UAAU,OAAO,KAAK,SACxB;GACF,CAAC;EAKD;CACF;CAMF,MAAM,mBACJ,OAAO,uBAAuB,IAC1B,IAAI,iBAAiB,OAAO,oBAAoB,IAChD,KAAA;CAIN,MAAM,uBAAuB,OAAO,iBAChC,qBAAqB,kBACnB,OAAO,4BACT,IACA,KAAA;CACJ,IAAI,sBAAsB;EACxB,GAAG,OACD,sCAAsC,qBAAqB,QAAQ,WACtD,qBAAqB,WAAW,QAAQ,EAAA,CAAG,QAAQ,CAAC,EAAE,0BAErE;EACA,iBACE,QACA,qCACA,oGAEF,CAAC,CAAC,aAAY,MAAK,EAAE,QAAQ,qBAAqB,YAAY,CAAC;EAC/D,iBAAiB,QAAQ,kCAAkC;GACzD,aACE;GAEF,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,MAAK,EAAE,QAAQ,qBAAqB,SAAS,CAAC;CAC/D;CAIA,MAAM,aAAa,OAAO,uBACtB,IAAI,WAAW;EACb,aAAa,OAAO;EACpB,eAAe,OAAO;EACtB;CACF,CAAC,IACD,KAAA;CACJ,IAAI,YAAY;EACd,iBAAiB,QAAQ,qBAAqB;GAC5C,aACE;GAEF,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,MAAK,EAAE,QAAQ,WAAW,KAAK,CAAC;EAC/C,iBAAiB,QAAQ,yBAAyB;GAChD,aACE;GAEF,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,MAAK,EAAE,QAAQ,WAAW,UAAU,CAAC;CACtD;CAIA,MAAM,aACJ,OAAO,IAAI,4BAA4B,IAAI,IAAI,WAAW,IAAI,KAAA;CAChE,MAAM,iBAAiB,YAAY,MACjC,GACG,YAAY,UAAU,OAAO,MAAM,CAAC,CACpC,YAAY,aAAa,aAAa,GACzC,OAAO,IAAI,4BAA4B,GACzC;CAEA,MAAM,qBACJ,IACA,KACA,qBACG;EACH,MAAM,SAAS,GACZ,YAAY,UAAU,OAAO,MAAM,CAAC,CACpC,YAAY,aAAa,aAAa,CAAC,CACvC,YAAY,SAAS,MAAM,KAAK,CAAC,CACjC,YAAY,YAAY,MAAM,QAAQ,CAAC,CACvC,YAAY,iBAAiB,EAAE,CAAC,CAChC,YAAY,YAAY,SAAS,CAAC;EAErC,MAAM,yBACJ,qBAAqB,IAAI,uBAAuB,QAAQ,KAAK;EAC/D,MAAM,qBAAqB,IAAI,6BAC7B,QACA,OAAO,KAAK,2BACZ,OAAO,KAAK,4BACZ,mBACA,YACA,iBACF;EAEA,GAAG,QACD,gDAAgD,OAAO,oBACzD;EAEA,MAAM,oBAAoB,IAAI,kBAAkB,sBAAsB;EAEtE,MAAM,oCAAoC,KAAK,IAC7C,OAAO,mBAAmB,GAC1B,CACF;EACA,MAAM,yBAAyB,KAAK,IAAI,OAAO,kBAAkB,CAAC;EAElE,OAAO,IAAI,kBACT,QACA,QACA,OACA,OAAO,QACP,IACA,OACA,IAAI,eACF,QACA,OAAO,KACP,IAAI,YACF,QACA,aACA,OACA,KAAA,GACA,gBACF,GACA,OACA,gBAAgB,yBAAyB,EAAE,GAC3C,IACA,yBAEE,oBAAoB,IAChB,oCACA,wBACN,OAAO,oBACP,QACA,sBACA,UACF,GACA,KACA,kBACA,OAAO,IAAI,sBACX,mBACA,oBACA,wBACA,eACA,KAAA,GACA,KAAA,GACA,KAAA,GACA,KAAA,GACA,UACF;CACF;CA2BA,MAAM,SAAS,IAAI,OACjB,IACA,QACA,mBA5BqB,cAClB,OACC,IAAI,eACF,GACG,YAAY,aAAa,SAAS,CAAC,CACnC,YAAY,iBAAiB,EAAE,GAClC,OACA,IACA,YACA,QACA,iBACF,IACF,KAAA,GAGF,eAAe,KAAA,IACX,KAAA,KACC,IAAY,uBACX,IAAI,cACF,QACA,GAAG,YAAY,iBAAiB,EAAE,GAClC,IACA,kBACF,GAQN,QACA,iBACF;CAEA,wBAAwB,IAAI,MAAM;CAElC,SAAc,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CAE9D,IAAI;EACF,OAAO,MAAM,eAAe,IAAI,QAAQ,MAAM;CAChD,UAAU;EAER,iBAAiB;CACnB;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}

package/out/zero-cache/src/server/worker-dispatcher.js

@@ -103,6 +103,6 @@ function parsePath(url) {
 	return URL_PATTERN.match(url.pathname) || void 0;
 }
 //#endregion
-export { WorkerDispatcher };
+export { WorkerDispatcher, parsePath };
 
 //# sourceMappingURL=worker-dispatcher.js.map

package/out/zero-cache/src/server/worker-dispatcher.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker-dispatcher.js","names":["#lc","#state","#assigner"],"sources":["../../../../../zero-cache/src/server/worker-dispatcher.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport UrlPattern from 'url-pattern';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n} from '../observability/metrics.ts';\nimport {RunningState} from '../services/running-state.ts';\nimport type {Service} from '../services/service.ts';\nimport type {IncomingMessageSubset} from '../types/http.ts';\nimport type {ClientGroupStatusMessage, Worker} from '../types/processes.ts';\nimport {installWebSocketHandoff} from '../types/websocket-handoff.ts';\nimport {getConnectParams} from '../workers/connect-params.ts';\nimport {SyncerAssigner} from './syncer-assigner.ts';\n\nexport class WorkerDispatcher implements Service {\n  readonly id = 'worker-dispatcher';\n  readonly #lc: LogContext;\n  readonly #state = new RunningState(this.id);\n  readonly #assigner: SyncerAssigner;\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    parent: Worker,\n    syncers: Worker[],\n    mutator: Worker | undefined,\n    changeStreamer: Worker | undefined,\n  ) {\n    this.#lc = lc;\n\n    const assigner = new SyncerAssigner(taskID, syncers.length);\n    this.#assigner = assigner;\n\n    const workerDispatchesCounter = getOrCreateCounter(\n      'sync',\n      'worker-dispatches',\n      'Dispatched client group connections to worker',\n    );\n\n    getOrCreateGauge(\n      'sync',\n      'worker-client-groups',\n      'Number of active client groups assigned to worker',\n    ).addCallback(result => {\n      for (let i = 0; i < syncers.length; i++) {\n        result.observe(assigner.getWorkerLoad(i), {worker: String(i)});\n      }\n    });\n\n    syncers.forEach((syncer, index) => {\n      syncer.onMessageType<ClientGroupStatusMessage>(\n        'clientGroupStatus',\n        ({clientGroupID, active, generation}) => {\n          if (!active) {\n            assigner.release(clientGroupID, index, generation);\n          }\n        },\n      );\n    });\n\n    function connectParams(req: IncomingMessageSubset) {\n      const {headers, url: u} = req;\n      const url = new URL(u ?? '', 'http://unused/');\n      const path = parsePath(url);\n      if (!path) {\n        throw new Error(`Invalid URL: ${u}`);\n      }\n      const version = Number(path.version);\n      if (Number.isNaN(version)) {\n        throw new Error(`Invalid version: ${u}`);\n      }\n      const {params, error} = getConnectParams(version, url, headers);\n      if (error !== null) {\n        throw new Error(error);\n      }\n      return params;\n    }\n\n    const handlePush = (req: IncomingMessageSubset) => {\n      assert(\n        mutator !== undefined,\n        'Received a push for a custom mutation but no `push.url` was configured.',\n      );\n      return {payload: connectParams(req), sender: mutator};\n    };\n\n    let maxProtocolVersion = 0;\n    getOrCreateGauge(\n      'sync',\n      'max-protocol-version',\n      'Latest sync protocol version from a connecting client',\n    ).addCallback(result => {\n      if (maxProtocolVersion) {\n        result.observe(maxProtocolVersion);\n      }\n    });\n\n    const handleSync = (req: IncomingMessageSubset) => {\n      assert(syncers.length, 'Received a sync request with no sync workers.');\n      const params = connectParams(req);\n      const {clientGroupID, protocolVersion} = params;\n      maxProtocolVersion = Math.max(maxProtocolVersion, protocolVersion);\n\n      const {worker: syncer, generation} = assigner.assign(clientGroupID);\n      workerDispatchesCounter.add(1, {worker: String(syncer)});\n\n      lc.debug?.(`connecting ${clientGroupID} to syncer ${syncer}`);\n      return {payload: {...params, generation}, sender: syncers[syncer]};\n    };\n\n    const handleChangeStream = (req: IncomingMessageSubset) => {\n      // Note: The change-streamer is generally not dispatched via the main\n      //       port, and in particular, should *not* be accessible via that\n      //       port in single-node mode. However, this plumbing is maintained\n      //       for the purpose of allowing --lazy-startup of the\n      //       replication-manager as a possible future feature.\n      assert(\n        syncers.length === 0 && mutator === undefined,\n        'Dispatch to the change-streamer via the main port ' +\n          'is only allowed in multi-node mode',\n      );\n      assert(\n        changeStreamer,\n        'Received a change-streamer request without a change-streamer worker',\n      );\n      const url = new URL(req.url ?? '', 'http://unused/');\n      const path = parsePath(url);\n      if (!path) {\n        throw new Error(`Invalid URL: ${req.url}`);\n      }\n\n      return {\n        payload: path.action,\n        sender: changeStreamer,\n      };\n    };\n\n    // handoff messages from this ZeroDispatcher to the appropriate worker (pool).\n    installWebSocketHandoff<unknown>(\n      lc,\n      request => {\n        const {url: u} = request;\n        const url = new URL(u ?? '', 'http://unused/');\n        const path = parsePath(url);\n        if (!path) {\n          throw new Error(`Invalid URL: ${u}`);\n        }\n        switch (path.worker) {\n          case 'sync':\n            return handleSync(request);\n          case 'replication':\n            return handleChangeStream(request);\n          case 'mutate':\n            return handlePush(request);\n          default:\n            throw new Error(`Invalid URL: ${u}`);\n        }\n      },\n      parent,\n    );\n  }\n\n  run() {\n    const readyStart = Date.now();\n    getOrCreateGauge('server', 'uptime', {\n      description: 'Cumulative uptime, starting from when requests are served',\n      unit: 's',\n    }).addCallback(result => result.observe((Date.now() - readyStart) / 1000));\n\n    return this.#state.stopped();\n  }\n\n  stop() {\n    this.#assigner.destroy();\n    this.#state.stop(this.#lc);\n    return this.#state.stopped();\n  }\n}\n\nconst URL_PATTERN = new UrlPattern('(/:base)/:worker/v:version/:action');\n\nexport function parsePath(url: URL):\n  | {\n      base?: string;\n      worker: 'sync' | 'mutate' | 'replication';\n      version: string;\n      action: string;\n    }\n  | undefined {\n  // The match() returns both null and undefined.\n  return URL_PATTERN.match(url.pathname) || undefined;\n} // The server allows the client to use any /:base/ path to facilitate\n// servicing requests on the same domain as the application.\n"],"mappings":";;;;;;;;AAeA,IAAa,mBAAb,MAAiD;CAC/C,KAAc;CACd;CACA,SAAkB,IAAI,aAAa,KAAK,EAAE;CAC1C;CAEA,YACE,IACA,QACA,QACA,SACA,SACA,gBACA;EACA,KAAKA,MAAM;EAEX,MAAM,WAAW,IAAI,eAAe,QAAQ,QAAQ,MAAM;EAC1D,KAAKE,YAAY;EAEjB,MAAM,0BAA0B,mBAC9B,QACA,qBACA,+CACF;EAEA,iBACE,QACA,wBACA,mDACF,EAAE,aAAY,WAAU;GACtB,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAClC,OAAO,QAAQ,SAAS,cAAc,CAAC,GAAG,EAAC,QAAQ,OAAO,CAAC,EAAC,CAAC;EAEjE,CAAC;EAED,QAAQ,SAAS,QAAQ,UAAU;GACjC,OAAO,cACL,sBACC,EAAC,eAAe,QAAQ,iBAAgB;IACvC,IAAI,CAAC,QACH,SAAS,QAAQ,eAAe,OAAO,UAAU;GAErD,CACF;EACF,CAAC;EAED,SAAS,cAAc,KAA4B;GACjD,MAAM,EAAC,SAAS,KAAK,MAAK;GAC1B,MAAM,MAAM,IAAI,IAAI,KAAK,IAAI,gBAAgB;GAC7C,MAAM,OAAO,UAAU,GAAG;GAC1B,IAAI,CAAC,MACH,MAAM,IAAI,MAAM,gBAAgB,GAAG;GAErC,MAAM,UAAU,OAAO,KAAK,OAAO;GACnC,IAAI,OAAO,MAAM,OAAO,GACtB,MAAM,IAAI,MAAM,oBAAoB,GAAG;GAEzC,MAAM,EAAC,QAAQ,UAAS,iBAAiB,SAAS,KAAK,OAAO;GAC9D,IAAI,UAAU,MACZ,MAAM,IAAI,MAAM,KAAK;GAEvB,OAAO;EACT;EAEA,MAAM,cAAc,QAA+B;GACjD,OACE,YAAY,KAAA,GACZ,yEACF;GACA,OAAO;IAAC,SAAS,cAAc,GAAG;IAAG,QAAQ;GAAO;EACtD;EAEA,IAAI,qBAAqB;EACzB,iBACE,QACA,wBACA,uDACF,EAAE,aAAY,WAAU;GACtB,IAAI,oBACF,OAAO,QAAQ,kBAAkB;EAErC,CAAC;EAED,MAAM,cAAc,QAA+B;GACjD,OAAO,QAAQ,QAAQ,+CAA+C;GACtE,MAAM,SAAS,cAAc,GAAG;GAChC,MAAM,EAAC,eAAe,oBAAmB;GACzC,qBAAqB,KAAK,IAAI,oBAAoB,eAAe;GAEjE,MAAM,EAAC,QAAQ,QAAQ,eAAc,SAAS,OAAO,aAAa;GAClE,wBAAwB,IAAI,GAAG,EAAC,QAAQ,OAAO,MAAM,EAAC,CAAC;GAEvD,GAAG,QAAQ,cAAc,cAAc,aAAa,QAAQ;GAC5D,OAAO;IAAC,SAAS;KAAC,GAAG;KAAQ;IAAU;IAAG,QAAQ,QAAQ;GAAO;EACnE;EAEA,MAAM,sBAAsB,QAA+B;GAMzD,OACE,QAAQ,WAAW,KAAK,YAAY,KAAA,GACpC,sFAEF;GACA,OACE,gBACA,qEACF;GAEA,MAAM,OAAO,UAAU,IADP,IAAI,IAAI,OAAO,IAAI,gBACZ,CAAG;GAC1B,IAAI,CAAC,MACH,MAAM,IAAI,MAAM,gBAAgB,IAAI,KAAK;GAG3C,OAAO;IACL,SAAS,KAAK;IACd,QAAQ;GACV;EACF;EAGA,wBACE,KACA,YAAW;GACT,MAAM,EAAC,KAAK,MAAK;GAEjB,MAAM,OAAO,UAAU,IADP,IAAI,KAAK,IAAI,gBACN,CAAG;GAC1B,IAAI,CAAC,MACH,MAAM,IAAI,MAAM,gBAAgB,GAAG;GAErC,QAAQ,KAAK,QAAb;IACE,KAAK,QACH,OAAO,WAAW,OAAO;IAC3B,KAAK,eACH,OAAO,mBAAmB,OAAO;IACnC,KAAK,UACH,OAAO,WAAW,OAAO;IAC3B,SACE,MAAM,IAAI,MAAM,gBAAgB,GAAG;GACvC;EACF,GACA,MACF;CACF;CAEA,MAAM;EACJ,MAAM,aAAa,KAAK,IAAI;EAC5B,iBAAiB,UAAU,UAAU;GACnC,aAAa;GACb,MAAM;EACR,CAAC,EAAE,aAAY,WAAU,OAAO,SAAS,KAAK,IAAI,IAAI,cAAc,GAAI,CAAC;EAEzE,OAAO,KAAKD,OAAO,QAAQ;CAC7B;CAEA,OAAO;EACL,KAAKC,UAAU,QAAQ;EACvB,KAAKD,OAAO,KAAK,KAAKD,GAAG;EACzB,OAAO,KAAKC,OAAO,QAAQ;CAC7B;AACF;AAEA,IAAM,cAAc,IAAI,WAAW,oCAAoC;AAEvE,SAAgB,UAAU,KAOZ;CAEZ,OAAO,YAAY,MAAM,IAAI,QAAQ,KAAK,KAAA;AAC5C"}
+{"version":3,"file":"worker-dispatcher.js","names":[],"sources":["../../../../../zero-cache/src/server/worker-dispatcher.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport UrlPattern from 'url-pattern';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n} from '../observability/metrics.ts';\nimport {RunningState} from '../services/running-state.ts';\nimport type {Service} from '../services/service.ts';\nimport type {IncomingMessageSubset} from '../types/http.ts';\nimport type {ClientGroupStatusMessage, Worker} from '../types/processes.ts';\nimport {installWebSocketHandoff} from '../types/websocket-handoff.ts';\nimport {getConnectParams} from '../workers/connect-params.ts';\nimport {SyncerAssigner} from './syncer-assigner.ts';\n\nexport class WorkerDispatcher implements Service {\n  readonly id = 'worker-dispatcher';\n  readonly #lc: LogContext;\n  readonly #state = new RunningState(this.id);\n  readonly #assigner: SyncerAssigner;\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    parent: Worker,\n    syncers: Worker[],\n    mutator: Worker | undefined,\n    changeStreamer: Worker | undefined,\n  ) {\n    this.#lc = lc;\n\n    const assigner = new SyncerAssigner(taskID, syncers.length);\n    this.#assigner = assigner;\n\n    const workerDispatchesCounter = getOrCreateCounter(\n      'sync',\n      'worker-dispatches',\n      'Dispatched client group connections to worker',\n    );\n\n    getOrCreateGauge(\n      'sync',\n      'worker-client-groups',\n      'Number of active client groups assigned to worker',\n    ).addCallback(result => {\n      for (let i = 0; i < syncers.length; i++) {\n        result.observe(assigner.getWorkerLoad(i), {worker: String(i)});\n      }\n    });\n\n    syncers.forEach((syncer, index) => {\n      syncer.onMessageType<ClientGroupStatusMessage>(\n        'clientGroupStatus',\n        ({clientGroupID, active, generation}) => {\n          if (!active) {\n            assigner.release(clientGroupID, index, generation);\n          }\n        },\n      );\n    });\n\n    function connectParams(req: IncomingMessageSubset) {\n      const {headers, url: u} = req;\n      const url = new URL(u ?? '', 'http://unused/');\n      const path = parsePath(url);\n      if (!path) {\n        throw new Error(`Invalid URL: ${u}`);\n      }\n      const version = Number(path.version);\n      if (Number.isNaN(version)) {\n        throw new Error(`Invalid version: ${u}`);\n      }\n      const {params, error} = getConnectParams(version, url, headers);\n      if (error !== null) {\n        throw new Error(error);\n      }\n      return params;\n    }\n\n    const handlePush = (req: IncomingMessageSubset) => {\n      assert(\n        mutator !== undefined,\n        'Received a push for a custom mutation but no `push.url` was configured.',\n      );\n      return {payload: connectParams(req), sender: mutator};\n    };\n\n    let maxProtocolVersion = 0;\n    getOrCreateGauge(\n      'sync',\n      'max-protocol-version',\n      'Latest sync protocol version from a connecting client',\n    ).addCallback(result => {\n      if (maxProtocolVersion) {\n        result.observe(maxProtocolVersion);\n      }\n    });\n\n    const handleSync = (req: IncomingMessageSubset) => {\n      assert(syncers.length, 'Received a sync request with no sync workers.');\n      const params = connectParams(req);\n      const {clientGroupID, protocolVersion} = params;\n      maxProtocolVersion = Math.max(maxProtocolVersion, protocolVersion);\n\n      const {worker: syncer, generation} = assigner.assign(clientGroupID);\n      workerDispatchesCounter.add(1, {worker: String(syncer)});\n\n      lc.debug?.(`connecting ${clientGroupID} to syncer ${syncer}`);\n      return {payload: {...params, generation}, sender: syncers[syncer]};\n    };\n\n    const handleChangeStream = (req: IncomingMessageSubset) => {\n      // Note: The change-streamer is generally not dispatched via the main\n      //       port, and in particular, should *not* be accessible via that\n      //       port in single-node mode. However, this plumbing is maintained\n      //       for the purpose of allowing --lazy-startup of the\n      //       replication-manager as a possible future feature.\n      assert(\n        syncers.length === 0 && mutator === undefined,\n        'Dispatch to the change-streamer via the main port ' +\n          'is only allowed in multi-node mode',\n      );\n      assert(\n        changeStreamer,\n        'Received a change-streamer request without a change-streamer worker',\n      );\n      const url = new URL(req.url ?? '', 'http://unused/');\n      const path = parsePath(url);\n      if (!path) {\n        throw new Error(`Invalid URL: ${req.url}`);\n      }\n\n      return {\n        payload: path.action,\n        sender: changeStreamer,\n      };\n    };\n\n    // handoff messages from this ZeroDispatcher to the appropriate worker (pool).\n    installWebSocketHandoff<unknown>(\n      lc,\n      request => {\n        const {url: u} = request;\n        const url = new URL(u ?? '', 'http://unused/');\n        const path = parsePath(url);\n        if (!path) {\n          throw new Error(`Invalid URL: ${u}`);\n        }\n        switch (path.worker) {\n          case 'sync':\n            return handleSync(request);\n          case 'replication':\n            return handleChangeStream(request);\n          case 'mutate':\n            return handlePush(request);\n          default:\n            throw new Error(`Invalid URL: ${u}`);\n        }\n      },\n      parent,\n    );\n  }\n\n  run() {\n    const readyStart = Date.now();\n    getOrCreateGauge('server', 'uptime', {\n      description: 'Cumulative uptime, starting from when requests are served',\n      unit: 's',\n    }).addCallback(result => result.observe((Date.now() - readyStart) / 1000));\n\n    return this.#state.stopped();\n  }\n\n  stop() {\n    this.#assigner.destroy();\n    this.#state.stop(this.#lc);\n    return this.#state.stopped();\n  }\n}\n\nconst URL_PATTERN = new UrlPattern('(/:base)/:worker/v:version/:action');\n\nexport function parsePath(url: URL):\n  | {\n      base?: string;\n      worker: 'sync' | 'mutate' | 'replication';\n      version: string;\n      action: string;\n    }\n  | undefined {\n  // The match() returns both null and undefined.\n  return URL_PATTERN.match(url.pathname) || undefined;\n} // The server allows the client to use any /:base/ path to facilitate\n// servicing requests on the same domain as the application.\n"],"mappings":";;;;;;;;AAeA,IAAa,mBAAb,MAAiD;CAC/C,KAAc;CACd;CACA,SAAkB,IAAI,aAAa,KAAK,EAAE;CAC1C;CAEA,YACE,IACA,QACA,QACA,SACA,SACA,gBACA;EACA,KAAK,MAAM;EAEX,MAAM,WAAW,IAAI,eAAe,QAAQ,QAAQ,MAAM;EAC1D,KAAK,YAAY;EAEjB,MAAM,0BAA0B,mBAC9B,QACA,qBACA,+CACF;EAEA,iBACE,QACA,wBACA,mDACF,CAAC,CAAC,aAAY,WAAU;GACtB,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAClC,OAAO,QAAQ,SAAS,cAAc,CAAC,GAAG,EAAC,QAAQ,OAAO,CAAC,EAAC,CAAC;EAEjE,CAAC;EAED,QAAQ,SAAS,QAAQ,UAAU;GACjC,OAAO,cACL,sBACC,EAAC,eAAe,QAAQ,iBAAgB;IACvC,IAAI,CAAC,QACH,SAAS,QAAQ,eAAe,OAAO,UAAU;GAErD,CACF;EACF,CAAC;EAED,SAAS,cAAc,KAA4B;GACjD,MAAM,EAAC,SAAS,KAAK,MAAK;GAC1B,MAAM,MAAM,IAAI,IAAI,KAAK,IAAI,gBAAgB;GAC7C,MAAM,OAAO,UAAU,GAAG;GAC1B,IAAI,CAAC,MACH,MAAM,IAAI,MAAM,gBAAgB,GAAG;GAErC,MAAM,UAAU,OAAO,KAAK,OAAO;GACnC,IAAI,OAAO,MAAM,OAAO,GACtB,MAAM,IAAI,MAAM,oBAAoB,GAAG;GAEzC,MAAM,EAAC,QAAQ,UAAS,iBAAiB,SAAS,KAAK,OAAO;GAC9D,IAAI,UAAU,MACZ,MAAM,IAAI,MAAM,KAAK;GAEvB,OAAO;EACT;EAEA,MAAM,cAAc,QAA+B;GACjD,OACE,YAAY,KAAA,GACZ,yEACF;GACA,OAAO;IAAC,SAAS,cAAc,GAAG;IAAG,QAAQ;GAAO;EACtD;EAEA,IAAI,qBAAqB;EACzB,iBACE,QACA,wBACA,uDACF,CAAC,CAAC,aAAY,WAAU;GACtB,IAAI,oBACF,OAAO,QAAQ,kBAAkB;EAErC,CAAC;EAED,MAAM,cAAc,QAA+B;GACjD,OAAO,QAAQ,QAAQ,+CAA+C;GACtE,MAAM,SAAS,cAAc,GAAG;GAChC,MAAM,EAAC,eAAe,oBAAmB;GACzC,qBAAqB,KAAK,IAAI,oBAAoB,eAAe;GAEjE,MAAM,EAAC,QAAQ,QAAQ,eAAc,SAAS,OAAO,aAAa;GAClE,wBAAwB,IAAI,GAAG,EAAC,QAAQ,OAAO,MAAM,EAAC,CAAC;GAEvD,GAAG,QAAQ,cAAc,cAAc,aAAa,QAAQ;GAC5D,OAAO;IAAC,SAAS;KAAC,GAAG;KAAQ;IAAU;IAAG,QAAQ,QAAQ;GAAO;EACnE;EAEA,MAAM,sBAAsB,QAA+B;GAMzD,OACE,QAAQ,WAAW,KAAK,YAAY,KAAA,GACpC,sFAEF;GACA,OACE,gBACA,qEACF;GAEA,MAAM,OAAO,UAAU,IADP,IAAI,IAAI,OAAO,IAAI,gBACZ,CAAG;GAC1B,IAAI,CAAC,MACH,MAAM,IAAI,MAAM,gBAAgB,IAAI,KAAK;GAG3C,OAAO;IACL,SAAS,KAAK;IACd,QAAQ;GACV;EACF;EAGA,wBACE,KACA,YAAW;GACT,MAAM,EAAC,KAAK,MAAK;GAEjB,MAAM,OAAO,UAAU,IADP,IAAI,KAAK,IAAI,gBACN,CAAG;GAC1B,IAAI,CAAC,MACH,MAAM,IAAI,MAAM,gBAAgB,GAAG;GAErC,QAAQ,KAAK,QAAb;IACE,KAAK,QACH,OAAO,WAAW,OAAO;IAC3B,KAAK,eACH,OAAO,mBAAmB,OAAO;IACnC,KAAK,UACH,OAAO,WAAW,OAAO;IAC3B,SACE,MAAM,IAAI,MAAM,gBAAgB,GAAG;GACvC;EACF,GACA,MACF;CACF;CAEA,MAAM;EACJ,MAAM,aAAa,KAAK,IAAI;EAC5B,iBAAiB,UAAU,UAAU;GACnC,aAAa;GACb,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,WAAU,OAAO,SAAS,KAAK,IAAI,IAAI,cAAc,GAAI,CAAC;EAEzE,OAAO,KAAK,OAAO,QAAQ;CAC7B;CAEA,OAAO;EACL,KAAK,UAAU,QAAQ;EACvB,KAAK,OAAO,KAAK,KAAK,GAAG;EACzB,OAAO,KAAK,OAAO,QAAQ;CAC7B;AACF;AAEA,IAAM,cAAc,IAAI,WAAW,oCAAoC;AAEvE,SAAgB,UAAU,KAOZ;CAEZ,OAAO,YAAY,MAAM,IAAI,QAAQ,KAAK,KAAA;AAC5C"}

package/out/zero-cache/src/server/worker-urls.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker-urls.js","names":[],"sources":["../../../../../zero-cache/src/server/worker-urls.ts"],"sourcesContent":["// This module provides URLs for worker files.\n\nconst tsRe = /\\.ts$/;\n\nfunction resolve(path: string): URL {\n  const {url} = import.meta;\n  if (url.endsWith('.js')) {\n    // When compiled, change .ts to .js\n    path = path.replace(tsRe, '.js');\n  }\n  return new URL(path, url);\n}\n\n// These URLs are part of the build process. See ../../zero/tool/build.ts\n// All these urls must be relative to this file and be located in the same directory.\n\nexport const CHANGE_STREAMER_URL = resolve('./change-streamer.ts');\nexport const MAIN_URL = resolve('./main.ts');\nexport const MUTATOR_URL = resolve('./mutator.ts');\nexport const REAPER_URL = resolve('./reaper.ts');\nexport const REPLICATOR_URL = resolve('./replicator.ts');\nexport const SHADOW_SYNCER_URL = resolve('./shadow-syncer.ts');\nexport const SLOT_KEEPER_URL = resolve('./slot-keeper.ts');\nexport const SYNCER_URL = resolve('./syncer.ts');\nexport const WRITE_WORKER_URL = resolve('./write-worker.ts');\n"],"mappings":";AAEA,IAAM,OAAO;AAEb,SAAS,QAAQ,MAAmB;CAClC,MAAM,EAAC,QAAO,OAAO;CACrB,IAAI,IAAI,SAAS,KAAK,GAEpB,OAAO,KAAK,QAAQ,MAAM,KAAK;CAEjC,OAAO,IAAI,IAAI,MAAM,GAAG;AAC1B;AAKA,IAAa,sBAAsB,QAAQ,sBAAsB;AACjE,IAAa,WAAW,QAAQ,WAAW;AAChB,QAAQ,cAAc;AACjD,IAAa,aAAa,QAAQ,aAAa;AAC/C,IAAa,iBAAiB,QAAQ,iBAAiB;AACvD,IAAa,oBAAoB,QAAQ,oBAAoB;AAC7D,IAAa,kBAAkB,QAAQ,kBAAkB;AACzD,IAAa,aAAa,QAAQ,aAAa;AAC/C,IAAa,mBAAmB,QAAQ,mBAAmB"}
+{"version":3,"file":"worker-urls.js","names":[],"sources":["../../../../../zero-cache/src/server/worker-urls.ts"],"sourcesContent":["// This module provides URLs for worker files.\n\nconst tsRe = /\\.ts$/;\n\nfunction resolve(path: string): URL {\n  const {url} = import.meta;\n  if (url.endsWith('.js')) {\n    // When compiled, change .ts to .js\n    path = path.replace(tsRe, '.js');\n  }\n  return new URL(path, url);\n}\n\n// These URLs are part of the build process. See ../../zero/tool/build.ts\n// All these urls must be relative to this file and be located in the same directory.\n\nexport const CHANGE_STREAMER_URL = resolve('./change-streamer.ts');\nexport const MAIN_URL = resolve('./main.ts');\nexport const MUTATOR_URL = resolve('./mutator.ts');\nexport const REAPER_URL = resolve('./reaper.ts');\nexport const REPLICATOR_URL = resolve('./replicator.ts');\nexport const SHADOW_SYNCER_URL = resolve('./shadow-syncer.ts');\nexport const SLOT_KEEPER_URL = resolve('./slot-keeper.ts');\nexport const SYNCER_URL = resolve('./syncer.ts');\nexport const WRITE_WORKER_URL = resolve('./write-worker.ts');\n"],"mappings":";AAEA,IAAM,OAAO;AAEb,SAAS,QAAQ,MAAmB;CAClC,MAAM,EAAC,QAAO;CACd,IAAI,IAAI,SAAS,KAAK,GAEpB,OAAO,KAAK,QAAQ,MAAM,KAAK;CAEjC,OAAO,IAAI,IAAI,MAAM,GAAG;AAC1B;AAKA,IAAa,sBAAsB,QAAQ,sBAAsB;AACjE,IAAa,WAAW,QAAQ,WAAW;AAChB,QAAQ,cAAc;AACjD,IAAa,aAAa,QAAQ,aAAa;AAC/C,IAAa,iBAAiB,QAAQ,iBAAiB;AACvD,IAAa,oBAAoB,QAAQ,oBAAoB;AAC7D,IAAa,kBAAkB,QAAQ,kBAAkB;AACzD,IAAa,aAAa,QAAQ,aAAa;AAC/C,IAAa,mBAAmB,QAAQ,mBAAmB"}

package/out/zero-cache/src/services/analyze.js

@@ -8,7 +8,7 @@ import { computeZqlSpecs, mustGetTableSpec } from "../db/lite-tables.js";
 import { createSQLiteCostModel } from "../../../zqlite/src/sqlite-cost-model.js";
 import { MAX_ANALYZE_ROWS, runAst } from "./run-ast.js";
 import { explainQueries } from "../../../zqlite/src/explain-queries.js";
-import { _usingCtx } from "../../../_virtual/_@oxc-project_runtime@0.130.0/helpers/usingCtx.js";
+import _usingCtx from "../../../_virtual/_@oxc-project_runtime@0.154.0/helpers/esm/usingCtx.js";
 import { TimeSliceTimer } from "./view-syncer/view-syncer.js";
 //#region ../zero-cache/src/services/analyze.ts
 var TIME_SLICE_LAP_THRESHOLD_MS = 200;

package/out/zero-cache/src/services/analyze.js.map

@@ -1 +1 @@
-{"version":3,"file":"analyze.js","names":[],"sources":["../../../../../zero-cache/src/services/analyze.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {getOrInsertComputed} from '../../../shared/src/map.ts';\nimport type {AnalyzeQueryResult} from '../../../zero-protocol/src/analyze-query-result.ts';\nimport type {AST} from '../../../zero-protocol/src/ast.ts';\nimport type {ClientSchema} from '../../../zero-protocol/src/client-schema.ts';\nimport type {PermissionsConfig} from '../../../zero-schema/src/compiled-permissions.ts';\nimport {Debug} from '../../../zql/src/builder/debug-delegate.ts';\nimport {MemoryStorage} from '../../../zql/src/ivm/memory-storage.ts';\nimport {\n  AccumulatorDebugger,\n  serializePlanDebugEvents,\n} from '../../../zql/src/planner/planner-debug.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport {explainQueries} from '../../../zqlite/src/explain-queries.ts';\nimport {createSQLiteCostModel} from '../../../zqlite/src/sqlite-cost-model.ts';\nimport {TableSource} from '../../../zqlite/src/table-source.ts';\nimport type {JWTAuth} from '../auth/auth.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {computeZqlSpecs, mustGetTableSpec} from '../db/lite-tables.ts';\nimport type {LiteAndZqlSpec, LiteTableSpec} from '../db/specs.ts';\nimport {MAX_ANALYZE_ROWS, runAst} from './run-ast.ts';\nimport {TimeSliceTimer} from './view-syncer/view-syncer.ts';\n\nconst TIME_SLICE_LAP_THRESHOLD_MS = 200;\n\nexport async function analyzeQuery(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  clientSchema: ClientSchema,\n  ast: AST,\n  syncedRows = true,\n  vendedRows = false,\n  permissions?: PermissionsConfig,\n  auth?: JWTAuth,\n  joinPlans = false,\n): Promise<AnalyzeQueryResult> {\n  using db = new Database(lc, config.replica.file);\n  const fullTables = new Map<string, LiteTableSpec>();\n  const tableSpecs = new Map<string, LiteAndZqlSpec>();\n  const tables = new Map<string, TableSource>();\n\n  computeZqlSpecs(\n    lc,\n    db,\n    {includeBackfillingColumns: false},\n    tableSpecs,\n    fullTables,\n  );\n\n  // Mirror production: the planner runs iff ZERO_ENABLE_QUERY_PLANNER is set\n  // on the server, so the analysis reflects what actually executes. Diagnostic\n  // event collection is orthogonal and opt-in via `joinPlans`.\n  const costModel = config.enableQueryPlanner\n    ? createSQLiteCostModel(db, tableSpecs)\n    : undefined;\n  const planDebugger = joinPlans ? new AccumulatorDebugger() : undefined;\n  const timer = await new TimeSliceTimer(lc).start();\n  const shouldYield = () => timer.elapsedLap() > TIME_SLICE_LAP_THRESHOLD_MS;\n  const yieldProcess = () => timer.yieldProcess();\n  const result = await runAst(\n    lc,\n    clientSchema,\n    ast,\n    true,\n    {\n      applyPermissions: permissions !== undefined,\n      syncedRows,\n      vendedRows,\n      auth,\n      db,\n      tableSpecs,\n      permissions,\n      costModel,\n      planDebugger,\n      maxSyncedRowsPerTable: MAX_ANALYZE_ROWS,\n      host: {\n        debug: new Debug(vendedRows, MAX_ANALYZE_ROWS),\n        enableNotExists: true,\n        // Mirror production, as with the planner above.\n        disableCorrelatedPredicatePushdown:\n          config.enableCorrelatedPredicatePushdown === false,\n        enablePlannerAwarePushdown: config.enablePlannerAwarePushdown !== false,\n        getSource(tableName: string) {\n          return getOrInsertComputed(tables, tableName, tableName => {\n            const tableSpec = mustGetTableSpec(tableSpecs, tableName);\n            const {primaryKey} = tableSpec.tableSpec;\n\n            return new TableSource(\n              lc,\n              config.log,\n              db,\n              tableName,\n              tableSpec.zqlSpec,\n              primaryKey,\n              shouldYield,\n            );\n          });\n        },\n        createStorage() {\n          return new MemoryStorage();\n        },\n        decorateSourceInput: input => input,\n        decorateInput: input => input,\n        addEdge() {},\n        decorateFilterInput: input => input,\n      },\n    },\n    yieldProcess,\n  );\n\n  // Fill in plans for any queries SQLite did not actually execute (and thus\n  // did not populate scanStatus EXPLAIN for) using the substituted-binding\n  // fallback. Plans captured at execution time use the real bindings and win.\n  const fallback = explainQueries(result.readRowCountsByQuery ?? {}, db);\n  const captured = result.sqlitePlans ?? {};\n  for (const [query, plan] of Object.entries(fallback)) {\n    if (!captured[query]) {\n      captured[query] = plan;\n    }\n  }\n  result.sqlitePlans = captured;\n\n  if (planDebugger) {\n    result.joinPlans = serializePlanDebugEvents(planDebugger.events);\n  }\n\n  return result;\n}\n"],"mappings":";;;;;;;;;;;;;AAuBA,IAAM,8BAA8B;AAEpC,eAAsB,aACpB,IACA,QACA,cACA,KACA,aAAa,MACb,aAAa,OACb,aACA,MACA,YAAY,OACiB;;;EAC7B,MAAM,KAAA,YAAA,EAAK,IAAI,SAAS,IAAI,OAAO,QAAQ,IAAI,CAAA;EAC/C,MAAM,6BAAa,IAAI,IAA2B;EAClD,MAAM,6BAAa,IAAI,IAA4B;EACnD,MAAM,yBAAS,IAAI,IAAyB;EAE5C,gBACE,IACA,IACA,EAAC,2BAA2B,MAAK,GACjC,YACA,UACF;EAKA,MAAM,YAAY,OAAO,qBACrB,sBAAsB,IAAI,UAAU,IACpC,KAAA;EACJ,MAAM,eAAe,YAAY,IAAI,oBAAoB,IAAI,KAAA;EAC7D,MAAM,QAAQ,MAAM,IAAI,eAAe,EAAE,EAAE,MAAM;EACjD,MAAM,oBAAoB,MAAM,WAAW,IAAI;EAC/C,MAAM,qBAAqB,MAAM,aAAa;EAC9C,MAAM,SAAS,MAAM,OACnB,IACA,cACA,KACA,MACA;GACE,kBAAkB,gBAAgB,KAAA;GAClC;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA,uBAAuB;GACvB,MAAM;IACJ,OAAO,IAAI,MAAM,YAAY,gBAAgB;IAC7C,iBAAiB;IAEjB,oCACE,OAAO,sCAAsC;IAC/C,4BAA4B,OAAO,+BAA+B;IAClE,UAAU,WAAmB;KAC3B,OAAO,oBAAoB,QAAQ,YAAW,cAAa;MACzD,MAAM,YAAY,iBAAiB,YAAY,SAAS;MACxD,MAAM,EAAC,eAAc,UAAU;MAE/B,OAAO,IAAI,YACT,IACA,OAAO,KACP,IACA,WACA,UAAU,SACV,YACA,WACF;KACF,CAAC;IACH;IACA,gBAAgB;KACd,OAAO,IAAI,cAAc;IAC3B;IACA,sBAAqB,UAAS;IAC9B,gBAAe,UAAS;IACxB,UAAU,CAAC;IACX,sBAAqB,UAAS;GAChC;EACF,GACA,YACF;EAKA,MAAM,WAAW,eAAe,OAAO,wBAAwB,CAAC,GAAG,EAAE;EACrE,MAAM,WAAW,OAAO,eAAe,CAAC;EACxC,KAAK,MAAM,CAAC,OAAO,SAAS,OAAO,QAAQ,QAAQ,GACjD,IAAI,CAAC,SAAS,QACZ,SAAS,SAAS;EAGtB,OAAO,cAAc;EAErB,IAAI,cACF,OAAO,YAAY,yBAAyB,aAAa,MAAM;EAGjE,OAAO;;;;;;AACT"}
+{"version":3,"file":"analyze.js","names":[],"sources":["../../../../../zero-cache/src/services/analyze.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {getOrInsertComputed} from '../../../shared/src/map.ts';\nimport type {AnalyzeQueryResult} from '../../../zero-protocol/src/analyze-query-result.ts';\nimport type {AST} from '../../../zero-protocol/src/ast.ts';\nimport type {ClientSchema} from '../../../zero-protocol/src/client-schema.ts';\nimport type {PermissionsConfig} from '../../../zero-schema/src/compiled-permissions.ts';\nimport {Debug} from '../../../zql/src/builder/debug-delegate.ts';\nimport {MemoryStorage} from '../../../zql/src/ivm/memory-storage.ts';\nimport {\n  AccumulatorDebugger,\n  serializePlanDebugEvents,\n} from '../../../zql/src/planner/planner-debug.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport {explainQueries} from '../../../zqlite/src/explain-queries.ts';\nimport {createSQLiteCostModel} from '../../../zqlite/src/sqlite-cost-model.ts';\nimport {TableSource} from '../../../zqlite/src/table-source.ts';\nimport type {JWTAuth} from '../auth/auth.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {computeZqlSpecs, mustGetTableSpec} from '../db/lite-tables.ts';\nimport type {LiteAndZqlSpec, LiteTableSpec} from '../db/specs.ts';\nimport {MAX_ANALYZE_ROWS, runAst} from './run-ast.ts';\nimport {TimeSliceTimer} from './view-syncer/view-syncer.ts';\n\nconst TIME_SLICE_LAP_THRESHOLD_MS = 200;\n\nexport async function analyzeQuery(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  clientSchema: ClientSchema,\n  ast: AST,\n  syncedRows = true,\n  vendedRows = false,\n  permissions?: PermissionsConfig,\n  auth?: JWTAuth,\n  joinPlans = false,\n): Promise<AnalyzeQueryResult> {\n  using db = new Database(lc, config.replica.file);\n  const fullTables = new Map<string, LiteTableSpec>();\n  const tableSpecs = new Map<string, LiteAndZqlSpec>();\n  const tables = new Map<string, TableSource>();\n\n  computeZqlSpecs(\n    lc,\n    db,\n    {includeBackfillingColumns: false},\n    tableSpecs,\n    fullTables,\n  );\n\n  // Mirror production: the planner runs iff ZERO_ENABLE_QUERY_PLANNER is set\n  // on the server, so the analysis reflects what actually executes. Diagnostic\n  // event collection is orthogonal and opt-in via `joinPlans`.\n  const costModel = config.enableQueryPlanner\n    ? createSQLiteCostModel(db, tableSpecs)\n    : undefined;\n  const planDebugger = joinPlans ? new AccumulatorDebugger() : undefined;\n  const timer = await new TimeSliceTimer(lc).start();\n  const shouldYield = () => timer.elapsedLap() > TIME_SLICE_LAP_THRESHOLD_MS;\n  const yieldProcess = () => timer.yieldProcess();\n  const result = await runAst(\n    lc,\n    clientSchema,\n    ast,\n    true,\n    {\n      applyPermissions: permissions !== undefined,\n      syncedRows,\n      vendedRows,\n      auth,\n      db,\n      tableSpecs,\n      permissions,\n      costModel,\n      planDebugger,\n      maxSyncedRowsPerTable: MAX_ANALYZE_ROWS,\n      host: {\n        debug: new Debug(vendedRows, MAX_ANALYZE_ROWS),\n        enableNotExists: true,\n        // Mirror production, as with the planner above.\n        disableCorrelatedPredicatePushdown:\n          config.enableCorrelatedPredicatePushdown === false,\n        enablePlannerAwarePushdown: config.enablePlannerAwarePushdown !== false,\n        getSource(tableName: string) {\n          return getOrInsertComputed(tables, tableName, tableName => {\n            const tableSpec = mustGetTableSpec(tableSpecs, tableName);\n            const {primaryKey} = tableSpec.tableSpec;\n\n            return new TableSource(\n              lc,\n              config.log,\n              db,\n              tableName,\n              tableSpec.zqlSpec,\n              primaryKey,\n              shouldYield,\n            );\n          });\n        },\n        createStorage() {\n          return new MemoryStorage();\n        },\n        decorateSourceInput: input => input,\n        decorateInput: input => input,\n        addEdge() {},\n        decorateFilterInput: input => input,\n      },\n    },\n    yieldProcess,\n  );\n\n  // Fill in plans for any queries SQLite did not actually execute (and thus\n  // did not populate scanStatus EXPLAIN for) using the substituted-binding\n  // fallback. Plans captured at execution time use the real bindings and win.\n  const fallback = explainQueries(result.readRowCountsByQuery ?? {}, db);\n  const captured = result.sqlitePlans ?? {};\n  for (const [query, plan] of Object.entries(fallback)) {\n    if (!captured[query]) {\n      captured[query] = plan;\n    }\n  }\n  result.sqlitePlans = captured;\n\n  if (planDebugger) {\n    result.joinPlans = serializePlanDebugEvents(planDebugger.events);\n  }\n\n  return result;\n}\n"],"mappings":";;;;;;;;;;;;;AAuBA,IAAM,8BAA8B;AAEpC,eAAsB,aACpB,IACA,QACA,cACA,KACA,aAAa,MACb,aAAa,OACb,aACA,MACA,YAAY,OACiB;;;EAC7B,MAAM,KAAA,YAAA,EAAK,IAAI,SAAS,IAAI,OAAO,QAAQ,IAAI,CAAA;EAC/C,MAAM,6BAAa,IAAI,IAA2B;EAClD,MAAM,6BAAa,IAAI,IAA4B;EACnD,MAAM,yBAAS,IAAI,IAAyB;EAE5C,gBACE,IACA,IACA,EAAC,2BAA2B,MAAK,GACjC,YACA,UACF;EAKA,MAAM,YAAY,OAAO,qBACrB,sBAAsB,IAAI,UAAU,IACpC,KAAA;EACJ,MAAM,eAAe,YAAY,IAAI,oBAAoB,IAAI,KAAA;EAC7D,MAAM,QAAQ,MAAM,IAAI,eAAe,EAAE,CAAC,CAAC,MAAM;EACjD,MAAM,oBAAoB,MAAM,WAAW,IAAI;EAC/C,MAAM,qBAAqB,MAAM,aAAa;EAC9C,MAAM,SAAS,MAAM,OACnB,IACA,cACA,KACA,MACA;GACE,kBAAkB,gBAAgB,KAAA;GAClC;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA,uBAAuB;GACvB,MAAM;IACJ,OAAO,IAAI,MAAM,YAAY,gBAAgB;IAC7C,iBAAiB;IAEjB,oCACE,OAAO,sCAAsC;IAC/C,4BAA4B,OAAO,+BAA+B;IAClE,UAAU,WAAmB;KAC3B,OAAO,oBAAoB,QAAQ,YAAW,cAAa;MACzD,MAAM,YAAY,iBAAiB,YAAY,SAAS;MACxD,MAAM,EAAC,eAAc,UAAU;MAE/B,OAAO,IAAI,YACT,IACA,OAAO,KACP,IACA,WACA,UAAU,SACV,YACA,WACF;KACF,CAAC;IACH;IACA,gBAAgB;KACd,OAAO,IAAI,cAAc;IAC3B;IACA,sBAAqB,UAAS;IAC9B,gBAAe,UAAS;IACxB,UAAU,CAAC;IACX,sBAAqB,UAAS;GAChC;EACF,GACA,YACF;EAKA,MAAM,WAAW,eAAe,OAAO,wBAAwB,CAAC,GAAG,EAAE;EACrE,MAAM,WAAW,OAAO,eAAe,CAAC;EACxC,KAAK,MAAM,CAAC,OAAO,SAAS,OAAO,QAAQ,QAAQ,GACjD,IAAI,CAAC,SAAS,QACZ,SAAS,SAAS;EAGtB,OAAO,cAAc;EAErB,IAAI,cACF,OAAO,YAAY,yBAAyB,aAAa,MAAM;EAGjE,OAAO;;;;;;AACT"}

package/out/zero-cache/src/services/change-source/common/backfill-manager.js

@@ -18,7 +18,7 @@ function tableKey({ schema, name }) {
 * backfill data have accumulated in it. Bounds the size of a single replica
 * COMMIT/checkpoint so it can't monopolize the replica.
 */
-var COMMIT_THRESHOLD_BYTES = 8 * 1024 * 1024;
+var COMMIT_THRESHOLD_BYTES = 8388608;
 /**
 * How many backfill messages the prefetch producer may read ahead of the
 * reservation-holding consumer loop. This bounds the in-memory buffer (~this

package/out/zero-cache/src/services/change-source/common/backfill-manager.js.map

@@ -1 +1 @@
-{"version":3,"file":"backfill-manager.js","names":["#lc","#requiredBackfills","#changeStreamer","#backfillStreamer","#jsonFormat","#awaitingStatusWatermarks","#commitThresholdBytes","#controller","#canceled","#minBackoffMs","#maxBackoffMs","#retryDelayMs","#lastStatusWatermark","#setRequiredBackfill","#checkAndStartBackfill","#backfillRetryTimer","#runningBackfill","#runBackfill","#stopRunningBackfill","#retryBackfillWithBackoff","#changeStreamReached","#maybeLogBackfillStats","#logBackfillStats","#currentTxWatermark","#setLastStatusWatermark","#backfillRunningFor","#deleteRequiredBackfill"],"sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {stringify} from '../../../../../shared/src/bigint-json.ts';\nimport {CustomKeyMap} from '../../../../../shared/src/custom-key-map.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {promiseOrAbort} from '../../../../../shared/src/promise-race.ts';\nimport {Queue} from '../../../../../shared/src/queue.ts';\nimport {randInt} from '../../../../../shared/src/rand.ts';\nimport {JSON_STRINGIFIED, type JSONFormat} from '../../../types/lite.ts';\nimport {\n  stateVersionFromString,\n  stateVersionToString,\n} from '../../../types/state-version.ts';\nimport type {\n  BackfillCompleted,\n  BackfillRequest,\n  ChangeStreamMessage,\n  Identifier,\n  MessageBackfill,\n} from '../protocol/current.ts';\nimport type {\n  Cancelable,\n  ChangeStreamMultiplexer,\n  Listener,\n} from './change-stream-multiplexer.ts';\n\nfunction tableKey({schema, name}: Identifier) {\n  return `${schema}.${name}`;\n}\n\n/**\n * A {@link MessageBackfill} or {@link BackfillCompleted} paired with the\n * approximate number of upstream bytes it represents. The BackfillManager uses\n * `byteSize` to bound the size of each replica transaction, committing and\n * reopening once {@link COMMIT_THRESHOLD_BYTES} is reached.\n */\nexport type BackfillMessage = {\n  message: MessageBackfill | BackfillCompleted;\n  byteSize: number;\n};\n\ntype BackfillStreamer = (\n  req: BackfillRequest,\n) => AsyncGenerator<BackfillMessage>;\n\n/**\n * Commit (and reopen) the current backfill transaction once this many bytes of\n * backfill data have accumulated in it. Bounds the size of a single replica\n * COMMIT/checkpoint so it can't monopolize the replica.\n */\nconst COMMIT_THRESHOLD_BYTES = 8 * 1024 * 1024;\n\n/**\n * How many backfill messages the prefetch producer may read ahead of the\n * reservation-holding consumer loop. This bounds the in-memory buffer (~this\n * many {@link BackfillMessage}s, each up to the stream's flush threshold, e.g.\n * 64 KiB) while giving the consumer a supply of ready messages so it never\n * has to wait on the upstream COPY while holding the change-stream reservation.\n */\nconst BACKFILL_PREFETCH_CREDITS = 16;\n\n/**\n * While the backfill holds an open transaction but its prefetch queue is empty\n * (i.e. the upstream is momentarily behind), the consumer polls for the next\n * message at this interval so that a replication transaction that begins\n * waiting mid-fetch can preempt the backfill within this bound, rather than\n * being blocked for the entire (possibly slow) upstream read.\n *\n * If stats indicate that these stalls are highly correlative with replication\n * lag (indicating that replication changes are waiting on these polls), we\n * can address this by adding an API to the change stream multiplexer that\n * would allow backfill to relinquish an in-progress commit on demand (i.e.\n * only if replication asks for a reservation).\n */\nconst BACKFILL_WAITER_POLL_MS = 10;\n\n/** Interval at which in-progress backfill timing stats are logged. */\nconst BACKFILL_STATS_LOG_INTERVAL_MS = 10_000;\n\n/** Sentinel enqueued by the prefetch producer when the backfill stream ends. */\nconst DONE = Symbol('backfill-done');\n/** Sentinel returned by a polled dequeue that timed out (see {@link BACKFILL_WAITER_POLL_MS}). */\nconst POLL = Symbol('backfill-poll');\n\ntype Pull = BackfillMessage | typeof DONE | typeof POLL;\n\n/**\n * Accumulated timing attribution for a single backfill run, logged\n * periodically to reveal where the backfill spends its time (and, in\n * particular, where the change-stream reservation is held). `produceMs`\n * accrues on the prefetch producer task and thus overlaps with the consumer's\n * `pushMs`; the others accrue on the reservation-holding consumer loop.\n */\ntype BackfillTimingStats = {\n  start: number;\n  lastLogged: number;\n  messages: number;\n  bytes: number;\n  commits: number;\n  stalls: number;\n  produceMs: number;\n  pushMs: number;\n  reserveMs: number;\n  stallMs: number;\n  waitMs: number;\n};\n\ntype RunningBackfillState = {\n  request: BackfillRequest;\n  minWatermark: string;\n  /**\n   * Aborted (with the reason, if any) when this backfill is stopped,\n   * e.g. canceled.\n   */\n  stopped: AbortController;\n};\n\nconst MIN_BACKOFF_INTERVAL_MS = 2_000;\nconst MAX_BACKOFF_INTERVAL_MS = 60_000;\n\ntype AwaitingStatusWatermark = {\n  watermark: string;\n  reached: () => void;\n};\n\n/**\n * The BackfillManager initiates backfills for BackfillRequests from the\n * change-streamer (i.e. unfinished backfills from previous sessions)\n * or for new backfills signaled by `create-table` or `add-column` messages\n * from the change-source.\n *\n * The BackfillManager registers itself as a change stream listener in order\n * to track necessary backfills, and potentially invalidate the in-progress\n * backfill (e.g. due to a schema change) so that it can be retried at a\n * new snapshot.\n *\n * The manager also handles low priority streaming of the backfill messages\n * using the {@link ChangeStreamMultiplexer}, implementing a policy of always\n * releasing its reservation if another producer (i.e. the main change stream)\n * has messages to stream.\n */\nexport class BackfillManager implements Cancelable, Listener {\n  readonly #lc: LogContext;\n\n  /**\n   * Tracks the metadata of required backfills based on schema changes\n   * and initial backfill requests.\n   */\n  readonly #requiredBackfills = new CustomKeyMap<Identifier, BackfillRequest>(\n    tableKey,\n  );\n  readonly #changeStreamer: ChangeStreamMultiplexer;\n  readonly #backfillStreamer: BackfillStreamer;\n  readonly #jsonFormat: JSONFormat;\n\n  /**\n   * The current running backfill. The backfill request is always also in\n   * `#requiredBackfills` (technically, it can be a subset of what's in\n   * `#requiredBackfills`); the request is removed from `#requiredBackfills`\n   * upon completion.\n   */\n  #runningBackfill: RunningBackfillState | null = null;\n\n  /** The last seen watermark in the change stream. */\n  #lastStatusWatermark: string | null = null;\n\n  readonly #awaitingStatusWatermarks: AwaitingStatusWatermark[] = [];\n\n  /** The watermark of the current transaction in the change stream. */\n  #currentTxWatermark: string | null = null;\n\n  readonly #commitThresholdBytes: number;\n\n  /**\n   * Aborted when the change stream is canceled. No further backfills are run.\n   *\n   * TODO: Thread this into the backfillStream so that upstream I/O can be\n   * aborted.\n   */\n  readonly #controller = new AbortController();\n\n  get #canceled() {\n    return this.#controller.signal.aborted;\n  }\n\n  constructor(\n    lc: LogContext,\n    changeStreamer: ChangeStreamMultiplexer,\n    backfillStreamer: BackfillStreamer,\n    jsonFormat: JSONFormat = JSON_STRINGIFIED,\n    minBackoffMs = MIN_BACKOFF_INTERVAL_MS,\n    maxBackoffMs = MAX_BACKOFF_INTERVAL_MS,\n    commitThresholdBytes = COMMIT_THRESHOLD_BYTES,\n  ) {\n    this.#lc = lc.withContext('component', 'backfill-manager');\n    this.#changeStreamer = changeStreamer;\n    this.#backfillStreamer = backfillStreamer;\n    this.#jsonFormat = jsonFormat;\n    this.#minBackoffMs = minBackoffMs;\n    this.#maxBackoffMs = maxBackoffMs;\n    this.#retryDelayMs = minBackoffMs;\n    this.#commitThresholdBytes = commitThresholdBytes;\n  }\n\n  run(lastWatermark: string, initialRequests: BackfillRequest[]) {\n    this.#lc.info?.(\n      `starting backfill manager with ${initialRequests.length} initial requests`,\n      {requests: initialRequests},\n    );\n    this.#lastStatusWatermark = lastWatermark;\n    initialRequests.forEach(req =>\n      this.#setRequiredBackfill('initial-request', req),\n    );\n    this.#checkAndStartBackfill();\n  }\n\n  #setLastStatusWatermark({watermark}: {watermark: string}) {\n    // Only allow the watermark to move forward. This prevents a backfill\n    // transaction (whose watermark is unrelated to change-stream state)\n    // from moving the watermark backwards.\n    if ((this.#lastStatusWatermark ?? '') < watermark) {\n      this.#lastStatusWatermark = watermark;\n      for (let i = this.#awaitingStatusWatermarks.length - 1; i >= 0; i--) {\n        const awaiting = this.#awaitingStatusWatermarks[i];\n        if (watermark >= awaiting.watermark) {\n          awaiting.reached();\n          this.#awaitingStatusWatermarks.splice(i, 1);\n        }\n      }\n    }\n  }\n\n  #changeStreamReached(\n    lc: LogContext,\n    watermark: string,\n  ): Promise<void> | null {\n    if ((this.#lastStatusWatermark ?? '') < watermark) {\n      const {promise, resolve: reached} = resolver();\n      this.#awaitingStatusWatermarks.push({watermark, reached});\n      lc.info?.(\n        `waiting for change stream (at ${this.#lastStatusWatermark}) to reach ${watermark}`,\n      );\n      return promise;\n    }\n    return null;\n  }\n\n  readonly #minBackoffMs: number;\n  readonly #maxBackoffMs: number;\n  #retryDelayMs: number;\n  #backfillRetryTimer: NodeJS.Timeout | undefined;\n\n  #checkAndStartBackfill() {\n    if (\n      !this.#canceled &&\n      !this.#backfillRetryTimer &&\n      !this.#runningBackfill &&\n      this.#requiredBackfills.size\n    ) {\n      // Pick a random backfill to avoid head-of-line blocking by a\n      // problematic backfill (e.g. awaiting a primary key). This is\n      // simpler that adding logic to classify (and declassify)\n      // problematic backfills.\n      const candidates = [...this.#requiredBackfills.values()];\n      const request = candidates[randInt(0, candidates.length - 1)];\n      const state = {\n        request,\n        minWatermark: '',\n        stopped: new AbortController(),\n      };\n      const lc = this.#lc.withContext('table', request.table.name);\n\n      this.#runningBackfill = state;\n      void this.#runBackfill(lc, state)\n        .then(() => {\n          this.#stopRunningBackfill('backfill exited', state);\n          this.#retryDelayMs = this.#minBackoffMs; // reset on success\n        })\n        // For unexpected errors (e.g. upstream replication slot\n        // unavailability), retry with exponential backoff.\n        .catch(e => {\n          this.#stopRunningBackfill(String(e), state);\n          this.#retryBackfillWithBackoff(e);\n        });\n    }\n  }\n\n  #retryBackfillWithBackoff(e: unknown) {\n    if (this.#canceled) {\n      this.#lc.debug?.(`not retrying backfill: change stream canceled`, e);\n      return;\n    }\n    this.#lc.warn?.(\n      `Error running backfill. Retrying in ${this.#retryDelayMs} ms`,\n      e,\n    );\n    this.#backfillRetryTimer = setTimeout(() => {\n      this.#backfillRetryTimer = undefined;\n      this.#checkAndStartBackfill();\n    }, this.#retryDelayMs);\n\n    this.#retryDelayMs = Math.min(this.#retryDelayMs * 2, this.#maxBackoffMs);\n  }\n\n  async #runBackfill(lc: LogContext, state: RunningBackfillState) {\n    const changeStream = this.#changeStreamer; // Purely for readability\n\n    // backfillTx is set if and only if a changeStreamer reservation has been\n    // acquired and the backfill stream is inside a transaction.\n    let backfillTx: string | null = null;\n    let uncommittedBytes = 0;\n\n    const stats: BackfillTimingStats = {\n      start: performance.now(),\n      lastLogged: performance.now(),\n      messages: 0,\n      bytes: 0,\n      commits: 0,\n      stalls: 0,\n      produceMs: 0,\n      pushMs: 0,\n      reserveMs: 0,\n      stallMs: 0,\n      waitMs: 0,\n    };\n\n    /**\n     * @returns the new tx watermark, or null if backfill was cancelled\n     */\n    const beginTxFor = async (\n      msg: MessageBackfill | BackfillCompleted,\n    ): Promise<string | null> => {\n      assert(backfillTx === null, 'Expected no active backfill transaction');\n      const lastWatermark = await changeStream.reserve('backfill');\n\n      // After obtaining the changeStream reservation, check if the stream\n      // had changes that resulted in invalidating / canceling this backfill.\n      if (\n        state.stopped.signal.aborted ||\n        (msg.tag === 'backfill' && msg.watermark < state.minWatermark)\n      ) {\n        if (!state.stopped.signal.aborted) {\n          assert(msg.tag === 'backfill', 'Expected backfill message tag'); // TypeScript should have figured this out.\n          this.#stopRunningBackfill(\n            `row key change at ${state.minWatermark} ` +\n              `postdates backfill watermark at ${msg.watermark}`,\n            state,\n          );\n        }\n        changeStream.release(lastWatermark);\n        return null;\n      }\n\n      const {major, minor = 0n} = stateVersionFromString(lastWatermark);\n      let tx = stateVersionToString({\n        major,\n        minor: BigInt(minor) + 1n,\n      });\n\n      if (tx < msg.watermark) {\n        // At this point it must be the case that the #changeStreamReached()\n        // the backfill watermark (possibly via a status message, which does\n        // not advance the reservation watermark). Given that guarantee,\n        // ensure that the version of the backfill transaction is at least up\n        // to the backfill watermark, so that the database state version is\n        // never earlier than the version of any backfilled rows.\n        tx = msg.watermark;\n      }\n\n      void changeStream.push([\n        'begin',\n        {tag: 'begin', json: this.#jsonFormat, skipAck: true},\n        {commitWatermark: tx},\n      ]);\n      return (backfillTx = tx);\n    };\n\n    const commitTx = () => {\n      if (backfillTx) {\n        void changeStream.push([\n          'commit',\n          {tag: 'commit'},\n          {watermark: backfillTx},\n        ]);\n        changeStream.release(backfillTx);\n        stats.commits++;\n      }\n      backfillTx = null;\n      uncommittedBytes = 0;\n    };\n\n    // === Prefetch producer ===\n    //\n    // The backfill stream (an upstream Postgres COPY) is drained on a separate\n    // task into a bounded queue, so that the reservation-holding consumer loop\n    // below never blocks on the upstream read while it holds the change-stream\n    // reservation. `credits` bounds how far the producer runs ahead; the\n    // consumer returns a credit for every message it takes.\n    const ready = new Queue<Pull>();\n    const credits = new Queue<1>();\n    for (let i = 0; i < BACKFILL_PREFETCH_CREDITS; i++) {\n      credits.enqueue(1);\n    }\n    let stopProducer = false;\n\n    // Wakes a consumer blocked awaiting the next prefetched message so a\n    // canceled backfill unwinds promptly.\n    const onCancel = () =>\n      ready.enqueueRejection(new AbortError('backfill canceled'));\n    if (!this.#canceled) {\n      this.#controller.signal.addEventListener('abort', onCancel);\n    }\n\n    const producer = (async () => {\n      try {\n        let t0 = performance.now();\n        for await (const value of this.#backfillStreamer(state.request)) {\n          stats.produceMs += performance.now() - t0;\n          await credits.dequeue(); // wait for buffer space (flow control)\n          if (stopProducer || this.#canceled) {\n            ready.enqueueRejection(new AbortError('backfill canceled'));\n            return;\n          }\n          ready.enqueue(value);\n          t0 = performance.now();\n        }\n        ready.enqueue(DONE);\n      } catch (e) {\n        ready.enqueueRejection(e);\n      }\n    })();\n    // The producer routes errors through `ready` and never rejects; guard\n    // anyway so that leaving it un-awaited on the cancellation path (below)\n    // cannot surface as an unhandled rejection.\n    void producer.catch(() => {});\n\n    // === Consumer ===\n\n    // Fetches the next item from the prefetch queue. If a transaction is open\n    // but the queue is empty (the producer/upstream is momentarily behind),\n    // polls so that a replication transaction that begins waiting during the\n    // upstream read can preempt the backfill: rather than hold the reservation\n    // for the whole read, commit and release, letting replication proceed.\n    const nextItem = async (): Promise<BackfillMessage | typeof DONE> => {\n      // Fast path: a message is already buffered, or there's no reservation to\n      // protect. Take it without altering transaction boundaries. `POLL` is\n      // never enqueued (only used as a dequeue timeout value), so the result is\n      // always a message or DONE.\n      if (backfillTx === null || ready.size() > 0) {\n        const n = ready.dequeue();\n        const v = n instanceof Promise ? await n : n;\n        return v as BackfillMessage | typeof DONE;\n      }\n      // Slow path: holding a reservation with an empty queue.\n      for (;;) {\n        if (backfillTx !== null && changeStream.waiterDelay() > 0) {\n          commitTx(); // yield the stream to the waiting replication txn\n        }\n        const n =\n          backfillTx !== null\n            ? ready.dequeue(POLL, BACKFILL_WAITER_POLL_MS)\n            : ready.dequeue(); // released: just wait for the next message\n        const t0 = performance.now();\n        const v = n instanceof Promise ? await n : n;\n        if (v === POLL) {\n          stats.stallMs += performance.now() - t0; // timed out; re-check waiter\n          continue;\n        }\n        if (n instanceof Promise) {\n          stats.stallMs += performance.now() - t0;\n          stats.stalls++;\n        }\n        return v;\n      }\n    };\n\n    try {\n      for (;;) {\n        const item = await nextItem();\n        if (item === DONE) {\n          break;\n        }\n        // Allow the producer to fetch the next message concurrently with the\n        // processing (and downstream push) of this one.\n        credits.enqueue(1);\n\n        if (this.#canceled) {\n          // Returning finalizes the backfill stream (and the upstream resources\n          // it holds) via the producer's `finally`. The reservation, if held,\n          // does not need to be released since the change stream is gone.\n          lc.info?.(`backfill stream canceled: change stream canceled`);\n          return;\n        }\n        const {message: msg, byteSize} = item;\n\n        // Before sending any backfill message, the main replication stream\n        // must reach the backfill (snapshot) watermark. Otherwise, snapshot\n        // rows can constitute \"future\" state (e.g. phantom rows at keys that\n        // the stream has yet to move or insert) that the subsequent\n        // replication changes conflict with (e.g. on unique indexes).\n        const waitForChangeStream = this.#changeStreamReached(\n          lc,\n          msg.watermark,\n        );\n\n        // Commit (and later reopen) the transaction if the main stream is\n        // waiting on the reservation, if we must wait for the main stream, or\n        // if the size of current transaction has reached the commit threshold.\n        if (\n          backfillTx &&\n          (changeStream.waiterDelay() > 0 ||\n            waitForChangeStream ||\n            uncommittedBytes >= this.#commitThresholdBytes)\n        ) {\n          commitTx();\n        }\n\n        if (waitForChangeStream) {\n          const t0 = performance.now();\n          // Exit promptly (via the error path) if the backfill is canceled\n          // while waiting, releasing its upstream resources.\n          await promiseOrAbort(waitForChangeStream, state.stopped.signal);\n          const elapsed = performance.now() - t0;\n          stats.waitMs += elapsed;\n          lc.info?.(\n            `change stream reached ${msg.watermark} ` +\n              `after waiting ${Math.round(elapsed)} ms`,\n          );\n        }\n\n        if (\n          msg.tag === 'backfill' &&\n          msg.rowValues.length > 0 &&\n          msg.relation.rowKey.columns.length === 0\n        ) {\n          throw new MissingRowKeyError(state.request);\n        }\n\n        // Reserve the changeStreamer if not in a transaction.\n        if (backfillTx === null) {\n          const t0 = performance.now();\n          backfillTx = await beginTxFor(msg);\n          stats.reserveMs += performance.now() - t0;\n          if (backfillTx === null) {\n            lc.info?.(\n              `backfill stream canceled: ${state.stopped.signal.reason}`,\n              state.request,\n            );\n            this.#checkAndStartBackfill(); // start the next backfill if present\n            return; // this backfill is canceled\n          }\n        }\n\n        // `await` to allow the change streamer to exert back pressure\n        // on backfills.\n        const t0 = performance.now();\n        await changeStream.push(['data', msg]);\n        stats.pushMs += performance.now() - t0;\n        uncommittedBytes += byteSize;\n        stats.messages++;\n        stats.bytes += byteSize;\n\n        this.#maybeLogBackfillStats(lc, stats);\n      }\n\n      // Flush any final tx and release the stream.\n      if (backfillTx) {\n        commitTx();\n      }\n      lc.debug?.(`backfill stream exited`);\n    } finally {\n      stopProducer = true;\n      credits.enqueue(1);\n      // The producer can potentially be hung on upstream I/O.\n      // TODO: Thread AbortSignal logic into the backfillStreamer.\n      if (!this.#canceled) {\n        await producer;\n      }\n      this.#controller.signal.removeEventListener('abort', onCancel);\n      this.#logBackfillStats(lc, stats);\n    }\n  }\n\n  #maybeLogBackfillStats(lc: LogContext, stats: BackfillTimingStats) {\n    const now = performance.now();\n    if (now - stats.lastLogged >= BACKFILL_STATS_LOG_INTERVAL_MS) {\n      this.#logBackfillStats(lc, stats);\n      stats.lastLogged = now;\n    }\n  }\n\n  /**\n   * Logs where the backfill spent its time. `produceMs` (upstream read +\n   * parse/decode) accrues on the prefetch producer concurrently with the\n   * consumer's `pushMs` (downstream backpressure), so the two can overlap and\n   * their sum can exceed `elapsedMs`. `reserveMs` is time spent (re)acquiring\n   * the change-stream reservation, and `stallMs` is time the consumer waited on\n   * an empty prefetch queue (i.e. blocked on the upstream) — a large `stallMs`\n   * indicates an upstream-bound backfill. `waitMs` is time spent waiting\n   * (without the reservation) for the change stream to reach the backfill\n   * watermark, during which the upstream snapshot is held open.\n   */\n  #logBackfillStats(lc: LogContext, stats: BackfillTimingStats) {\n    const elapsed = performance.now() - stats.start;\n    lc.info?.(`backfill timing`, {\n      messages: stats.messages,\n      bytes: stats.bytes,\n      commits: stats.commits,\n      stalls: stats.stalls,\n      elapsedMs: Math.round(elapsed),\n      produceMs: Math.round(stats.produceMs),\n      pushMs: Math.round(stats.pushMs),\n      reserveMs: Math.round(stats.reserveMs),\n      stallMs: Math.round(stats.stallMs),\n      waitMs: Math.round(stats.waitMs),\n      throughputKBps:\n        elapsed > 0 ? Math.round(stats.bytes / 1024 / (elapsed / 1000)) : 0,\n    });\n  }\n\n  #backfillRunningFor(table: Identifier): RunningBackfillState | null {\n    const state = this.#runningBackfill;\n    return state?.request.table.schema === table.schema &&\n      state.request.table.name === table.name\n      ? state\n      : null;\n  }\n\n  /**\n   * Stops the running backfill for the specified `reason`. If `instance` is\n   * specified, the running backfill is stopped only if it is that instance.\n   * This allows the running backfill itself to clear backfill state without\n   * accidentally stopping a different (e.g. subsequent) backfill.\n   */\n  #stopRunningBackfill(reason?: string, instance?: RunningBackfillState) {\n    const backfill = this.#runningBackfill;\n    if (backfill && backfill === (instance ?? backfill)) {\n      backfill.stopped.abort(reason);\n      this.#runningBackfill = null;\n      reason && this.#lc.info?.(`canceling backfill:`, reason);\n    }\n  }\n\n  #setRequiredBackfill(source: string, req: BackfillRequest) {\n    const action = this.#requiredBackfills.has(req.table) ? 'updated' : 'added';\n    this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n    this.#requiredBackfills.set(req.table, req);\n  }\n\n  #deleteRequiredBackfill(source: string, id: Identifier) {\n    const req = this.#requiredBackfills.get(id);\n    if (req) {\n      const action = source === 'backfill-completed' ? 'completed' : 'dropped';\n      this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n      this.#requiredBackfills.delete(id);\n    }\n  }\n\n  /**\n   * Implements {@link Listener.onChange()}, invoked by the\n   * {@link ChangeStreamMultiplexer}.\n   */\n  onChange(message: ChangeStreamMessage): void {\n    if (message[0] === 'begin') {\n      this.#currentTxWatermark = message[2].commitWatermark;\n      return;\n    }\n    if (message[0] === 'commit') {\n      this.#currentTxWatermark = null;\n      this.#setLastStatusWatermark(message[2]);\n      // Every commit is a candidate for starting the next backfill\n      // (if one is not currently running).\n      this.#checkAndStartBackfill();\n      return;\n    }\n    if (message[0] === 'status') {\n      this.#setLastStatusWatermark(message[2]);\n      return;\n    }\n    if (message[0] !== 'data') {\n      return;\n    }\n    const change = message[1];\n    const {tag} = change;\n    switch (tag) {\n      case 'update-table-metadata': {\n        const {table, new: metadata} = change;\n        const backfillRequest = this.#requiredBackfills.get(table);\n        if (backfillRequest) {\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            table: {...backfillRequest.table, metadata},\n          });\n          if (this.#backfillRunningFor(table)) {\n            this.#stopRunningBackfill(`TableMetadata updated`);\n          }\n        }\n        break;\n      }\n      case 'create-table': {\n        const {\n          spec: {schema, name},\n          metadata = null,\n          backfill,\n        } = change;\n\n        if (backfill) {\n          this.#setRequiredBackfill(tag, {\n            table: {schema, name, metadata},\n            columns: Object.fromEntries(\n              Object.entries(backfill).map(([col, id]) => [col, {id}]),\n            ),\n          });\n        }\n        break;\n      }\n      case 'rename-table': {\n        const {old, new: newTable} = change;\n        const backfillRequest = this.#requiredBackfills.get(old);\n        if (backfillRequest) {\n          const {schema, name} = newTable;\n          this.#deleteRequiredBackfill(tag, old);\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            table: {...backfillRequest.table, schema, name},\n          });\n          if (this.#backfillRunningFor(old)) {\n            this.#stopRunningBackfill(`table renamed`);\n          }\n        }\n        break;\n      }\n      case 'drop-table': {\n        const {id} = change;\n        const backfillRequest = this.#requiredBackfills.get(id);\n        if (backfillRequest) {\n          this.#deleteRequiredBackfill(tag, id);\n          if (this.#backfillRunningFor(id)) {\n            this.#stopRunningBackfill(`table dropped`);\n          }\n        }\n        break;\n      }\n      case 'add-column': {\n        const {\n          table,\n          tableMetadata: metadata = null,\n          column,\n          backfill,\n        } = change;\n        if (backfill) {\n          const backfillRequest = this.#requiredBackfills.get(table);\n          if (!backfillRequest) {\n            this.#setRequiredBackfill(tag, {\n              table: {...table, metadata},\n              columns: {[column.name]: {id: backfill}},\n            });\n          } else {\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              table: {...backfillRequest.table, metadata},\n              columns: {\n                ...backfillRequest.columns,\n                [column.name]: {id: backfill},\n              },\n            });\n            // Note: The running backfill need not be canceled if a\n            //   new column is added. The new column will be backfilled\n            //   by its own stream after the current backfill completes.\n          }\n        }\n        break;\n      }\n      case 'update-column': {\n        const {\n          table,\n          old: {name: oldName},\n          new: {name: newName},\n        } = change;\n        if (oldName !== newName) {\n          const backfillRequest = this.#requiredBackfills.get(table);\n          if (backfillRequest && oldName in backfillRequest.columns) {\n            const {[oldName]: colSpec, ...otherCols} = backfillRequest.columns;\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              columns: {...otherCols, [newName]: colSpec},\n            });\n            const backfill = this.#backfillRunningFor(table);\n            if (backfill && oldName in backfill.request.columns) {\n              this.#stopRunningBackfill(`column renamed`);\n            }\n          }\n        }\n        break;\n      }\n      case 'drop-column': {\n        const {table, column} = change;\n        const backfillRequest = this.#requiredBackfills.get(table);\n        if (backfillRequest && column in backfillRequest.columns) {\n          const {[column]: _excluded, ...remaining} = backfillRequest.columns;\n          if (Object.keys(remaining).length === 0) {\n            this.#deleteRequiredBackfill(tag, table);\n          } else {\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              columns: remaining,\n            });\n          }\n          const backfill = this.#backfillRunningFor(table);\n          if (backfill && column in backfill.request.columns) {\n            this.#stopRunningBackfill(`column dropped`);\n          }\n        }\n        break;\n      }\n      case 'update': {\n        const {relation, key, new: row} = change;\n        const backfill = this.#backfillRunningFor(relation);\n        const txWatermark = must(this.#currentTxWatermark, `not in a tx`);\n        if (backfill?.request.table.metadata && key !== null) {\n          // A corner case that backfill is unable to correctly handle is\n          // when a row's key changes; this is decomposed into a delete\n          // of the old key and a set of the new key in the replica change\n          // log, at which point the backfill algorithm assumes that the\n          // (old) row is deleted but does not know to backfill the new row.\n          // In these corner cases, the current backfill is canceled and\n          // retried if its version precedes this update.\n          for (const col of Object.keys(\n            backfill.request.table.metadata.rowKey,\n          )) {\n            if (key[col] !== row[col]) {\n              backfill.minWatermark = txWatermark;\n              this.#lc.info?.(\n                `key for row as changed (col: ${col}). ` +\n                  `backfill data must not predate ${backfill.minWatermark}`,\n              );\n              break;\n            }\n          }\n        }\n        break;\n      }\n      case 'backfill-completed': {\n        const {relation, columns} = change;\n        const backfillRequest = this.#requiredBackfills.get(relation);\n        assert(\n          backfillRequest,\n          () => `No BackfillRequest completed backfill ${stringify(change)}`,\n        );\n        const remaining = Object.entries(backfillRequest.columns).filter(\n          ([col]) =>\n            !(columns.includes(col) || relation.rowKey.columns.includes(col)),\n        );\n        if (remaining.length === 0) {\n          this.#deleteRequiredBackfill(tag, relation);\n        } else {\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            columns: Object.fromEntries(remaining),\n          });\n        }\n        // Technically the backfill is already stopping, but this method\n        // cleans up the state that tracks it.\n        this.#stopRunningBackfill();\n        break;\n      }\n    }\n  }\n\n  cancel(): void {\n    this.#controller.abort('canceled');\n    this.#stopRunningBackfill(`change stream canceled`);\n    clearTimeout(this.#backfillRetryTimer);\n    this.#backfillRetryTimer = undefined;\n\n    // Wake up a backfill that is waiting for the change stream to reach a\n    // watermark. The change stream never will, and the backfill must be\n    // allowed to unwind (and release its upstream resources) rather than\n    // remain suspended forever.\n    for (const {reached} of this.#awaitingStatusWatermarks.splice(0)) {\n      reached();\n    }\n  }\n}\n\nabstract class BackfillStreamError extends Error {\n  constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n    super(\n      `Cannot backfill ${bf.table.schema}.${bf.table.name}` +\n        `[${Object.keys(bf.columns).join(',')}]: ${msg}`,\n      {cause},\n    );\n  }\n}\n\n/**\n * Background: The zero-cache supports replication of tables without a\n * PRIMARY KEY to facilitate the onboarding process. These rows can be\n * INSERT'ed, but postgres will rightfully prohibit UPDATEs and DELETEs\n * on such tables because the rows cannot be identified by a key. Supporting\n * this mode of replication allows the user to \"fix\" the setup by adding the\n * primary key, after which the table can be published downstream without\n * requiring a resync of the data.\n *\n * In terms of backfill, however, non-empty tables without a row key **cannot**\n * be backfilled, because backfill retries would result in writing duplicating\n * rows. (Empty tables, on the other hand, are fine because there is no data\n * to be deduped.)\n *\n * The MissingRowKeyError is used to signal that the table cannot be backfilled\n * in its current state. For simplicity, it is handled like runtime errors and\n * retried with backoff, with which it can eventually succeed if (1) a primary\n * key is added or (2) the table is emptied, e.g. via a TRUNCATE.\n */\nclass MissingRowKeyError extends BackfillStreamError {\n  readonly name = 'MissingRowKeyError';\n\n  constructor(bf: BackfillRequest, cause?: unknown) {\n    super(bf, `\"${bf.table.name}\" is missing a PRIMARY KEY`, cause);\n  }\n}\n\n/**\n * Error type for backfill stream implementations to throw indicating that\n * the backfill request failed due to a schema incompatibility error. This\n * type of error does not need exponential backoff, as the retry happens\n * naturally once the invalidating schema change is processed and committed.\n */\nexport class SchemaIncompatibilityError extends BackfillStreamError {\n  readonly name = 'SchemaIncompatibilityError';\n\n  constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n    super(bf, msg, cause);\n  }\n}\n"],"mappings":";;;;;;;;;;;;AA4BA,SAAS,SAAS,EAAC,QAAQ,QAAmB;CAC5C,OAAO,GAAG,OAAO,GAAG;AACtB;;;;;;AAsBA,IAAM,yBAAyB,IAAI,OAAO;;;;;;;;AAS1C,IAAM,4BAA4B;;;;;;;;;;;;;;AAelC,IAAM,0BAA0B;;AAGhC,IAAM,iCAAiC;;AAGvC,IAAM,OAAO,OAAO,eAAe;;AAEnC,IAAM,OAAO,OAAO,eAAe;AAmCnC,IAAM,0BAA0B;AAChC,IAAM,0BAA0B;;;;;;;;;;;;;;;;;AAuBhC,IAAa,kBAAb,MAA6D;CAC3D;;;;;CAMA,qBAA8B,IAAI,aAChC,QACF;CACA;CACA;CACA;;;;;;;CAQA,mBAAgD;;CAGhD,uBAAsC;CAEtC,4BAAgE,CAAC;;CAGjE,sBAAqC;CAErC;;;;;;;CAQA,cAAuB,IAAI,gBAAgB;CAE3C,IAAIQ,YAAY;EACd,OAAO,KAAKD,YAAY,OAAO;CACjC;CAEA,YACE,IACA,gBACA,kBACA,aAAA,KACA,eAAe,yBACf,eAAe,yBACf,uBAAuB,wBACvB;EACA,KAAKP,MAAM,GAAG,YAAY,aAAa,kBAAkB;EACzD,KAAKE,kBAAkB;EACvB,KAAKC,oBAAoB;EACzB,KAAKC,cAAc;EACnB,KAAKK,gBAAgB;EACrB,KAAKC,gBAAgB;EACrB,KAAKC,gBAAgB;EACrB,KAAKL,wBAAwB;CAC/B;CAEA,IAAI,eAAuB,iBAAoC;EAC7D,KAAKN,IAAI,OACP,kCAAkC,gBAAgB,OAAO,oBACzD,EAAC,UAAU,gBAAe,CAC5B;EACA,KAAKY,uBAAuB;EAC5B,gBAAgB,SAAQ,QACtB,KAAKC,qBAAqB,mBAAmB,GAAG,CAClD;EACA,KAAKC,uBAAuB;CAC9B;CAEA,wBAAwB,EAAC,aAAiC;EAIxD,KAAK,KAAKF,wBAAwB,MAAM,WAAW;GACjD,KAAKA,uBAAuB;GAC5B,KAAK,IAAI,IAAI,KAAKP,0BAA0B,SAAS,GAAG,KAAK,GAAG,KAAK;IACnE,MAAM,WAAW,KAAKA,0BAA0B;IAChD,IAAI,aAAa,SAAS,WAAW;KACnC,SAAS,QAAQ;KACjB,KAAKA,0BAA0B,OAAO,GAAG,CAAC;IAC5C;GACF;EACF;CACF;CAEA,qBACE,IACA,WACsB;EACtB,KAAK,KAAKO,wBAAwB,MAAM,WAAW;GACjD,MAAM,EAAC,SAAS,SAAS,YAAW,SAAS;GAC7C,KAAKP,0BAA0B,KAAK;IAAC;IAAW;GAAO,CAAC;GACxD,GAAG,OACD,iCAAiC,KAAKO,qBAAqB,aAAa,WAC1E;GACA,OAAO;EACT;EACA,OAAO;CACT;CAEA;CACA;CACA;CACA;CAEA,yBAAyB;EACvB,IACE,CAAC,KAAKJ,aACN,CAAC,KAAKO,uBACN,CAAC,KAAKC,oBACN,KAAKf,mBAAmB,MACxB;GAKA,MAAM,aAAa,CAAC,GAAG,KAAKA,mBAAmB,OAAO,CAAC;GACvD,MAAM,UAAU,WAAW,QAAQ,GAAG,WAAW,SAAS,CAAC;GAC3D,MAAM,QAAQ;IACZ;IACA,cAAc;IACd,SAAS,IAAI,gBAAgB;GAC/B;GACA,MAAM,KAAK,KAAKD,IAAI,YAAY,SAAS,QAAQ,MAAM,IAAI;GAE3D,KAAKgB,mBAAmB;GACxB,KAAUC,aAAa,IAAI,KAAK,EAC7B,WAAW;IACV,KAAKC,qBAAqB,mBAAmB,KAAK;IAClD,KAAKP,gBAAgB,KAAKF;GAC5B,CAAC,EAGA,OAAM,MAAK;IACV,KAAKS,qBAAqB,OAAO,CAAC,GAAG,KAAK;IAC1C,KAAKC,0BAA0B,CAAC;GAClC,CAAC;EACL;CACF;CAEA,0BAA0B,GAAY;EACpC,IAAI,KAAKX,WAAW;GAClB,KAAKR,IAAI,QAAQ,iDAAiD,CAAC;GACnE;EACF;EACA,KAAKA,IAAI,OACP,uCAAuC,KAAKW,cAAc,MAC1D,CACF;EACA,KAAKI,sBAAsB,iBAAiB;GAC1C,KAAKA,sBAAsB,KAAA;GAC3B,KAAKD,uBAAuB;EAC9B,GAAG,KAAKH,aAAa;EAErB,KAAKA,gBAAgB,KAAK,IAAI,KAAKA,gBAAgB,GAAG,KAAKD,aAAa;CAC1E;CAEA,MAAMO,aAAa,IAAgB,OAA6B;EAC9D,MAAM,eAAe,KAAKf;EAI1B,IAAI,aAA4B;EAChC,IAAI,mBAAmB;EAEvB,MAAM,QAA6B;GACjC,OAAO,YAAY,IAAI;GACvB,YAAY,YAAY,IAAI;GAC5B,UAAU;GACV,OAAO;GACP,SAAS;GACT,QAAQ;GACR,WAAW;GACX,QAAQ;GACR,WAAW;GACX,SAAS;GACT,QAAQ;EACV;;;;EAKA,MAAM,aAAa,OACjB,QAC2B;GAC3B,OAAO,eAAe,MAAM,yCAAyC;GACrE,MAAM,gBAAgB,MAAM,aAAa,QAAQ,UAAU;GAI3D,IACE,MAAM,QAAQ,OAAO,WACpB,IAAI,QAAQ,cAAc,IAAI,YAAY,MAAM,cACjD;IACA,IAAI,CAAC,MAAM,QAAQ,OAAO,SAAS;KACjC,OAAO,IAAI,QAAQ,YAAY,+BAA+B;KAC9D,KAAKgB,qBACH,qBAAqB,MAAM,aAAa,mCACH,IAAI,aACzC,KACF;IACF;IACA,aAAa,QAAQ,aAAa;IAClC,OAAO;GACT;GAEA,MAAM,EAAC,OAAO,QAAQ,OAAM,uBAAuB,aAAa;GAChE,IAAI,KAAK,qBAAqB;IAC5B;IACA,OAAO,OAAO,KAAK,IAAI;GACzB,CAAC;GAED,IAAI,KAAK,IAAI,WAOX,KAAK,IAAI;GAGX,aAAkB,KAAK;IACrB;IACA;KAAC,KAAK;KAAS,MAAM,KAAKd;KAAa,SAAS;IAAI;IACpD,EAAC,iBAAiB,GAAE;GACtB,CAAC;GACD,OAAQ,aAAa;EACvB;EAEA,MAAM,iBAAiB;GACrB,IAAI,YAAY;IACd,aAAkB,KAAK;KACrB;KACA,EAAC,KAAK,SAAQ;KACd,EAAC,WAAW,WAAU;IACxB,CAAC;IACD,aAAa,QAAQ,UAAU;IAC/B,MAAM;GACR;GACA,aAAa;GACb,mBAAmB;EACrB;EASA,MAAM,QAAQ,IAAI,MAAY;EAC9B,MAAM,UAAU,IAAI,MAAS;EAC7B,KAAK,IAAI,IAAI,GAAG,IAAI,2BAA2B,KAC7C,QAAQ,QAAQ,CAAC;EAEnB,IAAI,eAAe;EAInB,MAAM,iBACJ,MAAM,iBAAiB,IAAI,WAAW,mBAAmB,CAAC;EAC5D,IAAI,CAAC,KAAKI,WACR,KAAKD,YAAY,OAAO,iBAAiB,SAAS,QAAQ;EAG5D,MAAM,YAAY,YAAY;GAC5B,IAAI;IACF,IAAI,KAAK,YAAY,IAAI;IACzB,WAAW,MAAM,SAAS,KAAKJ,kBAAkB,MAAM,OAAO,GAAG;KAC/D,MAAM,aAAa,YAAY,IAAI,IAAI;KACvC,MAAM,QAAQ,QAAQ;KACtB,IAAI,gBAAgB,KAAKK,WAAW;MAClC,MAAM,iBAAiB,IAAI,WAAW,mBAAmB,CAAC;MAC1D;KACF;KACA,MAAM,QAAQ,KAAK;KACnB,KAAK,YAAY,IAAI;IACvB;IACA,MAAM,QAAQ,IAAI;GACpB,SAAS,GAAG;IACV,MAAM,iBAAiB,CAAC;GAC1B;EACF,GAAG;EAIH,SAAc,YAAY,CAAC,CAAC;EAS5B,MAAM,WAAW,YAAoD;GAKnE,IAAI,eAAe,QAAQ,MAAM,KAAK,IAAI,GAAG;IAC3C,MAAM,IAAI,MAAM,QAAQ;IAExB,OADU,aAAa,UAAU,MAAM,IAAI;GAE7C;GAEA,SAAS;IACP,IAAI,eAAe,QAAQ,aAAa,YAAY,IAAI,GACtD,SAAS;IAEX,MAAM,IACJ,eAAe,OACX,MAAM,QAAQ,MAAM,uBAAuB,IAC3C,MAAM,QAAQ;IACpB,MAAM,KAAK,YAAY,IAAI;IAC3B,MAAM,IAAI,aAAa,UAAU,MAAM,IAAI;IAC3C,IAAI,MAAM,MAAM;KACd,MAAM,WAAW,YAAY,IAAI,IAAI;KACrC;IACF;IACA,IAAI,aAAa,SAAS;KACxB,MAAM,WAAW,YAAY,IAAI,IAAI;KACrC,MAAM;IACR;IACA,OAAO;GACT;EACF;EAEA,IAAI;GACF,SAAS;IACP,MAAM,OAAO,MAAM,SAAS;IAC5B,IAAI,SAAS,MACX;IAIF,QAAQ,QAAQ,CAAC;IAEjB,IAAI,KAAKA,WAAW;KAIlB,GAAG,OAAO,kDAAkD;KAC5D;IACF;IACA,MAAM,EAAC,SAAS,KAAK,aAAY;IAOjC,MAAM,sBAAsB,KAAKY,qBAC/B,IACA,IAAI,SACN;IAKA,IACE,eACC,aAAa,YAAY,IAAI,KAC5B,uBACA,oBAAoB,KAAKd,wBAE3B,SAAS;IAGX,IAAI,qBAAqB;KACvB,MAAM,KAAK,YAAY,IAAI;KAG3B,MAAM,eAAe,qBAAqB,MAAM,QAAQ,MAAM;KAC9D,MAAM,UAAU,YAAY,IAAI,IAAI;KACpC,MAAM,UAAU;KAChB,GAAG,OACD,yBAAyB,IAAI,UAAU,iBACpB,KAAK,MAAM,OAAO,EAAE,IACzC;IACF;IAEA,IACE,IAAI,QAAQ,cACZ,IAAI,UAAU,SAAS,KACvB,IAAI,SAAS,OAAO,QAAQ,WAAW,GAEvC,MAAM,IAAI,mBAAmB,MAAM,OAAO;IAI5C,IAAI,eAAe,MAAM;KACvB,MAAM,KAAK,YAAY,IAAI;KAC3B,aAAa,MAAM,WAAW,GAAG;KACjC,MAAM,aAAa,YAAY,IAAI,IAAI;KACvC,IAAI,eAAe,MAAM;MACvB,GAAG,OACD,6BAA6B,MAAM,QAAQ,OAAO,UAClD,MAAM,OACR;MACA,KAAKQ,uBAAuB;MAC5B;KACF;IACF;IAIA,MAAM,KAAK,YAAY,IAAI;IAC3B,MAAM,aAAa,KAAK,CAAC,QAAQ,GAAG,CAAC;IACrC,MAAM,UAAU,YAAY,IAAI,IAAI;IACpC,oBAAoB;IACpB,MAAM;IACN,MAAM,SAAS;IAEf,KAAKO,uBAAuB,IAAI,KAAK;GACvC;GAGA,IAAI,YACF,SAAS;GAEX,GAAG,QAAQ,wBAAwB;EACrC,UAAU;GACR,eAAe;GACf,QAAQ,QAAQ,CAAC;GAGjB,IAAI,CAAC,KAAKb,WACR,MAAM;GAER,KAAKD,YAAY,OAAO,oBAAoB,SAAS,QAAQ;GAC7D,KAAKe,kBAAkB,IAAI,KAAK;EAClC;CACF;CAEA,uBAAuB,IAAgB,OAA4B;EACjE,MAAM,MAAM,YAAY,IAAI;EAC5B,IAAI,MAAM,MAAM,cAAc,gCAAgC;GAC5D,KAAKA,kBAAkB,IAAI,KAAK;GAChC,MAAM,aAAa;EACrB;CACF;;;;;;;;;;;;CAaA,kBAAkB,IAAgB,OAA4B;EAC5D,MAAM,UAAU,YAAY,IAAI,IAAI,MAAM;EAC1C,GAAG,OAAO,mBAAmB;GAC3B,UAAU,MAAM;GAChB,OAAO,MAAM;GACb,SAAS,MAAM;GACf,QAAQ,MAAM;GACd,WAAW,KAAK,MAAM,OAAO;GAC7B,WAAW,KAAK,MAAM,MAAM,SAAS;GACrC,QAAQ,KAAK,MAAM,MAAM,MAAM;GAC/B,WAAW,KAAK,MAAM,MAAM,SAAS;GACrC,SAAS,KAAK,MAAM,MAAM,OAAO;GACjC,QAAQ,KAAK,MAAM,MAAM,MAAM;GAC/B,gBACE,UAAU,IAAI,KAAK,MAAM,MAAM,QAAQ,QAAQ,UAAU,IAAK,IAAI;EACtE,CAAC;CACH;CAEA,oBAAoB,OAAgD;EAClE,MAAM,QAAQ,KAAKN;EACnB,OAAO,OAAO,QAAQ,MAAM,WAAW,MAAM,UAC3C,MAAM,QAAQ,MAAM,SAAS,MAAM,OACjC,QACA;CACN;;;;;;;CAQA,qBAAqB,QAAiB,UAAiC;EACrE,MAAM,WAAW,KAAKA;EACtB,IAAI,YAAY,cAAc,YAAY,WAAW;GACnD,SAAS,QAAQ,MAAM,MAAM;GAC7B,KAAKA,mBAAmB;GACxB,UAAU,KAAKhB,IAAI,OAAO,uBAAuB,MAAM;EACzD;CACF;CAEA,qBAAqB,QAAgB,KAAsB;EACzD,MAAM,SAAS,KAAKC,mBAAmB,IAAI,IAAI,KAAK,IAAI,YAAY;EACpE,KAAKD,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,IAAG,CAAC;EAChE,KAAKC,mBAAmB,IAAI,IAAI,OAAO,GAAG;CAC5C;CAEA,wBAAwB,QAAgB,IAAgB;EACtD,MAAM,MAAM,KAAKA,mBAAmB,IAAI,EAAE;EAC1C,IAAI,KAAK;GACP,MAAM,SAAS,WAAW,uBAAuB,cAAc;GAC/D,KAAKD,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,IAAG,CAAC;GAChE,KAAKC,mBAAmB,OAAO,EAAE;EACnC;CACF;;;;;CAMA,SAAS,SAAoC;EAC3C,IAAI,QAAQ,OAAO,SAAS;GAC1B,KAAKsB,sBAAsB,QAAQ,GAAG;GACtC;EACF;EACA,IAAI,QAAQ,OAAO,UAAU;GAC3B,KAAKA,sBAAsB;GAC3B,KAAKC,wBAAwB,QAAQ,EAAE;GAGvC,KAAKV,uBAAuB;GAC5B;EACF;EACA,IAAI,QAAQ,OAAO,UAAU;GAC3B,KAAKU,wBAAwB,QAAQ,EAAE;GACvC;EACF;EACA,IAAI,QAAQ,OAAO,QACjB;EAEF,MAAM,SAAS,QAAQ;EACvB,MAAM,EAAC,QAAO;EACd,QAAQ,KAAR;GACE,KAAK,yBAAyB;IAC5B,MAAM,EAAC,OAAO,KAAK,aAAY;IAC/B,MAAM,kBAAkB,KAAKvB,mBAAmB,IAAI,KAAK;IACzD,IAAI,iBAAiB;KACnB,KAAKY,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;MAAQ;KAC5C,CAAC;KACD,IAAI,KAAKY,oBAAoB,KAAK,GAChC,KAAKP,qBAAqB,uBAAuB;IAErD;IACA;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,EACJ,MAAM,EAAC,QAAQ,QACf,WAAW,MACX,aACE;IAEJ,IAAI,UACF,KAAKL,qBAAqB,KAAK;KAC7B,OAAO;MAAC;MAAQ;MAAM;KAAQ;KAC9B,SAAS,OAAO,YACd,OAAO,QAAQ,QAAQ,EAAE,KAAK,CAAC,KAAK,QAAQ,CAAC,KAAK,EAAC,GAAE,CAAC,CAAC,CACzD;IACF,CAAC;IAEH;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,EAAC,KAAK,KAAK,aAAY;IAC7B,MAAM,kBAAkB,KAAKZ,mBAAmB,IAAI,GAAG;IACvD,IAAI,iBAAiB;KACnB,MAAM,EAAC,QAAQ,SAAQ;KACvB,KAAKyB,wBAAwB,KAAK,GAAG;KACrC,KAAKb,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;OAAQ;MAAI;KAChD,CAAC;KACD,IAAI,KAAKY,oBAAoB,GAAG,GAC9B,KAAKP,qBAAqB,eAAe;IAE7C;IACA;GACF;GACA,KAAK,cAAc;IACjB,MAAM,EAAC,OAAM;IAEb,IADwB,KAAKjB,mBAAmB,IAAI,EAChD,GAAiB;KACnB,KAAKyB,wBAAwB,KAAK,EAAE;KACpC,IAAI,KAAKD,oBAAoB,EAAE,GAC7B,KAAKP,qBAAqB,eAAe;IAE7C;IACA;GACF;GACA,KAAK,cAAc;IACjB,MAAM,EACJ,OACA,eAAe,WAAW,MAC1B,QACA,aACE;IACJ,IAAI,UAAU;KACZ,MAAM,kBAAkB,KAAKjB,mBAAmB,IAAI,KAAK;KACzD,IAAI,CAAC,iBACH,KAAKY,qBAAqB,KAAK;MAC7B,OAAO;OAAC,GAAG;OAAO;MAAQ;MAC1B,SAAS,GAAE,OAAO,OAAO,EAAC,IAAI,SAAQ,EAAC;KACzC,CAAC;UAED,KAAKA,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;MAAQ;MAC1C,SAAS;OACP,GAAG,gBAAgB;QAClB,OAAO,OAAO,EAAC,IAAI,SAAQ;MAC9B;KACF,CAAC;IAKL;IACA;GACF;GACA,KAAK,iBAAiB;IACpB,MAAM,EACJ,OACA,KAAK,EAAC,MAAM,WACZ,KAAK,EAAC,MAAM,cACV;IACJ,IAAI,YAAY,SAAS;KACvB,MAAM,kBAAkB,KAAKZ,mBAAmB,IAAI,KAAK;KACzD,IAAI,mBAAmB,WAAW,gBAAgB,SAAS;MACzD,MAAM,GAAE,UAAU,SAAS,GAAG,cAAa,gBAAgB;MAC3D,KAAKY,qBAAqB,KAAK;OAC7B,GAAG;OACH,SAAS;QAAC,GAAG;SAAY,UAAU;OAAO;MAC5C,CAAC;MACD,MAAM,WAAW,KAAKY,oBAAoB,KAAK;MAC/C,IAAI,YAAY,WAAW,SAAS,QAAQ,SAC1C,KAAKP,qBAAqB,gBAAgB;KAE9C;IACF;IACA;GACF;GACA,KAAK,eAAe;IAClB,MAAM,EAAC,OAAO,WAAU;IACxB,MAAM,kBAAkB,KAAKjB,mBAAmB,IAAI,KAAK;IACzD,IAAI,mBAAmB,UAAU,gBAAgB,SAAS;KACxD,MAAM,GAAE,SAAS,WAAW,GAAG,cAAa,gBAAgB;KAC5D,IAAI,OAAO,KAAK,SAAS,EAAE,WAAW,GACpC,KAAKyB,wBAAwB,KAAK,KAAK;UAEvC,KAAKb,qBAAqB,KAAK;MAC7B,GAAG;MACH,SAAS;KACX,CAAC;KAEH,MAAM,WAAW,KAAKY,oBAAoB,KAAK;KAC/C,IAAI,YAAY,UAAU,SAAS,QAAQ,SACzC,KAAKP,qBAAqB,gBAAgB;IAE9C;IACA;GACF;GACA,KAAK,UAAU;IACb,MAAM,EAAC,UAAU,KAAK,KAAK,QAAO;IAClC,MAAM,WAAW,KAAKO,oBAAoB,QAAQ;IAClD,MAAM,cAAc,KAAK,KAAKF,qBAAqB,aAAa;IAChE,IAAI,UAAU,QAAQ,MAAM,YAAY,QAAQ;UAQzC,MAAM,OAAO,OAAO,KACvB,SAAS,QAAQ,MAAM,SAAS,MAClC,GACE,IAAI,IAAI,SAAS,IAAI,MAAM;MACzB,SAAS,eAAe;MACxB,KAAKvB,IAAI,OACP,gCAAgC,IAAI,oCACA,SAAS,cAC/C;MACA;KACF;;IAGJ;GACF;GACA,KAAK,sBAAsB;IACzB,MAAM,EAAC,UAAU,YAAW;IAC5B,MAAM,kBAAkB,KAAKC,mBAAmB,IAAI,QAAQ;IAC5D,OACE,uBACM,yCAAyC,UAAU,MAAM,GACjE;IACA,MAAM,YAAY,OAAO,QAAQ,gBAAgB,OAAO,EAAE,QACvD,CAAC,SACA,EAAE,QAAQ,SAAS,GAAG,KAAK,SAAS,OAAO,QAAQ,SAAS,GAAG,EACnE;IACA,IAAI,UAAU,WAAW,GACvB,KAAKyB,wBAAwB,KAAK,QAAQ;SAE1C,KAAKb,qBAAqB,KAAK;KAC7B,GAAG;KACH,SAAS,OAAO,YAAY,SAAS;IACvC,CAAC;IAIH,KAAKK,qBAAqB;IAC1B;GACF;EACF;CACF;CAEA,SAAe;EACb,KAAKX,YAAY,MAAM,UAAU;EACjC,KAAKW,qBAAqB,wBAAwB;EAClD,aAAa,KAAKH,mBAAmB;EACrC,KAAKA,sBAAsB,KAAA;EAM3B,KAAK,MAAM,EAAC,aAAY,KAAKV,0BAA0B,OAAO,CAAC,GAC7D,QAAQ;CAEZ;AACF;AAEA,IAAe,sBAAf,cAA2C,MAAM;CAC/C,YAAY,IAAqB,KAAa,OAAiB;EAC7D,MACE,mBAAmB,GAAG,MAAM,OAAO,GAAG,GAAG,MAAM,KAAA,GACzC,OAAO,KAAK,GAAG,OAAO,EAAE,KAAK,GAAG,EAAE,KAAK,OAC7C,EAAC,MAAK,CACR;CACF;AACF;;;;;;;;;;;;;;;;;;;;AAqBA,IAAM,qBAAN,cAAiC,oBAAoB;CACnD,OAAgB;CAEhB,YAAY,IAAqB,OAAiB;EAChD,MAAM,IAAI,IAAI,GAAG,MAAM,KAAK,6BAA6B,KAAK;CAChE;AACF;;;;;;;AAQA,IAAa,6BAAb,cAAgD,oBAAoB;CAClE,OAAgB;CAEhB,YAAY,IAAqB,KAAa,OAAiB;EAC7D,MAAM,IAAI,KAAK,KAAK;CACtB;AACF"}
+{"version":3,"file":"backfill-manager.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {stringify} from '../../../../../shared/src/bigint-json.ts';\nimport {CustomKeyMap} from '../../../../../shared/src/custom-key-map.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {promiseOrAbort} from '../../../../../shared/src/promise-race.ts';\nimport {Queue} from '../../../../../shared/src/queue.ts';\nimport {randInt} from '../../../../../shared/src/rand.ts';\nimport {JSON_STRINGIFIED, type JSONFormat} from '../../../types/lite.ts';\nimport {\n  stateVersionFromString,\n  stateVersionToString,\n} from '../../../types/state-version.ts';\nimport type {\n  BackfillCompleted,\n  BackfillRequest,\n  ChangeStreamMessage,\n  Identifier,\n  MessageBackfill,\n} from '../protocol/current.ts';\nimport type {\n  Cancelable,\n  ChangeStreamMultiplexer,\n  Listener,\n} from './change-stream-multiplexer.ts';\n\nfunction tableKey({schema, name}: Identifier) {\n  return `${schema}.${name}`;\n}\n\n/**\n * A {@link MessageBackfill} or {@link BackfillCompleted} paired with the\n * approximate number of upstream bytes it represents. The BackfillManager uses\n * `byteSize` to bound the size of each replica transaction, committing and\n * reopening once {@link COMMIT_THRESHOLD_BYTES} is reached.\n */\nexport type BackfillMessage = {\n  message: MessageBackfill | BackfillCompleted;\n  byteSize: number;\n};\n\ntype BackfillStreamer = (\n  req: BackfillRequest,\n) => AsyncGenerator<BackfillMessage>;\n\n/**\n * Commit (and reopen) the current backfill transaction once this many bytes of\n * backfill data have accumulated in it. Bounds the size of a single replica\n * COMMIT/checkpoint so it can't monopolize the replica.\n */\nconst COMMIT_THRESHOLD_BYTES = 8 * 1024 * 1024;\n\n/**\n * How many backfill messages the prefetch producer may read ahead of the\n * reservation-holding consumer loop. This bounds the in-memory buffer (~this\n * many {@link BackfillMessage}s, each up to the stream's flush threshold, e.g.\n * 64 KiB) while giving the consumer a supply of ready messages so it never\n * has to wait on the upstream COPY while holding the change-stream reservation.\n */\nconst BACKFILL_PREFETCH_CREDITS = 16;\n\n/**\n * While the backfill holds an open transaction but its prefetch queue is empty\n * (i.e. the upstream is momentarily behind), the consumer polls for the next\n * message at this interval so that a replication transaction that begins\n * waiting mid-fetch can preempt the backfill within this bound, rather than\n * being blocked for the entire (possibly slow) upstream read.\n *\n * If stats indicate that these stalls are highly correlative with replication\n * lag (indicating that replication changes are waiting on these polls), we\n * can address this by adding an API to the change stream multiplexer that\n * would allow backfill to relinquish an in-progress commit on demand (i.e.\n * only if replication asks for a reservation).\n */\nconst BACKFILL_WAITER_POLL_MS = 10;\n\n/** Interval at which in-progress backfill timing stats are logged. */\nconst BACKFILL_STATS_LOG_INTERVAL_MS = 10_000;\n\n/** Sentinel enqueued by the prefetch producer when the backfill stream ends. */\nconst DONE = Symbol('backfill-done');\n/** Sentinel returned by a polled dequeue that timed out (see {@link BACKFILL_WAITER_POLL_MS}). */\nconst POLL = Symbol('backfill-poll');\n\ntype Pull = BackfillMessage | typeof DONE | typeof POLL;\n\n/**\n * Accumulated timing attribution for a single backfill run, logged\n * periodically to reveal where the backfill spends its time (and, in\n * particular, where the change-stream reservation is held). `produceMs`\n * accrues on the prefetch producer task and thus overlaps with the consumer's\n * `pushMs`; the others accrue on the reservation-holding consumer loop.\n */\ntype BackfillTimingStats = {\n  start: number;\n  lastLogged: number;\n  messages: number;\n  bytes: number;\n  commits: number;\n  stalls: number;\n  produceMs: number;\n  pushMs: number;\n  reserveMs: number;\n  stallMs: number;\n  waitMs: number;\n};\n\ntype RunningBackfillState = {\n  request: BackfillRequest;\n  minWatermark: string;\n  /**\n   * Aborted (with the reason, if any) when this backfill is stopped,\n   * e.g. canceled.\n   */\n  stopped: AbortController;\n};\n\nconst MIN_BACKOFF_INTERVAL_MS = 2_000;\nconst MAX_BACKOFF_INTERVAL_MS = 60_000;\n\ntype AwaitingStatusWatermark = {\n  watermark: string;\n  reached: () => void;\n};\n\n/**\n * The BackfillManager initiates backfills for BackfillRequests from the\n * change-streamer (i.e. unfinished backfills from previous sessions)\n * or for new backfills signaled by `create-table` or `add-column` messages\n * from the change-source.\n *\n * The BackfillManager registers itself as a change stream listener in order\n * to track necessary backfills, and potentially invalidate the in-progress\n * backfill (e.g. due to a schema change) so that it can be retried at a\n * new snapshot.\n *\n * The manager also handles low priority streaming of the backfill messages\n * using the {@link ChangeStreamMultiplexer}, implementing a policy of always\n * releasing its reservation if another producer (i.e. the main change stream)\n * has messages to stream.\n */\nexport class BackfillManager implements Cancelable, Listener {\n  readonly #lc: LogContext;\n\n  /**\n   * Tracks the metadata of required backfills based on schema changes\n   * and initial backfill requests.\n   */\n  readonly #requiredBackfills = new CustomKeyMap<Identifier, BackfillRequest>(\n    tableKey,\n  );\n  readonly #changeStreamer: ChangeStreamMultiplexer;\n  readonly #backfillStreamer: BackfillStreamer;\n  readonly #jsonFormat: JSONFormat;\n\n  /**\n   * The current running backfill. The backfill request is always also in\n   * `#requiredBackfills` (technically, it can be a subset of what's in\n   * `#requiredBackfills`); the request is removed from `#requiredBackfills`\n   * upon completion.\n   */\n  #runningBackfill: RunningBackfillState | null = null;\n\n  /** The last seen watermark in the change stream. */\n  #lastStatusWatermark: string | null = null;\n\n  readonly #awaitingStatusWatermarks: AwaitingStatusWatermark[] = [];\n\n  /** The watermark of the current transaction in the change stream. */\n  #currentTxWatermark: string | null = null;\n\n  readonly #commitThresholdBytes: number;\n\n  /**\n   * Aborted when the change stream is canceled. No further backfills are run.\n   *\n   * TODO: Thread this into the backfillStream so that upstream I/O can be\n   * aborted.\n   */\n  readonly #controller = new AbortController();\n\n  get #canceled() {\n    return this.#controller.signal.aborted;\n  }\n\n  constructor(\n    lc: LogContext,\n    changeStreamer: ChangeStreamMultiplexer,\n    backfillStreamer: BackfillStreamer,\n    jsonFormat: JSONFormat = JSON_STRINGIFIED,\n    minBackoffMs = MIN_BACKOFF_INTERVAL_MS,\n    maxBackoffMs = MAX_BACKOFF_INTERVAL_MS,\n    commitThresholdBytes = COMMIT_THRESHOLD_BYTES,\n  ) {\n    this.#lc = lc.withContext('component', 'backfill-manager');\n    this.#changeStreamer = changeStreamer;\n    this.#backfillStreamer = backfillStreamer;\n    this.#jsonFormat = jsonFormat;\n    this.#minBackoffMs = minBackoffMs;\n    this.#maxBackoffMs = maxBackoffMs;\n    this.#retryDelayMs = minBackoffMs;\n    this.#commitThresholdBytes = commitThresholdBytes;\n  }\n\n  run(lastWatermark: string, initialRequests: BackfillRequest[]) {\n    this.#lc.info?.(\n      `starting backfill manager with ${initialRequests.length} initial requests`,\n      {requests: initialRequests},\n    );\n    this.#lastStatusWatermark = lastWatermark;\n    initialRequests.forEach(req =>\n      this.#setRequiredBackfill('initial-request', req),\n    );\n    this.#checkAndStartBackfill();\n  }\n\n  #setLastStatusWatermark({watermark}: {watermark: string}) {\n    // Only allow the watermark to move forward. This prevents a backfill\n    // transaction (whose watermark is unrelated to change-stream state)\n    // from moving the watermark backwards.\n    if ((this.#lastStatusWatermark ?? '') < watermark) {\n      this.#lastStatusWatermark = watermark;\n      for (let i = this.#awaitingStatusWatermarks.length - 1; i >= 0; i--) {\n        const awaiting = this.#awaitingStatusWatermarks[i];\n        if (watermark >= awaiting.watermark) {\n          awaiting.reached();\n          this.#awaitingStatusWatermarks.splice(i, 1);\n        }\n      }\n    }\n  }\n\n  #changeStreamReached(\n    lc: LogContext,\n    watermark: string,\n  ): Promise<void> | null {\n    if ((this.#lastStatusWatermark ?? '') < watermark) {\n      const {promise, resolve: reached} = resolver();\n      this.#awaitingStatusWatermarks.push({watermark, reached});\n      lc.info?.(\n        `waiting for change stream (at ${this.#lastStatusWatermark}) to reach ${watermark}`,\n      );\n      return promise;\n    }\n    return null;\n  }\n\n  readonly #minBackoffMs: number;\n  readonly #maxBackoffMs: number;\n  #retryDelayMs: number;\n  #backfillRetryTimer: NodeJS.Timeout | undefined;\n\n  #checkAndStartBackfill() {\n    if (\n      !this.#canceled &&\n      !this.#backfillRetryTimer &&\n      !this.#runningBackfill &&\n      this.#requiredBackfills.size\n    ) {\n      // Pick a random backfill to avoid head-of-line blocking by a\n      // problematic backfill (e.g. awaiting a primary key). This is\n      // simpler that adding logic to classify (and declassify)\n      // problematic backfills.\n      const candidates = [...this.#requiredBackfills.values()];\n      const request = candidates[randInt(0, candidates.length - 1)];\n      const state = {\n        request,\n        minWatermark: '',\n        stopped: new AbortController(),\n      };\n      const lc = this.#lc.withContext('table', request.table.name);\n\n      this.#runningBackfill = state;\n      void this.#runBackfill(lc, state)\n        .then(() => {\n          this.#stopRunningBackfill('backfill exited', state);\n          this.#retryDelayMs = this.#minBackoffMs; // reset on success\n        })\n        // For unexpected errors (e.g. upstream replication slot\n        // unavailability), retry with exponential backoff.\n        .catch(e => {\n          this.#stopRunningBackfill(String(e), state);\n          this.#retryBackfillWithBackoff(e);\n        });\n    }\n  }\n\n  #retryBackfillWithBackoff(e: unknown) {\n    if (this.#canceled) {\n      this.#lc.debug?.(`not retrying backfill: change stream canceled`, e);\n      return;\n    }\n    this.#lc.warn?.(\n      `Error running backfill. Retrying in ${this.#retryDelayMs} ms`,\n      e,\n    );\n    this.#backfillRetryTimer = setTimeout(() => {\n      this.#backfillRetryTimer = undefined;\n      this.#checkAndStartBackfill();\n    }, this.#retryDelayMs);\n\n    this.#retryDelayMs = Math.min(this.#retryDelayMs * 2, this.#maxBackoffMs);\n  }\n\n  async #runBackfill(lc: LogContext, state: RunningBackfillState) {\n    const changeStream = this.#changeStreamer; // Purely for readability\n\n    // backfillTx is set if and only if a changeStreamer reservation has been\n    // acquired and the backfill stream is inside a transaction.\n    let backfillTx: string | null = null;\n    let uncommittedBytes = 0;\n\n    const stats: BackfillTimingStats = {\n      start: performance.now(),\n      lastLogged: performance.now(),\n      messages: 0,\n      bytes: 0,\n      commits: 0,\n      stalls: 0,\n      produceMs: 0,\n      pushMs: 0,\n      reserveMs: 0,\n      stallMs: 0,\n      waitMs: 0,\n    };\n\n    /**\n     * @returns the new tx watermark, or null if backfill was cancelled\n     */\n    const beginTxFor = async (\n      msg: MessageBackfill | BackfillCompleted,\n    ): Promise<string | null> => {\n      assert(backfillTx === null, 'Expected no active backfill transaction');\n      const lastWatermark = await changeStream.reserve('backfill');\n\n      // After obtaining the changeStream reservation, check if the stream\n      // had changes that resulted in invalidating / canceling this backfill.\n      if (\n        state.stopped.signal.aborted ||\n        (msg.tag === 'backfill' && msg.watermark < state.minWatermark)\n      ) {\n        if (!state.stopped.signal.aborted) {\n          assert(msg.tag === 'backfill', 'Expected backfill message tag'); // TypeScript should have figured this out.\n          this.#stopRunningBackfill(\n            `row key change at ${state.minWatermark} ` +\n              `postdates backfill watermark at ${msg.watermark}`,\n            state,\n          );\n        }\n        changeStream.release(lastWatermark);\n        return null;\n      }\n\n      const {major, minor = 0n} = stateVersionFromString(lastWatermark);\n      let tx = stateVersionToString({\n        major,\n        minor: BigInt(minor) + 1n,\n      });\n\n      if (tx < msg.watermark) {\n        // At this point it must be the case that the #changeStreamReached()\n        // the backfill watermark (possibly via a status message, which does\n        // not advance the reservation watermark). Given that guarantee,\n        // ensure that the version of the backfill transaction is at least up\n        // to the backfill watermark, so that the database state version is\n        // never earlier than the version of any backfilled rows.\n        tx = msg.watermark;\n      }\n\n      void changeStream.push([\n        'begin',\n        {tag: 'begin', json: this.#jsonFormat, skipAck: true},\n        {commitWatermark: tx},\n      ]);\n      return (backfillTx = tx);\n    };\n\n    const commitTx = () => {\n      if (backfillTx) {\n        void changeStream.push([\n          'commit',\n          {tag: 'commit'},\n          {watermark: backfillTx},\n        ]);\n        changeStream.release(backfillTx);\n        stats.commits++;\n      }\n      backfillTx = null;\n      uncommittedBytes = 0;\n    };\n\n    // === Prefetch producer ===\n    //\n    // The backfill stream (an upstream Postgres COPY) is drained on a separate\n    // task into a bounded queue, so that the reservation-holding consumer loop\n    // below never blocks on the upstream read while it holds the change-stream\n    // reservation. `credits` bounds how far the producer runs ahead; the\n    // consumer returns a credit for every message it takes.\n    const ready = new Queue<Pull>();\n    const credits = new Queue<1>();\n    for (let i = 0; i < BACKFILL_PREFETCH_CREDITS; i++) {\n      credits.enqueue(1);\n    }\n    let stopProducer = false;\n\n    // Wakes a consumer blocked awaiting the next prefetched message so a\n    // canceled backfill unwinds promptly.\n    const onCancel = () =>\n      ready.enqueueRejection(new AbortError('backfill canceled'));\n    if (!this.#canceled) {\n      this.#controller.signal.addEventListener('abort', onCancel);\n    }\n\n    const producer = (async () => {\n      try {\n        let t0 = performance.now();\n        for await (const value of this.#backfillStreamer(state.request)) {\n          stats.produceMs += performance.now() - t0;\n          await credits.dequeue(); // wait for buffer space (flow control)\n          if (stopProducer || this.#canceled) {\n            ready.enqueueRejection(new AbortError('backfill canceled'));\n            return;\n          }\n          ready.enqueue(value);\n          t0 = performance.now();\n        }\n        ready.enqueue(DONE);\n      } catch (e) {\n        ready.enqueueRejection(e);\n      }\n    })();\n    // The producer routes errors through `ready` and never rejects; guard\n    // anyway so that leaving it un-awaited on the cancellation path (below)\n    // cannot surface as an unhandled rejection.\n    void producer.catch(() => {});\n\n    // === Consumer ===\n\n    // Fetches the next item from the prefetch queue. If a transaction is open\n    // but the queue is empty (the producer/upstream is momentarily behind),\n    // polls so that a replication transaction that begins waiting during the\n    // upstream read can preempt the backfill: rather than hold the reservation\n    // for the whole read, commit and release, letting replication proceed.\n    const nextItem = async (): Promise<BackfillMessage | typeof DONE> => {\n      // Fast path: a message is already buffered, or there's no reservation to\n      // protect. Take it without altering transaction boundaries. `POLL` is\n      // never enqueued (only used as a dequeue timeout value), so the result is\n      // always a message or DONE.\n      if (backfillTx === null || ready.size() > 0) {\n        const n = ready.dequeue();\n        const v = n instanceof Promise ? await n : n;\n        return v as BackfillMessage | typeof DONE;\n      }\n      // Slow path: holding a reservation with an empty queue.\n      for (;;) {\n        if (backfillTx !== null && changeStream.waiterDelay() > 0) {\n          commitTx(); // yield the stream to the waiting replication txn\n        }\n        const n =\n          backfillTx !== null\n            ? ready.dequeue(POLL, BACKFILL_WAITER_POLL_MS)\n            : ready.dequeue(); // released: just wait for the next message\n        const t0 = performance.now();\n        const v = n instanceof Promise ? await n : n;\n        if (v === POLL) {\n          stats.stallMs += performance.now() - t0; // timed out; re-check waiter\n          continue;\n        }\n        if (n instanceof Promise) {\n          stats.stallMs += performance.now() - t0;\n          stats.stalls++;\n        }\n        return v;\n      }\n    };\n\n    try {\n      for (;;) {\n        const item = await nextItem();\n        if (item === DONE) {\n          break;\n        }\n        // Allow the producer to fetch the next message concurrently with the\n        // processing (and downstream push) of this one.\n        credits.enqueue(1);\n\n        if (this.#canceled) {\n          // Returning finalizes the backfill stream (and the upstream resources\n          // it holds) via the producer's `finally`. The reservation, if held,\n          // does not need to be released since the change stream is gone.\n          lc.info?.(`backfill stream canceled: change stream canceled`);\n          return;\n        }\n        const {message: msg, byteSize} = item;\n\n        // Before sending any backfill message, the main replication stream\n        // must reach the backfill (snapshot) watermark. Otherwise, snapshot\n        // rows can constitute \"future\" state (e.g. phantom rows at keys that\n        // the stream has yet to move or insert) that the subsequent\n        // replication changes conflict with (e.g. on unique indexes).\n        const waitForChangeStream = this.#changeStreamReached(\n          lc,\n          msg.watermark,\n        );\n\n        // Commit (and later reopen) the transaction if the main stream is\n        // waiting on the reservation, if we must wait for the main stream, or\n        // if the size of current transaction has reached the commit threshold.\n        if (\n          backfillTx &&\n          (changeStream.waiterDelay() > 0 ||\n            waitForChangeStream ||\n            uncommittedBytes >= this.#commitThresholdBytes)\n        ) {\n          commitTx();\n        }\n\n        if (waitForChangeStream) {\n          const t0 = performance.now();\n          // Exit promptly (via the error path) if the backfill is canceled\n          // while waiting, releasing its upstream resources.\n          await promiseOrAbort(waitForChangeStream, state.stopped.signal);\n          const elapsed = performance.now() - t0;\n          stats.waitMs += elapsed;\n          lc.info?.(\n            `change stream reached ${msg.watermark} ` +\n              `after waiting ${Math.round(elapsed)} ms`,\n          );\n        }\n\n        if (\n          msg.tag === 'backfill' &&\n          msg.rowValues.length > 0 &&\n          msg.relation.rowKey.columns.length === 0\n        ) {\n          throw new MissingRowKeyError(state.request);\n        }\n\n        // Reserve the changeStreamer if not in a transaction.\n        if (backfillTx === null) {\n          const t0 = performance.now();\n          backfillTx = await beginTxFor(msg);\n          stats.reserveMs += performance.now() - t0;\n          if (backfillTx === null) {\n            lc.info?.(\n              `backfill stream canceled: ${state.stopped.signal.reason}`,\n              state.request,\n            );\n            this.#checkAndStartBackfill(); // start the next backfill if present\n            return; // this backfill is canceled\n          }\n        }\n\n        // `await` to allow the change streamer to exert back pressure\n        // on backfills.\n        const t0 = performance.now();\n        await changeStream.push(['data', msg]);\n        stats.pushMs += performance.now() - t0;\n        uncommittedBytes += byteSize;\n        stats.messages++;\n        stats.bytes += byteSize;\n\n        this.#maybeLogBackfillStats(lc, stats);\n      }\n\n      // Flush any final tx and release the stream.\n      if (backfillTx) {\n        commitTx();\n      }\n      lc.debug?.(`backfill stream exited`);\n    } finally {\n      stopProducer = true;\n      credits.enqueue(1);\n      // The producer can potentially be hung on upstream I/O.\n      // TODO: Thread AbortSignal logic into the backfillStreamer.\n      if (!this.#canceled) {\n        await producer;\n      }\n      this.#controller.signal.removeEventListener('abort', onCancel);\n      this.#logBackfillStats(lc, stats);\n    }\n  }\n\n  #maybeLogBackfillStats(lc: LogContext, stats: BackfillTimingStats) {\n    const now = performance.now();\n    if (now - stats.lastLogged >= BACKFILL_STATS_LOG_INTERVAL_MS) {\n      this.#logBackfillStats(lc, stats);\n      stats.lastLogged = now;\n    }\n  }\n\n  /**\n   * Logs where the backfill spent its time. `produceMs` (upstream read +\n   * parse/decode) accrues on the prefetch producer concurrently with the\n   * consumer's `pushMs` (downstream backpressure), so the two can overlap and\n   * their sum can exceed `elapsedMs`. `reserveMs` is time spent (re)acquiring\n   * the change-stream reservation, and `stallMs` is time the consumer waited on\n   * an empty prefetch queue (i.e. blocked on the upstream) — a large `stallMs`\n   * indicates an upstream-bound backfill. `waitMs` is time spent waiting\n   * (without the reservation) for the change stream to reach the backfill\n   * watermark, during which the upstream snapshot is held open.\n   */\n  #logBackfillStats(lc: LogContext, stats: BackfillTimingStats) {\n    const elapsed = performance.now() - stats.start;\n    lc.info?.(`backfill timing`, {\n      messages: stats.messages,\n      bytes: stats.bytes,\n      commits: stats.commits,\n      stalls: stats.stalls,\n      elapsedMs: Math.round(elapsed),\n      produceMs: Math.round(stats.produceMs),\n      pushMs: Math.round(stats.pushMs),\n      reserveMs: Math.round(stats.reserveMs),\n      stallMs: Math.round(stats.stallMs),\n      waitMs: Math.round(stats.waitMs),\n      throughputKBps:\n        elapsed > 0 ? Math.round(stats.bytes / 1024 / (elapsed / 1000)) : 0,\n    });\n  }\n\n  #backfillRunningFor(table: Identifier): RunningBackfillState | null {\n    const state = this.#runningBackfill;\n    return state?.request.table.schema === table.schema &&\n      state.request.table.name === table.name\n      ? state\n      : null;\n  }\n\n  /**\n   * Stops the running backfill for the specified `reason`. If `instance` is\n   * specified, the running backfill is stopped only if it is that instance.\n   * This allows the running backfill itself to clear backfill state without\n   * accidentally stopping a different (e.g. subsequent) backfill.\n   */\n  #stopRunningBackfill(reason?: string, instance?: RunningBackfillState) {\n    const backfill = this.#runningBackfill;\n    if (backfill && backfill === (instance ?? backfill)) {\n      backfill.stopped.abort(reason);\n      this.#runningBackfill = null;\n      reason && this.#lc.info?.(`canceling backfill:`, reason);\n    }\n  }\n\n  #setRequiredBackfill(source: string, req: BackfillRequest) {\n    const action = this.#requiredBackfills.has(req.table) ? 'updated' : 'added';\n    this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n    this.#requiredBackfills.set(req.table, req);\n  }\n\n  #deleteRequiredBackfill(source: string, id: Identifier) {\n    const req = this.#requiredBackfills.get(id);\n    if (req) {\n      const action = source === 'backfill-completed' ? 'completed' : 'dropped';\n      this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n      this.#requiredBackfills.delete(id);\n    }\n  }\n\n  /**\n   * Implements {@link Listener.onChange()}, invoked by the\n   * {@link ChangeStreamMultiplexer}.\n   */\n  onChange(message: ChangeStreamMessage): void {\n    if (message[0] === 'begin') {\n      this.#currentTxWatermark = message[2].commitWatermark;\n      return;\n    }\n    if (message[0] === 'commit') {\n      this.#currentTxWatermark = null;\n      this.#setLastStatusWatermark(message[2]);\n      // Every commit is a candidate for starting the next backfill\n      // (if one is not currently running).\n      this.#checkAndStartBackfill();\n      return;\n    }\n    if (message[0] === 'status') {\n      this.#setLastStatusWatermark(message[2]);\n      return;\n    }\n    if (message[0] !== 'data') {\n      return;\n    }\n    const change = message[1];\n    const {tag} = change;\n    switch (tag) {\n      case 'update-table-metadata': {\n        const {table, new: metadata} = change;\n        const backfillRequest = this.#requiredBackfills.get(table);\n        if (backfillRequest) {\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            table: {...backfillRequest.table, metadata},\n          });\n          if (this.#backfillRunningFor(table)) {\n            this.#stopRunningBackfill(`TableMetadata updated`);\n          }\n        }\n        break;\n      }\n      case 'create-table': {\n        const {\n          spec: {schema, name},\n          metadata = null,\n          backfill,\n        } = change;\n\n        if (backfill) {\n          this.#setRequiredBackfill(tag, {\n            table: {schema, name, metadata},\n            columns: Object.fromEntries(\n              Object.entries(backfill).map(([col, id]) => [col, {id}]),\n            ),\n          });\n        }\n        break;\n      }\n      case 'rename-table': {\n        const {old, new: newTable} = change;\n        const backfillRequest = this.#requiredBackfills.get(old);\n        if (backfillRequest) {\n          const {schema, name} = newTable;\n          this.#deleteRequiredBackfill(tag, old);\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            table: {...backfillRequest.table, schema, name},\n          });\n          if (this.#backfillRunningFor(old)) {\n            this.#stopRunningBackfill(`table renamed`);\n          }\n        }\n        break;\n      }\n      case 'drop-table': {\n        const {id} = change;\n        const backfillRequest = this.#requiredBackfills.get(id);\n        if (backfillRequest) {\n          this.#deleteRequiredBackfill(tag, id);\n          if (this.#backfillRunningFor(id)) {\n            this.#stopRunningBackfill(`table dropped`);\n          }\n        }\n        break;\n      }\n      case 'add-column': {\n        const {\n          table,\n          tableMetadata: metadata = null,\n          column,\n          backfill,\n        } = change;\n        if (backfill) {\n          const backfillRequest = this.#requiredBackfills.get(table);\n          if (!backfillRequest) {\n            this.#setRequiredBackfill(tag, {\n              table: {...table, metadata},\n              columns: {[column.name]: {id: backfill}},\n            });\n          } else {\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              table: {...backfillRequest.table, metadata},\n              columns: {\n                ...backfillRequest.columns,\n                [column.name]: {id: backfill},\n              },\n            });\n            // Note: The running backfill need not be canceled if a\n            //   new column is added. The new column will be backfilled\n            //   by its own stream after the current backfill completes.\n          }\n        }\n        break;\n      }\n      case 'update-column': {\n        const {\n          table,\n          old: {name: oldName},\n          new: {name: newName},\n        } = change;\n        if (oldName !== newName) {\n          const backfillRequest = this.#requiredBackfills.get(table);\n          if (backfillRequest && oldName in backfillRequest.columns) {\n            const {[oldName]: colSpec, ...otherCols} = backfillRequest.columns;\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              columns: {...otherCols, [newName]: colSpec},\n            });\n            const backfill = this.#backfillRunningFor(table);\n            if (backfill && oldName in backfill.request.columns) {\n              this.#stopRunningBackfill(`column renamed`);\n            }\n          }\n        }\n        break;\n      }\n      case 'drop-column': {\n        const {table, column} = change;\n        const backfillRequest = this.#requiredBackfills.get(table);\n        if (backfillRequest && column in backfillRequest.columns) {\n          const {[column]: _excluded, ...remaining} = backfillRequest.columns;\n          if (Object.keys(remaining).length === 0) {\n            this.#deleteRequiredBackfill(tag, table);\n          } else {\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              columns: remaining,\n            });\n          }\n          const backfill = this.#backfillRunningFor(table);\n          if (backfill && column in backfill.request.columns) {\n            this.#stopRunningBackfill(`column dropped`);\n          }\n        }\n        break;\n      }\n      case 'update': {\n        const {relation, key, new: row} = change;\n        const backfill = this.#backfillRunningFor(relation);\n        const txWatermark = must(this.#currentTxWatermark, `not in a tx`);\n        if (backfill?.request.table.metadata && key !== null) {\n          // A corner case that backfill is unable to correctly handle is\n          // when a row's key changes; this is decomposed into a delete\n          // of the old key and a set of the new key in the replica change\n          // log, at which point the backfill algorithm assumes that the\n          // (old) row is deleted but does not know to backfill the new row.\n          // In these corner cases, the current backfill is canceled and\n          // retried if its version precedes this update.\n          for (const col of Object.keys(\n            backfill.request.table.metadata.rowKey,\n          )) {\n            if (key[col] !== row[col]) {\n              backfill.minWatermark = txWatermark;\n              this.#lc.info?.(\n                `key for row as changed (col: ${col}). ` +\n                  `backfill data must not predate ${backfill.minWatermark}`,\n              );\n              break;\n            }\n          }\n        }\n        break;\n      }\n      case 'backfill-completed': {\n        const {relation, columns} = change;\n        const backfillRequest = this.#requiredBackfills.get(relation);\n        assert(\n          backfillRequest,\n          () => `No BackfillRequest completed backfill ${stringify(change)}`,\n        );\n        const remaining = Object.entries(backfillRequest.columns).filter(\n          ([col]) =>\n            !(columns.includes(col) || relation.rowKey.columns.includes(col)),\n        );\n        if (remaining.length === 0) {\n          this.#deleteRequiredBackfill(tag, relation);\n        } else {\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            columns: Object.fromEntries(remaining),\n          });\n        }\n        // Technically the backfill is already stopping, but this method\n        // cleans up the state that tracks it.\n        this.#stopRunningBackfill();\n        break;\n      }\n    }\n  }\n\n  cancel(): void {\n    this.#controller.abort('canceled');\n    this.#stopRunningBackfill(`change stream canceled`);\n    clearTimeout(this.#backfillRetryTimer);\n    this.#backfillRetryTimer = undefined;\n\n    // Wake up a backfill that is waiting for the change stream to reach a\n    // watermark. The change stream never will, and the backfill must be\n    // allowed to unwind (and release its upstream resources) rather than\n    // remain suspended forever.\n    for (const {reached} of this.#awaitingStatusWatermarks.splice(0)) {\n      reached();\n    }\n  }\n}\n\nabstract class BackfillStreamError extends Error {\n  constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n    super(\n      `Cannot backfill ${bf.table.schema}.${bf.table.name}` +\n        `[${Object.keys(bf.columns).join(',')}]: ${msg}`,\n      {cause},\n    );\n  }\n}\n\n/**\n * Background: The zero-cache supports replication of tables without a\n * PRIMARY KEY to facilitate the onboarding process. These rows can be\n * INSERT'ed, but postgres will rightfully prohibit UPDATEs and DELETEs\n * on such tables because the rows cannot be identified by a key. Supporting\n * this mode of replication allows the user to \"fix\" the setup by adding the\n * primary key, after which the table can be published downstream without\n * requiring a resync of the data.\n *\n * In terms of backfill, however, non-empty tables without a row key **cannot**\n * be backfilled, because backfill retries would result in writing duplicating\n * rows. (Empty tables, on the other hand, are fine because there is no data\n * to be deduped.)\n *\n * The MissingRowKeyError is used to signal that the table cannot be backfilled\n * in its current state. For simplicity, it is handled like runtime errors and\n * retried with backoff, with which it can eventually succeed if (1) a primary\n * key is added or (2) the table is emptied, e.g. via a TRUNCATE.\n */\nclass MissingRowKeyError extends BackfillStreamError {\n  readonly name = 'MissingRowKeyError';\n\n  constructor(bf: BackfillRequest, cause?: unknown) {\n    super(bf, `\"${bf.table.name}\" is missing a PRIMARY KEY`, cause);\n  }\n}\n\n/**\n * Error type for backfill stream implementations to throw indicating that\n * the backfill request failed due to a schema incompatibility error. This\n * type of error does not need exponential backoff, as the retry happens\n * naturally once the invalidating schema change is processed and committed.\n */\nexport class SchemaIncompatibilityError extends BackfillStreamError {\n  readonly name = 'SchemaIncompatibilityError';\n\n  constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n    super(bf, msg, cause);\n  }\n}\n"],"mappings":";;;;;;;;;;;;AA4BA,SAAS,SAAS,EAAC,QAAQ,QAAmB;CAC5C,OAAO,GAAG,OAAO,GAAG;AACtB;;;;;;AAsBA,IAAM,yBAAyB;;;;;;;;AAS/B,IAAM,4BAA4B;;;;;;;;;;;;;;AAelC,IAAM,0BAA0B;;AAGhC,IAAM,iCAAiC;;AAGvC,IAAM,OAAO,OAAO,eAAe;;AAEnC,IAAM,OAAO,OAAO,eAAe;AAmCnC,IAAM,0BAA0B;AAChC,IAAM,0BAA0B;;;;;;;;;;;;;;;;;AAuBhC,IAAa,kBAAb,MAA6D;CAC3D;;;;;CAMA,qBAA8B,IAAI,aAChC,QACF;CACA;CACA;CACA;;;;;;;CAQA,mBAAgD;;CAGhD,uBAAsC;CAEtC,4BAAgE,CAAC;;CAGjE,sBAAqC;CAErC;;;;;;;CAQA,cAAuB,IAAI,gBAAgB;CAE3C,IAAI,YAAY;EACd,OAAO,KAAK,YAAY,OAAO;CACjC;CAEA,YACE,IACA,gBACA,kBACA,aAAA,KACA,eAAe,yBACf,eAAe,yBACf,uBAAuB,wBACvB;EACA,KAAK,MAAM,GAAG,YAAY,aAAa,kBAAkB;EACzD,KAAK,kBAAkB;EACvB,KAAK,oBAAoB;EACzB,KAAK,cAAc;EACnB,KAAK,gBAAgB;EACrB,KAAK,gBAAgB;EACrB,KAAK,gBAAgB;EACrB,KAAK,wBAAwB;CAC/B;CAEA,IAAI,eAAuB,iBAAoC;EAC7D,KAAK,IAAI,OACP,kCAAkC,gBAAgB,OAAO,oBACzD,EAAC,UAAU,gBAAe,CAC5B;EACA,KAAK,uBAAuB;EAC5B,gBAAgB,SAAQ,QACtB,KAAK,qBAAqB,mBAAmB,GAAG,CAClD;EACA,KAAK,uBAAuB;CAC9B;CAEA,wBAAwB,EAAC,aAAiC;EAIxD,KAAK,KAAK,wBAAwB,MAAM,WAAW;GACjD,KAAK,uBAAuB;GAC5B,KAAK,IAAI,IAAI,KAAK,0BAA0B,SAAS,GAAG,KAAK,GAAG,KAAK;IACnE,MAAM,WAAW,KAAK,0BAA0B;IAChD,IAAI,aAAa,SAAS,WAAW;KACnC,SAAS,QAAQ;KACjB,KAAK,0BAA0B,OAAO,GAAG,CAAC;IAC5C;GACF;EACF;CACF;CAEA,qBACE,IACA,WACsB;EACtB,KAAK,KAAK,wBAAwB,MAAM,WAAW;GACjD,MAAM,EAAC,SAAS,SAAS,YAAW,SAAS;GAC7C,KAAK,0BAA0B,KAAK;IAAC;IAAW;GAAO,CAAC;GACxD,GAAG,OACD,iCAAiC,KAAK,qBAAqB,aAAa,WAC1E;GACA,OAAO;EACT;EACA,OAAO;CACT;CAEA;CACA;CACA;CACA;CAEA,yBAAyB;EACvB,IACE,CAAC,KAAK,aACN,CAAC,KAAK,uBACN,CAAC,KAAK,oBACN,KAAK,mBAAmB,MACxB;GAKA,MAAM,aAAa,CAAC,GAAG,KAAK,mBAAmB,OAAO,CAAC;GACvD,MAAM,UAAU,WAAW,QAAQ,GAAG,WAAW,SAAS,CAAC;GAC3D,MAAM,QAAQ;IACZ;IACA,cAAc;IACd,SAAS,IAAI,gBAAgB;GAC/B;GACA,MAAM,KAAK,KAAK,IAAI,YAAY,SAAS,QAAQ,MAAM,IAAI;GAE3D,KAAK,mBAAmB;GACxB,KAAU,aAAa,IAAI,KAAK,CAAC,CAC9B,WAAW;IACV,KAAK,qBAAqB,mBAAmB,KAAK;IAClD,KAAK,gBAAgB,KAAK;GAC5B,CAAC,CAAC,CAGD,OAAM,MAAK;IACV,KAAK,qBAAqB,OAAO,CAAC,GAAG,KAAK;IAC1C,KAAK,0BAA0B,CAAC;GAClC,CAAC;EACL;CACF;CAEA,0BAA0B,GAAY;EACpC,IAAI,KAAK,WAAW;GAClB,KAAK,IAAI,QAAQ,iDAAiD,CAAC;GACnE;EACF;EACA,KAAK,IAAI,OACP,uCAAuC,KAAK,cAAc,MAC1D,CACF;EACA,KAAK,sBAAsB,iBAAiB;GAC1C,KAAK,sBAAsB,KAAA;GAC3B,KAAK,uBAAuB;EAC9B,GAAG,KAAK,aAAa;EAErB,KAAK,gBAAgB,KAAK,IAAI,KAAK,gBAAgB,GAAG,KAAK,aAAa;CAC1E;CAEA,MAAM,aAAa,IAAgB,OAA6B;EAC9D,MAAM,eAAe,KAAK;EAI1B,IAAI,aAA4B;EAChC,IAAI,mBAAmB;EAEvB,MAAM,QAA6B;GACjC,OAAO,YAAY,IAAI;GACvB,YAAY,YAAY,IAAI;GAC5B,UAAU;GACV,OAAO;GACP,SAAS;GACT,QAAQ;GACR,WAAW;GACX,QAAQ;GACR,WAAW;GACX,SAAS;GACT,QAAQ;EACV;;;;EAKA,MAAM,aAAa,OACjB,QAC2B;GAC3B,OAAO,eAAe,MAAM,yCAAyC;GACrE,MAAM,gBAAgB,MAAM,aAAa,QAAQ,UAAU;GAI3D,IACE,MAAM,QAAQ,OAAO,WACpB,IAAI,QAAQ,cAAc,IAAI,YAAY,MAAM,cACjD;IACA,IAAI,CAAC,MAAM,QAAQ,OAAO,SAAS;KACjC,OAAO,IAAI,QAAQ,YAAY,+BAA+B;KAC9D,KAAK,qBACH,qBAAqB,MAAM,aAAa,mCACH,IAAI,aACzC,KACF;IACF;IACA,aAAa,QAAQ,aAAa;IAClC,OAAO;GACT;GAEA,MAAM,EAAC,OAAO,QAAQ,OAAM,uBAAuB,aAAa;GAChE,IAAI,KAAK,qBAAqB;IAC5B;IACA,OAAO,OAAO,KAAK,IAAI;GACzB,CAAC;GAED,IAAI,KAAK,IAAI,WAOX,KAAK,IAAI;GAGX,aAAkB,KAAK;IACrB;IACA;KAAC,KAAK;KAAS,MAAM,KAAK;KAAa,SAAS;IAAI;IACpD,EAAC,iBAAiB,GAAE;GACtB,CAAC;GACD,OAAQ,aAAa;EACvB;EAEA,MAAM,iBAAiB;GACrB,IAAI,YAAY;IACd,aAAkB,KAAK;KACrB;KACA,EAAC,KAAK,SAAQ;KACd,EAAC,WAAW,WAAU;IACxB,CAAC;IACD,aAAa,QAAQ,UAAU;IAC/B,MAAM;GACR;GACA,aAAa;GACb,mBAAmB;EACrB;EASA,MAAM,QAAQ,IAAI,MAAY;EAC9B,MAAM,UAAU,IAAI,MAAS;EAC7B,KAAK,IAAI,IAAI,GAAG,IAAI,2BAA2B,KAC7C,QAAQ,QAAQ,CAAC;EAEnB,IAAI,eAAe;EAInB,MAAM,iBACJ,MAAM,iBAAiB,IAAI,WAAW,mBAAmB,CAAC;EAC5D,IAAI,CAAC,KAAK,WACR,KAAK,YAAY,OAAO,iBAAiB,SAAS,QAAQ;EAG5D,MAAM,YAAY,YAAY;GAC5B,IAAI;IACF,IAAI,KAAK,YAAY,IAAI;IACzB,WAAW,MAAM,SAAS,KAAK,kBAAkB,MAAM,OAAO,GAAG;KAC/D,MAAM,aAAa,YAAY,IAAI,IAAI;KACvC,MAAM,QAAQ,QAAQ;KACtB,IAAI,gBAAgB,KAAK,WAAW;MAClC,MAAM,iBAAiB,IAAI,WAAW,mBAAmB,CAAC;MAC1D;KACF;KACA,MAAM,QAAQ,KAAK;KACnB,KAAK,YAAY,IAAI;IACvB;IACA,MAAM,QAAQ,IAAI;GACpB,SAAS,GAAG;IACV,MAAM,iBAAiB,CAAC;GAC1B;EACF,EAAA,CAAG;EAIH,SAAc,YAAY,CAAC,CAAC;EAS5B,MAAM,WAAW,YAAoD;GAKnE,IAAI,eAAe,QAAQ,MAAM,KAAK,IAAI,GAAG;IAC3C,MAAM,IAAI,MAAM,QAAQ;IAExB,OADU,aAAa,UAAU,MAAM,IAAI;GAE7C;GAEA,SAAS;IACP,IAAI,eAAe,QAAQ,aAAa,YAAY,IAAI,GACtD,SAAS;IAEX,MAAM,IACJ,eAAe,OACX,MAAM,QAAQ,MAAM,uBAAuB,IAC3C,MAAM,QAAQ;IACpB,MAAM,KAAK,YAAY,IAAI;IAC3B,MAAM,IAAI,aAAa,UAAU,MAAM,IAAI;IAC3C,IAAI,MAAM,MAAM;KACd,MAAM,WAAW,YAAY,IAAI,IAAI;KACrC;IACF;IACA,IAAI,aAAa,SAAS;KACxB,MAAM,WAAW,YAAY,IAAI,IAAI;KACrC,MAAM;IACR;IACA,OAAO;GACT;EACF;EAEA,IAAI;GACF,SAAS;IACP,MAAM,OAAO,MAAM,SAAS;IAC5B,IAAI,SAAS,MACX;IAIF,QAAQ,QAAQ,CAAC;IAEjB,IAAI,KAAK,WAAW;KAIlB,GAAG,OAAO,kDAAkD;KAC5D;IACF;IACA,MAAM,EAAC,SAAS,KAAK,aAAY;IAOjC,MAAM,sBAAsB,KAAK,qBAC/B,IACA,IAAI,SACN;IAKA,IACE,eACC,aAAa,YAAY,IAAI,KAC5B,uBACA,oBAAoB,KAAK,wBAE3B,SAAS;IAGX,IAAI,qBAAqB;KACvB,MAAM,KAAK,YAAY,IAAI;KAG3B,MAAM,eAAe,qBAAqB,MAAM,QAAQ,MAAM;KAC9D,MAAM,UAAU,YAAY,IAAI,IAAI;KACpC,MAAM,UAAU;KAChB,GAAG,OACD,yBAAyB,IAAI,UAAU,iBACpB,KAAK,MAAM,OAAO,EAAE,IACzC;IACF;IAEA,IACE,IAAI,QAAQ,cACZ,IAAI,UAAU,SAAS,KACvB,IAAI,SAAS,OAAO,QAAQ,WAAW,GAEvC,MAAM,IAAI,mBAAmB,MAAM,OAAO;IAI5C,IAAI,eAAe,MAAM;KACvB,MAAM,KAAK,YAAY,IAAI;KAC3B,aAAa,MAAM,WAAW,GAAG;KACjC,MAAM,aAAa,YAAY,IAAI,IAAI;KACvC,IAAI,eAAe,MAAM;MACvB,GAAG,OACD,6BAA6B,MAAM,QAAQ,OAAO,UAClD,MAAM,OACR;MACA,KAAK,uBAAuB;MAC5B;KACF;IACF;IAIA,MAAM,KAAK,YAAY,IAAI;IAC3B,MAAM,aAAa,KAAK,CAAC,QAAQ,GAAG,CAAC;IACrC,MAAM,UAAU,YAAY,IAAI,IAAI;IACpC,oBAAoB;IACpB,MAAM;IACN,MAAM,SAAS;IAEf,KAAK,uBAAuB,IAAI,KAAK;GACvC;GAGA,IAAI,YACF,SAAS;GAEX,GAAG,QAAQ,wBAAwB;EACrC,UAAU;GACR,eAAe;GACf,QAAQ,QAAQ,CAAC;GAGjB,IAAI,CAAC,KAAK,WACR,MAAM;GAER,KAAK,YAAY,OAAO,oBAAoB,SAAS,QAAQ;GAC7D,KAAK,kBAAkB,IAAI,KAAK;EAClC;CACF;CAEA,uBAAuB,IAAgB,OAA4B;EACjE,MAAM,MAAM,YAAY,IAAI;EAC5B,IAAI,MAAM,MAAM,cAAc,gCAAgC;GAC5D,KAAK,kBAAkB,IAAI,KAAK;GAChC,MAAM,aAAa;EACrB;CACF;;;;;;;;;;;;CAaA,kBAAkB,IAAgB,OAA4B;EAC5D,MAAM,UAAU,YAAY,IAAI,IAAI,MAAM;EAC1C,GAAG,OAAO,mBAAmB;GAC3B,UAAU,MAAM;GAChB,OAAO,MAAM;GACb,SAAS,MAAM;GACf,QAAQ,MAAM;GACd,WAAW,KAAK,MAAM,OAAO;GAC7B,WAAW,KAAK,MAAM,MAAM,SAAS;GACrC,QAAQ,KAAK,MAAM,MAAM,MAAM;GAC/B,WAAW,KAAK,MAAM,MAAM,SAAS;GACrC,SAAS,KAAK,MAAM,MAAM,OAAO;GACjC,QAAQ,KAAK,MAAM,MAAM,MAAM;GAC/B,gBACE,UAAU,IAAI,KAAK,MAAM,MAAM,QAAQ,QAAQ,UAAU,IAAK,IAAI;EACtE,CAAC;CACH;CAEA,oBAAoB,OAAgD;EAClE,MAAM,QAAQ,KAAK;EACnB,OAAO,OAAO,QAAQ,MAAM,WAAW,MAAM,UAC3C,MAAM,QAAQ,MAAM,SAAS,MAAM,OACjC,QACA;CACN;;;;;;;CAQA,qBAAqB,QAAiB,UAAiC;EACrE,MAAM,WAAW,KAAK;EACtB,IAAI,YAAY,cAAc,YAAY,WAAW;GACnD,SAAS,QAAQ,MAAM,MAAM;GAC7B,KAAK,mBAAmB;GACxB,UAAU,KAAK,IAAI,OAAO,uBAAuB,MAAM;EACzD;CACF;CAEA,qBAAqB,QAAgB,KAAsB;EACzD,MAAM,SAAS,KAAK,mBAAmB,IAAI,IAAI,KAAK,IAAI,YAAY;EACpE,KAAK,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,IAAG,CAAC;EAChE,KAAK,mBAAmB,IAAI,IAAI,OAAO,GAAG;CAC5C;CAEA,wBAAwB,QAAgB,IAAgB;EACtD,MAAM,MAAM,KAAK,mBAAmB,IAAI,EAAE;EAC1C,IAAI,KAAK;GACP,MAAM,SAAS,WAAW,uBAAuB,cAAc;GAC/D,KAAK,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,IAAG,CAAC;GAChE,KAAK,mBAAmB,OAAO,EAAE;EACnC;CACF;;;;;CAMA,SAAS,SAAoC;EAC3C,IAAI,QAAQ,OAAO,SAAS;GAC1B,KAAK,sBAAsB,QAAQ,EAAE,CAAC;GACtC;EACF;EACA,IAAI,QAAQ,OAAO,UAAU;GAC3B,KAAK,sBAAsB;GAC3B,KAAK,wBAAwB,QAAQ,EAAE;GAGvC,KAAK,uBAAuB;GAC5B;EACF;EACA,IAAI,QAAQ,OAAO,UAAU;GAC3B,KAAK,wBAAwB,QAAQ,EAAE;GACvC;EACF;EACA,IAAI,QAAQ,OAAO,QACjB;EAEF,MAAM,SAAS,QAAQ;EACvB,MAAM,EAAC,QAAO;EACd,QAAQ,KAAR;GACE,KAAK,yBAAyB;IAC5B,MAAM,EAAC,OAAO,KAAK,aAAY;IAC/B,MAAM,kBAAkB,KAAK,mBAAmB,IAAI,KAAK;IACzD,IAAI,iBAAiB;KACnB,KAAK,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;MAAQ;KAC5C,CAAC;KACD,IAAI,KAAK,oBAAoB,KAAK,GAChC,KAAK,qBAAqB,uBAAuB;IAErD;IACA;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,EACJ,MAAM,EAAC,QAAQ,QACf,WAAW,MACX,aACE;IAEJ,IAAI,UACF,KAAK,qBAAqB,KAAK;KAC7B,OAAO;MAAC;MAAQ;MAAM;KAAQ;KAC9B,SAAS,OAAO,YACd,OAAO,QAAQ,QAAQ,CAAC,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAC,KAAK,EAAC,GAAE,CAAC,CAAC,CACzD;IACF,CAAC;IAEH;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,EAAC,KAAK,KAAK,aAAY;IAC7B,MAAM,kBAAkB,KAAK,mBAAmB,IAAI,GAAG;IACvD,IAAI,iBAAiB;KACnB,MAAM,EAAC,QAAQ,SAAQ;KACvB,KAAK,wBAAwB,KAAK,GAAG;KACrC,KAAK,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;OAAQ;MAAI;KAChD,CAAC;KACD,IAAI,KAAK,oBAAoB,GAAG,GAC9B,KAAK,qBAAqB,eAAe;IAE7C;IACA;GACF;GACA,KAAK,cAAc;IACjB,MAAM,EAAC,OAAM;IAEb,IADwB,KAAK,mBAAmB,IAAI,EAChD,GAAiB;KACnB,KAAK,wBAAwB,KAAK,EAAE;KACpC,IAAI,KAAK,oBAAoB,EAAE,GAC7B,KAAK,qBAAqB,eAAe;IAE7C;IACA;GACF;GACA,KAAK,cAAc;IACjB,MAAM,EACJ,OACA,eAAe,WAAW,MAC1B,QACA,aACE;IACJ,IAAI,UAAU;KACZ,MAAM,kBAAkB,KAAK,mBAAmB,IAAI,KAAK;KACzD,IAAI,CAAC,iBACH,KAAK,qBAAqB,KAAK;MAC7B,OAAO;OAAC,GAAG;OAAO;MAAQ;MAC1B,SAAS,GAAE,OAAO,OAAO,EAAC,IAAI,SAAQ,EAAC;KACzC,CAAC;UAED,KAAK,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;MAAQ;MAC1C,SAAS;OACP,GAAG,gBAAgB;QAClB,OAAO,OAAO,EAAC,IAAI,SAAQ;MAC9B;KACF,CAAC;IAKL;IACA;GACF;GACA,KAAK,iBAAiB;IACpB,MAAM,EACJ,OACA,KAAK,EAAC,MAAM,WACZ,KAAK,EAAC,MAAM,cACV;IACJ,IAAI,YAAY,SAAS;KACvB,MAAM,kBAAkB,KAAK,mBAAmB,IAAI,KAAK;KACzD,IAAI,mBAAmB,WAAW,gBAAgB,SAAS;MACzD,MAAM,GAAE,UAAU,SAAS,GAAG,cAAa,gBAAgB;MAC3D,KAAK,qBAAqB,KAAK;OAC7B,GAAG;OACH,SAAS;QAAC,GAAG;SAAY,UAAU;OAAO;MAC5C,CAAC;MACD,MAAM,WAAW,KAAK,oBAAoB,KAAK;MAC/C,IAAI,YAAY,WAAW,SAAS,QAAQ,SAC1C,KAAK,qBAAqB,gBAAgB;KAE9C;IACF;IACA;GACF;GACA,KAAK,eAAe;IAClB,MAAM,EAAC,OAAO,WAAU;IACxB,MAAM,kBAAkB,KAAK,mBAAmB,IAAI,KAAK;IACzD,IAAI,mBAAmB,UAAU,gBAAgB,SAAS;KACxD,MAAM,GAAE,SAAS,WAAW,GAAG,cAAa,gBAAgB;KAC5D,IAAI,OAAO,KAAK,SAAS,CAAC,CAAC,WAAW,GACpC,KAAK,wBAAwB,KAAK,KAAK;UAEvC,KAAK,qBAAqB,KAAK;MAC7B,GAAG;MACH,SAAS;KACX,CAAC;KAEH,MAAM,WAAW,KAAK,oBAAoB,KAAK;KAC/C,IAAI,YAAY,UAAU,SAAS,QAAQ,SACzC,KAAK,qBAAqB,gBAAgB;IAE9C;IACA;GACF;GACA,KAAK,UAAU;IACb,MAAM,EAAC,UAAU,KAAK,KAAK,QAAO;IAClC,MAAM,WAAW,KAAK,oBAAoB,QAAQ;IAClD,MAAM,cAAc,KAAK,KAAK,qBAAqB,aAAa;IAChE,IAAI,UAAU,QAAQ,MAAM,YAAY,QAAQ,MAQzC;UAAA,MAAM,OAAO,OAAO,KACvB,SAAS,QAAQ,MAAM,SAAS,MAClC,GACE,IAAI,IAAI,SAAS,IAAI,MAAM;MACzB,SAAS,eAAe;MACxB,KAAK,IAAI,OACP,gCAAgC,IAAI,oCACA,SAAS,cAC/C;MACA;KACF;;IAGJ;GACF;GACA,KAAK,sBAAsB;IACzB,MAAM,EAAC,UAAU,YAAW;IAC5B,MAAM,kBAAkB,KAAK,mBAAmB,IAAI,QAAQ;IAC5D,OACE,uBACM,yCAAyC,UAAU,MAAM,GACjE;IACA,MAAM,YAAY,OAAO,QAAQ,gBAAgB,OAAO,CAAC,CAAC,QACvD,CAAC,SACA,EAAE,QAAQ,SAAS,GAAG,KAAK,SAAS,OAAO,QAAQ,SAAS,GAAG,EACnE;IACA,IAAI,UAAU,WAAW,GACvB,KAAK,wBAAwB,KAAK,QAAQ;SAE1C,KAAK,qBAAqB,KAAK;KAC7B,GAAG;KACH,SAAS,OAAO,YAAY,SAAS;IACvC,CAAC;IAIH,KAAK,qBAAqB;IAC1B;GACF;EACF;CACF;CAEA,SAAe;EACb,KAAK,YAAY,MAAM,UAAU;EACjC,KAAK,qBAAqB,wBAAwB;EAClD,aAAa,KAAK,mBAAmB;EACrC,KAAK,sBAAsB,KAAA;EAM3B,KAAK,MAAM,EAAC,aAAY,KAAK,0BAA0B,OAAO,CAAC,GAC7D,QAAQ;CAEZ;AACF;AAEA,IAAe,sBAAf,cAA2C,MAAM;CAC/C,YAAY,IAAqB,KAAa,OAAiB;EAC7D,MACE,mBAAmB,GAAG,MAAM,OAAO,GAAG,GAAG,MAAM,KAAA,GACzC,OAAO,KAAK,GAAG,OAAO,CAAC,CAAC,KAAK,GAAG,EAAE,KAAK,OAC7C,EAAC,MAAK,CACR;CACF;AACF;;;;;;;;;;;;;;;;;;;;AAqBA,IAAM,qBAAN,cAAiC,oBAAoB;CACnD,OAAgB;CAEhB,YAAY,IAAqB,OAAiB;EAChD,MAAM,IAAI,IAAI,GAAG,MAAM,KAAK,6BAA6B,KAAK;CAChE;AACF;;;;;;;AAQA,IAAa,6BAAb,cAAgD,oBAAoB;CAClE,OAAgB;CAEhB,YAAY,IAAqB,KAAa,OAAiB;EAC7D,MAAM,IAAI,KAAK,KAAK;CACtB;AACF"}