@rocicorp/zero 1.12.0-canary.0 → 1.12.0-canary.10

package/out/zero-cache/src/server/replicator.js.map

@@ -1 +1 @@
-{"version":3,"file":"replicator.js","names":[],"sources":["../../../../../zero-cache/src/server/replicator.ts"],"sourcesContent":["import {stat} from 'node:fs/promises';\nimport {pid} from 'node:process';\nimport type {ObservableCallback} from '@opentelemetry/api';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport type {\n  LitestreamConfig,\n  NormalizedZeroConfig,\n} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {\n  ChangeStreamerHttpClient,\n  type ReservationFollowup,\n  type SnapshotReserver,\n} from '../services/change-streamer/change-streamer-http.ts';\nimport {reserveAndGetSnapshotStatus} from '../services/change-streamer/snapshot.ts';\nimport {\n  exitAfter,\n  runUntilKilled,\n  SupportingProcesses,\n} from '../services/life-cycle.ts';\nimport {\n  readLocalReplica,\n  startReplicaBackupProcess,\n  tryRestore,\n  type RestoreAttempt,\n  type RestoreResult,\n} from '../services/litestream/commands.ts';\nimport {\n  litestreamRestoreDuration,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreRuns,\n} from '../services/litestream/metrics.ts';\nimport {RestoreProgressReporter} from '../services/litestream/restore-progress.ts';\nimport {ReplicationStatusPublisher} from '../services/replicator/replication-status.ts';\nimport {\n  ReplicatorService,\n  type ReplicatorMode,\n} from '../services/replicator/replicator.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {\n  ThreadWriteWorkerClient,\n  type ForceCheckpointConfig,\n} from '../services/replicator/write-worker-client.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type DrainRequestedMessage,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {\n  deleteStaleChangeLog,\n  getPragmaConfig,\n  replicaFileModeSchema,\n  setUpMessageHandlers,\n  setupReplica,\n  type WalMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length > 0, `replicator mode not specified`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n  const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n  const mode: ReplicatorMode = fileMode === 'backup' ? 'backup' : 'serving';\n  const runningLocalChangeStreamer =\n    config.changeStreamer.mode === 'dedicated' && !config.changeStreamer.uri;\n  const workerName = `${mode}-replicator`;\n  installProfileHandler(parent, workerName);\n  startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);\n  lc = createLogContext(config, workerName);\n  const unregisterInitialCorruptionDiagnosticTarget =\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: `${workerName} replica`,\n        dbPath: config.replica.file,\n      },\n      config.sqliteCorruptionChecks,\n    );\n  initEventSink(lc, config);\n\n  const shard = getShardConfig(config);\n  const {\n    taskID,\n    change,\n    changeStreamer: {\n      port,\n      uri: changeStreamerURI = runningLocalChangeStreamer\n        ? `http://localhost:${port}/`\n        : undefined,\n    },\n    replicationLag: {initialReadinessThresholdMs, maxReadinessDelayMs},\n  } = config;\n  const changeStreamer = new ChangeStreamerHttpClient(\n    lc,\n    shard,\n    change.db,\n    changeStreamerURI,\n  );\n\n  // Remote view-syncers restore from the replication-manager, which supplies\n  // the backup URL. A local change-streamer owns the canonical replica and\n  // must not infer backup configuration from bundled executable paths.\n  //\n  // The reservation's connection (`initialConnection`) is threaded through to\n  // IncrementalSyncer's first subscribe(), so the replication-manager that\n  // reserved the change log for this restore is the one that serves the\n  // subscription (single-connection handoff).\n  let initialConnection: ReservationFollowup | undefined;\n  if (\n    fileMode === 'serving' &&\n    !runningLocalChangeStreamer &&\n    (config.litestream.executable || config.litestream.executableV5)\n  ) {\n    initialConnection = await restoreReplica(lc, config, changeStreamer);\n  }\n\n  const {\n    file: dbPath,\n    walMode,\n    pageSize,\n  } = await setupReplica(lc, fileMode, config.replica);\n  unregisterInitialCorruptionDiagnosticTarget();\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: `${workerName} replica`,\n      dbPath,\n    },\n    config.sqliteCorruptionChecks,\n  );\n\n  // A file left behind by a run with the writer enabled would otherwise hold\n  // local disk indefinitely. See `replicatorDeletesStaleChangeLog` for why\n  // this is keyed on the config flag and nothing else.\n  deleteStaleChangeLog(config.changeStreamer.sqliteChangeLogMode, dbPath);\n\n  setupMetrics(lc, dbPath, walMode, pageSize);\n\n  // Create the write worker for async SQLite writes. Enable write-path\n  // checkpoint backpressure only for the backup replicator on litestream v5,\n  // which owns the replica litestream backs up.\n  const pragmas = getPragmaConfig(fileMode);\n  const checkpoint: ForceCheckpointConfig | null =\n    mode === 'backup' &&\n    config.litestream.backupURL &&\n    config.litestream.backupUsingV5 &&\n    config.litestream.forceCheckpointThresholdMB\n      ? {\n          checkpointThresholdPages:\n            (config.litestream.forceCheckpointThresholdMB * 1024 ** 2) /\n            pageSize,\n          maxWalPages: !config.litestream.maxWalSizeMB\n            ? undefined\n            : (config.litestream.maxWalSizeMB * 1024 ** 2) / pageSize,\n        }\n      : null;\n\n  const workerClient = new ThreadWriteWorkerClient();\n  await workerClient.init(dbPath, mode, pragmas, config.log, checkpoint);\n\n  // The backup-replicator manages the litestream backup process. It is\n  // started after the write worker has prepared the db file in order to\n  // avoid contending on the sqlite lock.\n  const processes = new SupportingProcesses(lc, parent);\n  if (mode === 'backup') {\n    processes.add(\n      startReplicaBackupProcess(lc, config.litestream, dbPath),\n      'litestream',\n    );\n  }\n\n  const replicator = new ReplicatorService(\n    lc,\n    taskID,\n    `${workerName}-${pid}`,\n    mode,\n    changeStreamer,\n    workerClient,\n    runningLocalChangeStreamer\n      ? // publish ReplicationStatusEvents from backup-replicator only\n        ReplicationStatusPublisher.forReplicaFile(dbPath)\n      : null,\n    {\n      readinessThresholdMs: initialReadinessThresholdMs,\n      maxReadinessDelayMs,\n    },\n    initialConnection,\n    // View-syncers (i.e. with a remote change-streamer) switch over to\n    // another replication-manager when theirs drains.\n    runningLocalChangeStreamer\n      ? undefined\n      : {\n          onRestoreGranted: () =>\n            parent.send<DrainRequestedMessage>(['drainRequested', {}]),\n          replicaFile: dbPath,\n        },\n  );\n\n  setUpMessageHandlers(lc, replicator, parent);\n\n  void replicator.ready().then(() => {\n    parent.send(['ready', {ready: true}]);\n  });\n\n  return runUntilKilled(lc, parent, replicator, processes);\n}\n\nexport function setupMetrics(\n  lc: LogContext,\n  file: string,\n  walMode: WalMode,\n  pageSize: number,\n) {\n  let db: Database | undefined;\n  const getDb = () => {\n    if (!db) {\n      try {\n        db = new Database(lc, file, {readonly: true});\n      } catch (e) {\n        lc.warn?.(`unable to open ${file} for wal metrics`, e);\n      }\n    }\n    return db;\n  };\n\n  getOrCreateGauge('replica', 'db_size', {\n    description:\n      `The size of the replica's main db file, ` +\n      `which does not include the wal file(s)`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, file));\n\n  getOrCreateGauge('replica', 'wal_size', {\n    description: `The size of the replica's wal file`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, `${file}-wal`));\n\n  if (walMode === 'wal2') {\n    getOrCreateGauge('replica', 'wal2_size', {\n      description: `The size of the replica's wal2 file`,\n      unit: 'bytes',\n    }).addCallback(observeFileSize(lc, `${file}-wal2`));\n  }\n\n  getOrCreateGauge('replica', 'wal_active_bytes', {\n    description:\n      `The actual size of active WAL frames currently in use ` +\n      `(excluding pre-allocated or reclaimed space).`,\n    unit: 'bytes',\n  }).addCallback(observeWalActiveBytes(lc, getDb, pageSize));\n\n  getOrCreateGauge('replica', 'wal_uncheckpointed_bytes', {\n    description:\n      `The size of un-checkpointed WAL frames that have not yet ` +\n      `been backfilled into the main database.`,\n    unit: 'bytes',\n  }).addCallback(observeWalUncheckpointedBytes(lc, getDb, pageSize));\n\n  // A whole-database total rather than a per-file one. Its counterpart,\n  // `sqlite_change_log.file_bytes`, is emitted by the change-streamer, which is\n  // the process that writes the log.\n  getOrCreateGauge('replica', 'file_bytes', {\n    description:\n      `The replica's total on-disk footprint: its main db file plus its ` +\n      `wal sidecars.`,\n    unit: 'By',\n  }).addCallback(observeSQLiteFileBytes(lc, file));\n}\n\nfunction observeFileSize(lc: LogContext, file: string): ObservableCallback {\n  return async o => {\n    try {\n      const stats = await stat(file);\n      o.observe(stats.size);\n    } catch (e) {\n      lc.warn?.(`unable to stat ${file} for size metrics`, e);\n    }\n  };\n}\n\nfunction observeWalActiveBytes(\n  lc: LogContext,\n  getDb: () => Database | undefined,\n  pageSize: number,\n): ObservableCallback {\n  return o => {\n    try {\n      const db = getDb();\n      if (!db) {\n        return;\n      }\n      const [{log}] = db.pragma<{log: number}>('wal_checkpoint(NOOP)');\n      if (typeof log === 'number') {\n        o.observe(Math.max(0, log) * pageSize);\n      }\n    } catch (e) {\n      lc.warn?.('unable to inspect wal_checkpoint for active bytes metric', e);\n    }\n  };\n}\n\nfunction observeWalUncheckpointedBytes(\n  lc: LogContext,\n  getDb: () => Database | undefined,\n  pageSize: number,\n): ObservableCallback {\n  return o => {\n    try {\n      const db = getDb();\n      if (!db) {\n        return;\n      }\n      const [{log, checkpointed}] = db.pragma<{\n        log: number;\n        checkpointed: number;\n      }>('wal_checkpoint(NOOP)');\n      if (typeof log === 'number' && typeof checkpointed === 'number') {\n        o.observe(Math.max(0, log - checkpointed) * pageSize);\n      }\n    } catch (e) {\n      lc.warn?.(\n        'unable to inspect wal_checkpoint for uncheckpointed bytes metric',\n        e,\n      );\n    }\n  };\n}\n\nfunction observeSQLiteFileBytes(\n  lc: LogContext,\n  file: string,\n): ObservableCallback {\n  return async o => o.observe(await sqliteFileBytes(lc, file));\n}\n\nconst RETRY_INTERVAL_MS = 3000;\n\n// The number of times a view-syncer retries a snapshot reservation from a\n// replication-manager with an older replica version than its local replica\n// (e.g. a not-yet-terminated replication-manager during a resync rollout),\n// before accepting the older version (e.g. if the upstream was legitimately\n// replaced by one with a lower replica version).\nconst MAX_OLDER_REPLICA_VERSION_RETRIES = 20;\n\n// View-syncers wait indefinitely for the replication-manager to publish a\n// restorable backup. On a fresh stack the first backup is not durable until\n// the initial sync completes and litestream uploads the initial snapshot,\n// which can take many minutes for a large replica. The platform's startup\n// probe budget (which scales with replica size) is the backstop, so\n// restoreReplica must not impose its own shorter cap and self-terminate\n// while the backup is still being produced.\n//\n// Returns the `followup` of the reservation whose backup was actually\n// restored, still open, so the caller can hand it to IncrementalSyncer's\n// first subscribe() via `followup.subscribe()`: the same replication-manager\n// that reserved the change log is the one that serves the subscription.\n/** Exported for testing. */\nexport async function restoreReplica(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  changeStreamer: SnapshotReserver,\n): Promise<ReservationFollowup> {\n  const start = performance.now();\n  let backupURL: string | undefined;\n  let result: RestoreResult | undefined;\n  const progress = new RestoreProgressReporter(lc, config.replica.file);\n  let olderReplicaVersionRetries = 0;\n  // Describing the local replica (if any) allows the replication-manager to\n  // account for replicas that are being replaced (see RestoreAdmission). It\n  // is read once, before the first attempt, as a failed attempt may have\n  // already deleted it; it continues to describe the replica being replaced\n  // (and the version not to downgrade from) on every retry.\n  const localReplica = readLocalReplica(lc, config.replica.file);\n  const localReplicaVersion = localReplica?.replicaVersion;\n  try {\n    for (;;) {\n      const {reserved, followup} = await reserveAndGetSnapshotStatus(\n        lc,\n        config.taskID,\n        changeStreamer,\n        localReplica,\n      );\n      if (\n        localReplicaVersion !== undefined &&\n        reserved.replicaVersion < localReplicaVersion &&\n        olderReplicaVersionRetries++ < MAX_OLDER_REPLICA_VERSION_RETRIES\n      ) {\n        // Don't replace a newer local replica with an older one, e.g. from\n        // a replication-manager that has yet to be replaced by a resync.\n        followup.cancel();\n        lc.info?.(\n          `replication-manager is at an older replica version ` +\n            `(${reserved.replicaVersion}) than the local replica ` +\n            `(${localReplicaVersion}). retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n        );\n        await sleep(RETRY_INTERVAL_MS);\n        continue;\n      }\n      // The backupURL comes from the replication-manager's snapshot response.\n      ({backupURL} = reserved);\n      const litestream: LitestreamConfig = {...config.litestream, backupURL};\n      progress.start(reserved.replicaSize);\n      let attempt: RestoreAttempt;\n      try {\n        attempt = await tryRestore(\n          lc,\n          litestream,\n          config.replica.file,\n          reserved,\n          'view_syncer',\n          followup.signal, // abort the restore if the reservation disconnects\n        );\n      } catch (e) {\n        if (followup.signal.aborted) {\n          lc.info?.(\n            `reservation disconnected during restore. retrying`,\n            String(e),\n          );\n          continue;\n        }\n        followup.cancel();\n        throw e;\n      } finally {\n        progress.stop();\n      }\n      if (attempt.restored) {\n        result = attempt.result;\n        progress.done();\n        return followup; // connection to be converted to a subscription\n      }\n      // This attempt's reservation is stale; a fresh one is made on retry.\n      followup.cancel();\n      lc.info?.(\n        `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n      );\n      await sleep(RETRY_INTERVAL_MS);\n    }\n  } finally {\n    progress.stop();\n    const attrs = litestreamRestoreMetricAttrs(\n      config.litestream,\n      'view_syncer',\n      backupURL,\n    );\n    const labels = {...attrs, result: result ?? 'error'};\n    litestreamRestoreRuns().add(1, labels);\n    litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuEA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,SAAS,GAAG,+BAA+B;CACvD,MAAM,WAAW,MAAQ,KAAK,IAAI,qBAAqB;CAEvD,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CACjE,MAAM,OAAuB,aAAa,WAAW,WAAW;CAChE,MAAM,6BACJ,OAAO,eAAe,SAAS,eAAe,CAAC,OAAO,eAAe;CACvE,MAAM,aAAa,GAAG,KAAK;CAC3B,sBAAsB,QAAQ,UAAU;CACxC,cAAc,iBAAiB,QAAQ,YAAY,GAAG,KAAK,GAAG,YAAY,CAAC;CAC3E,KAAK,iBAAiB,QAAQ,UAAU;CACxC,MAAM,8CACJ,yCACE;EACE,WAAW,GAAG,WAAW;EACzB,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACF,cAAc,IAAI,MAAM;CAExB,MAAM,QAAQ,eAAe,MAAM;CACnC,MAAM,EACJ,QACA,QACA,gBAAgB,EACd,MACA,KAAK,oBAAoB,6BACrB,oBAAoB,KAAK,KACzB,KAAA,KAEN,gBAAgB,EAAC,6BAA6B,0BAC5C;CACJ,MAAM,iBAAiB,IAAI,yBACzB,IACA,OACA,OAAO,IACP,iBACF;CAUA,IAAI;CACJ,IACE,aAAa,aACb,CAAC,+BACA,OAAO,WAAW,cAAc,OAAO,WAAW,eAEnD,oBAAoB,MAAM,eAAe,IAAI,QAAQ,cAAc;CAGrE,MAAM,EACJ,MAAM,QACN,SACA,aACE,MAAM,aAAa,IAAI,UAAU,OAAO,OAAO;CACnD,4CAA4C;CAC5C,yCACE;EACE,WAAW,GAAG,WAAW;EACzB;CACF,GACA,OAAO,sBACT;CAKA,qBAAqB,OAAO,eAAe,qBAAqB,MAAM;CAEtE,aAAa,IAAI,QAAQ,SAAS,QAAQ;CAK1C,MAAM,UAAU,gBAAgB,QAAQ;CACxC,MAAM,aACJ,SAAS,YACT,OAAO,WAAW,aAClB,OAAO,WAAW,iBAClB,OAAO,WAAW,6BACd;EACE,0BACG,OAAO,WAAW,6BAA6B,QAAQ,IACxD;EACF,aAAa,CAAC,OAAO,WAAW,eAC5B,KAAA,IACC,OAAO,WAAW,eAAe,QAAQ,IAAK;CACrD,IACA;CAEN,MAAM,eAAe,IAAI,wBAAwB;CACjD,MAAM,aAAa,KAAK,QAAQ,MAAM,SAAS,OAAO,KAAK,UAAU;CAKrE,MAAM,YAAY,IAAI,oBAAoB,IAAI,MAAM;CACpD,IAAI,SAAS,UACX,UAAU,IACR,0BAA0B,IAAI,OAAO,YAAY,MAAM,GACvD,YACF;CAGF,MAAM,aAAa,IAAI,kBACrB,IACA,QACA,GAAG,WAAW,GAAG,OACjB,MACA,gBACA,cACA,6BAEI,2BAA2B,eAAe,MAAM,IAChD,MACJ;EACE,sBAAsB;EACtB;CACF,GACA,mBAGA,6BACI,KAAA,IACA;EACE,wBACE,OAAO,KAA4B,CAAC,kBAAkB,CAAC,CAAC,CAAC;EAC3D,aAAa;CACf,CACN;CAEA,qBAAqB,IAAI,YAAY,MAAM;CAE3C,WAAgB,MAAM,EAAE,WAAW;EACjC,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CACtC,CAAC;CAED,OAAO,eAAe,IAAI,QAAQ,YAAY,SAAS;AACzD;AAEA,SAAgB,aACd,IACA,MACA,SACA,UACA;CACA,IAAI;CACJ,MAAM,cAAc;EAClB,IAAI,CAAC,IACH,IAAI;GACF,KAAK,IAAI,SAAS,IAAI,MAAM,EAAC,UAAU,KAAI,CAAC;EAC9C,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,mBAAmB,CAAC;EACvD;EAEF,OAAO;CACT;CAEA,iBAAiB,WAAW,WAAW;EACrC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,IAAI,CAAC;CAExC,iBAAiB,WAAW,YAAY;EACtC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,KAAK,CAAC;CAEjD,IAAI,YAAY,QACd,iBAAiB,WAAW,aAAa;EACvC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,MAAM,CAAC;CAGpD,iBAAiB,WAAW,oBAAoB;EAC9C,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,sBAAsB,IAAI,OAAO,QAAQ,CAAC;CAEzD,iBAAiB,WAAW,4BAA4B;EACtD,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,8BAA8B,IAAI,OAAO,QAAQ,CAAC;CAKjE,iBAAiB,WAAW,cAAc;EACxC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,uBAAuB,IAAI,IAAI,CAAC;AACjD;AAEA,SAAS,gBAAgB,IAAgB,MAAkC;CACzE,OAAO,OAAM,MAAK;EAChB,IAAI;GACF,MAAM,QAAQ,MAAM,KAAK,IAAI;GAC7B,EAAE,QAAQ,MAAM,IAAI;EACtB,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;EACxD;CACF;AACF;AAEA,SAAS,sBACP,IACA,OACA,UACoB;CACpB,QAAO,MAAK;EACV,IAAI;GACF,MAAM,KAAK,MAAM;GACjB,IAAI,CAAC,IACH;GAEF,MAAM,CAAC,EAAC,SAAQ,GAAG,OAAsB,sBAAsB;GAC/D,IAAI,OAAO,QAAQ,UACjB,EAAE,QAAQ,KAAK,IAAI,GAAG,GAAG,IAAI,QAAQ;EAEzC,SAAS,GAAG;GACV,GAAG,OAAO,4DAA4D,CAAC;EACzE;CACF;AACF;AAEA,SAAS,8BACP,IACA,OACA,UACoB;CACpB,QAAO,MAAK;EACV,IAAI;GACF,MAAM,KAAK,MAAM;GACjB,IAAI,CAAC,IACH;GAEF,MAAM,CAAC,EAAC,KAAK,kBAAiB,GAAG,OAG9B,sBAAsB;GACzB,IAAI,OAAO,QAAQ,YAAY,OAAO,iBAAiB,UACrD,EAAE,QAAQ,KAAK,IAAI,GAAG,MAAM,YAAY,IAAI,QAAQ;EAExD,SAAS,GAAG;GACV,GAAG,OACD,oEACA,CACF;EACF;CACF;AACF;AAEA,SAAS,uBACP,IACA,MACoB;CACpB,OAAO,OAAM,MAAK,EAAE,QAAQ,MAAM,gBAAgB,IAAI,IAAI,CAAC;AAC7D;AAEA,IAAM,oBAAoB;AAO1B,IAAM,oCAAoC;;AAe1C,eAAsB,eACpB,IACA,QACA,gBAC8B;CAC9B,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;CACJ,MAAM,WAAW,IAAI,wBAAwB,IAAI,OAAO,QAAQ,IAAI;CACpE,IAAI,6BAA6B;CAMjC,MAAM,eAAe,iBAAiB,IAAI,OAAO,QAAQ,IAAI;CAC7D,MAAM,sBAAsB,cAAc;CAC1C,IAAI;EACF,SAAS;GACP,MAAM,EAAC,UAAU,aAAY,MAAM,4BACjC,IACA,OAAO,QACP,gBACA,YACF;GACA,IACE,wBAAwB,KAAA,KACxB,SAAS,iBAAiB,uBAC1B,+BAA+B,mCAC/B;IAGA,SAAS,OAAO;IAChB,GAAG,OACD,uDACM,SAAS,eAAe,4BACxB,oBAAoB,iBAAiB,oBAAoB,IAAK,SACtE;IACA,MAAM,MAAM,iBAAiB;IAC7B;GACF;GAEA,CAAC,CAAC,aAAa;GACf,MAAM,aAA+B;IAAC,GAAG,OAAO;IAAY;GAAS;GACrE,SAAS,MAAM,SAAS,WAAW;GACnC,IAAI;GACJ,IAAI;IACF,UAAU,MAAM,WACd,IACA,YACA,OAAO,QAAQ,MACf,UACA,eACA,SAAS,MACX;GACF,SAAS,GAAG;IACV,IAAI,SAAS,OAAO,SAAS;KAC3B,GAAG,OACD,qDACA,OAAO,CAAC,CACV;KACA;IACF;IACA,SAAS,OAAO;IAChB,MAAM;GACR,UAAU;IACR,SAAS,KAAK;GAChB;GACA,IAAI,QAAQ,UAAU;IACpB,SAAS,QAAQ;IACjB,SAAS,KAAK;IACd,OAAO;GACT;GAEA,SAAS,OAAO;GAChB,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,UAAU;EACR,SAAS,KAAK;EAMd,MAAM,SAAS;GAAC,GALF,6BACZ,OAAO,YACP,eACA,SAEiB;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,EAAE,IAAI,GAAG,MAAM;EACrC,0BAA0B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}
+{"version":3,"file":"replicator.js","names":["v.parse"],"sources":["../../../../../zero-cache/src/server/replicator.ts"],"sourcesContent":["import {stat} from 'node:fs/promises';\nimport {pid} from 'node:process';\nimport type {ObservableCallback} from '@opentelemetry/api';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport type {\n  LitestreamConfig,\n  NormalizedZeroConfig,\n} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {\n  ChangeStreamerHttpClient,\n  type ReservationFollowup,\n  type SnapshotReserver,\n} from '../services/change-streamer/change-streamer-http.ts';\nimport {reserveAndGetSnapshotStatus} from '../services/change-streamer/snapshot.ts';\nimport {\n  exitAfter,\n  runUntilKilled,\n  SupportingProcesses,\n} from '../services/life-cycle.ts';\nimport {\n  readLocalReplica,\n  startReplicaBackupProcess,\n  tryRestore,\n  type RestoreAttempt,\n  type RestoreResult,\n} from '../services/litestream/commands.ts';\nimport {\n  litestreamRestoreDuration,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreRuns,\n} from '../services/litestream/metrics.ts';\nimport {RestoreProgressReporter} from '../services/litestream/restore-progress.ts';\nimport {ReplicationStatusPublisher} from '../services/replicator/replication-status.ts';\nimport {\n  ReplicatorService,\n  type ReplicatorMode,\n} from '../services/replicator/replicator.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {\n  ThreadWriteWorkerClient,\n  type ForceCheckpointConfig,\n} from '../services/replicator/write-worker-client.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type DrainRequestedMessage,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {\n  deleteStaleChangeLog,\n  getPragmaConfig,\n  replicaFileModeSchema,\n  setUpMessageHandlers,\n  setupReplica,\n  type WalMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length > 0, `replicator mode not specified`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n  const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n  const mode: ReplicatorMode = fileMode === 'backup' ? 'backup' : 'serving';\n  const runningLocalChangeStreamer =\n    config.changeStreamer.mode === 'dedicated' && !config.changeStreamer.uri;\n  const workerName = `${mode}-replicator`;\n  installProfileHandler(parent, workerName);\n  startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);\n  lc = createLogContext(config, workerName);\n  const unregisterInitialCorruptionDiagnosticTarget =\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: `${workerName} replica`,\n        dbPath: config.replica.file,\n      },\n      config.sqliteCorruptionChecks,\n    );\n  initEventSink(lc, config);\n\n  const shard = getShardConfig(config);\n  const {\n    taskID,\n    change,\n    changeStreamer: {\n      port,\n      uri: changeStreamerURI = runningLocalChangeStreamer\n        ? `http://localhost:${port}/`\n        : undefined,\n    },\n    replicationLag: {initialReadinessThresholdMs, maxReadinessDelayMs},\n  } = config;\n  const changeStreamer = new ChangeStreamerHttpClient(\n    lc,\n    shard,\n    change.db,\n    changeStreamerURI,\n  );\n\n  // Remote view-syncers restore from the replication-manager, which supplies\n  // the backup URL. A local change-streamer owns the canonical replica and\n  // must not infer backup configuration from bundled executable paths.\n  //\n  // The reservation's connection (`initialConnection`) is threaded through to\n  // IncrementalSyncer's first subscribe(), so the replication-manager that\n  // reserved the change log for this restore is the one that serves the\n  // subscription (single-connection handoff).\n  let initialConnection: ReservationFollowup | undefined;\n  if (\n    fileMode === 'serving' &&\n    !runningLocalChangeStreamer &&\n    (config.litestream.executable || config.litestream.executableV5)\n  ) {\n    initialConnection = await restoreReplica(lc, config, changeStreamer);\n  }\n\n  const {\n    file: dbPath,\n    walMode,\n    pageSize,\n  } = await setupReplica(lc, fileMode, config.replica);\n  unregisterInitialCorruptionDiagnosticTarget();\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: `${workerName} replica`,\n      dbPath,\n    },\n    config.sqliteCorruptionChecks,\n  );\n\n  // A file left behind by a run with the writer enabled would otherwise hold\n  // local disk indefinitely. See `replicatorDeletesStaleChangeLog` for why\n  // this is keyed on the config flag and nothing else.\n  deleteStaleChangeLog(config.changeStreamer.sqliteChangeLogMode, dbPath);\n\n  setupMetrics(lc, dbPath, walMode, pageSize);\n\n  // Create the write worker for async SQLite writes. Enable write-path\n  // checkpoint backpressure only for the backup replicator on litestream v5,\n  // which owns the replica litestream backs up.\n  const pragmas = getPragmaConfig(fileMode);\n  const checkpoint: ForceCheckpointConfig | null =\n    mode === 'backup' &&\n    config.litestream.backupURL &&\n    config.litestream.backupUsingV5 &&\n    config.litestream.forceCheckpointThresholdMB\n      ? {\n          checkpointThresholdPages:\n            (config.litestream.forceCheckpointThresholdMB * 1024 ** 2) /\n            pageSize,\n          maxWalPages: !config.litestream.maxWalSizeMB\n            ? undefined\n            : (config.litestream.maxWalSizeMB * 1024 ** 2) / pageSize,\n        }\n      : null;\n\n  const workerClient = new ThreadWriteWorkerClient();\n  await workerClient.init(dbPath, mode, pragmas, config.log, checkpoint);\n\n  // The backup-replicator manages the litestream backup process. It is\n  // started after the write worker has prepared the db file in order to\n  // avoid contending on the sqlite lock.\n  const processes = new SupportingProcesses(lc, parent);\n  if (mode === 'backup') {\n    processes.add(\n      startReplicaBackupProcess(lc, config.litestream, dbPath),\n      'litestream',\n    );\n  }\n\n  const replicator = new ReplicatorService(\n    lc,\n    taskID,\n    `${workerName}-${pid}`,\n    mode,\n    changeStreamer,\n    workerClient,\n    runningLocalChangeStreamer\n      ? // publish ReplicationStatusEvents from backup-replicator only\n        ReplicationStatusPublisher.forReplicaFile(dbPath)\n      : null,\n    {\n      readinessThresholdMs: initialReadinessThresholdMs,\n      maxReadinessDelayMs,\n    },\n    initialConnection,\n    // View-syncers (i.e. with a remote change-streamer) switch over to\n    // another replication-manager when theirs drains.\n    runningLocalChangeStreamer\n      ? undefined\n      : {\n          onRestoreGranted: () =>\n            parent.send<DrainRequestedMessage>(['drainRequested', {}]),\n          replicaFile: dbPath,\n        },\n  );\n\n  setUpMessageHandlers(lc, replicator, parent);\n\n  void replicator.ready().then(() => {\n    parent.send(['ready', {ready: true}]);\n  });\n\n  return runUntilKilled(lc, parent, replicator, processes);\n}\n\nexport function setupMetrics(\n  lc: LogContext,\n  file: string,\n  walMode: WalMode,\n  pageSize: number,\n) {\n  let db: Database | undefined;\n  const getDb = () => {\n    if (!db) {\n      try {\n        db = new Database(lc, file, {readonly: true});\n      } catch (e) {\n        lc.warn?.(`unable to open ${file} for wal metrics`, e);\n      }\n    }\n    return db;\n  };\n\n  getOrCreateGauge('replica', 'db_size', {\n    description:\n      `The size of the replica's main db file, ` +\n      `which does not include the wal file(s)`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, file));\n\n  getOrCreateGauge('replica', 'wal_size', {\n    description: `The size of the replica's wal file`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, `${file}-wal`));\n\n  if (walMode === 'wal2') {\n    getOrCreateGauge('replica', 'wal2_size', {\n      description: `The size of the replica's wal2 file`,\n      unit: 'bytes',\n    }).addCallback(observeFileSize(lc, `${file}-wal2`));\n  }\n\n  getOrCreateGauge('replica', 'wal_active_bytes', {\n    description:\n      `The actual size of active WAL frames currently in use ` +\n      `(excluding pre-allocated or reclaimed space).`,\n    unit: 'bytes',\n  }).addCallback(observeWalActiveBytes(lc, getDb, pageSize));\n\n  getOrCreateGauge('replica', 'wal_uncheckpointed_bytes', {\n    description:\n      `The size of un-checkpointed WAL frames that have not yet ` +\n      `been backfilled into the main database.`,\n    unit: 'bytes',\n  }).addCallback(observeWalUncheckpointedBytes(lc, getDb, pageSize));\n\n  // A whole-database total rather than a per-file one. Its counterpart,\n  // `sqlite_change_log.file_bytes`, is emitted by the change-streamer, which is\n  // the process that writes the log.\n  getOrCreateGauge('replica', 'file_bytes', {\n    description:\n      `The replica's total on-disk footprint: its main db file plus its ` +\n      `wal sidecars.`,\n    unit: 'By',\n  }).addCallback(observeSQLiteFileBytes(lc, file));\n}\n\nfunction observeFileSize(lc: LogContext, file: string): ObservableCallback {\n  return async o => {\n    try {\n      const stats = await stat(file);\n      o.observe(stats.size);\n    } catch (e) {\n      lc.warn?.(`unable to stat ${file} for size metrics`, e);\n    }\n  };\n}\n\nfunction observeWalActiveBytes(\n  lc: LogContext,\n  getDb: () => Database | undefined,\n  pageSize: number,\n): ObservableCallback {\n  return o => {\n    try {\n      const db = getDb();\n      if (!db) {\n        return;\n      }\n      const [{log}] = db.pragma<{log: number}>('wal_checkpoint(NOOP)');\n      if (typeof log === 'number') {\n        o.observe(Math.max(0, log) * pageSize);\n      }\n    } catch (e) {\n      lc.warn?.('unable to inspect wal_checkpoint for active bytes metric', e);\n    }\n  };\n}\n\nfunction observeWalUncheckpointedBytes(\n  lc: LogContext,\n  getDb: () => Database | undefined,\n  pageSize: number,\n): ObservableCallback {\n  return o => {\n    try {\n      const db = getDb();\n      if (!db) {\n        return;\n      }\n      const [{log, checkpointed}] = db.pragma<{\n        log: number;\n        checkpointed: number;\n      }>('wal_checkpoint(NOOP)');\n      if (typeof log === 'number' && typeof checkpointed === 'number') {\n        o.observe(Math.max(0, log - checkpointed) * pageSize);\n      }\n    } catch (e) {\n      lc.warn?.(\n        'unable to inspect wal_checkpoint for uncheckpointed bytes metric',\n        e,\n      );\n    }\n  };\n}\n\nfunction observeSQLiteFileBytes(\n  lc: LogContext,\n  file: string,\n): ObservableCallback {\n  return async o => o.observe(await sqliteFileBytes(lc, file));\n}\n\nconst RETRY_INTERVAL_MS = 3000;\n\n// The number of times a view-syncer retries a snapshot reservation from a\n// replication-manager with an older replica version than its local replica\n// (e.g. a not-yet-terminated replication-manager during a resync rollout),\n// before accepting the older version (e.g. if the upstream was legitimately\n// replaced by one with a lower replica version).\nconst MAX_OLDER_REPLICA_VERSION_RETRIES = 20;\n\n// View-syncers wait indefinitely for the replication-manager to publish a\n// restorable backup. On a fresh stack the first backup is not durable until\n// the initial sync completes and litestream uploads the initial snapshot,\n// which can take many minutes for a large replica. The platform's startup\n// probe budget (which scales with replica size) is the backstop, so\n// restoreReplica must not impose its own shorter cap and self-terminate\n// while the backup is still being produced.\n//\n// Returns the `followup` of the reservation whose backup was actually\n// restored, still open, so the caller can hand it to IncrementalSyncer's\n// first subscribe() via `followup.subscribe()`: the same replication-manager\n// that reserved the change log is the one that serves the subscription.\n/** Exported for testing. */\nexport async function restoreReplica(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  changeStreamer: SnapshotReserver,\n): Promise<ReservationFollowup> {\n  const start = performance.now();\n  let backupURL: string | undefined;\n  let result: RestoreResult | undefined;\n  const progress = new RestoreProgressReporter(lc, config.replica.file);\n  let olderReplicaVersionRetries = 0;\n  // Describing the local replica (if any) allows the replication-manager to\n  // account for replicas that are being replaced (see RestoreAdmission). It\n  // is read once, before the first attempt, as a failed attempt may have\n  // already deleted it; it continues to describe the replica being replaced\n  // (and the version not to downgrade from) on every retry.\n  const localReplica = readLocalReplica(lc, config.replica.file);\n  const localReplicaVersion = localReplica?.replicaVersion;\n  try {\n    for (;;) {\n      const {reserved, followup} = await reserveAndGetSnapshotStatus(\n        lc,\n        config.taskID,\n        changeStreamer,\n        localReplica,\n      );\n      if (\n        localReplicaVersion !== undefined &&\n        reserved.replicaVersion < localReplicaVersion &&\n        olderReplicaVersionRetries++ < MAX_OLDER_REPLICA_VERSION_RETRIES\n      ) {\n        // Don't replace a newer local replica with an older one, e.g. from\n        // a replication-manager that has yet to be replaced by a resync.\n        followup.cancel();\n        lc.info?.(\n          `replication-manager is at an older replica version ` +\n            `(${reserved.replicaVersion}) than the local replica ` +\n            `(${localReplicaVersion}). retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n        );\n        await sleep(RETRY_INTERVAL_MS);\n        continue;\n      }\n      // The backupURL comes from the replication-manager's snapshot response.\n      ({backupURL} = reserved);\n      const litestream: LitestreamConfig = {...config.litestream, backupURL};\n      progress.start(reserved.replicaSize);\n      let attempt: RestoreAttempt;\n      try {\n        attempt = await tryRestore(\n          lc,\n          litestream,\n          config.replica.file,\n          reserved,\n          'view_syncer',\n          followup.signal, // abort the restore if the reservation disconnects\n        );\n      } catch (e) {\n        if (followup.signal.aborted) {\n          lc.info?.(\n            `reservation disconnected during restore. retrying`,\n            String(e),\n          );\n          continue;\n        }\n        followup.cancel();\n        throw e;\n      } finally {\n        progress.stop();\n      }\n      if (attempt.restored) {\n        result = attempt.result;\n        progress.done();\n        return followup; // connection to be converted to a subscription\n      }\n      // This attempt's reservation is stale; a fresh one is made on retry.\n      followup.cancel();\n      lc.info?.(\n        `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n      );\n      await sleep(RETRY_INTERVAL_MS);\n    }\n  } finally {\n    progress.stop();\n    const attrs = litestreamRestoreMetricAttrs(\n      config.litestream,\n      'view_syncer',\n      backupURL,\n    );\n    const labels = {...attrs, result: result ?? 'error'};\n    litestreamRestoreRuns().add(1, labels);\n    litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuEA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,SAAS,GAAG,+BAA+B;CACvD,MAAM,WAAWA,MAAQ,KAAK,IAAI,qBAAqB;CAEvD,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CACjE,MAAM,OAAuB,aAAa,WAAW,WAAW;CAChE,MAAM,6BACJ,OAAO,eAAe,SAAS,eAAe,CAAC,OAAO,eAAe;CACvE,MAAM,aAAa,GAAG,KAAK;CAC3B,sBAAsB,QAAQ,UAAU;CACxC,cAAc,iBAAiB,QAAQ,YAAY,GAAG,KAAK,GAAG,YAAY,CAAC;CAC3E,KAAK,iBAAiB,QAAQ,UAAU;CACxC,MAAM,8CACJ,yCACE;EACE,WAAW,GAAG,WAAW;EACzB,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACF,cAAc,IAAI,MAAM;CAExB,MAAM,QAAQ,eAAe,MAAM;CACnC,MAAM,EACJ,QACA,QACA,gBAAgB,EACd,MACA,KAAK,oBAAoB,6BACrB,oBAAoB,KAAK,KACzB,KAAA,KAEN,gBAAgB,EAAC,6BAA6B,0BAC5C;CACJ,MAAM,iBAAiB,IAAI,yBACzB,IACA,OACA,OAAO,IACP,iBACF;CAUA,IAAI;CACJ,IACE,aAAa,aACb,CAAC,+BACA,OAAO,WAAW,cAAc,OAAO,WAAW,eAEnD,oBAAoB,MAAM,eAAe,IAAI,QAAQ,cAAc;CAGrE,MAAM,EACJ,MAAM,QACN,SACA,aACE,MAAM,aAAa,IAAI,UAAU,OAAO,OAAO;CACnD,4CAA4C;CAC5C,yCACE;EACE,WAAW,GAAG,WAAW;EACzB;CACF,GACA,OAAO,sBACT;CAKA,qBAAqB,OAAO,eAAe,qBAAqB,MAAM;CAEtE,aAAa,IAAI,QAAQ,SAAS,QAAQ;CAK1C,MAAM,UAAU,gBAAgB,QAAQ;CACxC,MAAM,aACJ,SAAS,YACT,OAAO,WAAW,aAClB,OAAO,WAAW,iBAClB,OAAO,WAAW,6BACd;EACE,0BACG,OAAO,WAAW,6BAA6B,QAAQ,IACxD;EACF,aAAa,CAAC,OAAO,WAAW,eAC5B,KAAA,IACC,OAAO,WAAW,eAAe,QAAQ,IAAK;CACrD,IACA;CAEN,MAAM,eAAe,IAAI,wBAAwB;CACjD,MAAM,aAAa,KAAK,QAAQ,MAAM,SAAS,OAAO,KAAK,UAAU;CAKrE,MAAM,YAAY,IAAI,oBAAoB,IAAI,MAAM;CACpD,IAAI,SAAS,UACX,UAAU,IACR,0BAA0B,IAAI,OAAO,YAAY,MAAM,GACvD,YACF;CAGF,MAAM,aAAa,IAAI,kBACrB,IACA,QACA,GAAG,WAAW,GAAG,OACjB,MACA,gBACA,cACA,6BAEI,2BAA2B,eAAe,MAAM,IAChD,MACJ;EACE,sBAAsB;EACtB;CACF,GACA,mBAGA,6BACI,KAAA,IACA;EACE,wBACE,OAAO,KAA4B,CAAC,kBAAkB,CAAC,CAAC,CAAC;EAC3D,aAAa;CACf,CACN;CAEA,qBAAqB,IAAI,YAAY,MAAM;CAE3C,WAAgB,MAAM,CAAC,CAAC,WAAW;EACjC,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CACtC,CAAC;CAED,OAAO,eAAe,IAAI,QAAQ,YAAY,SAAS;AACzD;AAEA,SAAgB,aACd,IACA,MACA,SACA,UACA;CACA,IAAI;CACJ,MAAM,cAAc;EAClB,IAAI,CAAC,IACH,IAAI;GACF,KAAK,IAAI,SAAS,IAAI,MAAM,EAAC,UAAU,KAAI,CAAC;EAC9C,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,mBAAmB,CAAC;EACvD;EAEF,OAAO;CACT;CAEA,iBAAiB,WAAW,WAAW;EACrC,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,gBAAgB,IAAI,IAAI,CAAC;CAExC,iBAAiB,WAAW,YAAY;EACtC,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,gBAAgB,IAAI,GAAG,KAAK,KAAK,CAAC;CAEjD,IAAI,YAAY,QACd,iBAAiB,WAAW,aAAa;EACvC,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,gBAAgB,IAAI,GAAG,KAAK,MAAM,CAAC;CAGpD,iBAAiB,WAAW,oBAAoB;EAC9C,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,sBAAsB,IAAI,OAAO,QAAQ,CAAC;CAEzD,iBAAiB,WAAW,4BAA4B;EACtD,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,8BAA8B,IAAI,OAAO,QAAQ,CAAC;CAKjE,iBAAiB,WAAW,cAAc;EACxC,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,uBAAuB,IAAI,IAAI,CAAC;AACjD;AAEA,SAAS,gBAAgB,IAAgB,MAAkC;CACzE,OAAO,OAAM,MAAK;EAChB,IAAI;GACF,MAAM,QAAQ,MAAM,KAAK,IAAI;GAC7B,EAAE,QAAQ,MAAM,IAAI;EACtB,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;EACxD;CACF;AACF;AAEA,SAAS,sBACP,IACA,OACA,UACoB;CACpB,QAAO,MAAK;EACV,IAAI;GACF,MAAM,KAAK,MAAM;GACjB,IAAI,CAAC,IACH;GAEF,MAAM,CAAC,EAAC,SAAQ,GAAG,OAAsB,sBAAsB;GAC/D,IAAI,OAAO,QAAQ,UACjB,EAAE,QAAQ,KAAK,IAAI,GAAG,GAAG,IAAI,QAAQ;EAEzC,SAAS,GAAG;GACV,GAAG,OAAO,4DAA4D,CAAC;EACzE;CACF;AACF;AAEA,SAAS,8BACP,IACA,OACA,UACoB;CACpB,QAAO,MAAK;EACV,IAAI;GACF,MAAM,KAAK,MAAM;GACjB,IAAI,CAAC,IACH;GAEF,MAAM,CAAC,EAAC,KAAK,kBAAiB,GAAG,OAG9B,sBAAsB;GACzB,IAAI,OAAO,QAAQ,YAAY,OAAO,iBAAiB,UACrD,EAAE,QAAQ,KAAK,IAAI,GAAG,MAAM,YAAY,IAAI,QAAQ;EAExD,SAAS,GAAG;GACV,GAAG,OACD,oEACA,CACF;EACF;CACF;AACF;AAEA,SAAS,uBACP,IACA,MACoB;CACpB,OAAO,OAAM,MAAK,EAAE,QAAQ,MAAM,gBAAgB,IAAI,IAAI,CAAC;AAC7D;AAEA,IAAM,oBAAoB;AAO1B,IAAM,oCAAoC;;AAe1C,eAAsB,eACpB,IACA,QACA,gBAC8B;CAC9B,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;CACJ,MAAM,WAAW,IAAI,wBAAwB,IAAI,OAAO,QAAQ,IAAI;CACpE,IAAI,6BAA6B;CAMjC,MAAM,eAAe,iBAAiB,IAAI,OAAO,QAAQ,IAAI;CAC7D,MAAM,sBAAsB,cAAc;CAC1C,IAAI;EACF,SAAS;GACP,MAAM,EAAC,UAAU,aAAY,MAAM,4BACjC,IACA,OAAO,QACP,gBACA,YACF;GACA,IACE,wBAAwB,KAAA,KACxB,SAAS,iBAAiB,uBAC1B,+BAA+B,mCAC/B;IAGA,SAAS,OAAO;IAChB,GAAG,OACD,uDACM,SAAS,eAAe,4BACxB,oBAAoB,iBAAiB,oBAAoB,IAAK,SACtE;IACA,MAAM,MAAM,iBAAiB;IAC7B;GACF;GAEA,CAAC,CAAC,aAAa;GACf,MAAM,aAA+B;IAAC,GAAG,OAAO;IAAY;GAAS;GACrE,SAAS,MAAM,SAAS,WAAW;GACnC,IAAI;GACJ,IAAI;IACF,UAAU,MAAM,WACd,IACA,YACA,OAAO,QAAQ,MACf,UACA,eACA,SAAS,MACX;GACF,SAAS,GAAG;IACV,IAAI,SAAS,OAAO,SAAS;KAC3B,GAAG,OACD,qDACA,OAAO,CAAC,CACV;KACA;IACF;IACA,SAAS,OAAO;IAChB,MAAM;GACR,UAAU;IACR,SAAS,KAAK;GAChB;GACA,IAAI,QAAQ,UAAU;IACpB,SAAS,QAAQ;IACjB,SAAS,KAAK;IACd,OAAO;GACT;GAEA,SAAS,OAAO;GAChB,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,UAAU;EACR,SAAS,KAAK;EAMd,MAAM,SAAS;GAAC,GALF,6BACZ,OAAO,YACP,eACA,SAEiB;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,CAAC,CAAC,IAAI,GAAG,MAAM;EACrC,0BAA0B,CAAC,CAAC,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}

package/out/zero-cache/src/server/runner/run-worker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-worker.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/server/runner/run-worker.ts"],"names":[],"mappings":"AAAA,OAAO,kCAAkC,CAAC;AAa1C,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,0BAA0B,CAAC;AAsBlC;;;;;;GAMG;AACH,wBAAsB,SAAS,CAC7B,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,GAAG,EAAE,MAAM,CAAC,UAAU,GACrB,OAAO,CAAC,IAAI,CAAC,CAqEf"}
+{"version":3,"file":"run-worker.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/server/runner/run-worker.ts"],"names":[],"mappings":"AAAA,OAAO,kCAAkC,CAAC;AAa1C,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,0BAA0B,CAAC;AAsBlC;;;;;;GAMG;AACH,wBAAsB,SAAS,CAC7B,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,GAAG,EAAE,MAAM,CAAC,UAAU,GACrB,OAAO,CAAC,IAAI,CAAC,CAuEf"}

package/out/zero-cache/src/server/runner/run-worker.js

@@ -1,3 +1,4 @@
+import { resolver } from "../../../../shared/src/resolver.js";
 import "../../../../zero-protocol/src/protocol-version.js";
 import "../../../../shared/src/dotenv.js";
 import { colorConsole } from "../../../../shared/src/logging.js";
@@ -9,7 +10,6 @@ import { createLogContext } from "../logging.js";
 import { MAIN_URL } from "../worker-urls.js";
 import { getTaskID } from "./runtime.js";
 import { ZeroDispatcher } from "./zero-dispatcher.js";
-import { resolver } from "@rocicorp/resolver";
 import { styleText } from "node:util";
 //#region ../zero-cache/src/server/runner/run-worker.ts
 var startupMessageEnv = "ZERO_ENABLE_STARTUP_MESSAGE";
@@ -31,7 +31,8 @@ async function runWorker(parent, env) {
 		emitDeprecationWarnings: true
 	});
 	const lc = createLogContext(cfg, "runner");
-	const config = normalizeZeroConfig(lc, cfg, env, await getTaskID(lc));
+	const defaultTaskID = await getTaskID(lc);
+	const config = normalizeZeroConfig(lc, cfg, env, defaultTaskID);
 	const processes = new ProcessManager(lc, parent ?? process);
 	const { port, keepaliveTimeoutMs, lazyStartup, selfDrainDelayMs } = config;
 	const serverVersion = getServerVersion(config);
@@ -45,7 +46,7 @@ async function runWorker(parent, env) {
 	const dispatcher = new ZeroDispatcher(config, lc, {
 		port,
 		keepaliveTimeoutMs
-	}, startZeroCache, () => printStartupMessage(env));
+	}, startZeroCache, () => printStartupMessage(env), () => processes.shutdown());
 	const selfDrain = new SelfDrain(lc, parent ?? process, {
 		delayMs: selfDrainDelayMs,
 		onRequested: () => dispatcher.failHealthChecks()

package/out/zero-cache/src/server/runner/run-worker.js.map

@@ -1 +1 @@
-{"version":3,"file":"run-worker.js","names":[],"sources":["../../../../../../zero-cache/src/server/runner/run-worker.ts"],"sourcesContent":["import '../../../../shared/src/dotenv.ts';\n\nimport {styleText} from 'node:util';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {colorConsole} from '../../../../shared/src/logging.ts';\nimport {PROTOCOL_VERSION} from '../../../../zero-protocol/src/protocol-version.ts';\nimport {normalizeZeroConfig} from '../../config/normalize.ts';\nimport {getServerVersion, getZeroConfig} from '../../config/zero-config.ts';\nimport {\n  ProcessManager,\n  runUntilKilled,\n  SelfDrain,\n} from '../../services/life-cycle.ts';\nimport {\n  childWorker,\n  type DrainRequestedMessage,\n  type Worker,\n} from '../../types/processes.ts';\nimport {createLogContext} from '../logging.ts';\nimport {MAIN_URL} from '../worker-urls.ts';\nimport {getTaskID} from './runtime.ts';\nimport {ZeroDispatcher} from './zero-dispatcher.ts';\n\nconst startupMessageEnv = 'ZERO_ENABLE_STARTUP_MESSAGE';\n\nfunction printStartupMessage(env: NodeJS.ProcessEnv) {\n  if (env[startupMessageEnv] !== '1') {\n    return;\n  }\n\n  colorConsole.log(\n    `\\nBTW, ${styleText(['bold', 'cyan'], 'Cloud Zero')} ` +\n      'is now available - professional Zero hosting from the team that built it.\\n' +\n      `Get started now: ${styleText(['blue', 'underline'], 'https://zero.rocicorp.dev/cloud')}\\n\\n` +\n      styleText('dim', `Disable this message with ${startupMessageEnv}=0`) +\n      '\\n',\n  );\n}\n\n/**\n * Top-level `runner` entry point to the zero-cache. This layer is responsible for:\n * * runtime-based config normalization\n * * lazy startup\n * * serving /statsz\n * * auto-reset restarts (TODO)\n */\nexport async function runWorker(\n  parent: Worker | null,\n  env: NodeJS.ProcessEnv,\n): Promise<void> {\n  // Note: Deprecation warnings are only emitted at this top-level parse;\n  //       they are suppressed when parsed in subprocesses.\n  const cfg = getZeroConfig({env, emitDeprecationWarnings: true});\n  const lc = createLogContext(cfg, 'runner');\n\n  const defaultTaskID = await getTaskID(lc);\n  const config = normalizeZeroConfig(lc, cfg, env, defaultTaskID);\n  const processes = new ProcessManager(lc, parent ?? process);\n\n  const {port, keepaliveTimeoutMs, lazyStartup, selfDrainDelayMs} = config;\n  const serverVersion = getServerVersion(config);\n  lc.info?.(`starting server${!serverVersion ? '' : `@${serverVersion}`} `, {\n    protocolVersion: PROTOCOL_VERSION,\n    taskID: config.taskID,\n    app: config.app,\n    shard: config.shard,\n    port: config.port,\n  });\n\n  const dispatcher = new ZeroDispatcher(\n    config,\n    lc,\n    {port, keepaliveTimeoutMs},\n    startZeroCache,\n    () => printStartupMessage(env),\n  );\n  const selfDrain = new SelfDrain(lc, parent ?? process, {\n    delayMs: selfDrainDelayMs,\n    onRequested: () => dispatcher.failHealthChecks(),\n  });\n\n  let zeroCache: Resolver<Worker> | undefined;\n  function startZeroCache(): Promise<Worker> {\n    if (zeroCache === undefined) {\n      const startMs = performance.now();\n      lc.info?.('starting zero-cache');\n\n      const r = (zeroCache = resolver<Worker>());\n      const w = childWorker(MAIN_URL, env)\n        .once('message', () => {\n          r.resolve(w);\n          lc.info?.(`zero-cache ready (${performance.now() - startMs} ms)`);\n        })\n        .once('error', r.reject);\n      w.onMessageType<DrainRequestedMessage>('drainRequested', () =>\n        selfDrain.request(),\n      );\n\n      processes.addWorker(w, 'user-facing', 'zero-cache');\n    }\n    return zeroCache.promise;\n  }\n\n  // Eagerly start the zero-cache if it was not configured with --lazy-startup.\n  if (!lazyStartup) {\n    void startZeroCache();\n  }\n\n  await processes.allWorkersReady();\n  parent?.send(['ready', {ready: true}]);\n\n  try {\n    await runUntilKilled(lc, parent ?? process, dispatcher);\n  } catch (err) {\n    processes.logErrorAndExit(err, 'main');\n  } finally {\n    await processes.shutdown();\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAuBA,IAAM,oBAAoB;AAE1B,SAAS,oBAAoB,KAAwB;CACnD,IAAI,IAAI,uBAAuB,KAC7B;CAGF,aAAa,IACX,UAAU,UAAU,CAAC,QAAQ,MAAM,GAAG,YAAY,EAAE;mBAE9B,UAAU,CAAC,QAAQ,WAAW,GAAG,iCAAiC,EAAE,QACxF,UAAU,OAAO,6BAA6B,kBAAkB,GAAG,IACnE,IACJ;AACF;;;;;;;;AASA,eAAsB,UACpB,QACA,KACe;CAGf,MAAM,MAAM,cAAc;EAAC;EAAK,yBAAyB;CAAI,CAAC;CAC9D,MAAM,KAAK,iBAAiB,KAAK,QAAQ;CAGzC,MAAM,SAAS,oBAAoB,IAAI,KAAK,KAAK,MADrB,UAAU,EAAE,CACsB;CAC9D,MAAM,YAAY,IAAI,eAAe,IAAI,UAAU,OAAO;CAE1D,MAAM,EAAC,MAAM,oBAAoB,aAAa,qBAAoB;CAClE,MAAM,gBAAgB,iBAAiB,MAAM;CAC7C,GAAG,OAAO,kBAAkB,CAAC,gBAAgB,KAAK,IAAI,gBAAgB,IAAI;EACxE,iBAAA;EACA,QAAQ,OAAO;EACf,KAAK,OAAO;EACZ,OAAO,OAAO;EACd,MAAM,OAAO;CACf,CAAC;CAED,MAAM,aAAa,IAAI,eACrB,QACA,IACA;EAAC;EAAM;CAAkB,GACzB,sBACM,oBAAoB,GAAG,CAC/B;CACA,MAAM,YAAY,IAAI,UAAU,IAAI,UAAU,SAAS;EACrD,SAAS;EACT,mBAAmB,WAAW,iBAAiB;CACjD,CAAC;CAED,IAAI;CACJ,SAAS,iBAAkC;EACzC,IAAI,cAAc,KAAA,GAAW;GAC3B,MAAM,UAAU,YAAY,IAAI;GAChC,GAAG,OAAO,qBAAqB;GAE/B,MAAM,IAAK,YAAY,SAAiB;GACxC,MAAM,IAAI,YAAY,UAAU,GAAG,EAChC,KAAK,iBAAiB;IACrB,EAAE,QAAQ,CAAC;IACX,GAAG,OAAO,qBAAqB,YAAY,IAAI,IAAI,QAAQ,KAAK;GAClE,CAAC,EACA,KAAK,SAAS,EAAE,MAAM;GACzB,EAAE,cAAqC,wBACrC,UAAU,QAAQ,CACpB;GAEA,UAAU,UAAU,GAAG,eAAe,YAAY;EACpD;EACA,OAAO,UAAU;CACnB;CAGA,IAAI,CAAC,aACH,eAAoB;CAGtB,MAAM,UAAU,gBAAgB;CAChC,QAAQ,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAErC,IAAI;EACF,MAAM,eAAe,IAAI,UAAU,SAAS,UAAU;CACxD,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,MAAM;CACvC,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF"}
+{"version":3,"file":"run-worker.js","names":[],"sources":["../../../../../../zero-cache/src/server/runner/run-worker.ts"],"sourcesContent":["import '../../../../shared/src/dotenv.ts';\n\nimport {styleText} from 'node:util';\nimport {colorConsole} from '../../../../shared/src/logging.ts';\nimport {resolver, type Resolver} from '../../../../shared/src/resolver.ts';\nimport {PROTOCOL_VERSION} from '../../../../zero-protocol/src/protocol-version.ts';\nimport {normalizeZeroConfig} from '../../config/normalize.ts';\nimport {getServerVersion, getZeroConfig} from '../../config/zero-config.ts';\nimport {\n  ProcessManager,\n  runUntilKilled,\n  SelfDrain,\n} from '../../services/life-cycle.ts';\nimport {\n  childWorker,\n  type DrainRequestedMessage,\n  type Worker,\n} from '../../types/processes.ts';\nimport {createLogContext} from '../logging.ts';\nimport {MAIN_URL} from '../worker-urls.ts';\nimport {getTaskID} from './runtime.ts';\nimport {ZeroDispatcher} from './zero-dispatcher.ts';\n\nconst startupMessageEnv = 'ZERO_ENABLE_STARTUP_MESSAGE';\n\nfunction printStartupMessage(env: NodeJS.ProcessEnv) {\n  if (env[startupMessageEnv] !== '1') {\n    return;\n  }\n\n  colorConsole.log(\n    `\\nBTW, ${styleText(['bold', 'cyan'], 'Cloud Zero')} ` +\n      'is now available - professional Zero hosting from the team that built it.\\n' +\n      `Get started now: ${styleText(['blue', 'underline'], 'https://zero.rocicorp.dev/cloud')}\\n\\n` +\n      styleText('dim', `Disable this message with ${startupMessageEnv}=0`) +\n      '\\n',\n  );\n}\n\n/**\n * Top-level `runner` entry point to the zero-cache. This layer is responsible for:\n * * runtime-based config normalization\n * * lazy startup\n * * serving /statsz\n * * auto-reset restarts (TODO)\n */\nexport async function runWorker(\n  parent: Worker | null,\n  env: NodeJS.ProcessEnv,\n): Promise<void> {\n  // Note: Deprecation warnings are only emitted at this top-level parse;\n  //       they are suppressed when parsed in subprocesses.\n  const cfg = getZeroConfig({env, emitDeprecationWarnings: true});\n  const lc = createLogContext(cfg, 'runner');\n\n  const defaultTaskID = await getTaskID(lc);\n  const config = normalizeZeroConfig(lc, cfg, env, defaultTaskID);\n  const processes = new ProcessManager(lc, parent ?? process);\n\n  const {port, keepaliveTimeoutMs, lazyStartup, selfDrainDelayMs} = config;\n  const serverVersion = getServerVersion(config);\n  lc.info?.(`starting server${!serverVersion ? '' : `@${serverVersion}`} `, {\n    protocolVersion: PROTOCOL_VERSION,\n    taskID: config.taskID,\n    app: config.app,\n    shard: config.shard,\n    port: config.port,\n  });\n\n  const dispatcher = new ZeroDispatcher(\n    config,\n    lc,\n    {port, keepaliveTimeoutMs},\n    startZeroCache,\n    () => printStartupMessage(env),\n    // Keep the listener open until the zero-cache has drained and exited.\n    () => processes.shutdown(),\n  );\n  const selfDrain = new SelfDrain(lc, parent ?? process, {\n    delayMs: selfDrainDelayMs,\n    onRequested: () => dispatcher.failHealthChecks(),\n  });\n\n  let zeroCache: Resolver<Worker> | undefined;\n  function startZeroCache(): Promise<Worker> {\n    if (zeroCache === undefined) {\n      const startMs = performance.now();\n      lc.info?.('starting zero-cache');\n\n      const r = (zeroCache = resolver<Worker>());\n      const w = childWorker(MAIN_URL, env)\n        .once('message', () => {\n          r.resolve(w);\n          lc.info?.(`zero-cache ready (${performance.now() - startMs} ms)`);\n        })\n        .once('error', r.reject);\n      w.onMessageType<DrainRequestedMessage>('drainRequested', () =>\n        selfDrain.request(),\n      );\n\n      processes.addWorker(w, 'user-facing', 'zero-cache');\n    }\n    return zeroCache.promise;\n  }\n\n  // Eagerly start the zero-cache if it was not configured with --lazy-startup.\n  if (!lazyStartup) {\n    void startZeroCache();\n  }\n\n  await processes.allWorkersReady();\n  parent?.send(['ready', {ready: true}]);\n\n  try {\n    await runUntilKilled(lc, parent ?? process, dispatcher);\n  } catch (err) {\n    processes.logErrorAndExit(err, 'main');\n  } finally {\n    await processes.shutdown();\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAuBA,IAAM,oBAAoB;AAE1B,SAAS,oBAAoB,KAAwB;CACnD,IAAI,IAAI,uBAAuB,KAC7B;CAGF,aAAa,IACX,UAAU,UAAU,CAAC,QAAQ,MAAM,GAAG,YAAY,EAAE;mBAE9B,UAAU,CAAC,QAAQ,WAAW,GAAG,iCAAiC,EAAE,QACxF,UAAU,OAAO,6BAA6B,kBAAkB,GAAG,IACnE,IACJ;AACF;;;;;;;;AASA,eAAsB,UACpB,QACA,KACe;CAGf,MAAM,MAAM,cAAc;EAAC;EAAK,yBAAyB;CAAI,CAAC;CAC9D,MAAM,KAAK,iBAAiB,KAAK,QAAQ;CAEzC,MAAM,gBAAgB,MAAM,UAAU,EAAE;CACxC,MAAM,SAAS,oBAAoB,IAAI,KAAK,KAAK,aAAa;CAC9D,MAAM,YAAY,IAAI,eAAe,IAAI,UAAU,OAAO;CAE1D,MAAM,EAAC,MAAM,oBAAoB,aAAa,qBAAoB;CAClE,MAAM,gBAAgB,iBAAiB,MAAM;CAC7C,GAAG,OAAO,kBAAkB,CAAC,gBAAgB,KAAK,IAAI,gBAAgB,IAAI;EACxE,iBAAA;EACA,QAAQ,OAAO;EACf,KAAK,OAAO;EACZ,OAAO,OAAO;EACd,MAAM,OAAO;CACf,CAAC;CAED,MAAM,aAAa,IAAI,eACrB,QACA,IACA;EAAC;EAAM;CAAkB,GACzB,sBACM,oBAAoB,GAAG,SAEvB,UAAU,SAAS,CAC3B;CACA,MAAM,YAAY,IAAI,UAAU,IAAI,UAAU,SAAS;EACrD,SAAS;EACT,mBAAmB,WAAW,iBAAiB;CACjD,CAAC;CAED,IAAI;CACJ,SAAS,iBAAkC;EACzC,IAAI,cAAc,KAAA,GAAW;GAC3B,MAAM,UAAU,YAAY,IAAI;GAChC,GAAG,OAAO,qBAAqB;GAE/B,MAAM,IAAK,YAAY,SAAiB;GACxC,MAAM,IAAI,YAAY,UAAU,GAAG,CAAC,CACjC,KAAK,iBAAiB;IACrB,EAAE,QAAQ,CAAC;IACX,GAAG,OAAO,qBAAqB,YAAY,IAAI,IAAI,QAAQ,KAAK;GAClE,CAAC,CAAC,CACD,KAAK,SAAS,EAAE,MAAM;GACzB,EAAE,cAAqC,wBACrC,UAAU,QAAQ,CACpB;GAEA,UAAU,UAAU,GAAG,eAAe,YAAY;EACpD;EACA,OAAO,UAAU;CACnB;CAGA,IAAI,CAAC,aACH,eAAoB;CAGtB,MAAM,UAAU,gBAAgB;CAChC,QAAQ,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAErC,IAAI;EACF,MAAM,eAAe,IAAI,UAAU,SAAS,UAAU;CACxD,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,MAAM;CACvC,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF"}

package/out/zero-cache/src/server/runner/runtime.js

@@ -12,7 +12,8 @@ async function getTaskID(lc) {
 			lc.warn?.("unable to lookup container metadata", e);
 		}
 		try {
-			const metadata = parse(await (await fetch(`${containerURI}/task`)).json(), containerMetadataSchema, "passthrough");
+			const resp = await fetch(`${containerURI}/task`);
+			const metadata = parse(await resp.json(), containerMetadataSchema, "passthrough");
 			lc.info?.(`Task metadata`, { metadata });
 			const { TaskARN: taskID } = metadata;
 			const lastSlash = taskID.lastIndexOf("/");

package/out/zero-cache/src/server/runner/runtime.js.map

@@ -1 +1 @@
-{"version":3,"file":"runtime.js","names":[],"sources":["../../../../../../zero-cache/src/server/runner/runtime.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {nanoid} from 'nanoid';\nimport * as v from '../../../../shared/src/valita.ts';\n\n// https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint-v4-response.html\nconst containerMetadataSchema = v.object({['TaskARN']: v.string()});\n\nexport async function getTaskID(lc: LogContext) {\n  const containerURI = process.env['ECS_CONTAINER_METADATA_URI_V4'];\n  if (containerURI) {\n    try {\n      const resp = await fetch(`${containerURI}`);\n      const metadata = await resp.json();\n      // Logged purely for debugging.\n      lc.info?.(`Container metadata`, {metadata});\n    } catch (e) {\n      lc.warn?.('unable to lookup container metadata', e);\n    }\n\n    try {\n      const resp = await fetch(`${containerURI}/task`);\n      const metadata = v.parse(\n        await resp.json(),\n        containerMetadataSchema,\n        'passthrough',\n      );\n      lc.info?.(`Task metadata`, {metadata});\n      const {TaskARN: taskID} = metadata;\n      // Task ARN's are long, e.g.\n      // \"arn:aws:ecs:us-east-1:712907626835:task/zbugs-prod-Cluster-vvNFcPUVpGHr/0042ea25bf534dc19975e26f61441737\"\n      // We only care about the unique ID, i.e. the last path component.\n      const lastSlash = taskID.lastIndexOf('/');\n      return taskID.substring(lastSlash + 1); // works for lastSlash === -1 too\n    } catch (e) {\n      lc.warn?.('unable to determine task ID. falling back to random ID', e);\n    }\n  }\n  return nanoid();\n}\n"],"mappings":";;;AAKA,IAAM,0BAA0B,eAAE,OAAO,GAAE,YAAY,eAAE,OAAO,EAAC,CAAC;AAElE,eAAsB,UAAU,IAAgB;CAC9C,MAAM,eAAe,QAAQ,IAAI;CACjC,IAAI,cAAc;EAChB,IAAI;GAEF,MAAM,WAAW,OAAM,MADJ,MAAM,GAAG,cAAc,GACd,KAAK;GAEjC,GAAG,OAAO,sBAAsB,EAAC,SAAQ,CAAC;EAC5C,SAAS,GAAG;GACV,GAAG,OAAO,uCAAuC,CAAC;EACpD;EAEA,IAAI;GAEF,MAAM,WAAW,MACf,OAAM,MAFW,MAAM,GAAG,aAAa,MAAM,GAElC,KAAK,GAChB,yBACA,aACF;GACA,GAAG,OAAO,iBAAiB,EAAC,SAAQ,CAAC;GACrC,MAAM,EAAC,SAAS,WAAU;GAI1B,MAAM,YAAY,OAAO,YAAY,GAAG;GACxC,OAAO,OAAO,UAAU,YAAY,CAAC;EACvC,SAAS,GAAG;GACV,GAAG,OAAO,0DAA0D,CAAC;EACvE;CACF;CACA,OAAO,OAAO;AAChB"}
+{"version":3,"file":"runtime.js","names":["v","v.parse"],"sources":["../../../../../../zero-cache/src/server/runner/runtime.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {nanoid} from 'nanoid';\nimport * as v from '../../../../shared/src/valita.ts';\n\n// https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint-v4-response.html\nconst containerMetadataSchema = v.object({['TaskARN']: v.string()});\n\nexport async function getTaskID(lc: LogContext) {\n  const containerURI = process.env['ECS_CONTAINER_METADATA_URI_V4'];\n  if (containerURI) {\n    try {\n      const resp = await fetch(`${containerURI}`);\n      const metadata = await resp.json();\n      // Logged purely for debugging.\n      lc.info?.(`Container metadata`, {metadata});\n    } catch (e) {\n      lc.warn?.('unable to lookup container metadata', e);\n    }\n\n    try {\n      const resp = await fetch(`${containerURI}/task`);\n      const metadata = v.parse(\n        await resp.json(),\n        containerMetadataSchema,\n        'passthrough',\n      );\n      lc.info?.(`Task metadata`, {metadata});\n      const {TaskARN: taskID} = metadata;\n      // Task ARN's are long, e.g.\n      // \"arn:aws:ecs:us-east-1:712907626835:task/zbugs-prod-Cluster-vvNFcPUVpGHr/0042ea25bf534dc19975e26f61441737\"\n      // We only care about the unique ID, i.e. the last path component.\n      const lastSlash = taskID.lastIndexOf('/');\n      return taskID.substring(lastSlash + 1); // works for lastSlash === -1 too\n    } catch (e) {\n      lc.warn?.('unable to determine task ID. falling back to random ID', e);\n    }\n  }\n  return nanoid();\n}\n"],"mappings":";;;AAKA,IAAM,0BAA0BA,eAAE,OAAO,GAAE,YAAYA,eAAE,OAAO,EAAC,CAAC;AAElE,eAAsB,UAAU,IAAgB;CAC9C,MAAM,eAAe,QAAQ,IAAI;CACjC,IAAI,cAAc;EAChB,IAAI;GAEF,MAAM,WAAW,OAAM,MADJ,MAAM,GAAG,cAAc,EAAA,CACd,KAAK;GAEjC,GAAG,OAAO,sBAAsB,EAAC,SAAQ,CAAC;EAC5C,SAAS,GAAG;GACV,GAAG,OAAO,uCAAuC,CAAC;EACpD;EAEA,IAAI;GACF,MAAM,OAAO,MAAM,MAAM,GAAG,aAAa,MAAM;GAC/C,MAAM,WAAWC,MACf,MAAM,KAAK,KAAK,GAChB,yBACA,aACF;GACA,GAAG,OAAO,iBAAiB,EAAC,SAAQ,CAAC;GACrC,MAAM,EAAC,SAAS,WAAU;GAI1B,MAAM,YAAY,OAAO,YAAY,GAAG;GACxC,OAAO,OAAO,UAAU,YAAY,CAAC;EACvC,SAAS,GAAG;GACV,GAAG,OAAO,0DAA0D,CAAC;EACvE;CACF;CACA,OAAO,OAAO;AAChB"}

package/out/zero-cache/src/server/runner/zero-dispatcher.d.ts

@@ -5,7 +5,15 @@ import type { Worker } from '../../types/processes.ts';
 export declare class ZeroDispatcher extends HttpService {
     #private;
     readonly id = "zero-dispatcher";
-    constructor(config: NormalizedZeroConfig, lc: LogContext, opts: Options, getWorker: () => Promise<Worker>, onStart?: () => void);
+    /**
+     * @param whenDrained Resolves when the workers behind this dispatcher have
+     *        drained (exited). The listener stays open until then, rehoming new
+     *        sync connections, so that clients that are still routed here (e.g.
+     *        by a sticky cookie) are not refused while the load balancer
+     *        catches up.
+     */
+    constructor(config: NormalizedZeroConfig, lc: LogContext, opts: Options, getWorker: () => Promise<Worker>, onStart?: () => void, whenDrained?: () => Promise<void>);
+    drain(): Promise<void>;
     protected _onStart(): void;
 }
 //# sourceMappingURL=zero-dispatcher.d.ts.map

package/out/zero-cache/src/server/runner/zero-dispatcher.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"zero-dispatcher.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/server/runner/zero-dispatcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAEpE,OAAO,EAAC,WAAW,EAAE,KAAK,OAAO,EAAC,MAAM,gCAAgC,CAAC;AAQzE,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAMrD,qBAAa,cAAe,SAAQ,WAAW;;IAC7C,QAAQ,CAAC,EAAE,qBAAqB;IAIhC,YACE,MAAM,EAAE,oBAAoB,EAC5B,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,SAAS,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,EAChC,OAAO,CAAC,EAAE,MAAM,IAAI,EAsBrB;IAED,UAAmB,QAAQ,SAE1B;CAYF"}
+{"version":3,"file":"zero-dispatcher.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/server/runner/zero-dispatcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAEpE,OAAO,EAAC,WAAW,EAAE,KAAK,OAAO,EAAC,MAAM,gCAAgC,CAAC;AAQzE,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AASrD,qBAAa,cAAe,SAAQ,WAAW;;IAC7C,QAAQ,CAAC,EAAE,qBAAqB;IAOhC;;;;;;OAMG;IACH,YACE,MAAM,EAAE,oBAAoB,EAC5B,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,SAAS,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,EAChC,OAAO,CAAC,EAAE,MAAM,IAAI,EACpB,WAAW,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,EAyBlC;IAEK,KAAK,kBAIV;IAED,UAAmB,QAAQ,SAE1B;CAuBF"}

package/out/zero-cache/src/server/runner/zero-dispatcher.js

@@ -4,12 +4,24 @@ import { handlePlannerzRequest } from "../../services/plannerz.js";
 import { handleProfrmzRequest, handleProfzRequest } from "../../services/profz.js";
 import { handleStatzRequest } from "../../services/statz.js";
 import { installWebSocketHandoff } from "../../types/websocket-handoff.js";
+import { expiredAffinityCookieHeader, rehome } from "../../workers/rehome.js";
+import { parsePath } from "../worker-dispatcher.js";
 //#region ../zero-cache/src/server/runner/zero-dispatcher.ts
 var ZeroDispatcher = class extends HttpService {
 	id = "zero-dispatcher";
 	#getWorker;
 	#onStart;
-	constructor(config, lc, opts, getWorker, onStart) {
+	#whenDrained;
+	#drainingHeaders;
+	#draining = false;
+	/**
+	* @param whenDrained Resolves when the workers behind this dispatcher have
+	*        drained (exited). The listener stays open until then, rehoming new
+	*        sync connections, so that clients that are still routed here (e.g.
+	*        by a sticky cookie) are not refused while the load balancer
+	*        catches up.
+	*/
+	constructor(config, lc, opts, getWorker, onStart, whenDrained) {
 		super(`zero-dispatcher`, lc, opts, (fastify) => {
 			fastify.get("/statz", (req, res) => handleStatzRequest(lc, config, req, res));
 			fastify.get("/plannerz", (req, res) => handlePlannerzRequest(lc, config, req, res));
@@ -20,16 +32,31 @@ var ZeroDispatcher = class extends HttpService {
 		});
 		this.#getWorker = getWorker;
 		this.#onStart = onStart;
+		this.#whenDrained = whenDrained;
+		const expiredCookie = expiredAffinityCookieHeader(config);
+		this.#drainingHeaders = expiredCookie ? [expiredCookie] : [];
+	}
+	async drain() {
+		this.#draining = true;
+		this.failHealthChecks();
+		await this.#whenDrained?.();
 	}
 	_onStart() {
 		this.#onStart?.();
 	}
-	#handoff = (_req, dispatch, onError) => {
+	#handoff = (req, dispatch, onError, accept) => {
+		if (this.#draining && this.#isSync(req)) {
+			accept((ws) => rehome(this._lc, ws), this.#drainingHeaders);
+			return;
+		}
 		this.#getWorker().then((sender) => dispatch({
 			payload: "unused",
 			sender
 		}), onError);
 	};
+	#isSync({ url }) {
+		return parsePath(new URL(url ?? "", "http://unused/"))?.worker === "sync";
+	}
 };
 //#endregion
 export { ZeroDispatcher };

package/out/zero-cache/src/server/runner/zero-dispatcher.js.map

@@ -1 +1 @@
-{"version":3,"file":"zero-dispatcher.js","names":["#getWorker","#onStart","#handoff"],"sources":["../../../../../../zero-cache/src/server/runner/zero-dispatcher.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport {handleHeapzRequest} from '../../services/heapz.ts';\nimport {HttpService, type Options} from '../../services/http-service.ts';\nimport {handlePlannerzRequest} from '../../services/plannerz.ts';\nimport {\n  handleProfrmzRequest,\n  handleProfzRequest,\n} from '../../services/profz.ts';\nimport {handleStatzRequest} from '../../services/statz.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport type {Worker} from '../../types/processes.ts';\nimport {\n  installWebSocketHandoff,\n  type HandoffSpec,\n} from '../../types/websocket-handoff.ts';\n\nexport class ZeroDispatcher extends HttpService {\n  readonly id = 'zero-dispatcher';\n  readonly #getWorker: () => Promise<Worker>;\n  readonly #onStart: (() => void) | undefined;\n\n  constructor(\n    config: NormalizedZeroConfig,\n    lc: LogContext,\n    opts: Options,\n    getWorker: () => Promise<Worker>,\n    onStart?: () => void,\n  ) {\n    super(`zero-dispatcher`, lc, opts, fastify => {\n      fastify.get('/statz', (req, res) =>\n        handleStatzRequest(lc, config, req, res),\n      );\n      fastify.get('/plannerz', (req, res) =>\n        handlePlannerzRequest(lc, config, req, res),\n      );\n      fastify.get('/heapz', (req, res) =>\n        handleHeapzRequest(lc, config, req, res),\n      );\n      fastify.get('/profz', (req, res) =>\n        handleProfzRequest(lc, config, req, res, this.#getWorker),\n      );\n      fastify.get('/profrmz', (req, res) =>\n        handleProfrmzRequest(lc, config, req, res, this.#getWorker),\n      );\n      installWebSocketHandoff(lc, this.#handoff, fastify.server);\n    });\n    this.#getWorker = getWorker;\n    this.#onStart = onStart;\n  }\n\n  protected override _onStart() {\n    this.#onStart?.();\n  }\n\n  readonly #handoff = (\n    _req: IncomingMessageSubset,\n    dispatch: (h: HandoffSpec<string>) => void,\n    onError: (error: unknown) => void,\n  ) => {\n    void this.#getWorker().then(\n      sender => dispatch({payload: 'unused', sender}),\n      onError,\n    );\n  };\n}\n"],"mappings":";;;;;;;AAiBA,IAAa,iBAAb,cAAoC,YAAY;CAC9C,KAAc;CACd;CACA;CAEA,YACE,QACA,IACA,MACA,WACA,SACA;EACA,MAAM,mBAAmB,IAAI,OAAM,YAAW;GAC5C,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBAAmB,IAAI,QAAQ,KAAK,GAAG,CACzC;GACA,QAAQ,IAAI,cAAc,KAAK,QAC7B,sBAAsB,IAAI,QAAQ,KAAK,GAAG,CAC5C;GACA,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBAAmB,IAAI,QAAQ,KAAK,GAAG,CACzC;GACA,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBAAmB,IAAI,QAAQ,KAAK,KAAK,KAAKA,UAAU,CAC1D;GACA,QAAQ,IAAI,aAAa,KAAK,QAC5B,qBAAqB,IAAI,QAAQ,KAAK,KAAK,KAAKA,UAAU,CAC5D;GACA,wBAAwB,IAAI,KAAKE,UAAU,QAAQ,MAAM;EAC3D,CAAC;EACD,KAAKF,aAAa;EAClB,KAAKC,WAAW;CAClB;CAEA,WAA8B;EAC5B,KAAKA,WAAW;CAClB;CAEA,YACE,MACA,UACA,YACG;EACH,KAAUD,WAAW,EAAE,MACrB,WAAU,SAAS;GAAC,SAAS;GAAU;EAAM,CAAC,GAC9C,OACF;CACF;AACF"}
+{"version":3,"file":"zero-dispatcher.js","names":[],"sources":["../../../../../../zero-cache/src/server/runner/zero-dispatcher.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport {handleHeapzRequest} from '../../services/heapz.ts';\nimport {HttpService, type Options} from '../../services/http-service.ts';\nimport {handlePlannerzRequest} from '../../services/plannerz.ts';\nimport {\n  handleProfrmzRequest,\n  handleProfzRequest,\n} from '../../services/profz.ts';\nimport {handleStatzRequest} from '../../services/statz.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport type {Worker} from '../../types/processes.ts';\nimport {\n  installWebSocketHandoff,\n  type HandoffSpec,\n  type WebSocketHandoff,\n} from '../../types/websocket-handoff.ts';\nimport {expiredAffinityCookieHeader, rehome} from '../../workers/rehome.ts';\nimport {parsePath} from '../worker-dispatcher.ts';\n\nexport class ZeroDispatcher extends HttpService {\n  readonly id = 'zero-dispatcher';\n  readonly #getWorker: () => Promise<Worker>;\n  readonly #onStart: (() => void) | undefined;\n  readonly #whenDrained: (() => Promise<void>) | undefined;\n  readonly #drainingHeaders: string[];\n  #draining = false;\n\n  /**\n   * @param whenDrained Resolves when the workers behind this dispatcher have\n   *        drained (exited). The listener stays open until then, rehoming new\n   *        sync connections, so that clients that are still routed here (e.g.\n   *        by a sticky cookie) are not refused while the load balancer\n   *        catches up.\n   */\n  constructor(\n    config: NormalizedZeroConfig,\n    lc: LogContext,\n    opts: Options,\n    getWorker: () => Promise<Worker>,\n    onStart?: () => void,\n    whenDrained?: () => Promise<void>,\n  ) {\n    super(`zero-dispatcher`, lc, opts, fastify => {\n      fastify.get('/statz', (req, res) =>\n        handleStatzRequest(lc, config, req, res),\n      );\n      fastify.get('/plannerz', (req, res) =>\n        handlePlannerzRequest(lc, config, req, res),\n      );\n      fastify.get('/heapz', (req, res) =>\n        handleHeapzRequest(lc, config, req, res),\n      );\n      fastify.get('/profz', (req, res) =>\n        handleProfzRequest(lc, config, req, res, this.#getWorker),\n      );\n      fastify.get('/profrmz', (req, res) =>\n        handleProfrmzRequest(lc, config, req, res, this.#getWorker),\n      );\n      installWebSocketHandoff(lc, this.#handoff, fastify.server);\n    });\n    this.#getWorker = getWorker;\n    this.#onStart = onStart;\n    this.#whenDrained = whenDrained;\n    const expiredCookie = expiredAffinityCookieHeader(config);\n    this.#drainingHeaders = expiredCookie ? [expiredCookie] : [];\n  }\n\n  async drain() {\n    this.#draining = true;\n    this.failHealthChecks();\n    await this.#whenDrained?.();\n  }\n\n  protected override _onStart() {\n    this.#onStart?.();\n  }\n\n  readonly #handoff: WebSocketHandoff<string> = (\n    req: IncomingMessageSubset,\n    dispatch: (h: HandoffSpec<string>) => void,\n    onError: (error: unknown) => void,\n    accept,\n  ) => {\n    if (this.#draining && this.#isSync(req)) {\n      // Respond here rather than handing off to a sync worker, which may\n      // have already exited after draining its own clients.\n      accept(ws => rehome(this._lc, ws), this.#drainingHeaders);\n      return;\n    }\n    void this.#getWorker().then(\n      sender => dispatch({payload: 'unused', sender}),\n      onError,\n    );\n  };\n\n  #isSync({url}: IncomingMessageSubset) {\n    return parsePath(new URL(url ?? '', 'http://unused/'))?.worker === 'sync';\n  }\n}\n"],"mappings":";;;;;;;;;AAoBA,IAAa,iBAAb,cAAoC,YAAY;CAC9C,KAAc;CACd;CACA;CACA;CACA;CACA,YAAY;;;;;;;;CASZ,YACE,QACA,IACA,MACA,WACA,SACA,aACA;EACA,MAAM,mBAAmB,IAAI,OAAM,YAAW;GAC5C,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBAAmB,IAAI,QAAQ,KAAK,GAAG,CACzC;GACA,QAAQ,IAAI,cAAc,KAAK,QAC7B,sBAAsB,IAAI,QAAQ,KAAK,GAAG,CAC5C;GACA,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBAAmB,IAAI,QAAQ,KAAK,GAAG,CACzC;GACA,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBAAmB,IAAI,QAAQ,KAAK,KAAK,KAAK,UAAU,CAC1D;GACA,QAAQ,IAAI,aAAa,KAAK,QAC5B,qBAAqB,IAAI,QAAQ,KAAK,KAAK,KAAK,UAAU,CAC5D;GACA,wBAAwB,IAAI,KAAK,UAAU,QAAQ,MAAM;EAC3D,CAAC;EACD,KAAK,aAAa;EAClB,KAAK,WAAW;EAChB,KAAK,eAAe;EACpB,MAAM,gBAAgB,4BAA4B,MAAM;EACxD,KAAK,mBAAmB,gBAAgB,CAAC,aAAa,IAAI,CAAC;CAC7D;CAEA,MAAM,QAAQ;EACZ,KAAK,YAAY;EACjB,KAAK,iBAAiB;EACtB,MAAM,KAAK,eAAe;CAC5B;CAEA,WAA8B;EAC5B,KAAK,WAAW;CAClB;CAEA,YACE,KACA,UACA,SACA,WACG;EACH,IAAI,KAAK,aAAa,KAAK,QAAQ,GAAG,GAAG;GAGvC,QAAO,OAAM,OAAO,KAAK,KAAK,EAAE,GAAG,KAAK,gBAAgB;GACxD;EACF;EACA,KAAU,WAAW,CAAC,CAAC,MACrB,WAAU,SAAS;GAAC,SAAS;GAAU;EAAM,CAAC,GAC9C,OACF;CACF;CAEA,QAAQ,EAAC,OAA6B;EACpC,OAAO,UAAU,IAAI,IAAI,OAAO,IAAI,gBAAgB,CAAC,CAAC,EAAE,WAAW;CACrE;AACF"}

package/out/zero-cache/src/server/shadow-syncer.js

@@ -10,7 +10,7 @@ import { startOtelAuto } from "./otel-start.js";
 import { ShadowSyncService } from "../services/shadow-sync/shadow-sync-service.js";
 import { LogContext, consoleLogSink } from "@rocicorp/logger";
 //#region ../zero-cache/src/server/shadow-syncer.ts
-var MS_PER_HOUR = 1e3 * 60 * 60;
+var MS_PER_HOUR = 36e5;
 var lc = new LogContext("info", {}, consoleLogSink);
 function runWorker(parent, env, ...argv) {
 	const config = getNormalizedZeroConfig({

package/out/zero-cache/src/server/shadow-syncer.js.map

@@ -1 +1 @@
-{"version":3,"file":"shadow-syncer.js","names":[],"sources":["../../../../../zero-cache/src/server/shadow-syncer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {must} from '../../../shared/src/must.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {ShadowSyncService} from '../services/shadow-sync/shadow-sync-service.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\nconst MS_PER_HOUR = 1000 * 60 * 60;\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  const config = getNormalizedZeroConfig({env, argv});\n\n  startOtelAuto(\n    createLogContext(config, 'shadow-syncer', 0, false),\n    'shadow-syncer',\n    0,\n  );\n  lc = createLogContext(config, 'shadow-syncer');\n  initEventSink(lc, config);\n\n  const {shadowSync, upstream, initialSync} = config;\n  const shard = getShardConfig(config);\n  const service = new ShadowSyncService(\n    lc,\n    shard,\n    upstream.db,\n    getServerContext(config),\n    {\n      intervalMs: shadowSync.intervalHours * MS_PER_HOUR,\n      sampleRate: shadowSync.sampleRate,\n      maxRowsPerTable: shadowSync.maxRowsPerTable,\n      textCopy: initialSync.textCopy,\n    },\n  );\n\n  parent.send(['ready', {ready: true}]);\n\n  return runUntilKilled(lc, parent, service);\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;AAgBA,IAAM,cAAc,MAAO,KAAK;AAGhC,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,SAAwB,UACtB,QACA,KACA,GAAG,MACY;CACf,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAElD,cACE,iBAAiB,QAAQ,iBAAiB,GAAG,KAAK,GAClD,iBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,eAAe;CAC7C,cAAc,IAAI,MAAM;CAExB,MAAM,EAAC,YAAY,UAAU,gBAAe;CAC5C,MAAM,QAAQ,eAAe,MAAM;CACnC,MAAM,UAAU,IAAI,kBAClB,IACA,OACA,SAAS,IACT,iBAAiB,MAAM,GACvB;EACE,YAAY,WAAW,gBAAgB;EACvC,YAAY,WAAW;EACvB,iBAAiB,WAAW;EAC5B,UAAU,YAAY;CACxB,CACF;CAEA,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAEpC,OAAO,eAAe,IAAI,QAAQ,OAAO;AAC3C;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}
+{"version":3,"file":"shadow-syncer.js","names":[],"sources":["../../../../../zero-cache/src/server/shadow-syncer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {must} from '../../../shared/src/must.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {ShadowSyncService} from '../services/shadow-sync/shadow-sync-service.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\nconst MS_PER_HOUR = 1000 * 60 * 60;\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  const config = getNormalizedZeroConfig({env, argv});\n\n  startOtelAuto(\n    createLogContext(config, 'shadow-syncer', 0, false),\n    'shadow-syncer',\n    0,\n  );\n  lc = createLogContext(config, 'shadow-syncer');\n  initEventSink(lc, config);\n\n  const {shadowSync, upstream, initialSync} = config;\n  const shard = getShardConfig(config);\n  const service = new ShadowSyncService(\n    lc,\n    shard,\n    upstream.db,\n    getServerContext(config),\n    {\n      intervalMs: shadowSync.intervalHours * MS_PER_HOUR,\n      sampleRate: shadowSync.sampleRate,\n      maxRowsPerTable: shadowSync.maxRowsPerTable,\n      textCopy: initialSync.textCopy,\n    },\n  );\n\n  parent.send(['ready', {ready: true}]);\n\n  return runUntilKilled(lc, parent, service);\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;AAgBA,IAAM,cAAc;AAGpB,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,SAAwB,UACtB,QACA,KACA,GAAG,MACY;CACf,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAElD,cACE,iBAAiB,QAAQ,iBAAiB,GAAG,KAAK,GAClD,iBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,eAAe;CAC7C,cAAc,IAAI,MAAM;CAExB,MAAM,EAAC,YAAY,UAAU,gBAAe;CAC5C,MAAM,QAAQ,eAAe,MAAM;CACnC,MAAM,UAAU,IAAI,kBAClB,IACA,OACA,SAAS,IACT,iBAAiB,MAAM,GACvB;EACE,YAAY,WAAW,gBAAgB;EACvC,YAAY,WAAW;EACvB,iBAAiB,WAAW;EAC5B,UAAU,YAAY;CACxB,CACF;CAEA,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAEpC,OAAO,eAAe,IAAI,QAAQ,OAAO;AAC3C;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}

package/out/zero-cache/src/server/slot-keeper.js.map

@@ -1 +1 @@
-{"version":3,"file":"slot-keeper.js","names":[],"sources":["../../../../../zero-cache/src/server/slot-keeper.ts"],"sourcesContent":["import {resolve} from 'node:path';\nimport type {Readable} from 'node:stream';\nimport {fileURLToPath} from 'node:url';\nimport {PG_ADMIN_SHUTDOWN} from '@drdgvhbh/postgres-error-codes';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport {getLogConfig} from '../config/zero-config.ts';\nimport {\n  DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE,\n  makeAck,\n  startReplicationStream,\n  type PgConnectionConfig,\n} from '../services/change-source/pg/logical-replication/stream.ts';\nimport {toBigInt} from '../services/change-source/pg/lsn.ts';\nimport {exitAfter} from '../services/life-cycle.ts';\nimport {inactivityTimeoutSocket, isPostgresError} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {createLogContext} from './logging.ts';\n\nexport type SlotKeeperConfig = {\n  db: PgConnectionConfig;\n  slot: string;\n  dummyPublication: string;\n  lsn: string;\n  ackIntervalMs?: number | undefined;\n};\n\nexport function parseLsn(lsn: string): bigint {\n  return lsn.includes('/') ? toBigInt(lsn) : BigInt(lsn);\n}\n\nexport function waitForStartConfig(parent: Worker): Promise<SlotKeeperConfig> {\n  return new Promise<SlotKeeperConfig>((resolve, reject) => {\n    const cleanup = () => {\n      parent.off('disconnect', onDisconnect);\n      parent.off('close', onClose);\n      if (process.send) {\n        process.off('disconnect', onDisconnect);\n      }\n    };\n    const onDisconnect = () => {\n      cleanup();\n      reject(new Error('Parent disconnected before sending start config'));\n    };\n    const onClose = () => {\n      cleanup();\n      reject(new Error('Parent closed before sending start config'));\n    };\n    parent.once('disconnect', onDisconnect);\n    parent.once('close', onClose);\n    if (process.send) {\n      process.once('disconnect', onDisconnect);\n    }\n    parent.onceMessageType<['start', SlotKeeperConfig]>('start', cfg => {\n      cleanup();\n      resolve(cfg);\n    });\n    parent.send(['init', {}]);\n  });\n}\n\nexport type SlotKeeperDependencies = {\n  createSession?: (\n    config: SlotKeeperConfig,\n    lc: LogContext,\n  ) => postgres.Sql | {end: (opts?: {timeout?: number}) => Promise<void>};\n  startStream?: (\n    lc: LogContext,\n    session: postgres.Sql,\n    slot: string,\n    publications: string[],\n    startLsn: bigint,\n    maxAttempts?: number,\n  ) => Promise<[Readable, NodeJS.WritableStream]>;\n  sleepFn?: (ms: number) => Promise<void>;\n  logContext?: LogContext;\n};\n\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  depsOrArg?: string | SlotKeeperDependencies,\n  ...argv: string[]\n): Promise<void> {\n  // Only logging config is needed; avoid requiring the full (normalized) config.\n  const logConfig = getLogConfig({env, argv});\n  const deps: SlotKeeperDependencies =\n    typeof depsOrArg === 'object' && depsOrArg !== null ? depsOrArg : {};\n  const createSession =\n    deps.createSession ??\n    ((cfg: SlotKeeperConfig, log: LogContext) =>\n      postgres(\n        defu(\n          {\n            max: 1,\n            fetch_types: false,\n            idle_timeout: null,\n            max_lifetime: null as unknown as number,\n            connection: {\n              application_name: `slot-keeper-${cfg.slot}`,\n              replication: 'database',\n            },\n            socket: inactivityTimeoutSocket(log, 0, 60_000),\n          },\n          cfg.db as Options<Record<string, PostgresType>>,\n        ),\n      ));\n  const startStream = deps.startStream ?? startReplicationStream;\n  const sleepFn = deps.sleepFn ?? sleep;\n\n  const config = await waitForStartConfig(parent);\n\n  lc = (\n    deps.logContext ?? createLogContext(logConfig, 'slot-keeper')\n  ).withContext('slot', config.slot);\n  lc.info?.(`starting slot-keeper for ${config.slot}`);\n\n  let stopped = false;\n  let stopResolve: (() => void) | null = null;\n  let currentReadable: Readable | null = null;\n  let currentSession:\n    | postgres.Sql\n    | {end: (opts?: {timeout?: number}) => Promise<void>}\n    | null = null;\n  const lsn = parseLsn(config.lsn);\n  const ackIntervalMs = config.ackIntervalMs ?? 5_000;\n\n  const closeCurrent = async () => {\n    if (currentReadable) {\n      try {\n        currentReadable.destroy();\n      } catch {}\n      currentReadable = null;\n    }\n    if (currentSession) {\n      try {\n        await currentSession.end({timeout: 0});\n        lc.info?.(`ended slot-keeper replication session for ${config.slot}`);\n      } catch (e) {\n        lc.warn?.(`error ending slot-keeper session`, e);\n      }\n      currentSession = null;\n    }\n  };\n\n  const stop = async () => {\n    if (stopped) {\n      return;\n    }\n    stopped = true;\n    lc.info?.(`stopping slot-keeper for ${config.slot}`);\n    stopResolve?.();\n    await closeCurrent();\n  };\n\n  parent.onceMessageType('stop', () => void stop());\n  parent.on('SIGTERM', () => void stop());\n  parent.on('SIGINT', () => void stop());\n  parent.once('disconnect', () => void stop());\n  parent.once('close', () => void stop());\n  process.once('SIGTERM', () => void stop());\n  process.once('SIGINT', () => void stop());\n  if (process.send) {\n    process.once('disconnect', () => void stop());\n  }\n\n  let firstAttempt = true;\n  let delayMs = 1000;\n\n  while (!stopped) {\n    try {\n      lc.info?.(`connecting replication session for slot ${config.slot}`);\n      const session = createSession(config, lc);\n      currentSession = session;\n\n      const [readable, writeable] = await startStream(\n        lc,\n        session as postgres.Sql,\n        config.slot,\n        [config.dummyPublication],\n        lsn,\n        DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE + 1,\n      );\n      currentReadable = readable;\n      // Discard the streamed messages. Otherwise the paused socket backs up the\n      // wal_sender (e.g. with keepalives and logical messages) for the duration\n      // of the initial sync / restore, and a wal_sender blocked on writing to the\n      // client does not exit (and release the slot) when it is terminated by the\n      // takeover; it instead blocks on sending the termination error to the\n      // client, until signaled again.\n      readable.resume();\n\n      if (stopped) {\n        await closeCurrent();\n        break;\n      }\n\n      lc.info?.(`slot-keeper active for ${config.slot} at lsn ${config.lsn}`);\n      if (firstAttempt) {\n        firstAttempt = false;\n        parent.send(['ready', {ready: true}]);\n      }\n\n      // Reset backoff delay on successful connection\n      delayMs = 1000;\n\n      const ackInterval = setInterval(() => {\n        if (stopped) {\n          clearInterval(ackInterval);\n          return;\n        }\n        try {\n          writeable.write(makeAck(lsn));\n        } catch (e) {\n          lc.warn?.(`error writing ACK to replication stream`, e);\n        }\n      }, ackIntervalMs);\n\n      // Wait until stream closes or errors, or worker is stopped\n      await new Promise<void>((resolve, reject) => {\n        const onStop = () => {\n          cleanup();\n          resolve();\n        };\n        const onClose = () => {\n          cleanup();\n          resolve();\n        };\n        const onError = (err: unknown) => {\n          cleanup();\n          reject(err);\n        };\n        function cleanup() {\n          clearInterval(ackInterval);\n          readable.off('close', onClose);\n          readable.off('error', onError);\n        }\n        stopResolve = onStop;\n        readable.once('close', onClose);\n        readable.once('error', onError);\n      });\n\n      await closeCurrent();\n\n      if (stopped) {\n        break;\n      }\n    } catch (e) {\n      await closeCurrent();\n\n      if (stopped) {\n        break;\n      }\n\n      if (\n        isPostgresError(e, PG_ADMIN_SHUTDOWN) ||\n        (e instanceof Error &&\n          (e.message.includes('57P01') ||\n            e.message.includes(\n              'terminating connection due to administrator command',\n            )))\n      ) {\n        lc.info?.(\n          `slot ${config.slot} was taken over (admin shutdown / pg_terminate_backend)`,\n        );\n        break;\n      }\n\n      lc.warn?.(\n        `slot-keeper stream error for ${config.slot}, will reconnect in ${delayMs}ms`,\n        e,\n      );\n\n      if (firstAttempt) {\n        throw e;\n      }\n\n      await Promise.race([\n        sleepFn(delayMs),\n        new Promise<void>(resolve => {\n          stopResolve = resolve;\n        }),\n      ]);\n      delayMs = Math.min(delayMs * 2, 10_000);\n    }\n  }\n\n  lc.info?.(`slot-keeper finished for ${config.slot}`);\n}\n\n// fork()\nconst isMain =\n  typeof process.argv[1] === 'string' &&\n  fileURLToPath(import.meta.url) === resolve(process.argv[1]);\n\nconst parent = parentWorker;\nif (isMain && parent !== null && !singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(parent, process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;AAiCA,SAAgB,SAAS,KAAqB;CAC5C,OAAO,IAAI,SAAS,GAAG,IAAI,SAAS,GAAG,IAAI,OAAO,GAAG;AACvD;AAEA,SAAgB,mBAAmB,QAA2C;CAC5E,OAAO,IAAI,SAA2B,SAAS,WAAW;EACxD,MAAM,gBAAgB;GACpB,OAAO,IAAI,cAAc,YAAY;GACrC,OAAO,IAAI,SAAS,OAAO;GAC3B,IAAI,QAAQ,MACV,QAAQ,IAAI,cAAc,YAAY;EAE1C;EACA,MAAM,qBAAqB;GACzB,QAAQ;GACR,uBAAO,IAAI,MAAM,iDAAiD,CAAC;EACrE;EACA,MAAM,gBAAgB;GACpB,QAAQ;GACR,uBAAO,IAAI,MAAM,2CAA2C,CAAC;EAC/D;EACA,OAAO,KAAK,cAAc,YAAY;EACtC,OAAO,KAAK,SAAS,OAAO;EAC5B,IAAI,QAAQ,MACV,QAAQ,KAAK,cAAc,YAAY;EAEzC,OAAO,gBAA6C,UAAS,QAAO;GAClE,QAAQ;GACR,QAAQ,GAAG;EACb,CAAC;EACD,OAAO,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;CAC1B,CAAC;AACH;AAmBA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,WACA,GAAG,MACY;CAEf,MAAM,YAAY,aAAa;EAAC;EAAK;CAAI,CAAC;CAC1C,MAAM,OACJ,OAAO,cAAc,YAAY,cAAc,OAAO,YAAY,CAAC;CACrE,MAAM,gBACJ,KAAK,mBACH,KAAuB,QACvB,SACE,KACE;EACE,KAAK;EACL,aAAa;EACb,cAAc;EACd,cAAc;EACd,YAAY;GACV,kBAAkB,eAAe,IAAI;GACrC,aAAa;EACf;EACA,QAAQ,wBAAwB,KAAK,GAAG,GAAM;CAChD,GACA,IAAI,EACN,CACF;CACJ,MAAM,cAAc,KAAK,eAAe;CACxC,MAAM,UAAU,KAAK,WAAW;CAEhC,MAAM,SAAS,MAAM,mBAAmB,MAAM;CAE9C,MACE,KAAK,cAAc,iBAAiB,WAAW,aAAa,GAC5D,YAAY,QAAQ,OAAO,IAAI;CACjC,GAAG,OAAO,4BAA4B,OAAO,MAAM;CAEnD,IAAI,UAAU;CACd,IAAI,cAAmC;CACvC,IAAI,kBAAmC;CACvC,IAAI,iBAGO;CACX,MAAM,MAAM,SAAS,OAAO,GAAG;CAC/B,MAAM,gBAAgB,OAAO,iBAAiB;CAE9C,MAAM,eAAe,YAAY;EAC/B,IAAI,iBAAiB;GACnB,IAAI;IACF,gBAAgB,QAAQ;GAC1B,QAAQ,CAAC;GACT,kBAAkB;EACpB;EACA,IAAI,gBAAgB;GAClB,IAAI;IACF,MAAM,eAAe,IAAI,EAAC,SAAS,EAAC,CAAC;IACrC,GAAG,OAAO,6CAA6C,OAAO,MAAM;GACtE,SAAS,GAAG;IACV,GAAG,OAAO,oCAAoC,CAAC;GACjD;GACA,iBAAiB;EACnB;CACF;CAEA,MAAM,OAAO,YAAY;EACvB,IAAI,SACF;EAEF,UAAU;EACV,GAAG,OAAO,4BAA4B,OAAO,MAAM;EACnD,cAAc;EACd,MAAM,aAAa;CACrB;CAEA,OAAO,gBAAgB,cAAc,KAAK,KAAK,CAAC;CAChD,OAAO,GAAG,iBAAiB,KAAK,KAAK,CAAC;CACtC,OAAO,GAAG,gBAAgB,KAAK,KAAK,CAAC;CACrC,OAAO,KAAK,oBAAoB,KAAK,KAAK,CAAC;CAC3C,OAAO,KAAK,eAAe,KAAK,KAAK,CAAC;CACtC,QAAQ,KAAK,iBAAiB,KAAK,KAAK,CAAC;CACzC,QAAQ,KAAK,gBAAgB,KAAK,KAAK,CAAC;CACxC,IAAI,QAAQ,MACV,QAAQ,KAAK,oBAAoB,KAAK,KAAK,CAAC;CAG9C,IAAI,eAAe;CACnB,IAAI,UAAU;CAEd,OAAO,CAAC,SACN,IAAI;EACF,GAAG,OAAO,2CAA2C,OAAO,MAAM;EAClE,MAAM,UAAU,cAAc,QAAQ,EAAE;EACxC,iBAAiB;EAEjB,MAAM,CAAC,UAAU,aAAa,MAAM,YAClC,IACA,SACA,OAAO,MACP,CAAC,OAAO,gBAAgB,GACxB,KAAA,CAEF;EACA,kBAAkB;EAOlB,SAAS,OAAO;EAEhB,IAAI,SAAS;GACX,MAAM,aAAa;GACnB;EACF;EAEA,GAAG,OAAO,0BAA0B,OAAO,KAAK,UAAU,OAAO,KAAK;EACtE,IAAI,cAAc;GAChB,eAAe;GACf,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;EACtC;EAGA,UAAU;EAEV,MAAM,cAAc,kBAAkB;GACpC,IAAI,SAAS;IACX,cAAc,WAAW;IACzB;GACF;GACA,IAAI;IACF,UAAU,MAAM,QAAQ,GAAG,CAAC;GAC9B,SAAS,GAAG;IACV,GAAG,OAAO,2CAA2C,CAAC;GACxD;EACF,GAAG,aAAa;EAGhB,MAAM,IAAI,SAAe,SAAS,WAAW;GAC3C,MAAM,eAAe;IACnB,QAAQ;IACR,QAAQ;GACV;GACA,MAAM,gBAAgB;IACpB,QAAQ;IACR,QAAQ;GACV;GACA,MAAM,WAAW,QAAiB;IAChC,QAAQ;IACR,OAAO,GAAG;GACZ;GACA,SAAS,UAAU;IACjB,cAAc,WAAW;IACzB,SAAS,IAAI,SAAS,OAAO;IAC7B,SAAS,IAAI,SAAS,OAAO;GAC/B;GACA,cAAc;GACd,SAAS,KAAK,SAAS,OAAO;GAC9B,SAAS,KAAK,SAAS,OAAO;EAChC,CAAC;EAED,MAAM,aAAa;EAEnB,IAAI,SACF;CAEJ,SAAS,GAAG;EACV,MAAM,aAAa;EAEnB,IAAI,SACF;EAGF,IACE,gBAAgB,GAAG,iBAAiB,KACnC,aAAa,UACX,EAAE,QAAQ,SAAS,OAAO,KACzB,EAAE,QAAQ,SACR,qDACF,IACJ;GACA,GAAG,OACD,QAAQ,OAAO,KAAK,wDACtB;GACA;EACF;EAEA,GAAG,OACD,gCAAgC,OAAO,KAAK,sBAAsB,QAAQ,KAC1E,CACF;EAEA,IAAI,cACF,MAAM;EAGR,MAAM,QAAQ,KAAK,CACjB,QAAQ,OAAO,GACf,IAAI,SAAc,YAAW;GAC3B,cAAc;EAChB,CAAC,CACH,CAAC;EACD,UAAU,KAAK,IAAI,UAAU,GAAG,GAAM;CACxC;CAGF,GAAG,OAAO,4BAA4B,OAAO,MAAM;AACrD;AAGA,IAAM,SACJ,OAAO,QAAQ,KAAK,OAAO,YAC3B,cAAc,OAAO,KAAK,GAAG,MAAM,QAAQ,QAAQ,KAAK,EAAE;AAE5D,IAAM,SAAS;AACf,IAAI,UAAU,WAAW,QAAQ,CAAC,kBAAkB,GAClD,gBACQ,UACA,UAAU,QAAQ,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC/D"}
+{"version":3,"file":"slot-keeper.js","names":[],"sources":["../../../../../zero-cache/src/server/slot-keeper.ts"],"sourcesContent":["import {resolve} from 'node:path';\nimport type {Readable} from 'node:stream';\nimport {fileURLToPath} from 'node:url';\nimport {PG_ADMIN_SHUTDOWN} from '@drdgvhbh/postgres-error-codes';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport {getLogConfig} from '../config/zero-config.ts';\nimport {\n  DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE,\n  makeAck,\n  startReplicationStream,\n  type PgConnectionConfig,\n} from '../services/change-source/pg/logical-replication/stream.ts';\nimport {toBigInt} from '../services/change-source/pg/lsn.ts';\nimport {exitAfter} from '../services/life-cycle.ts';\nimport {inactivityTimeoutSocket, isPostgresError} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {createLogContext} from './logging.ts';\n\nexport type SlotKeeperConfig = {\n  db: PgConnectionConfig;\n  slot: string;\n  dummyPublication: string;\n  lsn: string;\n  ackIntervalMs?: number | undefined;\n};\n\nexport function parseLsn(lsn: string): bigint {\n  return lsn.includes('/') ? toBigInt(lsn) : BigInt(lsn);\n}\n\nexport function waitForStartConfig(parent: Worker): Promise<SlotKeeperConfig> {\n  return new Promise<SlotKeeperConfig>((resolve, reject) => {\n    const cleanup = () => {\n      parent.off('disconnect', onDisconnect);\n      parent.off('close', onClose);\n      if (process.send) {\n        process.off('disconnect', onDisconnect);\n      }\n    };\n    const onDisconnect = () => {\n      cleanup();\n      reject(new Error('Parent disconnected before sending start config'));\n    };\n    const onClose = () => {\n      cleanup();\n      reject(new Error('Parent closed before sending start config'));\n    };\n    parent.once('disconnect', onDisconnect);\n    parent.once('close', onClose);\n    if (process.send) {\n      process.once('disconnect', onDisconnect);\n    }\n    parent.onceMessageType<['start', SlotKeeperConfig]>('start', cfg => {\n      cleanup();\n      resolve(cfg);\n    });\n    parent.send(['init', {}]);\n  });\n}\n\nexport type SlotKeeperDependencies = {\n  createSession?: (\n    config: SlotKeeperConfig,\n    lc: LogContext,\n  ) => postgres.Sql | {end: (opts?: {timeout?: number}) => Promise<void>};\n  startStream?: (\n    lc: LogContext,\n    session: postgres.Sql,\n    slot: string,\n    publications: string[],\n    startLsn: bigint,\n    maxAttempts?: number,\n  ) => Promise<[Readable, NodeJS.WritableStream]>;\n  sleepFn?: (ms: number) => Promise<void>;\n  logContext?: LogContext;\n};\n\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  depsOrArg?: string | SlotKeeperDependencies,\n  ...argv: string[]\n): Promise<void> {\n  // Only logging config is needed; avoid requiring the full (normalized) config.\n  const logConfig = getLogConfig({env, argv});\n  const deps: SlotKeeperDependencies =\n    typeof depsOrArg === 'object' && depsOrArg !== null ? depsOrArg : {};\n  const createSession =\n    deps.createSession ??\n    ((cfg: SlotKeeperConfig, log: LogContext) =>\n      postgres(\n        defu(\n          {\n            max: 1,\n            fetch_types: false,\n            idle_timeout: null,\n            max_lifetime: null as unknown as number,\n            connection: {\n              application_name: `slot-keeper-${cfg.slot}`,\n              replication: 'database',\n            },\n            socket: inactivityTimeoutSocket(log, 0, 60_000),\n          },\n          cfg.db as Options<Record<string, PostgresType>>,\n        ),\n      ));\n  const startStream = deps.startStream ?? startReplicationStream;\n  const sleepFn = deps.sleepFn ?? sleep;\n\n  const config = await waitForStartConfig(parent);\n\n  lc = (\n    deps.logContext ?? createLogContext(logConfig, 'slot-keeper')\n  ).withContext('slot', config.slot);\n  lc.info?.(`starting slot-keeper for ${config.slot}`);\n\n  let stopped = false;\n  let stopResolve: (() => void) | null = null;\n  let currentReadable: Readable | null = null;\n  let currentSession:\n    | postgres.Sql\n    | {end: (opts?: {timeout?: number}) => Promise<void>}\n    | null = null;\n  const lsn = parseLsn(config.lsn);\n  const ackIntervalMs = config.ackIntervalMs ?? 5_000;\n\n  const closeCurrent = async () => {\n    if (currentReadable) {\n      try {\n        currentReadable.destroy();\n      } catch {}\n      currentReadable = null;\n    }\n    if (currentSession) {\n      try {\n        await currentSession.end({timeout: 0});\n        lc.info?.(`ended slot-keeper replication session for ${config.slot}`);\n      } catch (e) {\n        lc.warn?.(`error ending slot-keeper session`, e);\n      }\n      currentSession = null;\n    }\n  };\n\n  const stop = async () => {\n    if (stopped) {\n      return;\n    }\n    stopped = true;\n    lc.info?.(`stopping slot-keeper for ${config.slot}`);\n    stopResolve?.();\n    await closeCurrent();\n  };\n\n  parent.onceMessageType('stop', () => void stop());\n  parent.on('SIGTERM', () => void stop());\n  parent.on('SIGINT', () => void stop());\n  parent.once('disconnect', () => void stop());\n  parent.once('close', () => void stop());\n  process.once('SIGTERM', () => void stop());\n  process.once('SIGINT', () => void stop());\n  if (process.send) {\n    process.once('disconnect', () => void stop());\n  }\n\n  let firstAttempt = true;\n  let delayMs = 1000;\n\n  while (!stopped) {\n    try {\n      lc.info?.(`connecting replication session for slot ${config.slot}`);\n      const session = createSession(config, lc);\n      currentSession = session;\n\n      const [readable, writeable] = await startStream(\n        lc,\n        session as postgres.Sql,\n        config.slot,\n        [config.dummyPublication],\n        lsn,\n        DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE + 1,\n      );\n      currentReadable = readable;\n      // Discard the streamed messages. Otherwise the paused socket backs up the\n      // wal_sender (e.g. with keepalives and logical messages) for the duration\n      // of the initial sync / restore, and a wal_sender blocked on writing to the\n      // client does not exit (and release the slot) when it is terminated by the\n      // takeover; it instead blocks on sending the termination error to the\n      // client, until signaled again.\n      readable.resume();\n\n      if (stopped) {\n        await closeCurrent();\n        break;\n      }\n\n      lc.info?.(`slot-keeper active for ${config.slot} at lsn ${config.lsn}`);\n      if (firstAttempt) {\n        firstAttempt = false;\n        parent.send(['ready', {ready: true}]);\n      }\n\n      // Reset backoff delay on successful connection\n      delayMs = 1000;\n\n      const ackInterval = setInterval(() => {\n        if (stopped) {\n          clearInterval(ackInterval);\n          return;\n        }\n        try {\n          writeable.write(makeAck(lsn));\n        } catch (e) {\n          lc.warn?.(`error writing ACK to replication stream`, e);\n        }\n      }, ackIntervalMs);\n\n      // Wait until stream closes or errors, or worker is stopped\n      await new Promise<void>((resolve, reject) => {\n        const onStop = () => {\n          cleanup();\n          resolve();\n        };\n        const onClose = () => {\n          cleanup();\n          resolve();\n        };\n        const onError = (err: unknown) => {\n          cleanup();\n          reject(err);\n        };\n        function cleanup() {\n          clearInterval(ackInterval);\n          readable.off('close', onClose);\n          readable.off('error', onError);\n        }\n        stopResolve = onStop;\n        readable.once('close', onClose);\n        readable.once('error', onError);\n      });\n\n      await closeCurrent();\n\n      if (stopped) {\n        break;\n      }\n    } catch (e) {\n      await closeCurrent();\n\n      if (stopped) {\n        break;\n      }\n\n      if (\n        isPostgresError(e, PG_ADMIN_SHUTDOWN) ||\n        (e instanceof Error &&\n          (e.message.includes('57P01') ||\n            e.message.includes(\n              'terminating connection due to administrator command',\n            )))\n      ) {\n        lc.info?.(\n          `slot ${config.slot} was taken over (admin shutdown / pg_terminate_backend)`,\n        );\n        break;\n      }\n\n      lc.warn?.(\n        `slot-keeper stream error for ${config.slot}, will reconnect in ${delayMs}ms`,\n        e,\n      );\n\n      if (firstAttempt) {\n        throw e;\n      }\n\n      await Promise.race([\n        sleepFn(delayMs),\n        new Promise<void>(resolve => {\n          stopResolve = resolve;\n        }),\n      ]);\n      delayMs = Math.min(delayMs * 2, 10_000);\n    }\n  }\n\n  lc.info?.(`slot-keeper finished for ${config.slot}`);\n}\n\n// fork()\nconst isMain =\n  typeof process.argv[1] === 'string' &&\n  fileURLToPath(import.meta.url) === resolve(process.argv[1]);\n\nconst parent = parentWorker;\nif (isMain && parent !== null && !singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(parent, process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;AAiCA,SAAgB,SAAS,KAAqB;CAC5C,OAAO,IAAI,SAAS,GAAG,IAAI,SAAS,GAAG,IAAI,OAAO,GAAG;AACvD;AAEA,SAAgB,mBAAmB,QAA2C;CAC5E,OAAO,IAAI,SAA2B,SAAS,WAAW;EACxD,MAAM,gBAAgB;GACpB,OAAO,IAAI,cAAc,YAAY;GACrC,OAAO,IAAI,SAAS,OAAO;GAC3B,IAAI,QAAQ,MACV,QAAQ,IAAI,cAAc,YAAY;EAE1C;EACA,MAAM,qBAAqB;GACzB,QAAQ;GACR,uBAAO,IAAI,MAAM,iDAAiD,CAAC;EACrE;EACA,MAAM,gBAAgB;GACpB,QAAQ;GACR,uBAAO,IAAI,MAAM,2CAA2C,CAAC;EAC/D;EACA,OAAO,KAAK,cAAc,YAAY;EACtC,OAAO,KAAK,SAAS,OAAO;EAC5B,IAAI,QAAQ,MACV,QAAQ,KAAK,cAAc,YAAY;EAEzC,OAAO,gBAA6C,UAAS,QAAO;GAClE,QAAQ;GACR,QAAQ,GAAG;EACb,CAAC;EACD,OAAO,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;CAC1B,CAAC;AACH;AAmBA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,WACA,GAAG,MACY;CAEf,MAAM,YAAY,aAAa;EAAC;EAAK;CAAI,CAAC;CAC1C,MAAM,OACJ,OAAO,cAAc,YAAY,cAAc,OAAO,YAAY,CAAC;CACrE,MAAM,gBACJ,KAAK,mBACH,KAAuB,QACvB,SACE,KACE;EACE,KAAK;EACL,aAAa;EACb,cAAc;EACd,cAAc;EACd,YAAY;GACV,kBAAkB,eAAe,IAAI;GACrC,aAAa;EACf;EACA,QAAQ,wBAAwB,KAAK,GAAG,GAAM;CAChD,GACA,IAAI,EACN,CACF;CACJ,MAAM,cAAc,KAAK,eAAe;CACxC,MAAM,UAAU,KAAK,WAAW;CAEhC,MAAM,SAAS,MAAM,mBAAmB,MAAM;CAE9C,MACE,KAAK,cAAc,iBAAiB,WAAW,aAAa,EAAA,CAC5D,YAAY,QAAQ,OAAO,IAAI;CACjC,GAAG,OAAO,4BAA4B,OAAO,MAAM;CAEnD,IAAI,UAAU;CACd,IAAI,cAAmC;CACvC,IAAI,kBAAmC;CACvC,IAAI,iBAGO;CACX,MAAM,MAAM,SAAS,OAAO,GAAG;CAC/B,MAAM,gBAAgB,OAAO,iBAAiB;CAE9C,MAAM,eAAe,YAAY;EAC/B,IAAI,iBAAiB;GACnB,IAAI;IACF,gBAAgB,QAAQ;GAC1B,QAAQ,CAAC;GACT,kBAAkB;EACpB;EACA,IAAI,gBAAgB;GAClB,IAAI;IACF,MAAM,eAAe,IAAI,EAAC,SAAS,EAAC,CAAC;IACrC,GAAG,OAAO,6CAA6C,OAAO,MAAM;GACtE,SAAS,GAAG;IACV,GAAG,OAAO,oCAAoC,CAAC;GACjD;GACA,iBAAiB;EACnB;CACF;CAEA,MAAM,OAAO,YAAY;EACvB,IAAI,SACF;EAEF,UAAU;EACV,GAAG,OAAO,4BAA4B,OAAO,MAAM;EACnD,cAAc;EACd,MAAM,aAAa;CACrB;CAEA,OAAO,gBAAgB,cAAc,KAAK,KAAK,CAAC;CAChD,OAAO,GAAG,iBAAiB,KAAK,KAAK,CAAC;CACtC,OAAO,GAAG,gBAAgB,KAAK,KAAK,CAAC;CACrC,OAAO,KAAK,oBAAoB,KAAK,KAAK,CAAC;CAC3C,OAAO,KAAK,eAAe,KAAK,KAAK,CAAC;CACtC,QAAQ,KAAK,iBAAiB,KAAK,KAAK,CAAC;CACzC,QAAQ,KAAK,gBAAgB,KAAK,KAAK,CAAC;CACxC,IAAI,QAAQ,MACV,QAAQ,KAAK,oBAAoB,KAAK,KAAK,CAAC;CAG9C,IAAI,eAAe;CACnB,IAAI,UAAU;CAEd,OAAO,CAAC,SACN,IAAI;EACF,GAAG,OAAO,2CAA2C,OAAO,MAAM;EAClE,MAAM,UAAU,cAAc,QAAQ,EAAE;EACxC,iBAAiB;EAEjB,MAAM,CAAC,UAAU,aAAa,MAAM,YAClC,IACA,SACA,OAAO,MACP,CAAC,OAAO,gBAAgB,GACxB,KAAA,CAEF;EACA,kBAAkB;EAOlB,SAAS,OAAO;EAEhB,IAAI,SAAS;GACX,MAAM,aAAa;GACnB;EACF;EAEA,GAAG,OAAO,0BAA0B,OAAO,KAAK,UAAU,OAAO,KAAK;EACtE,IAAI,cAAc;GAChB,eAAe;GACf,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;EACtC;EAGA,UAAU;EAEV,MAAM,cAAc,kBAAkB;GACpC,IAAI,SAAS;IACX,cAAc,WAAW;IACzB;GACF;GACA,IAAI;IACF,UAAU,MAAM,QAAQ,GAAG,CAAC;GAC9B,SAAS,GAAG;IACV,GAAG,OAAO,2CAA2C,CAAC;GACxD;EACF,GAAG,aAAa;EAGhB,MAAM,IAAI,SAAe,SAAS,WAAW;GAC3C,MAAM,eAAe;IACnB,QAAQ;IACR,QAAQ;GACV;GACA,MAAM,gBAAgB;IACpB,QAAQ;IACR,QAAQ;GACV;GACA,MAAM,WAAW,QAAiB;IAChC,QAAQ;IACR,OAAO,GAAG;GACZ;GACA,SAAS,UAAU;IACjB,cAAc,WAAW;IACzB,SAAS,IAAI,SAAS,OAAO;IAC7B,SAAS,IAAI,SAAS,OAAO;GAC/B;GACA,cAAc;GACd,SAAS,KAAK,SAAS,OAAO;GAC9B,SAAS,KAAK,SAAS,OAAO;EAChC,CAAC;EAED,MAAM,aAAa;EAEnB,IAAI,SACF;CAEJ,SAAS,GAAG;EACV,MAAM,aAAa;EAEnB,IAAI,SACF;EAGF,IACE,gBAAgB,GAAG,iBAAiB,KACnC,aAAa,UACX,EAAE,QAAQ,SAAS,OAAO,KACzB,EAAE,QAAQ,SACR,qDACF,IACJ;GACA,GAAG,OACD,QAAQ,OAAO,KAAK,wDACtB;GACA;EACF;EAEA,GAAG,OACD,gCAAgC,OAAO,KAAK,sBAAsB,QAAQ,KAC1E,CACF;EAEA,IAAI,cACF,MAAM;EAGR,MAAM,QAAQ,KAAK,CACjB,QAAQ,OAAO,GACf,IAAI,SAAc,YAAW;GAC3B,cAAc;EAChB,CAAC,CACH,CAAC;EACD,UAAU,KAAK,IAAI,UAAU,GAAG,GAAM;CACxC;CAGF,GAAG,OAAO,4BAA4B,OAAO,MAAM;AACrD;AAGA,IAAM,SACJ,OAAO,QAAQ,KAAK,OAAO,YAC3B,cAAc,YAAY,GAAG,MAAM,QAAQ,QAAQ,KAAK,EAAE;AAE5D,IAAM,SAAS;AACf,IAAI,UAAU,WAAW,QAAQ,CAAC,kBAAkB,GAClD,gBACQ,UACA,UAAU,QAAQ,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC/D"}

package/out/zero-cache/src/server/syncer-assigner.js.map

@@ -1 +1 @@
-{"version":3,"file":"syncer-assigner.js","names":["#taskID","#numWorkers","#assignments","#workerLoads","#nextGeneration","#decrementLoad"],"sources":["../../../../../zero-cache/src/server/syncer-assigner.ts"],"sourcesContent":["import {h32} from '../../../shared/src/hash.ts';\n\nexport type Assignment = {\n  readonly worker: number;\n  readonly generation: number;\n};\n\nexport class SyncerAssigner {\n  readonly #taskID: string;\n  readonly #numWorkers: number;\n  readonly #assignments = new Map<string, Assignment>();\n  readonly #workerLoads: number[];\n  #nextGeneration = 0;\n\n  constructor(taskID: string, numWorkers: number) {\n    this.#taskID = taskID;\n    this.#numWorkers = numWorkers;\n    this.#workerLoads = new Array<number>(numWorkers).fill(0);\n  }\n\n  assign(clientGroupID: string): Assignment {\n    if (this.#numWorkers <= 0) {\n      return {worker: 0, generation: 0};\n    }\n\n    const generation = ++this.#nextGeneration;\n    const existing = this.#assignments.get(clientGroupID);\n    if (existing !== undefined) {\n      const updated: Assignment = {worker: existing.worker, generation};\n      this.#assignments.set(clientGroupID, updated);\n      return updated;\n    }\n\n    // Find worker(s) with minimum active load\n    let minLoad = Infinity;\n    const tied: number[] = [];\n    for (let i = 0; i < this.#numWorkers; i++) {\n      const load = this.#workerLoads[i];\n      if (load < minLoad) {\n        minLoad = load;\n        tied.length = 0;\n        tied.push(i);\n      } else if (load === minLoad) {\n        tied.push(i);\n      }\n    }\n\n    const chosen =\n      tied.length === 1\n        ? tied[0]\n        : tied[h32(this.#taskID + '/' + clientGroupID) % tied.length];\n\n    this.#workerLoads[chosen]++;\n    const assignment: Assignment = {worker: chosen, generation};\n    this.#assignments.set(clientGroupID, assignment);\n    return assignment;\n  }\n\n  release(\n    clientGroupID: string,\n    workerIndex: number,\n    generation?: number,\n  ): void {\n    const existing = this.#assignments.get(clientGroupID);\n    if (existing === undefined || existing.worker !== workerIndex) {\n      return;\n    }\n    if (generation !== undefined && existing.generation !== generation) {\n      return;\n    }\n    this.#decrementLoad(workerIndex);\n    this.#assignments.delete(clientGroupID);\n  }\n\n  getWorkerLoad(workerIndex: number): number {\n    return this.#workerLoads[workerIndex] ?? 0;\n  }\n\n  getAssignment(clientGroupID: string): number | undefined {\n    return this.#assignments.get(clientGroupID)?.worker;\n  }\n\n  #decrementLoad(worker: number): void {\n    if (this.#workerLoads[worker] > 0) {\n      this.#workerLoads[worker]--;\n    }\n  }\n\n  destroy(): void {\n    this.#assignments.clear();\n    this.#workerLoads.fill(0);\n  }\n}\n"],"mappings":";;AAOA,IAAa,iBAAb,MAA4B;CAC1B;CACA;CACA,+BAAwB,IAAI,IAAwB;CACpD;CACA,kBAAkB;CAElB,YAAY,QAAgB,YAAoB;EAC9C,KAAKA,UAAU;EACf,KAAKC,cAAc;EACnB,KAAKE,eAAe,IAAI,MAAc,UAAU,EAAE,KAAK,CAAC;CAC1D;CAEA,OAAO,eAAmC;EACxC,IAAI,KAAKF,eAAe,GACtB,OAAO;GAAC,QAAQ;GAAG,YAAY;EAAC;EAGlC,MAAM,aAAa,EAAE,KAAKG;EAC1B,MAAM,WAAW,KAAKF,aAAa,IAAI,aAAa;EACpD,IAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,UAAsB;IAAC,QAAQ,SAAS;IAAQ;GAAU;GAChE,KAAKA,aAAa,IAAI,eAAe,OAAO;GAC5C,OAAO;EACT;EAGA,IAAI,UAAU;EACd,MAAM,OAAiB,CAAC;EACxB,KAAK,IAAI,IAAI,GAAG,IAAI,KAAKD,aAAa,KAAK;GACzC,MAAM,OAAO,KAAKE,aAAa;GAC/B,IAAI,OAAO,SAAS;IAClB,UAAU;IACV,KAAK,SAAS;IACd,KAAK,KAAK,CAAC;GACb,OAAO,IAAI,SAAS,SAClB,KAAK,KAAK,CAAC;EAEf;EAEA,MAAM,SACJ,KAAK,WAAW,IACZ,KAAK,KACL,KAAK,IAAI,KAAKH,UAAU,MAAM,aAAa,IAAI,KAAK;EAE1D,KAAKG,aAAa;EAClB,MAAM,aAAyB;GAAC,QAAQ;GAAQ;EAAU;EAC1D,KAAKD,aAAa,IAAI,eAAe,UAAU;EAC/C,OAAO;CACT;CAEA,QACE,eACA,aACA,YACM;EACN,MAAM,WAAW,KAAKA,aAAa,IAAI,aAAa;EACpD,IAAI,aAAa,KAAA,KAAa,SAAS,WAAW,aAChD;EAEF,IAAI,eAAe,KAAA,KAAa,SAAS,eAAe,YACtD;EAEF,KAAKG,eAAe,WAAW;EAC/B,KAAKH,aAAa,OAAO,aAAa;CACxC;CAEA,cAAc,aAA6B;EACzC,OAAO,KAAKC,aAAa,gBAAgB;CAC3C;CAEA,cAAc,eAA2C;EACvD,OAAO,KAAKD,aAAa,IAAI,aAAa,GAAG;CAC/C;CAEA,eAAe,QAAsB;EACnC,IAAI,KAAKC,aAAa,UAAU,GAC9B,KAAKA,aAAa;CAEtB;CAEA,UAAgB;EACd,KAAKD,aAAa,MAAM;EACxB,KAAKC,aAAa,KAAK,CAAC;CAC1B;AACF"}
+{"version":3,"file":"syncer-assigner.js","names":[],"sources":["../../../../../zero-cache/src/server/syncer-assigner.ts"],"sourcesContent":["import {h32} from '../../../shared/src/hash.ts';\n\nexport type Assignment = {\n  readonly worker: number;\n  readonly generation: number;\n};\n\nexport class SyncerAssigner {\n  readonly #taskID: string;\n  readonly #numWorkers: number;\n  readonly #assignments = new Map<string, Assignment>();\n  readonly #workerLoads: number[];\n  #nextGeneration = 0;\n\n  constructor(taskID: string, numWorkers: number) {\n    this.#taskID = taskID;\n    this.#numWorkers = numWorkers;\n    this.#workerLoads = new Array<number>(numWorkers).fill(0);\n  }\n\n  assign(clientGroupID: string): Assignment {\n    if (this.#numWorkers <= 0) {\n      return {worker: 0, generation: 0};\n    }\n\n    const generation = ++this.#nextGeneration;\n    const existing = this.#assignments.get(clientGroupID);\n    if (existing !== undefined) {\n      const updated: Assignment = {worker: existing.worker, generation};\n      this.#assignments.set(clientGroupID, updated);\n      return updated;\n    }\n\n    // Find worker(s) with minimum active load\n    let minLoad = Infinity;\n    const tied: number[] = [];\n    for (let i = 0; i < this.#numWorkers; i++) {\n      const load = this.#workerLoads[i];\n      if (load < minLoad) {\n        minLoad = load;\n        tied.length = 0;\n        tied.push(i);\n      } else if (load === minLoad) {\n        tied.push(i);\n      }\n    }\n\n    const chosen =\n      tied.length === 1\n        ? tied[0]\n        : tied[h32(this.#taskID + '/' + clientGroupID) % tied.length];\n\n    this.#workerLoads[chosen]++;\n    const assignment: Assignment = {worker: chosen, generation};\n    this.#assignments.set(clientGroupID, assignment);\n    return assignment;\n  }\n\n  release(\n    clientGroupID: string,\n    workerIndex: number,\n    generation?: number,\n  ): void {\n    const existing = this.#assignments.get(clientGroupID);\n    if (existing === undefined || existing.worker !== workerIndex) {\n      return;\n    }\n    if (generation !== undefined && existing.generation !== generation) {\n      return;\n    }\n    this.#decrementLoad(workerIndex);\n    this.#assignments.delete(clientGroupID);\n  }\n\n  getWorkerLoad(workerIndex: number): number {\n    return this.#workerLoads[workerIndex] ?? 0;\n  }\n\n  getAssignment(clientGroupID: string): number | undefined {\n    return this.#assignments.get(clientGroupID)?.worker;\n  }\n\n  #decrementLoad(worker: number): void {\n    if (this.#workerLoads[worker] > 0) {\n      this.#workerLoads[worker]--;\n    }\n  }\n\n  destroy(): void {\n    this.#assignments.clear();\n    this.#workerLoads.fill(0);\n  }\n}\n"],"mappings":";;AAOA,IAAa,iBAAb,MAA4B;CAC1B;CACA;CACA,+BAAwB,IAAI,IAAwB;CACpD;CACA,kBAAkB;CAElB,YAAY,QAAgB,YAAoB;EAC9C,KAAK,UAAU;EACf,KAAK,cAAc;EACnB,KAAK,eAAe,IAAI,MAAc,UAAU,CAAC,CAAC,KAAK,CAAC;CAC1D;CAEA,OAAO,eAAmC;EACxC,IAAI,KAAK,eAAe,GACtB,OAAO;GAAC,QAAQ;GAAG,YAAY;EAAC;EAGlC,MAAM,aAAa,EAAE,KAAK;EAC1B,MAAM,WAAW,KAAK,aAAa,IAAI,aAAa;EACpD,IAAI,aAAa,KAAA,GAAW;GAC1B,MAAM,UAAsB;IAAC,QAAQ,SAAS;IAAQ;GAAU;GAChE,KAAK,aAAa,IAAI,eAAe,OAAO;GAC5C,OAAO;EACT;EAGA,IAAI,UAAU;EACd,MAAM,OAAiB,CAAC;EACxB,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,aAAa,KAAK;GACzC,MAAM,OAAO,KAAK,aAAa;GAC/B,IAAI,OAAO,SAAS;IAClB,UAAU;IACV,KAAK,SAAS;IACd,KAAK,KAAK,CAAC;GACb,OAAO,IAAI,SAAS,SAClB,KAAK,KAAK,CAAC;EAEf;EAEA,MAAM,SACJ,KAAK,WAAW,IACZ,KAAK,KACL,KAAK,IAAI,KAAK,UAAU,MAAM,aAAa,IAAI,KAAK;EAE1D,KAAK,aAAa,OAAO;EACzB,MAAM,aAAyB;GAAC,QAAQ;GAAQ;EAAU;EAC1D,KAAK,aAAa,IAAI,eAAe,UAAU;EAC/C,OAAO;CACT;CAEA,QACE,eACA,aACA,YACM;EACN,MAAM,WAAW,KAAK,aAAa,IAAI,aAAa;EACpD,IAAI,aAAa,KAAA,KAAa,SAAS,WAAW,aAChD;EAEF,IAAI,eAAe,KAAA,KAAa,SAAS,eAAe,YACtD;EAEF,KAAK,eAAe,WAAW;EAC/B,KAAK,aAAa,OAAO,aAAa;CACxC;CAEA,cAAc,aAA6B;EACzC,OAAO,KAAK,aAAa,gBAAgB;CAC3C;CAEA,cAAc,eAA2C;EACvD,OAAO,KAAK,aAAa,IAAI,aAAa,CAAC,EAAE;CAC/C;CAEA,eAAe,QAAsB;EACnC,IAAI,KAAK,aAAa,UAAU,GAC9B,KAAK,aAAa,OAAO;CAE7B;CAEA,UAAgB;EACd,KAAK,aAAa,MAAM;EACxB,KAAK,aAAa,KAAK,CAAC;CAC1B;AACF"}

package/out/zero-cache/src/server/syncer.js

@@ -6,14 +6,13 @@ import { parentWorker, singleProcessMode } from "../types/processes.js";
 import { getShardID } from "../types/shards.js";
 import { getNormalizedZeroConfig } from "../config/zero-config.js";
 import { randInt } from "../../../shared/src/rand.js";
-import { getOrCreateGauge } from "../observability/metrics.js";
-import { Snapshotter } from "../services/view-syncer/snapshotter.js";
-import { PipelineDriver } from "../services/view-syncer/pipeline-driver.js";
 import { connectPgClient } from "../types/pg.js";
 import { registerSQLiteCorruptionDiagnosticTarget } from "../db/sqlite-corruption.js";
+import { getOrCreateGauge } from "../observability/metrics.js";
 import { exitAfter, runUntilKilled } from "../services/life-cycle.js";
 import { createLogContext } from "./logging.js";
 import { installProfileHandler } from "../types/profiler.js";
+import { ProtocolErrorWithLevel } from "../types/error-with-level.js";
 import { warmupConnections } from "../db/warmup.js";
 import { initEventSink } from "../observability/events.js";
 import { startOtelAuto } from "./otel-start.js";
@@ -21,13 +20,14 @@ import { replicaFileModeSchema, replicaFileName } from "../workers/replicator.js
 import { startAnonymousTelemetry } from "./anonymous-otel-start.js";
 import { DatabaseStorage } from "../../../zqlite/src/database-storage.js";
 import { tokenConfigOptions, verifyToken } from "../auth/jwt.js";
-import { ProtocolErrorWithLevel } from "../types/error-with-level.js";
 import { CustomQueryTransformer } from "../custom-queries/transform-query.js";
 import { MutagenService } from "../services/mutagen/mutagen.js";
 import { PusherService } from "../services/mutagen/pusher.js";
 import { BatchPacer } from "../services/view-syncer/batch-pacer.js";
 import { ConnectionContextManagerImpl } from "../services/view-syncer/connection-context-manager.js";
 import { DeferredWritesBudget } from "../services/view-syncer/deferred-writes-budget.js";
+import { Snapshotter } from "../services/view-syncer/snapshotter.js";
+import { PipelineDriver } from "../services/view-syncer/pipeline-driver.js";
 import { QueryStats } from "../services/view-syncer/query-stats.js";
 import { SnapshotRowCache } from "../services/view-syncer/snapshot-row-cache.js";
 import { ViewSyncerService } from "../services/view-syncer/view-syncer.js";
@@ -39,6 +39,7 @@ import { pid } from "node:process";
 import { randomUUID } from "node:crypto";
 import { tmpdir } from "node:os";
 import path from "node:path";
+import { getHeapStatistics } from "node:v8";
 //#region ../zero-cache/src/server/syncer.ts
 function randomID() {
 	return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);
@@ -78,7 +79,11 @@ async function runWorker(parent, env, ...args) {
 	const upstreamDB = enableCrudMutations && upstream.type === "pg" ? await connectPgClient(lc, upstream.db, `sync-worker-${pid}-upstream`, { max: must(upstream.maxConnsPerWorker, "upstream.maxConnsPerWorker must be set") }) : void 0;
 	const dbWarmup = Promise.allSettled([warmupConnections(lc, cvrDB, "cvr"), upstreamDB ? warmupConnections(lc, upstreamDB, "upstream") : promiseVoid]);
 	const tmpDir = config.storageDBTmpDir ?? tmpdir();
-	const operatorStorage = DatabaseStorage.create(lc, path.join(tmpDir, `sync-worker-${randomUUID()}`));
+	const operatorStorage = DatabaseStorage.create(lc, path.join(tmpDir, `sync-worker-${randomUUID()}`), { maxMemoryBytes: Math.floor(getHeapStatistics().heap_size_limit * config.operatorStorageHeapProportion) });
+	getOrCreateGauge("sync", "ivm.operator-storage-memory-bytes", {
+		description: "Estimated bytes of IVM operator state that the client groups of a sync worker keep in memory rather than in SQLite",
+		unit: "By"
+	}).addCallback((o) => o.observe(operatorStorage.memoryBytes));
 	const writeAuthzStorage = DatabaseStorage.create(lc, path.join(tmpDir, `mutagen-${randomUUID()}`));
 	const shard = getShardID(config);
 	const customQueryConfig = getCustomQueryConfig(config);

package/out/zero-cache/src/server/syncer.js.map

@@ -1 +1 @@
-{"version":3,"file":"syncer.js","names":[],"sources":["../../../../../zero-cache/src/server/syncer.ts"],"sourcesContent":["import {randomUUID} from 'node:crypto';\nimport {tmpdir} from 'node:os';\nimport path from 'node:path';\nimport {pid} from 'node:process';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {DatabaseStorage} from '../../../zqlite/src/database-storage.ts';\nimport type {ValidateLegacyJWT} from '../auth/auth.ts';\nimport {tokenConfigOptions, verifyToken} from '../auth/jwt.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {CustomQueryTransformer} from '../custom-queries/transform-query.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {MutagenService} from '../services/mutagen/mutagen.ts';\nimport {PusherService} from '../services/mutagen/pusher.ts';\nimport type {ReplicaState} from '../services/replicator/replicator.ts';\nimport {BatchPacer} from '../services/view-syncer/batch-pacer.ts';\nimport {\n  type ConnectionContextManager,\n  ConnectionContextManagerImpl,\n} from '../services/view-syncer/connection-context-manager.ts';\nimport {DeferredWritesBudget} from '../services/view-syncer/deferred-writes-budget.ts';\nimport type {DrainCoordinator} from '../services/view-syncer/drain-coordinator.ts';\nimport {PipelineDriver} from '../services/view-syncer/pipeline-driver.ts';\nimport {QueryStats} from '../services/view-syncer/query-stats.ts';\nimport {SnapshotRowCache} from '../services/view-syncer/snapshot-row-cache.ts';\nimport {Snapshotter} from '../services/view-syncer/snapshotter.ts';\nimport {ViewSyncerService} from '../services/view-syncer/view-syncer.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardID} from '../types/shards.ts';\nimport type {Subscription} from '../types/subscription.ts';\nimport {replicaFileModeSchema, replicaFileName} from '../workers/replicator.ts';\nimport {Syncer} from '../workers/syncer.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {InspectorDelegate} from './inspector-delegate.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {isPriorityOpRunning, runPriorityOp} from './priority-op.ts';\n\nfunction randomID() {\n  return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);\n}\n\nfunction getCustomQueryConfig(\n  config: Pick<NormalizedZeroConfig, 'query' | 'getQueries'>,\n) {\n  const queryConfig = config.query?.url ? config.query : config.getQueries;\n\n  if (!queryConfig?.url) {\n    return undefined;\n  }\n\n  return {\n    url: queryConfig.url,\n    apiKey: queryConfig.apiKey,\n    allowedClientHeaders: queryConfig.allowedClientHeaders,\n    allowedRequestHeaders: queryConfig.allowedRequestHeaders,\n    forwardCookies: queryConfig.forwardCookies ?? false,\n  };\n}\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length >= 2, `expected [fileMode, workerIndex, ...flags]`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n  const workerIndex = Number(args[1]);\n  installProfileHandler(parent, `syncer-${workerIndex}`, workerIndex);\n  const config = getNormalizedZeroConfig({env, argv: args.slice(2)});\n\n  startOtelAuto(\n    createLogContext(config, 'syncer', workerIndex, false),\n    'syncer',\n    workerIndex,\n  );\n  lc = createLogContext(config, 'syncer', workerIndex);\n  initEventSink(lc, config);\n\n  const {cvr, upstream, enableCrudMutations} = config;\n\n  const replicaFile = replicaFileName(config.replica.file, fileMode);\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'syncer replica',\n      dbPath: replicaFile,\n    },\n    config.sqliteCorruptionChecks,\n  );\n  lc.debug?.(`running view-syncer on ${replicaFile}`);\n\n  const cvrDB = await connectPgClient(lc, cvr.db, `sync-worker-${pid}-cvr`, {\n    max: must(cvr.maxConnsPerWorker, 'cvr.maxConnsPerWorker must be set'),\n  });\n\n  const upstreamDB =\n    enableCrudMutations && upstream.type === 'pg'\n      ? await connectPgClient(lc, upstream.db, `sync-worker-${pid}-upstream`, {\n          max: must(\n            upstream.maxConnsPerWorker,\n            'upstream.maxConnsPerWorker must be set',\n          ),\n        })\n      : undefined;\n\n  const dbWarmup = Promise.allSettled([\n    warmupConnections(lc, cvrDB, 'cvr'),\n    upstreamDB ? warmupConnections(lc, upstreamDB, 'upstream') : promiseVoid,\n  ]);\n\n  const tmpDir = config.storageDBTmpDir ?? tmpdir();\n  const operatorStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `sync-worker-${randomUUID()}`),\n  );\n  const writeAuthzStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `mutagen-${randomUUID()}`),\n  );\n\n  const shard = getShardID(config);\n  const customQueryConfig = getCustomQueryConfig(config);\n  const pushConfig =\n    config.push.url === undefined && config.mutate.url === undefined\n      ? undefined\n      : {\n          ...config.push,\n          ...config.mutate,\n          url: must(\n            config.push.url ?? config.mutate.url,\n            'No push or mutate URL configured',\n          ),\n        };\n\n  /** @deprecated used in JWT validation */\n  let validateLegacyJWT: ValidateLegacyJWT | undefined = undefined;\n\n  const tokenOptions = tokenConfigOptions(config.auth ?? {});\n  if (tokenOptions.length === 1) {\n    validateLegacyJWT = async (token, {userID}) => {\n      if (!userID) {\n        throw new ProtocolErrorWithLevel(\n          {\n            kind: 'Unauthorized',\n            message: 'UserID is required for JWT validation.',\n            origin: 'zeroCache',\n          },\n          'warn',\n        );\n      }\n\n      const decoded = await verifyToken(config.auth, token, {\n        subject: userID,\n        ...(config.auth?.issuer && {issuer: config.auth.issuer}),\n        ...(config.auth?.audience && {\n          audience: config.auth.audience,\n        }),\n      });\n      return {\n        type: 'jwt',\n        raw: token,\n        decoded,\n      };\n    };\n  }\n\n  // Shared by all of the view-syncers on this worker so that the row reads\n  // performed when advancing their pipelines are done once per worker rather\n  // than once per client group.\n  const snapshotRowCache =\n    config.snapshotRowCacheSize > 0\n      ? new SnapshotRowCache(config.snapshotRowCacheSize)\n      : undefined;\n\n  // Shared by all of the view-syncers on this worker, which each hold their\n  // own copy of the changes they are advancing through.\n  const deferredWritesBudget = config.deferIvmWrites\n    ? DeferredWritesBudget.forHeapProportion(\n        config.deferIvmWritesHeapProportion,\n      )\n    : undefined;\n  if (deferredWritesBudget) {\n    lc.info?.(\n      `Deferred IVM writes may hold up to ${deferredWritesBudget.maxRows} ` +\n        `rows (~${(deferredWritesBudget.maxBytes / 1024 ** 2).toFixed(2)} MB) ` +\n        `across client groups`,\n    );\n    getOrCreateGauge(\n      'sync',\n      'ivm.deferred-writes-reserved-rows',\n      'Rows reserved by the client groups of a sync worker to hold IVM ' +\n        'changes in memory (deferIvmWrites)',\n    ).addCallback(o => o.observe(deferredWritesBudget.reservedRows));\n    getOrCreateGauge('sync', 'ivm.deferred-writes-held-bytes', {\n      description:\n        'Estimated bytes of the IVM changes that the client groups of a sync ' +\n        'worker hold in memory (deferIvmWrites)',\n      unit: 'By',\n    }).addCallback(o => o.observe(deferredWritesBudget.heldBytes));\n  }\n\n  // Shared by all of the view-syncers on this worker, whose batches of\n  // changes together are held to a share of the worker's time.\n  const batchPacer = config.costBasedSyncEnabled\n    ? new BatchPacer({\n        shareTarget: config.syncUpdateShareTarget,\n        maxIntervalMs: config.syncMaxBatchIntervalMs,\n        lc,\n      })\n    : undefined;\n  if (batchPacer) {\n    getOrCreateGauge('sync', 'worker-busy-share', {\n      description:\n        'Share of its time that a sync worker was busy over the last few ' +\n        'seconds, which the pacing of its client groups holds to a target',\n      unit: '1',\n    }).addCallback(o => o.observe(batchPacer.share));\n    getOrCreateGauge('sync', 'batch-wait-multiplier', {\n      description:\n        'How many times the processing time of a batch of changes the ' +\n        'client groups of a sync worker wait before their next batch',\n      unit: '1',\n    }).addCallback(o => o.observe(batchPacer.multiplier));\n  }\n\n  // Shared by all of the view-syncers on this worker, so that the work done\n  // for each query shape is logged once per worker per interval.\n  const queryStats =\n    config.log.queryStatsIntervalSeconds > 0 ? new QueryStats() : undefined;\n  const stopQueryStats = queryStats?.start(\n    lc\n      .withContext('taskID', config.taskID)\n      .withContext('component', 'view-syncer'),\n    config.log.queryStatsIntervalSeconds * 1000,\n  );\n\n  const viewSyncerFactory = (\n    id: string,\n    sub: Subscription<ReplicaState>,\n    drainCoordinator: DrainCoordinator,\n  ) => {\n    const logger = lc\n      .withContext('taskID', config.taskID)\n      .withContext('component', 'view-syncer')\n      .withContext('appID', shard.appID)\n      .withContext('shardNum', shard.shardNum)\n      .withContext('clientGroupID', id)\n      .withContext('instance', randomID());\n\n    const customQueryTransformer =\n      customQueryConfig && new CustomQueryTransformer(logger, shard);\n    const connContextManager = new ConnectionContextManagerImpl(\n      logger,\n      config.auth.revalidateIntervalSeconds,\n      config.auth.retransformIntervalSeconds,\n      customQueryConfig,\n      pushConfig,\n      validateLegacyJWT,\n    );\n\n    lc.debug?.(\n      `creating view syncer. Query Planner Enabled: ${config.enableQueryPlanner}`,\n    );\n\n    const inspectorDelegate = new InspectorDelegate(customQueryTransformer);\n\n    const priorityOpRunningYieldThresholdMs = Math.max(\n      config.yieldThresholdMs / 4,\n      2,\n    );\n    const normalYieldThresholdMs = Math.max(config.yieldThresholdMs, 2);\n\n    return new ViewSyncerService(\n      config,\n      logger,\n      shard,\n      config.taskID,\n      id,\n      cvrDB,\n      new PipelineDriver(\n        logger,\n        config.log,\n        new Snapshotter(\n          logger,\n          replicaFile,\n          shard,\n          undefined,\n          snapshotRowCache,\n        ),\n        shard,\n        operatorStorage.createClientGroupStorage(id),\n        id,\n        inspectorDelegate,\n        () =>\n          isPriorityOpRunning()\n            ? priorityOpRunningYieldThresholdMs\n            : normalYieldThresholdMs,\n        config.enableQueryPlanner,\n        config,\n        deferredWritesBudget,\n        queryStats,\n      ),\n      sub,\n      drainCoordinator,\n      config.log.slowHydrateThreshold,\n      inspectorDelegate,\n      connContextManager,\n      customQueryTransformer,\n      runPriorityOp,\n      undefined,\n      undefined,\n      undefined,\n      undefined,\n      batchPacer,\n    );\n  };\n\n  const mutagenFactory = upstreamDB\n    ? (id: string) =>\n        new MutagenService(\n          lc\n            .withContext('component', 'mutagen')\n            .withContext('clientGroupID', id),\n          shard,\n          id,\n          upstreamDB,\n          config,\n          writeAuthzStorage,\n        )\n    : undefined;\n\n  const pusherFactory =\n    pushConfig === undefined\n      ? undefined\n      : (id: string, connContextManager: ConnectionContextManager) =>\n          new PusherService(\n            config,\n            lc.withContext('clientGroupID', id),\n            id,\n            connContextManager,\n          );\n\n  const syncer = new Syncer(\n    lc,\n    config,\n    viewSyncerFactory,\n    mutagenFactory,\n    pusherFactory,\n    parent,\n    validateLegacyJWT,\n  );\n\n  startAnonymousTelemetry(lc, config);\n\n  void dbWarmup.then(() => parent.send(['ready', {ready: true}]));\n\n  try {\n    return await runUntilKilled(lc, parent, syncer);\n  } finally {\n    // Logs the work done since the last interval.\n    stopQueryStats?.();\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsDA,SAAS,WAAW;CAClB,OAAO,QAAQ,GAAG,OAAO,gBAAgB,EAAE,SAAS,EAAE;AACxD;AAEA,SAAS,qBACP,QACA;CACA,MAAM,cAAc,OAAO,OAAO,MAAM,OAAO,QAAQ,OAAO;CAE9D,IAAI,CAAC,aAAa,KAChB;CAGF,OAAO;EACL,KAAK,YAAY;EACjB,QAAQ,YAAY;EACpB,sBAAsB,YAAY;EAClC,uBAAuB,YAAY;EACnC,gBAAgB,YAAY,kBAAkB;CAChD;AACF;AAGA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,UAAU,GAAG,4CAA4C;CACrE,MAAM,WAAW,MAAQ,KAAK,IAAI,qBAAqB;CACvD,MAAM,cAAc,OAAO,KAAK,EAAE;CAClC,sBAAsB,QAAQ,UAAU,eAAe,WAAW;CAClE,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CAEjE,cACE,iBAAiB,QAAQ,UAAU,aAAa,KAAK,GACrD,UACA,WACF;CACA,KAAK,iBAAiB,QAAQ,UAAU,WAAW;CACnD,cAAc,IAAI,MAAM;CAExB,MAAM,EAAC,KAAK,UAAU,wBAAuB;CAE7C,MAAM,cAAc,gBAAgB,OAAO,QAAQ,MAAM,QAAQ;CACjE,yCACE;EACE,WAAW;EACX,QAAQ;CACV,GACA,OAAO,sBACT;CACA,GAAG,QAAQ,0BAA0B,aAAa;CAElD,MAAM,QAAQ,MAAM,gBAAgB,IAAI,IAAI,IAAI,eAAe,IAAI,OAAO,EACxE,KAAK,KAAK,IAAI,mBAAmB,mCAAmC,EACtE,CAAC;CAED,MAAM,aACJ,uBAAuB,SAAS,SAAS,OACrC,MAAM,gBAAgB,IAAI,SAAS,IAAI,eAAe,IAAI,YAAY,EACpE,KAAK,KACH,SAAS,mBACT,wCACF,EACF,CAAC,IACD,KAAA;CAEN,MAAM,WAAW,QAAQ,WAAW,CAClC,kBAAkB,IAAI,OAAO,KAAK,GAClC,aAAa,kBAAkB,IAAI,YAAY,UAAU,IAAI,WAC/D,CAAC;CAED,MAAM,SAAS,OAAO,mBAAmB,OAAO;CAChD,MAAM,kBAAkB,gBAAgB,OACtC,IACA,KAAK,KAAK,QAAQ,eAAe,WAAW,GAAG,CACjD;CACA,MAAM,oBAAoB,gBAAgB,OACxC,IACA,KAAK,KAAK,QAAQ,WAAW,WAAW,GAAG,CAC7C;CAEA,MAAM,QAAQ,WAAW,MAAM;CAC/B,MAAM,oBAAoB,qBAAqB,MAAM;CACrD,MAAM,aACJ,OAAO,KAAK,QAAQ,KAAA,KAAa,OAAO,OAAO,QAAQ,KAAA,IACnD,KAAA,IACA;EACE,GAAG,OAAO;EACV,GAAG,OAAO;EACV,KAAK,KACH,OAAO,KAAK,OAAO,OAAO,OAAO,KACjC,kCACF;CACF;;CAGN,IAAI,oBAAmD,KAAA;CAGvD,IADqB,mBAAmB,OAAO,QAAQ,CAAC,CACpD,EAAa,WAAW,GAC1B,oBAAoB,OAAO,OAAO,EAAC,aAAY;EAC7C,IAAI,CAAC,QACH,MAAM,IAAI,uBACR;GACE,MAAM;GACN,SAAS;GACT,QAAQ;EACV,GACA,MACF;EAUF,OAAO;GACL,MAAM;GACN,KAAK;GACL,SAAA,MAVoB,YAAY,OAAO,MAAM,OAAO;IACpD,SAAS;IACT,GAAI,OAAO,MAAM,UAAU,EAAC,QAAQ,OAAO,KAAK,OAAM;IACtD,GAAI,OAAO,MAAM,YAAY,EAC3B,UAAU,OAAO,KAAK,SACxB;GACF,CAAC;EAKD;CACF;CAMF,MAAM,mBACJ,OAAO,uBAAuB,IAC1B,IAAI,iBAAiB,OAAO,oBAAoB,IAChD,KAAA;CAIN,MAAM,uBAAuB,OAAO,iBAChC,qBAAqB,kBACnB,OAAO,4BACT,IACA,KAAA;CACJ,IAAI,sBAAsB;EACxB,GAAG,OACD,sCAAsC,qBAAqB,QAAQ,WACtD,qBAAqB,WAAW,QAAQ,GAAG,QAAQ,CAAC,EAAE,0BAErE;EACA,iBACE,QACA,qCACA,oGAEF,EAAE,aAAY,MAAK,EAAE,QAAQ,qBAAqB,YAAY,CAAC;EAC/D,iBAAiB,QAAQ,kCAAkC;GACzD,aACE;GAEF,MAAM;EACR,CAAC,EAAE,aAAY,MAAK,EAAE,QAAQ,qBAAqB,SAAS,CAAC;CAC/D;CAIA,MAAM,aAAa,OAAO,uBACtB,IAAI,WAAW;EACb,aAAa,OAAO;EACpB,eAAe,OAAO;EACtB;CACF,CAAC,IACD,KAAA;CACJ,IAAI,YAAY;EACd,iBAAiB,QAAQ,qBAAqB;GAC5C,aACE;GAEF,MAAM;EACR,CAAC,EAAE,aAAY,MAAK,EAAE,QAAQ,WAAW,KAAK,CAAC;EAC/C,iBAAiB,QAAQ,yBAAyB;GAChD,aACE;GAEF,MAAM;EACR,CAAC,EAAE,aAAY,MAAK,EAAE,QAAQ,WAAW,UAAU,CAAC;CACtD;CAIA,MAAM,aACJ,OAAO,IAAI,4BAA4B,IAAI,IAAI,WAAW,IAAI,KAAA;CAChE,MAAM,iBAAiB,YAAY,MACjC,GACG,YAAY,UAAU,OAAO,MAAM,EACnC,YAAY,aAAa,aAAa,GACzC,OAAO,IAAI,4BAA4B,GACzC;CAEA,MAAM,qBACJ,IACA,KACA,qBACG;EACH,MAAM,SAAS,GACZ,YAAY,UAAU,OAAO,MAAM,EACnC,YAAY,aAAa,aAAa,EACtC,YAAY,SAAS,MAAM,KAAK,EAChC,YAAY,YAAY,MAAM,QAAQ,EACtC,YAAY,iBAAiB,EAAE,EAC/B,YAAY,YAAY,SAAS,CAAC;EAErC,MAAM,yBACJ,qBAAqB,IAAI,uBAAuB,QAAQ,KAAK;EAC/D,MAAM,qBAAqB,IAAI,6BAC7B,QACA,OAAO,KAAK,2BACZ,OAAO,KAAK,4BACZ,mBACA,YACA,iBACF;EAEA,GAAG,QACD,gDAAgD,OAAO,oBACzD;EAEA,MAAM,oBAAoB,IAAI,kBAAkB,sBAAsB;EAEtE,MAAM,oCAAoC,KAAK,IAC7C,OAAO,mBAAmB,GAC1B,CACF;EACA,MAAM,yBAAyB,KAAK,IAAI,OAAO,kBAAkB,CAAC;EAElE,OAAO,IAAI,kBACT,QACA,QACA,OACA,OAAO,QACP,IACA,OACA,IAAI,eACF,QACA,OAAO,KACP,IAAI,YACF,QACA,aACA,OACA,KAAA,GACA,gBACF,GACA,OACA,gBAAgB,yBAAyB,EAAE,GAC3C,IACA,yBAEE,oBAAoB,IAChB,oCACA,wBACN,OAAO,oBACP,QACA,sBACA,UACF,GACA,KACA,kBACA,OAAO,IAAI,sBACX,mBACA,oBACA,wBACA,eACA,KAAA,GACA,KAAA,GACA,KAAA,GACA,KAAA,GACA,UACF;CACF;CA2BA,MAAM,SAAS,IAAI,OACjB,IACA,QACA,mBA5BqB,cAClB,OACC,IAAI,eACF,GACG,YAAY,aAAa,SAAS,EAClC,YAAY,iBAAiB,EAAE,GAClC,OACA,IACA,YACA,QACA,iBACF,IACF,KAAA,GAGF,eAAe,KAAA,IACX,KAAA,KACC,IAAY,uBACX,IAAI,cACF,QACA,GAAG,YAAY,iBAAiB,EAAE,GAClC,IACA,kBACF,GAQN,QACA,iBACF;CAEA,wBAAwB,IAAI,MAAM;CAElC,SAAc,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CAE9D,IAAI;EACF,OAAO,MAAM,eAAe,IAAI,QAAQ,MAAM;CAChD,UAAU;EAER,iBAAiB;CACnB;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}
+{"version":3,"file":"syncer.js","names":["v.parse"],"sources":["../../../../../zero-cache/src/server/syncer.ts"],"sourcesContent":["import {randomUUID} from 'node:crypto';\nimport {tmpdir} from 'node:os';\nimport path from 'node:path';\nimport {pid} from 'node:process';\nimport {getHeapStatistics} from 'node:v8';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {DatabaseStorage} from '../../../zqlite/src/database-storage.ts';\nimport type {ValidateLegacyJWT} from '../auth/auth.ts';\nimport {tokenConfigOptions, verifyToken} from '../auth/jwt.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {CustomQueryTransformer} from '../custom-queries/transform-query.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {MutagenService} from '../services/mutagen/mutagen.ts';\nimport {PusherService} from '../services/mutagen/pusher.ts';\nimport type {ReplicaState} from '../services/replicator/replicator.ts';\nimport {BatchPacer} from '../services/view-syncer/batch-pacer.ts';\nimport {\n  type ConnectionContextManager,\n  ConnectionContextManagerImpl,\n} from '../services/view-syncer/connection-context-manager.ts';\nimport {DeferredWritesBudget} from '../services/view-syncer/deferred-writes-budget.ts';\nimport type {DrainCoordinator} from '../services/view-syncer/drain-coordinator.ts';\nimport {PipelineDriver} from '../services/view-syncer/pipeline-driver.ts';\nimport {QueryStats} from '../services/view-syncer/query-stats.ts';\nimport {SnapshotRowCache} from '../services/view-syncer/snapshot-row-cache.ts';\nimport {Snapshotter} from '../services/view-syncer/snapshotter.ts';\nimport {ViewSyncerService} from '../services/view-syncer/view-syncer.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardID} from '../types/shards.ts';\nimport type {Subscription} from '../types/subscription.ts';\nimport {replicaFileModeSchema, replicaFileName} from '../workers/replicator.ts';\nimport {Syncer} from '../workers/syncer.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {InspectorDelegate} from './inspector-delegate.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {isPriorityOpRunning, runPriorityOp} from './priority-op.ts';\n\nfunction randomID() {\n  return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);\n}\n\nfunction getCustomQueryConfig(\n  config: Pick<NormalizedZeroConfig, 'query' | 'getQueries'>,\n) {\n  const queryConfig = config.query?.url ? config.query : config.getQueries;\n\n  if (!queryConfig?.url) {\n    return undefined;\n  }\n\n  return {\n    url: queryConfig.url,\n    apiKey: queryConfig.apiKey,\n    allowedClientHeaders: queryConfig.allowedClientHeaders,\n    allowedRequestHeaders: queryConfig.allowedRequestHeaders,\n    forwardCookies: queryConfig.forwardCookies ?? false,\n  };\n}\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length >= 2, `expected [fileMode, workerIndex, ...flags]`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n  const workerIndex = Number(args[1]);\n  installProfileHandler(parent, `syncer-${workerIndex}`, workerIndex);\n  const config = getNormalizedZeroConfig({env, argv: args.slice(2)});\n\n  startOtelAuto(\n    createLogContext(config, 'syncer', workerIndex, false),\n    'syncer',\n    workerIndex,\n  );\n  lc = createLogContext(config, 'syncer', workerIndex);\n  initEventSink(lc, config);\n\n  const {cvr, upstream, enableCrudMutations} = config;\n\n  const replicaFile = replicaFileName(config.replica.file, fileMode);\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'syncer replica',\n      dbPath: replicaFile,\n    },\n    config.sqliteCorruptionChecks,\n  );\n  lc.debug?.(`running view-syncer on ${replicaFile}`);\n\n  const cvrDB = await connectPgClient(lc, cvr.db, `sync-worker-${pid}-cvr`, {\n    max: must(cvr.maxConnsPerWorker, 'cvr.maxConnsPerWorker must be set'),\n  });\n\n  const upstreamDB =\n    enableCrudMutations && upstream.type === 'pg'\n      ? await connectPgClient(lc, upstream.db, `sync-worker-${pid}-upstream`, {\n          max: must(\n            upstream.maxConnsPerWorker,\n            'upstream.maxConnsPerWorker must be set',\n          ),\n        })\n      : undefined;\n\n  const dbWarmup = Promise.allSettled([\n    warmupConnections(lc, cvrDB, 'cvr'),\n    upstreamDB ? warmupConnections(lc, upstreamDB, 'upstream') : promiseVoid,\n  ]);\n\n  const tmpDir = config.storageDBTmpDir ?? tmpdir();\n  const operatorStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `sync-worker-${randomUUID()}`),\n    {\n      maxMemoryBytes: Math.floor(\n        getHeapStatistics().heap_size_limit *\n          config.operatorStorageHeapProportion,\n      ),\n    },\n  );\n  getOrCreateGauge('sync', 'ivm.operator-storage-memory-bytes', {\n    description:\n      'Estimated bytes of IVM operator state that the client groups of a ' +\n      'sync worker keep in memory rather than in SQLite',\n    unit: 'By',\n  }).addCallback(o => o.observe(operatorStorage.memoryBytes));\n  const writeAuthzStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `mutagen-${randomUUID()}`),\n  );\n\n  const shard = getShardID(config);\n  const customQueryConfig = getCustomQueryConfig(config);\n  const pushConfig =\n    config.push.url === undefined && config.mutate.url === undefined\n      ? undefined\n      : {\n          ...config.push,\n          ...config.mutate,\n          url: must(\n            config.push.url ?? config.mutate.url,\n            'No push or mutate URL configured',\n          ),\n        };\n\n  /** @deprecated used in JWT validation */\n  let validateLegacyJWT: ValidateLegacyJWT | undefined = undefined;\n\n  const tokenOptions = tokenConfigOptions(config.auth ?? {});\n  if (tokenOptions.length === 1) {\n    validateLegacyJWT = async (token, {userID}) => {\n      if (!userID) {\n        throw new ProtocolErrorWithLevel(\n          {\n            kind: 'Unauthorized',\n            message: 'UserID is required for JWT validation.',\n            origin: 'zeroCache',\n          },\n          'warn',\n        );\n      }\n\n      const decoded = await verifyToken(config.auth, token, {\n        subject: userID,\n        ...(config.auth?.issuer && {issuer: config.auth.issuer}),\n        ...(config.auth?.audience && {\n          audience: config.auth.audience,\n        }),\n      });\n      return {\n        type: 'jwt',\n        raw: token,\n        decoded,\n      };\n    };\n  }\n\n  // Shared by all of the view-syncers on this worker so that the row reads\n  // performed when advancing their pipelines are done once per worker rather\n  // than once per client group.\n  const snapshotRowCache =\n    config.snapshotRowCacheSize > 0\n      ? new SnapshotRowCache(config.snapshotRowCacheSize)\n      : undefined;\n\n  // Shared by all of the view-syncers on this worker, which each hold their\n  // own copy of the changes they are advancing through.\n  const deferredWritesBudget = config.deferIvmWrites\n    ? DeferredWritesBudget.forHeapProportion(\n        config.deferIvmWritesHeapProportion,\n      )\n    : undefined;\n  if (deferredWritesBudget) {\n    lc.info?.(\n      `Deferred IVM writes may hold up to ${deferredWritesBudget.maxRows} ` +\n        `rows (~${(deferredWritesBudget.maxBytes / 1024 ** 2).toFixed(2)} MB) ` +\n        `across client groups`,\n    );\n    getOrCreateGauge(\n      'sync',\n      'ivm.deferred-writes-reserved-rows',\n      'Rows reserved by the client groups of a sync worker to hold IVM ' +\n        'changes in memory (deferIvmWrites)',\n    ).addCallback(o => o.observe(deferredWritesBudget.reservedRows));\n    getOrCreateGauge('sync', 'ivm.deferred-writes-held-bytes', {\n      description:\n        'Estimated bytes of the IVM changes that the client groups of a sync ' +\n        'worker hold in memory (deferIvmWrites)',\n      unit: 'By',\n    }).addCallback(o => o.observe(deferredWritesBudget.heldBytes));\n  }\n\n  // Shared by all of the view-syncers on this worker, whose batches of\n  // changes together are held to a share of the worker's time.\n  const batchPacer = config.costBasedSyncEnabled\n    ? new BatchPacer({\n        shareTarget: config.syncUpdateShareTarget,\n        maxIntervalMs: config.syncMaxBatchIntervalMs,\n        lc,\n      })\n    : undefined;\n  if (batchPacer) {\n    getOrCreateGauge('sync', 'worker-busy-share', {\n      description:\n        'Share of its time that a sync worker was busy over the last few ' +\n        'seconds, which the pacing of its client groups holds to a target',\n      unit: '1',\n    }).addCallback(o => o.observe(batchPacer.share));\n    getOrCreateGauge('sync', 'batch-wait-multiplier', {\n      description:\n        'How many times the processing time of a batch of changes the ' +\n        'client groups of a sync worker wait before their next batch',\n      unit: '1',\n    }).addCallback(o => o.observe(batchPacer.multiplier));\n  }\n\n  // Shared by all of the view-syncers on this worker, so that the work done\n  // for each query shape is logged once per worker per interval.\n  const queryStats =\n    config.log.queryStatsIntervalSeconds > 0 ? new QueryStats() : undefined;\n  const stopQueryStats = queryStats?.start(\n    lc\n      .withContext('taskID', config.taskID)\n      .withContext('component', 'view-syncer'),\n    config.log.queryStatsIntervalSeconds * 1000,\n  );\n\n  const viewSyncerFactory = (\n    id: string,\n    sub: Subscription<ReplicaState>,\n    drainCoordinator: DrainCoordinator,\n  ) => {\n    const logger = lc\n      .withContext('taskID', config.taskID)\n      .withContext('component', 'view-syncer')\n      .withContext('appID', shard.appID)\n      .withContext('shardNum', shard.shardNum)\n      .withContext('clientGroupID', id)\n      .withContext('instance', randomID());\n\n    const customQueryTransformer =\n      customQueryConfig && new CustomQueryTransformer(logger, shard);\n    const connContextManager = new ConnectionContextManagerImpl(\n      logger,\n      config.auth.revalidateIntervalSeconds,\n      config.auth.retransformIntervalSeconds,\n      customQueryConfig,\n      pushConfig,\n      validateLegacyJWT,\n    );\n\n    lc.debug?.(\n      `creating view syncer. Query Planner Enabled: ${config.enableQueryPlanner}`,\n    );\n\n    const inspectorDelegate = new InspectorDelegate(customQueryTransformer);\n\n    const priorityOpRunningYieldThresholdMs = Math.max(\n      config.yieldThresholdMs / 4,\n      2,\n    );\n    const normalYieldThresholdMs = Math.max(config.yieldThresholdMs, 2);\n\n    return new ViewSyncerService(\n      config,\n      logger,\n      shard,\n      config.taskID,\n      id,\n      cvrDB,\n      new PipelineDriver(\n        logger,\n        config.log,\n        new Snapshotter(\n          logger,\n          replicaFile,\n          shard,\n          undefined,\n          snapshotRowCache,\n        ),\n        shard,\n        operatorStorage.createClientGroupStorage(id),\n        id,\n        inspectorDelegate,\n        () =>\n          isPriorityOpRunning()\n            ? priorityOpRunningYieldThresholdMs\n            : normalYieldThresholdMs,\n        config.enableQueryPlanner,\n        config,\n        deferredWritesBudget,\n        queryStats,\n      ),\n      sub,\n      drainCoordinator,\n      config.log.slowHydrateThreshold,\n      inspectorDelegate,\n      connContextManager,\n      customQueryTransformer,\n      runPriorityOp,\n      undefined,\n      undefined,\n      undefined,\n      undefined,\n      batchPacer,\n    );\n  };\n\n  const mutagenFactory = upstreamDB\n    ? (id: string) =>\n        new MutagenService(\n          lc\n            .withContext('component', 'mutagen')\n            .withContext('clientGroupID', id),\n          shard,\n          id,\n          upstreamDB,\n          config,\n          writeAuthzStorage,\n        )\n    : undefined;\n\n  const pusherFactory =\n    pushConfig === undefined\n      ? undefined\n      : (id: string, connContextManager: ConnectionContextManager) =>\n          new PusherService(\n            config,\n            lc.withContext('clientGroupID', id),\n            id,\n            connContextManager,\n          );\n\n  const syncer = new Syncer(\n    lc,\n    config,\n    viewSyncerFactory,\n    mutagenFactory,\n    pusherFactory,\n    parent,\n    validateLegacyJWT,\n  );\n\n  startAnonymousTelemetry(lc, config);\n\n  void dbWarmup.then(() => parent.send(['ready', {ready: true}]));\n\n  try {\n    return await runUntilKilled(lc, parent, syncer);\n  } finally {\n    // Logs the work done since the last interval.\n    stopQueryStats?.();\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuDA,SAAS,WAAW;CAClB,OAAO,QAAQ,GAAG,OAAO,gBAAgB,CAAC,CAAC,SAAS,EAAE;AACxD;AAEA,SAAS,qBACP,QACA;CACA,MAAM,cAAc,OAAO,OAAO,MAAM,OAAO,QAAQ,OAAO;CAE9D,IAAI,CAAC,aAAa,KAChB;CAGF,OAAO;EACL,KAAK,YAAY;EACjB,QAAQ,YAAY;EACpB,sBAAsB,YAAY;EAClC,uBAAuB,YAAY;EACnC,gBAAgB,YAAY,kBAAkB;CAChD;AACF;AAGA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,UAAU,GAAG,4CAA4C;CACrE,MAAM,WAAWA,MAAQ,KAAK,IAAI,qBAAqB;CACvD,MAAM,cAAc,OAAO,KAAK,EAAE;CAClC,sBAAsB,QAAQ,UAAU,eAAe,WAAW;CAClE,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CAEjE,cACE,iBAAiB,QAAQ,UAAU,aAAa,KAAK,GACrD,UACA,WACF;CACA,KAAK,iBAAiB,QAAQ,UAAU,WAAW;CACnD,cAAc,IAAI,MAAM;CAExB,MAAM,EAAC,KAAK,UAAU,wBAAuB;CAE7C,MAAM,cAAc,gBAAgB,OAAO,QAAQ,MAAM,QAAQ;CACjE,yCACE;EACE,WAAW;EACX,QAAQ;CACV,GACA,OAAO,sBACT;CACA,GAAG,QAAQ,0BAA0B,aAAa;CAElD,MAAM,QAAQ,MAAM,gBAAgB,IAAI,IAAI,IAAI,eAAe,IAAI,OAAO,EACxE,KAAK,KAAK,IAAI,mBAAmB,mCAAmC,EACtE,CAAC;CAED,MAAM,aACJ,uBAAuB,SAAS,SAAS,OACrC,MAAM,gBAAgB,IAAI,SAAS,IAAI,eAAe,IAAI,YAAY,EACpE,KAAK,KACH,SAAS,mBACT,wCACF,EACF,CAAC,IACD,KAAA;CAEN,MAAM,WAAW,QAAQ,WAAW,CAClC,kBAAkB,IAAI,OAAO,KAAK,GAClC,aAAa,kBAAkB,IAAI,YAAY,UAAU,IAAI,WAC/D,CAAC;CAED,MAAM,SAAS,OAAO,mBAAmB,OAAO;CAChD,MAAM,kBAAkB,gBAAgB,OACtC,IACA,KAAK,KAAK,QAAQ,eAAe,WAAW,GAAG,GAC/C,EACE,gBAAgB,KAAK,MACnB,kBAAkB,CAAC,CAAC,kBAClB,OAAO,6BACX,EACF,CACF;CACA,iBAAiB,QAAQ,qCAAqC;EAC5D,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,aAAY,MAAK,EAAE,QAAQ,gBAAgB,WAAW,CAAC;CAC1D,MAAM,oBAAoB,gBAAgB,OACxC,IACA,KAAK,KAAK,QAAQ,WAAW,WAAW,GAAG,CAC7C;CAEA,MAAM,QAAQ,WAAW,MAAM;CAC/B,MAAM,oBAAoB,qBAAqB,MAAM;CACrD,MAAM,aACJ,OAAO,KAAK,QAAQ,KAAA,KAAa,OAAO,OAAO,QAAQ,KAAA,IACnD,KAAA,IACA;EACE,GAAG,OAAO;EACV,GAAG,OAAO;EACV,KAAK,KACH,OAAO,KAAK,OAAO,OAAO,OAAO,KACjC,kCACF;CACF;;CAGN,IAAI,oBAAmD,KAAA;CAGvD,IADqB,mBAAmB,OAAO,QAAQ,CAAC,CACpD,CAAA,CAAa,WAAW,GAC1B,oBAAoB,OAAO,OAAO,EAAC,aAAY;EAC7C,IAAI,CAAC,QACH,MAAM,IAAI,uBACR;GACE,MAAM;GACN,SAAS;GACT,QAAQ;EACV,GACA,MACF;EAUF,OAAO;GACL,MAAM;GACN,KAAK;GACL,SAAA,MAVoB,YAAY,OAAO,MAAM,OAAO;IACpD,SAAS;IACT,GAAI,OAAO,MAAM,UAAU,EAAC,QAAQ,OAAO,KAAK,OAAM;IACtD,GAAI,OAAO,MAAM,YAAY,EAC3B,UAAU,OAAO,KAAK,SACxB;GACF,CAAC;EAKD;CACF;CAMF,MAAM,mBACJ,OAAO,uBAAuB,IAC1B,IAAI,iBAAiB,OAAO,oBAAoB,IAChD,KAAA;CAIN,MAAM,uBAAuB,OAAO,iBAChC,qBAAqB,kBACnB,OAAO,4BACT,IACA,KAAA;CACJ,IAAI,sBAAsB;EACxB,GAAG,OACD,sCAAsC,qBAAqB,QAAQ,WACtD,qBAAqB,WAAW,QAAQ,EAAA,CAAG,QAAQ,CAAC,EAAE,0BAErE;EACA,iBACE,QACA,qCACA,oGAEF,CAAC,CAAC,aAAY,MAAK,EAAE,QAAQ,qBAAqB,YAAY,CAAC;EAC/D,iBAAiB,QAAQ,kCAAkC;GACzD,aACE;GAEF,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,MAAK,EAAE,QAAQ,qBAAqB,SAAS,CAAC;CAC/D;CAIA,MAAM,aAAa,OAAO,uBACtB,IAAI,WAAW;EACb,aAAa,OAAO;EACpB,eAAe,OAAO;EACtB;CACF,CAAC,IACD,KAAA;CACJ,IAAI,YAAY;EACd,iBAAiB,QAAQ,qBAAqB;GAC5C,aACE;GAEF,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,MAAK,EAAE,QAAQ,WAAW,KAAK,CAAC;EAC/C,iBAAiB,QAAQ,yBAAyB;GAChD,aACE;GAEF,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,MAAK,EAAE,QAAQ,WAAW,UAAU,CAAC;CACtD;CAIA,MAAM,aACJ,OAAO,IAAI,4BAA4B,IAAI,IAAI,WAAW,IAAI,KAAA;CAChE,MAAM,iBAAiB,YAAY,MACjC,GACG,YAAY,UAAU,OAAO,MAAM,CAAC,CACpC,YAAY,aAAa,aAAa,GACzC,OAAO,IAAI,4BAA4B,GACzC;CAEA,MAAM,qBACJ,IACA,KACA,qBACG;EACH,MAAM,SAAS,GACZ,YAAY,UAAU,OAAO,MAAM,CAAC,CACpC,YAAY,aAAa,aAAa,CAAC,CACvC,YAAY,SAAS,MAAM,KAAK,CAAC,CACjC,YAAY,YAAY,MAAM,QAAQ,CAAC,CACvC,YAAY,iBAAiB,EAAE,CAAC,CAChC,YAAY,YAAY,SAAS,CAAC;EAErC,MAAM,yBACJ,qBAAqB,IAAI,uBAAuB,QAAQ,KAAK;EAC/D,MAAM,qBAAqB,IAAI,6BAC7B,QACA,OAAO,KAAK,2BACZ,OAAO,KAAK,4BACZ,mBACA,YACA,iBACF;EAEA,GAAG,QACD,gDAAgD,OAAO,oBACzD;EAEA,MAAM,oBAAoB,IAAI,kBAAkB,sBAAsB;EAEtE,MAAM,oCAAoC,KAAK,IAC7C,OAAO,mBAAmB,GAC1B,CACF;EACA,MAAM,yBAAyB,KAAK,IAAI,OAAO,kBAAkB,CAAC;EAElE,OAAO,IAAI,kBACT,QACA,QACA,OACA,OAAO,QACP,IACA,OACA,IAAI,eACF,QACA,OAAO,KACP,IAAI,YACF,QACA,aACA,OACA,KAAA,GACA,gBACF,GACA,OACA,gBAAgB,yBAAyB,EAAE,GAC3C,IACA,yBAEE,oBAAoB,IAChB,oCACA,wBACN,OAAO,oBACP,QACA,sBACA,UACF,GACA,KACA,kBACA,OAAO,IAAI,sBACX,mBACA,oBACA,wBACA,eACA,KAAA,GACA,KAAA,GACA,KAAA,GACA,KAAA,GACA,UACF;CACF;CA2BA,MAAM,SAAS,IAAI,OACjB,IACA,QACA,mBA5BqB,cAClB,OACC,IAAI,eACF,GACG,YAAY,aAAa,SAAS,CAAC,CACnC,YAAY,iBAAiB,EAAE,GAClC,OACA,IACA,YACA,QACA,iBACF,IACF,KAAA,GAGF,eAAe,KAAA,IACX,KAAA,KACC,IAAY,uBACX,IAAI,cACF,QACA,GAAG,YAAY,iBAAiB,EAAE,GAClC,IACA,kBACF,GAQN,QACA,iBACF;CAEA,wBAAwB,IAAI,MAAM;CAElC,SAAc,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CAE9D,IAAI;EACF,OAAO,MAAM,eAAe,IAAI,QAAQ,MAAM;CAChD,UAAU;EAER,iBAAiB;CACnB;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}

package/out/zero-cache/src/server/worker-dispatcher.js

@@ -103,6 +103,6 @@ function parsePath(url) {
 	return URL_PATTERN.match(url.pathname) || void 0;
 }
 //#endregion
-export { WorkerDispatcher };
+export { WorkerDispatcher, parsePath };
 
 //# sourceMappingURL=worker-dispatcher.js.map