@rocicorp/zero 1.12.0-canary.12 → 1.12.0-canary.13

package/out/zero-cache/src/server/replicator.js.map

@@ -1 +1 @@
-{"version":3,"file":"replicator.js","names":["v.parse"],"sources":["../../../../../zero-cache/src/server/replicator.ts"],"sourcesContent":["import {stat} from 'node:fs/promises';\nimport {pid} from 'node:process';\nimport type {ObservableCallback} from '@opentelemetry/api';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport type {\n  LitestreamConfig,\n  NormalizedZeroConfig,\n} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {\n  ChangeStreamerHttpClient,\n  type ReservationFollowup,\n  type SnapshotReserver,\n} from '../services/change-streamer/change-streamer-http.ts';\nimport {reserveAndGetSnapshotStatus} from '../services/change-streamer/snapshot.ts';\nimport {\n  exitAfter,\n  runUntilKilled,\n  SupportingProcesses,\n} from '../services/life-cycle.ts';\nimport {\n  readLocalReplica,\n  startReplicaBackupProcess,\n  tryRestore,\n  type RestoreAttempt,\n  type RestoreResult,\n} from '../services/litestream/commands.ts';\nimport {\n  litestreamRestoreDuration,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreRuns,\n} from '../services/litestream/metrics.ts';\nimport {RestoreProgressReporter} from '../services/litestream/restore-progress.ts';\nimport {ReplicationStatusPublisher} from '../services/replicator/replication-status.ts';\nimport {\n  ReplicatorService,\n  type ReplicatorMode,\n} from '../services/replicator/replicator.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {\n  ThreadWriteWorkerClient,\n  type ForceCheckpointConfig,\n} from '../services/replicator/write-worker-client.ts';\nimport {\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type DrainRequestedMessage,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {\n  deleteStaleChangeLog,\n  getPragmaConfig,\n  replicaFileModeSchema,\n  setUpMessageHandlers,\n  setupReplica,\n  type WalMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {SERVING_CHECKPOINTER_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length > 0, `replicator mode not specified`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n  const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n  const mode: ReplicatorMode = fileMode === 'backup' ? 'backup' : 'serving';\n  const runningLocalChangeStreamer =\n    config.changeStreamer.mode === 'dedicated' && !config.changeStreamer.uri;\n  const workerName = `${mode}-replicator`;\n  installProfileHandler(parent, workerName);\n  startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);\n  lc = createLogContext(config, workerName);\n  const unregisterInitialCorruptionDiagnosticTarget =\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: `${workerName} replica`,\n        dbPath: config.replica.file,\n      },\n      config.sqliteCorruptionChecks,\n    );\n  initEventSink(lc, config);\n\n  const shard = getShardConfig(config);\n  const {\n    taskID,\n    change,\n    changeStreamer: {\n      port,\n      uri: changeStreamerURI = runningLocalChangeStreamer\n        ? `http://localhost:${port}/`\n        : undefined,\n    },\n    replicationLag: {initialReadinessThresholdMs, maxReadinessDelayMs},\n  } = config;\n  const changeStreamer = new ChangeStreamerHttpClient(\n    lc,\n    shard,\n    change.db,\n    changeStreamerURI,\n  );\n\n  // Remote view-syncers restore from the replication-manager, which supplies\n  // the backup URL. A local change-streamer owns the canonical replica and\n  // must not infer backup configuration from bundled executable paths.\n  //\n  // The reservation's connection (`initialConnection`) is threaded through to\n  // IncrementalSyncer's first subscribe(), so the replication-manager that\n  // reserved the change log for this restore is the one that serves the\n  // subscription (single-connection handoff).\n  let initialConnection: ReservationFollowup | undefined;\n  if (\n    fileMode === 'serving' &&\n    !runningLocalChangeStreamer &&\n    (config.litestream.executable || config.litestream.executableV5)\n  ) {\n    initialConnection = await restoreReplica(lc, config, changeStreamer);\n  }\n\n  const {\n    file: dbPath,\n    walMode,\n    pageSize,\n  } = await setupReplica(lc, fileMode, config.replica);\n  unregisterInitialCorruptionDiagnosticTarget();\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: `${workerName} replica`,\n      dbPath,\n    },\n    config.sqliteCorruptionChecks,\n  );\n\n  // A file left behind by a run with the writer enabled would otherwise hold\n  // local disk indefinitely. See `replicatorDeletesStaleChangeLog` for why\n  // this is keyed on the config flag and nothing else.\n  deleteStaleChangeLog(config.changeStreamer.sqliteChangeLogMode, dbPath);\n\n  setupMetrics(lc, dbPath, walMode, pageSize);\n\n  // Create the write worker for async SQLite writes. Enable write-path\n  // checkpoint backpressure only for the backup replicator on litestream v5,\n  // which owns the replica litestream backs up.\n  const pragmas = getPragmaConfig();\n  const checkpoint: ForceCheckpointConfig | null =\n    mode === 'backup' &&\n    config.litestream.backupURL &&\n    config.litestream.backupUsingV5 &&\n    config.litestream.forceCheckpointThresholdMB\n      ? {\n          checkpointThresholdPages:\n            (config.litestream.forceCheckpointThresholdMB * 1024 ** 2) /\n            pageSize,\n          maxWalPages: !config.litestream.maxWalSizeMB\n            ? undefined\n            : (config.litestream.maxWalSizeMB * 1024 ** 2) / pageSize,\n        }\n      : null;\n\n  const workerClient = new ThreadWriteWorkerClient();\n  await workerClient.init(dbPath, mode, pragmas, config.log, checkpoint);\n\n  // The backup-replicator manages the litestream backup process, which\n  // checkpoints the backup replica. A serving replicator instead manages a\n  // checkpointer for its replica. Either is started after the write worker\n  // has prepared the db file in order to avoid contending on the sqlite lock.\n  const processes = new SupportingProcesses(lc, parent);\n  if (mode === 'backup') {\n    processes.add(\n      startReplicaBackupProcess(lc, config.litestream, dbPath),\n      'litestream',\n    );\n  } else {\n    processes.add(\n      childWorker(SERVING_CHECKPOINTER_URL, env, dbPath),\n      'serving-checkpointer',\n    );\n  }\n\n  const replicator = new ReplicatorService(\n    lc,\n    taskID,\n    `${workerName}-${pid}`,\n    mode,\n    changeStreamer,\n    workerClient,\n    runningLocalChangeStreamer\n      ? // publish ReplicationStatusEvents from backup-replicator only\n        ReplicationStatusPublisher.forReplicaFile(dbPath)\n      : null,\n    {\n      readinessThresholdMs: initialReadinessThresholdMs,\n      maxReadinessDelayMs,\n    },\n    initialConnection,\n    // View-syncers (i.e. with a remote change-streamer) switch over to\n    // another replication-manager when theirs drains.\n    runningLocalChangeStreamer\n      ? undefined\n      : {\n          onRestoreGranted: () =>\n            parent.send<DrainRequestedMessage>(['drainRequested', {}]),\n          replicaFile: dbPath,\n        },\n  );\n\n  setUpMessageHandlers(lc, replicator, parent);\n\n  void replicator.ready().then(() => {\n    parent.send(['ready', {ready: true}]);\n  });\n\n  return runUntilKilled(lc, parent, replicator, processes);\n}\n\nexport function setupMetrics(\n  lc: LogContext,\n  file: string,\n  walMode: WalMode,\n  pageSize: number,\n) {\n  let db: Database | undefined;\n  const getDb = () => {\n    if (!db) {\n      try {\n        db = new Database(lc, file, {readonly: true});\n      } catch (e) {\n        lc.warn?.(`unable to open ${file} for wal metrics`, e);\n      }\n    }\n    return db;\n  };\n\n  getOrCreateGauge('replica', 'db_size', {\n    description:\n      `The size of the replica's main db file, ` +\n      `which does not include the wal file(s)`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, file));\n\n  getOrCreateGauge('replica', 'wal_size', {\n    description: `The size of the replica's wal file`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, `${file}-wal`));\n\n  if (walMode === 'wal2') {\n    getOrCreateGauge('replica', 'wal2_size', {\n      description: `The size of the replica's wal2 file`,\n      unit: 'bytes',\n    }).addCallback(observeFileSize(lc, `${file}-wal2`));\n  }\n\n  getOrCreateGauge('replica', 'wal_active_bytes', {\n    description:\n      `The actual size of active WAL frames currently in use ` +\n      `(excluding pre-allocated or reclaimed space).`,\n    unit: 'bytes',\n  }).addCallback(observeWalActiveBytes(lc, getDb, pageSize));\n\n  getOrCreateGauge('replica', 'wal_uncheckpointed_bytes', {\n    description:\n      `The size of un-checkpointed WAL frames that have not yet ` +\n      `been backfilled into the main database.`,\n    unit: 'bytes',\n  }).addCallback(observeWalUncheckpointedBytes(lc, getDb, pageSize));\n\n  // A whole-database total rather than a per-file one. Its counterpart,\n  // `sqlite_change_log.file_bytes`, is emitted by the change-streamer, which is\n  // the process that writes the log.\n  getOrCreateGauge('replica', 'file_bytes', {\n    description:\n      `The replica's total on-disk footprint: its main db file plus its ` +\n      `wal sidecars.`,\n    unit: 'By',\n  }).addCallback(observeSQLiteFileBytes(lc, file));\n}\n\nfunction observeFileSize(lc: LogContext, file: string): ObservableCallback {\n  return async o => {\n    try {\n      const stats = await stat(file);\n      o.observe(stats.size);\n    } catch (e) {\n      lc.warn?.(`unable to stat ${file} for size metrics`, e);\n    }\n  };\n}\n\nfunction observeWalActiveBytes(\n  lc: LogContext,\n  getDb: () => Database | undefined,\n  pageSize: number,\n): ObservableCallback {\n  return o => {\n    try {\n      const db = getDb();\n      if (!db) {\n        return;\n      }\n      const [{log}] = db.pragma<{log: number}>('wal_checkpoint(NOOP)');\n      if (typeof log === 'number') {\n        o.observe(Math.max(0, log) * pageSize);\n      }\n    } catch (e) {\n      lc.warn?.('unable to inspect wal_checkpoint for active bytes metric', e);\n    }\n  };\n}\n\nfunction observeWalUncheckpointedBytes(\n  lc: LogContext,\n  getDb: () => Database | undefined,\n  pageSize: number,\n): ObservableCallback {\n  return o => {\n    try {\n      const db = getDb();\n      if (!db) {\n        return;\n      }\n      const [{log, checkpointed}] = db.pragma<{\n        log: number;\n        checkpointed: number;\n      }>('wal_checkpoint(NOOP)');\n      if (typeof log === 'number' && typeof checkpointed === 'number') {\n        o.observe(Math.max(0, log - checkpointed) * pageSize);\n      }\n    } catch (e) {\n      lc.warn?.(\n        'unable to inspect wal_checkpoint for uncheckpointed bytes metric',\n        e,\n      );\n    }\n  };\n}\n\nfunction observeSQLiteFileBytes(\n  lc: LogContext,\n  file: string,\n): ObservableCallback {\n  return async o => o.observe(await sqliteFileBytes(lc, file));\n}\n\nconst RETRY_INTERVAL_MS = 3000;\n\n// The number of times a view-syncer retries a snapshot reservation from a\n// replication-manager with an older replica version than its local replica\n// (e.g. a not-yet-terminated replication-manager during a resync rollout),\n// before accepting the older version (e.g. if the upstream was legitimately\n// replaced by one with a lower replica version).\nconst MAX_OLDER_REPLICA_VERSION_RETRIES = 20;\n\n// View-syncers wait indefinitely for the replication-manager to publish a\n// restorable backup. On a fresh stack the first backup is not durable until\n// the initial sync completes and litestream uploads the initial snapshot,\n// which can take many minutes for a large replica. The platform's startup\n// probe budget (which scales with replica size) is the backstop, so\n// restoreReplica must not impose its own shorter cap and self-terminate\n// while the backup is still being produced.\n//\n// Returns the `followup` of the reservation whose backup was actually\n// restored, still open, so the caller can hand it to IncrementalSyncer's\n// first subscribe() via `followup.subscribe()`: the same replication-manager\n// that reserved the change log is the one that serves the subscription.\n/** Exported for testing. */\nexport async function restoreReplica(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  changeStreamer: SnapshotReserver,\n): Promise<ReservationFollowup> {\n  const start = performance.now();\n  let backupURL: string | undefined;\n  let result: RestoreResult | undefined;\n  const progress = new RestoreProgressReporter(lc, config.replica.file);\n  let olderReplicaVersionRetries = 0;\n  // Describing the local replica (if any) allows the replication-manager to\n  // account for replicas that are being replaced (see RestoreAdmission). It\n  // is read once, before the first attempt, as a failed attempt may have\n  // already deleted it; it continues to describe the replica being replaced\n  // (and the version not to downgrade from) on every retry.\n  const localReplica = readLocalReplica(lc, config.replica.file);\n  const localReplicaVersion = localReplica?.replicaVersion;\n  try {\n    for (;;) {\n      const {reserved, followup} = await reserveAndGetSnapshotStatus(\n        lc,\n        config.taskID,\n        changeStreamer,\n        localReplica,\n      );\n      if (\n        localReplicaVersion !== undefined &&\n        reserved.replicaVersion < localReplicaVersion &&\n        olderReplicaVersionRetries++ < MAX_OLDER_REPLICA_VERSION_RETRIES\n      ) {\n        // Don't replace a newer local replica with an older one, e.g. from\n        // a replication-manager that has yet to be replaced by a resync.\n        followup.cancel();\n        lc.info?.(\n          `replication-manager is at an older replica version ` +\n            `(${reserved.replicaVersion}) than the local replica ` +\n            `(${localReplicaVersion}). retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n        );\n        await sleep(RETRY_INTERVAL_MS);\n        continue;\n      }\n      // The backupURL comes from the replication-manager's snapshot response.\n      ({backupURL} = reserved);\n      const litestream: LitestreamConfig = {...config.litestream, backupURL};\n      progress.start(reserved.replicaSize);\n      let attempt: RestoreAttempt;\n      try {\n        attempt = await tryRestore(\n          lc,\n          litestream,\n          config.replica.file,\n          reserved,\n          'view_syncer',\n          followup.signal, // abort the restore if the reservation disconnects\n        );\n      } catch (e) {\n        if (followup.signal.aborted) {\n          lc.info?.(\n            `reservation disconnected during restore. retrying`,\n            String(e),\n          );\n          continue;\n        }\n        followup.cancel();\n        throw e;\n      } finally {\n        progress.stop();\n      }\n      if (attempt.restored) {\n        result = attempt.result;\n        progress.done();\n        return followup; // connection to be converted to a subscription\n      }\n      // This attempt's reservation is stale; a fresh one is made on retry.\n      followup.cancel();\n      lc.info?.(\n        `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n      );\n      await sleep(RETRY_INTERVAL_MS);\n    }\n  } finally {\n    progress.stop();\n    const attrs = litestreamRestoreMetricAttrs(\n      config.litestream,\n      'view_syncer',\n      backupURL,\n    );\n    const labels = {...attrs, result: result ?? 'error'};\n    litestreamRestoreRuns().add(1, labels);\n    litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyEA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,SAAS,GAAG,+BAA+B;CACvD,MAAM,WAAWA,MAAQ,KAAK,IAAI,qBAAqB;CAEvD,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CACjE,MAAM,OAAuB,aAAa,WAAW,WAAW;CAChE,MAAM,6BACJ,OAAO,eAAe,SAAS,eAAe,CAAC,OAAO,eAAe;CACvE,MAAM,aAAa,GAAG,KAAK;CAC3B,sBAAsB,QAAQ,UAAU;CACxC,cAAc,iBAAiB,QAAQ,YAAY,GAAG,KAAK,GAAG,YAAY,CAAC;CAC3E,KAAK,iBAAiB,QAAQ,UAAU;CACxC,MAAM,8CACJ,yCACE;EACE,WAAW,GAAG,WAAW;EACzB,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACF,cAAc,IAAI,MAAM;CAExB,MAAM,QAAQ,eAAe,MAAM;CACnC,MAAM,EACJ,QACA,QACA,gBAAgB,EACd,MACA,KAAK,oBAAoB,6BACrB,oBAAoB,KAAK,KACzB,KAAA,KAEN,gBAAgB,EAAC,6BAA6B,0BAC5C;CACJ,MAAM,iBAAiB,IAAI,yBACzB,IACA,OACA,OAAO,IACP,iBACF;CAUA,IAAI;CACJ,IACE,aAAa,aACb,CAAC,+BACA,OAAO,WAAW,cAAc,OAAO,WAAW,eAEnD,oBAAoB,MAAM,eAAe,IAAI,QAAQ,cAAc;CAGrE,MAAM,EACJ,MAAM,QACN,SACA,aACE,MAAM,aAAa,IAAI,UAAU,OAAO,OAAO;CACnD,4CAA4C;CAC5C,yCACE;EACE,WAAW,GAAG,WAAW;EACzB;CACF,GACA,OAAO,sBACT;CAKA,qBAAqB,OAAO,eAAe,qBAAqB,MAAM;CAEtE,aAAa,IAAI,QAAQ,SAAS,QAAQ;CAK1C,MAAM,UAAU,gBAAgB;CAChC,MAAM,aACJ,SAAS,YACT,OAAO,WAAW,aAClB,OAAO,WAAW,iBAClB,OAAO,WAAW,6BACd;EACE,0BACG,OAAO,WAAW,6BAA6B,QAAQ,IACxD;EACF,aAAa,CAAC,OAAO,WAAW,eAC5B,KAAA,IACC,OAAO,WAAW,eAAe,QAAQ,IAAK;CACrD,IACA;CAEN,MAAM,eAAe,IAAI,wBAAwB;CACjD,MAAM,aAAa,KAAK,QAAQ,MAAM,SAAS,OAAO,KAAK,UAAU;CAMrE,MAAM,YAAY,IAAI,oBAAoB,IAAI,MAAM;CACpD,IAAI,SAAS,UACX,UAAU,IACR,0BAA0B,IAAI,OAAO,YAAY,MAAM,GACvD,YACF;MAEA,UAAU,IACR,YAAY,0BAA0B,KAAK,MAAM,GACjD,sBACF;CAGF,MAAM,aAAa,IAAI,kBACrB,IACA,QACA,GAAG,WAAW,GAAG,OACjB,MACA,gBACA,cACA,6BAEI,2BAA2B,eAAe,MAAM,IAChD,MACJ;EACE,sBAAsB;EACtB;CACF,GACA,mBAGA,6BACI,KAAA,IACA;EACE,wBACE,OAAO,KAA4B,CAAC,kBAAkB,CAAC,CAAC,CAAC;EAC3D,aAAa;CACf,CACN;CAEA,qBAAqB,IAAI,YAAY,MAAM;CAE3C,WAAgB,MAAM,CAAC,CAAC,WAAW;EACjC,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CACtC,CAAC;CAED,OAAO,eAAe,IAAI,QAAQ,YAAY,SAAS;AACzD;AAEA,SAAgB,aACd,IACA,MACA,SACA,UACA;CACA,IAAI;CACJ,MAAM,cAAc;EAClB,IAAI,CAAC,IACH,IAAI;GACF,KAAK,IAAI,SAAS,IAAI,MAAM,EAAC,UAAU,KAAI,CAAC;EAC9C,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,mBAAmB,CAAC;EACvD;EAEF,OAAO;CACT;CAEA,iBAAiB,WAAW,WAAW;EACrC,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,gBAAgB,IAAI,IAAI,CAAC;CAExC,iBAAiB,WAAW,YAAY;EACtC,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,gBAAgB,IAAI,GAAG,KAAK,KAAK,CAAC;CAEjD,IAAI,YAAY,QACd,iBAAiB,WAAW,aAAa;EACvC,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,gBAAgB,IAAI,GAAG,KAAK,MAAM,CAAC;CAGpD,iBAAiB,WAAW,oBAAoB;EAC9C,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,sBAAsB,IAAI,OAAO,QAAQ,CAAC;CAEzD,iBAAiB,WAAW,4BAA4B;EACtD,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,8BAA8B,IAAI,OAAO,QAAQ,CAAC;CAKjE,iBAAiB,WAAW,cAAc;EACxC,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,uBAAuB,IAAI,IAAI,CAAC;AACjD;AAEA,SAAS,gBAAgB,IAAgB,MAAkC;CACzE,OAAO,OAAM,MAAK;EAChB,IAAI;GACF,MAAM,QAAQ,MAAM,KAAK,IAAI;GAC7B,EAAE,QAAQ,MAAM,IAAI;EACtB,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;EACxD;CACF;AACF;AAEA,SAAS,sBACP,IACA,OACA,UACoB;CACpB,QAAO,MAAK;EACV,IAAI;GACF,MAAM,KAAK,MAAM;GACjB,IAAI,CAAC,IACH;GAEF,MAAM,CAAC,EAAC,SAAQ,GAAG,OAAsB,sBAAsB;GAC/D,IAAI,OAAO,QAAQ,UACjB,EAAE,QAAQ,KAAK,IAAI,GAAG,GAAG,IAAI,QAAQ;EAEzC,SAAS,GAAG;GACV,GAAG,OAAO,4DAA4D,CAAC;EACzE;CACF;AACF;AAEA,SAAS,8BACP,IACA,OACA,UACoB;CACpB,QAAO,MAAK;EACV,IAAI;GACF,MAAM,KAAK,MAAM;GACjB,IAAI,CAAC,IACH;GAEF,MAAM,CAAC,EAAC,KAAK,kBAAiB,GAAG,OAG9B,sBAAsB;GACzB,IAAI,OAAO,QAAQ,YAAY,OAAO,iBAAiB,UACrD,EAAE,QAAQ,KAAK,IAAI,GAAG,MAAM,YAAY,IAAI,QAAQ;EAExD,SAAS,GAAG;GACV,GAAG,OACD,oEACA,CACF;EACF;CACF;AACF;AAEA,SAAS,uBACP,IACA,MACoB;CACpB,OAAO,OAAM,MAAK,EAAE,QAAQ,MAAM,gBAAgB,IAAI,IAAI,CAAC;AAC7D;AAEA,IAAM,oBAAoB;AAO1B,IAAM,oCAAoC;;AAe1C,eAAsB,eACpB,IACA,QACA,gBAC8B;CAC9B,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;CACJ,MAAM,WAAW,IAAI,wBAAwB,IAAI,OAAO,QAAQ,IAAI;CACpE,IAAI,6BAA6B;CAMjC,MAAM,eAAe,iBAAiB,IAAI,OAAO,QAAQ,IAAI;CAC7D,MAAM,sBAAsB,cAAc;CAC1C,IAAI;EACF,SAAS;GACP,MAAM,EAAC,UAAU,aAAY,MAAM,4BACjC,IACA,OAAO,QACP,gBACA,YACF;GACA,IACE,wBAAwB,KAAA,KACxB,SAAS,iBAAiB,uBAC1B,+BAA+B,mCAC/B;IAGA,SAAS,OAAO;IAChB,GAAG,OACD,uDACM,SAAS,eAAe,4BACxB,oBAAoB,iBAAiB,oBAAoB,IAAK,SACtE;IACA,MAAM,MAAM,iBAAiB;IAC7B;GACF;GAEA,CAAC,CAAC,aAAa;GACf,MAAM,aAA+B;IAAC,GAAG,OAAO;IAAY;GAAS;GACrE,SAAS,MAAM,SAAS,WAAW;GACnC,IAAI;GACJ,IAAI;IACF,UAAU,MAAM,WACd,IACA,YACA,OAAO,QAAQ,MACf,UACA,eACA,SAAS,MACX;GACF,SAAS,GAAG;IACV,IAAI,SAAS,OAAO,SAAS;KAC3B,GAAG,OACD,qDACA,OAAO,CAAC,CACV;KACA;IACF;IACA,SAAS,OAAO;IAChB,MAAM;GACR,UAAU;IACR,SAAS,KAAK;GAChB;GACA,IAAI,QAAQ,UAAU;IACpB,SAAS,QAAQ;IACjB,SAAS,KAAK;IACd,OAAO;GACT;GAEA,SAAS,OAAO;GAChB,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,UAAU;EACR,SAAS,KAAK;EAMd,MAAM,SAAS;GAAC,GALF,6BACZ,OAAO,YACP,eACA,SAEiB;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,CAAC,CAAC,IAAI,GAAG,MAAM;EACrC,0BAA0B,CAAC,CAAC,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}
+{"version":3,"file":"replicator.js","names":["v.parse"],"sources":["../../../../../zero-cache/src/server/replicator.ts"],"sourcesContent":["import {stat} from 'node:fs/promises';\nimport {pid} from 'node:process';\nimport type {ObservableCallback} from '@opentelemetry/api';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport type {\n  LitestreamConfig,\n  NormalizedZeroConfig,\n} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {\n  ChangeStreamerHttpClient,\n  type ReservationFollowup,\n  type SnapshotReserver,\n} from '../services/change-streamer/change-streamer-http.ts';\nimport {reserveAndGetSnapshotStatus} from '../services/change-streamer/snapshot.ts';\nimport {\n  exitAfter,\n  runUntilKilled,\n  SupportingProcesses,\n} from '../services/life-cycle.ts';\nimport {\n  readLocalReplica,\n  startReplicaBackupProcess,\n  tryRestore,\n  type RestoreAttempt,\n  type RestoreResult,\n} from '../services/litestream/commands.ts';\nimport {\n  litestreamRestoreDuration,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreRuns,\n} from '../services/litestream/metrics.ts';\nimport {RestoreProgressReporter} from '../services/litestream/restore-progress.ts';\nimport {ReplicationStatusPublisher} from '../services/replicator/replication-status.ts';\nimport {\n  ReplicatorService,\n  type ReplicatorMode,\n} from '../services/replicator/replicator.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {\n  ThreadWriteWorkerClient,\n  type ForceCheckpointConfig,\n} from '../services/replicator/write-worker-client.ts';\nimport {\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type DrainRequestedMessage,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {\n  deleteStaleChangeLog,\n  getPragmaConfig,\n  replicaFileModeSchema,\n  setUpMessageHandlers,\n  setupReplica,\n  type WalMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {SERVING_CHECKPOINTER_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length > 0, `replicator mode not specified`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n  const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n  const mode: ReplicatorMode = fileMode === 'backup' ? 'backup' : 'serving';\n  const runningLocalChangeStreamer =\n    config.changeStreamer.mode === 'dedicated' && !config.changeStreamer.uri;\n  const workerName = `${mode}-replicator`;\n  installProfileHandler(parent, workerName);\n  startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);\n  lc = createLogContext(config, workerName);\n  const unregisterInitialCorruptionDiagnosticTarget =\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: `${workerName} replica`,\n        dbPath: config.replica.file,\n      },\n      config.sqliteCorruptionChecks,\n    );\n  initEventSink(lc, config);\n\n  const shard = getShardConfig(config);\n  const {\n    taskID,\n    change,\n    changeStreamer: {\n      port,\n      uri: changeStreamerURI = runningLocalChangeStreamer\n        ? `http://localhost:${port}/`\n        : undefined,\n      secret,\n    },\n    replicationLag: {initialReadinessThresholdMs, maxReadinessDelayMs},\n  } = config;\n  if (!runningLocalChangeStreamer && secret === undefined) {\n    lc.warn?.(\n      `No --change-streamer-secret is configured: connecting to the ` +\n        `change-streamer without one. Set ZERO_CHANGE_STREAMER_SECRET to ` +\n        `the same value on the replication-manager and on every view-syncer.`,\n    );\n  }\n  const changeStreamer = new ChangeStreamerHttpClient(\n    lc,\n    shard,\n    change.db,\n    changeStreamerURI,\n    secret,\n  );\n\n  // Remote view-syncers restore from the replication-manager, which supplies\n  // the backup URL. A local change-streamer owns the canonical replica and\n  // must not infer backup configuration from bundled executable paths.\n  //\n  // The reservation's connection (`initialConnection`) is threaded through to\n  // IncrementalSyncer's first subscribe(), so the replication-manager that\n  // reserved the change log for this restore is the one that serves the\n  // subscription (single-connection handoff).\n  let initialConnection: ReservationFollowup | undefined;\n  if (\n    fileMode === 'serving' &&\n    !runningLocalChangeStreamer &&\n    (config.litestream.executable || config.litestream.executableV5)\n  ) {\n    initialConnection = await restoreReplica(lc, config, changeStreamer);\n  }\n\n  const {\n    file: dbPath,\n    walMode,\n    pageSize,\n  } = await setupReplica(lc, fileMode, config.replica);\n  unregisterInitialCorruptionDiagnosticTarget();\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: `${workerName} replica`,\n      dbPath,\n    },\n    config.sqliteCorruptionChecks,\n  );\n\n  // A file left behind by a run with the writer enabled would otherwise hold\n  // local disk indefinitely. See `replicatorDeletesStaleChangeLog` for why\n  // this is keyed on the config flag and nothing else.\n  deleteStaleChangeLog(config.changeStreamer.sqliteChangeLogMode, dbPath);\n\n  setupMetrics(lc, dbPath, walMode, pageSize);\n\n  // Create the write worker for async SQLite writes. Enable write-path\n  // checkpoint backpressure only for the backup replicator on litestream v5,\n  // which owns the replica litestream backs up.\n  const pragmas = getPragmaConfig();\n  const checkpoint: ForceCheckpointConfig | null =\n    mode === 'backup' &&\n    config.litestream.backupURL &&\n    config.litestream.backupUsingV5 &&\n    config.litestream.forceCheckpointThresholdMB\n      ? {\n          checkpointThresholdPages:\n            (config.litestream.forceCheckpointThresholdMB * 1024 ** 2) /\n            pageSize,\n          maxWalPages: !config.litestream.maxWalSizeMB\n            ? undefined\n            : (config.litestream.maxWalSizeMB * 1024 ** 2) / pageSize,\n        }\n      : null;\n\n  const workerClient = new ThreadWriteWorkerClient();\n  await workerClient.init(dbPath, mode, pragmas, config.log, checkpoint);\n\n  // The backup-replicator manages the litestream backup process, which\n  // checkpoints the backup replica. A serving replicator instead manages a\n  // checkpointer for its replica. Either is started after the write worker\n  // has prepared the db file in order to avoid contending on the sqlite lock.\n  const processes = new SupportingProcesses(lc, parent);\n  if (mode === 'backup') {\n    processes.add(\n      startReplicaBackupProcess(lc, config.litestream, dbPath),\n      'litestream',\n    );\n  } else {\n    processes.add(\n      childWorker(SERVING_CHECKPOINTER_URL, env, dbPath),\n      'serving-checkpointer',\n    );\n  }\n\n  const replicator = new ReplicatorService(\n    lc,\n    taskID,\n    `${workerName}-${pid}`,\n    mode,\n    changeStreamer,\n    workerClient,\n    runningLocalChangeStreamer\n      ? // publish ReplicationStatusEvents from backup-replicator only\n        ReplicationStatusPublisher.forReplicaFile(dbPath)\n      : null,\n    {\n      readinessThresholdMs: initialReadinessThresholdMs,\n      maxReadinessDelayMs,\n    },\n    initialConnection,\n    // View-syncers (i.e. with a remote change-streamer) switch over to\n    // another replication-manager when theirs drains.\n    runningLocalChangeStreamer\n      ? undefined\n      : {\n          onRestoreGranted: () =>\n            parent.send<DrainRequestedMessage>(['drainRequested', {}]),\n          replicaFile: dbPath,\n        },\n  );\n\n  setUpMessageHandlers(lc, replicator, parent);\n\n  void replicator.ready().then(() => {\n    parent.send(['ready', {ready: true}]);\n  });\n\n  return runUntilKilled(lc, parent, replicator, processes);\n}\n\nexport function setupMetrics(\n  lc: LogContext,\n  file: string,\n  walMode: WalMode,\n  pageSize: number,\n) {\n  let db: Database | undefined;\n  const getDb = () => {\n    if (!db) {\n      try {\n        db = new Database(lc, file, {readonly: true});\n      } catch (e) {\n        lc.warn?.(`unable to open ${file} for wal metrics`, e);\n      }\n    }\n    return db;\n  };\n\n  getOrCreateGauge('replica', 'db_size', {\n    description:\n      `The size of the replica's main db file, ` +\n      `which does not include the wal file(s)`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, file));\n\n  getOrCreateGauge('replica', 'wal_size', {\n    description: `The size of the replica's wal file`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, `${file}-wal`));\n\n  if (walMode === 'wal2') {\n    getOrCreateGauge('replica', 'wal2_size', {\n      description: `The size of the replica's wal2 file`,\n      unit: 'bytes',\n    }).addCallback(observeFileSize(lc, `${file}-wal2`));\n  }\n\n  getOrCreateGauge('replica', 'wal_active_bytes', {\n    description:\n      `The actual size of active WAL frames currently in use ` +\n      `(excluding pre-allocated or reclaimed space).`,\n    unit: 'bytes',\n  }).addCallback(observeWalActiveBytes(lc, getDb, pageSize));\n\n  getOrCreateGauge('replica', 'wal_uncheckpointed_bytes', {\n    description:\n      `The size of un-checkpointed WAL frames that have not yet ` +\n      `been backfilled into the main database.`,\n    unit: 'bytes',\n  }).addCallback(observeWalUncheckpointedBytes(lc, getDb, pageSize));\n\n  // A whole-database total rather than a per-file one. Its counterpart,\n  // `sqlite_change_log.file_bytes`, is emitted by the change-streamer, which is\n  // the process that writes the log.\n  getOrCreateGauge('replica', 'file_bytes', {\n    description:\n      `The replica's total on-disk footprint: its main db file plus its ` +\n      `wal sidecars.`,\n    unit: 'By',\n  }).addCallback(observeSQLiteFileBytes(lc, file));\n}\n\nfunction observeFileSize(lc: LogContext, file: string): ObservableCallback {\n  return async o => {\n    try {\n      const stats = await stat(file);\n      o.observe(stats.size);\n    } catch (e) {\n      lc.warn?.(`unable to stat ${file} for size metrics`, e);\n    }\n  };\n}\n\nfunction observeWalActiveBytes(\n  lc: LogContext,\n  getDb: () => Database | undefined,\n  pageSize: number,\n): ObservableCallback {\n  return o => {\n    try {\n      const db = getDb();\n      if (!db) {\n        return;\n      }\n      const [{log}] = db.pragma<{log: number}>('wal_checkpoint(NOOP)');\n      if (typeof log === 'number') {\n        o.observe(Math.max(0, log) * pageSize);\n      }\n    } catch (e) {\n      lc.warn?.('unable to inspect wal_checkpoint for active bytes metric', e);\n    }\n  };\n}\n\nfunction observeWalUncheckpointedBytes(\n  lc: LogContext,\n  getDb: () => Database | undefined,\n  pageSize: number,\n): ObservableCallback {\n  return o => {\n    try {\n      const db = getDb();\n      if (!db) {\n        return;\n      }\n      const [{log, checkpointed}] = db.pragma<{\n        log: number;\n        checkpointed: number;\n      }>('wal_checkpoint(NOOP)');\n      if (typeof log === 'number' && typeof checkpointed === 'number') {\n        o.observe(Math.max(0, log - checkpointed) * pageSize);\n      }\n    } catch (e) {\n      lc.warn?.(\n        'unable to inspect wal_checkpoint for uncheckpointed bytes metric',\n        e,\n      );\n    }\n  };\n}\n\nfunction observeSQLiteFileBytes(\n  lc: LogContext,\n  file: string,\n): ObservableCallback {\n  return async o => o.observe(await sqliteFileBytes(lc, file));\n}\n\nconst RETRY_INTERVAL_MS = 3000;\n\n// The number of times a view-syncer retries a snapshot reservation from a\n// replication-manager with an older replica version than its local replica\n// (e.g. a not-yet-terminated replication-manager during a resync rollout),\n// before accepting the older version (e.g. if the upstream was legitimately\n// replaced by one with a lower replica version).\nconst MAX_OLDER_REPLICA_VERSION_RETRIES = 20;\n\n// View-syncers wait indefinitely for the replication-manager to publish a\n// restorable backup. On a fresh stack the first backup is not durable until\n// the initial sync completes and litestream uploads the initial snapshot,\n// which can take many minutes for a large replica. The platform's startup\n// probe budget (which scales with replica size) is the backstop, so\n// restoreReplica must not impose its own shorter cap and self-terminate\n// while the backup is still being produced.\n//\n// Returns the `followup` of the reservation whose backup was actually\n// restored, still open, so the caller can hand it to IncrementalSyncer's\n// first subscribe() via `followup.subscribe()`: the same replication-manager\n// that reserved the change log is the one that serves the subscription.\n/** Exported for testing. */\nexport async function restoreReplica(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  changeStreamer: SnapshotReserver,\n): Promise<ReservationFollowup> {\n  const start = performance.now();\n  let backupURL: string | undefined;\n  let result: RestoreResult | undefined;\n  const progress = new RestoreProgressReporter(lc, config.replica.file);\n  let olderReplicaVersionRetries = 0;\n  // Describing the local replica (if any) allows the replication-manager to\n  // account for replicas that are being replaced (see RestoreAdmission). It\n  // is read once, before the first attempt, as a failed attempt may have\n  // already deleted it; it continues to describe the replica being replaced\n  // (and the version not to downgrade from) on every retry.\n  const localReplica = readLocalReplica(lc, config.replica.file);\n  const localReplicaVersion = localReplica?.replicaVersion;\n  try {\n    for (;;) {\n      const {reserved, followup} = await reserveAndGetSnapshotStatus(\n        lc,\n        config.taskID,\n        changeStreamer,\n        localReplica,\n      );\n      if (\n        localReplicaVersion !== undefined &&\n        reserved.replicaVersion < localReplicaVersion &&\n        olderReplicaVersionRetries++ < MAX_OLDER_REPLICA_VERSION_RETRIES\n      ) {\n        // Don't replace a newer local replica with an older one, e.g. from\n        // a replication-manager that has yet to be replaced by a resync.\n        followup.cancel();\n        lc.info?.(\n          `replication-manager is at an older replica version ` +\n            `(${reserved.replicaVersion}) than the local replica ` +\n            `(${localReplicaVersion}). retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n        );\n        await sleep(RETRY_INTERVAL_MS);\n        continue;\n      }\n      // The backupURL comes from the replication-manager's snapshot response.\n      ({backupURL} = reserved);\n      const litestream: LitestreamConfig = {...config.litestream, backupURL};\n      progress.start(reserved.replicaSize);\n      let attempt: RestoreAttempt;\n      try {\n        attempt = await tryRestore(\n          lc,\n          litestream,\n          config.replica.file,\n          reserved,\n          'view_syncer',\n          followup.signal, // abort the restore if the reservation disconnects\n        );\n      } catch (e) {\n        if (followup.signal.aborted) {\n          lc.info?.(\n            `reservation disconnected during restore. retrying`,\n            String(e),\n          );\n          continue;\n        }\n        followup.cancel();\n        throw e;\n      } finally {\n        progress.stop();\n      }\n      if (attempt.restored) {\n        result = attempt.result;\n        progress.done();\n        return followup; // connection to be converted to a subscription\n      }\n      // This attempt's reservation is stale; a fresh one is made on retry.\n      followup.cancel();\n      lc.info?.(\n        `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n      );\n      await sleep(RETRY_INTERVAL_MS);\n    }\n  } finally {\n    progress.stop();\n    const attrs = litestreamRestoreMetricAttrs(\n      config.litestream,\n      'view_syncer',\n      backupURL,\n    );\n    const labels = {...attrs, result: result ?? 'error'};\n    litestreamRestoreRuns().add(1, labels);\n    litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyEA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,SAAS,GAAG,+BAA+B;CACvD,MAAM,WAAWA,MAAQ,KAAK,IAAI,qBAAqB;CAEvD,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CACjE,MAAM,OAAuB,aAAa,WAAW,WAAW;CAChE,MAAM,6BACJ,OAAO,eAAe,SAAS,eAAe,CAAC,OAAO,eAAe;CACvE,MAAM,aAAa,GAAG,KAAK;CAC3B,sBAAsB,QAAQ,UAAU;CACxC,cAAc,iBAAiB,QAAQ,YAAY,GAAG,KAAK,GAAG,YAAY,CAAC;CAC3E,KAAK,iBAAiB,QAAQ,UAAU;CACxC,MAAM,8CACJ,yCACE;EACE,WAAW,GAAG,WAAW;EACzB,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACF,cAAc,IAAI,MAAM;CAExB,MAAM,QAAQ,eAAe,MAAM;CACnC,MAAM,EACJ,QACA,QACA,gBAAgB,EACd,MACA,KAAK,oBAAoB,6BACrB,oBAAoB,KAAK,KACzB,KAAA,GACJ,UAEF,gBAAgB,EAAC,6BAA6B,0BAC5C;CACJ,IAAI,CAAC,8BAA8B,WAAW,KAAA,GAC5C,GAAG,OACD,kMAGF;CAEF,MAAM,iBAAiB,IAAI,yBACzB,IACA,OACA,OAAO,IACP,mBACA,MACF;CAUA,IAAI;CACJ,IACE,aAAa,aACb,CAAC,+BACA,OAAO,WAAW,cAAc,OAAO,WAAW,eAEnD,oBAAoB,MAAM,eAAe,IAAI,QAAQ,cAAc;CAGrE,MAAM,EACJ,MAAM,QACN,SACA,aACE,MAAM,aAAa,IAAI,UAAU,OAAO,OAAO;CACnD,4CAA4C;CAC5C,yCACE;EACE,WAAW,GAAG,WAAW;EACzB;CACF,GACA,OAAO,sBACT;CAKA,qBAAqB,OAAO,eAAe,qBAAqB,MAAM;CAEtE,aAAa,IAAI,QAAQ,SAAS,QAAQ;CAK1C,MAAM,UAAU,gBAAgB;CAChC,MAAM,aACJ,SAAS,YACT,OAAO,WAAW,aAClB,OAAO,WAAW,iBAClB,OAAO,WAAW,6BACd;EACE,0BACG,OAAO,WAAW,6BAA6B,QAAQ,IACxD;EACF,aAAa,CAAC,OAAO,WAAW,eAC5B,KAAA,IACC,OAAO,WAAW,eAAe,QAAQ,IAAK;CACrD,IACA;CAEN,MAAM,eAAe,IAAI,wBAAwB;CACjD,MAAM,aAAa,KAAK,QAAQ,MAAM,SAAS,OAAO,KAAK,UAAU;CAMrE,MAAM,YAAY,IAAI,oBAAoB,IAAI,MAAM;CACpD,IAAI,SAAS,UACX,UAAU,IACR,0BAA0B,IAAI,OAAO,YAAY,MAAM,GACvD,YACF;MAEA,UAAU,IACR,YAAY,0BAA0B,KAAK,MAAM,GACjD,sBACF;CAGF,MAAM,aAAa,IAAI,kBACrB,IACA,QACA,GAAG,WAAW,GAAG,OACjB,MACA,gBACA,cACA,6BAEI,2BAA2B,eAAe,MAAM,IAChD,MACJ;EACE,sBAAsB;EACtB;CACF,GACA,mBAGA,6BACI,KAAA,IACA;EACE,wBACE,OAAO,KAA4B,CAAC,kBAAkB,CAAC,CAAC,CAAC;EAC3D,aAAa;CACf,CACN;CAEA,qBAAqB,IAAI,YAAY,MAAM;CAE3C,WAAgB,MAAM,CAAC,CAAC,WAAW;EACjC,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CACtC,CAAC;CAED,OAAO,eAAe,IAAI,QAAQ,YAAY,SAAS;AACzD;AAEA,SAAgB,aACd,IACA,MACA,SACA,UACA;CACA,IAAI;CACJ,MAAM,cAAc;EAClB,IAAI,CAAC,IACH,IAAI;GACF,KAAK,IAAI,SAAS,IAAI,MAAM,EAAC,UAAU,KAAI,CAAC;EAC9C,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,mBAAmB,CAAC;EACvD;EAEF,OAAO;CACT;CAEA,iBAAiB,WAAW,WAAW;EACrC,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,gBAAgB,IAAI,IAAI,CAAC;CAExC,iBAAiB,WAAW,YAAY;EACtC,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,gBAAgB,IAAI,GAAG,KAAK,KAAK,CAAC;CAEjD,IAAI,YAAY,QACd,iBAAiB,WAAW,aAAa;EACvC,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,gBAAgB,IAAI,GAAG,KAAK,MAAM,CAAC;CAGpD,iBAAiB,WAAW,oBAAoB;EAC9C,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,sBAAsB,IAAI,OAAO,QAAQ,CAAC;CAEzD,iBAAiB,WAAW,4BAA4B;EACtD,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,8BAA8B,IAAI,OAAO,QAAQ,CAAC;CAKjE,iBAAiB,WAAW,cAAc;EACxC,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,uBAAuB,IAAI,IAAI,CAAC;AACjD;AAEA,SAAS,gBAAgB,IAAgB,MAAkC;CACzE,OAAO,OAAM,MAAK;EAChB,IAAI;GACF,MAAM,QAAQ,MAAM,KAAK,IAAI;GAC7B,EAAE,QAAQ,MAAM,IAAI;EACtB,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;EACxD;CACF;AACF;AAEA,SAAS,sBACP,IACA,OACA,UACoB;CACpB,QAAO,MAAK;EACV,IAAI;GACF,MAAM,KAAK,MAAM;GACjB,IAAI,CAAC,IACH;GAEF,MAAM,CAAC,EAAC,SAAQ,GAAG,OAAsB,sBAAsB;GAC/D,IAAI,OAAO,QAAQ,UACjB,EAAE,QAAQ,KAAK,IAAI,GAAG,GAAG,IAAI,QAAQ;EAEzC,SAAS,GAAG;GACV,GAAG,OAAO,4DAA4D,CAAC;EACzE;CACF;AACF;AAEA,SAAS,8BACP,IACA,OACA,UACoB;CACpB,QAAO,MAAK;EACV,IAAI;GACF,MAAM,KAAK,MAAM;GACjB,IAAI,CAAC,IACH;GAEF,MAAM,CAAC,EAAC,KAAK,kBAAiB,GAAG,OAG9B,sBAAsB;GACzB,IAAI,OAAO,QAAQ,YAAY,OAAO,iBAAiB,UACrD,EAAE,QAAQ,KAAK,IAAI,GAAG,MAAM,YAAY,IAAI,QAAQ;EAExD,SAAS,GAAG;GACV,GAAG,OACD,oEACA,CACF;EACF;CACF;AACF;AAEA,SAAS,uBACP,IACA,MACoB;CACpB,OAAO,OAAM,MAAK,EAAE,QAAQ,MAAM,gBAAgB,IAAI,IAAI,CAAC;AAC7D;AAEA,IAAM,oBAAoB;AAO1B,IAAM,oCAAoC;;AAe1C,eAAsB,eACpB,IACA,QACA,gBAC8B;CAC9B,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;CACJ,MAAM,WAAW,IAAI,wBAAwB,IAAI,OAAO,QAAQ,IAAI;CACpE,IAAI,6BAA6B;CAMjC,MAAM,eAAe,iBAAiB,IAAI,OAAO,QAAQ,IAAI;CAC7D,MAAM,sBAAsB,cAAc;CAC1C,IAAI;EACF,SAAS;GACP,MAAM,EAAC,UAAU,aAAY,MAAM,4BACjC,IACA,OAAO,QACP,gBACA,YACF;GACA,IACE,wBAAwB,KAAA,KACxB,SAAS,iBAAiB,uBAC1B,+BAA+B,mCAC/B;IAGA,SAAS,OAAO;IAChB,GAAG,OACD,uDACM,SAAS,eAAe,4BACxB,oBAAoB,iBAAiB,oBAAoB,IAAK,SACtE;IACA,MAAM,MAAM,iBAAiB;IAC7B;GACF;GAEA,CAAC,CAAC,aAAa;GACf,MAAM,aAA+B;IAAC,GAAG,OAAO;IAAY;GAAS;GACrE,SAAS,MAAM,SAAS,WAAW;GACnC,IAAI;GACJ,IAAI;IACF,UAAU,MAAM,WACd,IACA,YACA,OAAO,QAAQ,MACf,UACA,eACA,SAAS,MACX;GACF,SAAS,GAAG;IACV,IAAI,SAAS,OAAO,SAAS;KAC3B,GAAG,OACD,qDACA,OAAO,CAAC,CACV;KACA;IACF;IACA,SAAS,OAAO;IAChB,MAAM;GACR,UAAU;IACR,SAAS,KAAK;GAChB;GACA,IAAI,QAAQ,UAAU;IACpB,SAAS,QAAQ;IACjB,SAAS,KAAK;IACd,OAAO;GACT;GAEA,SAAS,OAAO;GAChB,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,UAAU;EACR,SAAS,KAAK;EAMd,MAAM,SAAS;GAAC,GALF,6BACZ,OAAO,YACP,eACA,SAEiB;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,CAAC,CAAC,IAAI,GAAG,MAAM;EACrC,0BAA0B,CAAC,CAAC,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}

package/out/zero-cache/src/server/syncer.js

@@ -13,6 +13,7 @@ import { exitAfter, runUntilKilled } from "../services/life-cycle.js";
 import { createLogContext } from "./logging.js";
 import { installProfileHandler } from "../types/profiler.js";
 import { ProtocolErrorWithLevel } from "../types/error-with-level.js";
+import { clientGroupLogID } from "../types/client-group-log-id.js";
 import { warmupConnections } from "../db/warmup.js";
 import { initEventSink } from "../observability/events.js";
 import { startOtelAuto } from "./otel-start.js";
@@ -75,7 +76,7 @@ async function runWorker(parent, env, ...args) {
 		dbPath: replicaFile
 	}, config.sqliteCorruptionChecks);
 	lc.debug?.(`running view-syncer on ${replicaFile}`);
-	const cvrDB = await connectPgClient(lc, cvr.db, `sync-worker-${pid}-cvr`, { max: must(cvr.maxConnsPerWorker, "cvr.maxConnsPerWorker must be set") });
+	const cvrDB = await connectPgClient(lc, cvr.db, `sync-worker-${pid}-cvr`, { max: must(cvr.maxConnsPerWorker, "cvr.maxConnsPerWorker must be set") }, { plainJSON: true });
 	const upstreamDB = enableCrudMutations && upstream.type === "pg" ? await connectPgClient(lc, upstream.db, `sync-worker-${pid}-upstream`, { max: must(upstream.maxConnsPerWorker, "upstream.maxConnsPerWorker must be set") }) : void 0;
 	const dbWarmup = Promise.allSettled([warmupConnections(lc, cvrDB, "cvr"), upstreamDB ? warmupConnections(lc, upstreamDB, "upstream") : promiseVoid]);
 	const tmpDir = config.storageDBTmpDir ?? tmpdir();
@@ -138,16 +139,16 @@ async function runWorker(parent, env, ...args) {
 	const queryStats = config.log.queryStatsIntervalSeconds > 0 ? new QueryStats() : void 0;
 	const stopQueryStats = queryStats?.start(lc.withContext("taskID", config.taskID).withContext("component", "view-syncer"), config.log.queryStatsIntervalSeconds * 1e3);
 	const viewSyncerFactory = (id, sub, drainCoordinator) => {
-		const logger = lc.withContext("taskID", config.taskID).withContext("component", "view-syncer").withContext("appID", shard.appID).withContext("shardNum", shard.shardNum).withContext("clientGroupID", id).withContext("instance", randomID());
+		const logger = lc.withContext("taskID", config.taskID).withContext("component", "view-syncer").withContext("appID", shard.appID).withContext("shardNum", shard.shardNum).withContext("clientGroupID", clientGroupLogID(id)).withContext("instance", randomID());
 		const customQueryTransformer = customQueryConfig && new CustomQueryTransformer(logger, shard);
-		const connContextManager = new ConnectionContextManagerImpl(logger, config.auth.revalidateIntervalSeconds, config.auth.retransformIntervalSeconds, customQueryConfig, pushConfig, validateLegacyJWT);
+		const connContextManager = new ConnectionContextManagerImpl(logger, config.auth.revalidateIntervalSeconds, config.auth.retransformIntervalSeconds, customQueryConfig, pushConfig, validateLegacyJWT, void 0, config.auth.requireServerUserID);
 		lc.debug?.(`creating view syncer. Query Planner Enabled: ${config.enableQueryPlanner}`);
 		const inspectorDelegate = new InspectorDelegate(customQueryTransformer);
 		const priorityOpRunningYieldThresholdMs = Math.max(config.yieldThresholdMs / 4, 2);
 		const normalYieldThresholdMs = Math.max(config.yieldThresholdMs, 2);
 		return new ViewSyncerService(config, logger, shard, config.taskID, id, cvrDB, new PipelineDriver(logger, config.log, new Snapshotter(logger, replicaFile, shard, void 0, snapshotRowCache), shard, operatorStorage.createClientGroupStorage(id), id, inspectorDelegate, () => isPriorityOpRunning() ? priorityOpRunningYieldThresholdMs : normalYieldThresholdMs, config.enableQueryPlanner, config, deferredWritesBudget, queryStats), sub, drainCoordinator, config.log.slowHydrateThreshold, inspectorDelegate, connContextManager, customQueryTransformer, runPriorityOp, void 0, void 0, void 0, void 0, batchPacer);
 	};
-	const syncer = new Syncer(lc, config, viewSyncerFactory, upstreamDB ? (id) => new MutagenService(lc.withContext("component", "mutagen").withContext("clientGroupID", id), shard, id, upstreamDB, config, writeAuthzStorage) : void 0, pushConfig === void 0 ? void 0 : (id, connContextManager) => new PusherService(config, lc.withContext("clientGroupID", id), id, connContextManager), parent, validateLegacyJWT);
+	const syncer = new Syncer(lc, config, viewSyncerFactory, upstreamDB ? (id) => new MutagenService(lc.withContext("component", "mutagen").withContext("clientGroupID", clientGroupLogID(id)), shard, id, upstreamDB, config, writeAuthzStorage) : void 0, pushConfig === void 0 ? void 0 : (id, connContextManager) => new PusherService(config, lc.withContext("clientGroupID", clientGroupLogID(id)), id, connContextManager), parent, validateLegacyJWT);
 	startAnonymousTelemetry(lc, config);
 	dbWarmup.then(() => parent.send(["ready", { ready: true }]));
 	try {

package/out/zero-cache/src/server/syncer.js.map

@@ -1 +1 @@
-{"version":3,"file":"syncer.js","names":["v.parse"],"sources":["../../../../../zero-cache/src/server/syncer.ts"],"sourcesContent":["import {randomUUID} from 'node:crypto';\nimport {tmpdir} from 'node:os';\nimport path from 'node:path';\nimport {pid} from 'node:process';\nimport {getHeapStatistics} from 'node:v8';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {DatabaseStorage} from '../../../zqlite/src/database-storage.ts';\nimport type {ValidateLegacyJWT} from '../auth/auth.ts';\nimport {tokenConfigOptions, verifyToken} from '../auth/jwt.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {CustomQueryTransformer} from '../custom-queries/transform-query.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {MutagenService} from '../services/mutagen/mutagen.ts';\nimport {PusherService} from '../services/mutagen/pusher.ts';\nimport type {ReplicaState} from '../services/replicator/replicator.ts';\nimport {BatchPacer} from '../services/view-syncer/batch-pacer.ts';\nimport {\n  type ConnectionContextManager,\n  ConnectionContextManagerImpl,\n} from '../services/view-syncer/connection-context-manager.ts';\nimport {DeferredWritesBudget} from '../services/view-syncer/deferred-writes-budget.ts';\nimport type {DrainCoordinator} from '../services/view-syncer/drain-coordinator.ts';\nimport {PipelineDriver} from '../services/view-syncer/pipeline-driver.ts';\nimport {QueryStats} from '../services/view-syncer/query-stats.ts';\nimport {SnapshotRowCache} from '../services/view-syncer/snapshot-row-cache.ts';\nimport {Snapshotter} from '../services/view-syncer/snapshotter.ts';\nimport {ViewSyncerService} from '../services/view-syncer/view-syncer.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardID} from '../types/shards.ts';\nimport type {Subscription} from '../types/subscription.ts';\nimport {replicaFileModeSchema, replicaFileName} from '../workers/replicator.ts';\nimport {Syncer} from '../workers/syncer.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {InspectorDelegate} from './inspector-delegate.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {isPriorityOpRunning, runPriorityOp} from './priority-op.ts';\n\nfunction randomID() {\n  return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);\n}\n\nfunction getCustomQueryConfig(\n  config: Pick<NormalizedZeroConfig, 'query' | 'getQueries'>,\n) {\n  const queryConfig = config.query?.url ? config.query : config.getQueries;\n\n  if (!queryConfig?.url) {\n    return undefined;\n  }\n\n  return {\n    url: queryConfig.url,\n    apiKey: queryConfig.apiKey,\n    allowedClientHeaders: queryConfig.allowedClientHeaders,\n    allowedRequestHeaders: queryConfig.allowedRequestHeaders,\n    forwardCookies: queryConfig.forwardCookies ?? false,\n  };\n}\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length >= 2, `expected [fileMode, workerIndex, ...flags]`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n  const workerIndex = Number(args[1]);\n  installProfileHandler(parent, `syncer-${workerIndex}`, workerIndex);\n  const config = getNormalizedZeroConfig({env, argv: args.slice(2)});\n\n  startOtelAuto(\n    createLogContext(config, 'syncer', workerIndex, false),\n    'syncer',\n    workerIndex,\n  );\n  lc = createLogContext(config, 'syncer', workerIndex);\n  initEventSink(lc, config);\n\n  const {cvr, upstream, enableCrudMutations} = config;\n\n  const replicaFile = replicaFileName(config.replica.file, fileMode);\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'syncer replica',\n      dbPath: replicaFile,\n    },\n    config.sqliteCorruptionChecks,\n  );\n  lc.debug?.(`running view-syncer on ${replicaFile}`);\n\n  const cvrDB = await connectPgClient(lc, cvr.db, `sync-worker-${pid}-cvr`, {\n    max: must(cvr.maxConnsPerWorker, 'cvr.maxConnsPerWorker must be set'),\n  });\n\n  const upstreamDB =\n    enableCrudMutations && upstream.type === 'pg'\n      ? await connectPgClient(lc, upstream.db, `sync-worker-${pid}-upstream`, {\n          max: must(\n            upstream.maxConnsPerWorker,\n            'upstream.maxConnsPerWorker must be set',\n          ),\n        })\n      : undefined;\n\n  const dbWarmup = Promise.allSettled([\n    warmupConnections(lc, cvrDB, 'cvr'),\n    upstreamDB ? warmupConnections(lc, upstreamDB, 'upstream') : promiseVoid,\n  ]);\n\n  const tmpDir = config.storageDBTmpDir ?? tmpdir();\n  const operatorStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `sync-worker-${randomUUID()}`),\n    {\n      maxMemoryBytes: Math.floor(\n        getHeapStatistics().heap_size_limit *\n          config.operatorStorageHeapProportion,\n      ),\n    },\n  );\n  getOrCreateGauge('sync', 'ivm.operator-storage-memory-bytes', {\n    description:\n      'Estimated bytes of IVM operator state that the client groups of a ' +\n      'sync worker keep in memory rather than in SQLite',\n    unit: 'By',\n  }).addCallback(o => o.observe(operatorStorage.memoryBytes));\n  const writeAuthzStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `mutagen-${randomUUID()}`),\n  );\n\n  const shard = getShardID(config);\n  const customQueryConfig = getCustomQueryConfig(config);\n  const pushConfig =\n    config.push.url === undefined && config.mutate.url === undefined\n      ? undefined\n      : {\n          ...config.push,\n          ...config.mutate,\n          url: must(\n            config.push.url ?? config.mutate.url,\n            'No push or mutate URL configured',\n          ),\n        };\n\n  /** @deprecated used in JWT validation */\n  let validateLegacyJWT: ValidateLegacyJWT | undefined = undefined;\n\n  const tokenOptions = tokenConfigOptions(config.auth ?? {});\n  if (tokenOptions.length === 1) {\n    validateLegacyJWT = async (token, {userID}) => {\n      if (!userID) {\n        throw new ProtocolErrorWithLevel(\n          {\n            kind: 'Unauthorized',\n            message: 'UserID is required for JWT validation.',\n            origin: 'zeroCache',\n          },\n          'warn',\n        );\n      }\n\n      const decoded = await verifyToken(config.auth, token, {\n        subject: userID,\n        ...(config.auth?.issuer && {issuer: config.auth.issuer}),\n        ...(config.auth?.audience && {\n          audience: config.auth.audience,\n        }),\n      });\n      return {\n        type: 'jwt',\n        raw: token,\n        decoded,\n      };\n    };\n  }\n\n  // Shared by all of the view-syncers on this worker so that the row reads\n  // performed when advancing their pipelines are done once per worker rather\n  // than once per client group.\n  const snapshotRowCache =\n    config.snapshotRowCacheSize > 0\n      ? new SnapshotRowCache(config.snapshotRowCacheSize)\n      : undefined;\n\n  // Shared by all of the view-syncers on this worker, which each hold their\n  // own copy of the changes they are advancing through.\n  const deferredWritesBudget = config.deferIvmWrites\n    ? DeferredWritesBudget.forHeapProportion(\n        config.deferIvmWritesHeapProportion,\n      )\n    : undefined;\n  if (deferredWritesBudget) {\n    lc.info?.(\n      `Deferred IVM writes may hold up to ${deferredWritesBudget.maxRows} ` +\n        `rows (~${(deferredWritesBudget.maxBytes / 1024 ** 2).toFixed(2)} MB) ` +\n        `across client groups`,\n    );\n    getOrCreateGauge(\n      'sync',\n      'ivm.deferred-writes-reserved-rows',\n      'Rows reserved by the client groups of a sync worker to hold IVM ' +\n        'changes in memory (deferIvmWrites)',\n    ).addCallback(o => o.observe(deferredWritesBudget.reservedRows));\n    getOrCreateGauge('sync', 'ivm.deferred-writes-held-bytes', {\n      description:\n        'Estimated bytes of the IVM changes that the client groups of a sync ' +\n        'worker hold in memory (deferIvmWrites)',\n      unit: 'By',\n    }).addCallback(o => o.observe(deferredWritesBudget.heldBytes));\n  }\n\n  // Shared by all of the view-syncers on this worker, whose batches of\n  // changes together are held to a share of the worker's time.\n  const batchPacer = config.costBasedSyncEnabled\n    ? new BatchPacer({\n        shareTarget: config.syncUpdateShareTarget,\n        maxIntervalMs: config.syncMaxBatchIntervalMs,\n        lc,\n      })\n    : undefined;\n  if (batchPacer) {\n    getOrCreateGauge('sync', 'worker-busy-share', {\n      description:\n        'Share of its time that a sync worker was busy over the last few ' +\n        'seconds, which the pacing of its client groups holds to a target',\n      unit: '1',\n    }).addCallback(o => o.observe(batchPacer.share));\n    getOrCreateGauge('sync', 'batch-wait-multiplier', {\n      description:\n        'How many times the processing time of a batch of changes the ' +\n        'client groups of a sync worker wait before their next batch',\n      unit: '1',\n    }).addCallback(o => o.observe(batchPacer.multiplier));\n  }\n\n  // Shared by all of the view-syncers on this worker, so that the work done\n  // for each query shape is logged once per worker per interval.\n  const queryStats =\n    config.log.queryStatsIntervalSeconds > 0 ? new QueryStats() : undefined;\n  const stopQueryStats = queryStats?.start(\n    lc\n      .withContext('taskID', config.taskID)\n      .withContext('component', 'view-syncer'),\n    config.log.queryStatsIntervalSeconds * 1000,\n  );\n\n  const viewSyncerFactory = (\n    id: string,\n    sub: Subscription<ReplicaState>,\n    drainCoordinator: DrainCoordinator,\n  ) => {\n    const logger = lc\n      .withContext('taskID', config.taskID)\n      .withContext('component', 'view-syncer')\n      .withContext('appID', shard.appID)\n      .withContext('shardNum', shard.shardNum)\n      .withContext('clientGroupID', id)\n      .withContext('instance', randomID());\n\n    const customQueryTransformer =\n      customQueryConfig && new CustomQueryTransformer(logger, shard);\n    const connContextManager = new ConnectionContextManagerImpl(\n      logger,\n      config.auth.revalidateIntervalSeconds,\n      config.auth.retransformIntervalSeconds,\n      customQueryConfig,\n      pushConfig,\n      validateLegacyJWT,\n    );\n\n    lc.debug?.(\n      `creating view syncer. Query Planner Enabled: ${config.enableQueryPlanner}`,\n    );\n\n    const inspectorDelegate = new InspectorDelegate(customQueryTransformer);\n\n    const priorityOpRunningYieldThresholdMs = Math.max(\n      config.yieldThresholdMs / 4,\n      2,\n    );\n    const normalYieldThresholdMs = Math.max(config.yieldThresholdMs, 2);\n\n    return new ViewSyncerService(\n      config,\n      logger,\n      shard,\n      config.taskID,\n      id,\n      cvrDB,\n      new PipelineDriver(\n        logger,\n        config.log,\n        new Snapshotter(\n          logger,\n          replicaFile,\n          shard,\n          undefined,\n          snapshotRowCache,\n        ),\n        shard,\n        operatorStorage.createClientGroupStorage(id),\n        id,\n        inspectorDelegate,\n        () =>\n          isPriorityOpRunning()\n            ? priorityOpRunningYieldThresholdMs\n            : normalYieldThresholdMs,\n        config.enableQueryPlanner,\n        config,\n        deferredWritesBudget,\n        queryStats,\n      ),\n      sub,\n      drainCoordinator,\n      config.log.slowHydrateThreshold,\n      inspectorDelegate,\n      connContextManager,\n      customQueryTransformer,\n      runPriorityOp,\n      undefined,\n      undefined,\n      undefined,\n      undefined,\n      batchPacer,\n    );\n  };\n\n  const mutagenFactory = upstreamDB\n    ? (id: string) =>\n        new MutagenService(\n          lc\n            .withContext('component', 'mutagen')\n            .withContext('clientGroupID', id),\n          shard,\n          id,\n          upstreamDB,\n          config,\n          writeAuthzStorage,\n        )\n    : undefined;\n\n  const pusherFactory =\n    pushConfig === undefined\n      ? undefined\n      : (id: string, connContextManager: ConnectionContextManager) =>\n          new PusherService(\n            config,\n            lc.withContext('clientGroupID', id),\n            id,\n            connContextManager,\n          );\n\n  const syncer = new Syncer(\n    lc,\n    config,\n    viewSyncerFactory,\n    mutagenFactory,\n    pusherFactory,\n    parent,\n    validateLegacyJWT,\n  );\n\n  startAnonymousTelemetry(lc, config);\n\n  void dbWarmup.then(() => parent.send(['ready', {ready: true}]));\n\n  try {\n    return await runUntilKilled(lc, parent, syncer);\n  } finally {\n    // Logs the work done since the last interval.\n    stopQueryStats?.();\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuDA,SAAS,WAAW;CAClB,OAAO,QAAQ,GAAG,OAAO,gBAAgB,CAAC,CAAC,SAAS,EAAE;AACxD;AAEA,SAAS,qBACP,QACA;CACA,MAAM,cAAc,OAAO,OAAO,MAAM,OAAO,QAAQ,OAAO;CAE9D,IAAI,CAAC,aAAa,KAChB;CAGF,OAAO;EACL,KAAK,YAAY;EACjB,QAAQ,YAAY;EACpB,sBAAsB,YAAY;EAClC,uBAAuB,YAAY;EACnC,gBAAgB,YAAY,kBAAkB;CAChD;AACF;AAGA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,UAAU,GAAG,4CAA4C;CACrE,MAAM,WAAWA,MAAQ,KAAK,IAAI,qBAAqB;CACvD,MAAM,cAAc,OAAO,KAAK,EAAE;CAClC,sBAAsB,QAAQ,UAAU,eAAe,WAAW;CAClE,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CAEjE,cACE,iBAAiB,QAAQ,UAAU,aAAa,KAAK,GACrD,UACA,WACF;CACA,KAAK,iBAAiB,QAAQ,UAAU,WAAW;CACnD,cAAc,IAAI,MAAM;CAExB,MAAM,EAAC,KAAK,UAAU,wBAAuB;CAE7C,MAAM,cAAc,gBAAgB,OAAO,QAAQ,MAAM,QAAQ;CACjE,yCACE;EACE,WAAW;EACX,QAAQ;CACV,GACA,OAAO,sBACT;CACA,GAAG,QAAQ,0BAA0B,aAAa;CAElD,MAAM,QAAQ,MAAM,gBAAgB,IAAI,IAAI,IAAI,eAAe,IAAI,OAAO,EACxE,KAAK,KAAK,IAAI,mBAAmB,mCAAmC,EACtE,CAAC;CAED,MAAM,aACJ,uBAAuB,SAAS,SAAS,OACrC,MAAM,gBAAgB,IAAI,SAAS,IAAI,eAAe,IAAI,YAAY,EACpE,KAAK,KACH,SAAS,mBACT,wCACF,EACF,CAAC,IACD,KAAA;CAEN,MAAM,WAAW,QAAQ,WAAW,CAClC,kBAAkB,IAAI,OAAO,KAAK,GAClC,aAAa,kBAAkB,IAAI,YAAY,UAAU,IAAI,WAC/D,CAAC;CAED,MAAM,SAAS,OAAO,mBAAmB,OAAO;CAChD,MAAM,kBAAkB,gBAAgB,OACtC,IACA,KAAK,KAAK,QAAQ,eAAe,WAAW,GAAG,GAC/C,EACE,gBAAgB,KAAK,MACnB,kBAAkB,CAAC,CAAC,kBAClB,OAAO,6BACX,EACF,CACF;CACA,iBAAiB,QAAQ,qCAAqC;EAC5D,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,aAAY,MAAK,EAAE,QAAQ,gBAAgB,WAAW,CAAC;CAC1D,MAAM,oBAAoB,gBAAgB,OACxC,IACA,KAAK,KAAK,QAAQ,WAAW,WAAW,GAAG,CAC7C;CAEA,MAAM,QAAQ,WAAW,MAAM;CAC/B,MAAM,oBAAoB,qBAAqB,MAAM;CACrD,MAAM,aACJ,OAAO,KAAK,QAAQ,KAAA,KAAa,OAAO,OAAO,QAAQ,KAAA,IACnD,KAAA,IACA;EACE,GAAG,OAAO;EACV,GAAG,OAAO;EACV,KAAK,KACH,OAAO,KAAK,OAAO,OAAO,OAAO,KACjC,kCACF;CACF;;CAGN,IAAI,oBAAmD,KAAA;CAGvD,IADqB,mBAAmB,OAAO,QAAQ,CAAC,CACpD,CAAA,CAAa,WAAW,GAC1B,oBAAoB,OAAO,OAAO,EAAC,aAAY;EAC7C,IAAI,CAAC,QACH,MAAM,IAAI,uBACR;GACE,MAAM;GACN,SAAS;GACT,QAAQ;EACV,GACA,MACF;EAUF,OAAO;GACL,MAAM;GACN,KAAK;GACL,SAAA,MAVoB,YAAY,OAAO,MAAM,OAAO;IACpD,SAAS;IACT,GAAI,OAAO,MAAM,UAAU,EAAC,QAAQ,OAAO,KAAK,OAAM;IACtD,GAAI,OAAO,MAAM,YAAY,EAC3B,UAAU,OAAO,KAAK,SACxB;GACF,CAAC;EAKD;CACF;CAMF,MAAM,mBACJ,OAAO,uBAAuB,IAC1B,IAAI,iBAAiB,OAAO,oBAAoB,IAChD,KAAA;CAIN,MAAM,uBAAuB,OAAO,iBAChC,qBAAqB,kBACnB,OAAO,4BACT,IACA,KAAA;CACJ,IAAI,sBAAsB;EACxB,GAAG,OACD,sCAAsC,qBAAqB,QAAQ,WACtD,qBAAqB,WAAW,QAAQ,EAAA,CAAG,QAAQ,CAAC,EAAE,0BAErE;EACA,iBACE,QACA,qCACA,oGAEF,CAAC,CAAC,aAAY,MAAK,EAAE,QAAQ,qBAAqB,YAAY,CAAC;EAC/D,iBAAiB,QAAQ,kCAAkC;GACzD,aACE;GAEF,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,MAAK,EAAE,QAAQ,qBAAqB,SAAS,CAAC;CAC/D;CAIA,MAAM,aAAa,OAAO,uBACtB,IAAI,WAAW;EACb,aAAa,OAAO;EACpB,eAAe,OAAO;EACtB;CACF,CAAC,IACD,KAAA;CACJ,IAAI,YAAY;EACd,iBAAiB,QAAQ,qBAAqB;GAC5C,aACE;GAEF,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,MAAK,EAAE,QAAQ,WAAW,KAAK,CAAC;EAC/C,iBAAiB,QAAQ,yBAAyB;GAChD,aACE;GAEF,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,MAAK,EAAE,QAAQ,WAAW,UAAU,CAAC;CACtD;CAIA,MAAM,aACJ,OAAO,IAAI,4BAA4B,IAAI,IAAI,WAAW,IAAI,KAAA;CAChE,MAAM,iBAAiB,YAAY,MACjC,GACG,YAAY,UAAU,OAAO,MAAM,CAAC,CACpC,YAAY,aAAa,aAAa,GACzC,OAAO,IAAI,4BAA4B,GACzC;CAEA,MAAM,qBACJ,IACA,KACA,qBACG;EACH,MAAM,SAAS,GACZ,YAAY,UAAU,OAAO,MAAM,CAAC,CACpC,YAAY,aAAa,aAAa,CAAC,CACvC,YAAY,SAAS,MAAM,KAAK,CAAC,CACjC,YAAY,YAAY,MAAM,QAAQ,CAAC,CACvC,YAAY,iBAAiB,EAAE,CAAC,CAChC,YAAY,YAAY,SAAS,CAAC;EAErC,MAAM,yBACJ,qBAAqB,IAAI,uBAAuB,QAAQ,KAAK;EAC/D,MAAM,qBAAqB,IAAI,6BAC7B,QACA,OAAO,KAAK,2BACZ,OAAO,KAAK,4BACZ,mBACA,YACA,iBACF;EAEA,GAAG,QACD,gDAAgD,OAAO,oBACzD;EAEA,MAAM,oBAAoB,IAAI,kBAAkB,sBAAsB;EAEtE,MAAM,oCAAoC,KAAK,IAC7C,OAAO,mBAAmB,GAC1B,CACF;EACA,MAAM,yBAAyB,KAAK,IAAI,OAAO,kBAAkB,CAAC;EAElE,OAAO,IAAI,kBACT,QACA,QACA,OACA,OAAO,QACP,IACA,OACA,IAAI,eACF,QACA,OAAO,KACP,IAAI,YACF,QACA,aACA,OACA,KAAA,GACA,gBACF,GACA,OACA,gBAAgB,yBAAyB,EAAE,GAC3C,IACA,yBAEE,oBAAoB,IAChB,oCACA,wBACN,OAAO,oBACP,QACA,sBACA,UACF,GACA,KACA,kBACA,OAAO,IAAI,sBACX,mBACA,oBACA,wBACA,eACA,KAAA,GACA,KAAA,GACA,KAAA,GACA,KAAA,GACA,UACF;CACF;CA2BA,MAAM,SAAS,IAAI,OACjB,IACA,QACA,mBA5BqB,cAClB,OACC,IAAI,eACF,GACG,YAAY,aAAa,SAAS,CAAC,CACnC,YAAY,iBAAiB,EAAE,GAClC,OACA,IACA,YACA,QACA,iBACF,IACF,KAAA,GAGF,eAAe,KAAA,IACX,KAAA,KACC,IAAY,uBACX,IAAI,cACF,QACA,GAAG,YAAY,iBAAiB,EAAE,GAClC,IACA,kBACF,GAQN,QACA,iBACF;CAEA,wBAAwB,IAAI,MAAM;CAElC,SAAc,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CAE9D,IAAI;EACF,OAAO,MAAM,eAAe,IAAI,QAAQ,MAAM;CAChD,UAAU;EAER,iBAAiB;CACnB;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}
+{"version":3,"file":"syncer.js","names":["v.parse"],"sources":["../../../../../zero-cache/src/server/syncer.ts"],"sourcesContent":["import {randomUUID} from 'node:crypto';\nimport {tmpdir} from 'node:os';\nimport path from 'node:path';\nimport {pid} from 'node:process';\nimport {getHeapStatistics} from 'node:v8';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {DatabaseStorage} from '../../../zqlite/src/database-storage.ts';\nimport type {ValidateLegacyJWT} from '../auth/auth.ts';\nimport {tokenConfigOptions, verifyToken} from '../auth/jwt.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {CustomQueryTransformer} from '../custom-queries/transform-query.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {MutagenService} from '../services/mutagen/mutagen.ts';\nimport {PusherService} from '../services/mutagen/pusher.ts';\nimport type {ReplicaState} from '../services/replicator/replicator.ts';\nimport {BatchPacer} from '../services/view-syncer/batch-pacer.ts';\nimport {\n  type ConnectionContextManager,\n  ConnectionContextManagerImpl,\n} from '../services/view-syncer/connection-context-manager.ts';\nimport {DeferredWritesBudget} from '../services/view-syncer/deferred-writes-budget.ts';\nimport type {DrainCoordinator} from '../services/view-syncer/drain-coordinator.ts';\nimport {PipelineDriver} from '../services/view-syncer/pipeline-driver.ts';\nimport {QueryStats} from '../services/view-syncer/query-stats.ts';\nimport {SnapshotRowCache} from '../services/view-syncer/snapshot-row-cache.ts';\nimport {Snapshotter} from '../services/view-syncer/snapshotter.ts';\nimport {ViewSyncerService} from '../services/view-syncer/view-syncer.ts';\nimport {clientGroupLogID} from '../types/client-group-log-id.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardID} from '../types/shards.ts';\nimport type {Subscription} from '../types/subscription.ts';\nimport {replicaFileModeSchema, replicaFileName} from '../workers/replicator.ts';\nimport {Syncer} from '../workers/syncer.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {InspectorDelegate} from './inspector-delegate.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {isPriorityOpRunning, runPriorityOp} from './priority-op.ts';\n\nfunction randomID() {\n  return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);\n}\n\nfunction getCustomQueryConfig(\n  config: Pick<NormalizedZeroConfig, 'query' | 'getQueries'>,\n) {\n  const queryConfig = config.query?.url ? config.query : config.getQueries;\n\n  if (!queryConfig?.url) {\n    return undefined;\n  }\n\n  return {\n    url: queryConfig.url,\n    apiKey: queryConfig.apiKey,\n    allowedClientHeaders: queryConfig.allowedClientHeaders,\n    allowedRequestHeaders: queryConfig.allowedRequestHeaders,\n    forwardCookies: queryConfig.forwardCookies ?? false,\n  };\n}\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length >= 2, `expected [fileMode, workerIndex, ...flags]`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n  const workerIndex = Number(args[1]);\n  installProfileHandler(parent, `syncer-${workerIndex}`, workerIndex);\n  const config = getNormalizedZeroConfig({env, argv: args.slice(2)});\n\n  startOtelAuto(\n    createLogContext(config, 'syncer', workerIndex, false),\n    'syncer',\n    workerIndex,\n  );\n  lc = createLogContext(config, 'syncer', workerIndex);\n  initEventSink(lc, config);\n\n  const {cvr, upstream, enableCrudMutations} = config;\n\n  const replicaFile = replicaFileName(config.replica.file, fileMode);\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'syncer replica',\n      dbPath: replicaFile,\n    },\n    config.sqliteCorruptionChecks,\n  );\n  lc.debug?.(`running view-syncer on ${replicaFile}`);\n\n  const cvrDB = await connectPgClient(\n    lc,\n    cvr.db,\n    `sync-worker-${pid}-cvr`,\n    {\n      max: must(cvr.maxConnsPerWorker, 'cvr.maxConnsPerWorker must be set'),\n    },\n    // Every number in the CVR is a JavaScript number, but JSONB returns a large\n    // one as digits, which would otherwise be parsed as a bigint.\n    {plainJSON: true},\n  );\n\n  const upstreamDB =\n    enableCrudMutations && upstream.type === 'pg'\n      ? await connectPgClient(lc, upstream.db, `sync-worker-${pid}-upstream`, {\n          max: must(\n            upstream.maxConnsPerWorker,\n            'upstream.maxConnsPerWorker must be set',\n          ),\n        })\n      : undefined;\n\n  const dbWarmup = Promise.allSettled([\n    warmupConnections(lc, cvrDB, 'cvr'),\n    upstreamDB ? warmupConnections(lc, upstreamDB, 'upstream') : promiseVoid,\n  ]);\n\n  const tmpDir = config.storageDBTmpDir ?? tmpdir();\n  const operatorStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `sync-worker-${randomUUID()}`),\n    {\n      maxMemoryBytes: Math.floor(\n        getHeapStatistics().heap_size_limit *\n          config.operatorStorageHeapProportion,\n      ),\n    },\n  );\n  getOrCreateGauge('sync', 'ivm.operator-storage-memory-bytes', {\n    description:\n      'Estimated bytes of IVM operator state that the client groups of a ' +\n      'sync worker keep in memory rather than in SQLite',\n    unit: 'By',\n  }).addCallback(o => o.observe(operatorStorage.memoryBytes));\n  const writeAuthzStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `mutagen-${randomUUID()}`),\n  );\n\n  const shard = getShardID(config);\n  const customQueryConfig = getCustomQueryConfig(config);\n  const pushConfig =\n    config.push.url === undefined && config.mutate.url === undefined\n      ? undefined\n      : {\n          ...config.push,\n          ...config.mutate,\n          url: must(\n            config.push.url ?? config.mutate.url,\n            'No push or mutate URL configured',\n          ),\n        };\n\n  /** @deprecated used in JWT validation */\n  let validateLegacyJWT: ValidateLegacyJWT | undefined = undefined;\n\n  const tokenOptions = tokenConfigOptions(config.auth ?? {});\n  if (tokenOptions.length === 1) {\n    validateLegacyJWT = async (token, {userID}) => {\n      if (!userID) {\n        throw new ProtocolErrorWithLevel(\n          {\n            kind: 'Unauthorized',\n            message: 'UserID is required for JWT validation.',\n            origin: 'zeroCache',\n          },\n          'warn',\n        );\n      }\n\n      const decoded = await verifyToken(config.auth, token, {\n        subject: userID,\n        ...(config.auth?.issuer && {issuer: config.auth.issuer}),\n        ...(config.auth?.audience && {\n          audience: config.auth.audience,\n        }),\n      });\n      return {\n        type: 'jwt',\n        raw: token,\n        decoded,\n      };\n    };\n  }\n\n  // Shared by all of the view-syncers on this worker so that the row reads\n  // performed when advancing their pipelines are done once per worker rather\n  // than once per client group.\n  const snapshotRowCache =\n    config.snapshotRowCacheSize > 0\n      ? new SnapshotRowCache(config.snapshotRowCacheSize)\n      : undefined;\n\n  // Shared by all of the view-syncers on this worker, which each hold their\n  // own copy of the changes they are advancing through.\n  const deferredWritesBudget = config.deferIvmWrites\n    ? DeferredWritesBudget.forHeapProportion(\n        config.deferIvmWritesHeapProportion,\n      )\n    : undefined;\n  if (deferredWritesBudget) {\n    lc.info?.(\n      `Deferred IVM writes may hold up to ${deferredWritesBudget.maxRows} ` +\n        `rows (~${(deferredWritesBudget.maxBytes / 1024 ** 2).toFixed(2)} MB) ` +\n        `across client groups`,\n    );\n    getOrCreateGauge(\n      'sync',\n      'ivm.deferred-writes-reserved-rows',\n      'Rows reserved by the client groups of a sync worker to hold IVM ' +\n        'changes in memory (deferIvmWrites)',\n    ).addCallback(o => o.observe(deferredWritesBudget.reservedRows));\n    getOrCreateGauge('sync', 'ivm.deferred-writes-held-bytes', {\n      description:\n        'Estimated bytes of the IVM changes that the client groups of a sync ' +\n        'worker hold in memory (deferIvmWrites)',\n      unit: 'By',\n    }).addCallback(o => o.observe(deferredWritesBudget.heldBytes));\n  }\n\n  // Shared by all of the view-syncers on this worker, whose batches of\n  // changes together are held to a share of the worker's time.\n  const batchPacer = config.costBasedSyncEnabled\n    ? new BatchPacer({\n        shareTarget: config.syncUpdateShareTarget,\n        maxIntervalMs: config.syncMaxBatchIntervalMs,\n        lc,\n      })\n    : undefined;\n  if (batchPacer) {\n    getOrCreateGauge('sync', 'worker-busy-share', {\n      description:\n        'Share of its time that a sync worker was busy over the last few ' +\n        'seconds, which the pacing of its client groups holds to a target',\n      unit: '1',\n    }).addCallback(o => o.observe(batchPacer.share));\n    getOrCreateGauge('sync', 'batch-wait-multiplier', {\n      description:\n        'How many times the processing time of a batch of changes the ' +\n        'client groups of a sync worker wait before their next batch',\n      unit: '1',\n    }).addCallback(o => o.observe(batchPacer.multiplier));\n  }\n\n  // Shared by all of the view-syncers on this worker, so that the work done\n  // for each query shape is logged once per worker per interval.\n  const queryStats =\n    config.log.queryStatsIntervalSeconds > 0 ? new QueryStats() : undefined;\n  const stopQueryStats = queryStats?.start(\n    lc\n      .withContext('taskID', config.taskID)\n      .withContext('component', 'view-syncer'),\n    config.log.queryStatsIntervalSeconds * 1000,\n  );\n\n  const viewSyncerFactory = (\n    id: string,\n    sub: Subscription<ReplicaState>,\n    drainCoordinator: DrainCoordinator,\n  ) => {\n    const logger = lc\n      .withContext('taskID', config.taskID)\n      .withContext('component', 'view-syncer')\n      .withContext('appID', shard.appID)\n      .withContext('shardNum', shard.shardNum)\n      .withContext('clientGroupID', clientGroupLogID(id))\n      .withContext('instance', randomID());\n\n    const customQueryTransformer =\n      customQueryConfig && new CustomQueryTransformer(logger, shard);\n    const connContextManager = new ConnectionContextManagerImpl(\n      logger,\n      config.auth.revalidateIntervalSeconds,\n      config.auth.retransformIntervalSeconds,\n      customQueryConfig,\n      pushConfig,\n      validateLegacyJWT,\n      undefined,\n      config.auth.requireServerUserID,\n    );\n\n    lc.debug?.(\n      `creating view syncer. Query Planner Enabled: ${config.enableQueryPlanner}`,\n    );\n\n    const inspectorDelegate = new InspectorDelegate(customQueryTransformer);\n\n    const priorityOpRunningYieldThresholdMs = Math.max(\n      config.yieldThresholdMs / 4,\n      2,\n    );\n    const normalYieldThresholdMs = Math.max(config.yieldThresholdMs, 2);\n\n    return new ViewSyncerService(\n      config,\n      logger,\n      shard,\n      config.taskID,\n      id,\n      cvrDB,\n      new PipelineDriver(\n        logger,\n        config.log,\n        new Snapshotter(\n          logger,\n          replicaFile,\n          shard,\n          undefined,\n          snapshotRowCache,\n        ),\n        shard,\n        operatorStorage.createClientGroupStorage(id),\n        id,\n        inspectorDelegate,\n        () =>\n          isPriorityOpRunning()\n            ? priorityOpRunningYieldThresholdMs\n            : normalYieldThresholdMs,\n        config.enableQueryPlanner,\n        config,\n        deferredWritesBudget,\n        queryStats,\n      ),\n      sub,\n      drainCoordinator,\n      config.log.slowHydrateThreshold,\n      inspectorDelegate,\n      connContextManager,\n      customQueryTransformer,\n      runPriorityOp,\n      undefined,\n      undefined,\n      undefined,\n      undefined,\n      batchPacer,\n    );\n  };\n\n  const mutagenFactory = upstreamDB\n    ? (id: string) =>\n        new MutagenService(\n          lc\n            .withContext('component', 'mutagen')\n            .withContext('clientGroupID', clientGroupLogID(id)),\n          shard,\n          id,\n          upstreamDB,\n          config,\n          writeAuthzStorage,\n        )\n    : undefined;\n\n  const pusherFactory =\n    pushConfig === undefined\n      ? undefined\n      : (id: string, connContextManager: ConnectionContextManager) =>\n          new PusherService(\n            config,\n            lc.withContext('clientGroupID', clientGroupLogID(id)),\n            id,\n            connContextManager,\n          );\n\n  const syncer = new Syncer(\n    lc,\n    config,\n    viewSyncerFactory,\n    mutagenFactory,\n    pusherFactory,\n    parent,\n    validateLegacyJWT,\n  );\n\n  startAnonymousTelemetry(lc, config);\n\n  void dbWarmup.then(() => parent.send(['ready', {ready: true}]));\n\n  try {\n    return await runUntilKilled(lc, parent, syncer);\n  } finally {\n    // Logs the work done since the last interval.\n    stopQueryStats?.();\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwDA,SAAS,WAAW;CAClB,OAAO,QAAQ,GAAG,OAAO,gBAAgB,CAAC,CAAC,SAAS,EAAE;AACxD;AAEA,SAAS,qBACP,QACA;CACA,MAAM,cAAc,OAAO,OAAO,MAAM,OAAO,QAAQ,OAAO;CAE9D,IAAI,CAAC,aAAa,KAChB;CAGF,OAAO;EACL,KAAK,YAAY;EACjB,QAAQ,YAAY;EACpB,sBAAsB,YAAY;EAClC,uBAAuB,YAAY;EACnC,gBAAgB,YAAY,kBAAkB;CAChD;AACF;AAGA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,UAAU,GAAG,4CAA4C;CACrE,MAAM,WAAWA,MAAQ,KAAK,IAAI,qBAAqB;CACvD,MAAM,cAAc,OAAO,KAAK,EAAE;CAClC,sBAAsB,QAAQ,UAAU,eAAe,WAAW;CAClE,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CAEjE,cACE,iBAAiB,QAAQ,UAAU,aAAa,KAAK,GACrD,UACA,WACF;CACA,KAAK,iBAAiB,QAAQ,UAAU,WAAW;CACnD,cAAc,IAAI,MAAM;CAExB,MAAM,EAAC,KAAK,UAAU,wBAAuB;CAE7C,MAAM,cAAc,gBAAgB,OAAO,QAAQ,MAAM,QAAQ;CACjE,yCACE;EACE,WAAW;EACX,QAAQ;CACV,GACA,OAAO,sBACT;CACA,GAAG,QAAQ,0BAA0B,aAAa;CAElD,MAAM,QAAQ,MAAM,gBAClB,IACA,IAAI,IACJ,eAAe,IAAI,OACnB,EACE,KAAK,KAAK,IAAI,mBAAmB,mCAAmC,EACtE,GAGA,EAAC,WAAW,KAAI,CAClB;CAEA,MAAM,aACJ,uBAAuB,SAAS,SAAS,OACrC,MAAM,gBAAgB,IAAI,SAAS,IAAI,eAAe,IAAI,YAAY,EACpE,KAAK,KACH,SAAS,mBACT,wCACF,EACF,CAAC,IACD,KAAA;CAEN,MAAM,WAAW,QAAQ,WAAW,CAClC,kBAAkB,IAAI,OAAO,KAAK,GAClC,aAAa,kBAAkB,IAAI,YAAY,UAAU,IAAI,WAC/D,CAAC;CAED,MAAM,SAAS,OAAO,mBAAmB,OAAO;CAChD,MAAM,kBAAkB,gBAAgB,OACtC,IACA,KAAK,KAAK,QAAQ,eAAe,WAAW,GAAG,GAC/C,EACE,gBAAgB,KAAK,MACnB,kBAAkB,CAAC,CAAC,kBAClB,OAAO,6BACX,EACF,CACF;CACA,iBAAiB,QAAQ,qCAAqC;EAC5D,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,aAAY,MAAK,EAAE,QAAQ,gBAAgB,WAAW,CAAC;CAC1D,MAAM,oBAAoB,gBAAgB,OACxC,IACA,KAAK,KAAK,QAAQ,WAAW,WAAW,GAAG,CAC7C;CAEA,MAAM,QAAQ,WAAW,MAAM;CAC/B,MAAM,oBAAoB,qBAAqB,MAAM;CACrD,MAAM,aACJ,OAAO,KAAK,QAAQ,KAAA,KAAa,OAAO,OAAO,QAAQ,KAAA,IACnD,KAAA,IACA;EACE,GAAG,OAAO;EACV,GAAG,OAAO;EACV,KAAK,KACH,OAAO,KAAK,OAAO,OAAO,OAAO,KACjC,kCACF;CACF;;CAGN,IAAI,oBAAmD,KAAA;CAGvD,IADqB,mBAAmB,OAAO,QAAQ,CAAC,CACpD,CAAA,CAAa,WAAW,GAC1B,oBAAoB,OAAO,OAAO,EAAC,aAAY;EAC7C,IAAI,CAAC,QACH,MAAM,IAAI,uBACR;GACE,MAAM;GACN,SAAS;GACT,QAAQ;EACV,GACA,MACF;EAUF,OAAO;GACL,MAAM;GACN,KAAK;GACL,SAAA,MAVoB,YAAY,OAAO,MAAM,OAAO;IACpD,SAAS;IACT,GAAI,OAAO,MAAM,UAAU,EAAC,QAAQ,OAAO,KAAK,OAAM;IACtD,GAAI,OAAO,MAAM,YAAY,EAC3B,UAAU,OAAO,KAAK,SACxB;GACF,CAAC;EAKD;CACF;CAMF,MAAM,mBACJ,OAAO,uBAAuB,IAC1B,IAAI,iBAAiB,OAAO,oBAAoB,IAChD,KAAA;CAIN,MAAM,uBAAuB,OAAO,iBAChC,qBAAqB,kBACnB,OAAO,4BACT,IACA,KAAA;CACJ,IAAI,sBAAsB;EACxB,GAAG,OACD,sCAAsC,qBAAqB,QAAQ,WACtD,qBAAqB,WAAW,QAAQ,EAAA,CAAG,QAAQ,CAAC,EAAE,0BAErE;EACA,iBACE,QACA,qCACA,oGAEF,CAAC,CAAC,aAAY,MAAK,EAAE,QAAQ,qBAAqB,YAAY,CAAC;EAC/D,iBAAiB,QAAQ,kCAAkC;GACzD,aACE;GAEF,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,MAAK,EAAE,QAAQ,qBAAqB,SAAS,CAAC;CAC/D;CAIA,MAAM,aAAa,OAAO,uBACtB,IAAI,WAAW;EACb,aAAa,OAAO;EACpB,eAAe,OAAO;EACtB;CACF,CAAC,IACD,KAAA;CACJ,IAAI,YAAY;EACd,iBAAiB,QAAQ,qBAAqB;GAC5C,aACE;GAEF,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,MAAK,EAAE,QAAQ,WAAW,KAAK,CAAC;EAC/C,iBAAiB,QAAQ,yBAAyB;GAChD,aACE;GAEF,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,MAAK,EAAE,QAAQ,WAAW,UAAU,CAAC;CACtD;CAIA,MAAM,aACJ,OAAO,IAAI,4BAA4B,IAAI,IAAI,WAAW,IAAI,KAAA;CAChE,MAAM,iBAAiB,YAAY,MACjC,GACG,YAAY,UAAU,OAAO,MAAM,CAAC,CACpC,YAAY,aAAa,aAAa,GACzC,OAAO,IAAI,4BAA4B,GACzC;CAEA,MAAM,qBACJ,IACA,KACA,qBACG;EACH,MAAM,SAAS,GACZ,YAAY,UAAU,OAAO,MAAM,CAAC,CACpC,YAAY,aAAa,aAAa,CAAC,CACvC,YAAY,SAAS,MAAM,KAAK,CAAC,CACjC,YAAY,YAAY,MAAM,QAAQ,CAAC,CACvC,YAAY,iBAAiB,iBAAiB,EAAE,CAAC,CAAC,CAClD,YAAY,YAAY,SAAS,CAAC;EAErC,MAAM,yBACJ,qBAAqB,IAAI,uBAAuB,QAAQ,KAAK;EAC/D,MAAM,qBAAqB,IAAI,6BAC7B,QACA,OAAO,KAAK,2BACZ,OAAO,KAAK,4BACZ,mBACA,YACA,mBACA,KAAA,GACA,OAAO,KAAK,mBACd;EAEA,GAAG,QACD,gDAAgD,OAAO,oBACzD;EAEA,MAAM,oBAAoB,IAAI,kBAAkB,sBAAsB;EAEtE,MAAM,oCAAoC,KAAK,IAC7C,OAAO,mBAAmB,GAC1B,CACF;EACA,MAAM,yBAAyB,KAAK,IAAI,OAAO,kBAAkB,CAAC;EAElE,OAAO,IAAI,kBACT,QACA,QACA,OACA,OAAO,QACP,IACA,OACA,IAAI,eACF,QACA,OAAO,KACP,IAAI,YACF,QACA,aACA,OACA,KAAA,GACA,gBACF,GACA,OACA,gBAAgB,yBAAyB,EAAE,GAC3C,IACA,yBAEE,oBAAoB,IAChB,oCACA,wBACN,OAAO,oBACP,QACA,sBACA,UACF,GACA,KACA,kBACA,OAAO,IAAI,sBACX,mBACA,oBACA,wBACA,eACA,KAAA,GACA,KAAA,GACA,KAAA,GACA,KAAA,GACA,UACF;CACF;CA2BA,MAAM,SAAS,IAAI,OACjB,IACA,QACA,mBA5BqB,cAClB,OACC,IAAI,eACF,GACG,YAAY,aAAa,SAAS,CAAC,CACnC,YAAY,iBAAiB,iBAAiB,EAAE,CAAC,GACpD,OACA,IACA,YACA,QACA,iBACF,IACF,KAAA,GAGF,eAAe,KAAA,IACX,KAAA,KACC,IAAY,uBACX,IAAI,cACF,QACA,GAAG,YAAY,iBAAiB,iBAAiB,EAAE,CAAC,GACpD,IACA,kBACF,GAQN,QACA,iBACF;CAEA,wBAAwB,IAAI,MAAM;CAElC,SAAc,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CAE9D,IAAI;EACF,OAAO,MAAM,eAAe,IAAI,QAAQ,MAAM;CAChD,UAAU;EAER,iBAAiB;CACnB;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}

package/out/zero-cache/src/server/worker-dispatcher.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"worker-dispatcher.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/worker-dispatcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,wBAAwB,CAAC;AAEpD,OAAO,KAAK,EAA2B,MAAM,EAAC,MAAM,uBAAuB,CAAC;AAK5E,qBAAa,gBAAiB,YAAW,OAAO;;IAC9C,QAAQ,CAAC,EAAE,uBAAuB;IAKlC,YACE,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,EAAE,MAAM,GAAG,SAAS,EAC3B,cAAc,EAAE,MAAM,GAAG,SAAS,EAsInC;IAED,GAAG,kBAQF;IAED,IAAI,kBAIH;CACF;AAID,wBAAgB,SAAS,CAAC,GAAG,EAAE,GAAG,GAC9B;IACE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,aAAa,CAAC;IAC1C,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB,GACD,SAAS,CAGZ"}
+{"version":3,"file":"worker-dispatcher.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/worker-dispatcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,wBAAwB,CAAC;AAGpD,OAAO,KAAK,EAA2B,MAAM,EAAC,MAAM,uBAAuB,CAAC;AAK5E,qBAAa,gBAAiB,YAAW,OAAO;;IAC9C,QAAQ,CAAC,EAAE,uBAAuB;IAKlC,YACE,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,EAAE,MAAM,GAAG,SAAS,EAC3B,cAAc,EAAE,MAAM,GAAG,SAAS,EAwInC;IAED,GAAG,kBAQF;IAED,IAAI,kBAIH;CACF;AAID,wBAAgB,SAAS,CAAC,GAAG,EAAE,GAAG,GAC9B;IACE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,aAAa,CAAC;IAC1C,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB,GACD,SAAS,CAGZ"}

package/out/zero-cache/src/server/worker-dispatcher.js

@@ -2,6 +2,7 @@ import { assert } from "../../../shared/src/asserts.js";
 import { getOrCreateCounter, getOrCreateGauge } from "../observability/metrics.js";
 import { RunningState } from "../services/running-state.js";
 import { installWebSocketHandoff } from "../types/websocket-handoff.js";
+import { clientGroupLogID } from "../types/client-group-log-id.js";
 import { getConnectParams } from "../workers/connect-params.js";
 import { SyncerAssigner } from "./syncer-assigner.js";
 import UrlPattern from "url-pattern";
@@ -53,7 +54,7 @@ var WorkerDispatcher = class {
 			maxProtocolVersion = Math.max(maxProtocolVersion, protocolVersion);
 			const { worker: syncer, generation } = assigner.assign(clientGroupID);
 			workerDispatchesCounter.add(1, { worker: String(syncer) });
-			lc.debug?.(`connecting ${clientGroupID} to syncer ${syncer}`);
+			lc.debug?.(`connecting ${clientGroupLogID(clientGroupID)} to syncer ${syncer}`);
 			return {
 				payload: {
 					...params,

package/out/zero-cache/src/server/worker-dispatcher.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker-dispatcher.js","names":[],"sources":["../../../../../zero-cache/src/server/worker-dispatcher.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport UrlPattern from 'url-pattern';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n} from '../observability/metrics.ts';\nimport {RunningState} from '../services/running-state.ts';\nimport type {Service} from '../services/service.ts';\nimport type {IncomingMessageSubset} from '../types/http.ts';\nimport type {ClientGroupStatusMessage, Worker} from '../types/processes.ts';\nimport {installWebSocketHandoff} from '../types/websocket-handoff.ts';\nimport {getConnectParams} from '../workers/connect-params.ts';\nimport {SyncerAssigner} from './syncer-assigner.ts';\n\nexport class WorkerDispatcher implements Service {\n  readonly id = 'worker-dispatcher';\n  readonly #lc: LogContext;\n  readonly #state = new RunningState(this.id);\n  readonly #assigner: SyncerAssigner;\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    parent: Worker,\n    syncers: Worker[],\n    mutator: Worker | undefined,\n    changeStreamer: Worker | undefined,\n  ) {\n    this.#lc = lc;\n\n    const assigner = new SyncerAssigner(taskID, syncers.length);\n    this.#assigner = assigner;\n\n    const workerDispatchesCounter = getOrCreateCounter(\n      'sync',\n      'worker-dispatches',\n      'Dispatched client group connections to worker',\n    );\n\n    getOrCreateGauge(\n      'sync',\n      'worker-client-groups',\n      'Number of active client groups assigned to worker',\n    ).addCallback(result => {\n      for (let i = 0; i < syncers.length; i++) {\n        result.observe(assigner.getWorkerLoad(i), {worker: String(i)});\n      }\n    });\n\n    syncers.forEach((syncer, index) => {\n      syncer.onMessageType<ClientGroupStatusMessage>(\n        'clientGroupStatus',\n        ({clientGroupID, active, generation}) => {\n          if (!active) {\n            assigner.release(clientGroupID, index, generation);\n          }\n        },\n      );\n    });\n\n    function connectParams(req: IncomingMessageSubset) {\n      const {headers, url: u} = req;\n      const url = new URL(u ?? '', 'http://unused/');\n      const path = parsePath(url);\n      if (!path) {\n        throw new Error(`Invalid URL: ${u}`);\n      }\n      const version = Number(path.version);\n      if (Number.isNaN(version)) {\n        throw new Error(`Invalid version: ${u}`);\n      }\n      const {params, error} = getConnectParams(version, url, headers);\n      if (error !== null) {\n        throw new Error(error);\n      }\n      return params;\n    }\n\n    const handlePush = (req: IncomingMessageSubset) => {\n      assert(\n        mutator !== undefined,\n        'Received a push for a custom mutation but no `push.url` was configured.',\n      );\n      return {payload: connectParams(req), sender: mutator};\n    };\n\n    let maxProtocolVersion = 0;\n    getOrCreateGauge(\n      'sync',\n      'max-protocol-version',\n      'Latest sync protocol version from a connecting client',\n    ).addCallback(result => {\n      if (maxProtocolVersion) {\n        result.observe(maxProtocolVersion);\n      }\n    });\n\n    const handleSync = (req: IncomingMessageSubset) => {\n      assert(syncers.length, 'Received a sync request with no sync workers.');\n      const params = connectParams(req);\n      const {clientGroupID, protocolVersion} = params;\n      maxProtocolVersion = Math.max(maxProtocolVersion, protocolVersion);\n\n      const {worker: syncer, generation} = assigner.assign(clientGroupID);\n      workerDispatchesCounter.add(1, {worker: String(syncer)});\n\n      lc.debug?.(`connecting ${clientGroupID} to syncer ${syncer}`);\n      return {payload: {...params, generation}, sender: syncers[syncer]};\n    };\n\n    const handleChangeStream = (req: IncomingMessageSubset) => {\n      // Note: The change-streamer is generally not dispatched via the main\n      //       port, and in particular, should *not* be accessible via that\n      //       port in single-node mode. However, this plumbing is maintained\n      //       for the purpose of allowing --lazy-startup of the\n      //       replication-manager as a possible future feature.\n      assert(\n        syncers.length === 0 && mutator === undefined,\n        'Dispatch to the change-streamer via the main port ' +\n          'is only allowed in multi-node mode',\n      );\n      assert(\n        changeStreamer,\n        'Received a change-streamer request without a change-streamer worker',\n      );\n      const url = new URL(req.url ?? '', 'http://unused/');\n      const path = parsePath(url);\n      if (!path) {\n        throw new Error(`Invalid URL: ${req.url}`);\n      }\n\n      return {\n        payload: path.action,\n        sender: changeStreamer,\n      };\n    };\n\n    // handoff messages from this ZeroDispatcher to the appropriate worker (pool).\n    installWebSocketHandoff<unknown>(\n      lc,\n      request => {\n        const {url: u} = request;\n        const url = new URL(u ?? '', 'http://unused/');\n        const path = parsePath(url);\n        if (!path) {\n          throw new Error(`Invalid URL: ${u}`);\n        }\n        switch (path.worker) {\n          case 'sync':\n            return handleSync(request);\n          case 'replication':\n            return handleChangeStream(request);\n          case 'mutate':\n            return handlePush(request);\n          default:\n            throw new Error(`Invalid URL: ${u}`);\n        }\n      },\n      parent,\n    );\n  }\n\n  run() {\n    const readyStart = Date.now();\n    getOrCreateGauge('server', 'uptime', {\n      description: 'Cumulative uptime, starting from when requests are served',\n      unit: 's',\n    }).addCallback(result => result.observe((Date.now() - readyStart) / 1000));\n\n    return this.#state.stopped();\n  }\n\n  stop() {\n    this.#assigner.destroy();\n    this.#state.stop(this.#lc);\n    return this.#state.stopped();\n  }\n}\n\nconst URL_PATTERN = new UrlPattern('(/:base)/:worker/v:version/:action');\n\nexport function parsePath(url: URL):\n  | {\n      base?: string;\n      worker: 'sync' | 'mutate' | 'replication';\n      version: string;\n      action: string;\n    }\n  | undefined {\n  // The match() returns both null and undefined.\n  return URL_PATTERN.match(url.pathname) || undefined;\n} // The server allows the client to use any /:base/ path to facilitate\n// servicing requests on the same domain as the application.\n"],"mappings":";;;;;;;;AAeA,IAAa,mBAAb,MAAiD;CAC/C,KAAc;CACd;CACA,SAAkB,IAAI,aAAa,KAAK,EAAE;CAC1C;CAEA,YACE,IACA,QACA,QACA,SACA,SACA,gBACA;EACA,KAAK,MAAM;EAEX,MAAM,WAAW,IAAI,eAAe,QAAQ,QAAQ,MAAM;EAC1D,KAAK,YAAY;EAEjB,MAAM,0BAA0B,mBAC9B,QACA,qBACA,+CACF;EAEA,iBACE,QACA,wBACA,mDACF,CAAC,CAAC,aAAY,WAAU;GACtB,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAClC,OAAO,QAAQ,SAAS,cAAc,CAAC,GAAG,EAAC,QAAQ,OAAO,CAAC,EAAC,CAAC;EAEjE,CAAC;EAED,QAAQ,SAAS,QAAQ,UAAU;GACjC,OAAO,cACL,sBACC,EAAC,eAAe,QAAQ,iBAAgB;IACvC,IAAI,CAAC,QACH,SAAS,QAAQ,eAAe,OAAO,UAAU;GAErD,CACF;EACF,CAAC;EAED,SAAS,cAAc,KAA4B;GACjD,MAAM,EAAC,SAAS,KAAK,MAAK;GAC1B,MAAM,MAAM,IAAI,IAAI,KAAK,IAAI,gBAAgB;GAC7C,MAAM,OAAO,UAAU,GAAG;GAC1B,IAAI,CAAC,MACH,MAAM,IAAI,MAAM,gBAAgB,GAAG;GAErC,MAAM,UAAU,OAAO,KAAK,OAAO;GACnC,IAAI,OAAO,MAAM,OAAO,GACtB,MAAM,IAAI,MAAM,oBAAoB,GAAG;GAEzC,MAAM,EAAC,QAAQ,UAAS,iBAAiB,SAAS,KAAK,OAAO;GAC9D,IAAI,UAAU,MACZ,MAAM,IAAI,MAAM,KAAK;GAEvB,OAAO;EACT;EAEA,MAAM,cAAc,QAA+B;GACjD,OACE,YAAY,KAAA,GACZ,yEACF;GACA,OAAO;IAAC,SAAS,cAAc,GAAG;IAAG,QAAQ;GAAO;EACtD;EAEA,IAAI,qBAAqB;EACzB,iBACE,QACA,wBACA,uDACF,CAAC,CAAC,aAAY,WAAU;GACtB,IAAI,oBACF,OAAO,QAAQ,kBAAkB;EAErC,CAAC;EAED,MAAM,cAAc,QAA+B;GACjD,OAAO,QAAQ,QAAQ,+CAA+C;GACtE,MAAM,SAAS,cAAc,GAAG;GAChC,MAAM,EAAC,eAAe,oBAAmB;GACzC,qBAAqB,KAAK,IAAI,oBAAoB,eAAe;GAEjE,MAAM,EAAC,QAAQ,QAAQ,eAAc,SAAS,OAAO,aAAa;GAClE,wBAAwB,IAAI,GAAG,EAAC,QAAQ,OAAO,MAAM,EAAC,CAAC;GAEvD,GAAG,QAAQ,cAAc,cAAc,aAAa,QAAQ;GAC5D,OAAO;IAAC,SAAS;KAAC,GAAG;KAAQ;IAAU;IAAG,QAAQ,QAAQ;GAAO;EACnE;EAEA,MAAM,sBAAsB,QAA+B;GAMzD,OACE,QAAQ,WAAW,KAAK,YAAY,KAAA,GACpC,sFAEF;GACA,OACE,gBACA,qEACF;GAEA,MAAM,OAAO,UAAU,IADP,IAAI,IAAI,OAAO,IAAI,gBACZ,CAAG;GAC1B,IAAI,CAAC,MACH,MAAM,IAAI,MAAM,gBAAgB,IAAI,KAAK;GAG3C,OAAO;IACL,SAAS,KAAK;IACd,QAAQ;GACV;EACF;EAGA,wBACE,KACA,YAAW;GACT,MAAM,EAAC,KAAK,MAAK;GAEjB,MAAM,OAAO,UAAU,IADP,IAAI,KAAK,IAAI,gBACN,CAAG;GAC1B,IAAI,CAAC,MACH,MAAM,IAAI,MAAM,gBAAgB,GAAG;GAErC,QAAQ,KAAK,QAAb;IACE,KAAK,QACH,OAAO,WAAW,OAAO;IAC3B,KAAK,eACH,OAAO,mBAAmB,OAAO;IACnC,KAAK,UACH,OAAO,WAAW,OAAO;IAC3B,SACE,MAAM,IAAI,MAAM,gBAAgB,GAAG;GACvC;EACF,GACA,MACF;CACF;CAEA,MAAM;EACJ,MAAM,aAAa,KAAK,IAAI;EAC5B,iBAAiB,UAAU,UAAU;GACnC,aAAa;GACb,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,WAAU,OAAO,SAAS,KAAK,IAAI,IAAI,cAAc,GAAI,CAAC;EAEzE,OAAO,KAAK,OAAO,QAAQ;CAC7B;CAEA,OAAO;EACL,KAAK,UAAU,QAAQ;EACvB,KAAK,OAAO,KAAK,KAAK,GAAG;EACzB,OAAO,KAAK,OAAO,QAAQ;CAC7B;AACF;AAEA,IAAM,cAAc,IAAI,WAAW,oCAAoC;AAEvE,SAAgB,UAAU,KAOZ;CAEZ,OAAO,YAAY,MAAM,IAAI,QAAQ,KAAK,KAAA;AAC5C"}
+{"version":3,"file":"worker-dispatcher.js","names":[],"sources":["../../../../../zero-cache/src/server/worker-dispatcher.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport UrlPattern from 'url-pattern';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n} from '../observability/metrics.ts';\nimport {RunningState} from '../services/running-state.ts';\nimport type {Service} from '../services/service.ts';\nimport {clientGroupLogID} from '../types/client-group-log-id.ts';\nimport type {IncomingMessageSubset} from '../types/http.ts';\nimport type {ClientGroupStatusMessage, Worker} from '../types/processes.ts';\nimport {installWebSocketHandoff} from '../types/websocket-handoff.ts';\nimport {getConnectParams} from '../workers/connect-params.ts';\nimport {SyncerAssigner} from './syncer-assigner.ts';\n\nexport class WorkerDispatcher implements Service {\n  readonly id = 'worker-dispatcher';\n  readonly #lc: LogContext;\n  readonly #state = new RunningState(this.id);\n  readonly #assigner: SyncerAssigner;\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    parent: Worker,\n    syncers: Worker[],\n    mutator: Worker | undefined,\n    changeStreamer: Worker | undefined,\n  ) {\n    this.#lc = lc;\n\n    const assigner = new SyncerAssigner(taskID, syncers.length);\n    this.#assigner = assigner;\n\n    const workerDispatchesCounter = getOrCreateCounter(\n      'sync',\n      'worker-dispatches',\n      'Dispatched client group connections to worker',\n    );\n\n    getOrCreateGauge(\n      'sync',\n      'worker-client-groups',\n      'Number of active client groups assigned to worker',\n    ).addCallback(result => {\n      for (let i = 0; i < syncers.length; i++) {\n        result.observe(assigner.getWorkerLoad(i), {worker: String(i)});\n      }\n    });\n\n    syncers.forEach((syncer, index) => {\n      syncer.onMessageType<ClientGroupStatusMessage>(\n        'clientGroupStatus',\n        ({clientGroupID, active, generation}) => {\n          if (!active) {\n            assigner.release(clientGroupID, index, generation);\n          }\n        },\n      );\n    });\n\n    function connectParams(req: IncomingMessageSubset) {\n      const {headers, url: u} = req;\n      const url = new URL(u ?? '', 'http://unused/');\n      const path = parsePath(url);\n      if (!path) {\n        throw new Error(`Invalid URL: ${u}`);\n      }\n      const version = Number(path.version);\n      if (Number.isNaN(version)) {\n        throw new Error(`Invalid version: ${u}`);\n      }\n      const {params, error} = getConnectParams(version, url, headers);\n      if (error !== null) {\n        throw new Error(error);\n      }\n      return params;\n    }\n\n    const handlePush = (req: IncomingMessageSubset) => {\n      assert(\n        mutator !== undefined,\n        'Received a push for a custom mutation but no `push.url` was configured.',\n      );\n      return {payload: connectParams(req), sender: mutator};\n    };\n\n    let maxProtocolVersion = 0;\n    getOrCreateGauge(\n      'sync',\n      'max-protocol-version',\n      'Latest sync protocol version from a connecting client',\n    ).addCallback(result => {\n      if (maxProtocolVersion) {\n        result.observe(maxProtocolVersion);\n      }\n    });\n\n    const handleSync = (req: IncomingMessageSubset) => {\n      assert(syncers.length, 'Received a sync request with no sync workers.');\n      const params = connectParams(req);\n      const {clientGroupID, protocolVersion} = params;\n      maxProtocolVersion = Math.max(maxProtocolVersion, protocolVersion);\n\n      const {worker: syncer, generation} = assigner.assign(clientGroupID);\n      workerDispatchesCounter.add(1, {worker: String(syncer)});\n\n      lc.debug?.(\n        `connecting ${clientGroupLogID(clientGroupID)} to syncer ${syncer}`,\n      );\n      return {payload: {...params, generation}, sender: syncers[syncer]};\n    };\n\n    const handleChangeStream = (req: IncomingMessageSubset) => {\n      // Note: The change-streamer is generally not dispatched via the main\n      //       port, and in particular, should *not* be accessible via that\n      //       port in single-node mode. However, this plumbing is maintained\n      //       for the purpose of allowing --lazy-startup of the\n      //       replication-manager as a possible future feature.\n      assert(\n        syncers.length === 0 && mutator === undefined,\n        'Dispatch to the change-streamer via the main port ' +\n          'is only allowed in multi-node mode',\n      );\n      assert(\n        changeStreamer,\n        'Received a change-streamer request without a change-streamer worker',\n      );\n      const url = new URL(req.url ?? '', 'http://unused/');\n      const path = parsePath(url);\n      if (!path) {\n        throw new Error(`Invalid URL: ${req.url}`);\n      }\n\n      return {\n        payload: path.action,\n        sender: changeStreamer,\n      };\n    };\n\n    // handoff messages from this ZeroDispatcher to the appropriate worker (pool).\n    installWebSocketHandoff<unknown>(\n      lc,\n      request => {\n        const {url: u} = request;\n        const url = new URL(u ?? '', 'http://unused/');\n        const path = parsePath(url);\n        if (!path) {\n          throw new Error(`Invalid URL: ${u}`);\n        }\n        switch (path.worker) {\n          case 'sync':\n            return handleSync(request);\n          case 'replication':\n            return handleChangeStream(request);\n          case 'mutate':\n            return handlePush(request);\n          default:\n            throw new Error(`Invalid URL: ${u}`);\n        }\n      },\n      parent,\n    );\n  }\n\n  run() {\n    const readyStart = Date.now();\n    getOrCreateGauge('server', 'uptime', {\n      description: 'Cumulative uptime, starting from when requests are served',\n      unit: 's',\n    }).addCallback(result => result.observe((Date.now() - readyStart) / 1000));\n\n    return this.#state.stopped();\n  }\n\n  stop() {\n    this.#assigner.destroy();\n    this.#state.stop(this.#lc);\n    return this.#state.stopped();\n  }\n}\n\nconst URL_PATTERN = new UrlPattern('(/:base)/:worker/v:version/:action');\n\nexport function parsePath(url: URL):\n  | {\n      base?: string;\n      worker: 'sync' | 'mutate' | 'replication';\n      version: string;\n      action: string;\n    }\n  | undefined {\n  // The match() returns both null and undefined.\n  return URL_PATTERN.match(url.pathname) || undefined;\n} // The server allows the client to use any /:base/ path to facilitate\n// servicing requests on the same domain as the application.\n"],"mappings":";;;;;;;;;AAgBA,IAAa,mBAAb,MAAiD;CAC/C,KAAc;CACd;CACA,SAAkB,IAAI,aAAa,KAAK,EAAE;CAC1C;CAEA,YACE,IACA,QACA,QACA,SACA,SACA,gBACA;EACA,KAAK,MAAM;EAEX,MAAM,WAAW,IAAI,eAAe,QAAQ,QAAQ,MAAM;EAC1D,KAAK,YAAY;EAEjB,MAAM,0BAA0B,mBAC9B,QACA,qBACA,+CACF;EAEA,iBACE,QACA,wBACA,mDACF,CAAC,CAAC,aAAY,WAAU;GACtB,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAClC,OAAO,QAAQ,SAAS,cAAc,CAAC,GAAG,EAAC,QAAQ,OAAO,CAAC,EAAC,CAAC;EAEjE,CAAC;EAED,QAAQ,SAAS,QAAQ,UAAU;GACjC,OAAO,cACL,sBACC,EAAC,eAAe,QAAQ,iBAAgB;IACvC,IAAI,CAAC,QACH,SAAS,QAAQ,eAAe,OAAO,UAAU;GAErD,CACF;EACF,CAAC;EAED,SAAS,cAAc,KAA4B;GACjD,MAAM,EAAC,SAAS,KAAK,MAAK;GAC1B,MAAM,MAAM,IAAI,IAAI,KAAK,IAAI,gBAAgB;GAC7C,MAAM,OAAO,UAAU,GAAG;GAC1B,IAAI,CAAC,MACH,MAAM,IAAI,MAAM,gBAAgB,GAAG;GAErC,MAAM,UAAU,OAAO,KAAK,OAAO;GACnC,IAAI,OAAO,MAAM,OAAO,GACtB,MAAM,IAAI,MAAM,oBAAoB,GAAG;GAEzC,MAAM,EAAC,QAAQ,UAAS,iBAAiB,SAAS,KAAK,OAAO;GAC9D,IAAI,UAAU,MACZ,MAAM,IAAI,MAAM,KAAK;GAEvB,OAAO;EACT;EAEA,MAAM,cAAc,QAA+B;GACjD,OACE,YAAY,KAAA,GACZ,yEACF;GACA,OAAO;IAAC,SAAS,cAAc,GAAG;IAAG,QAAQ;GAAO;EACtD;EAEA,IAAI,qBAAqB;EACzB,iBACE,QACA,wBACA,uDACF,CAAC,CAAC,aAAY,WAAU;GACtB,IAAI,oBACF,OAAO,QAAQ,kBAAkB;EAErC,CAAC;EAED,MAAM,cAAc,QAA+B;GACjD,OAAO,QAAQ,QAAQ,+CAA+C;GACtE,MAAM,SAAS,cAAc,GAAG;GAChC,MAAM,EAAC,eAAe,oBAAmB;GACzC,qBAAqB,KAAK,IAAI,oBAAoB,eAAe;GAEjE,MAAM,EAAC,QAAQ,QAAQ,eAAc,SAAS,OAAO,aAAa;GAClE,wBAAwB,IAAI,GAAG,EAAC,QAAQ,OAAO,MAAM,EAAC,CAAC;GAEvD,GAAG,QACD,cAAc,iBAAiB,aAAa,EAAE,aAAa,QAC7D;GACA,OAAO;IAAC,SAAS;KAAC,GAAG;KAAQ;IAAU;IAAG,QAAQ,QAAQ;GAAO;EACnE;EAEA,MAAM,sBAAsB,QAA+B;GAMzD,OACE,QAAQ,WAAW,KAAK,YAAY,KAAA,GACpC,sFAEF;GACA,OACE,gBACA,qEACF;GAEA,MAAM,OAAO,UAAU,IADP,IAAI,IAAI,OAAO,IAAI,gBACZ,CAAG;GAC1B,IAAI,CAAC,MACH,MAAM,IAAI,MAAM,gBAAgB,IAAI,KAAK;GAG3C,OAAO;IACL,SAAS,KAAK;IACd,QAAQ;GACV;EACF;EAGA,wBACE,KACA,YAAW;GACT,MAAM,EAAC,KAAK,MAAK;GAEjB,MAAM,OAAO,UAAU,IADP,IAAI,KAAK,IAAI,gBACN,CAAG;GAC1B,IAAI,CAAC,MACH,MAAM,IAAI,MAAM,gBAAgB,GAAG;GAErC,QAAQ,KAAK,QAAb;IACE,KAAK,QACH,OAAO,WAAW,OAAO;IAC3B,KAAK,eACH,OAAO,mBAAmB,OAAO;IACnC,KAAK,UACH,OAAO,WAAW,OAAO;IAC3B,SACE,MAAM,IAAI,MAAM,gBAAgB,GAAG;GACvC;EACF,GACA,MACF;CACF;CAEA,MAAM;EACJ,MAAM,aAAa,KAAK,IAAI;EAC5B,iBAAiB,UAAU,UAAU;GACnC,aAAa;GACb,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,WAAU,OAAO,SAAS,KAAK,IAAI,IAAI,cAAc,GAAI,CAAC;EAEzE,OAAO,KAAK,OAAO,QAAQ;CAC7B;CAEA,OAAO;EACL,KAAK,UAAU,QAAQ;EACvB,KAAK,OAAO,KAAK,KAAK,GAAG;EACzB,OAAO,KAAK,OAAO,QAAQ;CAC7B;AACF;AAEA,IAAM,cAAc,IAAI,WAAW,oCAAoC;AAEvE,SAAgB,UAAU,KAOZ;CAEZ,OAAO,YAAY,MAAM,IAAI,QAAQ,KAAK,KAAA;AAC5C"}

package/out/zero-cache/src/services/admin-auth.d.ts

@@ -0,0 +1,22 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { FastifyReply, FastifyRequest } from 'fastify';
+import type { NormalizedZeroConfig } from '../config/normalize.ts';
+import { type AdminAccess } from '../config/zero-config.ts';
+/**
+ * Checks the HTTP basic-auth credentials of a request to an endpoint guarded
+ * by the admin password, and limits how often a wrong password can be tried.
+ *
+ * Returns the access the credentials grant. When they grant none, the
+ * response has been sent (401 with a `WWW-Authenticate` challenge for
+ * `realm`) and `undefined` is returned. While attempts are blocked after
+ * repeated failures, every request with credentials is refused with 429 and a
+ * `Retry-After` header without checking the password, so that the response
+ * does not reveal whether it was right. A request without credentials is
+ * never counted as an attempt.
+ *
+ * @param required `'admin'` accepts only the admin password (see
+ *        {@link isAdminPasswordValid}); `'operator'` accepts the operator
+ *        password too (see {@link getOperatorAccess}).
+ */
+export declare function authorizeAdminRequest(lc: LogContext, config: Pick<NormalizedZeroConfig, 'adminPassword' | 'operatorPassword'>, req: FastifyRequest, res: FastifyReply, realm: string, required: AdminAccess): AdminAccess | undefined;
+//# sourceMappingURL=admin-auth.d.ts.map

package/out/zero-cache/src/services/admin-auth.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"admin-auth.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/admin-auth.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,SAAS,CAAC;AAC1D,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,wBAAwB,CAAC;AACjE,OAAO,EAGL,KAAK,WAAW,EACjB,MAAM,0BAA0B,CAAC;AAYlC;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,qBAAqB,CACnC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,IAAI,CAAC,oBAAoB,EAAE,eAAe,GAAG,kBAAkB,CAAC,EACxE,GAAG,EAAE,cAAc,EACnB,GAAG,EAAE,YAAY,EACjB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,WAAW,GACpB,WAAW,GAAG,SAAS,CAiCzB"}

package/out/zero-cache/src/services/admin-auth.js

@@ -0,0 +1,42 @@
+import { getOperatorAccess, isAdminPasswordValid } from "../config/zero-config.js";
+import { AuthAttemptCounter } from "./auth-attempt-counter.js";
+import auth from "basic-auth";
+//#region ../zero-cache/src/services/admin-auth.ts
+var attempts = new AuthAttemptCounter(5);
+/**
+* Checks the HTTP basic-auth credentials of a request to an endpoint guarded
+* by the admin password, and limits how often a wrong password can be tried.
+*
+* Returns the access the credentials grant. When they grant none, the
+* response has been sent (401 with a `WWW-Authenticate` challenge for
+* `realm`) and `undefined` is returned. While attempts are blocked after
+* repeated failures, every request with credentials is refused with 429 and a
+* `Retry-After` header without checking the password, so that the response
+* does not reveal whether it was right. A request without credentials is
+* never counted as an attempt.
+*
+* @param required `'admin'` accepts only the admin password (see
+*        {@link isAdminPasswordValid}); `'operator'` accepts the operator
+*        password too (see {@link getOperatorAccess}).
+*/
+function authorizeAdminRequest(lc, config, req, res, realm, required) {
+	const password = auth(req)?.pass;
+	const attempted = password !== void 0;
+	if (attempted && attempts.blockedFor() > 0) {
+		attempts.recordFailure();
+		lc.warn?.("Refusing admin password attempt while blocked");
+		res.code(429).header("Retry-After", Math.ceil(attempts.blockedFor() / 1e3)).send("Too Many Requests");
+		return;
+	}
+	const access = required === "admin" ? isAdminPasswordValid(lc, config, password) ? "admin" : void 0 : getOperatorAccess(lc, config, password);
+	if (access !== void 0) {
+		attempts.reset();
+		return access;
+	}
+	if (attempted) attempts.recordFailure();
+	res.code(401).header("WWW-Authenticate", `Basic realm="${realm}"`).send("Unauthorized");
+}
+//#endregion
+export { authorizeAdminRequest };
+
+//# sourceMappingURL=admin-auth.js.map

package/out/zero-cache/src/services/admin-auth.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"admin-auth.js","names":[],"sources":["../../../../../zero-cache/src/services/admin-auth.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport auth from 'basic-auth';\nimport type {FastifyReply, FastifyRequest} from 'fastify';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {\n  getOperatorAccess,\n  isAdminPasswordValid,\n  type AdminAccess,\n} from '../config/zero-config.ts';\nimport {AuthAttemptCounter} from './auth-attempt-counter.ts';\n\n/**\n * Failed attempts allowed before the process blocks further attempts. All\n * HTTP endpoints guarded by the admin (or operator) password share this one\n * counter, since the password is the same for all of them.\n */\nconst FREE_ATTEMPTS = 5;\n\nconst attempts = new AuthAttemptCounter(FREE_ATTEMPTS);\n\n/**\n * Checks the HTTP basic-auth credentials of a request to an endpoint guarded\n * by the admin password, and limits how often a wrong password can be tried.\n *\n * Returns the access the credentials grant. When they grant none, the\n * response has been sent (401 with a `WWW-Authenticate` challenge for\n * `realm`) and `undefined` is returned. While attempts are blocked after\n * repeated failures, every request with credentials is refused with 429 and a\n * `Retry-After` header without checking the password, so that the response\n * does not reveal whether it was right. A request without credentials is\n * never counted as an attempt.\n *\n * @param required `'admin'` accepts only the admin password (see\n *        {@link isAdminPasswordValid}); `'operator'` accepts the operator\n *        password too (see {@link getOperatorAccess}).\n */\nexport function authorizeAdminRequest(\n  lc: LogContext,\n  config: Pick<NormalizedZeroConfig, 'adminPassword' | 'operatorPassword'>,\n  req: FastifyRequest,\n  res: FastifyReply,\n  realm: string,\n  required: AdminAccess,\n): AdminAccess | undefined {\n  const password = auth(req)?.pass;\n  const attempted = password !== undefined;\n\n  if (attempted && attempts.blockedFor() > 0) {\n    attempts.recordFailure();\n    lc.warn?.('Refusing admin password attempt while blocked');\n    void res\n      .code(429)\n      .header('Retry-After', Math.ceil(attempts.blockedFor() / 1000))\n      .send('Too Many Requests');\n    return undefined;\n  }\n\n  const access =\n    required === 'admin'\n      ? isAdminPasswordValid(lc, config, password)\n        ? 'admin'\n        : undefined\n      : getOperatorAccess(lc, config, password);\n  if (access !== undefined) {\n    attempts.reset();\n    return access;\n  }\n\n  if (attempted) {\n    attempts.recordFailure();\n  }\n  void res\n    .code(401)\n    .header('WWW-Authenticate', `Basic realm=\"${realm}\"`)\n    .send('Unauthorized');\n  return undefined;\n}\n"],"mappings":";;;;AAkBA,IAAM,WAAW,IAAI,mBAAmB,CAAa;;;;;;;;;;;;;;;;;AAkBrD,SAAgB,sBACd,IACA,QACA,KACA,KACA,OACA,UACyB;CACzB,MAAM,WAAW,KAAK,GAAG,CAAC,EAAE;CAC5B,MAAM,YAAY,aAAa,KAAA;CAE/B,IAAI,aAAa,SAAS,WAAW,IAAI,GAAG;EAC1C,SAAS,cAAc;EACvB,GAAG,OAAO,+CAA+C;EACzD,IACG,KAAK,GAAG,CAAC,CACT,OAAO,eAAe,KAAK,KAAK,SAAS,WAAW,IAAI,GAAI,CAAC,CAAC,CAC9D,KAAK,mBAAmB;EAC3B;CACF;CAEA,MAAM,SACJ,aAAa,UACT,qBAAqB,IAAI,QAAQ,QAAQ,IACvC,UACA,KAAA,IACF,kBAAkB,IAAI,QAAQ,QAAQ;CAC5C,IAAI,WAAW,KAAA,GAAW;EACxB,SAAS,MAAM;EACf,OAAO;CACT;CAEA,IAAI,WACF,SAAS,cAAc;CAEzB,IACG,KAAK,GAAG,CAAC,CACT,OAAO,oBAAoB,gBAAgB,MAAM,EAAE,CAAC,CACpD,KAAK,cAAc;AAExB"}

package/out/zero-cache/src/services/auth-attempt-counter.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Counts failed password attempts and blocks further attempts after a few
+ * failures, for a period that doubles with every failure, up to a cap.
+ *
+ * A caller that finds the counter blocked must not check the password at all:
+ * it records the attempt as another failure, which extends the block, so that
+ * a correct password submitted during the block is refused too and the block
+ * cannot be ended by guessing right. The counter starts over after a quiet
+ * period without failures, or when the caller resets it on success.
+ *
+ * A blocked attempt is refused immediately rather than delayed, so that the
+ * caller never has to wait while holding a lock or an open request.
+ */
+export declare class AuthAttemptCounter {
+    #private;
+    /**
+     * @param freeAttempts Failures allowed before the counter blocks.
+     */
+    constructor(freeAttempts: number, now?: () => number);
+    /**
+     * Milliseconds until attempts are allowed again, or 0 when they are
+     * allowed now.
+     */
+    blockedFor(): number;
+    /**
+     * Records a failed (or blocked) attempt. Returns the number of failures
+     * since the counter last started over.
+     */
+    recordFailure(): number;
+    /** Forgets all failures, as after a successful attempt. */
+    reset(): void;
+}
+//# sourceMappingURL=auth-attempt-counter.d.ts.map

package/out/zero-cache/src/services/auth-attempt-counter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"auth-attempt-counter.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/auth-attempt-counter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAOH,qBAAa,kBAAkB;;IAO7B;;OAEG;IACH,YAAY,YAAY,EAAE,MAAM,EAAE,GAAG,GAAE,MAAM,MAAyB,EAGrE;IAED;;;OAGG;IACH,UAAU,IAAI,MAAM,CAEnB;IAED;;;OAGG;IACH,aAAa,IAAI,MAAM,CAgBtB;IAED,2DAA2D;IAC3D,KAAK,IAAI,IAAI,CAIZ;CACF"}

package/out/zero-cache/src/services/auth-attempt-counter.js

@@ -0,0 +1,61 @@
+//#region ../zero-cache/src/services/auth-attempt-counter.ts
+/**
+* Counts failed password attempts and blocks further attempts after a few
+* failures, for a period that doubles with every failure, up to a cap.
+*
+* A caller that finds the counter blocked must not check the password at all:
+* it records the attempt as another failure, which extends the block, so that
+* a correct password submitted during the block is refused too and the block
+* cannot be ended by guessing right. The counter starts over after a quiet
+* period without failures, or when the caller resets it on success.
+*
+* A blocked attempt is refused immediately rather than delayed, so that the
+* caller never has to wait while holding a lock or an open request.
+*/
+var BASE_BLOCK_MS = 1e3;
+var MAX_BLOCK_MS = 6e4;
+/** A counter with no failures for this long starts over. */
+var RESET_AFTER_MS = 9e5;
+var AuthAttemptCounter = class {
+	#freeAttempts;
+	#now;
+	#failures = 0;
+	#lastFailure = 0;
+	#blockedUntil = 0;
+	/**
+	* @param freeAttempts Failures allowed before the counter blocks.
+	*/
+	constructor(freeAttempts, now = () => Date.now()) {
+		this.#freeAttempts = freeAttempts;
+		this.#now = now;
+	}
+	/**
+	* Milliseconds until attempts are allowed again, or 0 when they are
+	* allowed now.
+	*/
+	blockedFor() {
+		return Math.max(0, this.#blockedUntil - this.#now());
+	}
+	/**
+	* Records a failed (or blocked) attempt. Returns the number of failures
+	* since the counter last started over.
+	*/
+	recordFailure() {
+		const now = this.#now();
+		if (now - this.#lastFailure > RESET_AFTER_MS) this.#failures = 0;
+		this.#failures++;
+		this.#lastFailure = now;
+		if (this.#failures >= this.#freeAttempts) this.#blockedUntil = now + Math.min(BASE_BLOCK_MS * 2 ** (this.#failures - this.#freeAttempts), MAX_BLOCK_MS);
+		return this.#failures;
+	}
+	/** Forgets all failures, as after a successful attempt. */
+	reset() {
+		this.#failures = 0;
+		this.#lastFailure = 0;
+		this.#blockedUntil = 0;
+	}
+};
+//#endregion
+export { AuthAttemptCounter };
+
+//# sourceMappingURL=auth-attempt-counter.js.map

package/out/zero-cache/src/services/auth-attempt-counter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"auth-attempt-counter.js","names":[],"sources":["../../../../../zero-cache/src/services/auth-attempt-counter.ts"],"sourcesContent":["/**\n * Counts failed password attempts and blocks further attempts after a few\n * failures, for a period that doubles with every failure, up to a cap.\n *\n * A caller that finds the counter blocked must not check the password at all:\n * it records the attempt as another failure, which extends the block, so that\n * a correct password submitted during the block is refused too and the block\n * cannot be ended by guessing right. The counter starts over after a quiet\n * period without failures, or when the caller resets it on success.\n *\n * A blocked attempt is refused immediately rather than delayed, so that the\n * caller never has to wait while holding a lock or an open request.\n */\n\nconst BASE_BLOCK_MS = 1_000;\nconst MAX_BLOCK_MS = 60_000;\n/** A counter with no failures for this long starts over. */\nconst RESET_AFTER_MS = 15 * 60_000;\n\nexport class AuthAttemptCounter {\n  readonly #freeAttempts: number;\n  readonly #now: () => number;\n  #failures = 0;\n  #lastFailure = 0;\n  #blockedUntil = 0;\n\n  /**\n   * @param freeAttempts Failures allowed before the counter blocks.\n   */\n  constructor(freeAttempts: number, now: () => number = () => Date.now()) {\n    this.#freeAttempts = freeAttempts;\n    this.#now = now;\n  }\n\n  /**\n   * Milliseconds until attempts are allowed again, or 0 when they are\n   * allowed now.\n   */\n  blockedFor(): number {\n    return Math.max(0, this.#blockedUntil - this.#now());\n  }\n\n  /**\n   * Records a failed (or blocked) attempt. Returns the number of failures\n   * since the counter last started over.\n   */\n  recordFailure(): number {\n    const now = this.#now();\n    if (now - this.#lastFailure > RESET_AFTER_MS) {\n      this.#failures = 0;\n    }\n    this.#failures++;\n    this.#lastFailure = now;\n    if (this.#failures >= this.#freeAttempts) {\n      this.#blockedUntil =\n        now +\n        Math.min(\n          BASE_BLOCK_MS * 2 ** (this.#failures - this.#freeAttempts),\n          MAX_BLOCK_MS,\n        );\n    }\n    return this.#failures;\n  }\n\n  /** Forgets all failures, as after a successful attempt. */\n  reset(): void {\n    this.#failures = 0;\n    this.#lastFailure = 0;\n    this.#blockedUntil = 0;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAcA,IAAM,gBAAgB;AACtB,IAAM,eAAe;;AAErB,IAAM,iBAAiB;AAEvB,IAAa,qBAAb,MAAgC;CAC9B;CACA;CACA,YAAY;CACZ,eAAe;CACf,gBAAgB;;;;CAKhB,YAAY,cAAsB,YAA0B,KAAK,IAAI,GAAG;EACtE,KAAK,gBAAgB;EACrB,KAAK,OAAO;CACd;;;;;CAMA,aAAqB;EACnB,OAAO,KAAK,IAAI,GAAG,KAAK,gBAAgB,KAAK,KAAK,CAAC;CACrD;;;;;CAMA,gBAAwB;EACtB,MAAM,MAAM,KAAK,KAAK;EACtB,IAAI,MAAM,KAAK,eAAe,gBAC5B,KAAK,YAAY;EAEnB,KAAK;EACL,KAAK,eAAe;EACpB,IAAI,KAAK,aAAa,KAAK,eACzB,KAAK,gBACH,MACA,KAAK,IACH,gBAAgB,MAAM,KAAK,YAAY,KAAK,gBAC5C,YACF;EAEJ,OAAO,KAAK;CACd;;CAGA,QAAc;EACZ,KAAK,YAAY;EACjB,KAAK,eAAe;EACpB,KAAK,gBAAgB;CACvB;AACF"}

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js

@@ -1,7 +1,7 @@
 import { assert } from "../../../../../shared/src/asserts.js";
+import { randomCharacters } from "../../../../../shared/src/random-values.js";
 import { parse } from "../../../../../shared/src/valita.js";
 import { must } from "../../../../../shared/src/must.js";
-import { randomCharacters } from "../../../../../shared/src/random-values.js";
 import { equals } from "../../../../../shared/src/set-utils.js";
 import { id } from "../../../types/sql.js";
 import { connectPgClient, pgClient } from "../../../types/pg.js";