@rocicorp/zero 1.12.0-canary.7 → 1.12.0-canary.9

package/out/zero-cache/src/services/replicator/change-log-db.js

@@ -487,6 +487,6 @@ function tableExists(db, table) {
       `).get(table) !== void 0;
 }
 //#endregion
-export { CHANGE_LOG_META_TABLE, CHANGE_LOG_STREAM_TABLE, CHANGE_LOG_STREAM_WRITE_TIME_INDEX, CREATE_CHANGE_LOG_STREAM_SCHEMA, ChangeLogRebuildRequired, MAX_RECONCILE_TRUNCATE_BYTES, MAX_RECONCILE_TRUNCATE_ROWS, applyChangeLogPragmas, changeLogFileName, changeLogWipeReason, deleteChangeLogDB, estimateChangeLogStreamRowBytes, openChangeLogDB, openChangeLogDBForWriting, readChangeLogBounds, readChangeLogHead, readChangeLogMeta, rebuildChangeLogDBForWriting, reconcileChangeLog, seedChangeLogStream };
+export { CHANGE_LOG_META_TABLE, CHANGE_LOG_STREAM_TABLE, CHANGE_LOG_STREAM_WRITE_TIME_INDEX, CREATE_CHANGE_LOG_STREAM_SCHEMA, ChangeLogRebuildRequired, MAX_RECONCILE_TRUNCATE_BYTES, MAX_RECONCILE_TRUNCATE_ROWS, applyChangeLogPragmas, changeLogFileName, changeLogWipeReason, deleteChangeLogDB, estimateChangeLogStreamRowBytes, openChangeLogDB, openChangeLogDBForWriting, readChangeLogBounds, readChangeLogHead, readChangeLogMeta, rebuildChangeLogDBForWriting, reconcileChangeLog };
 
 //# sourceMappingURL=change-log-db.js.map

package/out/zero-cache/src/services/replicator/change-log-db.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-log-db.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/change-log-db.ts"],"sourcesContent":["/**\n * The change log lives in its own SQLite database, `${replicaFile}-change-log`,\n * rather than in the replica file. It is a catchup cache: everything it holds is\n * either already durable in the replica's litestream backup or re-derivable from\n * the upstream replication slot, so it is deliberately excluded from the backup\n * and is never a source of truth.\n *\n * The change-streamer writes it, from the stream loop that fans changes out to\n * subscribers, and commits it before forwarding each transaction's `commit`\n * message. Its head is therefore always at or above the watermark the stream\n * would resume from, which is what {@link reconcileChangeLog} anchors on.\n *\n * Beside the buffer it holds the log's *cookie jar* — the backfill progress\n * state folded up to the same head. That is `change-log-cookies.ts`; this file\n * owns its tables' lifecycle, because a cookie set is only meaningful paired\n * with the watermark it was folded to, and so must be created, dropped, and —\n * whenever reconciliation moves the head — replaced with the anchor's, all with\n * the buffer rather than beside it.\n *\n * Because it is disposable, it has no migration framework. Any inconsistency —\n * a schema change, a leftover file from a different replica, a gap left by a\n * crash or by running with the writer disabled, a corrupt file — is resolved by\n * {@link reconcileChangeLog} and {@link openChangeLogDBForWriting}, which\n * truncate a bounded suffix or replace and reseed the log at the resume\n * watermark. The worst outcome is a `too-old` for a lagging subscriber; a\n * silently delivered gap is not reachable.\n *\n * Disk sizing: a task that writes the log needs room for the replica *plus* the\n * retained change stream — the log's main file, its wal2 sidecars, and pages a\n * purge has freed but not yet returned to the OS. The change-streamer schedules\n * purges against the same floor as the PG change log, so retention is bounded\n * by the confirmed backup watermark ANDed with the minimum retention window —\n * a function of the backup interval, not a constant: 15–30+ minutes behind an\n * RMv1 sync interval, ~60 seconds once litestream v5 backs up every 15–30 s.\n * A held snapshot reservation pins retention unboundedly for its duration, so\n * peak retained bytes is set by the slowest concurrent restore rather than by\n * the retention setting — size disk and alerts on that. The log is excluded\n * from the litestream backup, so this is local disk only and never S3.\n */\n\nimport {existsSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {\n  CHANGE_LOG_BACKFILLING_TABLE,\n  CHANGE_LOG_TABLE_METADATA_TABLE,\n  CREATE_CHANGE_LOG_COOKIE_SCHEMA,\n  DROP_CHANGE_LOG_COOKIE_TABLES,\n  replaceCookies,\n  type CookieSet,\n} from './change-log-cookies.ts';\n\n/**\n * Bumped whenever the change-log database's schema changes. Since the file is a\n * cache, a mismatch costs one reseed rather than a migration.\n *\n * v2 re-homed the writer to the change-streamer: the log anchors on the\n * watermark its stream connection resumes from rather than on the replica's\n * state version, and its meta row carries the replica's identity triple plus\n * the seed point.\n *\n * v3 added the cookie tables (see `change-log-cookies.ts`), i.e. the log's\n * second job. The reseed it costs on upgrade buys one retention window of\n * catchup reach on a log nothing reads yet.\n *\n * v4 stores each row's tag and estimated retained bytes alongside its JSON.\n * Readers can now filter without parsing arbitrary row payloads, and\n * reconciliation can bound truncation by bytes as well as rows.\n *\n * `auto_vacuum = INCREMENTAL` arrived within v2, deliberately without a bump:\n * it is a file-header property rather than a schema change, and the reseed a\n * version mismatch triggers cannot enable it (see\n * {@link applyChangeLogPragmas}), so {@link openChangeLogDBForWriting} guards\n * on the pragma itself instead of on this number.\n */\nexport const CHANGE_LOG_DB_SCHEMA_VERSION = 4;\n\n/** https://www.sqlite.org/pragma.html#pragma_auto_vacuum */\nconst AUTO_VACUUM_INCREMENTAL = 2;\n\nexport const CHANGE_LOG_STREAM_TABLE = '_zero.changeLogStream';\nexport const CHANGE_LOG_STREAM_WRITE_TIME_INDEX =\n  '_zero.changeLogStream_writeTimeMs';\n\nconst INTEGER_BYTES = 8;\n\n/**\n * Estimates the retained payload size of a stream row. This deliberately does\n * not claim to measure SQLite b-tree/page overhead; it is stable across the\n * write path, reconciliation, and the startup scan used by observability.\n */\nexport function estimateChangeLogStreamRowBytes(\n  watermark: string,\n  tag: string,\n  change: string,\n  precommit?: string,\n  hasWriteTime = false,\n): number {\n  return (\n    Buffer.byteLength(watermark) +\n    INTEGER_BYTES +\n    Buffer.byteLength(tag) +\n    Buffer.byteLength(change) +\n    (precommit === undefined ? 0 : Buffer.byteLength(precommit)) +\n    (hasWriteTime ? INTEGER_BYTES : 0)\n  );\n}\n\n// The index is partial because only commit rows carry a \"writeTimeMs\", so it\n// holds one entry per transaction rather than one per change. It is what the\n// purger scans to find transactions older than the retention window.\n//\n// Pre-release builds briefly carried this table in the replica; see the test\n// fixture `CREATE_V14_CHANGE_LOG_STREAM` in\n// `change-source/common/replica-schema.ts`. Its old shape must not be kept in\n// sync with this one.\nexport const CREATE_CHANGE_LOG_STREAM_SCHEMA = /*sql*/ `\n  CREATE TABLE \"${CHANGE_LOG_STREAM_TABLE}\" (\n    \"watermark\"      TEXT NOT NULL,\n    \"pos\"            INTEGER NOT NULL,\n    \"tag\"            TEXT NOT NULL,\n    \"estimatedBytes\"  INTEGER NOT NULL,\n    \"change\"          TEXT NOT NULL,\n    \"precommit\"       TEXT,\n    \"writeTimeMs\"     INTEGER,\n    PRIMARY KEY (\"watermark\", \"pos\")\n  );\n\n  CREATE INDEX \"${CHANGE_LOG_STREAM_WRITE_TIME_INDEX}\"\n    ON \"${CHANGE_LOG_STREAM_TABLE}\" (\"writeTimeMs\", \"watermark\")\n    WHERE \"writeTimeMs\" IS NOT NULL;\n`;\n\n/**\n * Inserts the synthetic begin/commit pair for the seed watermark.\n *\n * Both rows are inserted by one statement so a caller outside a larger\n * transaction cannot leave a partial seed.\n */\nconst SEED_CHANGE_LOG_STREAM_SQL = /*sql*/ `\n  INSERT INTO \"${CHANGE_LOG_STREAM_TABLE}\"\n    (\"watermark\", \"pos\", \"tag\", \"estimatedBytes\", \"change\", \"precommit\",\n     \"writeTimeMs\")\n  VALUES\n    (@watermark, 0, 'begin', @beginBytes, @beginChange, NULL, NULL),\n    (@watermark, 1, 'commit', @commitBytes, @commitChange, @watermark,\n     @writeTimeMs)\n  ON CONFLICT (\"watermark\", \"pos\") DO NOTHING\n`;\n\nexport const CHANGE_LOG_META_TABLE = '_zero.changeLogMeta';\n\n// The identity triple (see ChangeLogIdentity) detects a file left behind by a\n// different replica -- most concretely, a leftover log on a reused volume.\n//\n// \"seededAtMs\"    : when the log was seeded, i.e. how far its retention window\n//                   can possibly reach back. Slice 11 uses it as the warm-up\n//                   signal for canary reads.\n// \"seedWatermark\" : where the log was seeded. `min(\"watermark\")` cannot answer\n//                   this once purge has advanced the minimum, and the\n//                   difference is what distinguishes a log that purged past the\n//                   retention floor (an invariant violation) from one that\n//                   never held that history at all (routine at every fork).\n// \"lock\"          : enforces single-row semantics, as in\n//                   \"_zero.replicationConfig\".\nconst CREATE_CHANGE_LOG_META_SCHEMA = /*sql*/ `\n  CREATE TABLE \"${CHANGE_LOG_META_TABLE}\" (\n    \"epoch\"         TEXT,\n    \"generation\"    TEXT NOT NULL,\n    \"replicaID\"     TEXT,\n    \"schemaVersion\" INTEGER NOT NULL,\n    \"seededAtMs\"    INTEGER NOT NULL,\n    \"seedWatermark\" TEXT NOT NULL,\n    \"lock\"          INTEGER PRIMARY KEY DEFAULT 1 CHECK (\"lock\" = 1)\n  );\n`;\n\n/**\n * `${replicaFile}-change-log`, matching the `-serving-copy` convention.\n *\n * The name outlives the writer's move into the change-streamer: the\n * change-streamer derives the path from the replica file it owns, so renaming\n * would save no plumbing and would orphan the old file on every existing\n * volume, where nothing would delete it.\n *\n * Its `-wal` / `-wal2` / `-shm` sidecars are suffixes of *this* name, so they\n * never collide with the replica's own sidecars.\n */\nexport function changeLogFileName(replicaFile: string): string {\n  return `${replicaFile}-change-log`;\n}\n\n/**\n * Removes the change-log database and its `-wal` / `-wal2` / `-shm` sidecars.\n * Safe to call when the file does not exist.\n */\nexport function deleteChangeLogDB(replicaFile: string): void {\n  deleteLiteDB(changeLogFileName(replicaFile));\n}\n\n/**\n * Which replica the log's contents belong to.\n *\n * `generation` is RMv1's `replicaVersion`: the version at which initial sync\n * copied every row. It is shared by every sibling of a forked replica, which is\n * why it cannot be the whole of the identity under RMv2 — `epoch` and\n * `replicaID` are what distinguish two siblings' logs. Both are `null` until\n * RMv2 supplies them, and a `null` matches only another `null`, so a log\n * written before they exist is still recognized as this replica's.\n */\nexport type ChangeLogIdentity = {\n  readonly epoch: string | null;\n  readonly generation: string;\n  readonly replicaID: string | null;\n};\n\n/**\n * Where the stream connection resumes, and everything the log's state at that\n * position consists of.\n *\n * The two fields travel together because a cookie set is only meaningful paired\n * with the watermark it was folded up to (invariant 15). They are read from one\n * store, in one snapshot: `Storer.getStartStreamInitializationParameters()`\n * reads both from Postgres in a single REPEATABLE READ transaction. Pairing one\n * store's cookies with another's watermark would silently drop every backfill\n * that completed in the interval, and those rows are unrecoverable — backfill\n * transactions are synthesized by the change source rather than replayed from\n * the slot.\n */\nexport type ChangeLogResumePoint = {\n  /**\n   * The watermark this stream connection resumes from. The log's own head is\n   * measured against it, and a reseed seeds a synthetic transaction at it.\n   *\n   * Deliberately *not* \"the replica's state version\": after the writer moved to\n   * the change-streamer, the canonical replicator is a subscriber downstream of\n   * the write loop, so its state version is a consequence of this log rather\n   * than a constraint on it.\n   */\n  readonly resumeWatermark: string;\n  /**\n   * The cookie set that belongs to {@link resumeWatermark}. Any reconciliation\n   * that moves the head invalidates the set the log was holding — the cookies\n   * are unversioned point-in-time state, and truncating the stream above `W`\n   * does not roll them back to `W` — so the same transaction installs this one\n   * in its place (invariant 17).\n   *\n   * Postgres supplies it while it remains the store the stream resumes from. At\n   * the C5 flip the log's own head *is* the resume point, phantoms cannot\n   * exist, and both this field and the truncate-above path become dead code.\n   */\n  readonly cookies: CookieSet;\n};\n\n/**\n * What the log is reconciled against. Reconciliation runs once per stream\n * connection, since every reconnect can re-read a different resume point.\n */\nexport type ChangeLogAnchor = ChangeLogResumePoint & {\n  readonly identity: ChangeLogIdentity;\n  /**\n   * The change-streamer's clock, at reconciliation time. Supplied rather than\n   * read inline so that tests control it. Used for `seededAtMs` and for the\n   * seed transaction's `writeTimeMs`, which is what the purger's time floor\n   * measures.\n   */\n  readonly nowMs: number;\n};\n\n/** The meta row, i.e. everything the log records about itself. */\nexport type ChangeLogMeta = ChangeLogIdentity & {\n  readonly schemaVersion: number;\n  readonly seededAtMs: number;\n  readonly seedWatermark: string;\n};\n\n/**\n * Opens the change-log database beside `replicaFile`.\n *\n * A writable handle creates the file if it is absent; a `readonly` handle\n * throws, which is the normal state for a change-streamer whose writer is\n * disabled. Pragmas are the caller's responsibility, since only the writer sets\n * the persistent ones.\n */\nexport function openChangeLogDB(\n  lc: LogContext,\n  replicaFile: string,\n  opts: {readonly: boolean},\n): Database {\n  return new Database(\n    lc.withContext('component', 'change-log-db'),\n    changeLogFileName(replicaFile),\n    {readonly: opts.readonly},\n  );\n}\n\n/**\n * Configures the change-log database, which the change-streamer's writer owns\n * exclusively. These are the settings the `separate-files` mode of\n * `sqlite-change-log-ceiling.bench.ts` measured.\n *\n * `wal2` because the log is a continuous append that catchup reads\n * continuously, which is the case wal2 exists for: it avoids checkpoint\n * starvation without needing an autocheckpoint. `wal_autocheckpoint` therefore\n * keeps its default, unlike the backup replica's 0, which hands checkpointing\n * to litestream — nothing else owns this file.\n *\n * `synchronous = NORMAL` because the commit sits on the forward path, once per\n * upstream transaction, and NORMAL keeps that from costing an fsync. It is not\n * a durability compromise here: in WAL mode NORMAL still writes every commit\n * through to the OS, so it survives a process crash, an OOM kill, or a SIGKILL.\n * It is lost only to kernel panic or power loss, and those destroy the node —\n * the task restarts elsewhere, restores the replica from S3, and has no\n * change-log file at all. Whatever a power loss does take is detected by\n * {@link reconcileChangeLog} and reseeded.\n *\n * `auto_vacuum = INCREMENTAL` so that purge can return freed pages to the OS\n * rather than leaving the file at its high-water mark. **It must be the first\n * statement here.** SQLite honours a `none` -> `incremental` transition only\n * while the file is still empty, and `journal_mode = wal2` materializes page 1;\n * appending the pragma instead of prepending it silently leaves the file at\n * mode 0 forever, since neither `DROP TABLE` nor a reseed makes a file empty\n * again. {@link openChangeLogDBForWriting} verifies the result.\n */\nexport function applyChangeLogPragmas(db: Database): void {\n  db.pragma('auto_vacuum = INCREMENTAL');\n  db.pragma('busy_timeout = 30000');\n  db.pragma('analysis_limit = 1000');\n  db.pragma('journal_mode = wal2');\n  db.pragma('synchronous = NORMAL');\n}\n\nfunction readAutoVacuum(db: Database): number {\n  const [{auto_vacuum: mode}] = db.pragma<{auto_vacuum: number}>('auto_vacuum');\n  return mode;\n}\n\n/**\n * Opens the change-log database for the writer, applies its pragmas, and\n * reconciles it against `anchor`, returning the handle and what reconciliation\n * had to do.\n *\n * Corruption is answered with a rebuild rather than an escalation, which is a\n * deliberate divergence from how the replica's own corruption is handled. A\n * corrupt replica is a lost source of truth, so it ends in a restore or an\n * auto-reset; the change log holds nothing that is not either already in the\n * replica's litestream backup or re-derivable from the upstream slot, so the\n * entire response is to delete the file and seed a new one at the resume\n * watermark. The cost is one retention window of catchup — reconnecting\n * subscribers below the head get `too-old` — which is what every other\n * {@link ReseedReason} costs.\n *\n * Only corruption is rebuilt through. Anything else (a bad path, a permissions\n * error, a full disk) is a problem with the environment rather than with the\n * file's contents, and deleting a database in response would destroy state\n * without fixing it. Those propagate to the caller, which disables the writer\n * and keeps replicating.\n *\n * A file that predates `auto_vacuum = INCREMENTAL` takes the same rebuild, for\n * the same reason and at the same cost: the pragma only takes on an empty file,\n * so a reseed cannot enable it. Unlike corruption, a *second* failure there is\n * tolerated rather than thrown — see {@link ensureIncrementalAutoVacuum}.\n */\n/**\n * Returns an anchor after the database opens.\n *\n * The resolver can inspect the existing file before it selects the anchor. A\n * valid log can use its own head. A new or invalid log uses the replica seed.\n * The rebuild path calls the resolver again for the new file.\n */\nexport type AnchorResolver = (db: Database) => ChangeLogAnchor;\n\nexport function openChangeLogDBForWriting(\n  lc: LogContext,\n  replicaFile: string,\n  anchor: AnchorResolver,\n): {db: Database; result: ReconcileResult; replacedFile: boolean} {\n  const existed = existsSync(changeLogFileName(replicaFile));\n  const opened = openOrRebuildCorrupt(lc, replicaFile, anchor, existed);\n  const {db, result, replacedFile} = ensureIncrementalAutoVacuum(\n    lc,\n    replicaFile,\n    anchor,\n    opened,\n  );\n  return {db, result, replacedFile};\n}\n\nfunction openOrRebuildCorrupt(\n  lc: LogContext,\n  replicaFile: string,\n  anchor: AnchorResolver,\n  boundReconciliation: boolean,\n): OpenedChangeLog {\n  try {\n    return openAndReconcile(lc, replicaFile, anchor, boundReconciliation);\n  } catch (e) {\n    if (e instanceof ChangeLogRebuildRequired) {\n      return rebuildChangeLog(lc, replicaFile, anchor, e);\n    }\n    if (!isSQLiteCorruption(e)) {\n      throw e;\n    }\n    const file = changeLogFileName(replicaFile);\n    logSQLiteCorruptionDiagnostics(lc, 'change-log', file, e);\n    lc.error?.('rebuilding the corrupt SQLite change log', {\n      sqliteChangeLog: {file},\n    });\n    return rebuildChangeLog(lc, replicaFile, anchor);\n  }\n}\n\n/**\n * Rebuilds a log whose `auto_vacuum` mode is not `INCREMENTAL`, which is every\n * log created before the pragma shipped — including every v2 file the writer\n * wrote in the meantime, which is why the guard is on the pragma rather than\n * on the schema version. Deleting the file is the only way to enable it:\n * `DROP TABLE` leaves the header's mode intact, and the in-place alternative —\n * a full `VACUUM` — is an unbounded blocking statement on a file the writer\n * needs.\n *\n * A mode that is still wrong after the rebuild is logged and tolerated, which\n * is the one place this diverges from the corruption path. The two failures are\n * not alike: a file that stays corrupt after a rebuild is an environment\n * problem, but a pragma that stays wrong on a brand-new file can only mean\n * {@link applyChangeLogPragmas}' statement order is wrong — a code bug, present\n * on every task, on every start. Throwing there turns a lost optimization into\n * a crash loop, and rebuilding again turns it into a reseed per start, which is\n * worse than the freelist growth it was trying to avoid. `Database.compact()`\n * already warns and no-ops when the mode is not `INCREMENTAL`, so degrading is\n * the accepted fallback.\n */\nfunction ensureIncrementalAutoVacuum(\n  lc: LogContext,\n  replicaFile: string,\n  anchor: AnchorResolver,\n  opened: OpenedChangeLog,\n): OpenedChangeLog {\n  if (opened.autoVacuum === AUTO_VACUUM_INCREMENTAL) {\n    return opened;\n  }\n  const file = changeLogFileName(replicaFile);\n  lc.info?.('rebuilding the SQLite change log to enable incremental vacuum', {\n    sqliteChangeLog: {file, autoVacuum: opened.autoVacuum},\n  });\n  opened.db.close();\n  const rebuilt = rebuildChangeLog(lc, replicaFile, anchor);\n  if (rebuilt.autoVacuum !== AUTO_VACUUM_INCREMENTAL) {\n    lc.error?.(\n      'SQLite change log does not support incremental vacuum; ' +\n        'it will plateau at its high-water mark',\n      {sqliteChangeLog: {file, autoVacuum: rebuilt.autoVacuum}},\n    );\n  }\n  return rebuilt;\n}\n\n/** The shared \"delete the file and open a new one\" body. */\nfunction rebuildChangeLog(\n  lc: LogContext,\n  replicaFile: string,\n  anchor: AnchorResolver,\n  required?: ChangeLogRebuildRequired | undefined,\n): OpenedChangeLog {\n  if (required) {\n    lc.warn?.(\n      'rebuilding the SQLite change log instead of running an unbounded reconciliation',\n      {\n        sqliteChangeLogRebuild: {\n          reason: required.reason,\n          rows: required.rows,\n          estimatedBytes: required.estimatedBytes,\n        },\n      },\n    );\n  }\n  deleteChangeLogDB(replicaFile);\n  // The file is absent, so the reseed only drops empty/nonexistent tables. A\n  // second failure is the environment's, not the old file's.\n  const opened = openAndReconcile(\n    lc,\n    replicaFile,\n    anchor,\n    false,\n    required?.reason,\n  );\n  return {...opened, replacedFile: true};\n}\n\n/**\n * Replaces an already-closed change-log file after bounded reconciliation\n * declined its work. The caller closes cached readers before calling this so\n * none can continue serving the unlinked inode.\n */\nexport function rebuildChangeLogDBForWriting(\n  lc: LogContext,\n  replicaFile: string,\n  anchor: ChangeLogAnchor,\n  required: ChangeLogRebuildRequired,\n): {db: Database; result: ReconcileResult; replacedFile: true} {\n  const {db, result} = rebuildChangeLog(\n    lc,\n    replicaFile,\n    () => anchor,\n    required,\n  );\n  return {db, result, replacedFile: true};\n}\n\ntype OpenedChangeLog = {\n  db: Database;\n  result: ReconcileResult;\n  replacedFile: boolean;\n  /** The file's persistent `auto_vacuum` mode, read back after the pragmas. */\n  autoVacuum: number;\n};\n\nfunction openAndReconcile(\n  lc: LogContext,\n  replicaFile: string,\n  anchor: AnchorResolver,\n  boundReconciliation: boolean,\n  reseedReason?: ReseedReason | undefined,\n): OpenedChangeLog {\n  const db = openChangeLogDB(lc, replicaFile, {readonly: false});\n  try {\n    applyChangeLogPragmas(db);\n    const autoVacuum = readAutoVacuum(db);\n    return {\n      db,\n      result: reconcileChangeLog(lc, db, anchor(db), {\n        rebuildInsteadOfUnboundedWork: boundReconciliation,\n        reseedReason,\n      }),\n      replacedFile: false,\n      autoVacuum,\n    };\n  } catch (e) {\n    // The caller never sees this handle, so it closes here or leaks — and it\n    // must be closed before the corruption path deletes the file.\n    db.close();\n    throw e;\n  }\n}\n\nexport type ReseedReason =\n  | 'created' // file or table absent\n  | 'schema-mismatch'\n  | 'identity-mismatch'\n  | 'gap' // head does not land on the resume watermark after truncation\n  | 'oversized-truncate'; // preserving the prefix would require too much work\n\n/**\n * Reconciliation runs synchronously on the change-streamer's event loop. A\n * normal phantom is one transaction, so work above either limit is answered by\n * unlinking and reseeding this disposable cache rather than deleting an\n * arbitrarily large suffix in place.\n */\nexport const MAX_RECONCILE_TRUNCATE_ROWS = 10_000;\nexport const MAX_RECONCILE_TRUNCATE_BYTES = 16 * 1024 * 1024;\n\nexport class ChangeLogRebuildRequired extends Error {\n  override readonly name = 'ChangeLogRebuildRequired';\n  readonly reason: ReseedReason;\n  readonly rows?: number | undefined;\n  readonly estimatedBytes?: number | undefined;\n\n  constructor(\n    reason: ReseedReason,\n    rows?: number | undefined,\n    estimatedBytes?: number | undefined,\n  ) {\n    super(`SQLite change log requires a ${reason} rebuild`);\n    this.reason = reason;\n    this.rows = rows;\n    this.estimatedBytes = estimatedBytes;\n  }\n}\n\ntype ReconcileOptions = {\n  /** Throw before an unbounded delete/drop so the file owner can unlink it. */\n  readonly rebuildInsteadOfUnboundedWork?: boolean | undefined;\n  /** Preserve the reason that caused an old file to be unlinked. */\n  readonly reseedReason?: ReseedReason | undefined;\n};\n\n/**\n * `cookiesStale` records that the cookie set the log held did not survive\n * reconciliation. The cookies are unversioned point-in-time state, so any\n * reconciliation that moves the head invalidates them, and the same transaction\n * replaces them with the anchor's — the set that belongs to the resume\n * watermark (see `change-log-cookies.ts`). It is `action !== 'none'` today; it\n * is a field rather than a derivation because the replacement reads as intent\n * at the call site, and because it is what the `cookie_reseed` counter is keyed\n * on.\n */\nexport type ReconcileResult =\n  | {action: 'none'; head: string; cookiesStale: false}\n  | {action: 'truncated'; head: string; rows: number; cookiesStale: true}\n  | {\n      action: 'reseeded';\n      head: string;\n      reason: ReseedReason;\n      cookiesStale: true;\n    };\n\n/**\n * Brings the change log into agreement with the watermark its stream connection\n * resumes from, in one transaction on the change-log database:\n *\n * ```\n * head === resumeWatermark  → keep appending\n * head  >  resumeWatermark  → truncate above resumeWatermark\n * otherwise                 → wipe, seed at resumeWatermark, record the reason\n * ```\n *\n * The log commits before anything that can advance the resume watermark, so a\n * *phantom* — a transaction the log holds whose watermark the resumed stream\n * re-delivers — is the normal state, and truncate-above is the whole of\n * reconciliation rather than a special case. A phantom's rows all carry the\n * transaction's commit watermark, which is strictly greater than the resume\n * watermark, so deleting on that predicate removes phantoms whole and retains\n * every transaction at or below the resume point.\n *\n * It must run per stream connection, not once per process: the stream loop\n * re-reads its resume watermark on every reconnect, and the writer inserts with\n * a plain `INSERT` against `PRIMARY KEY (\"watermark\",\"pos\")`, so an\n * un-truncated overlap is a constraint violation on the first re-delivered\n * transaction.\n *\n * Anything truncation cannot resolve — the file was deleted, the writer was\n * disabled for a while, the replica was restored, power was lost with\n * `synchronous=NORMAL` — surfaces as a head that does not land on the resume\n * watermark, and is resolved by wiping and reseeding there.\n *\n * Whichever of the two moves the head also replaces the cookie set with the one\n * the anchor carries, in the same transaction: the head and the cookies are one\n * position, and nothing may observe them at two.\n */\nexport function reconcileChangeLog(\n  lc: LogContext,\n  db: Database,\n  anchor: ChangeLogAnchor,\n  opts: ReconcileOptions = {},\n): ReconcileResult {\n  db.exec('BEGIN IMMEDIATE');\n  try {\n    if (opts.rebuildInsteadOfUnboundedWork) {\n      const required = rebuildRequired(db, anchor);\n      if (required) {\n        throw required;\n      }\n    }\n    const result = opts.reseedReason\n      ? reseed(lc, db, anchor, opts.reseedReason)\n      : reconcile(lc, db, anchor);\n    db.exec('COMMIT');\n    return result;\n  } catch (e) {\n    if (db.inTransaction) {\n      db.exec('ROLLBACK');\n    }\n    throw e;\n  }\n}\n\n/**\n * Decides whether reconciliation can be performed within its synchronous work\n * budget. Every query is either an index seek or a scan capped at one row past\n * the delete limit. No state changes before the decision is complete.\n */\nfunction rebuildRequired(\n  db: Database,\n  anchor: ChangeLogAnchor,\n): ChangeLogRebuildRequired | undefined {\n  const wipe = changeLogWipeReason(db, anchor.identity);\n  if (wipe !== undefined) {\n    return new ChangeLogRebuildRequired(wipe);\n  }\n\n  const head = readChangeLogHead(db);\n  const {resumeWatermark} = anchor;\n  if (head === null || head < resumeWatermark) {\n    return new ChangeLogRebuildRequired('gap');\n  }\n  if (head === resumeWatermark) {\n    return undefined;\n  }\n\n  const boundary = db\n    .prepare(/*sql*/ `\n      SELECT 1 FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n        WHERE \"watermark\" = ?\n        LIMIT 1\n    `)\n    .get<{1: number} | undefined>(resumeWatermark);\n  if (boundary === undefined) {\n    return new ChangeLogRebuildRequired('gap');\n  }\n\n  const {rows, estimatedBytes} = db\n    .prepare(/*sql*/ `\n      SELECT count(*) AS \"rows\",\n             coalesce(sum(\"estimatedBytes\"), 0) AS \"estimatedBytes\"\n        FROM (\n          SELECT \"estimatedBytes\" FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n            WHERE \"watermark\" > ?\n            LIMIT ${MAX_RECONCILE_TRUNCATE_ROWS + 1}\n        )\n    `)\n    .get<{rows: number; estimatedBytes: number}>(resumeWatermark);\n  return rows > MAX_RECONCILE_TRUNCATE_ROWS ||\n    estimatedBytes > MAX_RECONCILE_TRUNCATE_BYTES\n    ? new ChangeLogRebuildRequired('oversized-truncate', rows, estimatedBytes)\n    : undefined;\n}\n\nfunction reconcile(\n  lc: LogContext,\n  db: Database,\n  anchor: ChangeLogAnchor,\n): ReconcileResult {\n  const wipe = changeLogWipeReason(db, anchor.identity);\n  if (wipe !== undefined) {\n    return reseed(lc, db, anchor, wipe);\n  }\n\n  const {resumeWatermark} = anchor;\n  let head = readChangeLogHead(db);\n  let rows = 0;\n  if (head !== null && head > resumeWatermark) {\n    rows = db\n      .prepare(/*sql*/ `\n        DELETE FROM \"${CHANGE_LOG_STREAM_TABLE}\" WHERE \"watermark\" > ?\n      `)\n      .run(resumeWatermark).changes;\n    head = readChangeLogHead(db);\n  }\n\n  if (head !== resumeWatermark) {\n    return reseed(lc, db, anchor, 'gap');\n  }\n  if (rows > 0) {\n    // The truncated transactions can have carried `create-table`,\n    // `add-column`, and `backfill-completed`, none of which the truncate rolls\n    // back, so the cookie set no longer belongs to the head. The anchor's set\n    // does, and it lands in this same transaction: invariant 17 holds at every\n    // point another connection could observe the log.\n    replaceCookies(db, anchor.cookies);\n    lc.info?.('truncated phantom transactions from the SQLite change log', {\n      sqliteChangeLogReconcile: {head, rows, cookies: cookieCounts(anchor)},\n    });\n    return {action: 'truncated', head, rows, cookiesStale: true};\n  }\n  lc.debug?.('SQLite change log is at the resume watermark', {\n    sqliteChangeLogReconcile: {head},\n  });\n  return {action: 'none', head, cookiesStale: false};\n}\n\n/**\n * Returns the reason that the log must be recreated. Returns `undefined` when\n * the writer can continue to use the log.\n *\n * This function takes only the expected identity because callers use the result\n * to select the anchor.\n */\nexport function changeLogWipeReason(\n  db: Database,\n  identity: ChangeLogIdentity,\n): ReseedReason | undefined {\n  if (\n    !tableExists(db, CHANGE_LOG_STREAM_TABLE) ||\n    !tableExists(db, CHANGE_LOG_META_TABLE)\n  ) {\n    return 'created';\n  }\n  // The schema version is read on its own because it is the one column every\n  // version of this table has had. Reading the rest of a v1 row would fail on\n  // its missing columns instead of reporting the mismatch that explains it.\n  const version = db\n    .prepare(/*sql*/ `\n      SELECT \"schemaVersion\" FROM \"${CHANGE_LOG_META_TABLE}\"\n    `)\n    .get<{schemaVersion: number} | undefined>();\n  if (version === undefined) {\n    return 'created';\n  }\n  if (version.schemaVersion !== CHANGE_LOG_DB_SCHEMA_VERSION) {\n    return 'schema-mismatch';\n  }\n  // Checked *after* the version, so that every pre-v4 file — which is every\n  // file in the fleet at the upgrade — reports the `schema-mismatch` that\n  // explains it rather than the `created` its absent cookie tables would\n  // otherwise look like.\n  if (\n    !tableExists(db, CHANGE_LOG_TABLE_METADATA_TABLE) ||\n    !tableExists(db, CHANGE_LOG_BACKFILLING_TABLE)\n  ) {\n    return 'created';\n  }\n  const {epoch, generation, replicaID} = readChangeLogMeta(db);\n  if (\n    epoch !== identity.epoch ||\n    generation !== identity.generation ||\n    replicaID !== identity.replicaID\n  ) {\n    return 'identity-mismatch';\n  }\n  return undefined;\n}\n\nfunction reseed(\n  lc: LogContext,\n  db: Database,\n  anchor: ChangeLogAnchor,\n  reason: ReseedReason,\n): ReconcileResult {\n  const {identity, resumeWatermark, nowMs} = anchor;\n  // Dropping the table drops its partial index with it. The cookie tables go\n  // with the buffer: a cookie set without the transactions it was folded from\n  // is not a smaller cookie set, it is a wrong one.\n  db.exec(/*sql*/ `\n    DROP TABLE IF EXISTS \"${CHANGE_LOG_STREAM_TABLE}\";\n    DROP TABLE IF EXISTS \"${CHANGE_LOG_META_TABLE}\";\n    ${DROP_CHANGE_LOG_COOKIE_TABLES}\n    ${CREATE_CHANGE_LOG_STREAM_SCHEMA}\n    ${CREATE_CHANGE_LOG_META_SCHEMA}\n    ${CREATE_CHANGE_LOG_COOKIE_SCHEMA}\n  `);\n  db.prepare(/*sql*/ `\n    INSERT INTO \"${CHANGE_LOG_META_TABLE}\"\n      (\"epoch\", \"generation\", \"replicaID\", \"schemaVersion\", \"seededAtMs\",\n       \"seedWatermark\")\n      VALUES (@epoch, @generation, @replicaID, @schemaVersion, @seededAtMs,\n              @seedWatermark)\n  `).run({\n    epoch: identity.epoch,\n    generation: identity.generation,\n    replicaID: identity.replicaID,\n    schemaVersion: CHANGE_LOG_DB_SCHEMA_VERSION,\n    seededAtMs: nowMs,\n    seedWatermark: resumeWatermark,\n  });\n  seedChangeLogStream(db, anchor);\n  // Into tables this statement just created, so the wholesale replacement is\n  // all insert. It is still the same call the truncate path makes: a reseed is\n  // the case where the log's cookie set was not merely moved but discarded, and\n  // the anchor's is equally what belongs at the head either way.\n  replaceCookies(db, anchor.cookies);\n\n  lc.info?.('reseeded the SQLite change log', {\n    sqliteChangeLogReconcile: {\n      reason,\n      head: resumeWatermark,\n      ...identity,\n      schemaVersion: CHANGE_LOG_DB_SCHEMA_VERSION,\n      seededAtMs: nowMs,\n      cookies: cookieCounts(anchor),\n    },\n  });\n  return {\n    action: 'reseeded',\n    head: resumeWatermark,\n    reason,\n    cookiesStale: true,\n  };\n}\n\n/**\n * Seeds a valid synthetic transaction at the anchor's resume watermark, making\n * an otherwise empty log serviceable as a catchup boundary for a subscriber\n * that is exactly at that watermark.\n */\nexport function seedChangeLogStream(\n  db: Database,\n  anchor: ChangeLogAnchor,\n): void {\n  const beginChange = '{\"tag\":\"begin\"}';\n  const commitChange = '{\"tag\":\"commit\"}';\n  db.prepare(SEED_CHANGE_LOG_STREAM_SQL).run({\n    watermark: anchor.resumeWatermark,\n    beginBytes: estimateChangeLogStreamRowBytes(\n      anchor.resumeWatermark,\n      'begin',\n      beginChange,\n    ),\n    beginChange,\n    commitBytes: estimateChangeLogStreamRowBytes(\n      anchor.resumeWatermark,\n      'commit',\n      commitChange,\n      anchor.resumeWatermark,\n      true,\n    ),\n    commitChange,\n    writeTimeMs: anchor.nowMs,\n  });\n}\n\n/** Reads the whole meta row. Throws if the log has not been reconciled. */\nexport function readChangeLogMeta(db: Database): ChangeLogMeta {\n  const meta = db\n    .prepare(/*sql*/ `\n      SELECT \"epoch\", \"generation\", \"replicaID\", \"schemaVersion\",\n             \"seededAtMs\", \"seedWatermark\"\n        FROM \"${CHANGE_LOG_META_TABLE}\"\n    `)\n    .get<ChangeLogMeta | undefined>();\n  if (meta === undefined) {\n    throw new Error('the SQLite change log has no meta row');\n  }\n  return meta;\n}\n\n/**\n * `max()` on the primary key's leading column, i.e. an index seek. Null when\n * the log is empty.\n */\nexport function readChangeLogHead(db: Database): string | null {\n  const {head} = db\n    .prepare(/*sql*/ `\n      SELECT max(\"watermark\") AS \"head\" FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n    `)\n    .get<{head: string | null}>();\n  return head;\n}\n\n/**\n * The covered watermark range. Both scalar subqueries seek the watermark\n * index, so unlike a `count()` this does not scan the log. Both bounds are\n * null when it is empty.\n */\nexport function readChangeLogBounds(db: Database): ChangeLogBounds {\n  return db\n    .prepare(/*sql*/ `\n      SELECT\n        (SELECT min(\"watermark\") FROM \"${CHANGE_LOG_STREAM_TABLE}\")\n          AS \"minWatermark\",\n        (SELECT max(\"watermark\") FROM \"${CHANGE_LOG_STREAM_TABLE}\")\n          AS \"headWatermark\"\n    `)\n    .get<ChangeLogBounds>();\n}\n\nexport type ChangeLogBounds = {\n  readonly minWatermark: string | null;\n  readonly headWatermark: string | null;\n};\n\n/**\n * What was installed, for the reconcile log line. Normally `{0, 0}`: a nonzero\n * `backfilling` means the resumed stream is being handed backfills to re-request.\n */\nfunction cookieCounts({cookies}: ChangeLogAnchor) {\n  return {\n    tableMetadata: cookies.tableMetadata.length,\n    backfilling: cookies.backfilling.length,\n  };\n}\n\nfunction tableExists(db: Database, table: string): boolean {\n  return (\n    db\n      .prepare(/*sql*/ `\n        SELECT 1 FROM \"sqlite_master\" WHERE \"type\" = 'table' AND \"name\" = ?\n      `)\n      .get<{1: number} | undefined>(table) !== undefined\n  );\n}\n"],"mappings":";;;;;;AAmFA,IAAM,0BAA0B;AAEhC,IAAa,0BAA0B;AACvC,IAAa,qCACX;AAEF,IAAM,gBAAgB;;;;;;AAOtB,SAAgB,gCACd,WACA,KACA,QACA,WACA,eAAe,OACP;CACR,OACE,OAAO,WAAW,SAAS,IAC3B,gBACA,OAAO,WAAW,GAAG,IACrB,OAAO,WAAW,MAAM,KACvB,cAAc,KAAA,IAAY,IAAI,OAAO,WAAW,SAAS,MACzD,eAAe,gBAAgB;AAEpC;AAUA,IAAa,kCAA0C;kBACrC,wBAAwB;;;;;;;;;;;kBAWxB,mCAAmC;UAC3C,wBAAwB;;;;;;;;;AAUlC,IAAM,6BAAqC;iBAC1B,wBAAwB;;;;;;;;;AAUzC,IAAa,wBAAwB;AAerC,IAAM,gCAAwC;kBAC5B,sBAAsB;;;;;;;;;;;;;;;;;;;;;AAsBxC,SAAgB,kBAAkB,aAA6B;CAC7D,OAAO,GAAG,YAAY;AACxB;;;;;AAMA,SAAgB,kBAAkB,aAA2B;CAC3D,aAAa,kBAAkB,WAAW,CAAC;AAC7C;;;;;;;;;AAsFA,SAAgB,gBACd,IACA,aACA,MACU;CACV,OAAO,IAAI,SACT,GAAG,YAAY,aAAa,eAAe,GAC3C,kBAAkB,WAAW,GAC7B,EAAC,UAAU,KAAK,SAAQ,CAC1B;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,SAAgB,sBAAsB,IAAoB;CACxD,GAAG,OAAO,2BAA2B;CACrC,GAAG,OAAO,sBAAsB;CAChC,GAAG,OAAO,uBAAuB;CACjC,GAAG,OAAO,qBAAqB;CAC/B,GAAG,OAAO,sBAAsB;AAClC;AAEA,SAAS,eAAe,IAAsB;CAC5C,MAAM,CAAC,EAAC,aAAa,UAAS,GAAG,OAA8B,aAAa;CAC5E,OAAO;AACT;AAqCA,SAAgB,0BACd,IACA,aACA,QACgE;CAGhE,MAAM,EAAC,IAAI,QAAQ,iBAAgB,4BACjC,IACA,aACA,QAJa,qBAAqB,IAAI,aAAa,QADrC,WAAW,kBAAkB,WAAW,CACK,CAK3D,CACF;CACA,OAAO;EAAC;EAAI;EAAQ;CAAY;AAClC;AAEA,SAAS,qBACP,IACA,aACA,QACA,qBACiB;CACjB,IAAI;EACF,OAAO,iBAAiB,IAAI,aAAa,QAAQ,mBAAmB;CACtE,SAAS,GAAG;EACV,IAAI,aAAa,0BACf,OAAO,iBAAiB,IAAI,aAAa,QAAQ,CAAC;EAEpD,IAAI,CAAC,mBAAmB,CAAC,GACvB,MAAM;EAER,MAAM,OAAO,kBAAkB,WAAW;EAC1C,+BAA+B,IAAI,cAAc,MAAM,CAAC;EACxD,GAAG,QAAQ,4CAA4C,EACrD,iBAAiB,EAAC,KAAI,EACxB,CAAC;EACD,OAAO,iBAAiB,IAAI,aAAa,MAAM;CACjD;AACF;;;;;;;;;;;;;;;;;;;;;AAsBA,SAAS,4BACP,IACA,aACA,QACA,QACiB;CACjB,IAAI,OAAO,eAAe,yBACxB,OAAO;CAET,MAAM,OAAO,kBAAkB,WAAW;CAC1C,GAAG,OAAO,iEAAiE,EACzE,iBAAiB;EAAC;EAAM,YAAY,OAAO;CAAU,EACvD,CAAC;CACD,OAAO,GAAG,MAAM;CAChB,MAAM,UAAU,iBAAiB,IAAI,aAAa,MAAM;CACxD,IAAI,QAAQ,eAAe,yBACzB,GAAG,QACD,iGAEA,EAAC,iBAAiB;EAAC;EAAM,YAAY,QAAQ;CAAU,EAAC,CAC1D;CAEF,OAAO;AACT;;AAGA,SAAS,iBACP,IACA,aACA,QACA,UACiB;CACjB,IAAI,UACF,GAAG,OACD,mFACA,EACE,wBAAwB;EACtB,QAAQ,SAAS;EACjB,MAAM,SAAS;EACf,gBAAgB,SAAS;CAC3B,EACF,CACF;CAEF,kBAAkB,WAAW;CAU7B,OAAO;EAAC,GAPO,iBACb,IACA,aACA,QACA,OACA,UAAU,MAED;EAAQ,cAAc;CAAI;AACvC;;;;;;AAOA,SAAgB,6BACd,IACA,aACA,QACA,UAC6D;CAC7D,MAAM,EAAC,IAAI,WAAU,iBACnB,IACA,mBACM,QACN,QACF;CACA,OAAO;EAAC;EAAI;EAAQ,cAAc;CAAI;AACxC;AAUA,SAAS,iBACP,IACA,aACA,QACA,qBACA,cACiB;CACjB,MAAM,KAAK,gBAAgB,IAAI,aAAa,EAAC,UAAU,MAAK,CAAC;CAC7D,IAAI;EACF,sBAAsB,EAAE;EACxB,MAAM,aAAa,eAAe,EAAE;EACpC,OAAO;GACL;GACA,QAAQ,mBAAmB,IAAI,IAAI,OAAO,EAAE,GAAG;IAC7C,+BAA+B;IAC/B;GACF,CAAC;GACD,cAAc;GACd;EACF;CACF,SAAS,GAAG;EAGV,GAAG,MAAM;EACT,MAAM;CACR;AACF;;;;;;;AAeA,IAAa,8BAA8B;AAC3C,IAAa,+BAA+B;AAE5C,IAAa,2BAAb,cAA8C,MAAM;CAClD,OAAyB;CACzB;CACA;CACA;CAEA,YACE,QACA,MACA,gBACA;EACA,MAAM,gCAAgC,OAAO,SAAS;EACtD,KAAK,SAAS;EACd,KAAK,OAAO;EACZ,KAAK,iBAAiB;CACxB;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8DA,SAAgB,mBACd,IACA,IACA,QACA,OAAyB,CAAC,GACT;CACjB,GAAG,KAAK,iBAAiB;CACzB,IAAI;EACF,IAAI,KAAK,+BAA+B;GACtC,MAAM,WAAW,gBAAgB,IAAI,MAAM;GAC3C,IAAI,UACF,MAAM;EAEV;EACA,MAAM,SAAS,KAAK,eAChB,OAAO,IAAI,IAAI,QAAQ,KAAK,YAAY,IACxC,UAAU,IAAI,IAAI,MAAM;EAC5B,GAAG,KAAK,QAAQ;EAChB,OAAO;CACT,SAAS,GAAG;EACV,IAAI,GAAG,eACL,GAAG,KAAK,UAAU;EAEpB,MAAM;CACR;AACF;;;;;;AAOA,SAAS,gBACP,IACA,QACsC;CACtC,MAAM,OAAO,oBAAoB,IAAI,OAAO,QAAQ;CACpD,IAAI,SAAS,KAAA,GACX,OAAO,IAAI,yBAAyB,IAAI;CAG1C,MAAM,OAAO,kBAAkB,EAAE;CACjC,MAAM,EAAC,oBAAmB;CAC1B,IAAI,SAAS,QAAQ,OAAO,iBAC1B,OAAO,IAAI,yBAAyB,KAAK;CAE3C,IAAI,SAAS,iBACX;CAUF,IAPiB,GACd,QAAgB;;;;KAIhB,CAAC,CACD,IAA6B,eAC5B,MAAa,KAAA,GACf,OAAO,IAAI,yBAAyB,KAAK;CAG3C,MAAM,EAAC,MAAM,mBAAkB,GAC5B,QAAgB;;;;0CAIqB,wBAAwB;;;;KAI7D,CAAC,CACD,IAA4C,eAAe;CAC9D,OAAO,OAAA,OACL,iBAAA,WACE,IAAI,yBAAyB,sBAAsB,MAAM,cAAc,IACvE,KAAA;AACN;AAEA,SAAS,UACP,IACA,IACA,QACiB;CACjB,MAAM,OAAO,oBAAoB,IAAI,OAAO,QAAQ;CACpD,IAAI,SAAS,KAAA,GACX,OAAO,OAAO,IAAI,IAAI,QAAQ,IAAI;CAGpC,MAAM,EAAC,oBAAmB;CAC1B,IAAI,OAAO,kBAAkB,EAAE;CAC/B,IAAI,OAAO;CACX,IAAI,SAAS,QAAQ,OAAO,iBAAiB;EAC3C,OAAO,GACJ,QAAgB;uBACA,wBAAwB;OACxC,CAAC,CACD,IAAI,eAAe,CAAC,CAAC;EACxB,OAAO,kBAAkB,EAAE;CAC7B;CAEA,IAAI,SAAS,iBACX,OAAO,OAAO,IAAI,IAAI,QAAQ,KAAK;CAErC,IAAI,OAAO,GAAG;EAMZ,eAAe,IAAI,OAAO,OAAO;EACjC,GAAG,OAAO,6DAA6D,EACrE,0BAA0B;GAAC;GAAM;GAAM,SAAS,aAAa,MAAM;EAAC,EACtE,CAAC;EACD,OAAO;GAAC,QAAQ;GAAa;GAAM;GAAM,cAAc;EAAI;CAC7D;CACA,GAAG,QAAQ,gDAAgD,EACzD,0BAA0B,EAAC,KAAI,EACjC,CAAC;CACD,OAAO;EAAC,QAAQ;EAAQ;EAAM,cAAc;CAAK;AACnD;;;;;;;;AASA,SAAgB,oBACd,IACA,UAC0B;CAC1B,IACE,CAAC,YAAY,IAAA,uBAA2B,KACxC,CAAC,YAAY,IAAA,qBAAyB,GAEtC,OAAO;CAKT,MAAM,UAAU,GACb,QAAgB;qCACgB,sBAAsB;KACtD,CAAC,CACD,IAAyC;CAC5C,IAAI,YAAY,KAAA,GACd,OAAO;CAET,IAAI,QAAQ,kBAAA,GACV,OAAO;CAMT,IACE,CAAC,YAAY,IAAA,8BAAmC,KAChD,CAAC,YAAY,IAAA,4BAAgC,GAE7C,OAAO;CAET,MAAM,EAAC,OAAO,YAAY,cAAa,kBAAkB,EAAE;CAC3D,IACE,UAAU,SAAS,SACnB,eAAe,SAAS,cACxB,cAAc,SAAS,WAEvB,OAAO;AAGX;AAEA,SAAS,OACP,IACA,IACA,QACA,QACiB;CACjB,MAAM,EAAC,UAAU,iBAAiB,UAAS;CAI3C,GAAG,KAAa;4BACU,wBAAwB;4BACxB,sBAAsB;MAC5C,8BAA8B;MAC9B,gCAAgC;MAChC,8BAA8B;MAC9B,gCAAgC;GACnC;CACD,GAAG,QAAgB;mBACF,sBAAsB;;;;;GAKtC,CAAC,CAAC,IAAI;EACL,OAAO,SAAS;EAChB,YAAY,SAAS;EACrB,WAAW,SAAS;EACpB,eAAA;EACA,YAAY;EACZ,eAAe;CACjB,CAAC;CACD,oBAAoB,IAAI,MAAM;CAK9B,eAAe,IAAI,OAAO,OAAO;CAEjC,GAAG,OAAO,kCAAkC,EAC1C,0BAA0B;EACxB;EACA,MAAM;EACN,GAAG;EACH,eAAA;EACA,YAAY;EACZ,SAAS,aAAa,MAAM;CAC9B,EACF,CAAC;CACD,OAAO;EACL,QAAQ;EACR,MAAM;EACN;EACA,cAAc;CAChB;AACF;;;;;;AAOA,SAAgB,oBACd,IACA,QACM;CACN,MAAM,cAAc;CACpB,MAAM,eAAe;CACrB,GAAG,QAAQ,0BAA0B,CAAC,CAAC,IAAI;EACzC,WAAW,OAAO;EAClB,YAAY,gCACV,OAAO,iBACP,SACA,WACF;EACA;EACA,aAAa,gCACX,OAAO,iBACP,UACA,cACA,OAAO,iBACP,IACF;EACA;EACA,aAAa,OAAO;CACtB,CAAC;AACH;;AAGA,SAAgB,kBAAkB,IAA6B;CAC7D,MAAM,OAAO,GACV,QAAgB;;;gBAGL,sBAAsB;KACjC,CAAC,CACD,IAA+B;CAClC,IAAI,SAAS,KAAA,GACX,MAAM,IAAI,MAAM,uCAAuC;CAEzD,OAAO;AACT;;;;;AAMA,SAAgB,kBAAkB,IAA6B;CAC7D,MAAM,EAAC,SAAQ,GACZ,QAAgB;gDAC2B,wBAAwB;KACnE,CAAC,CACD,IAA2B;CAC9B,OAAO;AACT;;;;;;AAOA,SAAgB,oBAAoB,IAA+B;CACjE,OAAO,GACJ,QAAgB;;yCAEoB,wBAAwB;;yCAExB,wBAAwB;;KAE5D,CAAC,CACD,IAAqB;AAC1B;;;;;AAWA,SAAS,aAAa,EAAC,WAA2B;CAChD,OAAO;EACL,eAAe,QAAQ,cAAc;EACrC,aAAa,QAAQ,YAAY;CACnC;AACF;AAEA,SAAS,YAAY,IAAc,OAAwB;CACzD,OACE,GACG,QAAgB;;OAEhB,CAAC,CACD,IAA6B,KAAK,MAAM,KAAA;AAE/C"}
+{"version":3,"file":"change-log-db.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/change-log-db.ts"],"sourcesContent":["/**\n * The change log lives in its own SQLite database, `${replicaFile}-change-log`,\n * rather than in the replica file. It is a catchup cache: everything it holds is\n * either already durable in the replica's litestream backup or re-derivable from\n * the upstream replication slot, so it is deliberately excluded from the backup\n * and is never a source of truth.\n *\n * The change-streamer writes it, from the stream loop that fans changes out to\n * subscribers, and commits it before forwarding each transaction's `commit`\n * message. Its head is therefore always at or above the watermark the stream\n * would resume from, which is what {@link reconcileChangeLog} anchors on.\n *\n * Beside the buffer it holds the log's *cookie jar* — the backfill progress\n * state folded up to the same head. That is `change-log-cookies.ts`; this file\n * owns its tables' lifecycle, because a cookie set is only meaningful paired\n * with the watermark it was folded to, and so must be created, dropped, and —\n * whenever reconciliation moves the head — replaced with the anchor's, all with\n * the buffer rather than beside it.\n *\n * Because it is disposable, it has no migration framework. Any inconsistency —\n * a schema change, a leftover file from a different replica, a gap left by a\n * crash or by running with the writer disabled, a corrupt file — is resolved by\n * {@link reconcileChangeLog} and {@link openChangeLogDBForWriting}, which\n * truncate a bounded suffix or replace and reseed the log at the resume\n * watermark. The worst outcome is a `too-old` for a lagging subscriber; a\n * silently delivered gap is not reachable.\n *\n * Disk sizing: a task that writes the log needs room for the replica *plus* the\n * retained change stream — the log's main file, its wal2 sidecars, and pages a\n * purge has freed but not yet returned to the OS. The change-streamer schedules\n * purges against the same floor as the PG change log, so retention is bounded\n * by the confirmed backup watermark ANDed with the minimum retention window —\n * a function of the backup interval, not a constant: 15–30+ minutes behind an\n * RMv1 sync interval, ~60 seconds once litestream v5 backs up every 15–30 s.\n * A held snapshot reservation pins retention unboundedly for its duration, so\n * peak retained bytes is set by the slowest concurrent restore rather than by\n * the retention setting — size disk and alerts on that. The log is excluded\n * from the litestream backup, so this is local disk only and never S3.\n */\n\nimport {existsSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {\n  CHANGE_LOG_BACKFILLING_TABLE,\n  CHANGE_LOG_TABLE_METADATA_TABLE,\n  CREATE_CHANGE_LOG_COOKIE_SCHEMA,\n  DROP_CHANGE_LOG_COOKIE_TABLES,\n  replaceCookies,\n  type CookieSet,\n} from './change-log-cookies.ts';\n\n/**\n * Bumped whenever the change-log database's schema changes. Since the file is a\n * cache, a mismatch costs one reseed rather than a migration.\n *\n * v2 re-homed the writer to the change-streamer: the log anchors on the\n * watermark its stream connection resumes from rather than on the replica's\n * state version, and its meta row carries the replica's identity triple plus\n * the seed point.\n *\n * v3 added the cookie tables (see `change-log-cookies.ts`), i.e. the log's\n * second job. The reseed it costs on upgrade buys one retention window of\n * catchup reach on a log nothing reads yet.\n *\n * v4 stores each row's tag and estimated retained bytes alongside its JSON.\n * Readers can now filter without parsing arbitrary row payloads, and\n * reconciliation can bound truncation by bytes as well as rows.\n *\n * `auto_vacuum = INCREMENTAL` arrived within v2, deliberately without a bump:\n * it is a file-header property rather than a schema change, and the reseed a\n * version mismatch triggers cannot enable it (see\n * {@link applyChangeLogPragmas}), so {@link openChangeLogDBForWriting} guards\n * on the pragma itself instead of on this number.\n */\nexport const CHANGE_LOG_DB_SCHEMA_VERSION = 4;\n\n/** https://www.sqlite.org/pragma.html#pragma_auto_vacuum */\nconst AUTO_VACUUM_INCREMENTAL = 2;\n\nexport const CHANGE_LOG_STREAM_TABLE = '_zero.changeLogStream';\nexport const CHANGE_LOG_STREAM_WRITE_TIME_INDEX =\n  '_zero.changeLogStream_writeTimeMs';\n\nconst INTEGER_BYTES = 8;\n\n/**\n * Estimates the retained payload size of a stream row. This deliberately does\n * not claim to measure SQLite b-tree/page overhead; it is stable across the\n * write path, reconciliation, and the startup scan used by observability.\n */\nexport function estimateChangeLogStreamRowBytes(\n  watermark: string,\n  tag: string,\n  change: string,\n  precommit?: string,\n  hasWriteTime = false,\n): number {\n  return (\n    Buffer.byteLength(watermark) +\n    INTEGER_BYTES +\n    Buffer.byteLength(tag) +\n    Buffer.byteLength(change) +\n    (precommit === undefined ? 0 : Buffer.byteLength(precommit)) +\n    (hasWriteTime ? INTEGER_BYTES : 0)\n  );\n}\n\n// The index is partial because only commit rows carry a \"writeTimeMs\", so it\n// holds one entry per transaction rather than one per change. It is what the\n// purger scans to find transactions older than the retention window.\n//\n// Pre-release builds briefly carried this table in the replica; see the test\n// fixture `CREATE_V14_CHANGE_LOG_STREAM` in\n// `change-source/common/replica-schema.ts`. Its old shape must not be kept in\n// sync with this one.\nexport const CREATE_CHANGE_LOG_STREAM_SCHEMA = /*sql*/ `\n  CREATE TABLE \"${CHANGE_LOG_STREAM_TABLE}\" (\n    \"watermark\"      TEXT NOT NULL,\n    \"pos\"            INTEGER NOT NULL,\n    \"tag\"            TEXT NOT NULL,\n    \"estimatedBytes\"  INTEGER NOT NULL,\n    \"change\"          TEXT NOT NULL,\n    \"precommit\"       TEXT,\n    \"writeTimeMs\"     INTEGER,\n    PRIMARY KEY (\"watermark\", \"pos\")\n  );\n\n  CREATE INDEX \"${CHANGE_LOG_STREAM_WRITE_TIME_INDEX}\"\n    ON \"${CHANGE_LOG_STREAM_TABLE}\" (\"writeTimeMs\", \"watermark\")\n    WHERE \"writeTimeMs\" IS NOT NULL;\n`;\n\n/**\n * Inserts the synthetic begin/commit pair for the seed watermark.\n *\n * Both rows are inserted by one statement so a caller outside a larger\n * transaction cannot leave a partial seed.\n */\nconst SEED_CHANGE_LOG_STREAM_SQL = /*sql*/ `\n  INSERT INTO \"${CHANGE_LOG_STREAM_TABLE}\"\n    (\"watermark\", \"pos\", \"tag\", \"estimatedBytes\", \"change\", \"precommit\",\n     \"writeTimeMs\")\n  VALUES\n    (@watermark, 0, 'begin', @beginBytes, @beginChange, NULL, NULL),\n    (@watermark, 1, 'commit', @commitBytes, @commitChange, @watermark,\n     @writeTimeMs)\n  ON CONFLICT (\"watermark\", \"pos\") DO NOTHING\n`;\n\nexport const CHANGE_LOG_META_TABLE = '_zero.changeLogMeta';\n\n// The identity triple (see ChangeLogIdentity) detects a file left behind by a\n// different replica -- most concretely, a leftover log on a reused volume.\n//\n// \"seededAtMs\"    : when the log was seeded, i.e. how far its retention window\n//                   can possibly reach back. Slice 11 uses it as the warm-up\n//                   signal for canary reads.\n// \"seedWatermark\" : where the log was seeded. `min(\"watermark\")` cannot answer\n//                   this once purge has advanced the minimum, and the\n//                   difference is what distinguishes a log that purged past the\n//                   retention floor (an invariant violation) from one that\n//                   never held that history at all (routine at every fork).\n// \"lock\"          : enforces single-row semantics, as in\n//                   \"_zero.replicationConfig\".\nconst CREATE_CHANGE_LOG_META_SCHEMA = /*sql*/ `\n  CREATE TABLE \"${CHANGE_LOG_META_TABLE}\" (\n    \"epoch\"         TEXT,\n    \"generation\"    TEXT NOT NULL,\n    \"replicaID\"     TEXT,\n    \"schemaVersion\" INTEGER NOT NULL,\n    \"seededAtMs\"    INTEGER NOT NULL,\n    \"seedWatermark\" TEXT NOT NULL,\n    \"lock\"          INTEGER PRIMARY KEY DEFAULT 1 CHECK (\"lock\" = 1)\n  );\n`;\n\n/**\n * `${replicaFile}-change-log`, matching the `-serving-copy` convention.\n *\n * The name outlives the writer's move into the change-streamer: the\n * change-streamer derives the path from the replica file it owns, so renaming\n * would save no plumbing and would orphan the old file on every existing\n * volume, where nothing would delete it.\n *\n * Its `-wal` / `-wal2` / `-shm` sidecars are suffixes of *this* name, so they\n * never collide with the replica's own sidecars.\n */\nexport function changeLogFileName(replicaFile: string): string {\n  return `${replicaFile}-change-log`;\n}\n\n/**\n * Removes the change-log database and its `-wal` / `-wal2` / `-shm` sidecars.\n * Safe to call when the file does not exist.\n */\nexport function deleteChangeLogDB(replicaFile: string): void {\n  deleteLiteDB(changeLogFileName(replicaFile));\n}\n\n/**\n * Which replica the log's contents belong to.\n *\n * `generation` is RMv1's `replicaVersion`: the version at which initial sync\n * copied every row. It is shared by every sibling of a forked replica, which is\n * why it cannot be the whole of the identity under RMv2 — `epoch` and\n * `replicaID` are what distinguish two siblings' logs. Both are `null` until\n * RMv2 supplies them, and a `null` matches only another `null`, so a log\n * written before they exist is still recognized as this replica's.\n */\nexport type ChangeLogIdentity = {\n  readonly epoch: string | null;\n  readonly generation: string;\n  readonly replicaID: string | null;\n};\n\n/**\n * Where the stream connection resumes, and everything the log's state at that\n * position consists of.\n *\n * The two fields travel together because a cookie set is only meaningful paired\n * with the watermark it was folded up to (invariant 15). They are read from one\n * store, in one snapshot: `Storer.getStartStreamInitializationParameters()`\n * reads both from Postgres in a single REPEATABLE READ transaction. Pairing one\n * store's cookies with another's watermark would silently drop every backfill\n * that completed in the interval, and those rows are unrecoverable — backfill\n * transactions are synthesized by the change source rather than replayed from\n * the slot.\n */\nexport type ChangeLogResumePoint = {\n  /**\n   * The watermark this stream connection resumes from. The log's own head is\n   * measured against it, and a reseed seeds a synthetic transaction at it.\n   *\n   * Deliberately *not* \"the replica's state version\": after the writer moved to\n   * the change-streamer, the canonical replicator is a subscriber downstream of\n   * the write loop, so its state version is a consequence of this log rather\n   * than a constraint on it.\n   */\n  readonly resumeWatermark: string;\n  /**\n   * The cookie set that belongs to {@link resumeWatermark}. Any reconciliation\n   * that moves the head invalidates the set the log was holding — the cookies\n   * are unversioned point-in-time state, and truncating the stream above `W`\n   * does not roll them back to `W` — so the same transaction installs this one\n   * in its place (invariant 17).\n   *\n   * Postgres supplies it while it remains the store the stream resumes from. At\n   * the C5 flip the log's own head *is* the resume point, phantoms cannot\n   * exist, and both this field and the truncate-above path become dead code.\n   */\n  readonly cookies: CookieSet;\n};\n\n/**\n * What the log is reconciled against. Reconciliation runs once per stream\n * connection, since every reconnect can re-read a different resume point.\n */\nexport type ChangeLogAnchor = ChangeLogResumePoint & {\n  readonly identity: ChangeLogIdentity;\n  /**\n   * The change-streamer's clock, at reconciliation time. Supplied rather than\n   * read inline so that tests control it. Used for `seededAtMs` and for the\n   * seed transaction's `writeTimeMs`, which is what the purger's time floor\n   * measures.\n   */\n  readonly nowMs: number;\n};\n\n/** The meta row, i.e. everything the log records about itself. */\nexport type ChangeLogMeta = ChangeLogIdentity & {\n  readonly schemaVersion: number;\n  readonly seededAtMs: number;\n  readonly seedWatermark: string;\n};\n\n/**\n * Opens the change-log database beside `replicaFile`.\n *\n * A writable handle creates the file if it is absent; a `readonly` handle\n * throws, which is the normal state for a change-streamer whose writer is\n * disabled. Pragmas are the caller's responsibility, since only the writer sets\n * the persistent ones.\n */\nexport function openChangeLogDB(\n  lc: LogContext,\n  replicaFile: string,\n  opts: {readonly: boolean},\n): Database {\n  return new Database(\n    lc.withContext('component', 'change-log-db'),\n    changeLogFileName(replicaFile),\n    {readonly: opts.readonly},\n  );\n}\n\n/**\n * Configures the change-log database, which the change-streamer's writer owns\n * exclusively. These are the settings the `separate-files` mode of\n * `sqlite-change-log-ceiling.bench.ts` measured.\n *\n * `wal2` because the log is a continuous append that catchup reads\n * continuously, which is the case wal2 exists for: it avoids checkpoint\n * starvation without needing an autocheckpoint. `wal_autocheckpoint` therefore\n * keeps its default, unlike the backup replica's 0, which hands checkpointing\n * to litestream — nothing else owns this file.\n *\n * `synchronous = NORMAL` because the commit sits on the forward path, once per\n * upstream transaction, and NORMAL keeps that from costing an fsync. It is not\n * a durability compromise here: in WAL mode NORMAL still writes every commit\n * through to the OS, so it survives a process crash, an OOM kill, or a SIGKILL.\n * It is lost only to kernel panic or power loss, and those destroy the node —\n * the task restarts elsewhere, restores the replica from S3, and has no\n * change-log file at all. Whatever a power loss does take is detected by\n * {@link reconcileChangeLog} and reseeded.\n *\n * `auto_vacuum = INCREMENTAL` so that purge can return freed pages to the OS\n * rather than leaving the file at its high-water mark. **It must be the first\n * statement here.** SQLite honours a `none` -> `incremental` transition only\n * while the file is still empty, and `journal_mode = wal2` materializes page 1;\n * appending the pragma instead of prepending it silently leaves the file at\n * mode 0 forever, since neither `DROP TABLE` nor a reseed makes a file empty\n * again. {@link openChangeLogDBForWriting} verifies the result.\n */\nexport function applyChangeLogPragmas(db: Database): void {\n  db.pragma('auto_vacuum = INCREMENTAL');\n  db.pragma('busy_timeout = 30000');\n  db.pragma('analysis_limit = 1000');\n  db.pragma('journal_mode = wal2');\n  db.pragma('synchronous = NORMAL');\n}\n\nfunction readAutoVacuum(db: Database): number {\n  const [{auto_vacuum: mode}] = db.pragma<{auto_vacuum: number}>('auto_vacuum');\n  return mode;\n}\n\n/**\n * Opens the change-log database for the writer, applies its pragmas, and\n * reconciles it against `anchor`, returning the handle and what reconciliation\n * had to do.\n *\n * Corruption is answered with a rebuild rather than an escalation, which is a\n * deliberate divergence from how the replica's own corruption is handled. A\n * corrupt replica is a lost source of truth, so it ends in a restore or an\n * auto-reset; the change log holds nothing that is not either already in the\n * replica's litestream backup or re-derivable from the upstream slot, so the\n * entire response is to delete the file and seed a new one at the resume\n * watermark. The cost is one retention window of catchup — reconnecting\n * subscribers below the head get `too-old` — which is what every other\n * {@link ReseedReason} costs.\n *\n * Only corruption is rebuilt through. Anything else (a bad path, a permissions\n * error, a full disk) is a problem with the environment rather than with the\n * file's contents, and deleting a database in response would destroy state\n * without fixing it. Those propagate to the caller, which disables the writer\n * and keeps replicating.\n *\n * A file that predates `auto_vacuum = INCREMENTAL` takes the same rebuild, for\n * the same reason and at the same cost: the pragma only takes on an empty file,\n * so a reseed cannot enable it. Unlike corruption, a *second* failure there is\n * tolerated rather than thrown — see {@link ensureIncrementalAutoVacuum}.\n */\n/**\n * Returns an anchor after the database opens.\n *\n * The resolver can inspect the existing file before it selects the anchor. A\n * valid log can use its own head. A new or invalid log uses the replica seed.\n * The rebuild path calls the resolver again for the new file.\n */\nexport type AnchorResolver = (db: Database) => ChangeLogAnchor;\n\nexport function openChangeLogDBForWriting(\n  lc: LogContext,\n  replicaFile: string,\n  anchor: AnchorResolver,\n): {db: Database; result: ReconcileResult; replacedFile: boolean} {\n  const existed = existsSync(changeLogFileName(replicaFile));\n  const opened = openOrRebuildCorrupt(lc, replicaFile, anchor, existed);\n  const {db, result, replacedFile} = ensureIncrementalAutoVacuum(\n    lc,\n    replicaFile,\n    anchor,\n    opened,\n  );\n  return {db, result, replacedFile};\n}\n\nfunction openOrRebuildCorrupt(\n  lc: LogContext,\n  replicaFile: string,\n  anchor: AnchorResolver,\n  boundReconciliation: boolean,\n): OpenedChangeLog {\n  try {\n    return openAndReconcile(lc, replicaFile, anchor, boundReconciliation);\n  } catch (e) {\n    if (e instanceof ChangeLogRebuildRequired) {\n      return rebuildChangeLog(lc, replicaFile, anchor, e);\n    }\n    if (!isSQLiteCorruption(e)) {\n      throw e;\n    }\n    const file = changeLogFileName(replicaFile);\n    logSQLiteCorruptionDiagnostics(lc, 'change-log', file, e);\n    lc.error?.('rebuilding the corrupt SQLite change log', {\n      sqliteChangeLog: {file},\n    });\n    return rebuildChangeLog(lc, replicaFile, anchor);\n  }\n}\n\n/**\n * Rebuilds a log whose `auto_vacuum` mode is not `INCREMENTAL`, which is every\n * log created before the pragma shipped — including every v2 file the writer\n * wrote in the meantime, which is why the guard is on the pragma rather than\n * on the schema version. Deleting the file is the only way to enable it:\n * `DROP TABLE` leaves the header's mode intact, and the in-place alternative —\n * a full `VACUUM` — is an unbounded blocking statement on a file the writer\n * needs.\n *\n * A mode that is still wrong after the rebuild is logged and tolerated, which\n * is the one place this diverges from the corruption path. The two failures are\n * not alike: a file that stays corrupt after a rebuild is an environment\n * problem, but a pragma that stays wrong on a brand-new file can only mean\n * {@link applyChangeLogPragmas}' statement order is wrong — a code bug, present\n * on every task, on every start. Throwing there turns a lost optimization into\n * a crash loop, and rebuilding again turns it into a reseed per start, which is\n * worse than the freelist growth it was trying to avoid. `Database.compact()`\n * already warns and no-ops when the mode is not `INCREMENTAL`, so degrading is\n * the accepted fallback.\n */\nfunction ensureIncrementalAutoVacuum(\n  lc: LogContext,\n  replicaFile: string,\n  anchor: AnchorResolver,\n  opened: OpenedChangeLog,\n): OpenedChangeLog {\n  if (opened.autoVacuum === AUTO_VACUUM_INCREMENTAL) {\n    return opened;\n  }\n  const file = changeLogFileName(replicaFile);\n  lc.info?.('rebuilding the SQLite change log to enable incremental vacuum', {\n    sqliteChangeLog: {file, autoVacuum: opened.autoVacuum},\n  });\n  opened.db.close();\n  const rebuilt = rebuildChangeLog(lc, replicaFile, anchor);\n  if (rebuilt.autoVacuum !== AUTO_VACUUM_INCREMENTAL) {\n    lc.error?.(\n      'SQLite change log does not support incremental vacuum; ' +\n        'it will plateau at its high-water mark',\n      {sqliteChangeLog: {file, autoVacuum: rebuilt.autoVacuum}},\n    );\n  }\n  return rebuilt;\n}\n\n/** The shared \"delete the file and open a new one\" body. */\nfunction rebuildChangeLog(\n  lc: LogContext,\n  replicaFile: string,\n  anchor: AnchorResolver,\n  required?: ChangeLogRebuildRequired | undefined,\n): OpenedChangeLog {\n  if (required) {\n    lc.warn?.(\n      'rebuilding the SQLite change log instead of running an unbounded reconciliation',\n      {\n        sqliteChangeLogRebuild: {\n          reason: required.reason,\n          rows: required.rows,\n          estimatedBytes: required.estimatedBytes,\n        },\n      },\n    );\n  }\n  deleteChangeLogDB(replicaFile);\n  // The file is absent, so the reseed only drops empty/nonexistent tables. A\n  // second failure is the environment's, not the old file's.\n  const opened = openAndReconcile(\n    lc,\n    replicaFile,\n    anchor,\n    false,\n    required?.reason,\n  );\n  return {...opened, replacedFile: true};\n}\n\n/**\n * Replaces an already-closed change-log file after bounded reconciliation\n * declined its work. The caller closes cached readers before calling this so\n * none can continue serving the unlinked inode.\n */\nexport function rebuildChangeLogDBForWriting(\n  lc: LogContext,\n  replicaFile: string,\n  anchor: ChangeLogAnchor,\n  required: ChangeLogRebuildRequired,\n): {db: Database; result: ReconcileResult; replacedFile: true} {\n  const {db, result} = rebuildChangeLog(\n    lc,\n    replicaFile,\n    () => anchor,\n    required,\n  );\n  return {db, result, replacedFile: true};\n}\n\ntype OpenedChangeLog = {\n  db: Database;\n  result: ReconcileResult;\n  replacedFile: boolean;\n  /** The file's persistent `auto_vacuum` mode, read back after the pragmas. */\n  autoVacuum: number;\n};\n\nfunction openAndReconcile(\n  lc: LogContext,\n  replicaFile: string,\n  anchor: AnchorResolver,\n  boundReconciliation: boolean,\n  reseedReason?: ReseedReason | undefined,\n): OpenedChangeLog {\n  const db = openChangeLogDB(lc, replicaFile, {readonly: false});\n  try {\n    applyChangeLogPragmas(db);\n    const autoVacuum = readAutoVacuum(db);\n    return {\n      db,\n      result: reconcileChangeLog(lc, db, anchor(db), {\n        rebuildInsteadOfUnboundedWork: boundReconciliation,\n        reseedReason,\n      }),\n      replacedFile: false,\n      autoVacuum,\n    };\n  } catch (e) {\n    // The caller never sees this handle, so it closes here or leaks — and it\n    // must be closed before the corruption path deletes the file.\n    db.close();\n    throw e;\n  }\n}\n\nexport type ReseedReason =\n  | 'created' // file or table absent\n  | 'schema-mismatch'\n  | 'identity-mismatch'\n  | 'gap' // head does not land on the resume watermark after truncation\n  | 'oversized-truncate'; // preserving the prefix would require too much work\n\n/**\n * Reconciliation runs synchronously on the change-streamer's event loop. A\n * normal phantom is one transaction, so work above either limit is answered by\n * unlinking and reseeding this disposable cache rather than deleting an\n * arbitrarily large suffix in place.\n */\nexport const MAX_RECONCILE_TRUNCATE_ROWS = 10_000;\nexport const MAX_RECONCILE_TRUNCATE_BYTES = 16 * 1024 * 1024;\n\nexport class ChangeLogRebuildRequired extends Error {\n  override readonly name = 'ChangeLogRebuildRequired';\n  readonly reason: ReseedReason;\n  readonly rows?: number | undefined;\n  readonly estimatedBytes?: number | undefined;\n\n  constructor(\n    reason: ReseedReason,\n    rows?: number | undefined,\n    estimatedBytes?: number | undefined,\n  ) {\n    super(`SQLite change log requires a ${reason} rebuild`);\n    this.reason = reason;\n    this.rows = rows;\n    this.estimatedBytes = estimatedBytes;\n  }\n}\n\ntype ReconcileOptions = {\n  /** Throw before an unbounded delete/drop so the file owner can unlink it. */\n  readonly rebuildInsteadOfUnboundedWork?: boolean | undefined;\n  /** Preserve the reason that caused an old file to be unlinked. */\n  readonly reseedReason?: ReseedReason | undefined;\n};\n\n/**\n * `cookiesStale` records that the cookie set the log held did not survive\n * reconciliation. The cookies are unversioned point-in-time state, so any\n * reconciliation that moves the head invalidates them, and the same transaction\n * replaces them with the anchor's — the set that belongs to the resume\n * watermark (see `change-log-cookies.ts`). It is `action !== 'none'` today; it\n * is a field rather than a derivation because the replacement reads as intent\n * at the call site, and because it is what the `cookie_reseed` counter is keyed\n * on.\n */\nexport type ReconcileResult =\n  | {action: 'none'; head: string; cookiesStale: false}\n  | {action: 'truncated'; head: string; rows: number; cookiesStale: true}\n  | {\n      action: 'reseeded';\n      head: string;\n      reason: ReseedReason;\n      cookiesStale: true;\n    };\n\n/**\n * Brings the change log into agreement with the watermark its stream connection\n * resumes from, in one transaction on the change-log database:\n *\n * ```\n * head === resumeWatermark  → keep appending\n * head  >  resumeWatermark  → truncate above resumeWatermark\n * otherwise                 → wipe, seed at resumeWatermark, record the reason\n * ```\n *\n * The log commits before anything that can advance the resume watermark, so a\n * *phantom* — a transaction the log holds whose watermark the resumed stream\n * re-delivers — is the normal state, and truncate-above is the whole of\n * reconciliation rather than a special case. A phantom's rows all carry the\n * transaction's commit watermark, which is strictly greater than the resume\n * watermark, so deleting on that predicate removes phantoms whole and retains\n * every transaction at or below the resume point.\n *\n * It must run per stream connection, not once per process: the stream loop\n * re-reads its resume watermark on every reconnect, and the writer inserts with\n * a plain `INSERT` against `PRIMARY KEY (\"watermark\",\"pos\")`, so an\n * un-truncated overlap is a constraint violation on the first re-delivered\n * transaction.\n *\n * Anything truncation cannot resolve — the file was deleted, the writer was\n * disabled for a while, the replica was restored, power was lost with\n * `synchronous=NORMAL` — surfaces as a head that does not land on the resume\n * watermark, and is resolved by wiping and reseeding there.\n *\n * Whichever of the two moves the head also replaces the cookie set with the one\n * the anchor carries, in the same transaction: the head and the cookies are one\n * position, and nothing may observe them at two.\n */\nexport function reconcileChangeLog(\n  lc: LogContext,\n  db: Database,\n  anchor: ChangeLogAnchor,\n  opts: ReconcileOptions = {},\n): ReconcileResult {\n  db.exec('BEGIN IMMEDIATE');\n  try {\n    if (opts.rebuildInsteadOfUnboundedWork) {\n      const required = rebuildRequired(db, anchor);\n      if (required) {\n        throw required;\n      }\n    }\n    const result = opts.reseedReason\n      ? reseed(lc, db, anchor, opts.reseedReason)\n      : reconcile(lc, db, anchor);\n    db.exec('COMMIT');\n    return result;\n  } catch (e) {\n    if (db.inTransaction) {\n      db.exec('ROLLBACK');\n    }\n    throw e;\n  }\n}\n\n/**\n * Decides whether reconciliation can be performed within its synchronous work\n * budget. Every query is either an index seek or a scan capped at one row past\n * the delete limit. No state changes before the decision is complete.\n */\nfunction rebuildRequired(\n  db: Database,\n  anchor: ChangeLogAnchor,\n): ChangeLogRebuildRequired | undefined {\n  const wipe = changeLogWipeReason(db, anchor.identity);\n  if (wipe !== undefined) {\n    return new ChangeLogRebuildRequired(wipe);\n  }\n\n  const head = readChangeLogHead(db);\n  const {resumeWatermark} = anchor;\n  if (head === null || head < resumeWatermark) {\n    return new ChangeLogRebuildRequired('gap');\n  }\n  if (head === resumeWatermark) {\n    return undefined;\n  }\n\n  const boundary = db\n    .prepare(/*sql*/ `\n      SELECT 1 FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n        WHERE \"watermark\" = ?\n        LIMIT 1\n    `)\n    .get<{1: number} | undefined>(resumeWatermark);\n  if (boundary === undefined) {\n    return new ChangeLogRebuildRequired('gap');\n  }\n\n  const {rows, estimatedBytes} = db\n    .prepare(/*sql*/ `\n      SELECT count(*) AS \"rows\",\n             coalesce(sum(\"estimatedBytes\"), 0) AS \"estimatedBytes\"\n        FROM (\n          SELECT \"estimatedBytes\" FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n            WHERE \"watermark\" > ?\n            LIMIT ${MAX_RECONCILE_TRUNCATE_ROWS + 1}\n        )\n    `)\n    .get<{rows: number; estimatedBytes: number}>(resumeWatermark);\n  return rows > MAX_RECONCILE_TRUNCATE_ROWS ||\n    estimatedBytes > MAX_RECONCILE_TRUNCATE_BYTES\n    ? new ChangeLogRebuildRequired('oversized-truncate', rows, estimatedBytes)\n    : undefined;\n}\n\nfunction reconcile(\n  lc: LogContext,\n  db: Database,\n  anchor: ChangeLogAnchor,\n): ReconcileResult {\n  const wipe = changeLogWipeReason(db, anchor.identity);\n  if (wipe !== undefined) {\n    return reseed(lc, db, anchor, wipe);\n  }\n\n  const {resumeWatermark} = anchor;\n  let head = readChangeLogHead(db);\n  let rows = 0;\n  if (head !== null && head > resumeWatermark) {\n    rows = db\n      .prepare(/*sql*/ `\n        DELETE FROM \"${CHANGE_LOG_STREAM_TABLE}\" WHERE \"watermark\" > ?\n      `)\n      .run(resumeWatermark).changes;\n    head = readChangeLogHead(db);\n  }\n\n  if (head !== resumeWatermark) {\n    return reseed(lc, db, anchor, 'gap');\n  }\n  if (rows > 0) {\n    // The truncated transactions can have carried `create-table`,\n    // `add-column`, and `backfill-completed`, none of which the truncate rolls\n    // back, so the cookie set no longer belongs to the head. The anchor's set\n    // does, and it lands in this same transaction: invariant 17 holds at every\n    // point another connection could observe the log.\n    replaceCookies(db, anchor.cookies);\n    lc.info?.('truncated phantom transactions from the SQLite change log', {\n      sqliteChangeLogReconcile: {head, rows, cookies: cookieCounts(anchor)},\n    });\n    return {action: 'truncated', head, rows, cookiesStale: true};\n  }\n  lc.debug?.('SQLite change log is at the resume watermark', {\n    sqliteChangeLogReconcile: {head},\n  });\n  return {action: 'none', head, cookiesStale: false};\n}\n\n/**\n * Returns the reason that the log must be recreated. Returns `undefined` when\n * the writer can continue to use the log.\n *\n * This function takes only the expected identity because callers use the result\n * to select the anchor.\n */\nexport function changeLogWipeReason(\n  db: Database,\n  identity: ChangeLogIdentity,\n): ReseedReason | undefined {\n  if (\n    !tableExists(db, CHANGE_LOG_STREAM_TABLE) ||\n    !tableExists(db, CHANGE_LOG_META_TABLE)\n  ) {\n    return 'created';\n  }\n  // The schema version is read on its own because it is the one column every\n  // version of this table has had. Reading the rest of a v1 row would fail on\n  // its missing columns instead of reporting the mismatch that explains it.\n  const version = db\n    .prepare(/*sql*/ `\n      SELECT \"schemaVersion\" FROM \"${CHANGE_LOG_META_TABLE}\"\n    `)\n    .get<{schemaVersion: number} | undefined>();\n  if (version === undefined) {\n    return 'created';\n  }\n  if (version.schemaVersion !== CHANGE_LOG_DB_SCHEMA_VERSION) {\n    return 'schema-mismatch';\n  }\n  // Checked *after* the version, so that every pre-v4 file — which is every\n  // file in the fleet at the upgrade — reports the `schema-mismatch` that\n  // explains it rather than the `created` its absent cookie tables would\n  // otherwise look like.\n  if (\n    !tableExists(db, CHANGE_LOG_TABLE_METADATA_TABLE) ||\n    !tableExists(db, CHANGE_LOG_BACKFILLING_TABLE)\n  ) {\n    return 'created';\n  }\n  const {epoch, generation, replicaID} = readChangeLogMeta(db);\n  if (\n    epoch !== identity.epoch ||\n    generation !== identity.generation ||\n    replicaID !== identity.replicaID\n  ) {\n    return 'identity-mismatch';\n  }\n  return undefined;\n}\n\nfunction reseed(\n  lc: LogContext,\n  db: Database,\n  anchor: ChangeLogAnchor,\n  reason: ReseedReason,\n): ReconcileResult {\n  const {identity, resumeWatermark, nowMs} = anchor;\n  // Dropping the table drops its partial index with it. The cookie tables go\n  // with the buffer: a cookie set without the transactions it was folded from\n  // is not a smaller cookie set, it is a wrong one.\n  db.exec(/*sql*/ `\n    DROP TABLE IF EXISTS \"${CHANGE_LOG_STREAM_TABLE}\";\n    DROP TABLE IF EXISTS \"${CHANGE_LOG_META_TABLE}\";\n    ${DROP_CHANGE_LOG_COOKIE_TABLES}\n    ${CREATE_CHANGE_LOG_STREAM_SCHEMA}\n    ${CREATE_CHANGE_LOG_META_SCHEMA}\n    ${CREATE_CHANGE_LOG_COOKIE_SCHEMA}\n  `);\n  db.prepare(/*sql*/ `\n    INSERT INTO \"${CHANGE_LOG_META_TABLE}\"\n      (\"epoch\", \"generation\", \"replicaID\", \"schemaVersion\", \"seededAtMs\",\n       \"seedWatermark\")\n      VALUES (@epoch, @generation, @replicaID, @schemaVersion, @seededAtMs,\n              @seedWatermark)\n  `).run({\n    epoch: identity.epoch,\n    generation: identity.generation,\n    replicaID: identity.replicaID,\n    schemaVersion: CHANGE_LOG_DB_SCHEMA_VERSION,\n    seededAtMs: nowMs,\n    seedWatermark: resumeWatermark,\n  });\n  seedChangeLogStream(db, anchor);\n  // Into tables this statement just created, so the wholesale replacement is\n  // all insert. It is still the same call the truncate path makes: a reseed is\n  // the case where the log's cookie set was not merely moved but discarded, and\n  // the anchor's is equally what belongs at the head either way.\n  replaceCookies(db, anchor.cookies);\n\n  lc.info?.('reseeded the SQLite change log', {\n    sqliteChangeLogReconcile: {\n      reason,\n      head: resumeWatermark,\n      ...identity,\n      schemaVersion: CHANGE_LOG_DB_SCHEMA_VERSION,\n      seededAtMs: nowMs,\n      cookies: cookieCounts(anchor),\n    },\n  });\n  return {\n    action: 'reseeded',\n    head: resumeWatermark,\n    reason,\n    cookiesStale: true,\n  };\n}\n\n/**\n * Seeds a valid synthetic transaction at the anchor's resume watermark, making\n * an otherwise empty log serviceable as a catchup boundary for a subscriber\n * that is exactly at that watermark.\n */\nfunction seedChangeLogStream(db: Database, anchor: ChangeLogAnchor): void {\n  const beginChange = '{\"tag\":\"begin\"}';\n  const commitChange = '{\"tag\":\"commit\"}';\n  db.prepare(SEED_CHANGE_LOG_STREAM_SQL).run({\n    watermark: anchor.resumeWatermark,\n    beginBytes: estimateChangeLogStreamRowBytes(\n      anchor.resumeWatermark,\n      'begin',\n      beginChange,\n    ),\n    beginChange,\n    commitBytes: estimateChangeLogStreamRowBytes(\n      anchor.resumeWatermark,\n      'commit',\n      commitChange,\n      anchor.resumeWatermark,\n      true,\n    ),\n    commitChange,\n    writeTimeMs: anchor.nowMs,\n  });\n}\n\n/** Reads the whole meta row. Throws if the log has not been reconciled. */\nexport function readChangeLogMeta(db: Database): ChangeLogMeta {\n  const meta = db\n    .prepare(/*sql*/ `\n      SELECT \"epoch\", \"generation\", \"replicaID\", \"schemaVersion\",\n             \"seededAtMs\", \"seedWatermark\"\n        FROM \"${CHANGE_LOG_META_TABLE}\"\n    `)\n    .get<ChangeLogMeta | undefined>();\n  if (meta === undefined) {\n    throw new Error('the SQLite change log has no meta row');\n  }\n  return meta;\n}\n\n/**\n * `max()` on the primary key's leading column, i.e. an index seek. Null when\n * the log is empty.\n */\nexport function readChangeLogHead(db: Database): string | null {\n  const {head} = db\n    .prepare(/*sql*/ `\n      SELECT max(\"watermark\") AS \"head\" FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n    `)\n    .get<{head: string | null}>();\n  return head;\n}\n\n/**\n * The covered watermark range. Both scalar subqueries seek the watermark\n * index, so unlike a `count()` this does not scan the log. Both bounds are\n * null when it is empty.\n */\nexport function readChangeLogBounds(db: Database): ChangeLogBounds {\n  return db\n    .prepare(/*sql*/ `\n      SELECT\n        (SELECT min(\"watermark\") FROM \"${CHANGE_LOG_STREAM_TABLE}\")\n          AS \"minWatermark\",\n        (SELECT max(\"watermark\") FROM \"${CHANGE_LOG_STREAM_TABLE}\")\n          AS \"headWatermark\"\n    `)\n    .get<ChangeLogBounds>();\n}\n\nexport type ChangeLogBounds = {\n  readonly minWatermark: string | null;\n  readonly headWatermark: string | null;\n};\n\n/**\n * What was installed, for the reconcile log line. Normally `{0, 0}`: a nonzero\n * `backfilling` means the resumed stream is being handed backfills to re-request.\n */\nfunction cookieCounts({cookies}: ChangeLogAnchor) {\n  return {\n    tableMetadata: cookies.tableMetadata.length,\n    backfilling: cookies.backfilling.length,\n  };\n}\n\nfunction tableExists(db: Database, table: string): boolean {\n  return (\n    db\n      .prepare(/*sql*/ `\n        SELECT 1 FROM \"sqlite_master\" WHERE \"type\" = 'table' AND \"name\" = ?\n      `)\n      .get<{1: number} | undefined>(table) !== undefined\n  );\n}\n"],"mappings":";;;;;;AAmFA,IAAM,0BAA0B;AAEhC,IAAa,0BAA0B;AACvC,IAAa,qCACX;AAEF,IAAM,gBAAgB;;;;;;AAOtB,SAAgB,gCACd,WACA,KACA,QACA,WACA,eAAe,OACP;CACR,OACE,OAAO,WAAW,SAAS,IAC3B,gBACA,OAAO,WAAW,GAAG,IACrB,OAAO,WAAW,MAAM,KACvB,cAAc,KAAA,IAAY,IAAI,OAAO,WAAW,SAAS,MACzD,eAAe,gBAAgB;AAEpC;AAUA,IAAa,kCAA0C;kBACrC,wBAAwB;;;;;;;;;;;kBAWxB,mCAAmC;UAC3C,wBAAwB;;;;;;;;;AAUlC,IAAM,6BAAqC;iBAC1B,wBAAwB;;;;;;;;;AAUzC,IAAa,wBAAwB;AAerC,IAAM,gCAAwC;kBAC5B,sBAAsB;;;;;;;;;;;;;;;;;;;;;AAsBxC,SAAgB,kBAAkB,aAA6B;CAC7D,OAAO,GAAG,YAAY;AACxB;;;;;AAMA,SAAgB,kBAAkB,aAA2B;CAC3D,aAAa,kBAAkB,WAAW,CAAC;AAC7C;;;;;;;;;AAsFA,SAAgB,gBACd,IACA,aACA,MACU;CACV,OAAO,IAAI,SACT,GAAG,YAAY,aAAa,eAAe,GAC3C,kBAAkB,WAAW,GAC7B,EAAC,UAAU,KAAK,SAAQ,CAC1B;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,SAAgB,sBAAsB,IAAoB;CACxD,GAAG,OAAO,2BAA2B;CACrC,GAAG,OAAO,sBAAsB;CAChC,GAAG,OAAO,uBAAuB;CACjC,GAAG,OAAO,qBAAqB;CAC/B,GAAG,OAAO,sBAAsB;AAClC;AAEA,SAAS,eAAe,IAAsB;CAC5C,MAAM,CAAC,EAAC,aAAa,UAAS,GAAG,OAA8B,aAAa;CAC5E,OAAO;AACT;AAqCA,SAAgB,0BACd,IACA,aACA,QACgE;CAGhE,MAAM,EAAC,IAAI,QAAQ,iBAAgB,4BACjC,IACA,aACA,QAJa,qBAAqB,IAAI,aAAa,QADrC,WAAW,kBAAkB,WAAW,CACK,CAK3D,CACF;CACA,OAAO;EAAC;EAAI;EAAQ;CAAY;AAClC;AAEA,SAAS,qBACP,IACA,aACA,QACA,qBACiB;CACjB,IAAI;EACF,OAAO,iBAAiB,IAAI,aAAa,QAAQ,mBAAmB;CACtE,SAAS,GAAG;EACV,IAAI,aAAa,0BACf,OAAO,iBAAiB,IAAI,aAAa,QAAQ,CAAC;EAEpD,IAAI,CAAC,mBAAmB,CAAC,GACvB,MAAM;EAER,MAAM,OAAO,kBAAkB,WAAW;EAC1C,+BAA+B,IAAI,cAAc,MAAM,CAAC;EACxD,GAAG,QAAQ,4CAA4C,EACrD,iBAAiB,EAAC,KAAI,EACxB,CAAC;EACD,OAAO,iBAAiB,IAAI,aAAa,MAAM;CACjD;AACF;;;;;;;;;;;;;;;;;;;;;AAsBA,SAAS,4BACP,IACA,aACA,QACA,QACiB;CACjB,IAAI,OAAO,eAAe,yBACxB,OAAO;CAET,MAAM,OAAO,kBAAkB,WAAW;CAC1C,GAAG,OAAO,iEAAiE,EACzE,iBAAiB;EAAC;EAAM,YAAY,OAAO;CAAU,EACvD,CAAC;CACD,OAAO,GAAG,MAAM;CAChB,MAAM,UAAU,iBAAiB,IAAI,aAAa,MAAM;CACxD,IAAI,QAAQ,eAAe,yBACzB,GAAG,QACD,iGAEA,EAAC,iBAAiB;EAAC;EAAM,YAAY,QAAQ;CAAU,EAAC,CAC1D;CAEF,OAAO;AACT;;AAGA,SAAS,iBACP,IACA,aACA,QACA,UACiB;CACjB,IAAI,UACF,GAAG,OACD,mFACA,EACE,wBAAwB;EACtB,QAAQ,SAAS;EACjB,MAAM,SAAS;EACf,gBAAgB,SAAS;CAC3B,EACF,CACF;CAEF,kBAAkB,WAAW;CAU7B,OAAO;EAAC,GAPO,iBACb,IACA,aACA,QACA,OACA,UAAU,MAED;EAAQ,cAAc;CAAI;AACvC;;;;;;AAOA,SAAgB,6BACd,IACA,aACA,QACA,UAC6D;CAC7D,MAAM,EAAC,IAAI,WAAU,iBACnB,IACA,mBACM,QACN,QACF;CACA,OAAO;EAAC;EAAI;EAAQ,cAAc;CAAI;AACxC;AAUA,SAAS,iBACP,IACA,aACA,QACA,qBACA,cACiB;CACjB,MAAM,KAAK,gBAAgB,IAAI,aAAa,EAAC,UAAU,MAAK,CAAC;CAC7D,IAAI;EACF,sBAAsB,EAAE;EACxB,MAAM,aAAa,eAAe,EAAE;EACpC,OAAO;GACL;GACA,QAAQ,mBAAmB,IAAI,IAAI,OAAO,EAAE,GAAG;IAC7C,+BAA+B;IAC/B;GACF,CAAC;GACD,cAAc;GACd;EACF;CACF,SAAS,GAAG;EAGV,GAAG,MAAM;EACT,MAAM;CACR;AACF;;;;;;;AAeA,IAAa,8BAA8B;AAC3C,IAAa,+BAA+B;AAE5C,IAAa,2BAAb,cAA8C,MAAM;CAClD,OAAyB;CACzB;CACA;CACA;CAEA,YACE,QACA,MACA,gBACA;EACA,MAAM,gCAAgC,OAAO,SAAS;EACtD,KAAK,SAAS;EACd,KAAK,OAAO;EACZ,KAAK,iBAAiB;CACxB;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8DA,SAAgB,mBACd,IACA,IACA,QACA,OAAyB,CAAC,GACT;CACjB,GAAG,KAAK,iBAAiB;CACzB,IAAI;EACF,IAAI,KAAK,+BAA+B;GACtC,MAAM,WAAW,gBAAgB,IAAI,MAAM;GAC3C,IAAI,UACF,MAAM;EAEV;EACA,MAAM,SAAS,KAAK,eAChB,OAAO,IAAI,IAAI,QAAQ,KAAK,YAAY,IACxC,UAAU,IAAI,IAAI,MAAM;EAC5B,GAAG,KAAK,QAAQ;EAChB,OAAO;CACT,SAAS,GAAG;EACV,IAAI,GAAG,eACL,GAAG,KAAK,UAAU;EAEpB,MAAM;CACR;AACF;;;;;;AAOA,SAAS,gBACP,IACA,QACsC;CACtC,MAAM,OAAO,oBAAoB,IAAI,OAAO,QAAQ;CACpD,IAAI,SAAS,KAAA,GACX,OAAO,IAAI,yBAAyB,IAAI;CAG1C,MAAM,OAAO,kBAAkB,EAAE;CACjC,MAAM,EAAC,oBAAmB;CAC1B,IAAI,SAAS,QAAQ,OAAO,iBAC1B,OAAO,IAAI,yBAAyB,KAAK;CAE3C,IAAI,SAAS,iBACX;CAUF,IAPiB,GACd,QAAgB;;;;KAIhB,CAAC,CACD,IAA6B,eAC5B,MAAa,KAAA,GACf,OAAO,IAAI,yBAAyB,KAAK;CAG3C,MAAM,EAAC,MAAM,mBAAkB,GAC5B,QAAgB;;;;0CAIqB,wBAAwB;;;;KAI7D,CAAC,CACD,IAA4C,eAAe;CAC9D,OAAO,OAAA,OACL,iBAAA,WACE,IAAI,yBAAyB,sBAAsB,MAAM,cAAc,IACvE,KAAA;AACN;AAEA,SAAS,UACP,IACA,IACA,QACiB;CACjB,MAAM,OAAO,oBAAoB,IAAI,OAAO,QAAQ;CACpD,IAAI,SAAS,KAAA,GACX,OAAO,OAAO,IAAI,IAAI,QAAQ,IAAI;CAGpC,MAAM,EAAC,oBAAmB;CAC1B,IAAI,OAAO,kBAAkB,EAAE;CAC/B,IAAI,OAAO;CACX,IAAI,SAAS,QAAQ,OAAO,iBAAiB;EAC3C,OAAO,GACJ,QAAgB;uBACA,wBAAwB;OACxC,CAAC,CACD,IAAI,eAAe,CAAC,CAAC;EACxB,OAAO,kBAAkB,EAAE;CAC7B;CAEA,IAAI,SAAS,iBACX,OAAO,OAAO,IAAI,IAAI,QAAQ,KAAK;CAErC,IAAI,OAAO,GAAG;EAMZ,eAAe,IAAI,OAAO,OAAO;EACjC,GAAG,OAAO,6DAA6D,EACrE,0BAA0B;GAAC;GAAM;GAAM,SAAS,aAAa,MAAM;EAAC,EACtE,CAAC;EACD,OAAO;GAAC,QAAQ;GAAa;GAAM;GAAM,cAAc;EAAI;CAC7D;CACA,GAAG,QAAQ,gDAAgD,EACzD,0BAA0B,EAAC,KAAI,EACjC,CAAC;CACD,OAAO;EAAC,QAAQ;EAAQ;EAAM,cAAc;CAAK;AACnD;;;;;;;;AASA,SAAgB,oBACd,IACA,UAC0B;CAC1B,IACE,CAAC,YAAY,IAAA,uBAA2B,KACxC,CAAC,YAAY,IAAA,qBAAyB,GAEtC,OAAO;CAKT,MAAM,UAAU,GACb,QAAgB;qCACgB,sBAAsB;KACtD,CAAC,CACD,IAAyC;CAC5C,IAAI,YAAY,KAAA,GACd,OAAO;CAET,IAAI,QAAQ,kBAAA,GACV,OAAO;CAMT,IACE,CAAC,YAAY,IAAA,8BAAmC,KAChD,CAAC,YAAY,IAAA,4BAAgC,GAE7C,OAAO;CAET,MAAM,EAAC,OAAO,YAAY,cAAa,kBAAkB,EAAE;CAC3D,IACE,UAAU,SAAS,SACnB,eAAe,SAAS,cACxB,cAAc,SAAS,WAEvB,OAAO;AAGX;AAEA,SAAS,OACP,IACA,IACA,QACA,QACiB;CACjB,MAAM,EAAC,UAAU,iBAAiB,UAAS;CAI3C,GAAG,KAAa;4BACU,wBAAwB;4BACxB,sBAAsB;MAC5C,8BAA8B;MAC9B,gCAAgC;MAChC,8BAA8B;MAC9B,gCAAgC;GACnC;CACD,GAAG,QAAgB;mBACF,sBAAsB;;;;;GAKtC,CAAC,CAAC,IAAI;EACL,OAAO,SAAS;EAChB,YAAY,SAAS;EACrB,WAAW,SAAS;EACpB,eAAA;EACA,YAAY;EACZ,eAAe;CACjB,CAAC;CACD,oBAAoB,IAAI,MAAM;CAK9B,eAAe,IAAI,OAAO,OAAO;CAEjC,GAAG,OAAO,kCAAkC,EAC1C,0BAA0B;EACxB;EACA,MAAM;EACN,GAAG;EACH,eAAA;EACA,YAAY;EACZ,SAAS,aAAa,MAAM;CAC9B,EACF,CAAC;CACD,OAAO;EACL,QAAQ;EACR,MAAM;EACN;EACA,cAAc;CAChB;AACF;;;;;;AAOA,SAAS,oBAAoB,IAAc,QAA+B;CACxE,MAAM,cAAc;CACpB,MAAM,eAAe;CACrB,GAAG,QAAQ,0BAA0B,CAAC,CAAC,IAAI;EACzC,WAAW,OAAO;EAClB,YAAY,gCACV,OAAO,iBACP,SACA,WACF;EACA;EACA,aAAa,gCACX,OAAO,iBACP,UACA,cACA,OAAO,iBACP,IACF;EACA;EACA,aAAa,OAAO;CACtB,CAAC;AACH;;AAGA,SAAgB,kBAAkB,IAA6B;CAC7D,MAAM,OAAO,GACV,QAAgB;;;gBAGL,sBAAsB;KACjC,CAAC,CACD,IAA+B;CAClC,IAAI,SAAS,KAAA,GACX,MAAM,IAAI,MAAM,uCAAuC;CAEzD,OAAO;AACT;;;;;AAMA,SAAgB,kBAAkB,IAA6B;CAC7D,MAAM,EAAC,SAAQ,GACZ,QAAgB;gDAC2B,wBAAwB;KACnE,CAAC,CACD,IAA2B;CAC9B,OAAO;AACT;;;;;;AAOA,SAAgB,oBAAoB,IAA+B;CACjE,OAAO,GACJ,QAAgB;;yCAEoB,wBAAwB;;yCAExB,wBAAwB;;KAE5D,CAAC,CACD,IAAqB;AAC1B;;;;;AAWA,SAAS,aAAa,EAAC,WAA2B;CAChD,OAAO;EACL,eAAe,QAAQ,cAAc;EACrC,aAAa,QAAQ,YAAY;CACnC;AACF;AAEA,SAAS,YAAY,IAAc,OAAwB;CACzD,OACE,GACG,QAAgB;;OAEhB,CAAC,CACD,IAA6B,KAAK,MAAM,KAAA;AAE/C"}

package/out/zero-cache/src/services/replicator/change-log-stream-writer.js

@@ -143,6 +143,6 @@ var ChangeLogStreamWriter = class {
 	}
 };
 //#endregion
-export { ChangeLogStreamInvariantError, ChangeLogStreamWriter };
+export { ChangeLogStreamWriter };
 
 //# sourceMappingURL=change-log-stream-writer.js.map

package/out/zero-cache/src/services/replicator/change-log-stream-writer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-log-stream-writer.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/change-log-stream-writer.ts"],"sourcesContent":["import type {Statement} from '../../../../zqlite/src/db.ts';\nimport type {StatementRunner} from '../../db/statements.ts';\nimport {\n  isSchemaChange,\n  type DataOrSchemaChange,\n} from '../change-source/protocol/current/data.ts';\nimport {extractChangeSubstring} from '../change-streamer/change-log-codec.ts';\nimport {ChangeLogTransactionHasher} from '../change-streamer/change-log-transaction-hash.ts';\nimport {ChangeLogCookieWriter, type CookieOpTag} from './change-log-cookies.ts';\nimport {\n  CHANGE_LOG_STREAM_TABLE,\n  estimateChangeLogStreamRowBytes,\n} from './change-log-db.ts';\n\nexport type ChangeLogStreamTransactionStats = {\n  readonly rows: number;\n  readonly estimatedBytes: number;\n  /** Present only for transactions selected for shadow-write validation. */\n  readonly hash?: string | undefined;\n  /**\n   * The cookie mutations this transaction's schema changes folded in, in stream\n   * order. Empty for the overwhelming majority of transactions.\n   */\n  readonly cookieMutations: readonly CookieOpTag[];\n};\n\nexport class ChangeLogStreamInvariantError extends Error {\n  override readonly name = 'ChangeLogStreamInvariantError';\n}\n\nfunction assertInvariant(\n  condition: unknown,\n  message: string,\n): asserts condition {\n  if (!condition) {\n    throw new ChangeLogStreamInvariantError(message);\n  }\n}\n\n/**\n * Appends the canonical downstream stream to the change-log database, which is\n * a separate file from the replica, and folds its schema changes into the log's\n * cookie jar (`change-log-cookies.ts`) as it goes.\n *\n * This class owns the change log's transaction, and its transaction alone. It is\n * driven from the change-streamer's stream loop, which commits it before\n * forwarding a transaction's `commit` message; see `SQLiteChangeLogWriter` in\n * `change-streamer/sqlite-change-log-writer.ts` for the ordering that makes a\n * hole unreachable, and for the fail-soft policy that wraps every call here.\n *\n * Because the cookies are written in that same transaction, they are atomic with\n * the head and {@link rollback} discards them with the rows they were folded\n * from. Neither needs any handling of its own.\n */\nexport class ChangeLogStreamWriter {\n  readonly #db: StatementRunner;\n  readonly #insertChange: Statement;\n  readonly #insertCommit: Statement;\n  readonly #cookies: ChangeLogCookieWriter;\n\n  #watermark: string | undefined;\n  #pos = 0;\n  #estimatedBytes = 0;\n  #hasher: ChangeLogTransactionHasher | undefined;\n  #cookieMutations: CookieOpTag[] = [];\n\n  constructor(db: StatementRunner) {\n    this.#db = db;\n    this.#insertChange = db.db.prepare(/*sql*/ `\n      INSERT INTO \"${CHANGE_LOG_STREAM_TABLE}\"\n        (\"watermark\", \"pos\", \"tag\", \"estimatedBytes\", \"change\")\n        VALUES (?, ?, ?, ?, ?)\n    `);\n    this.#insertCommit = db.db.prepare(/*sql*/ `\n      INSERT INTO \"${CHANGE_LOG_STREAM_TABLE}\"\n        (\"watermark\", \"pos\", \"tag\", \"estimatedBytes\", \"change\", \"precommit\",\n         \"writeTimeMs\")\n        VALUES (?, ?, 'commit', ?, ?, ?, ?)\n    `);\n    this.#cookies = new ChangeLogCookieWriter(db.db);\n  }\n\n  begin(\n    watermark: string,\n    json: string,\n    storedChange = extractChangeSubstring(json, 'begin'),\n    hash = true,\n  ): void {\n    assertInvariant(\n      this.#watermark === undefined,\n      `change-log stream transaction already open at ${this.#watermark}`,\n    );\n    // This process is the sole writer of the change-log database, so the write\n    // lock is taken up front rather than risking a failed upgrade from a\n    // deferred transaction partway through the append.\n    this.#db.beginImmediate();\n    this.#watermark = watermark;\n    this.#pos = 0;\n    const change = storedChange;\n    const estimatedBytes = estimateChangeLogStreamRowBytes(\n      watermark,\n      'begin',\n      change,\n    );\n    this.#insertChange.run(\n      watermark,\n      this.#pos,\n      'begin',\n      estimatedBytes,\n      change,\n    );\n    if (hash) {\n      this.#hasher = new ChangeLogTransactionHasher();\n      this.#hasher.add({\n        watermark,\n        pos: this.#pos,\n        tag: 'begin',\n        change,\n        precommit: null,\n      });\n    }\n    this.#estimatedBytes = estimatedBytes;\n  }\n\n  /**\n   * Appends one data or schema change. The parsed message is taken rather than\n   * just its tag because a schema change also mutates the cookie jar, and the\n   * caller already holds it — so the fold costs no re-parse.\n   *\n   * The cookie statements run here, i.e. inside this transaction and in stream\n   * order relative to the rows they were folded from. The Postgres change log\n   * has to flush its buffered batch to get the same ordering\n   * (`Storer.#processQueue()`); SQLite gets it from statement order.\n   */\n  append(\n    json: string,\n    change: DataOrSchemaChange,\n    storedChange = extractChangeSubstring(json, change.tag),\n  ): void {\n    const watermark = this.#requireWatermark();\n    const {tag} = change;\n    const stored = storedChange;\n    const estimatedBytes = estimateChangeLogStreamRowBytes(\n      watermark,\n      tag,\n      stored,\n    );\n    this.#insertChange.run(watermark, ++this.#pos, tag, estimatedBytes, stored);\n    this.#hasher?.add({\n      watermark,\n      pos: this.#pos,\n      tag,\n      change: stored,\n      precommit: null,\n    });\n    this.#estimatedBytes += estimatedBytes;\n\n    if (isSchemaChange(change)) {\n      for (const op of this.#cookies.apply(change)) {\n        this.#cookieMutations.push(op.op);\n      }\n    }\n  }\n\n  commit(\n    watermark: string,\n    json: string,\n    writeTimeMs: number,\n    storedChange = extractChangeSubstring(json, 'commit'),\n  ): ChangeLogStreamTransactionStats {\n    const precommit = this.#requireWatermark();\n    assertInvariant(\n      watermark === precommit,\n      `change-log stream commit ${watermark} does not match begin ${precommit}`,\n    );\n    const change = storedChange;\n    const estimatedBytes = estimateChangeLogStreamRowBytes(\n      watermark,\n      'commit',\n      change,\n      precommit,\n      true,\n    );\n    this.#insertCommit.run(\n      watermark,\n      ++this.#pos,\n      estimatedBytes,\n      change,\n      precommit,\n      writeTimeMs,\n    );\n    const hasher = this.#hasher;\n    hasher?.add({\n      watermark,\n      pos: this.#pos,\n      tag: 'commit',\n      change,\n      precommit,\n    });\n    const stats = {\n      rows: this.#pos + 1,\n      estimatedBytes: this.#estimatedBytes + estimatedBytes,\n      hash: hasher?.digest(),\n      cookieMutations: this.#cookieMutations,\n    };\n    // The log is durable at `watermark` from here on, and nothing that can\n    // advance the watermark the stream would resume from has run yet. A crash in\n    // that window leaves a transaction the resumed stream re-delivers, which\n    // reconciliation truncates.\n    this.#db.commit();\n    this.#reset();\n    return stats;\n  }\n\n  /**\n   * Discards the open transaction, if any. Safe to call when none is open — the\n   * caller rolls back on an interrupted change stream, which can arrive between\n   * transactions as well as inside one.\n   */\n  rollback(): void {\n    const {inTransaction} = this.#db.db;\n    this.#reset();\n    if (inTransaction) {\n      this.#db.rollback();\n    }\n  }\n\n  #reset(): void {\n    this.#watermark = undefined;\n    this.#pos = 0;\n    this.#estimatedBytes = 0;\n    this.#hasher = undefined;\n    // A fresh array rather than a truncation: the committed stats hold a\n    // reference to the previous one.\n    this.#cookieMutations = [];\n  }\n\n  #requireWatermark(): string {\n    assertInvariant(\n      this.#watermark !== undefined,\n      'change-log stream message received outside of a transaction',\n    );\n    return this.#watermark;\n  }\n}\n"],"mappings":";;;;;;AA0BA,IAAa,gCAAb,cAAmD,MAAM;CACvD,OAAyB;AAC3B;AAEA,SAAS,gBACP,WACA,SACmB;CACnB,IAAI,CAAC,WACH,MAAM,IAAI,8BAA8B,OAAO;AAEnD;;;;;;;;;;;;;;;;AAiBA,IAAa,wBAAb,MAAmC;CACjC;CACA;CACA;CACA;CAEA;CACA,OAAO;CACP,kBAAkB;CAClB;CACA,mBAAkC,CAAC;CAEnC,YAAY,IAAqB;EAC/B,KAAK,MAAM;EACX,KAAK,gBAAgB,GAAG,GAAG,QAAgB;qBAC1B,wBAAwB;;;KAGxC;EACD,KAAK,gBAAgB,GAAG,GAAG,QAAgB;qBAC1B,wBAAwB;;;;KAIxC;EACD,KAAK,WAAW,IAAI,sBAAsB,GAAG,EAAE;CACjD;CAEA,MACE,WACA,MACA,eAAe,uBAAuB,MAAM,OAAO,GACnD,OAAO,MACD;EACN,gBACE,KAAK,eAAe,KAAA,GACpB,iDAAiD,KAAK,YACxD;EAIA,KAAK,IAAI,eAAe;EACxB,KAAK,aAAa;EAClB,KAAK,OAAO;EACZ,MAAM,SAAS;EACf,MAAM,iBAAiB,gCACrB,WACA,SACA,MACF;EACA,KAAK,cAAc,IACjB,WACA,KAAK,MACL,SACA,gBACA,MACF;EACA,IAAI,MAAM;GACR,KAAK,UAAU,IAAI,2BAA2B;GAC9C,KAAK,QAAQ,IAAI;IACf;IACA,KAAK,KAAK;IACV,KAAK;IACL;IACA,WAAW;GACb,CAAC;EACH;EACA,KAAK,kBAAkB;CACzB;;;;;;;;;;;CAYA,OACE,MACA,QACA,eAAe,uBAAuB,MAAM,OAAO,GAAG,GAChD;EACN,MAAM,YAAY,KAAK,kBAAkB;EACzC,MAAM,EAAC,QAAO;EACd,MAAM,SAAS;EACf,MAAM,iBAAiB,gCACrB,WACA,KACA,MACF;EACA,KAAK,cAAc,IAAI,WAAW,EAAE,KAAK,MAAM,KAAK,gBAAgB,MAAM;EAC1E,KAAK,SAAS,IAAI;GAChB;GACA,KAAK,KAAK;GACV;GACA,QAAQ;GACR,WAAW;EACb,CAAC;EACD,KAAK,mBAAmB;EAExB,IAAI,eAAe,MAAM,GACvB,KAAK,MAAM,MAAM,KAAK,SAAS,MAAM,MAAM,GACzC,KAAK,iBAAiB,KAAK,GAAG,EAAE;CAGtC;CAEA,OACE,WACA,MACA,aACA,eAAe,uBAAuB,MAAM,QAAQ,GACnB;EACjC,MAAM,YAAY,KAAK,kBAAkB;EACzC,gBACE,cAAc,WACd,4BAA4B,UAAU,wBAAwB,WAChE;EACA,MAAM,SAAS;EACf,MAAM,iBAAiB,gCACrB,WACA,UACA,QACA,WACA,IACF;EACA,KAAK,cAAc,IACjB,WACA,EAAE,KAAK,MACP,gBACA,QACA,WACA,WACF;EACA,MAAM,SAAS,KAAK;EACpB,QAAQ,IAAI;GACV;GACA,KAAK,KAAK;GACV,KAAK;GACL;GACA;EACF,CAAC;EACD,MAAM,QAAQ;GACZ,MAAM,KAAK,OAAO;GAClB,gBAAgB,KAAK,kBAAkB;GACvC,MAAM,QAAQ,OAAO;GACrB,iBAAiB,KAAK;EACxB;EAKA,KAAK,IAAI,OAAO;EAChB,KAAK,OAAO;EACZ,OAAO;CACT;;;;;;CAOA,WAAiB;EACf,MAAM,EAAC,kBAAiB,KAAK,IAAI;EACjC,KAAK,OAAO;EACZ,IAAI,eACF,KAAK,IAAI,SAAS;CAEtB;CAEA,SAAe;EACb,KAAK,aAAa,KAAA;EAClB,KAAK,OAAO;EACZ,KAAK,kBAAkB;EACvB,KAAK,UAAU,KAAA;EAGf,KAAK,mBAAmB,CAAC;CAC3B;CAEA,oBAA4B;EAC1B,gBACE,KAAK,eAAe,KAAA,GACpB,6DACF;EACA,OAAO,KAAK;CACd;AACF"}
+{"version":3,"file":"change-log-stream-writer.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/change-log-stream-writer.ts"],"sourcesContent":["import type {Statement} from '../../../../zqlite/src/db.ts';\nimport type {StatementRunner} from '../../db/statements.ts';\nimport {\n  isSchemaChange,\n  type DataOrSchemaChange,\n} from '../change-source/protocol/current/data.ts';\nimport {extractChangeSubstring} from '../change-streamer/change-log-codec.ts';\nimport {ChangeLogTransactionHasher} from '../change-streamer/change-log-transaction-hash.ts';\nimport {ChangeLogCookieWriter, type CookieOpTag} from './change-log-cookies.ts';\nimport {\n  CHANGE_LOG_STREAM_TABLE,\n  estimateChangeLogStreamRowBytes,\n} from './change-log-db.ts';\n\nexport type ChangeLogStreamTransactionStats = {\n  readonly rows: number;\n  readonly estimatedBytes: number;\n  /** Present only for transactions selected for shadow-write validation. */\n  readonly hash?: string | undefined;\n  /**\n   * The cookie mutations this transaction's schema changes folded in, in stream\n   * order. Empty for the overwhelming majority of transactions.\n   */\n  readonly cookieMutations: readonly CookieOpTag[];\n};\n\nclass ChangeLogStreamInvariantError extends Error {\n  override readonly name = 'ChangeLogStreamInvariantError';\n}\n\nfunction assertInvariant(\n  condition: unknown,\n  message: string,\n): asserts condition {\n  if (!condition) {\n    throw new ChangeLogStreamInvariantError(message);\n  }\n}\n\n/**\n * Appends the canonical downstream stream to the change-log database, which is\n * a separate file from the replica, and folds its schema changes into the log's\n * cookie jar (`change-log-cookies.ts`) as it goes.\n *\n * This class owns the change log's transaction, and its transaction alone. It is\n * driven from the change-streamer's stream loop, which commits it before\n * forwarding a transaction's `commit` message; see `SQLiteChangeLogWriter` in\n * `change-streamer/sqlite-change-log-writer.ts` for the ordering that makes a\n * hole unreachable, and for the fail-soft policy that wraps every call here.\n *\n * Because the cookies are written in that same transaction, they are atomic with\n * the head and {@link rollback} discards them with the rows they were folded\n * from. Neither needs any handling of its own.\n */\nexport class ChangeLogStreamWriter {\n  readonly #db: StatementRunner;\n  readonly #insertChange: Statement;\n  readonly #insertCommit: Statement;\n  readonly #cookies: ChangeLogCookieWriter;\n\n  #watermark: string | undefined;\n  #pos = 0;\n  #estimatedBytes = 0;\n  #hasher: ChangeLogTransactionHasher | undefined;\n  #cookieMutations: CookieOpTag[] = [];\n\n  constructor(db: StatementRunner) {\n    this.#db = db;\n    this.#insertChange = db.db.prepare(/*sql*/ `\n      INSERT INTO \"${CHANGE_LOG_STREAM_TABLE}\"\n        (\"watermark\", \"pos\", \"tag\", \"estimatedBytes\", \"change\")\n        VALUES (?, ?, ?, ?, ?)\n    `);\n    this.#insertCommit = db.db.prepare(/*sql*/ `\n      INSERT INTO \"${CHANGE_LOG_STREAM_TABLE}\"\n        (\"watermark\", \"pos\", \"tag\", \"estimatedBytes\", \"change\", \"precommit\",\n         \"writeTimeMs\")\n        VALUES (?, ?, 'commit', ?, ?, ?, ?)\n    `);\n    this.#cookies = new ChangeLogCookieWriter(db.db);\n  }\n\n  begin(\n    watermark: string,\n    json: string,\n    storedChange = extractChangeSubstring(json, 'begin'),\n    hash = true,\n  ): void {\n    assertInvariant(\n      this.#watermark === undefined,\n      `change-log stream transaction already open at ${this.#watermark}`,\n    );\n    // This process is the sole writer of the change-log database, so the write\n    // lock is taken up front rather than risking a failed upgrade from a\n    // deferred transaction partway through the append.\n    this.#db.beginImmediate();\n    this.#watermark = watermark;\n    this.#pos = 0;\n    const change = storedChange;\n    const estimatedBytes = estimateChangeLogStreamRowBytes(\n      watermark,\n      'begin',\n      change,\n    );\n    this.#insertChange.run(\n      watermark,\n      this.#pos,\n      'begin',\n      estimatedBytes,\n      change,\n    );\n    if (hash) {\n      this.#hasher = new ChangeLogTransactionHasher();\n      this.#hasher.add({\n        watermark,\n        pos: this.#pos,\n        tag: 'begin',\n        change,\n        precommit: null,\n      });\n    }\n    this.#estimatedBytes = estimatedBytes;\n  }\n\n  /**\n   * Appends one data or schema change. The parsed message is taken rather than\n   * just its tag because a schema change also mutates the cookie jar, and the\n   * caller already holds it — so the fold costs no re-parse.\n   *\n   * The cookie statements run here, i.e. inside this transaction and in stream\n   * order relative to the rows they were folded from. The Postgres change log\n   * has to flush its buffered batch to get the same ordering\n   * (`Storer.#processQueue()`); SQLite gets it from statement order.\n   */\n  append(\n    json: string,\n    change: DataOrSchemaChange,\n    storedChange = extractChangeSubstring(json, change.tag),\n  ): void {\n    const watermark = this.#requireWatermark();\n    const {tag} = change;\n    const stored = storedChange;\n    const estimatedBytes = estimateChangeLogStreamRowBytes(\n      watermark,\n      tag,\n      stored,\n    );\n    this.#insertChange.run(watermark, ++this.#pos, tag, estimatedBytes, stored);\n    this.#hasher?.add({\n      watermark,\n      pos: this.#pos,\n      tag,\n      change: stored,\n      precommit: null,\n    });\n    this.#estimatedBytes += estimatedBytes;\n\n    if (isSchemaChange(change)) {\n      for (const op of this.#cookies.apply(change)) {\n        this.#cookieMutations.push(op.op);\n      }\n    }\n  }\n\n  commit(\n    watermark: string,\n    json: string,\n    writeTimeMs: number,\n    storedChange = extractChangeSubstring(json, 'commit'),\n  ): ChangeLogStreamTransactionStats {\n    const precommit = this.#requireWatermark();\n    assertInvariant(\n      watermark === precommit,\n      `change-log stream commit ${watermark} does not match begin ${precommit}`,\n    );\n    const change = storedChange;\n    const estimatedBytes = estimateChangeLogStreamRowBytes(\n      watermark,\n      'commit',\n      change,\n      precommit,\n      true,\n    );\n    this.#insertCommit.run(\n      watermark,\n      ++this.#pos,\n      estimatedBytes,\n      change,\n      precommit,\n      writeTimeMs,\n    );\n    const hasher = this.#hasher;\n    hasher?.add({\n      watermark,\n      pos: this.#pos,\n      tag: 'commit',\n      change,\n      precommit,\n    });\n    const stats = {\n      rows: this.#pos + 1,\n      estimatedBytes: this.#estimatedBytes + estimatedBytes,\n      hash: hasher?.digest(),\n      cookieMutations: this.#cookieMutations,\n    };\n    // The log is durable at `watermark` from here on, and nothing that can\n    // advance the watermark the stream would resume from has run yet. A crash in\n    // that window leaves a transaction the resumed stream re-delivers, which\n    // reconciliation truncates.\n    this.#db.commit();\n    this.#reset();\n    return stats;\n  }\n\n  /**\n   * Discards the open transaction, if any. Safe to call when none is open — the\n   * caller rolls back on an interrupted change stream, which can arrive between\n   * transactions as well as inside one.\n   */\n  rollback(): void {\n    const {inTransaction} = this.#db.db;\n    this.#reset();\n    if (inTransaction) {\n      this.#db.rollback();\n    }\n  }\n\n  #reset(): void {\n    this.#watermark = undefined;\n    this.#pos = 0;\n    this.#estimatedBytes = 0;\n    this.#hasher = undefined;\n    // A fresh array rather than a truncation: the committed stats hold a\n    // reference to the previous one.\n    this.#cookieMutations = [];\n  }\n\n  #requireWatermark(): string {\n    assertInvariant(\n      this.#watermark !== undefined,\n      'change-log stream message received outside of a transaction',\n    );\n    return this.#watermark;\n  }\n}\n"],"mappings":";;;;;;AA0BA,IAAM,gCAAN,cAA4C,MAAM;CAChD,OAAyB;AAC3B;AAEA,SAAS,gBACP,WACA,SACmB;CACnB,IAAI,CAAC,WACH,MAAM,IAAI,8BAA8B,OAAO;AAEnD;;;;;;;;;;;;;;;;AAiBA,IAAa,wBAAb,MAAmC;CACjC;CACA;CACA;CACA;CAEA;CACA,OAAO;CACP,kBAAkB;CAClB;CACA,mBAAkC,CAAC;CAEnC,YAAY,IAAqB;EAC/B,KAAK,MAAM;EACX,KAAK,gBAAgB,GAAG,GAAG,QAAgB;qBAC1B,wBAAwB;;;KAGxC;EACD,KAAK,gBAAgB,GAAG,GAAG,QAAgB;qBAC1B,wBAAwB;;;;KAIxC;EACD,KAAK,WAAW,IAAI,sBAAsB,GAAG,EAAE;CACjD;CAEA,MACE,WACA,MACA,eAAe,uBAAuB,MAAM,OAAO,GACnD,OAAO,MACD;EACN,gBACE,KAAK,eAAe,KAAA,GACpB,iDAAiD,KAAK,YACxD;EAIA,KAAK,IAAI,eAAe;EACxB,KAAK,aAAa;EAClB,KAAK,OAAO;EACZ,MAAM,SAAS;EACf,MAAM,iBAAiB,gCACrB,WACA,SACA,MACF;EACA,KAAK,cAAc,IACjB,WACA,KAAK,MACL,SACA,gBACA,MACF;EACA,IAAI,MAAM;GACR,KAAK,UAAU,IAAI,2BAA2B;GAC9C,KAAK,QAAQ,IAAI;IACf;IACA,KAAK,KAAK;IACV,KAAK;IACL;IACA,WAAW;GACb,CAAC;EACH;EACA,KAAK,kBAAkB;CACzB;;;;;;;;;;;CAYA,OACE,MACA,QACA,eAAe,uBAAuB,MAAM,OAAO,GAAG,GAChD;EACN,MAAM,YAAY,KAAK,kBAAkB;EACzC,MAAM,EAAC,QAAO;EACd,MAAM,SAAS;EACf,MAAM,iBAAiB,gCACrB,WACA,KACA,MACF;EACA,KAAK,cAAc,IAAI,WAAW,EAAE,KAAK,MAAM,KAAK,gBAAgB,MAAM;EAC1E,KAAK,SAAS,IAAI;GAChB;GACA,KAAK,KAAK;GACV;GACA,QAAQ;GACR,WAAW;EACb,CAAC;EACD,KAAK,mBAAmB;EAExB,IAAI,eAAe,MAAM,GACvB,KAAK,MAAM,MAAM,KAAK,SAAS,MAAM,MAAM,GACzC,KAAK,iBAAiB,KAAK,GAAG,EAAE;CAGtC;CAEA,OACE,WACA,MACA,aACA,eAAe,uBAAuB,MAAM,QAAQ,GACnB;EACjC,MAAM,YAAY,KAAK,kBAAkB;EACzC,gBACE,cAAc,WACd,4BAA4B,UAAU,wBAAwB,WAChE;EACA,MAAM,SAAS;EACf,MAAM,iBAAiB,gCACrB,WACA,UACA,QACA,WACA,IACF;EACA,KAAK,cAAc,IACjB,WACA,EAAE,KAAK,MACP,gBACA,QACA,WACA,WACF;EACA,MAAM,SAAS,KAAK;EACpB,QAAQ,IAAI;GACV;GACA,KAAK,KAAK;GACV,KAAK;GACL;GACA;EACF,CAAC;EACD,MAAM,QAAQ;GACZ,MAAM,KAAK,OAAO;GAClB,gBAAgB,KAAK,kBAAkB;GACvC,MAAM,QAAQ,OAAO;GACrB,iBAAiB,KAAK;EACxB;EAKA,KAAK,IAAI,OAAO;EAChB,KAAK,OAAO;EACZ,OAAO;CACT;;;;;;CAOA,WAAiB;EACf,MAAM,EAAC,kBAAiB,KAAK,IAAI;EACjC,KAAK,OAAO;EACZ,IAAI,eACF,KAAK,IAAI,SAAS;CAEtB;CAEA,SAAe;EACb,KAAK,aAAa,KAAA;EAClB,KAAK,OAAO;EACZ,KAAK,kBAAkB;EACvB,KAAK,UAAU,KAAA;EAGf,KAAK,mBAAmB,CAAC;CAC3B;CAEA,oBAA4B;EAC1B,gBACE,KAAK,eAAe,KAAA,GACpB,6DACF;EACA,OAAO,KAAK;CACd;AACF"}

package/out/zero-cache/src/services/replicator/drain-handler.js

@@ -1,5 +1,5 @@
+import { resolver } from "../../../../shared/src/resolver.js";
 import { readHead } from "../../types/streams.js";
-import { resolver } from "@rocicorp/resolver";
 //#region ../zero-cache/src/services/replicator/drain-handler.ts
 var DEFAULT_SWITCHOVER_TIMEOUT_MS = 1e4;
 var MIN_SWITCHOVER_RETRY_DELAY_MS = 1e3;

package/out/zero-cache/src/services/replicator/drain-handler.js.map

@@ -1 +1 @@
-{"version":3,"file":"drain-handler.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/drain-handler.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {readHead, type Sized, type Source} from '../../types/streams.ts';\nimport type {\n  ChangeStreamer,\n  Status,\n  SubscriberContext,\n} from '../change-streamer/change-streamer.ts';\nimport type {SubscribeDownstream} from '../change-streamer/subscribe.ts';\nimport type {RunningState} from '../running-state.ts';\n\n// Bounds the time to wait for a switchover subscription to be established\n// and to receive its first message.\nconst DEFAULT_SWITCHOVER_TIMEOUT_MS = 10_000;\nconst MIN_SWITCHOVER_RETRY_DELAY_MS = 1_000;\nconst MAX_SWITCHOVER_RETRY_DELAY_MS = 30_000;\n\n/**\n * Bounds the time to search for another replication-manager to switch over\n * to, after which the current subscription is closed so that the draining\n * replication-manager can shut down (e.g. when it is the only one). The time\n * during which a subscription is held by another replication-manager (i.e.\n * waiting for permission to restore) does not count towards it.\n *\n * This is well below the default termination grace period of container\n * runners (30 seconds in Kubernetes and ECS), so that the draining\n * replication-manager can shut down cleanly when there is no other.\n */\nconst DEFAULT_SWITCHOVER_SEARCH_TIMEOUT_MS = 20_000;\n\nexport type SwitchoverOptions = {\n  /**\n   * Called when a replication-manager with a newer replica version grants\n   * permission to restore its replica. This should initiate a graceful\n   * restart of the server (during which the current subscription continues\n   * to be served).\n   */\n  onRestoreGranted: () => void;\n\n  /**\n   * The replica file, from which the state of the last committed\n   * transaction is read for switchover subscriptions.\n   */\n  replicaFile: string;\n\n  /** For testing. */\n  timeoutMs?: number | undefined;\n\n  /** For testing. */\n  searchTimeoutMs?: number | undefined;\n};\n\nexport type Downstream = Source<Sized<SubscribeDownstream>>;\n\ntype AttemptOutcome =\n  | 'switched'\n  | 'restoring'\n  // The subscription was held for restore, but closed before a grant.\n  | 'released'\n  | 'retry';\n\n/**\n * Handles the switchover of a serving replica's subscription from a draining\n * replication-manager to another one, without interrupting the current\n * subscription.\n *\n * Upon {@link onChangeStreamerDrain}, the handler repeatedly (with backoff)\n * opens a new `switchover` subscription, at the watermark of the replica's\n * last committed transaction, to whichever replication-manager the\n * connection is routed to. The first message on the new subscription\n * determines the outcome:\n * * `status`: The subscription was accepted (i.e. the replica is compatible).\n *   The current subscription is cancelled (aborting any transaction in\n *   progress), and the new one is made available via\n *   {@link getNextConnection}. Note that the new subscription may start with\n *   transactions that were committed from the current subscription in the\n *   meantime, which the consumer must skip.\n * * `status` with `restore: queued`: The replication-manager has a newer\n *   replica version, and is holding the subscription until it grants\n *   permission to restore. The held subscription is monitored for the grant.\n * * `status` with `restore: granted`: Permission to restore is granted, and\n *   {@link SwitchoverOptions.onRestoreGranted} is called. The granted\n *   subscription is kept open until the process exits, so that the\n *   replication-manager can track the grant until the restore begins.\n * * Anything else (e.g. an error, or a `draining` replication-manager):\n *   The switchover is retried later.\n *\n * If no other replication-manager accepts or holds a switchover subscription\n * within {@link DEFAULT_SWITCHOVER_SEARCH_TIMEOUT_MS} (e.g. because the\n * draining one is the only one), the current subscription is closed so that\n * the draining replication-manager can shut down, and the consumer reconnects\n * on its own.\n *\n * The current subscription continues to be consumed throughout. A\n * switchover is scoped to the subscription from which it switches: If that\n * subscription ends first (e.g. the replication-manager shuts down), the\n * switchover is abandoned, including any held (but not granted)\n * subscription, as the consumer reconnects on its own.\n */\nexport class DrainHandler {\n  readonly #lc: LogContext;\n  readonly #changeStreamer: ChangeStreamer;\n  readonly #state: RunningState;\n  readonly #options: SwitchoverOptions;\n  readonly #committedContext: () => SubscriberContext;\n\n  // The subscription currently being switched from.\n  #draining: Downstream | undefined;\n  #restoreGranted = false;\n  #next: Downstream | undefined;\n\n  /**\n   * @param committedContext Returns the subscriber context as of the\n   *        replica's last committed transaction.\n   */\n  constructor(\n    lc: LogContext,\n    changeStreamer: ChangeStreamer,\n    state: RunningState,\n    options: SwitchoverOptions,\n    committedContext: () => SubscriberContext,\n  ) {\n    this.#lc = lc.withContext('component', 'drain-handler');\n    this.#changeStreamer = changeStreamer;\n    this.#state = state;\n    this.#options = options;\n    this.#committedContext = committedContext;\n  }\n\n  /**\n   * Called when the change-streamer of the `current` subscription is\n   * draining.\n   */\n  onChangeStreamerDrain(current: Downstream) {\n    if (\n      this.#restoreGranted ||\n      this.#draining === current ||\n      !this.#isActive(current)\n    ) {\n      return;\n    }\n    this.#draining = current;\n    this.#lc.info?.(\n      `replication-manager is draining. switching over to another`,\n    );\n    void this.#switchover(current);\n  }\n\n  get hasNextConnection(): boolean {\n    return this.#next !== undefined;\n  }\n\n  /**\n   * Returns (and clears) the subscription that replaces the (now cancelled)\n   * current one, if a switchover succeeded.\n   */\n  getNextConnection(): Downstream | undefined {\n    const next = this.#next;\n    this.#next = undefined;\n    return next;\n  }\n\n  #isActive(current: Downstream) {\n    return !current.signal.aborted && this.#state.shouldRun();\n  }\n\n  async #switchover(current: Downstream) {\n    const searchTimeoutMs =\n      this.#options.searchTimeoutMs ?? DEFAULT_SWITCHOVER_SEARCH_TIMEOUT_MS;\n    let failures = 0;\n    let searchStart = Date.now();\n    while (this.#isActive(current)) {\n      const outcome = await this.#attempt(current);\n      if (outcome === 'switched' || outcome === 'restoring') {\n        return;\n      }\n      if (outcome === 'released') {\n        // Time spent held by another replication-manager does not count\n        // towards the search.\n        failures = 0;\n        searchStart = Date.now();\n      }\n      const remaining = searchStart + searchTimeoutMs - Date.now();\n      if (remaining > 0) {\n        const delay = Math.min(\n          MIN_SWITCHOVER_RETRY_DELAY_MS * 2 ** failures++,\n          MAX_SWITCHOVER_RETRY_DELAY_MS,\n        );\n        await this.#state.sleep(Math.min(delay, remaining));\n        continue;\n      }\n      // The search has timed out.\n      if (this.#isActive(current)) {\n        this.#lc.warn?.(\n          `no other replication-manager found within ${searchTimeoutMs} ms. ` +\n            `disconnecting from the draining replication-manager`,\n        );\n        current.cancel();\n      }\n      return;\n    }\n  }\n\n  async #attempt(current: Downstream): Promise<AttemptOutcome> {\n    const lc = this.#lc;\n    const timeoutMs = this.#options.timeoutMs ?? DEFAULT_SWITCHOVER_TIMEOUT_MS;\n    let candidate: Downstream | undefined;\n    let timer: ReturnType<typeof setTimeout> | undefined;\n    let timedOut = false;\n    try {\n      const ctx = this.#committedContext();\n      lc.info?.(`attempting switchover at ${ctx.watermark}`);\n      const timeout = new Promise<never>((_, reject) => {\n        timer = setTimeout(() => {\n          timedOut = true;\n          reject(new Error(`timed out after ${timeoutMs} ms`));\n        }, timeoutMs);\n      });\n      const subscribing = this.#changeStreamer.subscribe({\n        ...ctx,\n        switchover: true,\n      });\n      // Clean up a connection that is established after a timeout.\n      void subscribing.then(\n        sub => (timedOut ? sub.cancel() : undefined),\n        () => {},\n      );\n      candidate = await Promise.race([subscribing, timeout]);\n      const {head, rest} = readHead(candidate);\n      const first = await Promise.race([head.next(), timeout]);\n      const msg = first.done ? undefined : first.value.data;\n\n      if (!this.#isActive(current)) {\n        // e.g. The current subscription failed in the meantime.\n      } else if (msg?.[0] === 'status') {\n        const {restore, draining} = msg[1];\n        if (restore) {\n          const granted = await this.#holdForRestore(current, rest, restore);\n          if (granted) {\n            return 'restoring';\n          }\n          lc.info?.(`held switchover subscription closed. will retry`);\n          return 'released';\n        }\n        if (!draining) {\n          lc.info?.(`switched over to a new subscription at ${ctx.watermark}`);\n          this.#next = rest;\n          current.cancel();\n          return 'switched';\n        }\n      }\n      lc.warn?.(`switchover rejected. continuing current subscription`, msg);\n    } catch (e) {\n      lc.warn?.(`switchover failed. continuing current subscription`, e);\n    } finally {\n      clearTimeout(timer);\n    }\n    candidate?.cancel();\n    return 'retry';\n  }\n\n  /**\n   * Keeps the held (or granted) subscription open, and initiates a graceful\n   * restart when permission to restore is granted. A held subscription that\n   * has yet to be granted is cancelled if the `current` subscription ends.\n   *\n   * @returns `true` once permission is granted, or `false` if the held\n   *          subscription closes before then.\n   */\n  #holdForRestore(\n    current: Downstream,\n    held: Downstream,\n    restore: NonNullable<Status['restore']>,\n  ): Promise<boolean> {\n    const lc = this.#lc;\n    const result = resolver<boolean>();\n    const unregister = this.#state.cancelOnStop(held);\n    const onCurrentEnded = () => {\n      if (!this.#restoreGranted) {\n        lc.info?.(`abandoning held switchover subscription`);\n        held.cancel();\n      }\n    };\n    current.signal.addEventListener('abort', onCurrentEnded, {once: true});\n\n    const onRestore = (r: NonNullable<Status['restore']>) => {\n      if (r.state === 'queued') {\n        lc.info?.(\n          `waiting for permission to restore a new replica (position ${r.position})`,\n        );\n      } else if (!this.#restoreGranted) {\n        lc.info?.(\n          `granted permission to restore a new replica. restarting gracefully`,\n        );\n        this.#restoreGranted = true;\n        this.#options.onRestoreGranted();\n        result.resolve(true);\n      }\n    };\n    onRestore(restore);\n    void (async () => {\n      try {\n        // Note: The loop must not be exited early, as that would close the\n        // connection.\n        for await (const {data: msg} of held) {\n          if (msg[0] === 'status' && msg[1].restore) {\n            onRestore(msg[1].restore);\n          }\n        }\n      } catch (e) {\n        lc.warn?.(`held switchover subscription failed`, e);\n      } finally {\n        unregister();\n        current.signal.removeEventListener('abort', onCurrentEnded);\n        result.resolve(false);\n      }\n    })();\n    return result.promise;\n  }\n}\n"],"mappings":";;;AAaA,IAAM,gCAAgC;AACtC,IAAM,gCAAgC;AACtC,IAAM,gCAAgC;;;;;;;;;;;;AAatC,IAAM,uCAAuC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuE7C,IAAa,eAAb,MAA0B;CACxB;CACA;CACA;CACA;CACA;CAGA;CACA,kBAAkB;CAClB;;;;;CAMA,YACE,IACA,gBACA,OACA,SACA,kBACA;EACA,KAAK,MAAM,GAAG,YAAY,aAAa,eAAe;EACtD,KAAK,kBAAkB;EACvB,KAAK,SAAS;EACd,KAAK,WAAW;EAChB,KAAK,oBAAoB;CAC3B;;;;;CAMA,sBAAsB,SAAqB;EACzC,IACE,KAAK,mBACL,KAAK,cAAc,WACnB,CAAC,KAAK,UAAU,OAAO,GAEvB;EAEF,KAAK,YAAY;EACjB,KAAK,IAAI,OACP,4DACF;EACA,KAAU,YAAY,OAAO;CAC/B;CAEA,IAAI,oBAA6B;EAC/B,OAAO,KAAK,UAAU,KAAA;CACxB;;;;;CAMA,oBAA4C;EAC1C,MAAM,OAAO,KAAK;EAClB,KAAK,QAAQ,KAAA;EACb,OAAO;CACT;CAEA,UAAU,SAAqB;EAC7B,OAAO,CAAC,QAAQ,OAAO,WAAW,KAAK,OAAO,UAAU;CAC1D;CAEA,MAAM,YAAY,SAAqB;EACrC,MAAM,kBACJ,KAAK,SAAS,mBAAmB;EACnC,IAAI,WAAW;EACf,IAAI,cAAc,KAAK,IAAI;EAC3B,OAAO,KAAK,UAAU,OAAO,GAAG;GAC9B,MAAM,UAAU,MAAM,KAAK,SAAS,OAAO;GAC3C,IAAI,YAAY,cAAc,YAAY,aACxC;GAEF,IAAI,YAAY,YAAY;IAG1B,WAAW;IACX,cAAc,KAAK,IAAI;GACzB;GACA,MAAM,YAAY,cAAc,kBAAkB,KAAK,IAAI;GAC3D,IAAI,YAAY,GAAG;IACjB,MAAM,QAAQ,KAAK,IACjB,gCAAgC,KAAK,YACrC,6BACF;IACA,MAAM,KAAK,OAAO,MAAM,KAAK,IAAI,OAAO,SAAS,CAAC;IAClD;GACF;GAEA,IAAI,KAAK,UAAU,OAAO,GAAG;IAC3B,KAAK,IAAI,OACP,6CAA6C,gBAAgB,yDAE/D;IACA,QAAQ,OAAO;GACjB;GACA;EACF;CACF;CAEA,MAAM,SAAS,SAA8C;EAC3D,MAAM,KAAK,KAAK;EAChB,MAAM,YAAY,KAAK,SAAS,aAAa;EAC7C,IAAI;EACJ,IAAI;EACJ,IAAI,WAAW;EACf,IAAI;GACF,MAAM,MAAM,KAAK,kBAAkB;GACnC,GAAG,OAAO,4BAA4B,IAAI,WAAW;GACrD,MAAM,UAAU,IAAI,SAAgB,GAAG,WAAW;IAChD,QAAQ,iBAAiB;KACvB,WAAW;KACX,uBAAO,IAAI,MAAM,mBAAmB,UAAU,IAAI,CAAC;IACrD,GAAG,SAAS;GACd,CAAC;GACD,MAAM,cAAc,KAAK,gBAAgB,UAAU;IACjD,GAAG;IACH,YAAY;GACd,CAAC;GAED,YAAiB,MACf,QAAQ,WAAW,IAAI,OAAO,IAAI,KAAA,SAC5B,CAAC,CACT;GACA,YAAY,MAAM,QAAQ,KAAK,CAAC,aAAa,OAAO,CAAC;GACrD,MAAM,EAAC,MAAM,SAAQ,SAAS,SAAS;GACvC,MAAM,QAAQ,MAAM,QAAQ,KAAK,CAAC,KAAK,KAAK,GAAG,OAAO,CAAC;GACvD,MAAM,MAAM,MAAM,OAAO,KAAA,IAAY,MAAM,MAAM;GAEjD,IAAI,CAAC,KAAK,UAAU,OAAO,GAAG,CAE9B,OAAO,IAAI,MAAM,OAAO,UAAU;IAChC,MAAM,EAAC,SAAS,aAAY,IAAI;IAChC,IAAI,SAAS;KAEX,IAAI,MADkB,KAAK,gBAAgB,SAAS,MAAM,OAAO,GAE/D,OAAO;KAET,GAAG,OAAO,iDAAiD;KAC3D,OAAO;IACT;IACA,IAAI,CAAC,UAAU;KACb,GAAG,OAAO,0CAA0C,IAAI,WAAW;KACnE,KAAK,QAAQ;KACb,QAAQ,OAAO;KACf,OAAO;IACT;GACF;GACA,GAAG,OAAO,wDAAwD,GAAG;EACvE,SAAS,GAAG;GACV,GAAG,OAAO,sDAAsD,CAAC;EACnE,UAAU;GACR,aAAa,KAAK;EACpB;EACA,WAAW,OAAO;EAClB,OAAO;CACT;;;;;;;;;CAUA,gBACE,SACA,MACA,SACkB;EAClB,MAAM,KAAK,KAAK;EAChB,MAAM,SAAS,SAAkB;EACjC,MAAM,aAAa,KAAK,OAAO,aAAa,IAAI;EAChD,MAAM,uBAAuB;GAC3B,IAAI,CAAC,KAAK,iBAAiB;IACzB,GAAG,OAAO,yCAAyC;IACnD,KAAK,OAAO;GACd;EACF;EACA,QAAQ,OAAO,iBAAiB,SAAS,gBAAgB,EAAC,MAAM,KAAI,CAAC;EAErE,MAAM,aAAa,MAAsC;GACvD,IAAI,EAAE,UAAU,UACd,GAAG,OACD,6DAA6D,EAAE,SAAS,EAC1E;QACK,IAAI,CAAC,KAAK,iBAAiB;IAChC,GAAG,OACD,oEACF;IACA,KAAK,kBAAkB;IACvB,KAAK,SAAS,iBAAiB;IAC/B,OAAO,QAAQ,IAAI;GACrB;EACF;EACA,UAAU,OAAO;EACjB,CAAM,YAAY;GAChB,IAAI;IAGF,WAAW,MAAM,EAAC,MAAM,SAAQ,MAC9B,IAAI,IAAI,OAAO,YAAY,IAAI,EAAE,CAAC,SAChC,UAAU,IAAI,EAAE,CAAC,OAAO;GAG9B,SAAS,GAAG;IACV,GAAG,OAAO,uCAAuC,CAAC;GACpD,UAAU;IACR,WAAW;IACX,QAAQ,OAAO,oBAAoB,SAAS,cAAc;IAC1D,OAAO,QAAQ,KAAK;GACtB;EACF,EAAA,CAAG;EACH,OAAO,OAAO;CAChB;AACF"}
+{"version":3,"file":"drain-handler.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/drain-handler.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '../../../../shared/src/resolver.ts';\nimport {readHead, type Sized, type Source} from '../../types/streams.ts';\nimport type {\n  ChangeStreamer,\n  Status,\n  SubscriberContext,\n} from '../change-streamer/change-streamer.ts';\nimport type {SubscribeDownstream} from '../change-streamer/subscribe.ts';\nimport type {RunningState} from '../running-state.ts';\n\n// Bounds the time to wait for a switchover subscription to be established\n// and to receive its first message.\nconst DEFAULT_SWITCHOVER_TIMEOUT_MS = 10_000;\nconst MIN_SWITCHOVER_RETRY_DELAY_MS = 1_000;\nconst MAX_SWITCHOVER_RETRY_DELAY_MS = 30_000;\n\n/**\n * Bounds the time to search for another replication-manager to switch over\n * to, after which the current subscription is closed so that the draining\n * replication-manager can shut down (e.g. when it is the only one). The time\n * during which a subscription is held by another replication-manager (i.e.\n * waiting for permission to restore) does not count towards it.\n *\n * This is well below the default termination grace period of container\n * runners (30 seconds in Kubernetes and ECS), so that the draining\n * replication-manager can shut down cleanly when there is no other.\n */\nconst DEFAULT_SWITCHOVER_SEARCH_TIMEOUT_MS = 20_000;\n\nexport type SwitchoverOptions = {\n  /**\n   * Called when a replication-manager with a newer replica version grants\n   * permission to restore its replica. This should initiate a graceful\n   * restart of the server (during which the current subscription continues\n   * to be served).\n   */\n  onRestoreGranted: () => void;\n\n  /**\n   * The replica file, from which the state of the last committed\n   * transaction is read for switchover subscriptions.\n   */\n  replicaFile: string;\n\n  /** For testing. */\n  timeoutMs?: number | undefined;\n\n  /** For testing. */\n  searchTimeoutMs?: number | undefined;\n};\n\nexport type Downstream = Source<Sized<SubscribeDownstream>>;\n\ntype AttemptOutcome =\n  | 'switched'\n  | 'restoring'\n  // The subscription was held for restore, but closed before a grant.\n  | 'released'\n  | 'retry';\n\n/**\n * Handles the switchover of a serving replica's subscription from a draining\n * replication-manager to another one, without interrupting the current\n * subscription.\n *\n * Upon {@link onChangeStreamerDrain}, the handler repeatedly (with backoff)\n * opens a new `switchover` subscription, at the watermark of the replica's\n * last committed transaction, to whichever replication-manager the\n * connection is routed to. The first message on the new subscription\n * determines the outcome:\n * * `status`: The subscription was accepted (i.e. the replica is compatible).\n *   The current subscription is cancelled (aborting any transaction in\n *   progress), and the new one is made available via\n *   {@link getNextConnection}. Note that the new subscription may start with\n *   transactions that were committed from the current subscription in the\n *   meantime, which the consumer must skip.\n * * `status` with `restore: queued`: The replication-manager has a newer\n *   replica version, and is holding the subscription until it grants\n *   permission to restore. The held subscription is monitored for the grant.\n * * `status` with `restore: granted`: Permission to restore is granted, and\n *   {@link SwitchoverOptions.onRestoreGranted} is called. The granted\n *   subscription is kept open until the process exits, so that the\n *   replication-manager can track the grant until the restore begins.\n * * Anything else (e.g. an error, or a `draining` replication-manager):\n *   The switchover is retried later.\n *\n * If no other replication-manager accepts or holds a switchover subscription\n * within {@link DEFAULT_SWITCHOVER_SEARCH_TIMEOUT_MS} (e.g. because the\n * draining one is the only one), the current subscription is closed so that\n * the draining replication-manager can shut down, and the consumer reconnects\n * on its own.\n *\n * The current subscription continues to be consumed throughout. A\n * switchover is scoped to the subscription from which it switches: If that\n * subscription ends first (e.g. the replication-manager shuts down), the\n * switchover is abandoned, including any held (but not granted)\n * subscription, as the consumer reconnects on its own.\n */\nexport class DrainHandler {\n  readonly #lc: LogContext;\n  readonly #changeStreamer: ChangeStreamer;\n  readonly #state: RunningState;\n  readonly #options: SwitchoverOptions;\n  readonly #committedContext: () => SubscriberContext;\n\n  // The subscription currently being switched from.\n  #draining: Downstream | undefined;\n  #restoreGranted = false;\n  #next: Downstream | undefined;\n\n  /**\n   * @param committedContext Returns the subscriber context as of the\n   *        replica's last committed transaction.\n   */\n  constructor(\n    lc: LogContext,\n    changeStreamer: ChangeStreamer,\n    state: RunningState,\n    options: SwitchoverOptions,\n    committedContext: () => SubscriberContext,\n  ) {\n    this.#lc = lc.withContext('component', 'drain-handler');\n    this.#changeStreamer = changeStreamer;\n    this.#state = state;\n    this.#options = options;\n    this.#committedContext = committedContext;\n  }\n\n  /**\n   * Called when the change-streamer of the `current` subscription is\n   * draining.\n   */\n  onChangeStreamerDrain(current: Downstream) {\n    if (\n      this.#restoreGranted ||\n      this.#draining === current ||\n      !this.#isActive(current)\n    ) {\n      return;\n    }\n    this.#draining = current;\n    this.#lc.info?.(\n      `replication-manager is draining. switching over to another`,\n    );\n    void this.#switchover(current);\n  }\n\n  get hasNextConnection(): boolean {\n    return this.#next !== undefined;\n  }\n\n  /**\n   * Returns (and clears) the subscription that replaces the (now cancelled)\n   * current one, if a switchover succeeded.\n   */\n  getNextConnection(): Downstream | undefined {\n    const next = this.#next;\n    this.#next = undefined;\n    return next;\n  }\n\n  #isActive(current: Downstream) {\n    return !current.signal.aborted && this.#state.shouldRun();\n  }\n\n  async #switchover(current: Downstream) {\n    const searchTimeoutMs =\n      this.#options.searchTimeoutMs ?? DEFAULT_SWITCHOVER_SEARCH_TIMEOUT_MS;\n    let failures = 0;\n    let searchStart = Date.now();\n    while (this.#isActive(current)) {\n      const outcome = await this.#attempt(current);\n      if (outcome === 'switched' || outcome === 'restoring') {\n        return;\n      }\n      if (outcome === 'released') {\n        // Time spent held by another replication-manager does not count\n        // towards the search.\n        failures = 0;\n        searchStart = Date.now();\n      }\n      const remaining = searchStart + searchTimeoutMs - Date.now();\n      if (remaining > 0) {\n        const delay = Math.min(\n          MIN_SWITCHOVER_RETRY_DELAY_MS * 2 ** failures++,\n          MAX_SWITCHOVER_RETRY_DELAY_MS,\n        );\n        await this.#state.sleep(Math.min(delay, remaining));\n        continue;\n      }\n      // The search has timed out.\n      if (this.#isActive(current)) {\n        this.#lc.warn?.(\n          `no other replication-manager found within ${searchTimeoutMs} ms. ` +\n            `disconnecting from the draining replication-manager`,\n        );\n        current.cancel();\n      }\n      return;\n    }\n  }\n\n  async #attempt(current: Downstream): Promise<AttemptOutcome> {\n    const lc = this.#lc;\n    const timeoutMs = this.#options.timeoutMs ?? DEFAULT_SWITCHOVER_TIMEOUT_MS;\n    let candidate: Downstream | undefined;\n    let timer: ReturnType<typeof setTimeout> | undefined;\n    let timedOut = false;\n    try {\n      const ctx = this.#committedContext();\n      lc.info?.(`attempting switchover at ${ctx.watermark}`);\n      const timeout = new Promise<never>((_, reject) => {\n        timer = setTimeout(() => {\n          timedOut = true;\n          reject(new Error(`timed out after ${timeoutMs} ms`));\n        }, timeoutMs);\n      });\n      const subscribing = this.#changeStreamer.subscribe({\n        ...ctx,\n        switchover: true,\n      });\n      // Clean up a connection that is established after a timeout.\n      void subscribing.then(\n        sub => (timedOut ? sub.cancel() : undefined),\n        () => {},\n      );\n      candidate = await Promise.race([subscribing, timeout]);\n      const {head, rest} = readHead(candidate);\n      const first = await Promise.race([head.next(), timeout]);\n      const msg = first.done ? undefined : first.value.data;\n\n      if (!this.#isActive(current)) {\n        // e.g. The current subscription failed in the meantime.\n      } else if (msg?.[0] === 'status') {\n        const {restore, draining} = msg[1];\n        if (restore) {\n          const granted = await this.#holdForRestore(current, rest, restore);\n          if (granted) {\n            return 'restoring';\n          }\n          lc.info?.(`held switchover subscription closed. will retry`);\n          return 'released';\n        }\n        if (!draining) {\n          lc.info?.(`switched over to a new subscription at ${ctx.watermark}`);\n          this.#next = rest;\n          current.cancel();\n          return 'switched';\n        }\n      }\n      lc.warn?.(`switchover rejected. continuing current subscription`, msg);\n    } catch (e) {\n      lc.warn?.(`switchover failed. continuing current subscription`, e);\n    } finally {\n      clearTimeout(timer);\n    }\n    candidate?.cancel();\n    return 'retry';\n  }\n\n  /**\n   * Keeps the held (or granted) subscription open, and initiates a graceful\n   * restart when permission to restore is granted. A held subscription that\n   * has yet to be granted is cancelled if the `current` subscription ends.\n   *\n   * @returns `true` once permission is granted, or `false` if the held\n   *          subscription closes before then.\n   */\n  #holdForRestore(\n    current: Downstream,\n    held: Downstream,\n    restore: NonNullable<Status['restore']>,\n  ): Promise<boolean> {\n    const lc = this.#lc;\n    const result = resolver<boolean>();\n    const unregister = this.#state.cancelOnStop(held);\n    const onCurrentEnded = () => {\n      if (!this.#restoreGranted) {\n        lc.info?.(`abandoning held switchover subscription`);\n        held.cancel();\n      }\n    };\n    current.signal.addEventListener('abort', onCurrentEnded, {once: true});\n\n    const onRestore = (r: NonNullable<Status['restore']>) => {\n      if (r.state === 'queued') {\n        lc.info?.(\n          `waiting for permission to restore a new replica (position ${r.position})`,\n        );\n      } else if (!this.#restoreGranted) {\n        lc.info?.(\n          `granted permission to restore a new replica. restarting gracefully`,\n        );\n        this.#restoreGranted = true;\n        this.#options.onRestoreGranted();\n        result.resolve(true);\n      }\n    };\n    onRestore(restore);\n    void (async () => {\n      try {\n        // Note: The loop must not be exited early, as that would close the\n        // connection.\n        for await (const {data: msg} of held) {\n          if (msg[0] === 'status' && msg[1].restore) {\n            onRestore(msg[1].restore);\n          }\n        }\n      } catch (e) {\n        lc.warn?.(`held switchover subscription failed`, e);\n      } finally {\n        unregister();\n        current.signal.removeEventListener('abort', onCurrentEnded);\n        result.resolve(false);\n      }\n    })();\n    return result.promise;\n  }\n}\n"],"mappings":";;;AAaA,IAAM,gCAAgC;AACtC,IAAM,gCAAgC;AACtC,IAAM,gCAAgC;;;;;;;;;;;;AAatC,IAAM,uCAAuC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuE7C,IAAa,eAAb,MAA0B;CACxB;CACA;CACA;CACA;CACA;CAGA;CACA,kBAAkB;CAClB;;;;;CAMA,YACE,IACA,gBACA,OACA,SACA,kBACA;EACA,KAAK,MAAM,GAAG,YAAY,aAAa,eAAe;EACtD,KAAK,kBAAkB;EACvB,KAAK,SAAS;EACd,KAAK,WAAW;EAChB,KAAK,oBAAoB;CAC3B;;;;;CAMA,sBAAsB,SAAqB;EACzC,IACE,KAAK,mBACL,KAAK,cAAc,WACnB,CAAC,KAAK,UAAU,OAAO,GAEvB;EAEF,KAAK,YAAY;EACjB,KAAK,IAAI,OACP,4DACF;EACA,KAAU,YAAY,OAAO;CAC/B;CAEA,IAAI,oBAA6B;EAC/B,OAAO,KAAK,UAAU,KAAA;CACxB;;;;;CAMA,oBAA4C;EAC1C,MAAM,OAAO,KAAK;EAClB,KAAK,QAAQ,KAAA;EACb,OAAO;CACT;CAEA,UAAU,SAAqB;EAC7B,OAAO,CAAC,QAAQ,OAAO,WAAW,KAAK,OAAO,UAAU;CAC1D;CAEA,MAAM,YAAY,SAAqB;EACrC,MAAM,kBACJ,KAAK,SAAS,mBAAmB;EACnC,IAAI,WAAW;EACf,IAAI,cAAc,KAAK,IAAI;EAC3B,OAAO,KAAK,UAAU,OAAO,GAAG;GAC9B,MAAM,UAAU,MAAM,KAAK,SAAS,OAAO;GAC3C,IAAI,YAAY,cAAc,YAAY,aACxC;GAEF,IAAI,YAAY,YAAY;IAG1B,WAAW;IACX,cAAc,KAAK,IAAI;GACzB;GACA,MAAM,YAAY,cAAc,kBAAkB,KAAK,IAAI;GAC3D,IAAI,YAAY,GAAG;IACjB,MAAM,QAAQ,KAAK,IACjB,gCAAgC,KAAK,YACrC,6BACF;IACA,MAAM,KAAK,OAAO,MAAM,KAAK,IAAI,OAAO,SAAS,CAAC;IAClD;GACF;GAEA,IAAI,KAAK,UAAU,OAAO,GAAG;IAC3B,KAAK,IAAI,OACP,6CAA6C,gBAAgB,yDAE/D;IACA,QAAQ,OAAO;GACjB;GACA;EACF;CACF;CAEA,MAAM,SAAS,SAA8C;EAC3D,MAAM,KAAK,KAAK;EAChB,MAAM,YAAY,KAAK,SAAS,aAAa;EAC7C,IAAI;EACJ,IAAI;EACJ,IAAI,WAAW;EACf,IAAI;GACF,MAAM,MAAM,KAAK,kBAAkB;GACnC,GAAG,OAAO,4BAA4B,IAAI,WAAW;GACrD,MAAM,UAAU,IAAI,SAAgB,GAAG,WAAW;IAChD,QAAQ,iBAAiB;KACvB,WAAW;KACX,uBAAO,IAAI,MAAM,mBAAmB,UAAU,IAAI,CAAC;IACrD,GAAG,SAAS;GACd,CAAC;GACD,MAAM,cAAc,KAAK,gBAAgB,UAAU;IACjD,GAAG;IACH,YAAY;GACd,CAAC;GAED,YAAiB,MACf,QAAQ,WAAW,IAAI,OAAO,IAAI,KAAA,SAC5B,CAAC,CACT;GACA,YAAY,MAAM,QAAQ,KAAK,CAAC,aAAa,OAAO,CAAC;GACrD,MAAM,EAAC,MAAM,SAAQ,SAAS,SAAS;GACvC,MAAM,QAAQ,MAAM,QAAQ,KAAK,CAAC,KAAK,KAAK,GAAG,OAAO,CAAC;GACvD,MAAM,MAAM,MAAM,OAAO,KAAA,IAAY,MAAM,MAAM;GAEjD,IAAI,CAAC,KAAK,UAAU,OAAO,GAAG,CAE9B,OAAO,IAAI,MAAM,OAAO,UAAU;IAChC,MAAM,EAAC,SAAS,aAAY,IAAI;IAChC,IAAI,SAAS;KAEX,IAAI,MADkB,KAAK,gBAAgB,SAAS,MAAM,OAAO,GAE/D,OAAO;KAET,GAAG,OAAO,iDAAiD;KAC3D,OAAO;IACT;IACA,IAAI,CAAC,UAAU;KACb,GAAG,OAAO,0CAA0C,IAAI,WAAW;KACnE,KAAK,QAAQ;KACb,QAAQ,OAAO;KACf,OAAO;IACT;GACF;GACA,GAAG,OAAO,wDAAwD,GAAG;EACvE,SAAS,GAAG;GACV,GAAG,OAAO,sDAAsD,CAAC;EACnE,UAAU;GACR,aAAa,KAAK;EACpB;EACA,WAAW,OAAO;EAClB,OAAO;CACT;;;;;;;;;CAUA,gBACE,SACA,MACA,SACkB;EAClB,MAAM,KAAK,KAAK;EAChB,MAAM,SAAS,SAAkB;EACjC,MAAM,aAAa,KAAK,OAAO,aAAa,IAAI;EAChD,MAAM,uBAAuB;GAC3B,IAAI,CAAC,KAAK,iBAAiB;IACzB,GAAG,OAAO,yCAAyC;IACnD,KAAK,OAAO;GACd;EACF;EACA,QAAQ,OAAO,iBAAiB,SAAS,gBAAgB,EAAC,MAAM,KAAI,CAAC;EAErE,MAAM,aAAa,MAAsC;GACvD,IAAI,EAAE,UAAU,UACd,GAAG,OACD,6DAA6D,EAAE,SAAS,EAC1E;QACK,IAAI,CAAC,KAAK,iBAAiB;IAChC,GAAG,OACD,oEACF;IACA,KAAK,kBAAkB;IACvB,KAAK,SAAS,iBAAiB;IAC/B,OAAO,QAAQ,IAAI;GACrB;EACF;EACA,UAAU,OAAO;EACjB,CAAM,YAAY;GAChB,IAAI;IAGF,WAAW,MAAM,EAAC,MAAM,SAAQ,MAC9B,IAAI,IAAI,OAAO,YAAY,IAAI,EAAE,CAAC,SAChC,UAAU,IAAI,EAAE,CAAC,OAAO;GAG9B,SAAS,GAAG;IACV,GAAG,OAAO,uCAAuC,CAAC;GACpD,UAAU;IACR,WAAW;IACX,QAAQ,OAAO,oBAAoB,SAAS,cAAc;IAC1D,OAAO,QAAQ,KAAK;GACtB;EACF,EAAA,CAAG;EACH,OAAO,OAAO;CAChB;AACF"}

package/out/zero-cache/src/services/replicator/reporter/recorder.js

@@ -1,5 +1,5 @@
+import { resolver } from "../../../../../shared/src/resolver.js";
 import { getOrCreateGauge } from "../../../observability/metrics.js";
-import { resolver } from "@rocicorp/resolver";
 //#region ../zero-cache/src/services/replicator/reporter/recorder.ts
 var LOG_ALL_REPLICATION_REPORTS_AT_DEBUG = process.env.ZERO_LOG_ALL_REPLICATION_REPORTS_AT_DEBUG === "1";
 var ReplicationReportRecorder = class {

package/out/zero-cache/src/services/replicator/reporter/recorder.js.map

@@ -1 +1 @@
-{"version":3,"file":"recorder.js","names":[],"sources":["../../../../../../../zero-cache/src/services/replicator/reporter/recorder.ts"],"sourcesContent":["import type {ObservableResult} from '@opentelemetry/api';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {getOrCreateGauge} from '../../../observability/metrics.ts';\nimport type {ReplicationReport} from './report-schema.ts';\n\n// Hook for sanity checking lag reports in development.\nconst LOG_ALL_REPLICATION_REPORTS_AT_DEBUG =\n  process.env.ZERO_LOG_ALL_REPLICATION_REPORTS_AT_DEBUG === '1';\n\nexport type ReplicationLagReadinessOptions = {\n  readinessThresholdMs?: number | undefined;\n  maxReadinessDelayMs?: number | undefined;\n  setTimeoutFn?: typeof setTimeout | undefined;\n  clearTimeoutFn?: typeof clearTimeout | undefined;\n};\n\nexport class ReplicationReportRecorder {\n  readonly #lc: LogContext;\n  readonly #readinessThresholdMs: number;\n  readonly #readyGate = resolver();\n  readonly #clearTimeout: typeof clearTimeout;\n  readonly #readinessTimer: ReturnType<typeof setTimeout>;\n  #ready = false;\n  #last: ReplicationReport | null = null;\n\n  constructor(\n    lc: LogContext,\n    {\n      readinessThresholdMs = 1000,\n      maxReadinessDelayMs = 600_000,\n      setTimeoutFn = setTimeout,\n      clearTimeoutFn = clearTimeout,\n    }: ReplicationLagReadinessOptions = {},\n  ) {\n    this.#lc = lc;\n    this.#readinessThresholdMs = readinessThresholdMs;\n    this.#clearTimeout = clearTimeoutFn;\n\n    this.#readinessTimer = setTimeoutFn(\n      () =>\n        this.#setReady(\n          'warn',\n          `timed out waiting for replication lag readiness threshold of ${this.#readinessThresholdMs} ms`,\n        ),\n      maxReadinessDelayMs,\n    );\n    // The timer should not keep the process alive.\n    this.#readinessTimer?.unref?.();\n  }\n\n  ready(): Promise<void> {\n    return this.#readyGate.promise;\n  }\n\n  record(report: ReplicationReport) {\n    if (isLagReportingDisabledSignal(report)) {\n      this.#setReady(\n        'info',\n        `replica is caught up (replication lag reporting is disabled upstream)`,\n      );\n      return;\n    }\n    this.#last = report;\n\n    const {lastTimings} = report;\n    if (lastTimings) {\n      const total = lastTimings.replicateTimeMs - lastTimings.sendTimeMs;\n      if (total > 10_000) {\n        const log = this.#ready ? 'warn' : 'info';\n        this.#lc[log]?.(`high replication lag: ${total} ms`, {\n          ready: this.#ready,\n          report,\n        });\n      } else if (total > 1_000) {\n        this.#lc.info?.(`replication lag: ${total} ms`, {\n          ready: this.#ready,\n          report,\n        });\n      }\n      if (LOG_ALL_REPLICATION_REPORTS_AT_DEBUG) {\n        this.#lc.debug?.(`replication lag ${total} ms`, {\n          ready: this.#ready,\n          report,\n        });\n      }\n      if (!this.#ready && total <= this.#readinessThresholdMs) {\n        this.#setReady(\n          'info',\n          `replicator is now ready with replication lag (${total} ms) less than ${this.#readinessThresholdMs} ms readiness threshold`,\n        );\n      }\n    }\n  }\n\n  #setReady(log: LogLevel, message: string) {\n    if (this.#ready) {\n      return;\n    }\n    this.#lc[log]?.(message, {lastReport: this.#last});\n    this.#ready = true;\n    this.#clearTimeout(this.#readinessTimer);\n    this.#readyGate.resolve();\n\n    getOrCreateGauge('replication', 'upstream_lag', {\n      description:\n        'Latency from sending an upstream replication report ' +\n        'to receiving it in the replication stream',\n      unit: 'millisecond',\n    }).addCallback(this.reportUpstreamLag);\n\n    getOrCreateGauge('replication', 'replica_lag', {\n      description:\n        'Latency from receiving an upstream replication report ' +\n        'to its reaching the replica',\n      unit: 'millisecond',\n    }).addCallback(this.reportReplicaLag);\n\n    getOrCreateGauge('replication', 'total_lag', {\n      description:\n        'Latency from sending an upstream replication report to its ' +\n        'reaching the replica. This reflects the actual measured ' +\n        'round-trip of the most recently received report and does not ' +\n        'grow when reports stop arriving.',\n      unit: 'millisecond',\n    }).addCallback(this.reportTotalLag);\n\n    getOrCreateGauge('replication', 'last_total_lag', {\n      description:\n        'Latency from sending the most recently received upstream ' +\n        'replication report to its reaching the replica. This is an alias ' +\n        'of replication.total_lag retained for dashboards that explicitly ' +\n        'want the non-extrapolated value.',\n      unit: 'millisecond',\n    }).addCallback(this.reportLastTotalLag);\n\n    getOrCreateGauge('replication', 'upstream_clock_skew', {\n      description:\n        'Estimated offset of the upstream database clock relative to ' +\n        \"zero-cache's. Positive means upstream is running ahead. \" +\n        'Derived from the lag report round-trip, whose send and receive ' +\n        'times are stamped on our clock either side of the upstream ' +\n        'commit timestamp. This matters because ' +\n        'sync.e2e_serving_lag subtracts an upstream commit timestamp from ' +\n        'a local clock, so a skewed upstream clock biases it directly -- ' +\n        'and an upstream clock running ahead biases it *low*, which looks ' +\n        'like healthy serving rather than a broken measurement.',\n      unit: 'millisecond',\n    }).addCallback(this.reportUpstreamClockSkew);\n  }\n\n  readonly reportUpstreamLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.receiveTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportReplicaLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.receiveTimeMs);\n    }\n  };\n\n  readonly reportTotalLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportLastTotalLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportUpstreamClockSkew = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(estimateUpstreamClockSkewMs(last));\n    }\n  };\n}\n\n/**\n * Estimates how far the upstream database's clock is from ours, in\n * milliseconds, positive meaning upstream is ahead.\n *\n * `sendTimeMs` and `receiveTimeMs` are both stamped in the\n * replication-manager, either side of `commitTimeMs`, which upstream stamps.\n * Assuming the upstream commit fell midway through that round trip, the\n * difference between where it actually landed and that midpoint is the clock\n * offset -- the same round-trip estimate NTP uses.\n *\n * The estimate is biased by any asymmetry between the two legs: the outbound\n * leg is a SQL round trip, while the return leg additionally waits on a WAL\n * flush and the walsender (which the pre-PG-17 path notes can add 50~100ms on\n * an idle database). So treat this as a detector of gross skew -- the kind\n * that invalidates sync.e2e_serving_lag -- and not as a sub-100ms correction.\n *\n * The rigorous bound, if a tighter reading is ever needed: the commit\n * genuinely happened between send and receive, so a `commitTimeMs` outside\n * that window proves skew of at least the amount by which it falls outside,\n * independent of any latency assumption.\n */\nexport function estimateUpstreamClockSkewMs({\n  sendTimeMs,\n  commitTimeMs,\n  receiveTimeMs,\n}: {\n  sendTimeMs: number;\n  commitTimeMs: number;\n  receiveTimeMs: number;\n}): number {\n  return commitTimeMs - (sendTimeMs + receiveTimeMs) / 2;\n}\n\n/**\n * Returns whether the `report` is the signal sent by the change-streamer,\n * once the subscriber is caught up, to indicate that lag reporting is\n * disabled upstream (and thus readiness cannot be gated on lag reports).\n *\n * This is a lag report with a `nextSendTimeMs` of 0 and no `lastTimings`.\n */\nexport function isLagReportingDisabledSignal(\n  report: ReplicationReport,\n): boolean {\n  return report.nextSendTimeMs === 0 && report.lastTimings === undefined;\n}\n"],"mappings":";;;AAOA,IAAM,uCACJ,QAAQ,IAAI,8CAA8C;AAS5D,IAAa,4BAAb,MAAuC;CACrC;CACA;CACA,aAAsB,SAAS;CAC/B;CACA;CACA,SAAS;CACT,QAAkC;CAElC,YACE,IACA,EACE,uBAAuB,KACvB,sBAAsB,KACtB,eAAe,YACf,iBAAiB,iBACiB,CAAC,GACrC;EACA,KAAK,MAAM;EACX,KAAK,wBAAwB;EAC7B,KAAK,gBAAgB;EAErB,KAAK,kBAAkB,mBAEnB,KAAK,UACH,QACA,gEAAgE,KAAK,sBAAsB,IAC7F,GACF,mBACF;EAEA,KAAK,iBAAiB,QAAQ;CAChC;CAEA,QAAuB;EACrB,OAAO,KAAK,WAAW;CACzB;CAEA,OAAO,QAA2B;EAChC,IAAI,6BAA6B,MAAM,GAAG;GACxC,KAAK,UACH,QACA,uEACF;GACA;EACF;EACA,KAAK,QAAQ;EAEb,MAAM,EAAC,gBAAe;EACtB,IAAI,aAAa;GACf,MAAM,QAAQ,YAAY,kBAAkB,YAAY;GACxD,IAAI,QAAQ,KAAQ;IAClB,MAAM,MAAM,KAAK,SAAS,SAAS;IACnC,KAAK,IAAI,IAAI,GAAG,yBAAyB,MAAM,MAAM;KACnD,OAAO,KAAK;KACZ;IACF,CAAC;GACH,OAAO,IAAI,QAAQ,KACjB,KAAK,IAAI,OAAO,oBAAoB,MAAM,MAAM;IAC9C,OAAO,KAAK;IACZ;GACF,CAAC;GAEH,IAAI,sCACF,KAAK,IAAI,QAAQ,mBAAmB,MAAM,MAAM;IAC9C,OAAO,KAAK;IACZ;GACF,CAAC;GAEH,IAAI,CAAC,KAAK,UAAU,SAAS,KAAK,uBAChC,KAAK,UACH,QACA,iDAAiD,MAAM,iBAAiB,KAAK,sBAAsB,wBACrG;EAEJ;CACF;CAEA,UAAU,KAAe,SAAiB;EACxC,IAAI,KAAK,QACP;EAEF,KAAK,IAAI,IAAI,GAAG,SAAS,EAAC,YAAY,KAAK,MAAK,CAAC;EACjD,KAAK,SAAS;EACd,KAAK,cAAc,KAAK,eAAe;EACvC,KAAK,WAAW,QAAQ;EAExB,iBAAiB,eAAe,gBAAgB;GAC9C,aACE;GAEF,MAAM;EACR,CAAC,CAAC,CAAC,YAAY,KAAK,iBAAiB;EAErC,iBAAiB,eAAe,eAAe;GAC7C,aACE;GAEF,MAAM;EACR,CAAC,CAAC,CAAC,YAAY,KAAK,gBAAgB;EAEpC,iBAAiB,eAAe,aAAa;GAC3C,aACE;GAIF,MAAM;EACR,CAAC,CAAC,CAAC,YAAY,KAAK,cAAc;EAElC,iBAAiB,eAAe,kBAAkB;GAChD,aACE;GAIF,MAAM;EACR,CAAC,CAAC,CAAC,YAAY,KAAK,kBAAkB;EAEtC,iBAAiB,eAAe,uBAAuB;GACrD,aACE;GASF,MAAM;EACR,CAAC,CAAC,CAAC,YAAY,KAAK,uBAAuB;CAC7C;CAEA,qBAA8B,MAAwB;EACpD,MAAM,OAAO,KAAK,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,gBAAgB,KAAK,UAAU;CAElD;CAEA,oBAA6B,MAAwB;EACnD,MAAM,OAAO,KAAK,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,aAAa;CAEvD;CAEA,kBAA2B,MAAwB;EACjD,MAAM,OAAO,KAAK,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,UAAU;CAEpD;CAEA,sBAA+B,MAAwB;EACrD,MAAM,OAAO,KAAK,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,UAAU;CAEpD;CAEA,2BAAoC,MAAwB;EAC1D,MAAM,OAAO,KAAK,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,4BAA4B,IAAI,CAAC;CAE/C;AACF;;;;;;;;;;;;;;;;;;;;;;AAuBA,SAAgB,4BAA4B,EAC1C,YACA,cACA,iBAKS;CACT,OAAO,gBAAgB,aAAa,iBAAiB;AACvD;;;;;;;;AASA,SAAgB,6BACd,QACS;CACT,OAAO,OAAO,mBAAmB,KAAK,OAAO,gBAAgB,KAAA;AAC/D"}
+{"version":3,"file":"recorder.js","names":[],"sources":["../../../../../../../zero-cache/src/services/replicator/reporter/recorder.ts"],"sourcesContent":["import type {ObservableResult} from '@opentelemetry/api';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '../../../../../shared/src/resolver.ts';\nimport {getOrCreateGauge} from '../../../observability/metrics.ts';\nimport type {ReplicationReport} from './report-schema.ts';\n\n// Hook for sanity checking lag reports in development.\nconst LOG_ALL_REPLICATION_REPORTS_AT_DEBUG =\n  process.env.ZERO_LOG_ALL_REPLICATION_REPORTS_AT_DEBUG === '1';\n\nexport type ReplicationLagReadinessOptions = {\n  readinessThresholdMs?: number | undefined;\n  maxReadinessDelayMs?: number | undefined;\n  setTimeoutFn?: typeof setTimeout | undefined;\n  clearTimeoutFn?: typeof clearTimeout | undefined;\n};\n\nexport class ReplicationReportRecorder {\n  readonly #lc: LogContext;\n  readonly #readinessThresholdMs: number;\n  readonly #readyGate = resolver();\n  readonly #clearTimeout: typeof clearTimeout;\n  readonly #readinessTimer: ReturnType<typeof setTimeout>;\n  #ready = false;\n  #last: ReplicationReport | null = null;\n\n  constructor(\n    lc: LogContext,\n    {\n      readinessThresholdMs = 1000,\n      maxReadinessDelayMs = 600_000,\n      setTimeoutFn = setTimeout,\n      clearTimeoutFn = clearTimeout,\n    }: ReplicationLagReadinessOptions = {},\n  ) {\n    this.#lc = lc;\n    this.#readinessThresholdMs = readinessThresholdMs;\n    this.#clearTimeout = clearTimeoutFn;\n\n    this.#readinessTimer = setTimeoutFn(\n      () =>\n        this.#setReady(\n          'warn',\n          `timed out waiting for replication lag readiness threshold of ${this.#readinessThresholdMs} ms`,\n        ),\n      maxReadinessDelayMs,\n    );\n    // The timer should not keep the process alive.\n    this.#readinessTimer?.unref?.();\n  }\n\n  ready(): Promise<void> {\n    return this.#readyGate.promise;\n  }\n\n  record(report: ReplicationReport) {\n    if (isLagReportingDisabledSignal(report)) {\n      this.#setReady(\n        'info',\n        `replica is caught up (replication lag reporting is disabled upstream)`,\n      );\n      return;\n    }\n    this.#last = report;\n\n    const {lastTimings} = report;\n    if (lastTimings) {\n      const total = lastTimings.replicateTimeMs - lastTimings.sendTimeMs;\n      if (total > 10_000) {\n        const log = this.#ready ? 'warn' : 'info';\n        this.#lc[log]?.(`high replication lag: ${total} ms`, {\n          ready: this.#ready,\n          report,\n        });\n      } else if (total > 1_000) {\n        this.#lc.info?.(`replication lag: ${total} ms`, {\n          ready: this.#ready,\n          report,\n        });\n      }\n      if (LOG_ALL_REPLICATION_REPORTS_AT_DEBUG) {\n        this.#lc.debug?.(`replication lag ${total} ms`, {\n          ready: this.#ready,\n          report,\n        });\n      }\n      if (!this.#ready && total <= this.#readinessThresholdMs) {\n        this.#setReady(\n          'info',\n          `replicator is now ready with replication lag (${total} ms) less than ${this.#readinessThresholdMs} ms readiness threshold`,\n        );\n      }\n    }\n  }\n\n  #setReady(log: LogLevel, message: string) {\n    if (this.#ready) {\n      return;\n    }\n    this.#lc[log]?.(message, {lastReport: this.#last});\n    this.#ready = true;\n    this.#clearTimeout(this.#readinessTimer);\n    this.#readyGate.resolve();\n\n    getOrCreateGauge('replication', 'upstream_lag', {\n      description:\n        'Latency from sending an upstream replication report ' +\n        'to receiving it in the replication stream',\n      unit: 'millisecond',\n    }).addCallback(this.reportUpstreamLag);\n\n    getOrCreateGauge('replication', 'replica_lag', {\n      description:\n        'Latency from receiving an upstream replication report ' +\n        'to its reaching the replica',\n      unit: 'millisecond',\n    }).addCallback(this.reportReplicaLag);\n\n    getOrCreateGauge('replication', 'total_lag', {\n      description:\n        'Latency from sending an upstream replication report to its ' +\n        'reaching the replica. This reflects the actual measured ' +\n        'round-trip of the most recently received report and does not ' +\n        'grow when reports stop arriving.',\n      unit: 'millisecond',\n    }).addCallback(this.reportTotalLag);\n\n    getOrCreateGauge('replication', 'last_total_lag', {\n      description:\n        'Latency from sending the most recently received upstream ' +\n        'replication report to its reaching the replica. This is an alias ' +\n        'of replication.total_lag retained for dashboards that explicitly ' +\n        'want the non-extrapolated value.',\n      unit: 'millisecond',\n    }).addCallback(this.reportLastTotalLag);\n\n    getOrCreateGauge('replication', 'upstream_clock_skew', {\n      description:\n        'Estimated offset of the upstream database clock relative to ' +\n        \"zero-cache's. Positive means upstream is running ahead. \" +\n        'Derived from the lag report round-trip, whose send and receive ' +\n        'times are stamped on our clock either side of the upstream ' +\n        'commit timestamp. This matters because ' +\n        'sync.e2e_serving_lag subtracts an upstream commit timestamp from ' +\n        'a local clock, so a skewed upstream clock biases it directly -- ' +\n        'and an upstream clock running ahead biases it *low*, which looks ' +\n        'like healthy serving rather than a broken measurement.',\n      unit: 'millisecond',\n    }).addCallback(this.reportUpstreamClockSkew);\n  }\n\n  readonly reportUpstreamLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.receiveTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportReplicaLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.receiveTimeMs);\n    }\n  };\n\n  readonly reportTotalLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportLastTotalLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportUpstreamClockSkew = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(estimateUpstreamClockSkewMs(last));\n    }\n  };\n}\n\n/**\n * Estimates how far the upstream database's clock is from ours, in\n * milliseconds, positive meaning upstream is ahead.\n *\n * `sendTimeMs` and `receiveTimeMs` are both stamped in the\n * replication-manager, either side of `commitTimeMs`, which upstream stamps.\n * Assuming the upstream commit fell midway through that round trip, the\n * difference between where it actually landed and that midpoint is the clock\n * offset -- the same round-trip estimate NTP uses.\n *\n * The estimate is biased by any asymmetry between the two legs: the outbound\n * leg is a SQL round trip, while the return leg additionally waits on a WAL\n * flush and the walsender (which the pre-PG-17 path notes can add 50~100ms on\n * an idle database). So treat this as a detector of gross skew -- the kind\n * that invalidates sync.e2e_serving_lag -- and not as a sub-100ms correction.\n *\n * The rigorous bound, if a tighter reading is ever needed: the commit\n * genuinely happened between send and receive, so a `commitTimeMs` outside\n * that window proves skew of at least the amount by which it falls outside,\n * independent of any latency assumption.\n */\nexport function estimateUpstreamClockSkewMs({\n  sendTimeMs,\n  commitTimeMs,\n  receiveTimeMs,\n}: {\n  sendTimeMs: number;\n  commitTimeMs: number;\n  receiveTimeMs: number;\n}): number {\n  return commitTimeMs - (sendTimeMs + receiveTimeMs) / 2;\n}\n\n/**\n * Returns whether the `report` is the signal sent by the change-streamer,\n * once the subscriber is caught up, to indicate that lag reporting is\n * disabled upstream (and thus readiness cannot be gated on lag reports).\n *\n * This is a lag report with a `nextSendTimeMs` of 0 and no `lastTimings`.\n */\nexport function isLagReportingDisabledSignal(\n  report: ReplicationReport,\n): boolean {\n  return report.nextSendTimeMs === 0 && report.lastTimings === undefined;\n}\n"],"mappings":";;;AAOA,IAAM,uCACJ,QAAQ,IAAI,8CAA8C;AAS5D,IAAa,4BAAb,MAAuC;CACrC;CACA;CACA,aAAsB,SAAS;CAC/B;CACA;CACA,SAAS;CACT,QAAkC;CAElC,YACE,IACA,EACE,uBAAuB,KACvB,sBAAsB,KACtB,eAAe,YACf,iBAAiB,iBACiB,CAAC,GACrC;EACA,KAAK,MAAM;EACX,KAAK,wBAAwB;EAC7B,KAAK,gBAAgB;EAErB,KAAK,kBAAkB,mBAEnB,KAAK,UACH,QACA,gEAAgE,KAAK,sBAAsB,IAC7F,GACF,mBACF;EAEA,KAAK,iBAAiB,QAAQ;CAChC;CAEA,QAAuB;EACrB,OAAO,KAAK,WAAW;CACzB;CAEA,OAAO,QAA2B;EAChC,IAAI,6BAA6B,MAAM,GAAG;GACxC,KAAK,UACH,QACA,uEACF;GACA;EACF;EACA,KAAK,QAAQ;EAEb,MAAM,EAAC,gBAAe;EACtB,IAAI,aAAa;GACf,MAAM,QAAQ,YAAY,kBAAkB,YAAY;GACxD,IAAI,QAAQ,KAAQ;IAClB,MAAM,MAAM,KAAK,SAAS,SAAS;IACnC,KAAK,IAAI,IAAI,GAAG,yBAAyB,MAAM,MAAM;KACnD,OAAO,KAAK;KACZ;IACF,CAAC;GACH,OAAO,IAAI,QAAQ,KACjB,KAAK,IAAI,OAAO,oBAAoB,MAAM,MAAM;IAC9C,OAAO,KAAK;IACZ;GACF,CAAC;GAEH,IAAI,sCACF,KAAK,IAAI,QAAQ,mBAAmB,MAAM,MAAM;IAC9C,OAAO,KAAK;IACZ;GACF,CAAC;GAEH,IAAI,CAAC,KAAK,UAAU,SAAS,KAAK,uBAChC,KAAK,UACH,QACA,iDAAiD,MAAM,iBAAiB,KAAK,sBAAsB,wBACrG;EAEJ;CACF;CAEA,UAAU,KAAe,SAAiB;EACxC,IAAI,KAAK,QACP;EAEF,KAAK,IAAI,IAAI,GAAG,SAAS,EAAC,YAAY,KAAK,MAAK,CAAC;EACjD,KAAK,SAAS;EACd,KAAK,cAAc,KAAK,eAAe;EACvC,KAAK,WAAW,QAAQ;EAExB,iBAAiB,eAAe,gBAAgB;GAC9C,aACE;GAEF,MAAM;EACR,CAAC,CAAC,CAAC,YAAY,KAAK,iBAAiB;EAErC,iBAAiB,eAAe,eAAe;GAC7C,aACE;GAEF,MAAM;EACR,CAAC,CAAC,CAAC,YAAY,KAAK,gBAAgB;EAEpC,iBAAiB,eAAe,aAAa;GAC3C,aACE;GAIF,MAAM;EACR,CAAC,CAAC,CAAC,YAAY,KAAK,cAAc;EAElC,iBAAiB,eAAe,kBAAkB;GAChD,aACE;GAIF,MAAM;EACR,CAAC,CAAC,CAAC,YAAY,KAAK,kBAAkB;EAEtC,iBAAiB,eAAe,uBAAuB;GACrD,aACE;GASF,MAAM;EACR,CAAC,CAAC,CAAC,YAAY,KAAK,uBAAuB;CAC7C;CAEA,qBAA8B,MAAwB;EACpD,MAAM,OAAO,KAAK,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,gBAAgB,KAAK,UAAU;CAElD;CAEA,oBAA6B,MAAwB;EACnD,MAAM,OAAO,KAAK,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,aAAa;CAEvD;CAEA,kBAA2B,MAAwB;EACjD,MAAM,OAAO,KAAK,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,UAAU;CAEpD;CAEA,sBAA+B,MAAwB;EACrD,MAAM,OAAO,KAAK,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,UAAU;CAEpD;CAEA,2BAAoC,MAAwB;EAC1D,MAAM,OAAO,KAAK,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,4BAA4B,IAAI,CAAC;CAE/C;AACF;;;;;;;;;;;;;;;;;;;;;;AAuBA,SAAgB,4BAA4B,EAC1C,YACA,cACA,iBAKS;CACT,OAAO,gBAAgB,aAAa,iBAAiB;AACvD;;;;;;;;AASA,SAAgB,6BACd,QACS;CACT,OAAO,OAAO,mBAAmB,KAAK,OAAO,gBAAgB,KAAA;AAC/D"}

package/out/zero-cache/src/services/replicator/reporter/report-schema.d.ts

@@ -12,14 +12,7 @@ export declare const changeSourceReportSchema: v.ObjectType<{
     }, undefined>;
     nextSendTimeMs: v.Type<number>;
 }, undefined>;
-export declare const replicationTimingsSchema: v.ObjectType<Omit<{
-    sendTimeMs: v.Type<number>;
-    commitTimeMs: v.Type<number>;
-    receiveTimeMs: v.Type<number>;
-}, "replicateTimeMs"> & {
-    replicateTimeMs: v.Type<number>;
-}, undefined>;
-export declare const replicationReportSchema: v.ObjectType<{
+declare const replicationReportSchema: v.ObjectType<{
     lastTimings: v.Optional<{
         sendTimeMs: number;
         commitTimeMs: number;
@@ -28,8 +21,6 @@ export declare const replicationReportSchema: v.ObjectType<{
     }>;
     nextSendTimeMs: v.Type<number>;
 }, undefined>;
-export type ChangeSourceTimings = v.Infer<typeof changeSourceTimingsSchema>;
-export type ChangeSourceReport = v.Infer<typeof changeSourceReportSchema>;
-export type ReplicationTimings = v.Infer<typeof replicationTimingsSchema>;
 export type ReplicationReport = v.Infer<typeof replicationReportSchema>;
+export {};
 //# sourceMappingURL=report-schema.d.ts.map

package/out/zero-cache/src/services/replicator/reporter/report-schema.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"report-schema.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/replicator/reporter/report-schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AAEzD,eAAO,MAAM,yBAAyB;;;;aAIpC,CAAC;AAEH,eAAO,MAAM,wBAAwB;;;;;;;aAGnC,CAAC;AAEH,eAAO,MAAM,wBAAwB;;;;;;aAEnC,CAAC;AAEH,eAAO,MAAM,uBAAuB;;;;;;;;aAGlC,CAAC;AAEH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAE1E,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAC1E,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC"}
+{"version":3,"file":"report-schema.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/replicator/reporter/report-schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AAEzD,eAAO,MAAM,yBAAyB;;;;aAIpC,CAAC;AAEH,eAAO,MAAM,wBAAwB;;;;;;;aAGnC,CAAC;AAMH,QAAA,MAAM,uBAAuB;;;;;;;;aAG3B,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC"}

package/out/zero-cache/src/services/replicator/reporter/report-schema.js

@@ -15,6 +15,6 @@ valita_exports.object({
 	nextSendTimeMs: valita_exports.number()
 });
 //#endregion
-export { changeSourceReportSchema, changeSourceTimingsSchema, replicationTimingsSchema };
+export { changeSourceReportSchema, changeSourceTimingsSchema };
 
 //# sourceMappingURL=report-schema.js.map

package/out/zero-cache/src/services/replicator/reporter/report-schema.js.map

@@ -1 +1 @@
-{"version":3,"file":"report-schema.js","names":["v"],"sources":["../../../../../../../zero-cache/src/services/replicator/reporter/report-schema.ts"],"sourcesContent":["import * as v from '../../../../../shared/src/valita.ts';\n\nexport const changeSourceTimingsSchema = v.object({\n  sendTimeMs: v.number(),\n  commitTimeMs: v.number(),\n  receiveTimeMs: v.number(),\n});\n\nexport const changeSourceReportSchema = v.object({\n  lastTimings: changeSourceTimingsSchema,\n  nextSendTimeMs: v.number(),\n});\n\nexport const replicationTimingsSchema = changeSourceTimingsSchema.extend({\n  replicateTimeMs: v.number(),\n});\n\nexport const replicationReportSchema = v.object({\n  lastTimings: replicationTimingsSchema.optional(),\n  nextSendTimeMs: v.number(),\n});\n\nexport type ChangeSourceTimings = v.Infer<typeof changeSourceTimingsSchema>;\nexport type ChangeSourceReport = v.Infer<typeof changeSourceReportSchema>;\n\nexport type ReplicationTimings = v.Infer<typeof replicationTimingsSchema>;\nexport type ReplicationReport = v.Infer<typeof replicationReportSchema>;\n"],"mappings":";;AAEA,IAAa,4BAA4BA,eAAE,OAAO;CAChD,YAAYA,eAAE,OAAO;CACrB,cAAcA,eAAE,OAAO;CACvB,eAAeA,eAAE,OAAO;AAC1B,CAAC;AAED,IAAa,2BAA2BA,eAAE,OAAO;CAC/C,aAAa;CACb,gBAAgBA,eAAE,OAAO;AAC3B,CAAC;AAED,IAAa,2BAA2B,0BAA0B,OAAO,EACvE,iBAAiBA,eAAE,OAAO,EAC5B,CAAC;AAEsCA,eAAE,OAAO;CAC9C,aAAa,yBAAyB,SAAS;CAC/C,gBAAgBA,eAAE,OAAO;AAC3B,CAAC"}
+{"version":3,"file":"report-schema.js","names":["v"],"sources":["../../../../../../../zero-cache/src/services/replicator/reporter/report-schema.ts"],"sourcesContent":["import * as v from '../../../../../shared/src/valita.ts';\n\nexport const changeSourceTimingsSchema = v.object({\n  sendTimeMs: v.number(),\n  commitTimeMs: v.number(),\n  receiveTimeMs: v.number(),\n});\n\nexport const changeSourceReportSchema = v.object({\n  lastTimings: changeSourceTimingsSchema,\n  nextSendTimeMs: v.number(),\n});\n\nconst replicationTimingsSchema = changeSourceTimingsSchema.extend({\n  replicateTimeMs: v.number(),\n});\n\nconst replicationReportSchema = v.object({\n  lastTimings: replicationTimingsSchema.optional(),\n  nextSendTimeMs: v.number(),\n});\n\nexport type ReplicationReport = v.Infer<typeof replicationReportSchema>;\n"],"mappings":";;AAEA,IAAa,4BAA4BA,eAAE,OAAO;CAChD,YAAYA,eAAE,OAAO;CACrB,cAAcA,eAAE,OAAO;CACvB,eAAeA,eAAE,OAAO;AAC1B,CAAC;AAED,IAAa,2BAA2BA,eAAE,OAAO;CAC/C,aAAa;CACb,gBAAgBA,eAAE,OAAO;AAC3B,CAAC;AAED,IAAM,2BAA2B,0BAA0B,OAAO,EAChE,iBAAiBA,eAAE,OAAO,EAC5B,CAAC;AAE+BA,eAAE,OAAO;CACvC,aAAa,yBAAyB,SAAS;CAC/C,gBAAgBA,eAAE,OAAO;AAC3B,CAAC"}

package/out/zero-cache/src/services/replicator/schema/change-log.d.ts

@@ -1,5 +1,4 @@
 import { type JSONValue } from '../../../../../shared/src/bigint-json.ts';
-import * as v from '../../../../../shared/src/valita.ts';
 import type { Database } from '../../../../../zqlite/src/db.ts';
 import type { LexiVersion } from '../../../types/lexi-version.ts';
 import type { LiteRowKey } from '../../../types/lite.ts';
@@ -62,14 +61,6 @@ export type ChangeLogEntry = {
  * {@link ChangeLog}).
  */
 export declare function parseChangeLogEntry(stateVersion: string, table: string, rowKey: string, op: string): ChangeLogEntry;
-declare const rawChangeLogEntrySchema: v.ObjectType<{
-    stateVersion: v.Type<string>;
-    table: v.Type<string>;
-    rowKey: v.Type<string>;
-    op: v.Type<"d" | "r" | "s" | "t">;
-    backfillingColumnVersions: v.Type<Record<string, string>>;
-}, undefined>;
-export type RawChangeLogEntry = v.Infer<typeof rawChangeLogEntrySchema>;
 export declare class ChangeLog {
     #private;
     constructor(db: Database);
@@ -96,5 +87,4 @@ export declare class ChangeLog {
     logTruncateOp(version: LexiVersion, table: string): void;
     logResetOp(version: LexiVersion, table: string): void;
 }
-export {};
 //# sourceMappingURL=change-log.d.ts.map

package/out/zero-cache/src/services/replicator/schema/change-log.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-log.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/replicator/schema/change-log.ts"],"names":[],"mappings":"AACA,OAAO,EAGL,KAAK,SAAS,EACf,MAAM,0CAA0C,CAAC;AAClD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AACzD,OAAO,KAAK,EAAC,QAAQ,EAAY,MAAM,iCAAiC,CAAC;AACzE,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAChE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,wBAAwB,CAAC;AAGvD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,eAAO,MAAM,MAAM,MAAM,CAAC;AAC1B,eAAO,MAAM,MAAM,MAAM,CAAC;AAC1B,eAAO,MAAM,WAAW,MAAM,CAAC;AAC/B,eAAO,MAAM,QAAQ,MAAM,CAAC;AAG5B,eAAO,MAAM,uBAAuB,sbAmCjC,CAAC;AAEJ;;;;;GAKG;AACH,MAAM,MAAM,cAAc,GACtB;IACE,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;IACrD,QAAQ,CAAC,EAAE,EAAE,OAAO,MAAM,GAAG,OAAO,MAAM,CAAC;CAC5C,GACD;IACE,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAEvB,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC;IACtB,QAAQ,CAAC,EAAE,EAAE,OAAO,WAAW,GAAG,OAAO,QAAQ,CAAC;CACnD,CAAC;AAEN;;;;;GAKG;AACH,wBAAgB,mBAAmB,CACjC,YAAY,EAAE,MAAM,EACpB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,MAAM,GACT,cAAc,CAWhB;AAWD,QAAA,MAAM,uBAAuB;;;;;;aAQ3B,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,qBAAa,SAAS;;IAMpB,YAAY,EAAE,EAAE,QAAQ,EAqCvB;IAED;;;;;;;;;;OAUG;IACH,QAAQ,CACN,OAAO,EAAE,WAAW,EACpB,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,UAAU,EACf,UAAU,EAAE,MAAM,EAAE,GAAG,SAAS,GAC/B,MAAM,CAER;IAED,WAAW,CACT,OAAO,EAAE,WAAW,EACpB,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,UAAU,GACd,MAAM,CAKR;IAED,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU;;;;;;kBAM5C;IA8BD,aAAa,CAAC,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,QAEhD;IAED,UAAU,CAAC,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,QAE7C;CACF"}
+{"version":3,"file":"change-log.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/replicator/schema/change-log.ts"],"names":[],"mappings":"AACA,OAAO,EAGL,KAAK,SAAS,EACf,MAAM,0CAA0C,CAAC;AAElD,OAAO,KAAK,EAAC,QAAQ,EAAY,MAAM,iCAAiC,CAAC;AACzE,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAChE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,wBAAwB,CAAC;AAGvD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,eAAO,MAAM,MAAM,MAAM,CAAC;AAC1B,eAAO,MAAM,MAAM,MAAM,CAAC;AAC1B,eAAO,MAAM,WAAW,MAAM,CAAC;AAC/B,eAAO,MAAM,QAAQ,MAAM,CAAC;AAG5B,eAAO,MAAM,uBAAuB,sbAmCjC,CAAC;AAEJ;;;;;GAKG;AACH,MAAM,MAAM,cAAc,GACtB;IACE,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;IACrD,QAAQ,CAAC,EAAE,EAAE,OAAO,MAAM,GAAG,OAAO,MAAM,CAAC;CAC5C,GACD;IACE,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAEvB,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC;IACtB,QAAQ,CAAC,EAAE,EAAE,OAAO,WAAW,GAAG,OAAO,QAAQ,CAAC;CACnD,CAAC;AAEN;;;;;GAKG;AACH,wBAAgB,mBAAmB,CACjC,YAAY,EAAE,MAAM,EACpB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,MAAM,GACT,cAAc,CAWhB;AAqBD,qBAAa,SAAS;;IAMpB,YAAY,EAAE,EAAE,QAAQ,EAqCvB;IAED;;;;;;;;;;OAUG;IACH,QAAQ,CACN,OAAO,EAAE,WAAW,EACpB,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,UAAU,EACf,UAAU,EAAE,MAAM,EAAE,GAAG,SAAS,GAC/B,MAAM,CAER;IAED,WAAW,CACT,OAAO,EAAE,WAAW,EACpB,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,UAAU,GACd,MAAM,CAKR;IAED,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU;;;;;;kBAM5C;IA8BD,aAAa,CAAC,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,QAEhD;IAED,UAAU,CAAC,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,QAE7C;CACF"}