@rocicorp/zero 1.12.0-canary.0 → 1.12.0-canary.10

package/out/zero-cache/src/services/replicator/schema/column-metadata.js.map

@@ -1 +1 @@
-{"version":3,"file":"column-metadata.js","names":["#instances","#insertStmt","#updateStmt","#clearBackfillStmt","#deleteColumnStmt","#deleteTableStmt","#renameTableStmt","#getColumnStmt","#getTableStmt","#hasTableStmt","#insertMetadata"],"sources":["../../../../../../../zero-cache/src/services/replicator/schema/column-metadata.ts"],"sourcesContent":["/**\n * Column metadata table for storing upstream PostgreSQL schema information.\n *\n * Previously, upstream type metadata was embedded in SQLite column type strings\n * using pipe-delimited notation (e.g., \"int8|NOT_NULL|TEXT_ENUM\"). This caused\n * issues with SQLite type affinity and made schema inspection difficult.\n *\n * This table stores that metadata separately, allowing SQLite columns to use\n * plain type names while preserving all necessary upstream type information.\n */\n\nimport type {Database, Statement} from '../../../../../zqlite/src/db.ts';\nimport {isArrayColumn, isEnumColumn} from '../../../db/pg-to-lite.ts';\nimport type {ColumnSpec, LiteTableSpec} from '../../../db/specs.ts';\nimport {\n  isArray as checkIsArray,\n  isEnum as checkIsEnum,\n  liteTypeString,\n  nullableUpstream,\n  upstreamDataType,\n} from '../../../types/lite.ts';\nimport type {BackfillID} from '../../change-source/protocol/current.ts';\n\n/**\n * Structured column metadata, replacing the old pipe-delimited string format.\n */\nexport interface ColumnMetadata {\n  /** PostgreSQL type name, e.g., 'int8', 'varchar', 'text[]', 'user_role' */\n  upstreamType: string;\n  isNotNull: boolean;\n  isEnum: boolean;\n  isArray: boolean;\n  /** Maximum character length for varchar/char types */\n  characterMaxLength?: number | null;\n  isBackfilling: boolean;\n}\n\ntype ColumnMetadataRow = {\n  upstream_type: string;\n  is_not_null: number;\n  is_enum: number;\n  is_array: number;\n  character_max_length: number | null;\n  backfill: string | null;\n};\n\nexport const CREATE_COLUMN_METADATA_TABLE = `\n  CREATE TABLE \"_zero.column_metadata\" (\n    table_name TEXT NOT NULL,\n    column_name TEXT NOT NULL,\n    upstream_type TEXT NOT NULL,\n    is_not_null INTEGER NOT NULL,\n    is_enum INTEGER NOT NULL,\n    is_array INTEGER NOT NULL,\n    character_max_length INTEGER,\n    backfill TEXT,\n    PRIMARY KEY (table_name, column_name)\n  );\n`;\n\n/**\n * Efficient column metadata store that prepares all statements upfront.\n * Use this class to avoid re-preparing statements on every operation.\n *\n * Access via `ColumnMetadataStore.getInstance(db)`, which returns `undefined`\n * if the metadata table doesn't exist yet.\n */\nexport class ColumnMetadataStore {\n  static #instances = new WeakMap<Database, ColumnMetadataStore>();\n\n  readonly #insertStmt: Statement;\n  readonly #updateStmt: Statement;\n  readonly #clearBackfillStmt: Statement;\n  readonly #deleteColumnStmt: Statement;\n  readonly #deleteTableStmt: Statement;\n  readonly #renameTableStmt: Statement;\n  readonly #getColumnStmt: Statement;\n  readonly #getTableStmt: Statement;\n  readonly #hasTableStmt: Statement;\n\n  private constructor(db: Database) {\n    this.#insertStmt = db.prepare(`\n      INSERT INTO \"_zero.column_metadata\"\n        (table_name, column_name, upstream_type, is_not_null, is_enum, is_array, character_max_length, backfill)\n      VALUES (?, ?, ?, ?, ?, ?, ?, ?)\n    `);\n\n    this.#updateStmt = db.prepare(`\n      UPDATE \"_zero.column_metadata\"\n      SET column_name = ?,\n          upstream_type = ?,\n          is_not_null = ?,\n          is_enum = ?,\n          is_array = ?,\n          character_max_length = ?\n      WHERE table_name = ? AND column_name = ?\n    `);\n\n    this.#clearBackfillStmt = db.prepare(/*sql*/ `\n      UPDATE \"_zero.column_metadata\"\n      SET backfill = NULL\n      WHERE table_name = ? AND column_name = ?\n    `);\n\n    this.#deleteColumnStmt = db.prepare(`\n      DELETE FROM \"_zero.column_metadata\"\n      WHERE table_name = ? AND column_name = ?\n    `);\n\n    this.#deleteTableStmt = db.prepare(`\n      DELETE FROM \"_zero.column_metadata\"\n      WHERE table_name = ?\n    `);\n\n    this.#renameTableStmt = db.prepare(`\n      UPDATE \"_zero.column_metadata\"\n      SET table_name = ?\n      WHERE table_name = ?\n    `);\n\n    this.#getColumnStmt = db.prepare(`\n      SELECT upstream_type, is_not_null, is_enum, is_array, character_max_length, backfill\n      FROM \"_zero.column_metadata\"\n      WHERE table_name = ? AND column_name = ?\n    `);\n\n    this.#getTableStmt = db.prepare(`\n      SELECT column_name, upstream_type, is_not_null, is_enum, is_array, character_max_length, backfill\n      FROM \"_zero.column_metadata\"\n      WHERE table_name = ?\n      ORDER BY column_name\n    `);\n\n    this.#hasTableStmt = db.prepare(`\n      SELECT 1 FROM sqlite_master\n      WHERE type = 'table' AND name = '_zero.column_metadata'\n    `);\n  }\n\n  /**\n   * Gets the singleton instance of ColumnMetadataStore for the given database.\n   * Returns `undefined` if the metadata table doesn't exist yet.\n   */\n  static getInstance(db: Database): ColumnMetadataStore | undefined {\n    let instance = ColumnMetadataStore.#instances.get(db);\n    if (!instance) {\n      // Check if table exists\n      const tableExists = db\n        .prepare(\n          `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_zero.column_metadata'`,\n        )\n        .get();\n\n      if (!tableExists) {\n        return undefined;\n      }\n\n      instance = new ColumnMetadataStore(db);\n      ColumnMetadataStore.#instances.set(db, instance);\n    }\n    return instance;\n  }\n\n  insert(\n    tableName: string,\n    columnName: string,\n    spec: ColumnSpec,\n    backfill?: BackfillID,\n  ): void {\n    const metadata = pgColumnSpecToMetadata(spec);\n    this.#insertMetadata(tableName, columnName, metadata, backfill);\n  }\n\n  #insertMetadata(\n    tableName: string,\n    columnName: string,\n    metadata: Omit<ColumnMetadata, 'isBackfilling'>,\n    backfill?: BackfillID,\n  ): void {\n    this.#insertStmt.run(\n      tableName,\n      columnName,\n      metadata.upstreamType,\n      metadata.isNotNull ? 1 : 0,\n      metadata.isEnum ? 1 : 0,\n      metadata.isArray ? 1 : 0,\n      metadata.characterMaxLength ?? null,\n      backfill ? JSON.stringify(backfill) : null,\n    );\n  }\n\n  update(\n    tableName: string,\n    oldColumnName: string,\n    newColumnName: string,\n    spec: ColumnSpec,\n  ): void {\n    const metadata = pgColumnSpecToMetadata(spec);\n    this.#updateStmt.run(\n      newColumnName,\n      metadata.upstreamType,\n      metadata.isNotNull ? 1 : 0,\n      metadata.isEnum ? 1 : 0,\n      metadata.isArray ? 1 : 0,\n      metadata.characterMaxLength ?? null,\n      tableName,\n      oldColumnName,\n    );\n  }\n\n  clearBackfilling(tableName: string, columnName: string): void {\n    this.#clearBackfillStmt.run(tableName, columnName);\n  }\n\n  deleteColumn(tableName: string, columnName: string): void {\n    this.#deleteColumnStmt.run(tableName, columnName);\n  }\n\n  deleteTable(tableName: string): void {\n    this.#deleteTableStmt.run(tableName);\n  }\n\n  renameTable(oldTableName: string, newTableName: string): void {\n    this.#renameTableStmt.run(newTableName, oldTableName);\n  }\n\n  getColumn(tableName: string, columnName: string): ColumnMetadata | undefined {\n    const row = this.#getColumnStmt.get(tableName, columnName) as\n      | ColumnMetadataRow\n      | undefined;\n\n    if (!row) {\n      return undefined;\n    }\n\n    return {\n      upstreamType: row.upstream_type,\n      isNotNull: row.is_not_null !== 0,\n      isEnum: row.is_enum !== 0,\n      isArray: row.is_array !== 0,\n      characterMaxLength: row.character_max_length,\n      isBackfilling: row.backfill !== null,\n    };\n  }\n\n  getTable(tableName: string): Map<string, ColumnMetadata> {\n    const rows = this.#getTableStmt.all(tableName) as Array<\n      ColumnMetadataRow & {column_name: string}\n    >;\n\n    const metadata = new Map<string, ColumnMetadata>();\n    for (const row of rows) {\n      metadata.set(row.column_name, {\n        upstreamType: row.upstream_type,\n        isNotNull: row.is_not_null !== 0,\n        isEnum: row.is_enum !== 0,\n        isArray: row.is_array !== 0,\n        characterMaxLength: row.character_max_length,\n        isBackfilling: row.backfill !== null,\n      });\n    }\n\n    return metadata;\n  }\n\n  hasTable(): boolean {\n    const result = this.#hasTableStmt.get();\n    return result !== undefined;\n  }\n}\n\n/**\n * Populates metadata table from existing tables that use pipe notation.\n * This is used during migration v8 to backfill the metadata table.\n */\nexport function populateFromExistingTables(\n  db: Database,\n  tables: LiteTableSpec[],\n): void {\n  // The backfill column is not relevant here, and does not exist on\n  // older versions of the replica.\n  const legacyInsertStmt = db.prepare(`\n      INSERT INTO \"_zero.column_metadata\"\n        (table_name, column_name, upstream_type, is_not_null, is_enum, is_array, character_max_length)\n      VALUES (?, ?, ?, ?, ?, ?, ?)\n    `);\n\n  for (const table of tables) {\n    for (const [columnName, columnSpec] of Object.entries(table.columns)) {\n      const metadata = liteTypeStringToMetadata(\n        columnSpec.dataType,\n        columnSpec.characterMaximumLength,\n      );\n      legacyInsertStmt.run(\n        table.name,\n        columnName,\n        metadata.upstreamType,\n        metadata.isNotNull ? 1 : 0,\n        metadata.isEnum ? 1 : 0,\n        metadata.isArray ? 1 : 0,\n        metadata.characterMaxLength ?? null,\n      );\n    }\n  }\n}\n\n/**\n * Converts pipe-delimited LiteTypeString to structured ColumnMetadata.\n * This is a compatibility helper for the migration period.\n */\nexport function liteTypeStringToMetadata(\n  liteTypeString: string,\n  characterMaxLength?: number | null,\n): ColumnMetadata {\n  const baseType = upstreamDataType(liteTypeString);\n  const isArrayType = checkIsArray(liteTypeString);\n\n  // Reconstruct the full upstream type including array notation\n  // For new-style arrays like 'text[]', upstreamDataType returns 'text[]'\n  // For old-style arrays like 'int4|NOT_NULL[]', upstreamDataType returns 'int4', so we append '[]'\n  const fullUpstreamType =\n    isArrayType && !baseType.includes('[]') ? `${baseType}[]` : baseType;\n\n  return {\n    upstreamType: fullUpstreamType,\n    isNotNull: !nullableUpstream(liteTypeString),\n    isEnum: checkIsEnum(liteTypeString),\n    isArray: isArrayType,\n    characterMaxLength: characterMaxLength ?? null,\n    isBackfilling: false,\n  };\n}\n\n/**\n * Converts structured ColumnMetadata back to pipe-delimited LiteTypeString.\n * This is a compatibility helper for the migration period.\n */\nexport function metadataToLiteTypeString(metadata: ColumnMetadata): string {\n  return liteTypeString(\n    metadata.upstreamType,\n    metadata.isNotNull,\n    metadata.isEnum,\n    metadata.isArray,\n  );\n}\n\n/**\n * Converts PostgreSQL ColumnSpec to structured ColumnMetadata.\n * Used during replication to populate the metadata table from upstream schema.\n *\n * Uses the same logic as liteTypeString() and mapPostgresToLiteColumn() via shared helpers.\n */\nexport function pgColumnSpecToMetadata(spec: ColumnSpec): ColumnMetadata {\n  return {\n    upstreamType: spec.dataType,\n    isNotNull: spec.notNull ?? false,\n    isEnum: isEnumColumn(spec),\n    isArray: isArrayColumn(spec),\n    characterMaxLength: spec.characterMaximumLength ?? null,\n    isBackfilling: false,\n  };\n}\n"],"mappings":";;;AA8CA,IAAa,+BAA+B;;;;;;;;;;;;;;;;;;;;AAqB5C,IAAa,sBAAb,MAAa,oBAAoB;CAC/B,OAAOA,6BAAa,IAAI,QAAuC;CAE/D;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAoB,IAAc;EAChC,KAAKC,cAAc,GAAG,QAAQ;;;;KAI7B;EAED,KAAKC,cAAc,GAAG,QAAQ;;;;;;;;;KAS7B;EAED,KAAKC,qBAAqB,GAAG,QAAgB;;;;KAI5C;EAED,KAAKC,oBAAoB,GAAG,QAAQ;;;KAGnC;EAED,KAAKC,mBAAmB,GAAG,QAAQ;;;KAGlC;EAED,KAAKC,mBAAmB,GAAG,QAAQ;;;;KAIlC;EAED,KAAKC,iBAAiB,GAAG,QAAQ;;;;KAIhC;EAED,KAAKC,gBAAgB,GAAG,QAAQ;;;;;KAK/B;EAED,KAAKC,gBAAgB,GAAG,QAAQ;;;KAG/B;CACH;;;;;CAMA,OAAO,YAAY,IAA+C;EAChE,IAAI,WAAW,oBAAoBT,WAAW,IAAI,EAAE;EACpD,IAAI,CAAC,UAAU;GAQb,IAAI,CANgB,GACjB,QACC,qFACF,EACC,IAEE,GACH;GAGF,WAAW,IAAI,oBAAoB,EAAE;GACrC,oBAAoBA,WAAW,IAAI,IAAI,QAAQ;EACjD;EACA,OAAO;CACT;CAEA,OACE,WACA,YACA,MACA,UACM;EACN,MAAM,WAAW,uBAAuB,IAAI;EAC5C,KAAKU,gBAAgB,WAAW,YAAY,UAAU,QAAQ;CAChE;CAEA,gBACE,WACA,YACA,UACA,UACM;EACN,KAAKT,YAAY,IACf,WACA,YACA,SAAS,cACT,SAAS,YAAY,IAAI,GACzB,SAAS,SAAS,IAAI,GACtB,SAAS,UAAU,IAAI,GACvB,SAAS,sBAAsB,MAC/B,WAAW,KAAK,UAAU,QAAQ,IAAI,IACxC;CACF;CAEA,OACE,WACA,eACA,eACA,MACM;EACN,MAAM,WAAW,uBAAuB,IAAI;EAC5C,KAAKC,YAAY,IACf,eACA,SAAS,cACT,SAAS,YAAY,IAAI,GACzB,SAAS,SAAS,IAAI,GACtB,SAAS,UAAU,IAAI,GACvB,SAAS,sBAAsB,MAC/B,WACA,aACF;CACF;CAEA,iBAAiB,WAAmB,YAA0B;EAC5D,KAAKC,mBAAmB,IAAI,WAAW,UAAU;CACnD;CAEA,aAAa,WAAmB,YAA0B;EACxD,KAAKC,kBAAkB,IAAI,WAAW,UAAU;CAClD;CAEA,YAAY,WAAyB;EACnC,KAAKC,iBAAiB,IAAI,SAAS;CACrC;CAEA,YAAY,cAAsB,cAA4B;EAC5D,KAAKC,iBAAiB,IAAI,cAAc,YAAY;CACtD;CAEA,UAAU,WAAmB,YAAgD;EAC3E,MAAM,MAAM,KAAKC,eAAe,IAAI,WAAW,UAAU;EAIzD,IAAI,CAAC,KACH;EAGF,OAAO;GACL,cAAc,IAAI;GAClB,WAAW,IAAI,gBAAgB;GAC/B,QAAQ,IAAI,YAAY;GACxB,SAAS,IAAI,aAAa;GAC1B,oBAAoB,IAAI;GACxB,eAAe,IAAI,aAAa;EAClC;CACF;CAEA,SAAS,WAAgD;EACvD,MAAM,OAAO,KAAKC,cAAc,IAAI,SAAS;EAI7C,MAAM,2BAAW,IAAI,IAA4B;EACjD,KAAK,MAAM,OAAO,MAChB,SAAS,IAAI,IAAI,aAAa;GAC5B,cAAc,IAAI;GAClB,WAAW,IAAI,gBAAgB;GAC/B,QAAQ,IAAI,YAAY;GACxB,SAAS,IAAI,aAAa;GAC1B,oBAAoB,IAAI;GACxB,eAAe,IAAI,aAAa;EAClC,CAAC;EAGH,OAAO;CACT;CAEA,WAAoB;EAElB,OADe,KAAKC,cAAc,IAC3B,MAAW,KAAA;CACpB;AACF;;;;;AAMA,SAAgB,2BACd,IACA,QACM;CAGN,MAAM,mBAAmB,GAAG,QAAQ;;;;KAIjC;CAEH,KAAK,MAAM,SAAS,QAClB,KAAK,MAAM,CAAC,YAAY,eAAe,OAAO,QAAQ,MAAM,OAAO,GAAG;EACpE,MAAM,WAAW,yBACf,WAAW,UACX,WAAW,sBACb;EACA,iBAAiB,IACf,MAAM,MACN,YACA,SAAS,cACT,SAAS,YAAY,IAAI,GACzB,SAAS,SAAS,IAAI,GACtB,SAAS,UAAU,IAAI,GACvB,SAAS,sBAAsB,IACjC;CACF;AAEJ;;;;;AAMA,SAAgB,yBACd,gBACA,oBACgB;CAChB,MAAM,WAAW,iBAAiB,cAAc;CAChD,MAAM,cAAc,QAAa,cAAc;CAQ/C,OAAO;EACL,cAHA,eAAe,CAAC,SAAS,SAAS,IAAI,IAAI,GAAG,SAAS,MAAM;EAI5D,WAAW,CAAC,iBAAiB,cAAc;EAC3C,QAAQ,OAAY,cAAc;EAClC,SAAS;EACT,oBAAoB,sBAAsB;EAC1C,eAAe;CACjB;AACF;;;;;AAMA,SAAgB,yBAAyB,UAAkC;CACzE,OAAO,eACL,SAAS,cACT,SAAS,WACT,SAAS,QACT,SAAS,OACX;AACF;;;;;;;AAQA,SAAgB,uBAAuB,MAAkC;CACvE,OAAO;EACL,cAAc,KAAK;EACnB,WAAW,KAAK,WAAW;EAC3B,QAAQ,aAAa,IAAI;EACzB,SAAS,cAAc,IAAI;EAC3B,oBAAoB,KAAK,0BAA0B;EACnD,eAAe;CACjB;AACF"}
+{"version":3,"file":"column-metadata.js","names":["checkIsArray","checkIsEnum"],"sources":["../../../../../../../zero-cache/src/services/replicator/schema/column-metadata.ts"],"sourcesContent":["/**\n * Column metadata table for storing upstream PostgreSQL schema information.\n *\n * Previously, upstream type metadata was embedded in SQLite column type strings\n * using pipe-delimited notation (e.g., \"int8|NOT_NULL|TEXT_ENUM\"). This caused\n * issues with SQLite type affinity and made schema inspection difficult.\n *\n * This table stores that metadata separately, allowing SQLite columns to use\n * plain type names while preserving all necessary upstream type information.\n */\n\nimport type {Database, Statement} from '../../../../../zqlite/src/db.ts';\nimport {isArrayColumn, isEnumColumn} from '../../../db/pg-to-lite.ts';\nimport type {ColumnSpec, LiteTableSpec} from '../../../db/specs.ts';\nimport {\n  isArray as checkIsArray,\n  isEnum as checkIsEnum,\n  liteTypeString,\n  nullableUpstream,\n  upstreamDataType,\n} from '../../../types/lite.ts';\nimport type {BackfillID} from '../../change-source/protocol/current.ts';\n\n/**\n * Structured column metadata, replacing the old pipe-delimited string format.\n */\nexport interface ColumnMetadata {\n  /** PostgreSQL type name, e.g., 'int8', 'varchar', 'text[]', 'user_role' */\n  upstreamType: string;\n  isNotNull: boolean;\n  isEnum: boolean;\n  isArray: boolean;\n  /** Maximum character length for varchar/char types */\n  characterMaxLength?: number | null;\n  isBackfilling: boolean;\n}\n\ntype ColumnMetadataRow = {\n  upstream_type: string;\n  is_not_null: number;\n  is_enum: number;\n  is_array: number;\n  character_max_length: number | null;\n  backfill: string | null;\n};\n\nexport const CREATE_COLUMN_METADATA_TABLE = `\n  CREATE TABLE \"_zero.column_metadata\" (\n    table_name TEXT NOT NULL,\n    column_name TEXT NOT NULL,\n    upstream_type TEXT NOT NULL,\n    is_not_null INTEGER NOT NULL,\n    is_enum INTEGER NOT NULL,\n    is_array INTEGER NOT NULL,\n    character_max_length INTEGER,\n    backfill TEXT,\n    PRIMARY KEY (table_name, column_name)\n  );\n`;\n\n/**\n * Efficient column metadata store that prepares all statements upfront.\n * Use this class to avoid re-preparing statements on every operation.\n *\n * Access via `ColumnMetadataStore.getInstance(db)`, which returns `undefined`\n * if the metadata table doesn't exist yet.\n */\nexport class ColumnMetadataStore {\n  static #instances = new WeakMap<Database, ColumnMetadataStore>();\n\n  readonly #insertStmt: Statement;\n  readonly #updateStmt: Statement;\n  readonly #clearBackfillStmt: Statement;\n  readonly #deleteColumnStmt: Statement;\n  readonly #deleteTableStmt: Statement;\n  readonly #renameTableStmt: Statement;\n  readonly #getColumnStmt: Statement;\n  readonly #getTableStmt: Statement;\n  readonly #hasTableStmt: Statement;\n\n  private constructor(db: Database) {\n    this.#insertStmt = db.prepare(`\n      INSERT INTO \"_zero.column_metadata\"\n        (table_name, column_name, upstream_type, is_not_null, is_enum, is_array, character_max_length, backfill)\n      VALUES (?, ?, ?, ?, ?, ?, ?, ?)\n    `);\n\n    this.#updateStmt = db.prepare(`\n      UPDATE \"_zero.column_metadata\"\n      SET column_name = ?,\n          upstream_type = ?,\n          is_not_null = ?,\n          is_enum = ?,\n          is_array = ?,\n          character_max_length = ?\n      WHERE table_name = ? AND column_name = ?\n    `);\n\n    this.#clearBackfillStmt = db.prepare(/*sql*/ `\n      UPDATE \"_zero.column_metadata\"\n      SET backfill = NULL\n      WHERE table_name = ? AND column_name = ?\n    `);\n\n    this.#deleteColumnStmt = db.prepare(`\n      DELETE FROM \"_zero.column_metadata\"\n      WHERE table_name = ? AND column_name = ?\n    `);\n\n    this.#deleteTableStmt = db.prepare(`\n      DELETE FROM \"_zero.column_metadata\"\n      WHERE table_name = ?\n    `);\n\n    this.#renameTableStmt = db.prepare(`\n      UPDATE \"_zero.column_metadata\"\n      SET table_name = ?\n      WHERE table_name = ?\n    `);\n\n    this.#getColumnStmt = db.prepare(`\n      SELECT upstream_type, is_not_null, is_enum, is_array, character_max_length, backfill\n      FROM \"_zero.column_metadata\"\n      WHERE table_name = ? AND column_name = ?\n    `);\n\n    this.#getTableStmt = db.prepare(`\n      SELECT column_name, upstream_type, is_not_null, is_enum, is_array, character_max_length, backfill\n      FROM \"_zero.column_metadata\"\n      WHERE table_name = ?\n      ORDER BY column_name\n    `);\n\n    this.#hasTableStmt = db.prepare(`\n      SELECT 1 FROM sqlite_master\n      WHERE type = 'table' AND name = '_zero.column_metadata'\n    `);\n  }\n\n  /**\n   * Gets the singleton instance of ColumnMetadataStore for the given database.\n   * Returns `undefined` if the metadata table doesn't exist yet.\n   */\n  static getInstance(db: Database): ColumnMetadataStore | undefined {\n    let instance = ColumnMetadataStore.#instances.get(db);\n    if (!instance) {\n      // Check if table exists\n      const tableExists = db\n        .prepare(\n          `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_zero.column_metadata'`,\n        )\n        .get();\n\n      if (!tableExists) {\n        return undefined;\n      }\n\n      instance = new ColumnMetadataStore(db);\n      ColumnMetadataStore.#instances.set(db, instance);\n    }\n    return instance;\n  }\n\n  insert(\n    tableName: string,\n    columnName: string,\n    spec: ColumnSpec,\n    backfill?: BackfillID,\n  ): void {\n    const metadata = pgColumnSpecToMetadata(spec);\n    this.#insertMetadata(tableName, columnName, metadata, backfill);\n  }\n\n  #insertMetadata(\n    tableName: string,\n    columnName: string,\n    metadata: Omit<ColumnMetadata, 'isBackfilling'>,\n    backfill?: BackfillID,\n  ): void {\n    this.#insertStmt.run(\n      tableName,\n      columnName,\n      metadata.upstreamType,\n      metadata.isNotNull ? 1 : 0,\n      metadata.isEnum ? 1 : 0,\n      metadata.isArray ? 1 : 0,\n      metadata.characterMaxLength ?? null,\n      backfill ? JSON.stringify(backfill) : null,\n    );\n  }\n\n  update(\n    tableName: string,\n    oldColumnName: string,\n    newColumnName: string,\n    spec: ColumnSpec,\n  ): void {\n    const metadata = pgColumnSpecToMetadata(spec);\n    this.#updateStmt.run(\n      newColumnName,\n      metadata.upstreamType,\n      metadata.isNotNull ? 1 : 0,\n      metadata.isEnum ? 1 : 0,\n      metadata.isArray ? 1 : 0,\n      metadata.characterMaxLength ?? null,\n      tableName,\n      oldColumnName,\n    );\n  }\n\n  clearBackfilling(tableName: string, columnName: string): void {\n    this.#clearBackfillStmt.run(tableName, columnName);\n  }\n\n  deleteColumn(tableName: string, columnName: string): void {\n    this.#deleteColumnStmt.run(tableName, columnName);\n  }\n\n  deleteTable(tableName: string): void {\n    this.#deleteTableStmt.run(tableName);\n  }\n\n  renameTable(oldTableName: string, newTableName: string): void {\n    this.#renameTableStmt.run(newTableName, oldTableName);\n  }\n\n  getColumn(tableName: string, columnName: string): ColumnMetadata | undefined {\n    const row = this.#getColumnStmt.get(tableName, columnName) as\n      | ColumnMetadataRow\n      | undefined;\n\n    if (!row) {\n      return undefined;\n    }\n\n    return {\n      upstreamType: row.upstream_type,\n      isNotNull: row.is_not_null !== 0,\n      isEnum: row.is_enum !== 0,\n      isArray: row.is_array !== 0,\n      characterMaxLength: row.character_max_length,\n      isBackfilling: row.backfill !== null,\n    };\n  }\n\n  getTable(tableName: string): Map<string, ColumnMetadata> {\n    const rows = this.#getTableStmt.all(tableName) as Array<\n      ColumnMetadataRow & {column_name: string}\n    >;\n\n    const metadata = new Map<string, ColumnMetadata>();\n    for (const row of rows) {\n      metadata.set(row.column_name, {\n        upstreamType: row.upstream_type,\n        isNotNull: row.is_not_null !== 0,\n        isEnum: row.is_enum !== 0,\n        isArray: row.is_array !== 0,\n        characterMaxLength: row.character_max_length,\n        isBackfilling: row.backfill !== null,\n      });\n    }\n\n    return metadata;\n  }\n\n  hasTable(): boolean {\n    const result = this.#hasTableStmt.get();\n    return result !== undefined;\n  }\n}\n\n/**\n * Populates metadata table from existing tables that use pipe notation.\n * This is used during migration v8 to backfill the metadata table.\n */\nexport function populateFromExistingTables(\n  db: Database,\n  tables: LiteTableSpec[],\n): void {\n  // The backfill column is not relevant here, and does not exist on\n  // older versions of the replica.\n  const legacyInsertStmt = db.prepare(`\n      INSERT INTO \"_zero.column_metadata\"\n        (table_name, column_name, upstream_type, is_not_null, is_enum, is_array, character_max_length)\n      VALUES (?, ?, ?, ?, ?, ?, ?)\n    `);\n\n  for (const table of tables) {\n    for (const [columnName, columnSpec] of Object.entries(table.columns)) {\n      const metadata = liteTypeStringToMetadata(\n        columnSpec.dataType,\n        columnSpec.characterMaximumLength,\n      );\n      legacyInsertStmt.run(\n        table.name,\n        columnName,\n        metadata.upstreamType,\n        metadata.isNotNull ? 1 : 0,\n        metadata.isEnum ? 1 : 0,\n        metadata.isArray ? 1 : 0,\n        metadata.characterMaxLength ?? null,\n      );\n    }\n  }\n}\n\n/**\n * Converts pipe-delimited LiteTypeString to structured ColumnMetadata.\n * This is a compatibility helper for the migration period.\n */\nexport function liteTypeStringToMetadata(\n  liteTypeString: string,\n  characterMaxLength?: number | null,\n): ColumnMetadata {\n  const baseType = upstreamDataType(liteTypeString);\n  const isArrayType = checkIsArray(liteTypeString);\n\n  // Reconstruct the full upstream type including array notation\n  // For new-style arrays like 'text[]', upstreamDataType returns 'text[]'\n  // For old-style arrays like 'int4|NOT_NULL[]', upstreamDataType returns 'int4', so we append '[]'\n  const fullUpstreamType =\n    isArrayType && !baseType.includes('[]') ? `${baseType}[]` : baseType;\n\n  return {\n    upstreamType: fullUpstreamType,\n    isNotNull: !nullableUpstream(liteTypeString),\n    isEnum: checkIsEnum(liteTypeString),\n    isArray: isArrayType,\n    characterMaxLength: characterMaxLength ?? null,\n    isBackfilling: false,\n  };\n}\n\n/**\n * Converts structured ColumnMetadata back to pipe-delimited LiteTypeString.\n * This is a compatibility helper for the migration period.\n */\nexport function metadataToLiteTypeString(metadata: ColumnMetadata): string {\n  return liteTypeString(\n    metadata.upstreamType,\n    metadata.isNotNull,\n    metadata.isEnum,\n    metadata.isArray,\n  );\n}\n\n/**\n * Converts PostgreSQL ColumnSpec to structured ColumnMetadata.\n * Used during replication to populate the metadata table from upstream schema.\n *\n * Uses the same logic as liteTypeString() and mapPostgresToLiteColumn() via shared helpers.\n */\nfunction pgColumnSpecToMetadata(spec: ColumnSpec): ColumnMetadata {\n  return {\n    upstreamType: spec.dataType,\n    isNotNull: spec.notNull ?? false,\n    isEnum: isEnumColumn(spec),\n    isArray: isArrayColumn(spec),\n    characterMaxLength: spec.characterMaximumLength ?? null,\n    isBackfilling: false,\n  };\n}\n"],"mappings":";;;AA8CA,IAAa,+BAA+B;;;;;;;;;;;;;;;;;;;;AAqB5C,IAAa,sBAAb,MAAa,oBAAoB;CAC/B,OAAO,6BAAa,IAAI,QAAuC;CAE/D;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAoB,IAAc;EAChC,KAAK,cAAc,GAAG,QAAQ;;;;KAI7B;EAED,KAAK,cAAc,GAAG,QAAQ;;;;;;;;;KAS7B;EAED,KAAK,qBAAqB,GAAG,QAAgB;;;;KAI5C;EAED,KAAK,oBAAoB,GAAG,QAAQ;;;KAGnC;EAED,KAAK,mBAAmB,GAAG,QAAQ;;;KAGlC;EAED,KAAK,mBAAmB,GAAG,QAAQ;;;;KAIlC;EAED,KAAK,iBAAiB,GAAG,QAAQ;;;;KAIhC;EAED,KAAK,gBAAgB,GAAG,QAAQ;;;;;KAK/B;EAED,KAAK,gBAAgB,GAAG,QAAQ;;;KAG/B;CACH;;;;;CAMA,OAAO,YAAY,IAA+C;EAChE,IAAI,WAAW,oBAAoB,WAAW,IAAI,EAAE;EACpD,IAAI,CAAC,UAAU;GAQb,IAAI,CANgB,GACjB,QACC,qFACF,CAAC,CACA,IAEE,GACH;GAGF,WAAW,IAAI,oBAAoB,EAAE;GACrC,oBAAoB,WAAW,IAAI,IAAI,QAAQ;EACjD;EACA,OAAO;CACT;CAEA,OACE,WACA,YACA,MACA,UACM;EACN,MAAM,WAAW,uBAAuB,IAAI;EAC5C,KAAK,gBAAgB,WAAW,YAAY,UAAU,QAAQ;CAChE;CAEA,gBACE,WACA,YACA,UACA,UACM;EACN,KAAK,YAAY,IACf,WACA,YACA,SAAS,cACT,SAAS,YAAY,IAAI,GACzB,SAAS,SAAS,IAAI,GACtB,SAAS,UAAU,IAAI,GACvB,SAAS,sBAAsB,MAC/B,WAAW,KAAK,UAAU,QAAQ,IAAI,IACxC;CACF;CAEA,OACE,WACA,eACA,eACA,MACM;EACN,MAAM,WAAW,uBAAuB,IAAI;EAC5C,KAAK,YAAY,IACf,eACA,SAAS,cACT,SAAS,YAAY,IAAI,GACzB,SAAS,SAAS,IAAI,GACtB,SAAS,UAAU,IAAI,GACvB,SAAS,sBAAsB,MAC/B,WACA,aACF;CACF;CAEA,iBAAiB,WAAmB,YAA0B;EAC5D,KAAK,mBAAmB,IAAI,WAAW,UAAU;CACnD;CAEA,aAAa,WAAmB,YAA0B;EACxD,KAAK,kBAAkB,IAAI,WAAW,UAAU;CAClD;CAEA,YAAY,WAAyB;EACnC,KAAK,iBAAiB,IAAI,SAAS;CACrC;CAEA,YAAY,cAAsB,cAA4B;EAC5D,KAAK,iBAAiB,IAAI,cAAc,YAAY;CACtD;CAEA,UAAU,WAAmB,YAAgD;EAC3E,MAAM,MAAM,KAAK,eAAe,IAAI,WAAW,UAAU;EAIzD,IAAI,CAAC,KACH;EAGF,OAAO;GACL,cAAc,IAAI;GAClB,WAAW,IAAI,gBAAgB;GAC/B,QAAQ,IAAI,YAAY;GACxB,SAAS,IAAI,aAAa;GAC1B,oBAAoB,IAAI;GACxB,eAAe,IAAI,aAAa;EAClC;CACF;CAEA,SAAS,WAAgD;EACvD,MAAM,OAAO,KAAK,cAAc,IAAI,SAAS;EAI7C,MAAM,2BAAW,IAAI,IAA4B;EACjD,KAAK,MAAM,OAAO,MAChB,SAAS,IAAI,IAAI,aAAa;GAC5B,cAAc,IAAI;GAClB,WAAW,IAAI,gBAAgB;GAC/B,QAAQ,IAAI,YAAY;GACxB,SAAS,IAAI,aAAa;GAC1B,oBAAoB,IAAI;GACxB,eAAe,IAAI,aAAa;EAClC,CAAC;EAGH,OAAO;CACT;CAEA,WAAoB;EAElB,OADe,KAAK,cAAc,IAC3B,MAAW,KAAA;CACpB;AACF;;;;;AAMA,SAAgB,2BACd,IACA,QACM;CAGN,MAAM,mBAAmB,GAAG,QAAQ;;;;KAIjC;CAEH,KAAK,MAAM,SAAS,QAClB,KAAK,MAAM,CAAC,YAAY,eAAe,OAAO,QAAQ,MAAM,OAAO,GAAG;EACpE,MAAM,WAAW,yBACf,WAAW,UACX,WAAW,sBACb;EACA,iBAAiB,IACf,MAAM,MACN,YACA,SAAS,cACT,SAAS,YAAY,IAAI,GACzB,SAAS,SAAS,IAAI,GACtB,SAAS,UAAU,IAAI,GACvB,SAAS,sBAAsB,IACjC;CACF;AAEJ;;;;;AAMA,SAAgB,yBACd,gBACA,oBACgB;CAChB,MAAM,WAAW,iBAAiB,cAAc;CAChD,MAAM,cAAcA,QAAa,cAAc;CAQ/C,OAAO;EACL,cAHA,eAAe,CAAC,SAAS,SAAS,IAAI,IAAI,GAAG,SAAS,MAAM;EAI5D,WAAW,CAAC,iBAAiB,cAAc;EAC3C,QAAQC,OAAY,cAAc;EAClC,SAAS;EACT,oBAAoB,sBAAsB;EAC1C,eAAe;CACjB;AACF;;;;;AAMA,SAAgB,yBAAyB,UAAkC;CACzE,OAAO,eACL,SAAS,cACT,SAAS,WACT,SAAS,QACT,SAAS,OACX;AACF;;;;;;;AAQA,SAAS,uBAAuB,MAAkC;CAChE,OAAO;EACL,cAAc,KAAK;EACnB,WAAW,KAAK,WAAW;EAC3B,QAAQ,aAAa,IAAI;EACzB,SAAS,cAAc,IAAI;EAC3B,oBAAoB,KAAK,0BAA0B;EACnD,eAAe;CACjB;AACF"}

package/out/zero-cache/src/services/replicator/schema/replication-state.js

@@ -1,6 +1,6 @@
 import { parse, valita_exports } from "../../../../../shared/src/valita.js";
 import { jsonObjectSchema, stringify } from "../../../../../shared/src/bigint-json.js";
-import "./constants.js";
+import { ZERO_VERSION_COLUMN_NAME } from "./constants.js";
 import { CREATE_COLUMN_METADATA_TABLE } from "./column-metadata.js";
 import { CREATE_TABLE_METADATA_TABLE } from "./table-metadata.js";
 import { CREATE_BACKFILLING_TABLE } from "./backfilling.js";
@@ -73,21 +73,23 @@ function getAscendingEvents(db) {
 	}));
 }
 function getSubscriptionState(db) {
-	return parse(db.get(`
+	const result = db.get(`
       SELECT c.replicaVersion, c.publications, s.stateVersion as watermark
         FROM "_zero.replicationConfig" as c
         JOIN "_zero.replicationState" as s
         ON c.lock = s.lock
-    `), subscriptionStateSchema);
+    `);
+	return parse(result, subscriptionStateSchema);
 }
 function getSubscriptionStateAndContext(db) {
-	return parse(db.get(`
+	const result = db.get(`
       SELECT c.replicaVersion, c.publications, c.initialSyncContext,
         s.stateVersion as watermark
         FROM "_zero.replicationConfig" as c
         JOIN "_zero.replicationState" as s
         ON c.lock = s.lock
-    `), subscriptionStateAndContextSchema);
+    `);
+	return parse(result, subscriptionStateAndContextSchema);
 }
 function updateReplicationWatermark(db, watermark, writeTimeMs) {
 	if (writeTimeMs === void 0) db.run(`
@@ -98,9 +100,10 @@ function updateReplicationWatermark(db, watermark, writeTimeMs) {
         SET stateVersion=?, writeTimeMs=?`, watermark, writeTimeMs);
 }
 function getReplicationState(db) {
-	return parse(db.get(`SELECT stateVersion FROM "_zero.replicationState"`), replicationStateSchema);
+	const result = db.get(`SELECT stateVersion FROM "_zero.replicationState"`);
+	return parse(result, replicationStateSchema);
 }
 //#endregion
-export { CREATE_RUNTIME_EVENTS_TABLE, createReplicationStateTables, getAscendingEvents, getReplicationState, getSubscriptionState, getSubscriptionStateAndContext, initReplicationState, recordEvent, updateReplicationWatermark };
+export { CREATE_RUNTIME_EVENTS_TABLE, ZERO_VERSION_COLUMN_NAME, createReplicationStateTables, getAscendingEvents, getReplicationState, getSubscriptionState, getSubscriptionStateAndContext, initReplicationState, recordEvent, updateReplicationWatermark };
 
 //# sourceMappingURL=replication-state.js.map

package/out/zero-cache/src/services/replicator/schema/replication-state.js.map

@@ -1 +1 @@
-{"version":3,"file":"replication-state.js","names":[],"sources":["../../../../../../../zero-cache/src/services/replicator/schema/replication-state.ts"],"sourcesContent":["/**\n * Replication metadata, used for incremental view maintenance and catchup.\n *\n * These tables are created atomically in {@link setupReplicationTables}\n * after the logical replication handoff when initial data synchronization has completed.\n */\n\nimport {\n  jsonObjectSchema,\n  stringify,\n  type JSONObject,\n} from '../../../../../shared/src/bigint-json.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport type {StatementRunner} from '../../../db/statements.ts';\nimport {CREATE_BACKFILLING_TABLE} from './backfilling.ts';\nimport {CREATE_CHANGELOG_SCHEMA} from './change-log.ts';\nimport {CREATE_COLUMN_METADATA_TABLE} from './column-metadata.ts';\nimport {ZERO_VERSION_COLUMN_NAME} from './constants.ts';\nimport {CREATE_TABLE_METADATA_TABLE} from './table-metadata.ts';\n\nexport {ZERO_VERSION_COLUMN_NAME};\n\nexport type RuntimeEvent = 'sync' | 'upgrade' | 'vacuum';\n\n// event     : The RuntimeEvent. Only one row per event is tracked.\n//             Inserting an event will REPLACE any row for the same event.\n// timestamp : SQLite timestamp string, e.g. \"2024-04-12 11:37:46\".\n//             Append a `Z` when parsing with `new Date(...)`;\nexport const CREATE_RUNTIME_EVENTS_TABLE = `\n  CREATE TABLE \"_zero.runtimeEvents\" (\n    event TEXT PRIMARY KEY ON CONFLICT REPLACE,\n    timestamp TEXT NOT NULL DEFAULT (current_timestamp)\n  );\n`;\n\nconst CREATE_REPLICATION_STATE_SCHEMA =\n  // replicaVersion     : A value identifying the version at which the initial sync happened, i.e.\n  //                      the version at which all rows were copied, and to `_0_version` was set.\n  //                      This value is used to distinguish data from other replicas (e.g. if a\n  //                      replica is reset or if there are ever multiple replicas).\n  // publications       : JSON stringified array of publication names\n  // initialSyncContext : Metadata related to the context of when and how the replica was initially\n  //                      synced. This corresponds with the same column stored in upstream and is\n  //                      used for debugging replica version mismatches, which can arise from a number\n  //                      of misconfigurations, such as dueling replication-managers, or restores of\n  //                      stale litestream backups.\n  // lock               : Auto-magic column for enforcing single-row semantics.\n  /*sql*/ `\n  CREATE TABLE \"_zero.replicationConfig\" (\n    replicaVersion TEXT NOT NULL,\n    publications TEXT NOT NULL,\n    initialSyncContext TEXT DEFAULT '{}',\n    lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n  ` +\n  // stateVersion     : The latest version replicated from upstream, starting with the initial\n  //                    `replicaVersion` and moving forward to each subsequent commit watermark\n  //                    (e.g. corresponding to a Postgres LSN). Versions are represented as\n  //                    lexicographically sortable watermarks (e.g. LexiVersions).\n  // writeTimeMs      : The millisecond epoch at which this version was written to the replica.\n  //\n  /*sql*/ `\n  CREATE TABLE \"_zero.replicationState\" (\n    stateVersion TEXT NOT NULL,\n    writeTimeMs INTEGER NOT NULL,\n    lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n  ` +\n  CREATE_CHANGELOG_SCHEMA +\n  CREATE_RUNTIME_EVENTS_TABLE +\n  CREATE_COLUMN_METADATA_TABLE +\n  CREATE_TABLE_METADATA_TABLE +\n  CREATE_BACKFILLING_TABLE;\n\nconst stringArray = v.array(v.string());\n\nconst subscriptionStateSchema = v\n  .object({\n    replicaVersion: v.string(),\n    publications: v.string(),\n    watermark: v.string(),\n  })\n  .map(s => ({\n    ...s,\n    publications: v.parse(JSON.parse(s.publications), stringArray),\n  }));\n\nexport type SubscriptionState = v.Infer<typeof subscriptionStateSchema>;\n\nconst subscriptionStateAndContextSchema = v\n  .object({\n    replicaVersion: v.string(),\n    publications: v.string(),\n    initialSyncContext: v.string(),\n    watermark: v.string(),\n  })\n  .map(s => ({\n    ...s,\n    publications: v.parse(JSON.parse(s.publications), stringArray),\n    initialSyncContext: v.parse(\n      JSON.parse(s.initialSyncContext),\n      jsonObjectSchema,\n    ),\n  }));\n\nexport type SubscriptionStateAndContext = v.Infer<\n  typeof subscriptionStateAndContextSchema\n>;\n\nconst replicationStateSchema = v.object({\n  stateVersion: v.string(),\n});\n\nexport type ReplicationState = v.Infer<typeof replicationStateSchema>;\n\nexport function initReplicationState(\n  db: Database,\n  publications: string[],\n  watermark: string,\n  initialSyncContext: JSONObject = {},\n  createTables = true,\n) {\n  if (createTables) {\n    createReplicationStateTables(db);\n  }\n  db.prepare(\n    `\n    INSERT INTO \"_zero.replicationConfig\" \n       (replicaVersion, publications, initialSyncContext) VALUES (?, ?, ?)\n    `,\n  ).run(\n    watermark,\n    JSON.stringify(publications.sort()),\n    stringify(initialSyncContext),\n  );\n  db.prepare(/*sql*/ `\n    INSERT INTO \"_zero.replicationState\" (stateVersion, writeTimeMs) \n      VALUES (?, unixepoch('subsec') * 1000)\n    `).run(watermark);\n  recordEvent(db, 'sync');\n}\n\n/**\n * Exposed as a separate function for the custom change source,\n * which needs the tables to be created in order to construct\n * ChangeProcessor before it knows the initial watermark.\n */\nexport function createReplicationStateTables(db: Database) {\n  db.exec(CREATE_REPLICATION_STATE_SCHEMA);\n}\n\nexport function recordEvent(db: Database, event: RuntimeEvent) {\n  db.prepare(\n    `\n    INSERT INTO \"_zero.runtimeEvents\" (event) VALUES (?) \n    `,\n  ).run(event);\n}\n\nexport function getAscendingEvents(db: Database) {\n  const result = db\n    .prepare(\n      `SELECT event, timestamp FROM \"_zero.runtimeEvents\" \n         ORDER BY timestamp ASC\n    `,\n    )\n    .all<{event: string; timestamp: string}>();\n  return result.map(({event, timestamp}) => ({\n    event,\n    timestamp: new Date(timestamp + 'Z'),\n  }));\n}\n\nexport function getSubscriptionState(db: StatementRunner): SubscriptionState {\n  const result = db.get(/*sql*/ `\n      SELECT c.replicaVersion, c.publications, s.stateVersion as watermark\n        FROM \"_zero.replicationConfig\" as c\n        JOIN \"_zero.replicationState\" as s\n        ON c.lock = s.lock\n    `);\n  return v.parse(result, subscriptionStateSchema);\n}\n\nexport function getSubscriptionStateAndContext(\n  db: StatementRunner,\n): SubscriptionStateAndContext {\n  const result = db.get(/*sql*/ `\n      SELECT c.replicaVersion, c.publications, c.initialSyncContext,\n        s.stateVersion as watermark\n        FROM \"_zero.replicationConfig\" as c\n        JOIN \"_zero.replicationState\" as s\n        ON c.lock = s.lock\n    `);\n  return v.parse(result, subscriptionStateAndContextSchema);\n}\n\nexport function updateReplicationWatermark(\n  db: StatementRunner,\n  watermark: string,\n  writeTimeMs?: number | undefined,\n) {\n  if (writeTimeMs === undefined) {\n    db.run(\n      /*sql*/ `\n      UPDATE \"_zero.replicationState\"\n        SET stateVersion=?, writeTimeMs=unixepoch('subsec') * 1000`,\n      watermark,\n    );\n  } else {\n    db.run(\n      /*sql*/ `\n      UPDATE \"_zero.replicationState\"\n        SET stateVersion=?, writeTimeMs=?`,\n      watermark,\n      writeTimeMs,\n    );\n  }\n}\n\nexport function getReplicationState(db: StatementRunner): ReplicationState {\n  const result = db.get(`SELECT stateVersion FROM \"_zero.replicationState\"`);\n  return v.parse(result, replicationStateSchema);\n}\n"],"mappings":";;;;;;;;;;;;;;AA6BA,IAAa,8BAA8B;;;;;;AAO3C,IAAM,kCAYI,qZAqBR,0BACA,8BACA,+BACA,8BACA;AAEF,IAAM,cAAc,eAAE,MAAM,eAAE,OAAO,CAAC;AAEtC,IAAM,0BAA0B,eAC7B,OAAO;CACN,gBAAgB,eAAE,OAAO;CACzB,cAAc,eAAE,OAAO;CACvB,WAAW,eAAE,OAAO;AACtB,CAAC,EACA,KAAI,OAAM;CACT,GAAG;CACH,cAAc,MAAQ,KAAK,MAAM,EAAE,YAAY,GAAG,WAAW;AAC/D,EAAE;AAIJ,IAAM,oCAAoC,eACvC,OAAO;CACN,gBAAgB,eAAE,OAAO;CACzB,cAAc,eAAE,OAAO;CACvB,oBAAoB,eAAE,OAAO;CAC7B,WAAW,eAAE,OAAO;AACtB,CAAC,EACA,KAAI,OAAM;CACT,GAAG;CACH,cAAc,MAAQ,KAAK,MAAM,EAAE,YAAY,GAAG,WAAW;CAC7D,oBAAoB,MAClB,KAAK,MAAM,EAAE,kBAAkB,GAC/B,gBACF;AACF,EAAE;AAMJ,IAAM,yBAAyB,eAAE,OAAO,EACtC,cAAc,eAAE,OAAO,EACzB,CAAC;AAID,SAAgB,qBACd,IACA,cACA,WACA,qBAAiC,CAAC,GAClC,eAAe,MACf;CACA,IAAI,cACF,6BAA6B,EAAE;CAEjC,GAAG,QACD;;;KAIF,EAAE,IACA,WACA,KAAK,UAAU,aAAa,KAAK,CAAC,GAClC,UAAU,kBAAkB,CAC9B;CACA,GAAG,QAAgB;;;KAGhB,EAAE,IAAI,SAAS;CAClB,YAAY,IAAI,MAAM;AACxB;;;;;;AAOA,SAAgB,6BAA6B,IAAc;CACzD,GAAG,KAAK,+BAA+B;AACzC;AAEA,SAAgB,YAAY,IAAc,OAAqB;CAC7D,GAAG,QACD;;KAGF,EAAE,IAAI,KAAK;AACb;AAEA,SAAgB,mBAAmB,IAAc;CAQ/C,OAPe,GACZ,QACC;;KAGF,EACC,IACI,EAAO,KAAK,EAAC,OAAO,iBAAgB;EACzC;EACA,2BAAW,IAAI,KAAK,YAAY,GAAG;CACrC,EAAE;AACJ;AAEA,SAAgB,qBAAqB,IAAwC;CAO3E,OAAO,MANQ,GAAG,IAAY;;;;;KAMf,GAAQ,uBAAuB;AAChD;AAEA,SAAgB,+BACd,IAC6B;CAQ7B,OAAO,MAPQ,GAAG,IAAY;;;;;;KAOf,GAAQ,iCAAiC;AAC1D;AAEA,SAAgB,2BACd,IACA,WACA,aACA;CACA,IAAI,gBAAgB,KAAA,GAClB,GAAG,IACO;;qEAGR,SACF;MAEA,GAAG,IACO;;4CAGR,WACA,WACF;AAEJ;AAEA,SAAgB,oBAAoB,IAAuC;CAEzE,OAAO,MADQ,GAAG,IAAI,mDACP,GAAQ,sBAAsB;AAC/C"}
+{"version":3,"file":"replication-state.js","names":["v","v.parse"],"sources":["../../../../../../../zero-cache/src/services/replicator/schema/replication-state.ts"],"sourcesContent":["/**\n * Replication metadata, used for incremental view maintenance and catchup.\n *\n * These tables are created atomically in {@link setupReplicationTables}\n * after the logical replication handoff when initial data synchronization has completed.\n */\n\nimport {\n  jsonObjectSchema,\n  stringify,\n  type JSONObject,\n} from '../../../../../shared/src/bigint-json.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport type {StatementRunner} from '../../../db/statements.ts';\nimport {CREATE_BACKFILLING_TABLE} from './backfilling.ts';\nimport {CREATE_CHANGELOG_SCHEMA} from './change-log.ts';\nimport {CREATE_COLUMN_METADATA_TABLE} from './column-metadata.ts';\nimport {ZERO_VERSION_COLUMN_NAME} from './constants.ts';\nimport {CREATE_TABLE_METADATA_TABLE} from './table-metadata.ts';\n\nexport {ZERO_VERSION_COLUMN_NAME};\n\nexport type RuntimeEvent = 'sync' | 'upgrade' | 'vacuum';\n\n// event     : The RuntimeEvent. Only one row per event is tracked.\n//             Inserting an event will REPLACE any row for the same event.\n// timestamp : SQLite timestamp string, e.g. \"2024-04-12 11:37:46\".\n//             Append a `Z` when parsing with `new Date(...)`;\nexport const CREATE_RUNTIME_EVENTS_TABLE = `\n  CREATE TABLE \"_zero.runtimeEvents\" (\n    event TEXT PRIMARY KEY ON CONFLICT REPLACE,\n    timestamp TEXT NOT NULL DEFAULT (current_timestamp)\n  );\n`;\n\nconst CREATE_REPLICATION_STATE_SCHEMA =\n  // replicaVersion     : A value identifying the version at which the initial sync happened, i.e.\n  //                      the version at which all rows were copied, and to `_0_version` was set.\n  //                      This value is used to distinguish data from other replicas (e.g. if a\n  //                      replica is reset or if there are ever multiple replicas).\n  // publications       : JSON stringified array of publication names\n  // initialSyncContext : Metadata related to the context of when and how the replica was initially\n  //                      synced. This corresponds with the same column stored in upstream and is\n  //                      used for debugging replica version mismatches, which can arise from a number\n  //                      of misconfigurations, such as dueling replication-managers, or restores of\n  //                      stale litestream backups.\n  // lock               : Auto-magic column for enforcing single-row semantics.\n  /*sql*/ `\n  CREATE TABLE \"_zero.replicationConfig\" (\n    replicaVersion TEXT NOT NULL,\n    publications TEXT NOT NULL,\n    initialSyncContext TEXT DEFAULT '{}',\n    lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n  ` +\n  // stateVersion     : The latest version replicated from upstream, starting with the initial\n  //                    `replicaVersion` and moving forward to each subsequent commit watermark\n  //                    (e.g. corresponding to a Postgres LSN). Versions are represented as\n  //                    lexicographically sortable watermarks (e.g. LexiVersions).\n  // writeTimeMs      : The millisecond epoch at which this version was written to the replica.\n  //\n  /*sql*/ `\n  CREATE TABLE \"_zero.replicationState\" (\n    stateVersion TEXT NOT NULL,\n    writeTimeMs INTEGER NOT NULL,\n    lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n  ` +\n  CREATE_CHANGELOG_SCHEMA +\n  CREATE_RUNTIME_EVENTS_TABLE +\n  CREATE_COLUMN_METADATA_TABLE +\n  CREATE_TABLE_METADATA_TABLE +\n  CREATE_BACKFILLING_TABLE;\n\nconst stringArray = v.array(v.string());\n\nconst subscriptionStateSchema = v\n  .object({\n    replicaVersion: v.string(),\n    publications: v.string(),\n    watermark: v.string(),\n  })\n  .map(s => ({\n    ...s,\n    publications: v.parse(JSON.parse(s.publications), stringArray),\n  }));\n\nexport type SubscriptionState = v.Infer<typeof subscriptionStateSchema>;\n\nconst subscriptionStateAndContextSchema = v\n  .object({\n    replicaVersion: v.string(),\n    publications: v.string(),\n    initialSyncContext: v.string(),\n    watermark: v.string(),\n  })\n  .map(s => ({\n    ...s,\n    publications: v.parse(JSON.parse(s.publications), stringArray),\n    initialSyncContext: v.parse(\n      JSON.parse(s.initialSyncContext),\n      jsonObjectSchema,\n    ),\n  }));\n\nexport type SubscriptionStateAndContext = v.Infer<\n  typeof subscriptionStateAndContextSchema\n>;\n\nconst replicationStateSchema = v.object({\n  stateVersion: v.string(),\n});\n\nexport type ReplicationState = v.Infer<typeof replicationStateSchema>;\n\nexport function initReplicationState(\n  db: Database,\n  publications: string[],\n  watermark: string,\n  initialSyncContext: JSONObject = {},\n  createTables = true,\n) {\n  if (createTables) {\n    createReplicationStateTables(db);\n  }\n  db.prepare(\n    `\n    INSERT INTO \"_zero.replicationConfig\" \n       (replicaVersion, publications, initialSyncContext) VALUES (?, ?, ?)\n    `,\n  ).run(\n    watermark,\n    JSON.stringify(publications.sort()),\n    stringify(initialSyncContext),\n  );\n  db.prepare(/*sql*/ `\n    INSERT INTO \"_zero.replicationState\" (stateVersion, writeTimeMs) \n      VALUES (?, unixepoch('subsec') * 1000)\n    `).run(watermark);\n  recordEvent(db, 'sync');\n}\n\n/**\n * Exposed as a separate function for the custom change source,\n * which needs the tables to be created in order to construct\n * ChangeProcessor before it knows the initial watermark.\n */\nexport function createReplicationStateTables(db: Database) {\n  db.exec(CREATE_REPLICATION_STATE_SCHEMA);\n}\n\nexport function recordEvent(db: Database, event: RuntimeEvent) {\n  db.prepare(\n    `\n    INSERT INTO \"_zero.runtimeEvents\" (event) VALUES (?) \n    `,\n  ).run(event);\n}\n\nexport function getAscendingEvents(db: Database) {\n  const result = db\n    .prepare(\n      `SELECT event, timestamp FROM \"_zero.runtimeEvents\" \n         ORDER BY timestamp ASC\n    `,\n    )\n    .all<{event: string; timestamp: string}>();\n  return result.map(({event, timestamp}) => ({\n    event,\n    timestamp: new Date(timestamp + 'Z'),\n  }));\n}\n\nexport function getSubscriptionState(db: StatementRunner): SubscriptionState {\n  const result = db.get(/*sql*/ `\n      SELECT c.replicaVersion, c.publications, s.stateVersion as watermark\n        FROM \"_zero.replicationConfig\" as c\n        JOIN \"_zero.replicationState\" as s\n        ON c.lock = s.lock\n    `);\n  return v.parse(result, subscriptionStateSchema);\n}\n\nexport function getSubscriptionStateAndContext(\n  db: StatementRunner,\n): SubscriptionStateAndContext {\n  const result = db.get(/*sql*/ `\n      SELECT c.replicaVersion, c.publications, c.initialSyncContext,\n        s.stateVersion as watermark\n        FROM \"_zero.replicationConfig\" as c\n        JOIN \"_zero.replicationState\" as s\n        ON c.lock = s.lock\n    `);\n  return v.parse(result, subscriptionStateAndContextSchema);\n}\n\nexport function updateReplicationWatermark(\n  db: StatementRunner,\n  watermark: string,\n  writeTimeMs?: number | undefined,\n) {\n  if (writeTimeMs === undefined) {\n    db.run(\n      /*sql*/ `\n      UPDATE \"_zero.replicationState\"\n        SET stateVersion=?, writeTimeMs=unixepoch('subsec') * 1000`,\n      watermark,\n    );\n  } else {\n    db.run(\n      /*sql*/ `\n      UPDATE \"_zero.replicationState\"\n        SET stateVersion=?, writeTimeMs=?`,\n      watermark,\n      writeTimeMs,\n    );\n  }\n}\n\nexport function getReplicationState(db: StatementRunner): ReplicationState {\n  const result = db.get(`SELECT stateVersion FROM \"_zero.replicationState\"`);\n  return v.parse(result, replicationStateSchema);\n}\n"],"mappings":";;;;;;;;;;;;;;AA6BA,IAAa,8BAA8B;;;;;;AAO3C,IAAM,kCAYI,qZAqBR,0BACA,8BACA,+BACA,8BACA;AAEF,IAAM,cAAcA,eAAE,MAAMA,eAAE,OAAO,CAAC;AAEtC,IAAM,0BAA0BA,eAC7B,OAAO;CACN,gBAAgBA,eAAE,OAAO;CACzB,cAAcA,eAAE,OAAO;CACvB,WAAWA,eAAE,OAAO;AACtB,CAAC,CAAC,CACD,KAAI,OAAM;CACT,GAAG;CACH,cAAcC,MAAQ,KAAK,MAAM,EAAE,YAAY,GAAG,WAAW;AAC/D,EAAE;AAIJ,IAAM,oCAAoCD,eACvC,OAAO;CACN,gBAAgBA,eAAE,OAAO;CACzB,cAAcA,eAAE,OAAO;CACvB,oBAAoBA,eAAE,OAAO;CAC7B,WAAWA,eAAE,OAAO;AACtB,CAAC,CAAC,CACD,KAAI,OAAM;CACT,GAAG;CACH,cAAcC,MAAQ,KAAK,MAAM,EAAE,YAAY,GAAG,WAAW;CAC7D,oBAAoBA,MAClB,KAAK,MAAM,EAAE,kBAAkB,GAC/B,gBACF;AACF,EAAE;AAMJ,IAAM,yBAAyBD,eAAE,OAAO,EACtC,cAAcA,eAAE,OAAO,EACzB,CAAC;AAID,SAAgB,qBACd,IACA,cACA,WACA,qBAAiC,CAAC,GAClC,eAAe,MACf;CACA,IAAI,cACF,6BAA6B,EAAE;CAEjC,GAAG,QACD;;;KAIF,CAAC,CAAC,IACA,WACA,KAAK,UAAU,aAAa,KAAK,CAAC,GAClC,UAAU,kBAAkB,CAC9B;CACA,GAAG,QAAgB;;;KAGhB,CAAC,CAAC,IAAI,SAAS;CAClB,YAAY,IAAI,MAAM;AACxB;;;;;;AAOA,SAAgB,6BAA6B,IAAc;CACzD,GAAG,KAAK,+BAA+B;AACzC;AAEA,SAAgB,YAAY,IAAc,OAAqB;CAC7D,GAAG,QACD;;KAGF,CAAC,CAAC,IAAI,KAAK;AACb;AAEA,SAAgB,mBAAmB,IAAc;CAQ/C,OAPe,GACZ,QACC;;KAGF,CAAC,CACA,IACI,CAAA,CAAO,KAAK,EAAC,OAAO,iBAAgB;EACzC;EACA,2BAAW,IAAI,KAAK,YAAY,GAAG;CACrC,EAAE;AACJ;AAEA,SAAgB,qBAAqB,IAAwC;CAC3E,MAAM,SAAS,GAAG,IAAY;;;;;KAK3B;CACH,OAAOC,MAAQ,QAAQ,uBAAuB;AAChD;AAEA,SAAgB,+BACd,IAC6B;CAC7B,MAAM,SAAS,GAAG,IAAY;;;;;;KAM3B;CACH,OAAOA,MAAQ,QAAQ,iCAAiC;AAC1D;AAEA,SAAgB,2BACd,IACA,WACA,aACA;CACA,IAAI,gBAAgB,KAAA,GAClB,GAAG,IACO;;qEAGR,SACF;MAEA,GAAG,IACO;;4CAGR,WACA,WACF;AAEJ;AAEA,SAAgB,oBAAoB,IAAuC;CACzE,MAAM,SAAS,GAAG,IAAI,mDAAmD;CACzE,OAAOA,MAAQ,QAAQ,sBAAsB;AAC/C"}

package/out/zero-cache/src/services/replicator/schema/table-metadata.js.map

@@ -1 +1 @@
-{"version":3,"file":"table-metadata.js","names":["#db","#setUpstreamMetadata","#setMinRowVersion","#getMinRowVersions","#rename","#drop"],"sources":["../../../../../../../zero-cache/src/services/replicator/schema/table-metadata.ts"],"sourcesContent":["import type {Database, Statement} from '../../../../../zqlite/src/db.ts';\nimport {liteTableName} from '../../../types/names.ts';\nimport type {\n  Identifier,\n  TableMetadata,\n} from '../../change-source/protocol/current.ts';\n\n/**\n * Table-level controls for handling replicated data.\n *\n * ### Columns\n *\n * `minRowVersion`: the minimum `_0_version` value to apply to\n *   all rows in the table. This overrides any per-row\n *   `_0_version` value that is smaller (i.e. earlier).\n *   The `minRowVersion` column is used to force a re-download\n *   of all rows after a table-wide schema change (by giving\n *   each row a version that's newer than what's in any CVR).\n *   The naive, brute-force method of updating all of the rows\n *   requires re-writing the entire table into the WAL as one\n *   SQLite operation, which is too costly from both latency\n *   and storage space.\n *\n * `upstreamMetadata`: the replica-level analog of tableMetadata in\n *   change-streamer/schema. Per the requirement of the backfill\n *   protocol, backfill metadata must be tracked outside of the\n *   change source (otherwise the change source would have to be\n *   able to compute the state of the metadata at arbitrary points\n *   in the past).\n *\n * `metadata`: the previous name of the `upstreamMetadata` column,\n *   kept for backwards compatibility.\n *\n * This tracking is done:\n * 1. at the Change DB level, by the change-streamer\n * 2. at the replica level, in order to support the eventual configuration\n *    of ephemeral Change DBs (on SQLite) that are initialized from data\n *    in the replica.\n */\nexport const CREATE_TABLE_METADATA_TABLE = /*sql*/ `\n  CREATE TABLE \"_zero.tableMetadata\" (\n    \"schema\"           TEXT NOT NULL,\n    \"table\"            TEXT NOT NULL,\n    \"minRowVersion\"    TEXT NOT NULL DEFAULT \"00\",\n    \"upstreamMetadata\" TEXT,\n    \"metadata\"         TEXT,  -- deprecated\n    PRIMARY KEY (\"schema\", \"table\")\n  );\n`;\n\nexport class TableMetadataTracker {\n  readonly #db: Database;\n\n  // All statements are lazily created.\n  #setUpstreamMetadata: Statement | undefined;\n  #setMinRowVersion: Statement | undefined;\n  #getMinRowVersions: Statement | undefined;\n  #rename: Statement | undefined;\n  #drop: Statement | undefined;\n\n  constructor(db: Database) {\n    this.#db = db;\n  }\n\n  setUpstreamMetadata({schema, name}: Identifier, metadata: TableMetadata) {\n    (this.#setUpstreamMetadata ??= this.#db.prepare(/*sql*/ `\n      INSERT INTO \"_zero.tableMetadata\" (\"schema\", \"table\", \"upstreamMetadata\") \n        VALUES (@schema, @name, @metadata)\n        ON CONFLICT (\"schema\", \"table\")\n        DO UPDATE SET \"upstreamMetadata\" = @metadata\n    `)).run({\n      schema,\n      name,\n      metadata: JSON.stringify(metadata),\n    });\n  }\n\n  setMinRowVersion({schema, name}: Identifier, version: string) {\n    (this.#setMinRowVersion ??= this.#db.prepare(/*sql*/ `\n      INSERT INTO \"_zero.tableMetadata\" (\"schema\", \"table\", \"minRowVersion\") \n        VALUES (@schema, @name, @version)\n        ON CONFLICT (\"schema\", \"table\")\n        DO UPDATE SET \"minRowVersion\" = @version;\n    `)).run({schema, name, version});\n  }\n\n  getMinRowVersions(): Map<string, string> {\n    const results = (this.#getMinRowVersions ??= this.#db.prepare(/*sql*/ `\n      SELECT \"schema\", \"table\" as \"name\", \"minRowVersion\" FROM \"_zero.tableMetadata\"\n    `)).all<{schema: string; name: string; minRowVersion: string}>();\n    return new Map(\n      results.map(({schema, name, minRowVersion}) => [\n        liteTableName({schema, name}),\n        minRowVersion,\n      ]),\n    );\n  }\n\n  rename(oldTable: Identifier, newTable: Identifier) {\n    (this.#rename ??= this.#db.prepare(/*sql*/ `\n      UPDATE \"_zero.tableMetadata\" SET \"schema\" = ?, \"table\" = ?\n        WHERE \"schema\" = ? AND \"table\" = ?\n    `)).run(newTable.schema, newTable.name, oldTable.schema, oldTable.name);\n  }\n\n  drop({schema, name}: Identifier) {\n    (this.#drop ??= this.#db.prepare(/*sql*/ `\n      DELETE FROM \"_zero.tableMetadata\" WHERE \"schema\" = ? AND \"table\" = ?\n    `)).run(schema, name);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuCA,IAAa,8BAAsC;;;;;;;;;;AAWnD,IAAa,uBAAb,MAAkC;CAChC;CAGA;CACA;CACA;CACA;CACA;CAEA,YAAY,IAAc;EACxB,KAAKA,MAAM;CACb;CAEA,oBAAoB,EAAC,QAAQ,QAAmB,UAAyB;EACvE,CAAC,KAAKC,yBAAyB,KAAKD,IAAI,QAAgB;;;;;KAKvD,GAAG,IAAI;GACN;GACA;GACA,UAAU,KAAK,UAAU,QAAQ;EACnC,CAAC;CACH;CAEA,iBAAiB,EAAC,QAAQ,QAAmB,SAAiB;EAC5D,CAAC,KAAKE,sBAAsB,KAAKF,IAAI,QAAgB;;;;;KAKpD,GAAG,IAAI;GAAC;GAAQ;GAAM;EAAO,CAAC;CACjC;CAEA,oBAAyC;EACvC,MAAM,WAAW,KAAKG,uBAAuB,KAAKH,IAAI,QAAgB;;KAErE,GAAG,IAA2D;EAC/D,OAAO,IAAI,IACT,QAAQ,KAAK,EAAC,QAAQ,MAAM,oBAAmB,CAC7C,cAAc;GAAC;GAAQ;EAAI,CAAC,GAC5B,aACF,CAAC,CACH;CACF;CAEA,OAAO,UAAsB,UAAsB;EACjD,CAAC,KAAKI,YAAY,KAAKJ,IAAI,QAAgB;;;KAG1C,GAAG,IAAI,SAAS,QAAQ,SAAS,MAAM,SAAS,QAAQ,SAAS,IAAI;CACxE;CAEA,KAAK,EAAC,QAAQ,QAAmB;EAC/B,CAAC,KAAKK,UAAU,KAAKL,IAAI,QAAgB;;KAExC,GAAG,IAAI,QAAQ,IAAI;CACtB;AACF"}
+{"version":3,"file":"table-metadata.js","names":[],"sources":["../../../../../../../zero-cache/src/services/replicator/schema/table-metadata.ts"],"sourcesContent":["import type {Database, Statement} from '../../../../../zqlite/src/db.ts';\nimport {liteTableName} from '../../../types/names.ts';\nimport type {\n  Identifier,\n  TableMetadata,\n} from '../../change-source/protocol/current.ts';\n\n/**\n * Table-level controls for handling replicated data.\n *\n * ### Columns\n *\n * `minRowVersion`: the minimum `_0_version` value to apply to\n *   all rows in the table. This overrides any per-row\n *   `_0_version` value that is smaller (i.e. earlier).\n *   The `minRowVersion` column is used to force a re-download\n *   of all rows after a table-wide schema change (by giving\n *   each row a version that's newer than what's in any CVR).\n *   The naive, brute-force method of updating all of the rows\n *   requires re-writing the entire table into the WAL as one\n *   SQLite operation, which is too costly from both latency\n *   and storage space.\n *\n * `upstreamMetadata`: the replica-level analog of tableMetadata in\n *   change-streamer/schema. Per the requirement of the backfill\n *   protocol, backfill metadata must be tracked outside of the\n *   change source (otherwise the change source would have to be\n *   able to compute the state of the metadata at arbitrary points\n *   in the past).\n *\n * `metadata`: the previous name of the `upstreamMetadata` column,\n *   kept for backwards compatibility.\n *\n * This tracking is done:\n * 1. at the Change DB level, by the change-streamer\n * 2. at the replica level, in order to support the eventual configuration\n *    of ephemeral Change DBs (on SQLite) that are initialized from data\n *    in the replica.\n */\nexport const CREATE_TABLE_METADATA_TABLE = /*sql*/ `\n  CREATE TABLE \"_zero.tableMetadata\" (\n    \"schema\"           TEXT NOT NULL,\n    \"table\"            TEXT NOT NULL,\n    \"minRowVersion\"    TEXT NOT NULL DEFAULT \"00\",\n    \"upstreamMetadata\" TEXT,\n    \"metadata\"         TEXT,  -- deprecated\n    PRIMARY KEY (\"schema\", \"table\")\n  );\n`;\n\nexport class TableMetadataTracker {\n  readonly #db: Database;\n\n  // All statements are lazily created.\n  #setUpstreamMetadata: Statement | undefined;\n  #setMinRowVersion: Statement | undefined;\n  #getMinRowVersions: Statement | undefined;\n  #rename: Statement | undefined;\n  #drop: Statement | undefined;\n\n  constructor(db: Database) {\n    this.#db = db;\n  }\n\n  setUpstreamMetadata({schema, name}: Identifier, metadata: TableMetadata) {\n    (this.#setUpstreamMetadata ??= this.#db.prepare(/*sql*/ `\n      INSERT INTO \"_zero.tableMetadata\" (\"schema\", \"table\", \"upstreamMetadata\") \n        VALUES (@schema, @name, @metadata)\n        ON CONFLICT (\"schema\", \"table\")\n        DO UPDATE SET \"upstreamMetadata\" = @metadata\n    `)).run({\n      schema,\n      name,\n      metadata: JSON.stringify(metadata),\n    });\n  }\n\n  setMinRowVersion({schema, name}: Identifier, version: string) {\n    (this.#setMinRowVersion ??= this.#db.prepare(/*sql*/ `\n      INSERT INTO \"_zero.tableMetadata\" (\"schema\", \"table\", \"minRowVersion\") \n        VALUES (@schema, @name, @version)\n        ON CONFLICT (\"schema\", \"table\")\n        DO UPDATE SET \"minRowVersion\" = @version;\n    `)).run({schema, name, version});\n  }\n\n  getMinRowVersions(): Map<string, string> {\n    const results = (this.#getMinRowVersions ??= this.#db.prepare(/*sql*/ `\n      SELECT \"schema\", \"table\" as \"name\", \"minRowVersion\" FROM \"_zero.tableMetadata\"\n    `)).all<{schema: string; name: string; minRowVersion: string}>();\n    return new Map(\n      results.map(({schema, name, minRowVersion}) => [\n        liteTableName({schema, name}),\n        minRowVersion,\n      ]),\n    );\n  }\n\n  rename(oldTable: Identifier, newTable: Identifier) {\n    (this.#rename ??= this.#db.prepare(/*sql*/ `\n      UPDATE \"_zero.tableMetadata\" SET \"schema\" = ?, \"table\" = ?\n        WHERE \"schema\" = ? AND \"table\" = ?\n    `)).run(newTable.schema, newTable.name, oldTable.schema, oldTable.name);\n  }\n\n  drop({schema, name}: Identifier) {\n    (this.#drop ??= this.#db.prepare(/*sql*/ `\n      DELETE FROM \"_zero.tableMetadata\" WHERE \"schema\" = ? AND \"table\" = ?\n    `)).run(schema, name);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuCA,IAAa,8BAAsC;;;;;;;;;;AAWnD,IAAa,uBAAb,MAAkC;CAChC;CAGA;CACA;CACA;CACA;CACA;CAEA,YAAY,IAAc;EACxB,KAAK,MAAM;CACb;CAEA,oBAAoB,EAAC,QAAQ,QAAmB,UAAyB;EACvE,CAAC,KAAK,yBAAyB,KAAK,IAAI,QAAgB;;;;;KAKvD,EAAA,CAAG,IAAI;GACN;GACA;GACA,UAAU,KAAK,UAAU,QAAQ;EACnC,CAAC;CACH;CAEA,iBAAiB,EAAC,QAAQ,QAAmB,SAAiB;EAC5D,CAAC,KAAK,sBAAsB,KAAK,IAAI,QAAgB;;;;;KAKpD,EAAA,CAAG,IAAI;GAAC;GAAQ;GAAM;EAAO,CAAC;CACjC;CAEA,oBAAyC;EACvC,MAAM,WAAW,KAAK,uBAAuB,KAAK,IAAI,QAAgB;;KAErE,EAAA,CAAG,IAA2D;EAC/D,OAAO,IAAI,IACT,QAAQ,KAAK,EAAC,QAAQ,MAAM,oBAAmB,CAC7C,cAAc;GAAC;GAAQ;EAAI,CAAC,GAC5B,aACF,CAAC,CACH;CACF;CAEA,OAAO,UAAsB,UAAsB;EACjD,CAAC,KAAK,YAAY,KAAK,IAAI,QAAgB;;;KAG1C,EAAA,CAAG,IAAI,SAAS,QAAQ,SAAS,MAAM,SAAS,QAAQ,SAAS,IAAI;CACxE;CAEA,KAAK,EAAC,QAAQ,QAAmB;EAC/B,CAAC,KAAK,UAAU,KAAK,IAAI,QAAgB;;KAExC,EAAA,CAAG,IAAI,QAAQ,IAAI;CACtB;AACF"}

package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js

@@ -415,6 +415,6 @@ function watermarkDistance(receivedHead, sqliteHead) {
 	}
 }
 //#endregion
-export { SQLiteChangeLogObserver, getSQLiteChangeLogInfo, logSQLiteChangeLogStartup, recordSQLiteChangeLogReconcile, sqliteFileBytes };
+export { SQLiteChangeLogObserver, getSQLiteChangeLogInfo, getSQLiteChangeLogStartupInfo, logSQLiteChangeLogStartup, recordSQLiteChangeLogReconcile, sqliteFileBytes };
 
 //# sourceMappingURL=sqlite-change-log-observability.js.map

package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js.map

@@ -1 +1 @@
-{"version":3,"file":"sqlite-change-log-observability.js","names":["#lc","#messageProcessing","#logCommitDuration","#transactionRows","#rollbackCounter","#invariantFailureCounter","#hashComparisonCounter","#hashUnpairedCounter","#cookieMutationCounter","#receivedHead","#sqliteHead","#rows","#estimatedBytes","#cookieRows","#sourceHasher","#invariantFailure","#sourcePos","#sourceCommitHash","#resetSourceHash","#transactionWatermark","#recordRollback","#recordUnpaired","#compareHashes","#cookieMutations","#rollbacks","#invariantFailures","#hashMatches","#hashMismatches","#hashUnpaired"],"sources":["../../../../../../zero-cache/src/services/replicator/sqlite-change-log-observability.ts"],"sourcesContent":["/**\n * Observability for the SQLite change log.\n *\n * Everything here now runs in the **change-streamer** process, which is where\n * the writer lives. The module stays beside the rest of the change-log code\n * under `services/replicator/` for the same reason `change-log-db.ts` and\n * `change-log-stream-writer.ts` do: moving files would obscure the topology\n * diff, and the writer's home is a property of who calls it rather than of\n * where it sits. Metric names are unchanged from 7H — only their emitting\n * process, and therefore their resource attributes, moved.\n */\n\nimport {stat} from 'node:fs/promises';\nimport type {LogContext} from '@rocicorp/logger';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n  getOrCreateLatencyHistogram,\n  getOrCreateValueHistogram,\n} from '../../observability/metrics.ts';\nimport {versionFromLexi} from '../../types/lexi-version.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {extractChangeSubstring} from '../change-streamer/change-log-codec.ts';\nimport {ChangeLogTransactionHasher} from '../change-streamer/change-log-transaction-hash.ts';\nimport {\n  readCookieRowCounts,\n  type CookieOpTag,\n  type CookieRowCounts,\n} from './change-log-cookies.ts';\nimport {\n  CHANGE_LOG_STREAM_TABLE,\n  readChangeLogBounds,\n  readChangeLogMeta,\n  type ChangeLogMeta,\n  type ReconcileResult,\n} from './change-log-db.ts';\nimport type {ChangeLogStreamTransactionStats} from './change-log-stream-writer.ts';\n\n/**\n * Reports the reconciliation of the change-log database against the watermark\n * the stream connection resumes from. This runs once per stream connection, so\n * a reconnect storm is visible here as well as a restart.\n *\n * A wipe is not an error — it is how every abnormal change-log state is\n * collapsed into a correct one — but each wipe costs the retention window, so\n * every reconnecting subscriber below the head gets `too-old`. The `gap` reason\n * in particular means the log neither held the resume watermark nor could be\n * truncated to it, so a nonzero rate in steady state is an alert, not a\n * statistic.\n */\nexport function recordSQLiteChangeLogReconcile(\n  lc: LogContext,\n  result: ReconcileResult,\n): void {\n  switch (result.action) {\n    case 'truncated':\n      getOrCreateCounter(\n        'replica',\n        'sqlite_change_log.reconcile_truncated_rows',\n        'Phantom change-log rows removed at reconciliation, i.e. rows above ' +\n          'the watermark the change stream resumed from.',\n      ).add(result.rows);\n      break;\n    case 'reseeded':\n      getOrCreateCounter(\n        'replica',\n        'sqlite_change_log.reconcile_wipes',\n        'Change-log wipes at reconciliation, by reason. Each one resets the ' +\n          'retention window, so subscribers below the head get `too-old`.',\n      ).add(1, {reason: result.reason});\n      break;\n    case 'none':\n      break;\n    default:\n      unreachable(result);\n  }\n  if (result.cookiesStale) {\n    // The invariant-17 counter: after any reconcile that moved the head, the\n    // log's cookie set has to be replaced with the one that belongs to the\n    // resume watermark. A nonzero `truncated` series in steady state means\n    // phantom truncation is happening on a shard where it should not.\n    getOrCreateCounter(\n      'replica',\n      'sqlite_change_log.cookie_reseed',\n      'Change-log cookie sets invalidated at reconciliation, by the ' +\n        'reconcile action that invalidated them.',\n    ).add(1, {action: result.action});\n  }\n  lc.debug?.('SQLite change-log reconciliation', {\n    sqliteChangeLogReconcile: result,\n  });\n}\n\n/**\n * What the log says about itself. Nothing here reads the replica: after the\n * writer moved to the change-streamer, the replica's state version is a\n * consequence of this log rather than a fact about it.\n */\nexport type SQLiteChangeLogStartupInfo = ChangeLogMeta & {\n  readonly minWatermark: string;\n  readonly headWatermark: string;\n};\n\nexport type SQLiteChangeLogInfo = SQLiteChangeLogStartupInfo & {\n  readonly rows: number;\n  readonly estimatedBytes: number;\n  readonly cookieRows: CookieRowCounts;\n};\n\n/**\n * Reads the meta row and covered range. Unlike {@link getSQLiteChangeLogInfo}\n * this does not scan the log; see {@link readChangeLogBounds}.\n *\n * Call this after reconciliation, which is the only point at which the head is\n * known to be the resume watermark.\n */\nexport function getSQLiteChangeLogStartupInfo(\n  changeLog: Database,\n): SQLiteChangeLogStartupInfo {\n  const meta = readChangeLogMeta(changeLog);\n  const {minWatermark, headWatermark} = readChangeLogBounds(changeLog);\n  if (minWatermark === null || headWatermark === null) {\n    throw new Error('the SQLite change log must contain its seed transaction');\n  }\n  return {...meta, minWatermark, headWatermark};\n}\n\n/**\n * Extends {@link getSQLiteChangeLogStartupInfo} with the retained row and\n * estimated byte totals. Computing these scans the entire change-log table, so\n * only call it when the totals are consumed, i.e. when the writer (and thus the\n * observer) is enabled.\n */\nexport function getSQLiteChangeLogInfo(\n  changeLog: Database,\n): SQLiteChangeLogInfo {\n  const startup = getSQLiteChangeLogStartupInfo(changeLog);\n  const aggregate = changeLog\n    .prepare(/*sql*/ `\n      SELECT count(*) AS \"rows\",\n             coalesce(sum(\"estimatedBytes\"), 0) AS \"estimatedBytes\"\n        FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n    `)\n    .get<{rows: number; estimatedBytes: number}>();\n\n  return {\n    ...startup,\n    rows: aggregate.rows,\n    estimatedBytes: aggregate.estimatedBytes,\n    cookieRows: readCookieRowCounts(changeLog),\n  };\n}\n\nexport function logSQLiteChangeLogStartup(\n  lc: LogContext,\n  file: string,\n  info: SQLiteChangeLogStartupInfo,\n): void {\n  lc.info?.('SQLite change-log startup', {\n    sqliteChangeLog: {\n      file,\n      epoch: info.epoch,\n      generation: info.generation,\n      replicaID: info.replicaID,\n      schemaVersion: info.schemaVersion,\n      seedWatermark: info.seedWatermark,\n      seededAtMs: info.seededAtMs,\n      minWatermark: info.minWatermark,\n      headWatermark: info.headWatermark,\n    },\n  });\n}\n\n/**\n * The on-disk footprint of a SQLite database: its main file plus its `-wal` and\n * `-wal2` sidecars, whichever exist. `-shm` is excluded — it is a\n * shared-memory index rather than retained bytes.\n *\n * A missing file counts as zero rather than warning. Which sidecars exist\n * depends on the journal mode, and the change-log database is absent entirely\n * until the writer creates it.\n */\nexport async function sqliteFileBytes(\n  lc: LogContext,\n  file: string,\n): Promise<number> {\n  const sizes = await Promise.all(\n    [file, `${file}-wal`, `${file}-wal2`].map(async path => {\n      try {\n        return (await stat(path)).size;\n      } catch (e) {\n        if ((e as NodeJS.ErrnoException).code !== 'ENOENT') {\n          lc.warn?.(`unable to stat ${path} for size metrics`, e);\n        }\n        return 0;\n      }\n    }),\n  );\n  return sizes.reduce((total, size) => total + size, 0);\n}\n\n/** What the writer reports back for a committed transaction. */\nexport type ChangeLogCommit = {\n  readonly watermark: string;\n  readonly stats: ChangeLogStreamTransactionStats;\n  /**\n   * Appending the commit row and committing the log's transaction. This is the\n   * cost that sits on the forward path, once per upstream transaction.\n   */\n  readonly logCommitMs: number;\n};\n\nexport type SQLiteChangeLogObservabilityState = {\n  readonly receivedHead: string;\n  readonly sqliteHead: string;\n  readonly headLag: number | undefined;\n  readonly rows: number;\n  readonly estimatedBytes: number;\n  readonly cookieRows: CookieRowCounts;\n  readonly cookieMutations: number;\n  readonly rollbacks: number;\n  readonly invariantFailures: number;\n  readonly hashMatches: number;\n  readonly hashMismatches: number;\n  readonly hashUnpaired: number;\n};\n\nconst TRANSACTION_ROW_BUCKETS = [2, 3, 5, 10, 25, 50, 100, 250, 1000, 5000];\n\n/**\n * Records writer metrics in the change-streamer process. State changes only\n * after the corresponding write succeeds, except receivedHead, which advances\n * when a commit message arrives so that transient lag behind the log is\n * observable.\n */\nexport class SQLiteChangeLogObserver {\n  readonly #lc: LogContext;\n  readonly #messageProcessing = getOrCreateLatencyHistogram(\n    'replica',\n    'sqlite_change_log.message_processing_duration',\n    'Sampled time to write a replication message to the SQLite change log.',\n  );\n  // The commit that sits on the forward path. There is no second commit to\n  // order against any more: the replica's belongs to a subscriber downstream of\n  // this loop, so `sqlite_change_log.replica_commit_duration` and the phantom\n  // window it measured are gone. Sampled failed commits are still timed by\n  // `message_processing_duration` keyed `tag=\"commit\"`.\n  readonly #logCommitDuration = getOrCreateLatencyHistogram(\n    'replica',\n    'sqlite_change_log.log_commit_duration',\n    'Time to append the commit row to the SQLite change log and commit its ' +\n      'transaction. This is on the forward path: it precedes the forward of ' +\n      \"that transaction's commit message.\",\n  );\n  readonly #transactionRows = getOrCreateValueHistogram(\n    'replica',\n    'sqlite_change_log.transaction_rows',\n    {\n      description: 'Rows stored per committed SQLite change-log transaction.',\n      unit: '{row}',\n      bucketBoundaries: TRANSACTION_ROW_BUCKETS,\n    },\n  );\n  readonly #rollbackCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.rollbacks',\n    'SQLite change-log transactions rolled back before commit.',\n  );\n  readonly #invariantFailureCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.invariant_failures',\n    'Detected SQLite change-log writer invariant failures.',\n  );\n  readonly #hashComparisonCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.hash_comparisons',\n    'Sampled ephemeral transaction hash comparisons between the received change stream and SQLite change-log writes.',\n  );\n  readonly #hashUnpairedCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.hash_unpaired',\n    'Committed change-log transactions without both ephemeral hashes available for comparison.',\n  );\n  // Near zero at steady state: cookie state only moves on schema changes.\n  readonly #cookieMutationCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.cookie_mutations',\n    'Change-log cookie mutations folded from schema changes, by operation.',\n  );\n\n  #receivedHead: string;\n  #sqliteHead: string;\n  #rows: number;\n  #estimatedBytes: number;\n  #cookieRows: CookieRowCounts;\n  #cookieMutations = 0;\n  #transactionWatermark: string | undefined;\n  #sourceHasher: ChangeLogTransactionHasher | undefined;\n  #sourcePos = 0;\n  #sourceCommitHash: string | undefined;\n  #rollbacks = 0;\n  #invariantFailures = 0;\n  #hashMatches = 0;\n  #hashMismatches = 0;\n  #hashUnpaired = 0;\n\n  constructor(lc: LogContext, info: SQLiteChangeLogInfo) {\n    this.#lc = lc.withContext('component', 'sqlite-change-log-observer');\n    // Reconciliation has just put the head at the resume watermark, which is\n    // also the point the stream is about to be received from.\n    this.#receivedHead = info.headWatermark;\n    this.#sqliteHead = info.headWatermark;\n    this.#rows = info.rows;\n    this.#estimatedBytes = info.estimatedBytes;\n    this.#cookieRows = info.cookieRows;\n\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.rows',\n      'Rows retained in the SQLite change log.',\n    ).addCallback(result => result.observe(this.#rows));\n    // Nonzero with no backfill in flight is a leaked cookie, i.e. a backfill\n    // that will be re-requested forever.\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.cookie_rows',\n      'Cookie rows retained in the SQLite change log, by cookie table.',\n    ).addCallback(result => {\n      result.observe(this.#cookieRows.tableMetadata, {table: 'tableMetadata'});\n      result.observe(this.#cookieRows.backfilling, {table: 'backfilling'});\n    });\n    getOrCreateGauge('replica', 'sqlite_change_log.retained_bytes', {\n      description:\n        'Estimated UTF-8 payload bytes retained in the SQLite change log.',\n      unit: 'By',\n    }).addCallback(result => result.observe(this.#estimatedBytes));\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.head',\n      'SQLite change-log head converted from its lexicographic watermark.',\n    ).addCallback(result => {\n      const head = watermarkValue(this.#sqliteHead);\n      if (head !== undefined) {\n        result.observe(head);\n      }\n    });\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.head_lag',\n      'Distance from the latest received commit to the SQLite change-log ' +\n        'head. The log commits before the forward, so this is zero except ' +\n        'within a transaction.',\n    ).addCallback(result => {\n      const lag = watermarkDistance(this.#receivedHead, this.#sqliteHead);\n      if (lag !== undefined) {\n        result.observe(lag);\n      }\n    });\n  }\n\n  /**\n   * Called as each message arrives, before it is written. For a sampled\n   * transaction, hashes the *received* stream and compares it with the hash the\n   * writer computes over what it *stored*.\n   */\n  messageReceived(\n    data: ChangeStreamData,\n    json: string,\n    validateHash = true,\n    storedChange = extractChangeSubstring(json, data[1].tag),\n  ): void {\n    const tag = data[1].tag;\n    if (data[0] === 'begin') {\n      if (this.#sourceHasher !== undefined) {\n        this.#invariantFailure(\n          'Received change-log begin while a source hash is already open',\n          {receivedWatermark: data[2].commitWatermark},\n        );\n      }\n      const watermark = data[2].commitWatermark;\n      this.#sourceHasher = validateHash\n        ? new ChangeLogTransactionHasher()\n        : undefined;\n      this.#sourcePos = 0;\n      this.#sourceCommitHash = undefined;\n      this.#sourceHasher?.add({\n        watermark,\n        pos: this.#sourcePos,\n        tag,\n        change: storedChange,\n        precommit: null,\n      });\n      return;\n    }\n\n    if (tag === 'rollback') {\n      this.#resetSourceHash();\n      return;\n    }\n\n    if (data[0] === 'commit') {\n      this.#receivedHead = data[2].watermark;\n    }\n\n    if (!validateHash) {\n      return;\n    }\n\n    const watermark = this.#transactionWatermark;\n    if (this.#sourceHasher === undefined || watermark === undefined) {\n      this.#invariantFailure(\n        'Received change-log message without an observed transaction',\n        {tag},\n      );\n      return;\n    }\n\n    const commitWatermark =\n      data[0] === 'commit' ? data[2].watermark : watermark;\n    this.#sourceHasher.add({\n      watermark: commitWatermark,\n      pos: ++this.#sourcePos,\n      tag,\n      change: storedChange,\n      precommit: data[0] === 'commit' ? watermark : null,\n    });\n    if (data[0] === 'commit') {\n      this.#sourceCommitHash = this.#sourceHasher.digest();\n      this.#sourceHasher = undefined;\n    }\n  }\n\n  /** Called after the message has been written to the log. */\n  messageProcessed(\n    data: ChangeStreamData,\n    commit: ChangeLogCommit | null,\n    durationMs: number | undefined,\n  ): void {\n    const tag = data[1].tag;\n    if (durationMs !== undefined) {\n      this.#messageProcessing.recordMs(durationMs, {tag, outcome: 'success'});\n    }\n\n    if (data[0] === 'begin') {\n      if (this.#transactionWatermark !== undefined) {\n        this.#invariantFailure('SQLite change-log transaction already open', {\n          openWatermark: this.#transactionWatermark,\n          receivedWatermark: data[2].commitWatermark,\n        });\n      }\n      this.#transactionWatermark = data[2].commitWatermark;\n      return;\n    }\n\n    if (tag === 'rollback') {\n      this.#recordRollback('upstream');\n      this.#transactionWatermark = undefined;\n      this.#resetSourceHash();\n      return;\n    }\n\n    if (data[0] !== 'commit') {\n      return;\n    }\n\n    const watermark = data[2].watermark;\n    if (commit !== null) {\n      this.#logCommitDuration.recordMs(commit.logCommitMs);\n    }\n    if (this.#transactionWatermark !== watermark) {\n      this.#invariantFailure(\n        'SQLite change-log commit does not match the observed begin',\n        {openWatermark: this.#transactionWatermark, commitWatermark: watermark},\n      );\n    }\n    if (commit === null) {\n      this.#invariantFailure(\n        'SQLite change-log commit result is missing writer statistics',\n        {commitWatermark: watermark},\n      );\n      this.#recordUnpaired(watermark, 'missing-sqlite-hash');\n    } else {\n      if (commit.watermark !== watermark) {\n        this.#invariantFailure(\n          'SQLite change-log commit result has an unexpected watermark',\n          {commitWatermark: watermark, resultWatermark: commit.watermark},\n        );\n      }\n      this.#rows += commit.stats.rows;\n      this.#estimatedBytes += commit.stats.estimatedBytes;\n      this.#transactionRows.record(commit.stats.rows);\n      if (\n        this.#sourceCommitHash !== undefined ||\n        commit.stats.hash !== undefined\n      ) {\n        this.#compareHashes(\n          watermark,\n          this.#sourceCommitHash,\n          commit.stats.hash,\n        );\n      }\n    }\n    if (watermark < this.#sqliteHead) {\n      this.#invariantFailure('SQLite change-log head regressed', {\n        previousHead: this.#sqliteHead,\n        commitWatermark: watermark,\n      });\n    }\n    this.#sqliteHead = watermark;\n    this.#transactionWatermark = undefined;\n    this.#resetSourceHash();\n  }\n\n  messageFailed(\n    data: ChangeStreamData,\n    error: unknown,\n    durationMs: number | undefined,\n  ): void {\n    const tag = data[1].tag;\n    if (durationMs !== undefined) {\n      this.#messageProcessing.recordMs(durationMs, {tag, outcome: 'error'});\n    }\n    if (data[0] === 'commit' && this.#sourceCommitHash !== undefined) {\n      // No commit duration here: a failed commit produces no statistics, so\n      // there is nothing to attribute. The attempt is timed by\n      // `message_processing_duration{tag=\"commit\",outcome=\"error\"}` above.\n      this.#recordUnpaired(data[2].watermark, 'sqlite-commit-failed');\n    }\n    if (\n      error instanceof Error &&\n      error.name === 'ChangeLogStreamInvariantError'\n    ) {\n      this.#invariantFailure(error.message);\n    }\n    if (this.#transactionWatermark !== undefined || data[0] === 'begin') {\n      this.#recordRollback('processing-error');\n    }\n    this.#transactionWatermark = undefined;\n    this.#resetSourceHash();\n  }\n\n  /**\n   * A committed transaction folded schema changes into the cookie jar. Reported\n   * separately from {@link messageProcessed} because it costs a read of the\n   * cookie tables, which is only worth taking when they actually moved.\n   */\n  cookiesMutated(ops: readonly CookieOpTag[], rows: CookieRowCounts): void {\n    for (const op of ops) {\n      this.#cookieMutations++;\n      this.#cookieMutationCounter.add(1, {op});\n    }\n    this.#cookieRows = rows;\n  }\n\n  /**\n   * A missed truncate-above surfaces as a constraint violation on the writer's\n   * plain `INSERT`, which the fail-soft policy would otherwise absorb quietly.\n   * Classifying it as an invariant failure is what keeps a reconciliation bug\n   * from looking like a lost cache.\n   */\n  invariantFailure(message: string, details?: Record<string, unknown>): void {\n    this.#invariantFailure(message, details);\n  }\n\n  /** The source interrupted the stream mid-transaction. */\n  abort(): void {\n    if (this.#transactionWatermark !== undefined) {\n      this.#recordRollback('source-interruption');\n      this.#transactionWatermark = undefined;\n    }\n    this.#resetSourceHash();\n  }\n\n  /** Reconciliation moved the head, so the observer's totals are restated. */\n  reconciled(info: SQLiteChangeLogInfo): void {\n    this.#receivedHead = info.headWatermark;\n    this.#sqliteHead = info.headWatermark;\n    this.#rows = info.rows;\n    this.#estimatedBytes = info.estimatedBytes;\n    this.#cookieRows = info.cookieRows;\n    this.#transactionWatermark = undefined;\n    this.#resetSourceHash();\n  }\n\n  state(): SQLiteChangeLogObservabilityState {\n    return {\n      receivedHead: this.#receivedHead,\n      sqliteHead: this.#sqliteHead,\n      headLag: watermarkDistance(this.#receivedHead, this.#sqliteHead),\n      rows: this.#rows,\n      estimatedBytes: this.#estimatedBytes,\n      cookieRows: this.#cookieRows,\n      cookieMutations: this.#cookieMutations,\n      rollbacks: this.#rollbacks,\n      invariantFailures: this.#invariantFailures,\n      hashMatches: this.#hashMatches,\n      hashMismatches: this.#hashMismatches,\n      hashUnpaired: this.#hashUnpaired,\n    };\n  }\n\n  #recordRollback(\n    reason: 'upstream' | 'processing-error' | 'source-interruption',\n  ) {\n    this.#rollbacks++;\n    this.#rollbackCounter.add(1, {reason});\n  }\n\n  #invariantFailure(message: string, details?: Record<string, unknown>) {\n    this.#invariantFailures++;\n    this.#invariantFailureCounter.add(1);\n    this.#lc.error?.(message, details);\n  }\n\n  #compareHashes(\n    watermark: string,\n    sourceHash: string | undefined,\n    sqliteHash: string | undefined,\n  ): void {\n    if (sourceHash === undefined) {\n      this.#recordUnpaired(watermark, 'missing-source-hash');\n      return;\n    }\n    if (sqliteHash === undefined) {\n      this.#recordUnpaired(watermark, 'missing-sqlite-hash');\n      return;\n    }\n    if (sourceHash === sqliteHash) {\n      this.#hashMatches++;\n      this.#hashComparisonCounter.add(1, {outcome: 'match'});\n      return;\n    }\n    this.#hashMismatches++;\n    this.#hashComparisonCounter.add(1, {outcome: 'mismatch'});\n    this.#lc.error?.('SQLite change-log transaction hash mismatch', {\n      sqliteChangeLogHashComparison: {\n        watermark,\n        sourceHash: sourceHash.slice(0, 16),\n        sqliteHash: sqliteHash.slice(0, 16),\n      },\n    });\n  }\n\n  #recordUnpaired(\n    watermark: string,\n    reason:\n      | 'missing-source-hash'\n      | 'missing-sqlite-hash'\n      | 'sqlite-commit-failed',\n  ): void {\n    this.#hashUnpaired++;\n    this.#hashUnpairedCounter.add(1, {reason});\n    this.#lc.warn?.('SQLite change-log transaction hash was not compared', {\n      sqliteChangeLogHashComparison: {watermark, reason},\n    });\n  }\n\n  #resetSourceHash(): void {\n    this.#sourceHasher = undefined;\n    this.#sourcePos = 0;\n    this.#sourceCommitHash = undefined;\n  }\n}\n\nfunction watermarkValue(watermark: string): number | undefined {\n  try {\n    const value = Number(versionFromLexi(watermark));\n    return Number.isFinite(value) ? value : undefined;\n  } catch {\n    return undefined;\n  }\n}\n\nfunction watermarkDistance(\n  receivedHead: string,\n  sqliteHead: string,\n): number | undefined {\n  if (receivedHead <= sqliteHead) {\n    return 0;\n  }\n  try {\n    const distance =\n      versionFromLexi(receivedHead) - versionFromLexi(sqliteHead);\n    const value = Number(distance);\n    return Number.isFinite(value) ? value : undefined;\n  } catch {\n    return undefined;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoDA,SAAgB,+BACd,IACA,QACM;CACN,QAAQ,OAAO,QAAf;EACE,KAAK;GACH,mBACE,WACA,8CACA,kHAEF,EAAE,IAAI,OAAO,IAAI;GACjB;EACF,KAAK;GACH,mBACE,WACA,qCACA,mIAEF,EAAE,IAAI,GAAG,EAAC,QAAQ,OAAO,OAAM,CAAC;GAChC;EACF,KAAK,QACH;EACF,SACE,YAAY,MAAM;CACtB;CACA,IAAI,OAAO,cAKT,mBACE,WACA,mCACA,sGAEF,EAAE,IAAI,GAAG,EAAC,QAAQ,OAAO,OAAM,CAAC;CAElC,GAAG,QAAQ,oCAAoC,EAC7C,0BAA0B,OAC5B,CAAC;AACH;;;;;;;;AAyBA,SAAgB,8BACd,WAC4B;CAC5B,MAAM,OAAO,kBAAkB,SAAS;CACxC,MAAM,EAAC,cAAc,kBAAiB,oBAAoB,SAAS;CACnE,IAAI,iBAAiB,QAAQ,kBAAkB,MAC7C,MAAM,IAAI,MAAM,yDAAyD;CAE3E,OAAO;EAAC,GAAG;EAAM;EAAc;CAAa;AAC9C;;;;;;;AAQA,SAAgB,uBACd,WACqB;CACrB,MAAM,UAAU,8BAA8B,SAAS;CACvD,MAAM,YAAY,UACf,QAAgB;;;gBAGL,wBAAwB;KACnC,EACA,IAA4C;CAE/C,OAAO;EACL,GAAG;EACH,MAAM,UAAU;EAChB,gBAAgB,UAAU;EAC1B,YAAY,oBAAoB,SAAS;CAC3C;AACF;AAEA,SAAgB,0BACd,IACA,MACA,MACM;CACN,GAAG,OAAO,6BAA6B,EACrC,iBAAiB;EACf;EACA,OAAO,KAAK;EACZ,YAAY,KAAK;EACjB,WAAW,KAAK;EAChB,eAAe,KAAK;EACpB,eAAe,KAAK;EACpB,YAAY,KAAK;EACjB,cAAc,KAAK;EACnB,eAAe,KAAK;CACtB,EACF,CAAC;AACH;;;;;;;;;;AAWA,eAAsB,gBACpB,IACA,MACiB;CAajB,QAAO,MAZa,QAAQ,IAC1B;EAAC;EAAM,GAAG,KAAK;EAAO,GAAG,KAAK;CAAM,EAAE,IAAI,OAAM,SAAQ;EACtD,IAAI;GACF,QAAQ,MAAM,KAAK,IAAI,GAAG;EAC5B,SAAS,GAAG;GACV,IAAK,EAA4B,SAAS,UACxC,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;GAExD,OAAO;EACT;CACF,CAAC,CACH,GACa,QAAQ,OAAO,SAAS,QAAQ,MAAM,CAAC;AACtD;AA4BA,IAAM,0BAA0B;CAAC;CAAG;CAAG;CAAG;CAAI;CAAI;CAAI;CAAK;CAAK;CAAM;AAAI;;;;;;;AAQ1E,IAAa,0BAAb,MAAqC;CACnC;CACA,qBAA8B,4BAC5B,WACA,iDACA,uEACF;CAMA,qBAA8B,4BAC5B,WACA,yCACA,+KAGF;CACA,mBAA4B,0BAC1B,WACA,sCACA;EACE,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CACF;CACA,mBAA4B,mBAC1B,WACA,+BACA,2DACF;CACA,2BAAoC,mBAClC,WACA,wCACA,uDACF;CACA,yBAAkC,mBAChC,WACA,sCACA,iHACF;CACA,uBAAgC,mBAC9B,WACA,mCACA,2FACF;CAEA,yBAAkC,mBAChC,WACA,sCACA,uEACF;CAEA;CACA;CACA;CACA;CACA;CACA,mBAAmB;CACnB;CACA;CACA,aAAa;CACb;CACA,aAAa;CACb,qBAAqB;CACrB,eAAe;CACf,kBAAkB;CAClB,gBAAgB;CAEhB,YAAY,IAAgB,MAA2B;EACrD,KAAKA,MAAM,GAAG,YAAY,aAAa,4BAA4B;EAGnE,KAAKS,gBAAgB,KAAK;EAC1B,KAAKC,cAAc,KAAK;EACxB,KAAKC,QAAQ,KAAK;EAClB,KAAKC,kBAAkB,KAAK;EAC5B,KAAKC,cAAc,KAAK;EAExB,iBACE,WACA,0BACA,yCACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKF,KAAK,CAAC;EAGlD,iBACE,WACA,iCACA,iEACF,EAAE,aAAY,WAAU;GACtB,OAAO,QAAQ,KAAKE,YAAY,eAAe,EAAC,OAAO,gBAAe,CAAC;GACvE,OAAO,QAAQ,KAAKA,YAAY,aAAa,EAAC,OAAO,cAAa,CAAC;EACrE,CAAC;EACD,iBAAiB,WAAW,oCAAoC;GAC9D,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKD,eAAe,CAAC;EAC7D,iBACE,WACA,0BACA,oEACF,EAAE,aAAY,WAAU;GACtB,MAAM,OAAO,eAAe,KAAKF,WAAW;GAC5C,IAAI,SAAS,KAAA,GACX,OAAO,QAAQ,IAAI;EAEvB,CAAC;EACD,iBACE,WACA,8BACA,0JAGF,EAAE,aAAY,WAAU;GACtB,MAAM,MAAM,kBAAkB,KAAKD,eAAe,KAAKC,WAAW;GAClE,IAAI,QAAQ,KAAA,GACV,OAAO,QAAQ,GAAG;EAEtB,CAAC;CACH;;;;;;CAOA,gBACE,MACA,MACA,eAAe,MACf,eAAe,uBAAuB,MAAM,KAAK,GAAG,GAAG,GACjD;EACN,MAAM,MAAM,KAAK,GAAG;EACpB,IAAI,KAAK,OAAO,SAAS;GACvB,IAAI,KAAKI,kBAAkB,KAAA,GACzB,KAAKC,kBACH,iEACA,EAAC,mBAAmB,KAAK,GAAG,gBAAe,CAC7C;GAEF,MAAM,YAAY,KAAK,GAAG;GAC1B,KAAKD,gBAAgB,eACjB,IAAI,2BAA2B,IAC/B,KAAA;GACJ,KAAKE,aAAa;GAClB,KAAKC,oBAAoB,KAAA;GACzB,KAAKH,eAAe,IAAI;IACtB;IACA,KAAK,KAAKE;IACV;IACA,QAAQ;IACR,WAAW;GACb,CAAC;GACD;EACF;EAEA,IAAI,QAAQ,YAAY;GACtB,KAAKE,iBAAiB;GACtB;EACF;EAEA,IAAI,KAAK,OAAO,UACd,KAAKT,gBAAgB,KAAK,GAAG;EAG/B,IAAI,CAAC,cACH;EAGF,MAAM,YAAY,KAAKU;EACvB,IAAI,KAAKL,kBAAkB,KAAA,KAAa,cAAc,KAAA,GAAW;GAC/D,KAAKC,kBACH,+DACA,EAAC,IAAG,CACN;GACA;EACF;EAEA,MAAM,kBACJ,KAAK,OAAO,WAAW,KAAK,GAAG,YAAY;EAC7C,KAAKD,cAAc,IAAI;GACrB,WAAW;GACX,KAAK,EAAE,KAAKE;GACZ;GACA,QAAQ;GACR,WAAW,KAAK,OAAO,WAAW,YAAY;EAChD,CAAC;EACD,IAAI,KAAK,OAAO,UAAU;GACxB,KAAKC,oBAAoB,KAAKH,cAAc,OAAO;GACnD,KAAKA,gBAAgB,KAAA;EACvB;CACF;;CAGA,iBACE,MACA,QACA,YACM;EACN,MAAM,MAAM,KAAK,GAAG;EACpB,IAAI,eAAe,KAAA,GACjB,KAAKb,mBAAmB,SAAS,YAAY;GAAC;GAAK,SAAS;EAAS,CAAC;EAGxE,IAAI,KAAK,OAAO,SAAS;GACvB,IAAI,KAAKkB,0BAA0B,KAAA,GACjC,KAAKJ,kBAAkB,8CAA8C;IACnE,eAAe,KAAKI;IACpB,mBAAmB,KAAK,GAAG;GAC7B,CAAC;GAEH,KAAKA,wBAAwB,KAAK,GAAG;GACrC;EACF;EAEA,IAAI,QAAQ,YAAY;GACtB,KAAKC,gBAAgB,UAAU;GAC/B,KAAKD,wBAAwB,KAAA;GAC7B,KAAKD,iBAAiB;GACtB;EACF;EAEA,IAAI,KAAK,OAAO,UACd;EAGF,MAAM,YAAY,KAAK,GAAG;EAC1B,IAAI,WAAW,MACb,KAAKhB,mBAAmB,SAAS,OAAO,WAAW;EAErD,IAAI,KAAKiB,0BAA0B,WACjC,KAAKJ,kBACH,8DACA;GAAC,eAAe,KAAKI;GAAuB,iBAAiB;EAAS,CACxE;EAEF,IAAI,WAAW,MAAM;GACnB,KAAKJ,kBACH,gEACA,EAAC,iBAAiB,UAAS,CAC7B;GACA,KAAKM,gBAAgB,WAAW,qBAAqB;EACvD,OAAO;GACL,IAAI,OAAO,cAAc,WACvB,KAAKN,kBACH,+DACA;IAAC,iBAAiB;IAAW,iBAAiB,OAAO;GAAS,CAChE;GAEF,KAAKJ,SAAS,OAAO,MAAM;GAC3B,KAAKC,mBAAmB,OAAO,MAAM;GACrC,KAAKT,iBAAiB,OAAO,OAAO,MAAM,IAAI;GAC9C,IACE,KAAKc,sBAAsB,KAAA,KAC3B,OAAO,MAAM,SAAS,KAAA,GAEtB,KAAKK,eACH,WACA,KAAKL,mBACL,OAAO,MAAM,IACf;EAEJ;EACA,IAAI,YAAY,KAAKP,aACnB,KAAKK,kBAAkB,oCAAoC;GACzD,cAAc,KAAKL;GACnB,iBAAiB;EACnB,CAAC;EAEH,KAAKA,cAAc;EACnB,KAAKS,wBAAwB,KAAA;EAC7B,KAAKD,iBAAiB;CACxB;CAEA,cACE,MACA,OACA,YACM;EACN,MAAM,MAAM,KAAK,GAAG;EACpB,IAAI,eAAe,KAAA,GACjB,KAAKjB,mBAAmB,SAAS,YAAY;GAAC;GAAK,SAAS;EAAO,CAAC;EAEtE,IAAI,KAAK,OAAO,YAAY,KAAKgB,sBAAsB,KAAA,GAIrD,KAAKI,gBAAgB,KAAK,GAAG,WAAW,sBAAsB;EAEhE,IACE,iBAAiB,SACjB,MAAM,SAAS,iCAEf,KAAKN,kBAAkB,MAAM,OAAO;EAEtC,IAAI,KAAKI,0BAA0B,KAAA,KAAa,KAAK,OAAO,SAC1D,KAAKC,gBAAgB,kBAAkB;EAEzC,KAAKD,wBAAwB,KAAA;EAC7B,KAAKD,iBAAiB;CACxB;;;;;;CAOA,eAAe,KAA6B,MAA6B;EACvE,KAAK,MAAM,MAAM,KAAK;GACpB,KAAKK;GACL,KAAKf,uBAAuB,IAAI,GAAG,EAAC,GAAE,CAAC;EACzC;EACA,KAAKK,cAAc;CACrB;;;;;;;CAQA,iBAAiB,SAAiB,SAAyC;EACzE,KAAKE,kBAAkB,SAAS,OAAO;CACzC;;CAGA,QAAc;EACZ,IAAI,KAAKI,0BAA0B,KAAA,GAAW;GAC5C,KAAKC,gBAAgB,qBAAqB;GAC1C,KAAKD,wBAAwB,KAAA;EAC/B;EACA,KAAKD,iBAAiB;CACxB;;CAGA,WAAW,MAAiC;EAC1C,KAAKT,gBAAgB,KAAK;EAC1B,KAAKC,cAAc,KAAK;EACxB,KAAKC,QAAQ,KAAK;EAClB,KAAKC,kBAAkB,KAAK;EAC5B,KAAKC,cAAc,KAAK;EACxB,KAAKM,wBAAwB,KAAA;EAC7B,KAAKD,iBAAiB;CACxB;CAEA,QAA2C;EACzC,OAAO;GACL,cAAc,KAAKT;GACnB,YAAY,KAAKC;GACjB,SAAS,kBAAkB,KAAKD,eAAe,KAAKC,WAAW;GAC/D,MAAM,KAAKC;GACX,gBAAgB,KAAKC;GACrB,YAAY,KAAKC;GACjB,iBAAiB,KAAKU;GACtB,WAAW,KAAKC;GAChB,mBAAmB,KAAKC;GACxB,aAAa,KAAKC;GAClB,gBAAgB,KAAKC;GACrB,cAAc,KAAKC;EACrB;CACF;CAEA,gBACE,QACA;EACA,KAAKJ;EACL,KAAKpB,iBAAiB,IAAI,GAAG,EAAC,OAAM,CAAC;CACvC;CAEA,kBAAkB,SAAiB,SAAmC;EACpE,KAAKqB;EACL,KAAKpB,yBAAyB,IAAI,CAAC;EACnC,KAAKL,IAAI,QAAQ,SAAS,OAAO;CACnC;CAEA,eACE,WACA,YACA,YACM;EACN,IAAI,eAAe,KAAA,GAAW;GAC5B,KAAKqB,gBAAgB,WAAW,qBAAqB;GACrD;EACF;EACA,IAAI,eAAe,KAAA,GAAW;GAC5B,KAAKA,gBAAgB,WAAW,qBAAqB;GACrD;EACF;EACA,IAAI,eAAe,YAAY;GAC7B,KAAKK;GACL,KAAKpB,uBAAuB,IAAI,GAAG,EAAC,SAAS,QAAO,CAAC;GACrD;EACF;EACA,KAAKqB;EACL,KAAKrB,uBAAuB,IAAI,GAAG,EAAC,SAAS,WAAU,CAAC;EACxD,KAAKN,IAAI,QAAQ,+CAA+C,EAC9D,+BAA+B;GAC7B;GACA,YAAY,WAAW,MAAM,GAAG,EAAE;GAClC,YAAY,WAAW,MAAM,GAAG,EAAE;EACpC,EACF,CAAC;CACH;CAEA,gBACE,WACA,QAIM;EACN,KAAK4B;EACL,KAAKrB,qBAAqB,IAAI,GAAG,EAAC,OAAM,CAAC;EACzC,KAAKP,IAAI,OAAO,uDAAuD,EACrE,+BAA+B;GAAC;GAAW;EAAM,EACnD,CAAC;CACH;CAEA,mBAAyB;EACvB,KAAKc,gBAAgB,KAAA;EACrB,KAAKE,aAAa;EAClB,KAAKC,oBAAoB,KAAA;CAC3B;AACF;AAEA,SAAS,eAAe,WAAuC;CAC7D,IAAI;EACF,MAAM,QAAQ,OAAO,gBAAgB,SAAS,CAAC;EAC/C,OAAO,OAAO,SAAS,KAAK,IAAI,QAAQ,KAAA;CAC1C,QAAQ;EACN;CACF;AACF;AAEA,SAAS,kBACP,cACA,YACoB;CACpB,IAAI,gBAAgB,YAClB,OAAO;CAET,IAAI;EACF,MAAM,WACJ,gBAAgB,YAAY,IAAI,gBAAgB,UAAU;EAC5D,MAAM,QAAQ,OAAO,QAAQ;EAC7B,OAAO,OAAO,SAAS,KAAK,IAAI,QAAQ,KAAA;CAC1C,QAAQ;EACN;CACF;AACF"}
+{"version":3,"file":"sqlite-change-log-observability.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/sqlite-change-log-observability.ts"],"sourcesContent":["/**\n * Observability for the SQLite change log.\n *\n * Everything here now runs in the **change-streamer** process, which is where\n * the writer lives. The module stays beside the rest of the change-log code\n * under `services/replicator/` for the same reason `change-log-db.ts` and\n * `change-log-stream-writer.ts` do: moving files would obscure the topology\n * diff, and the writer's home is a property of who calls it rather than of\n * where it sits. Metric names are unchanged from 7H — only their emitting\n * process, and therefore their resource attributes, moved.\n */\n\nimport {stat} from 'node:fs/promises';\nimport type {LogContext} from '@rocicorp/logger';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n  getOrCreateLatencyHistogram,\n  getOrCreateValueHistogram,\n} from '../../observability/metrics.ts';\nimport {versionFromLexi} from '../../types/lexi-version.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {extractChangeSubstring} from '../change-streamer/change-log-codec.ts';\nimport {ChangeLogTransactionHasher} from '../change-streamer/change-log-transaction-hash.ts';\nimport {\n  readCookieRowCounts,\n  type CookieOpTag,\n  type CookieRowCounts,\n} from './change-log-cookies.ts';\nimport {\n  CHANGE_LOG_STREAM_TABLE,\n  readChangeLogBounds,\n  readChangeLogMeta,\n  type ChangeLogMeta,\n  type ReconcileResult,\n} from './change-log-db.ts';\nimport type {ChangeLogStreamTransactionStats} from './change-log-stream-writer.ts';\n\n/**\n * Reports the reconciliation of the change-log database against the watermark\n * the stream connection resumes from. This runs once per stream connection, so\n * a reconnect storm is visible here as well as a restart.\n *\n * A wipe is not an error — it is how every abnormal change-log state is\n * collapsed into a correct one — but each wipe costs the retention window, so\n * every reconnecting subscriber below the head gets `too-old`. The `gap` reason\n * in particular means the log neither held the resume watermark nor could be\n * truncated to it, so a nonzero rate in steady state is an alert, not a\n * statistic.\n */\nexport function recordSQLiteChangeLogReconcile(\n  lc: LogContext,\n  result: ReconcileResult,\n): void {\n  switch (result.action) {\n    case 'truncated':\n      getOrCreateCounter(\n        'replica',\n        'sqlite_change_log.reconcile_truncated_rows',\n        'Phantom change-log rows removed at reconciliation, i.e. rows above ' +\n          'the watermark the change stream resumed from.',\n      ).add(result.rows);\n      break;\n    case 'reseeded':\n      getOrCreateCounter(\n        'replica',\n        'sqlite_change_log.reconcile_wipes',\n        'Change-log wipes at reconciliation, by reason. Each one resets the ' +\n          'retention window, so subscribers below the head get `too-old`.',\n      ).add(1, {reason: result.reason});\n      break;\n    case 'none':\n      break;\n    default:\n      unreachable(result);\n  }\n  if (result.cookiesStale) {\n    // The invariant-17 counter: after any reconcile that moved the head, the\n    // log's cookie set has to be replaced with the one that belongs to the\n    // resume watermark. A nonzero `truncated` series in steady state means\n    // phantom truncation is happening on a shard where it should not.\n    getOrCreateCounter(\n      'replica',\n      'sqlite_change_log.cookie_reseed',\n      'Change-log cookie sets invalidated at reconciliation, by the ' +\n        'reconcile action that invalidated them.',\n    ).add(1, {action: result.action});\n  }\n  lc.debug?.('SQLite change-log reconciliation', {\n    sqliteChangeLogReconcile: result,\n  });\n}\n\n/**\n * What the log says about itself. Nothing here reads the replica: after the\n * writer moved to the change-streamer, the replica's state version is a\n * consequence of this log rather than a fact about it.\n */\nexport type SQLiteChangeLogStartupInfo = ChangeLogMeta & {\n  readonly minWatermark: string;\n  readonly headWatermark: string;\n};\n\nexport type SQLiteChangeLogInfo = SQLiteChangeLogStartupInfo & {\n  readonly rows: number;\n  readonly estimatedBytes: number;\n  readonly cookieRows: CookieRowCounts;\n};\n\n/**\n * Reads the meta row and covered range. Unlike {@link getSQLiteChangeLogInfo}\n * this does not scan the log; see {@link readChangeLogBounds}.\n *\n * Call this after reconciliation, which is the only point at which the head is\n * known to be the resume watermark.\n */\nexport function getSQLiteChangeLogStartupInfo(\n  changeLog: Database,\n): SQLiteChangeLogStartupInfo {\n  const meta = readChangeLogMeta(changeLog);\n  const {minWatermark, headWatermark} = readChangeLogBounds(changeLog);\n  if (minWatermark === null || headWatermark === null) {\n    throw new Error('the SQLite change log must contain its seed transaction');\n  }\n  return {...meta, minWatermark, headWatermark};\n}\n\n/**\n * Extends {@link getSQLiteChangeLogStartupInfo} with the retained row and\n * estimated byte totals. Computing these scans the entire change-log table, so\n * only call it when the totals are consumed, i.e. when the writer (and thus the\n * observer) is enabled.\n */\nexport function getSQLiteChangeLogInfo(\n  changeLog: Database,\n): SQLiteChangeLogInfo {\n  const startup = getSQLiteChangeLogStartupInfo(changeLog);\n  const aggregate = changeLog\n    .prepare(/*sql*/ `\n      SELECT count(*) AS \"rows\",\n             coalesce(sum(\"estimatedBytes\"), 0) AS \"estimatedBytes\"\n        FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n    `)\n    .get<{rows: number; estimatedBytes: number}>();\n\n  return {\n    ...startup,\n    rows: aggregate.rows,\n    estimatedBytes: aggregate.estimatedBytes,\n    cookieRows: readCookieRowCounts(changeLog),\n  };\n}\n\nexport function logSQLiteChangeLogStartup(\n  lc: LogContext,\n  file: string,\n  info: SQLiteChangeLogStartupInfo,\n): void {\n  lc.info?.('SQLite change-log startup', {\n    sqliteChangeLog: {\n      file,\n      epoch: info.epoch,\n      generation: info.generation,\n      replicaID: info.replicaID,\n      schemaVersion: info.schemaVersion,\n      seedWatermark: info.seedWatermark,\n      seededAtMs: info.seededAtMs,\n      minWatermark: info.minWatermark,\n      headWatermark: info.headWatermark,\n    },\n  });\n}\n\n/**\n * The on-disk footprint of a SQLite database: its main file plus its `-wal` and\n * `-wal2` sidecars, whichever exist. `-shm` is excluded — it is a\n * shared-memory index rather than retained bytes.\n *\n * A missing file counts as zero rather than warning. Which sidecars exist\n * depends on the journal mode, and the change-log database is absent entirely\n * until the writer creates it.\n */\nexport async function sqliteFileBytes(\n  lc: LogContext,\n  file: string,\n): Promise<number> {\n  const sizes = await Promise.all(\n    [file, `${file}-wal`, `${file}-wal2`].map(async path => {\n      try {\n        return (await stat(path)).size;\n      } catch (e) {\n        if ((e as NodeJS.ErrnoException).code !== 'ENOENT') {\n          lc.warn?.(`unable to stat ${path} for size metrics`, e);\n        }\n        return 0;\n      }\n    }),\n  );\n  return sizes.reduce((total, size) => total + size, 0);\n}\n\n/** What the writer reports back for a committed transaction. */\nexport type ChangeLogCommit = {\n  readonly watermark: string;\n  readonly stats: ChangeLogStreamTransactionStats;\n  /**\n   * Appending the commit row and committing the log's transaction. This is the\n   * cost that sits on the forward path, once per upstream transaction.\n   */\n  readonly logCommitMs: number;\n};\n\nexport type SQLiteChangeLogObservabilityState = {\n  readonly receivedHead: string;\n  readonly sqliteHead: string;\n  readonly headLag: number | undefined;\n  readonly rows: number;\n  readonly estimatedBytes: number;\n  readonly cookieRows: CookieRowCounts;\n  readonly cookieMutations: number;\n  readonly rollbacks: number;\n  readonly invariantFailures: number;\n  readonly hashMatches: number;\n  readonly hashMismatches: number;\n  readonly hashUnpaired: number;\n};\n\nconst TRANSACTION_ROW_BUCKETS = [2, 3, 5, 10, 25, 50, 100, 250, 1000, 5000];\n\n/**\n * Records writer metrics in the change-streamer process. State changes only\n * after the corresponding write succeeds, except receivedHead, which advances\n * when a commit message arrives so that transient lag behind the log is\n * observable.\n */\nexport class SQLiteChangeLogObserver {\n  readonly #lc: LogContext;\n  readonly #messageProcessing = getOrCreateLatencyHistogram(\n    'replica',\n    'sqlite_change_log.message_processing_duration',\n    'Sampled time to write a replication message to the SQLite change log.',\n  );\n  // The commit that sits on the forward path. There is no second commit to\n  // order against any more: the replica's belongs to a subscriber downstream of\n  // this loop, so `sqlite_change_log.replica_commit_duration` and the phantom\n  // window it measured are gone. Sampled failed commits are still timed by\n  // `message_processing_duration` keyed `tag=\"commit\"`.\n  readonly #logCommitDuration = getOrCreateLatencyHistogram(\n    'replica',\n    'sqlite_change_log.log_commit_duration',\n    'Time to append the commit row to the SQLite change log and commit its ' +\n      'transaction. This is on the forward path: it precedes the forward of ' +\n      \"that transaction's commit message.\",\n  );\n  readonly #transactionRows = getOrCreateValueHistogram(\n    'replica',\n    'sqlite_change_log.transaction_rows',\n    {\n      description: 'Rows stored per committed SQLite change-log transaction.',\n      unit: '{row}',\n      bucketBoundaries: TRANSACTION_ROW_BUCKETS,\n    },\n  );\n  readonly #rollbackCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.rollbacks',\n    'SQLite change-log transactions rolled back before commit.',\n  );\n  readonly #invariantFailureCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.invariant_failures',\n    'Detected SQLite change-log writer invariant failures.',\n  );\n  readonly #hashComparisonCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.hash_comparisons',\n    'Sampled ephemeral transaction hash comparisons between the received change stream and SQLite change-log writes.',\n  );\n  readonly #hashUnpairedCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.hash_unpaired',\n    'Committed change-log transactions without both ephemeral hashes available for comparison.',\n  );\n  // Near zero at steady state: cookie state only moves on schema changes.\n  readonly #cookieMutationCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.cookie_mutations',\n    'Change-log cookie mutations folded from schema changes, by operation.',\n  );\n\n  #receivedHead: string;\n  #sqliteHead: string;\n  #rows: number;\n  #estimatedBytes: number;\n  #cookieRows: CookieRowCounts;\n  #cookieMutations = 0;\n  #transactionWatermark: string | undefined;\n  #sourceHasher: ChangeLogTransactionHasher | undefined;\n  #sourcePos = 0;\n  #sourceCommitHash: string | undefined;\n  #rollbacks = 0;\n  #invariantFailures = 0;\n  #hashMatches = 0;\n  #hashMismatches = 0;\n  #hashUnpaired = 0;\n\n  constructor(lc: LogContext, info: SQLiteChangeLogInfo) {\n    this.#lc = lc.withContext('component', 'sqlite-change-log-observer');\n    // Reconciliation has just put the head at the resume watermark, which is\n    // also the point the stream is about to be received from.\n    this.#receivedHead = info.headWatermark;\n    this.#sqliteHead = info.headWatermark;\n    this.#rows = info.rows;\n    this.#estimatedBytes = info.estimatedBytes;\n    this.#cookieRows = info.cookieRows;\n\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.rows',\n      'Rows retained in the SQLite change log.',\n    ).addCallback(result => result.observe(this.#rows));\n    // Nonzero with no backfill in flight is a leaked cookie, i.e. a backfill\n    // that will be re-requested forever.\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.cookie_rows',\n      'Cookie rows retained in the SQLite change log, by cookie table.',\n    ).addCallback(result => {\n      result.observe(this.#cookieRows.tableMetadata, {table: 'tableMetadata'});\n      result.observe(this.#cookieRows.backfilling, {table: 'backfilling'});\n    });\n    getOrCreateGauge('replica', 'sqlite_change_log.retained_bytes', {\n      description:\n        'Estimated UTF-8 payload bytes retained in the SQLite change log.',\n      unit: 'By',\n    }).addCallback(result => result.observe(this.#estimatedBytes));\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.head',\n      'SQLite change-log head converted from its lexicographic watermark.',\n    ).addCallback(result => {\n      const head = watermarkValue(this.#sqliteHead);\n      if (head !== undefined) {\n        result.observe(head);\n      }\n    });\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.head_lag',\n      'Distance from the latest received commit to the SQLite change-log ' +\n        'head. The log commits before the forward, so this is zero except ' +\n        'within a transaction.',\n    ).addCallback(result => {\n      const lag = watermarkDistance(this.#receivedHead, this.#sqliteHead);\n      if (lag !== undefined) {\n        result.observe(lag);\n      }\n    });\n  }\n\n  /**\n   * Called as each message arrives, before it is written. For a sampled\n   * transaction, hashes the *received* stream and compares it with the hash the\n   * writer computes over what it *stored*.\n   */\n  messageReceived(\n    data: ChangeStreamData,\n    json: string,\n    validateHash = true,\n    storedChange = extractChangeSubstring(json, data[1].tag),\n  ): void {\n    const tag = data[1].tag;\n    if (data[0] === 'begin') {\n      if (this.#sourceHasher !== undefined) {\n        this.#invariantFailure(\n          'Received change-log begin while a source hash is already open',\n          {receivedWatermark: data[2].commitWatermark},\n        );\n      }\n      const watermark = data[2].commitWatermark;\n      this.#sourceHasher = validateHash\n        ? new ChangeLogTransactionHasher()\n        : undefined;\n      this.#sourcePos = 0;\n      this.#sourceCommitHash = undefined;\n      this.#sourceHasher?.add({\n        watermark,\n        pos: this.#sourcePos,\n        tag,\n        change: storedChange,\n        precommit: null,\n      });\n      return;\n    }\n\n    if (tag === 'rollback') {\n      this.#resetSourceHash();\n      return;\n    }\n\n    if (data[0] === 'commit') {\n      this.#receivedHead = data[2].watermark;\n    }\n\n    if (!validateHash) {\n      return;\n    }\n\n    const watermark = this.#transactionWatermark;\n    if (this.#sourceHasher === undefined || watermark === undefined) {\n      this.#invariantFailure(\n        'Received change-log message without an observed transaction',\n        {tag},\n      );\n      return;\n    }\n\n    const commitWatermark =\n      data[0] === 'commit' ? data[2].watermark : watermark;\n    this.#sourceHasher.add({\n      watermark: commitWatermark,\n      pos: ++this.#sourcePos,\n      tag,\n      change: storedChange,\n      precommit: data[0] === 'commit' ? watermark : null,\n    });\n    if (data[0] === 'commit') {\n      this.#sourceCommitHash = this.#sourceHasher.digest();\n      this.#sourceHasher = undefined;\n    }\n  }\n\n  /** Called after the message has been written to the log. */\n  messageProcessed(\n    data: ChangeStreamData,\n    commit: ChangeLogCommit | null,\n    durationMs: number | undefined,\n  ): void {\n    const tag = data[1].tag;\n    if (durationMs !== undefined) {\n      this.#messageProcessing.recordMs(durationMs, {tag, outcome: 'success'});\n    }\n\n    if (data[0] === 'begin') {\n      if (this.#transactionWatermark !== undefined) {\n        this.#invariantFailure('SQLite change-log transaction already open', {\n          openWatermark: this.#transactionWatermark,\n          receivedWatermark: data[2].commitWatermark,\n        });\n      }\n      this.#transactionWatermark = data[2].commitWatermark;\n      return;\n    }\n\n    if (tag === 'rollback') {\n      this.#recordRollback('upstream');\n      this.#transactionWatermark = undefined;\n      this.#resetSourceHash();\n      return;\n    }\n\n    if (data[0] !== 'commit') {\n      return;\n    }\n\n    const watermark = data[2].watermark;\n    if (commit !== null) {\n      this.#logCommitDuration.recordMs(commit.logCommitMs);\n    }\n    if (this.#transactionWatermark !== watermark) {\n      this.#invariantFailure(\n        'SQLite change-log commit does not match the observed begin',\n        {openWatermark: this.#transactionWatermark, commitWatermark: watermark},\n      );\n    }\n    if (commit === null) {\n      this.#invariantFailure(\n        'SQLite change-log commit result is missing writer statistics',\n        {commitWatermark: watermark},\n      );\n      this.#recordUnpaired(watermark, 'missing-sqlite-hash');\n    } else {\n      if (commit.watermark !== watermark) {\n        this.#invariantFailure(\n          'SQLite change-log commit result has an unexpected watermark',\n          {commitWatermark: watermark, resultWatermark: commit.watermark},\n        );\n      }\n      this.#rows += commit.stats.rows;\n      this.#estimatedBytes += commit.stats.estimatedBytes;\n      this.#transactionRows.record(commit.stats.rows);\n      if (\n        this.#sourceCommitHash !== undefined ||\n        commit.stats.hash !== undefined\n      ) {\n        this.#compareHashes(\n          watermark,\n          this.#sourceCommitHash,\n          commit.stats.hash,\n        );\n      }\n    }\n    if (watermark < this.#sqliteHead) {\n      this.#invariantFailure('SQLite change-log head regressed', {\n        previousHead: this.#sqliteHead,\n        commitWatermark: watermark,\n      });\n    }\n    this.#sqliteHead = watermark;\n    this.#transactionWatermark = undefined;\n    this.#resetSourceHash();\n  }\n\n  messageFailed(\n    data: ChangeStreamData,\n    error: unknown,\n    durationMs: number | undefined,\n  ): void {\n    const tag = data[1].tag;\n    if (durationMs !== undefined) {\n      this.#messageProcessing.recordMs(durationMs, {tag, outcome: 'error'});\n    }\n    if (data[0] === 'commit' && this.#sourceCommitHash !== undefined) {\n      // No commit duration here: a failed commit produces no statistics, so\n      // there is nothing to attribute. The attempt is timed by\n      // `message_processing_duration{tag=\"commit\",outcome=\"error\"}` above.\n      this.#recordUnpaired(data[2].watermark, 'sqlite-commit-failed');\n    }\n    if (\n      error instanceof Error &&\n      error.name === 'ChangeLogStreamInvariantError'\n    ) {\n      this.#invariantFailure(error.message);\n    }\n    if (this.#transactionWatermark !== undefined || data[0] === 'begin') {\n      this.#recordRollback('processing-error');\n    }\n    this.#transactionWatermark = undefined;\n    this.#resetSourceHash();\n  }\n\n  /**\n   * A committed transaction folded schema changes into the cookie jar. Reported\n   * separately from {@link messageProcessed} because it costs a read of the\n   * cookie tables, which is only worth taking when they actually moved.\n   */\n  cookiesMutated(ops: readonly CookieOpTag[], rows: CookieRowCounts): void {\n    for (const op of ops) {\n      this.#cookieMutations++;\n      this.#cookieMutationCounter.add(1, {op});\n    }\n    this.#cookieRows = rows;\n  }\n\n  /**\n   * A missed truncate-above surfaces as a constraint violation on the writer's\n   * plain `INSERT`, which the fail-soft policy would otherwise absorb quietly.\n   * Classifying it as an invariant failure is what keeps a reconciliation bug\n   * from looking like a lost cache.\n   */\n  invariantFailure(message: string, details?: Record<string, unknown>): void {\n    this.#invariantFailure(message, details);\n  }\n\n  /** The source interrupted the stream mid-transaction. */\n  abort(): void {\n    if (this.#transactionWatermark !== undefined) {\n      this.#recordRollback('source-interruption');\n      this.#transactionWatermark = undefined;\n    }\n    this.#resetSourceHash();\n  }\n\n  /** Reconciliation moved the head, so the observer's totals are restated. */\n  reconciled(info: SQLiteChangeLogInfo): void {\n    this.#receivedHead = info.headWatermark;\n    this.#sqliteHead = info.headWatermark;\n    this.#rows = info.rows;\n    this.#estimatedBytes = info.estimatedBytes;\n    this.#cookieRows = info.cookieRows;\n    this.#transactionWatermark = undefined;\n    this.#resetSourceHash();\n  }\n\n  state(): SQLiteChangeLogObservabilityState {\n    return {\n      receivedHead: this.#receivedHead,\n      sqliteHead: this.#sqliteHead,\n      headLag: watermarkDistance(this.#receivedHead, this.#sqliteHead),\n      rows: this.#rows,\n      estimatedBytes: this.#estimatedBytes,\n      cookieRows: this.#cookieRows,\n      cookieMutations: this.#cookieMutations,\n      rollbacks: this.#rollbacks,\n      invariantFailures: this.#invariantFailures,\n      hashMatches: this.#hashMatches,\n      hashMismatches: this.#hashMismatches,\n      hashUnpaired: this.#hashUnpaired,\n    };\n  }\n\n  #recordRollback(\n    reason: 'upstream' | 'processing-error' | 'source-interruption',\n  ) {\n    this.#rollbacks++;\n    this.#rollbackCounter.add(1, {reason});\n  }\n\n  #invariantFailure(message: string, details?: Record<string, unknown>) {\n    this.#invariantFailures++;\n    this.#invariantFailureCounter.add(1);\n    this.#lc.error?.(message, details);\n  }\n\n  #compareHashes(\n    watermark: string,\n    sourceHash: string | undefined,\n    sqliteHash: string | undefined,\n  ): void {\n    if (sourceHash === undefined) {\n      this.#recordUnpaired(watermark, 'missing-source-hash');\n      return;\n    }\n    if (sqliteHash === undefined) {\n      this.#recordUnpaired(watermark, 'missing-sqlite-hash');\n      return;\n    }\n    if (sourceHash === sqliteHash) {\n      this.#hashMatches++;\n      this.#hashComparisonCounter.add(1, {outcome: 'match'});\n      return;\n    }\n    this.#hashMismatches++;\n    this.#hashComparisonCounter.add(1, {outcome: 'mismatch'});\n    this.#lc.error?.('SQLite change-log transaction hash mismatch', {\n      sqliteChangeLogHashComparison: {\n        watermark,\n        sourceHash: sourceHash.slice(0, 16),\n        sqliteHash: sqliteHash.slice(0, 16),\n      },\n    });\n  }\n\n  #recordUnpaired(\n    watermark: string,\n    reason:\n      | 'missing-source-hash'\n      | 'missing-sqlite-hash'\n      | 'sqlite-commit-failed',\n  ): void {\n    this.#hashUnpaired++;\n    this.#hashUnpairedCounter.add(1, {reason});\n    this.#lc.warn?.('SQLite change-log transaction hash was not compared', {\n      sqliteChangeLogHashComparison: {watermark, reason},\n    });\n  }\n\n  #resetSourceHash(): void {\n    this.#sourceHasher = undefined;\n    this.#sourcePos = 0;\n    this.#sourceCommitHash = undefined;\n  }\n}\n\nfunction watermarkValue(watermark: string): number | undefined {\n  try {\n    const value = Number(versionFromLexi(watermark));\n    return Number.isFinite(value) ? value : undefined;\n  } catch {\n    return undefined;\n  }\n}\n\nfunction watermarkDistance(\n  receivedHead: string,\n  sqliteHead: string,\n): number | undefined {\n  if (receivedHead <= sqliteHead) {\n    return 0;\n  }\n  try {\n    const distance =\n      versionFromLexi(receivedHead) - versionFromLexi(sqliteHead);\n    const value = Number(distance);\n    return Number.isFinite(value) ? value : undefined;\n  } catch {\n    return undefined;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoDA,SAAgB,+BACd,IACA,QACM;CACN,QAAQ,OAAO,QAAf;EACE,KAAK;GACH,mBACE,WACA,8CACA,kHAEF,CAAC,CAAC,IAAI,OAAO,IAAI;GACjB;EACF,KAAK;GACH,mBACE,WACA,qCACA,mIAEF,CAAC,CAAC,IAAI,GAAG,EAAC,QAAQ,OAAO,OAAM,CAAC;GAChC;EACF,KAAK,QACH;EACF,SACE,YAAY,MAAM;CACtB;CACA,IAAI,OAAO,cAKT,mBACE,WACA,mCACA,sGAEF,CAAC,CAAC,IAAI,GAAG,EAAC,QAAQ,OAAO,OAAM,CAAC;CAElC,GAAG,QAAQ,oCAAoC,EAC7C,0BAA0B,OAC5B,CAAC;AACH;;;;;;;;AAyBA,SAAgB,8BACd,WAC4B;CAC5B,MAAM,OAAO,kBAAkB,SAAS;CACxC,MAAM,EAAC,cAAc,kBAAiB,oBAAoB,SAAS;CACnE,IAAI,iBAAiB,QAAQ,kBAAkB,MAC7C,MAAM,IAAI,MAAM,yDAAyD;CAE3E,OAAO;EAAC,GAAG;EAAM;EAAc;CAAa;AAC9C;;;;;;;AAQA,SAAgB,uBACd,WACqB;CACrB,MAAM,UAAU,8BAA8B,SAAS;CACvD,MAAM,YAAY,UACf,QAAgB;;;gBAGL,wBAAwB;KACnC,CAAC,CACD,IAA4C;CAE/C,OAAO;EACL,GAAG;EACH,MAAM,UAAU;EAChB,gBAAgB,UAAU;EAC1B,YAAY,oBAAoB,SAAS;CAC3C;AACF;AAEA,SAAgB,0BACd,IACA,MACA,MACM;CACN,GAAG,OAAO,6BAA6B,EACrC,iBAAiB;EACf;EACA,OAAO,KAAK;EACZ,YAAY,KAAK;EACjB,WAAW,KAAK;EAChB,eAAe,KAAK;EACpB,eAAe,KAAK;EACpB,YAAY,KAAK;EACjB,cAAc,KAAK;EACnB,eAAe,KAAK;CACtB,EACF,CAAC;AACH;;;;;;;;;;AAWA,eAAsB,gBACpB,IACA,MACiB;CAajB,QAAO,MAZa,QAAQ,IAC1B;EAAC;EAAM,GAAG,KAAK;EAAO,GAAG,KAAK;CAAM,CAAC,CAAC,IAAI,OAAM,SAAQ;EACtD,IAAI;GACF,QAAQ,MAAM,KAAK,IAAI,EAAA,CAAG;EAC5B,SAAS,GAAG;GACV,IAAK,EAA4B,SAAS,UACxC,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;GAExD,OAAO;EACT;CACF,CAAC,CACH,EAAA,CACa,QAAQ,OAAO,SAAS,QAAQ,MAAM,CAAC;AACtD;AA4BA,IAAM,0BAA0B;CAAC;CAAG;CAAG;CAAG;CAAI;CAAI;CAAI;CAAK;CAAK;CAAM;AAAI;;;;;;;AAQ1E,IAAa,0BAAb,MAAqC;CACnC;CACA,qBAA8B,4BAC5B,WACA,iDACA,uEACF;CAMA,qBAA8B,4BAC5B,WACA,yCACA,+KAGF;CACA,mBAA4B,0BAC1B,WACA,sCACA;EACE,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CACF;CACA,mBAA4B,mBAC1B,WACA,+BACA,2DACF;CACA,2BAAoC,mBAClC,WACA,wCACA,uDACF;CACA,yBAAkC,mBAChC,WACA,sCACA,iHACF;CACA,uBAAgC,mBAC9B,WACA,mCACA,2FACF;CAEA,yBAAkC,mBAChC,WACA,sCACA,uEACF;CAEA;CACA;CACA;CACA;CACA;CACA,mBAAmB;CACnB;CACA;CACA,aAAa;CACb;CACA,aAAa;CACb,qBAAqB;CACrB,eAAe;CACf,kBAAkB;CAClB,gBAAgB;CAEhB,YAAY,IAAgB,MAA2B;EACrD,KAAK,MAAM,GAAG,YAAY,aAAa,4BAA4B;EAGnE,KAAK,gBAAgB,KAAK;EAC1B,KAAK,cAAc,KAAK;EACxB,KAAK,QAAQ,KAAK;EAClB,KAAK,kBAAkB,KAAK;EAC5B,KAAK,cAAc,KAAK;EAExB,iBACE,WACA,0BACA,yCACF,CAAC,CAAC,aAAY,WAAU,OAAO,QAAQ,KAAK,KAAK,CAAC;EAGlD,iBACE,WACA,iCACA,iEACF,CAAC,CAAC,aAAY,WAAU;GACtB,OAAO,QAAQ,KAAK,YAAY,eAAe,EAAC,OAAO,gBAAe,CAAC;GACvE,OAAO,QAAQ,KAAK,YAAY,aAAa,EAAC,OAAO,cAAa,CAAC;EACrE,CAAC;EACD,iBAAiB,WAAW,oCAAoC;GAC9D,aACE;GACF,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,WAAU,OAAO,QAAQ,KAAK,eAAe,CAAC;EAC7D,iBACE,WACA,0BACA,oEACF,CAAC,CAAC,aAAY,WAAU;GACtB,MAAM,OAAO,eAAe,KAAK,WAAW;GAC5C,IAAI,SAAS,KAAA,GACX,OAAO,QAAQ,IAAI;EAEvB,CAAC;EACD,iBACE,WACA,8BACA,0JAGF,CAAC,CAAC,aAAY,WAAU;GACtB,MAAM,MAAM,kBAAkB,KAAK,eAAe,KAAK,WAAW;GAClE,IAAI,QAAQ,KAAA,GACV,OAAO,QAAQ,GAAG;EAEtB,CAAC;CACH;;;;;;CAOA,gBACE,MACA,MACA,eAAe,MACf,eAAe,uBAAuB,MAAM,KAAK,EAAE,CAAC,GAAG,GACjD;EACN,MAAM,MAAM,KAAK,EAAE,CAAC;EACpB,IAAI,KAAK,OAAO,SAAS;GACvB,IAAI,KAAK,kBAAkB,KAAA,GACzB,KAAK,kBACH,iEACA,EAAC,mBAAmB,KAAK,EAAE,CAAC,gBAAe,CAC7C;GAEF,MAAM,YAAY,KAAK,EAAE,CAAC;GAC1B,KAAK,gBAAgB,eACjB,IAAI,2BAA2B,IAC/B,KAAA;GACJ,KAAK,aAAa;GAClB,KAAK,oBAAoB,KAAA;GACzB,KAAK,eAAe,IAAI;IACtB;IACA,KAAK,KAAK;IACV;IACA,QAAQ;IACR,WAAW;GACb,CAAC;GACD;EACF;EAEA,IAAI,QAAQ,YAAY;GACtB,KAAK,iBAAiB;GACtB;EACF;EAEA,IAAI,KAAK,OAAO,UACd,KAAK,gBAAgB,KAAK,EAAE,CAAC;EAG/B,IAAI,CAAC,cACH;EAGF,MAAM,YAAY,KAAK;EACvB,IAAI,KAAK,kBAAkB,KAAA,KAAa,cAAc,KAAA,GAAW;GAC/D,KAAK,kBACH,+DACA,EAAC,IAAG,CACN;GACA;EACF;EAEA,MAAM,kBACJ,KAAK,OAAO,WAAW,KAAK,EAAE,CAAC,YAAY;EAC7C,KAAK,cAAc,IAAI;GACrB,WAAW;GACX,KAAK,EAAE,KAAK;GACZ;GACA,QAAQ;GACR,WAAW,KAAK,OAAO,WAAW,YAAY;EAChD,CAAC;EACD,IAAI,KAAK,OAAO,UAAU;GACxB,KAAK,oBAAoB,KAAK,cAAc,OAAO;GACnD,KAAK,gBAAgB,KAAA;EACvB;CACF;;CAGA,iBACE,MACA,QACA,YACM;EACN,MAAM,MAAM,KAAK,EAAE,CAAC;EACpB,IAAI,eAAe,KAAA,GACjB,KAAK,mBAAmB,SAAS,YAAY;GAAC;GAAK,SAAS;EAAS,CAAC;EAGxE,IAAI,KAAK,OAAO,SAAS;GACvB,IAAI,KAAK,0BAA0B,KAAA,GACjC,KAAK,kBAAkB,8CAA8C;IACnE,eAAe,KAAK;IACpB,mBAAmB,KAAK,EAAE,CAAC;GAC7B,CAAC;GAEH,KAAK,wBAAwB,KAAK,EAAE,CAAC;GACrC;EACF;EAEA,IAAI,QAAQ,YAAY;GACtB,KAAK,gBAAgB,UAAU;GAC/B,KAAK,wBAAwB,KAAA;GAC7B,KAAK,iBAAiB;GACtB;EACF;EAEA,IAAI,KAAK,OAAO,UACd;EAGF,MAAM,YAAY,KAAK,EAAE,CAAC;EAC1B,IAAI,WAAW,MACb,KAAK,mBAAmB,SAAS,OAAO,WAAW;EAErD,IAAI,KAAK,0BAA0B,WACjC,KAAK,kBACH,8DACA;GAAC,eAAe,KAAK;GAAuB,iBAAiB;EAAS,CACxE;EAEF,IAAI,WAAW,MAAM;GACnB,KAAK,kBACH,gEACA,EAAC,iBAAiB,UAAS,CAC7B;GACA,KAAK,gBAAgB,WAAW,qBAAqB;EACvD,OAAO;GACL,IAAI,OAAO,cAAc,WACvB,KAAK,kBACH,+DACA;IAAC,iBAAiB;IAAW,iBAAiB,OAAO;GAAS,CAChE;GAEF,KAAK,SAAS,OAAO,MAAM;GAC3B,KAAK,mBAAmB,OAAO,MAAM;GACrC,KAAK,iBAAiB,OAAO,OAAO,MAAM,IAAI;GAC9C,IACE,KAAK,sBAAsB,KAAA,KAC3B,OAAO,MAAM,SAAS,KAAA,GAEtB,KAAK,eACH,WACA,KAAK,mBACL,OAAO,MAAM,IACf;EAEJ;EACA,IAAI,YAAY,KAAK,aACnB,KAAK,kBAAkB,oCAAoC;GACzD,cAAc,KAAK;GACnB,iBAAiB;EACnB,CAAC;EAEH,KAAK,cAAc;EACnB,KAAK,wBAAwB,KAAA;EAC7B,KAAK,iBAAiB;CACxB;CAEA,cACE,MACA,OACA,YACM;EACN,MAAM,MAAM,KAAK,EAAE,CAAC;EACpB,IAAI,eAAe,KAAA,GACjB,KAAK,mBAAmB,SAAS,YAAY;GAAC;GAAK,SAAS;EAAO,CAAC;EAEtE,IAAI,KAAK,OAAO,YAAY,KAAK,sBAAsB,KAAA,GAIrD,KAAK,gBAAgB,KAAK,EAAE,CAAC,WAAW,sBAAsB;EAEhE,IACE,iBAAiB,SACjB,MAAM,SAAS,iCAEf,KAAK,kBAAkB,MAAM,OAAO;EAEtC,IAAI,KAAK,0BAA0B,KAAA,KAAa,KAAK,OAAO,SAC1D,KAAK,gBAAgB,kBAAkB;EAEzC,KAAK,wBAAwB,KAAA;EAC7B,KAAK,iBAAiB;CACxB;;;;;;CAOA,eAAe,KAA6B,MAA6B;EACvE,KAAK,MAAM,MAAM,KAAK;GACpB,KAAK;GACL,KAAK,uBAAuB,IAAI,GAAG,EAAC,GAAE,CAAC;EACzC;EACA,KAAK,cAAc;CACrB;;;;;;;CAQA,iBAAiB,SAAiB,SAAyC;EACzE,KAAK,kBAAkB,SAAS,OAAO;CACzC;;CAGA,QAAc;EACZ,IAAI,KAAK,0BAA0B,KAAA,GAAW;GAC5C,KAAK,gBAAgB,qBAAqB;GAC1C,KAAK,wBAAwB,KAAA;EAC/B;EACA,KAAK,iBAAiB;CACxB;;CAGA,WAAW,MAAiC;EAC1C,KAAK,gBAAgB,KAAK;EAC1B,KAAK,cAAc,KAAK;EACxB,KAAK,QAAQ,KAAK;EAClB,KAAK,kBAAkB,KAAK;EAC5B,KAAK,cAAc,KAAK;EACxB,KAAK,wBAAwB,KAAA;EAC7B,KAAK,iBAAiB;CACxB;CAEA,QAA2C;EACzC,OAAO;GACL,cAAc,KAAK;GACnB,YAAY,KAAK;GACjB,SAAS,kBAAkB,KAAK,eAAe,KAAK,WAAW;GAC/D,MAAM,KAAK;GACX,gBAAgB,KAAK;GACrB,YAAY,KAAK;GACjB,iBAAiB,KAAK;GACtB,WAAW,KAAK;GAChB,mBAAmB,KAAK;GACxB,aAAa,KAAK;GAClB,gBAAgB,KAAK;GACrB,cAAc,KAAK;EACrB;CACF;CAEA,gBACE,QACA;EACA,KAAK;EACL,KAAK,iBAAiB,IAAI,GAAG,EAAC,OAAM,CAAC;CACvC;CAEA,kBAAkB,SAAiB,SAAmC;EACpE,KAAK;EACL,KAAK,yBAAyB,IAAI,CAAC;EACnC,KAAK,IAAI,QAAQ,SAAS,OAAO;CACnC;CAEA,eACE,WACA,YACA,YACM;EACN,IAAI,eAAe,KAAA,GAAW;GAC5B,KAAK,gBAAgB,WAAW,qBAAqB;GACrD;EACF;EACA,IAAI,eAAe,KAAA,GAAW;GAC5B,KAAK,gBAAgB,WAAW,qBAAqB;GACrD;EACF;EACA,IAAI,eAAe,YAAY;GAC7B,KAAK;GACL,KAAK,uBAAuB,IAAI,GAAG,EAAC,SAAS,QAAO,CAAC;GACrD;EACF;EACA,KAAK;EACL,KAAK,uBAAuB,IAAI,GAAG,EAAC,SAAS,WAAU,CAAC;EACxD,KAAK,IAAI,QAAQ,+CAA+C,EAC9D,+BAA+B;GAC7B;GACA,YAAY,WAAW,MAAM,GAAG,EAAE;GAClC,YAAY,WAAW,MAAM,GAAG,EAAE;EACpC,EACF,CAAC;CACH;CAEA,gBACE,WACA,QAIM;EACN,KAAK;EACL,KAAK,qBAAqB,IAAI,GAAG,EAAC,OAAM,CAAC;EACzC,KAAK,IAAI,OAAO,uDAAuD,EACrE,+BAA+B;GAAC;GAAW;EAAM,EACnD,CAAC;CACH;CAEA,mBAAyB;EACvB,KAAK,gBAAgB,KAAA;EACrB,KAAK,aAAa;EAClB,KAAK,oBAAoB,KAAA;CAC3B;AACF;AAEA,SAAS,eAAe,WAAuC;CAC7D,IAAI;EACF,MAAM,QAAQ,OAAO,gBAAgB,SAAS,CAAC;EAC/C,OAAO,OAAO,SAAS,KAAK,IAAI,QAAQ,KAAA;CAC1C,QAAQ;EACN;CACF;AACF;AAEA,SAAS,kBACP,cACA,YACoB;CACpB,IAAI,gBAAgB,YAClB,OAAO;CAET,IAAI;EACF,MAAM,WACJ,gBAAgB,YAAY,IAAI,gBAAgB,UAAU;EAC5D,MAAM,QAAQ,OAAO,QAAQ;EAC7B,OAAO,OAAO,SAAS,KAAK,IAAI,QAAQ,KAAA;CAC1C,QAAQ;EACN;CACF;AACF"}

package/out/zero-cache/src/services/replicator/sqlite-change-log-purger.js

@@ -278,6 +278,6 @@ var SQLiteChangeLogPurger = class {
 	}
 };
 //#endregion
-export { SQLiteChangeLogPurger };
+export { SQLITE_CHANGE_LOG_COUNT_TRANSACTION_SQL, SQLITE_CHANGE_LOG_ELIGIBLE_COMMITS_SQL, SQLiteChangeLogPurger };
 
 //# sourceMappingURL=sqlite-change-log-purger.js.map

package/out/zero-cache/src/services/replicator/sqlite-change-log-purger.js.map

@@ -1 +1 @@
-{"version":3,"file":"sqlite-change-log-purger.js","names":["#db","#now","#stmts","#budgetRows","#purge","#budget","#tear","#timed","#anyEligible"],"sources":["../../../../../../zero-cache/src/services/replicator/sqlite-change-log-purger.ts"],"sourcesContent":["/**\n * Deletes aged-out transactions from the change-log database, one bounded write\n * transaction at a time.\n *\n * The purger is a pure function of one database: it takes the floors its caller\n * has already gated on and narrows them, never widens them, and it reads no\n * replication state. That is what makes it testable without a scheduler and\n * what lets slice 9 run it from the change-streamer's stream loop, between the\n * writer's commits, on the writer's own connection.\n */\n\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport type {Database, Statement} from '../../../../zqlite/src/db.ts';\nimport {CHANGE_LOG_STREAM_TABLE} from './change-log-db.ts';\n\n/**\n * The eligible set, as three ANDed bounds. Only the `externalFloor` is a safety\n * property; the other two make the log serviceable and make purge conservative.\n *\n * `\"watermark\" < @externalFloor` is deliberately strict. A follower that lost\n * its replica restores it from S3 at watermark `W` and then asks this log for\n * everything after `W`, which requires the `commit` row *at* `W` to still be\n * here (`SQLITE_CHANGE_LOG_BOUNDARY_SQL`). Deleting it returns `too-old` to the\n * very subscriber the floor exists to protect.\n *\n * The head cap reads `max(\"watermark\")` from the log's own database, which\n * keeps the purger a pure function of one file. The cap is inert in normal\n * operation: the floor is derived from a backup of a replica that consumes this\n * loop's output, and the log commits before that output is forwarded, so\n * `externalFloor <= head` holds by construction. It engages only on a *stale*\n * log — writer stopped, mode flipped to `off`, another task took over — where\n * deleting to a floor that has run past a frozen log would empty the file. Note\n * that head can be a phantom — committed here but not yet re-deliverable by a\n * resumed stream — for as long as it takes the watermark the stream would\n * resume from to advance past it. That is harmless precisely because head is\n * never a purge candidate, and it is why the purger asserts nothing about\n * anything outside its own file.\n *\n * Ordered by `(\"writeTimeMs\", \"watermark\")` so it rides the partial index,\n * which carries one entry per transaction rather than one per change. The\n * `LIMIT` keeps a drain from re-scanning the remaining history on every batch,\n * which would be quadratic in the number of eligible transactions and would do\n * it inside the `BEGIN IMMEDIATE`, holding the write lock alongside the delete.\n *\n * Exported so the focused query-plan test covers the production query.\n */\nexport const SQLITE_CHANGE_LOG_ELIGIBLE_COMMITS_SQL = /*sql*/ `\n  SELECT \"watermark\"\n  FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n  WHERE \"writeTimeMs\" IS NOT NULL\n    AND \"writeTimeMs\" < @retentionCutoffMs\n    AND \"watermark\" < @externalFloor\n    AND \"watermark\" < (SELECT max(\"watermark\") FROM \"${CHANGE_LOG_STREAM_TABLE}\")\n  ORDER BY \"writeTimeMs\", \"watermark\"\n  LIMIT @limit\n`;\n\n/**\n * Counts the rows at `@watermark`, capped at `@limit`. The cap is what keeps\n * batch sizing O(budget): this runs per candidate inside the\n * `BEGIN IMMEDIATE`, and the transaction being sized can be arbitrarily\n * large — an uncapped count of a multi-million-row backfill would rescan\n * every remaining row on every tear batch, holding the write lock (and the\n * appending path behind it) for the duration, quadratically over the drain.\n * The caller only needs to know whether the transaction fits its remaining\n * budget, so it passes `remaining + 1` and reads a count above `remaining`\n * as \"does not fit\".\n *\n * Exported so the focused tests cover the production query.\n */\nexport const SQLITE_CHANGE_LOG_COUNT_TRANSACTION_SQL = /*sql*/ `\n  SELECT count(*) AS \"rows\" FROM (\n    SELECT 1\n    FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n    WHERE \"watermark\" = @watermark\n    LIMIT @limit\n  )\n`;\n\n/**\n * A prefix delete, so no transaction between the minimum and the head is ever\n * split. The `<=` is correct and is not the floor comparison: `deletedThrough`\n * is a watermark this batch already established as wholly eligible.\n */\nconst DELETE_THROUGH_SQL = /*sql*/ `\n  DELETE FROM \"${CHANGE_LOG_STREAM_TABLE}\" WHERE \"watermark\" <= ?\n`;\n\nconst MIN_WATERMARK_SQL = /*sql*/ `\n  SELECT min(\"watermark\") AS \"minWatermark\" FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n`;\n\n/**\n * The `pos` at `@offset` rows into the torn transaction, ascending. `undefined`\n * when fewer than `@offset` rows remain, which is how the tear knows it is on\n * its final chunk.\n */\nconst TEAR_BOUNDARY_SQL = /*sql*/ `\n  SELECT \"pos\"\n  FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n  WHERE \"watermark\" = @watermark\n  ORDER BY \"pos\"\n  LIMIT 1 OFFSET @offset\n`;\n\n/**\n * A range scan on the primary key. Expressed as `\"pos\" < @pos` rather than\n * `DELETE ... LIMIT n`, which is both non-portable and the wrong bound: it\n * carries no guarantee about *which* rows it removes, and the commit row must\n * survive until the chunk that finishes the tear.\n */\nconst DELETE_BELOW_POS_SQL = /*sql*/ `\n  DELETE FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n  WHERE \"watermark\" = @watermark AND \"pos\" < @pos\n`;\n\nexport type PurgeBatchResult = {\n  readonly deletedRows: number;\n  /**\n   * The watermark through which whole transactions were removed. `undefined`\n   * when the batch only tore rows off the minimum transaction without\n   * completing it — in that case `deletedRows > 0` and `moreEligible` is true.\n   */\n  readonly deletedThrough: string | undefined;\n  /** True while a tear is in progress, as well as while whole transactions remain eligible. */\n  readonly moreEligible: boolean;\n};\n\nexport type PurgeBatchOptions = {\n  /**\n   * The floor the caller has already gated on subscriber ACKs and snapshot\n   * reservations — normally the confirmed S3 backup watermark. Transactions at\n   * or above it are NOT durable anywhere else: the change log is excluded from\n   * the litestream backup, so this is the boundary a follower restores to and\n   * then catches up from. Compared strictly (`<`), so the transaction *at* the\n   * floor survives as that follower's catchup boundary. The purger narrows this\n   * input and never widens it.\n   */\n  readonly externalFloor: string;\n  /**\n   * A *minimum* retention, ANDed on top of `externalFloor` rather than an\n   * alternative to it. Which of the two binds depends on the backup interval.\n   */\n  readonly retentionCutoffMs: number;\n  /** Starting chunk size. Not the bound — `maxDurationMs` is. */\n  readonly maxRows: number;\n  /**\n   * The statement duration target, enforced by feedback rather than by\n   * interruption: SQLite cannot yield mid-statement, so the purger measures\n   * each delete and shrinks the next batch's row budget when one overruns\n   * (see {@link SQLiteChangeLogPurger.budgetRows}) — which means the first\n   * delete after a restart, sized by `maxRows` alone, can still overrun it.\n   * The target is this consequential because purge shares the writer's\n   * connection: there is no lock to arbitrate and no `busy_timeout` to bound\n   * the wait — a long statement blocks the appending path, and the forward\n   * behind it, directly and for its full duration. That is why this is a\n   * precondition of purging from the stream loop rather than a tuning knob.\n   */\n  readonly maxDurationMs: number;\n};\n\ntype Statements = {\n  readonly eligibleCommits: Statement;\n  readonly countTransaction: Statement;\n  readonly deleteThrough: Statement;\n  readonly minWatermark: Statement;\n  readonly tearBoundary: Statement;\n  readonly deleteBelowPos: Statement;\n};\n\nexport class SQLiteChangeLogPurger {\n  readonly #db: Database;\n  readonly #now: () => number;\n  readonly #stmts: Statements;\n\n  /**\n   * The measured row budget, carried across batches. Row cost varies with\n   * payload size, so `maxRows` alone cannot bound a statement's duration; this\n   * shrinks when a delete overruns `maxDurationMs` and grows back when the\n   * budget — rather than the eligible set — was what limited the batch.\n   */\n  #budgetRows: number | undefined;\n\n  /**\n   * @param db a writable connection to a change log the writer has already\n   *     created and reconciled — in slice 9, the writer's own connection.\n   *     Statements are prepared eagerly, so a caller that may run before the\n   *     log exists opens the connection lazily and treats construction failure\n   *     as a skipped cycle.\n   */\n  constructor(\n    db: Database,\n    now: () => number = performance.now.bind(performance),\n  ) {\n    this.#db = db;\n    this.#now = now;\n    this.#stmts = {\n      eligibleCommits: db.prepare(SQLITE_CHANGE_LOG_ELIGIBLE_COMMITS_SQL),\n      countTransaction: db.prepare(SQLITE_CHANGE_LOG_COUNT_TRANSACTION_SQL),\n      deleteThrough: db.prepare(DELETE_THROUGH_SQL),\n      minWatermark: db.prepare(MIN_WATERMARK_SQL),\n      tearBoundary: db.prepare(TEAR_BOUNDARY_SQL),\n      deleteBelowPos: db.prepare(DELETE_BELOW_POS_SQL),\n    };\n  }\n\n  /** The row budget the next batch would use. Exposed for tests and metrics. */\n  get budgetRows(): number | undefined {\n    return this.#budgetRows;\n  }\n\n  /**\n   * Removes one bounded batch of aged-out history, in one `BEGIN IMMEDIATE`\n   * transaction. Callers loop while `moreEligible`, releasing whatever guards\n   * they hold between batches.\n   */\n  purgeBatch(opts: PurgeBatchOptions): PurgeBatchResult {\n    const {maxRows, maxDurationMs} = opts;\n    assert(\n      Number.isSafeInteger(maxRows) && maxRows > 0,\n      'change-log purge maxRows must be a positive safe integer',\n    );\n    assert(\n      maxDurationMs > 0,\n      'change-log purge maxDurationMs must be positive',\n    );\n\n    this.#db.exec('BEGIN IMMEDIATE');\n    try {\n      const result = this.#purge(opts);\n      this.#db.exec('COMMIT');\n      return result;\n    } catch (e) {\n      if (this.#db.inTransaction) {\n        this.#db.exec('ROLLBACK');\n      }\n      throw e;\n    }\n  }\n\n  #purge(opts: PurgeBatchOptions): PurgeBatchResult {\n    const {externalFloor, retentionCutoffMs} = opts;\n    const budget = this.#budget(opts.maxRows);\n    // Each transaction is at least a `begin` and a `commit`, so this is the\n    // most candidates that can fit in the budget.\n    const limit = Math.max(1, Math.ceil(budget / 2));\n    const candidates = this.#stmts.eligibleCommits.all<{watermark: string}>({\n      retentionCutoffMs,\n      externalFloor,\n      limit,\n    });\n    if (candidates.length === 0) {\n      return {deletedRows: 0, deletedThrough: undefined, moreEligible: false};\n    }\n\n    // Accounted per candidate rather than per prefix, which is exact as long as\n    // the candidates arrive in watermark order — they do, because the single\n    // writer stamps `writeTimeMs` at commit time. A clock that went backwards\n    // can put a non-candidate below `deletedThrough`, and the prefix delete\n    // then removes more rows than were counted. Not a safety problem: both\n    // watermark bounds are prefix-closed, only the minimum-retention courtesy\n    // is crossed, and `maxRows` is the starting chunk size rather than the\n    // bound — `maxDurationMs` is, and the governor measures what actually ran.\n    let rows = 0;\n    let deletedThrough: string | undefined;\n    for (const {watermark} of candidates) {\n      // The count is capped at one past the remaining budget: `n > remaining`\n      // is exactly `rows + n > budget` for a fitting transaction, and a\n      // too-large one stops scanning as soon as it is known not to fit.\n      const remaining = budget - rows;\n      const {rows: n} = this.#stmts.countTransaction.get<{rows: number}>({\n        watermark,\n        limit: remaining + 1,\n      });\n      if (n > remaining) {\n        break;\n      }\n      rows += n;\n      deletedThrough = watermark;\n    }\n\n    if (deletedThrough === undefined) {\n      return this.#tear(opts, budget);\n    }\n    const deletedRows = this.#timed(\n      opts,\n      budget,\n      () => this.#stmts.deleteThrough.run(deletedThrough).changes,\n    );\n    return {\n      deletedRows,\n      deletedThrough,\n      moreEligible: this.#anyEligible(opts),\n    };\n  }\n\n  /**\n   * Removes one bounded chunk of the minimum transaction, which is the only\n   * transaction that is ever torn.\n   *\n   * A single upstream transaction can be arbitrarily large — a backfill, a\n   * schema migration — and SQLite cannot yield mid-statement, so the only way\n   * to bound the lock hold is to delete it across several statements. That is\n   * unobservable given three constraints, all of which live here:\n   *\n   * 1. Only `min(\"watermark\")` is torn, so at most one transaction is ever\n   *    partial. A reader never reads *into* a transaction — `read()` starts\n   *    strictly after `(fromWatermark, MAX_SAFE_INTEGER)` — so the only\n   *    transaction a tear can touch is one a subscriber reads *from*, and\n   *    reading from it does not read its rows.\n   * 2. Rows go ascending by `pos`, commit row last. `plan()`'s boundary check\n   *    reads the *highest* `pos` at the watermark and requires it to be the\n   *    `commit`; removing that row first would give a subscriber sitting\n   *    exactly there a spurious `too-old`.\n   * 3. The commit row is removed by a prefix delete, which advances\n   *    `min(\"watermark\")` in the same SQLite transaction, so no state exposes\n   *    the boundary as gone while the minimum still points at it. Normally that\n   *    is {@link SQLiteChangeLogPurger.purgeBatch}'s ordinary whole-transaction\n   *    delete, taken on the batch where what is left of the torn transaction\n   *    finally fits the budget; the branch below is the same delete for the\n   *    case where the minimum is not itself in the eligible set.\n   *\n   * A crash mid-tear needs no recovery state: the torn transaction is still the\n   * minimum and still eligible, so the next batch continues it. Reconciliation\n   * is unaffected — it tests the head, and a tear is at the tail.\n   *\n   * Tearing the minimum is safe even in the unusual case where it is not itself\n   * in the eligible set (a writer whose clock went backwards can put a later\n   * watermark at an earlier `writeTimeMs`). Both watermark bounds are\n   * prefix-closed — a non-empty eligible set implies `min < externalFloor` and\n   * `min < head` — and the prefix delete this batch would otherwise run removes\n   * the minimum anyway. Only the time floor, which is a minimum-retention\n   * courtesy rather than a safety property, can be crossed that way.\n   */\n  #tear(opts: PurgeBatchOptions, budget: number): PurgeBatchResult {\n    const {minWatermark} = this.#stmts.minWatermark.get<{\n      minWatermark: string | null;\n    }>();\n    assert(\n      minWatermark !== null,\n      'change-log purge found eligible transactions in an empty log',\n    );\n\n    const boundary = this.#stmts.tearBoundary.get<{pos: number} | undefined>({\n      watermark: minWatermark,\n      offset: budget,\n    });\n    if (boundary === undefined) {\n      // At most `budget` rows are left at the minimum, which is reachable only\n      // when the minimum is not the transaction that failed to fit — i.e. when\n      // it is not itself eligible. The prefix delete removes it whole and\n      // advances `min(\"watermark\")`.\n      const deletedRows = this.#timed(\n        opts,\n        budget,\n        () => this.#stmts.deleteThrough.run(minWatermark).changes,\n      );\n      return {\n        deletedRows,\n        deletedThrough: minWatermark,\n        moreEligible: this.#anyEligible(opts),\n      };\n    }\n    // `boundary.pos` is the `budget`-th row by `pos`, so this removes exactly\n    // `budget` rows and always makes progress. The commit row is the highest\n    // `pos`, so it can only be reached by the branch above.\n    const deletedRows = this.#timed(\n      opts,\n      budget,\n      () =>\n        this.#stmts.deleteBelowPos.run({\n          watermark: minWatermark,\n          pos: boundary.pos,\n        }).changes,\n    );\n    return {deletedRows, deletedThrough: undefined, moreEligible: true};\n  }\n\n  #anyEligible({externalFloor, retentionCutoffMs}: PurgeBatchOptions): boolean {\n    return (\n      this.#stmts.eligibleCommits.get<{watermark: string} | undefined>({\n        retentionCutoffMs,\n        externalFloor,\n        limit: 1,\n      }) !== undefined\n    );\n  }\n\n  #budget(maxRows: number): number {\n    return Math.max(1, Math.min(maxRows, this.#budgetRows ?? maxRows));\n  }\n\n  /**\n   * Runs a delete and feeds its measured duration back into the row budget.\n   * Growth is gated on the budget having been the binding constraint: a batch\n   * that finished quickly because only four rows were eligible is no evidence\n   * that a larger statement would stay under the target.\n   */\n  #timed(\n    {maxRows, maxDurationMs}: PurgeBatchOptions,\n    budget: number,\n    run: () => number,\n  ): number {\n    const start = this.#now();\n    const deletedRows = run();\n    const elapsedMs = this.#now() - start;\n\n    if (elapsedMs > maxDurationMs) {\n      this.#budgetRows = Math.max(1, Math.floor(budget / 2));\n    } else if (deletedRows >= budget && elapsedMs * 2 <= maxDurationMs) {\n      this.#budgetRows = Math.min(maxRows, budget * 2);\n    } else {\n      this.#budgetRows = budget;\n    }\n    return deletedRows;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8CA,IAAa,yCAAiD;;UAEpD,wBAAwB;;;;uDAIqB,wBAAwB;;;;;;;;;;;;;;;;;AAkB/E,IAAa,0CAAkD;;;YAGnD,wBAAwB;;;;;;;;;;AAWpC,IAAM,qBAA6B;iBAClB,wBAAwB;;AAGzC,IAAM,oBAA4B;oDACkB,wBAAwB;;;;;;;AAQ5E,IAAM,oBAA4B;;UAExB,wBAAwB;;;;;;;;;;;AAYlC,IAAM,uBAA+B;iBACpB,wBAAwB;;;AA0DzC,IAAa,wBAAb,MAAmC;CACjC;CACA;CACA;;;;;;;CAQA;;;;;;;;CASA,YACE,IACA,MAAoB,YAAY,IAAI,KAAK,WAAW,GACpD;EACA,KAAKA,MAAM;EACX,KAAKC,OAAO;EACZ,KAAKC,SAAS;GACZ,iBAAiB,GAAG,QAAQ,sCAAsC;GAClE,kBAAkB,GAAG,QAAQ,uCAAuC;GACpE,eAAe,GAAG,QAAQ,kBAAkB;GAC5C,cAAc,GAAG,QAAQ,iBAAiB;GAC1C,cAAc,GAAG,QAAQ,iBAAiB;GAC1C,gBAAgB,GAAG,QAAQ,oBAAoB;EACjD;CACF;;CAGA,IAAI,aAAiC;EACnC,OAAO,KAAKC;CACd;;;;;;CAOA,WAAW,MAA2C;EACpD,MAAM,EAAC,SAAS,kBAAiB;EACjC,OACE,OAAO,cAAc,OAAO,KAAK,UAAU,GAC3C,0DACF;EACA,OACE,gBAAgB,GAChB,iDACF;EAEA,KAAKH,IAAI,KAAK,iBAAiB;EAC/B,IAAI;GACF,MAAM,SAAS,KAAKI,OAAO,IAAI;GAC/B,KAAKJ,IAAI,KAAK,QAAQ;GACtB,OAAO;EACT,SAAS,GAAG;GACV,IAAI,KAAKA,IAAI,eACX,KAAKA,IAAI,KAAK,UAAU;GAE1B,MAAM;EACR;CACF;CAEA,OAAO,MAA2C;EAChD,MAAM,EAAC,eAAe,sBAAqB;EAC3C,MAAM,SAAS,KAAKK,QAAQ,KAAK,OAAO;EAGxC,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,KAAK,SAAS,CAAC,CAAC;EAC/C,MAAM,aAAa,KAAKH,OAAO,gBAAgB,IAAyB;GACtE;GACA;GACA;EACF,CAAC;EACD,IAAI,WAAW,WAAW,GACxB,OAAO;GAAC,aAAa;GAAG,gBAAgB,KAAA;GAAW,cAAc;EAAK;EAWxE,IAAI,OAAO;EACX,IAAI;EACJ,KAAK,MAAM,EAAC,eAAc,YAAY;GAIpC,MAAM,YAAY,SAAS;GAC3B,MAAM,EAAC,MAAM,MAAK,KAAKA,OAAO,iBAAiB,IAAoB;IACjE;IACA,OAAO,YAAY;GACrB,CAAC;GACD,IAAI,IAAI,WACN;GAEF,QAAQ;GACR,iBAAiB;EACnB;EAEA,IAAI,mBAAmB,KAAA,GACrB,OAAO,KAAKI,MAAM,MAAM,MAAM;EAOhC,OAAO;GACL,aANkB,KAAKC,OACvB,MACA,cACM,KAAKL,OAAO,cAAc,IAAI,cAAc,EAAE,OAGpD;GACA;GACA,cAAc,KAAKM,aAAa,IAAI;EACtC;CACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwCA,MAAM,MAAyB,QAAkC;EAC/D,MAAM,EAAC,iBAAgB,KAAKN,OAAO,aAAa,IAE7C;EACH,OACE,iBAAiB,MACjB,8DACF;EAEA,MAAM,WAAW,KAAKA,OAAO,aAAa,IAA+B;GACvE,WAAW;GACX,QAAQ;EACV,CAAC;EACD,IAAI,aAAa,KAAA,GAUf,OAAO;GACL,aANkB,KAAKK,OACvB,MACA,cACM,KAAKL,OAAO,cAAc,IAAI,YAAY,EAAE,OAGlD;GACA,gBAAgB;GAChB,cAAc,KAAKM,aAAa,IAAI;EACtC;EAcF,OAAO;GAAC,aATY,KAAKD,OACvB,MACA,cAEE,KAAKL,OAAO,eAAe,IAAI;IAC7B,WAAW;IACX,KAAK,SAAS;GAChB,CAAC,EAAE,OAEC;GAAa,gBAAgB,KAAA;GAAW,cAAc;EAAI;CACpE;CAEA,aAAa,EAAC,eAAe,qBAAgD;EAC3E,OACE,KAAKA,OAAO,gBAAgB,IAAqC;GAC/D;GACA;GACA,OAAO;EACT,CAAC,MAAM,KAAA;CAEX;CAEA,QAAQ,SAAyB;EAC/B,OAAO,KAAK,IAAI,GAAG,KAAK,IAAI,SAAS,KAAKC,eAAe,OAAO,CAAC;CACnE;;;;;;;CAQA,OACE,EAAC,SAAS,iBACV,QACA,KACQ;EACR,MAAM,QAAQ,KAAKF,KAAK;EACxB,MAAM,cAAc,IAAI;EACxB,MAAM,YAAY,KAAKA,KAAK,IAAI;EAEhC,IAAI,YAAY,eACd,KAAKE,cAAc,KAAK,IAAI,GAAG,KAAK,MAAM,SAAS,CAAC,CAAC;OAChD,IAAI,eAAe,UAAU,YAAY,KAAK,eACnD,KAAKA,cAAc,KAAK,IAAI,SAAS,SAAS,CAAC;OAE/C,KAAKA,cAAc;EAErB,OAAO;CACT;AACF"}
+{"version":3,"file":"sqlite-change-log-purger.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/sqlite-change-log-purger.ts"],"sourcesContent":["/**\n * Deletes aged-out transactions from the change-log database, one bounded write\n * transaction at a time.\n *\n * The purger is a pure function of one database: it takes the floors its caller\n * has already gated on and narrows them, never widens them, and it reads no\n * replication state. That is what makes it testable without a scheduler and\n * what lets slice 9 run it from the change-streamer's stream loop, between the\n * writer's commits, on the writer's own connection.\n */\n\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport type {Database, Statement} from '../../../../zqlite/src/db.ts';\nimport {CHANGE_LOG_STREAM_TABLE} from './change-log-db.ts';\n\n/**\n * The eligible set, as three ANDed bounds. Only the `externalFloor` is a safety\n * property; the other two make the log serviceable and make purge conservative.\n *\n * `\"watermark\" < @externalFloor` is deliberately strict. A follower that lost\n * its replica restores it from S3 at watermark `W` and then asks this log for\n * everything after `W`, which requires the `commit` row *at* `W` to still be\n * here (`SQLITE_CHANGE_LOG_BOUNDARY_SQL`). Deleting it returns `too-old` to the\n * very subscriber the floor exists to protect.\n *\n * The head cap reads `max(\"watermark\")` from the log's own database, which\n * keeps the purger a pure function of one file. The cap is inert in normal\n * operation: the floor is derived from a backup of a replica that consumes this\n * loop's output, and the log commits before that output is forwarded, so\n * `externalFloor <= head` holds by construction. It engages only on a *stale*\n * log — writer stopped, mode flipped to `off`, another task took over — where\n * deleting to a floor that has run past a frozen log would empty the file. Note\n * that head can be a phantom — committed here but not yet re-deliverable by a\n * resumed stream — for as long as it takes the watermark the stream would\n * resume from to advance past it. That is harmless precisely because head is\n * never a purge candidate, and it is why the purger asserts nothing about\n * anything outside its own file.\n *\n * Ordered by `(\"writeTimeMs\", \"watermark\")` so it rides the partial index,\n * which carries one entry per transaction rather than one per change. The\n * `LIMIT` keeps a drain from re-scanning the remaining history on every batch,\n * which would be quadratic in the number of eligible transactions and would do\n * it inside the `BEGIN IMMEDIATE`, holding the write lock alongside the delete.\n *\n * Exported so the focused query-plan test covers the production query.\n */\nexport const SQLITE_CHANGE_LOG_ELIGIBLE_COMMITS_SQL = /*sql*/ `\n  SELECT \"watermark\"\n  FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n  WHERE \"writeTimeMs\" IS NOT NULL\n    AND \"writeTimeMs\" < @retentionCutoffMs\n    AND \"watermark\" < @externalFloor\n    AND \"watermark\" < (SELECT max(\"watermark\") FROM \"${CHANGE_LOG_STREAM_TABLE}\")\n  ORDER BY \"writeTimeMs\", \"watermark\"\n  LIMIT @limit\n`;\n\n/**\n * Counts the rows at `@watermark`, capped at `@limit`. The cap is what keeps\n * batch sizing O(budget): this runs per candidate inside the\n * `BEGIN IMMEDIATE`, and the transaction being sized can be arbitrarily\n * large — an uncapped count of a multi-million-row backfill would rescan\n * every remaining row on every tear batch, holding the write lock (and the\n * appending path behind it) for the duration, quadratically over the drain.\n * The caller only needs to know whether the transaction fits its remaining\n * budget, so it passes `remaining + 1` and reads a count above `remaining`\n * as \"does not fit\".\n *\n * Exported so the focused tests cover the production query.\n */\nexport const SQLITE_CHANGE_LOG_COUNT_TRANSACTION_SQL = /*sql*/ `\n  SELECT count(*) AS \"rows\" FROM (\n    SELECT 1\n    FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n    WHERE \"watermark\" = @watermark\n    LIMIT @limit\n  )\n`;\n\n/**\n * A prefix delete, so no transaction between the minimum and the head is ever\n * split. The `<=` is correct and is not the floor comparison: `deletedThrough`\n * is a watermark this batch already established as wholly eligible.\n */\nconst DELETE_THROUGH_SQL = /*sql*/ `\n  DELETE FROM \"${CHANGE_LOG_STREAM_TABLE}\" WHERE \"watermark\" <= ?\n`;\n\nconst MIN_WATERMARK_SQL = /*sql*/ `\n  SELECT min(\"watermark\") AS \"minWatermark\" FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n`;\n\n/**\n * The `pos` at `@offset` rows into the torn transaction, ascending. `undefined`\n * when fewer than `@offset` rows remain, which is how the tear knows it is on\n * its final chunk.\n */\nconst TEAR_BOUNDARY_SQL = /*sql*/ `\n  SELECT \"pos\"\n  FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n  WHERE \"watermark\" = @watermark\n  ORDER BY \"pos\"\n  LIMIT 1 OFFSET @offset\n`;\n\n/**\n * A range scan on the primary key. Expressed as `\"pos\" < @pos` rather than\n * `DELETE ... LIMIT n`, which is both non-portable and the wrong bound: it\n * carries no guarantee about *which* rows it removes, and the commit row must\n * survive until the chunk that finishes the tear.\n */\nconst DELETE_BELOW_POS_SQL = /*sql*/ `\n  DELETE FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n  WHERE \"watermark\" = @watermark AND \"pos\" < @pos\n`;\n\nexport type PurgeBatchResult = {\n  readonly deletedRows: number;\n  /**\n   * The watermark through which whole transactions were removed. `undefined`\n   * when the batch only tore rows off the minimum transaction without\n   * completing it — in that case `deletedRows > 0` and `moreEligible` is true.\n   */\n  readonly deletedThrough: string | undefined;\n  /** True while a tear is in progress, as well as while whole transactions remain eligible. */\n  readonly moreEligible: boolean;\n};\n\nexport type PurgeBatchOptions = {\n  /**\n   * The floor the caller has already gated on subscriber ACKs and snapshot\n   * reservations — normally the confirmed S3 backup watermark. Transactions at\n   * or above it are NOT durable anywhere else: the change log is excluded from\n   * the litestream backup, so this is the boundary a follower restores to and\n   * then catches up from. Compared strictly (`<`), so the transaction *at* the\n   * floor survives as that follower's catchup boundary. The purger narrows this\n   * input and never widens it.\n   */\n  readonly externalFloor: string;\n  /**\n   * A *minimum* retention, ANDed on top of `externalFloor` rather than an\n   * alternative to it. Which of the two binds depends on the backup interval.\n   */\n  readonly retentionCutoffMs: number;\n  /** Starting chunk size. Not the bound — `maxDurationMs` is. */\n  readonly maxRows: number;\n  /**\n   * The statement duration target, enforced by feedback rather than by\n   * interruption: SQLite cannot yield mid-statement, so the purger measures\n   * each delete and shrinks the next batch's row budget when one overruns\n   * (see {@link SQLiteChangeLogPurger.budgetRows}) — which means the first\n   * delete after a restart, sized by `maxRows` alone, can still overrun it.\n   * The target is this consequential because purge shares the writer's\n   * connection: there is no lock to arbitrate and no `busy_timeout` to bound\n   * the wait — a long statement blocks the appending path, and the forward\n   * behind it, directly and for its full duration. That is why this is a\n   * precondition of purging from the stream loop rather than a tuning knob.\n   */\n  readonly maxDurationMs: number;\n};\n\ntype Statements = {\n  readonly eligibleCommits: Statement;\n  readonly countTransaction: Statement;\n  readonly deleteThrough: Statement;\n  readonly minWatermark: Statement;\n  readonly tearBoundary: Statement;\n  readonly deleteBelowPos: Statement;\n};\n\nexport class SQLiteChangeLogPurger {\n  readonly #db: Database;\n  readonly #now: () => number;\n  readonly #stmts: Statements;\n\n  /**\n   * The measured row budget, carried across batches. Row cost varies with\n   * payload size, so `maxRows` alone cannot bound a statement's duration; this\n   * shrinks when a delete overruns `maxDurationMs` and grows back when the\n   * budget — rather than the eligible set — was what limited the batch.\n   */\n  #budgetRows: number | undefined;\n\n  /**\n   * @param db a writable connection to a change log the writer has already\n   *     created and reconciled — in slice 9, the writer's own connection.\n   *     Statements are prepared eagerly, so a caller that may run before the\n   *     log exists opens the connection lazily and treats construction failure\n   *     as a skipped cycle.\n   */\n  constructor(\n    db: Database,\n    now: () => number = performance.now.bind(performance),\n  ) {\n    this.#db = db;\n    this.#now = now;\n    this.#stmts = {\n      eligibleCommits: db.prepare(SQLITE_CHANGE_LOG_ELIGIBLE_COMMITS_SQL),\n      countTransaction: db.prepare(SQLITE_CHANGE_LOG_COUNT_TRANSACTION_SQL),\n      deleteThrough: db.prepare(DELETE_THROUGH_SQL),\n      minWatermark: db.prepare(MIN_WATERMARK_SQL),\n      tearBoundary: db.prepare(TEAR_BOUNDARY_SQL),\n      deleteBelowPos: db.prepare(DELETE_BELOW_POS_SQL),\n    };\n  }\n\n  /** The row budget the next batch would use. Exposed for tests and metrics. */\n  get budgetRows(): number | undefined {\n    return this.#budgetRows;\n  }\n\n  /**\n   * Removes one bounded batch of aged-out history, in one `BEGIN IMMEDIATE`\n   * transaction. Callers loop while `moreEligible`, releasing whatever guards\n   * they hold between batches.\n   */\n  purgeBatch(opts: PurgeBatchOptions): PurgeBatchResult {\n    const {maxRows, maxDurationMs} = opts;\n    assert(\n      Number.isSafeInteger(maxRows) && maxRows > 0,\n      'change-log purge maxRows must be a positive safe integer',\n    );\n    assert(\n      maxDurationMs > 0,\n      'change-log purge maxDurationMs must be positive',\n    );\n\n    this.#db.exec('BEGIN IMMEDIATE');\n    try {\n      const result = this.#purge(opts);\n      this.#db.exec('COMMIT');\n      return result;\n    } catch (e) {\n      if (this.#db.inTransaction) {\n        this.#db.exec('ROLLBACK');\n      }\n      throw e;\n    }\n  }\n\n  #purge(opts: PurgeBatchOptions): PurgeBatchResult {\n    const {externalFloor, retentionCutoffMs} = opts;\n    const budget = this.#budget(opts.maxRows);\n    // Each transaction is at least a `begin` and a `commit`, so this is the\n    // most candidates that can fit in the budget.\n    const limit = Math.max(1, Math.ceil(budget / 2));\n    const candidates = this.#stmts.eligibleCommits.all<{watermark: string}>({\n      retentionCutoffMs,\n      externalFloor,\n      limit,\n    });\n    if (candidates.length === 0) {\n      return {deletedRows: 0, deletedThrough: undefined, moreEligible: false};\n    }\n\n    // Accounted per candidate rather than per prefix, which is exact as long as\n    // the candidates arrive in watermark order — they do, because the single\n    // writer stamps `writeTimeMs` at commit time. A clock that went backwards\n    // can put a non-candidate below `deletedThrough`, and the prefix delete\n    // then removes more rows than were counted. Not a safety problem: both\n    // watermark bounds are prefix-closed, only the minimum-retention courtesy\n    // is crossed, and `maxRows` is the starting chunk size rather than the\n    // bound — `maxDurationMs` is, and the governor measures what actually ran.\n    let rows = 0;\n    let deletedThrough: string | undefined;\n    for (const {watermark} of candidates) {\n      // The count is capped at one past the remaining budget: `n > remaining`\n      // is exactly `rows + n > budget` for a fitting transaction, and a\n      // too-large one stops scanning as soon as it is known not to fit.\n      const remaining = budget - rows;\n      const {rows: n} = this.#stmts.countTransaction.get<{rows: number}>({\n        watermark,\n        limit: remaining + 1,\n      });\n      if (n > remaining) {\n        break;\n      }\n      rows += n;\n      deletedThrough = watermark;\n    }\n\n    if (deletedThrough === undefined) {\n      return this.#tear(opts, budget);\n    }\n    const deletedRows = this.#timed(\n      opts,\n      budget,\n      () => this.#stmts.deleteThrough.run(deletedThrough).changes,\n    );\n    return {\n      deletedRows,\n      deletedThrough,\n      moreEligible: this.#anyEligible(opts),\n    };\n  }\n\n  /**\n   * Removes one bounded chunk of the minimum transaction, which is the only\n   * transaction that is ever torn.\n   *\n   * A single upstream transaction can be arbitrarily large — a backfill, a\n   * schema migration — and SQLite cannot yield mid-statement, so the only way\n   * to bound the lock hold is to delete it across several statements. That is\n   * unobservable given three constraints, all of which live here:\n   *\n   * 1. Only `min(\"watermark\")` is torn, so at most one transaction is ever\n   *    partial. A reader never reads *into* a transaction — `read()` starts\n   *    strictly after `(fromWatermark, MAX_SAFE_INTEGER)` — so the only\n   *    transaction a tear can touch is one a subscriber reads *from*, and\n   *    reading from it does not read its rows.\n   * 2. Rows go ascending by `pos`, commit row last. `plan()`'s boundary check\n   *    reads the *highest* `pos` at the watermark and requires it to be the\n   *    `commit`; removing that row first would give a subscriber sitting\n   *    exactly there a spurious `too-old`.\n   * 3. The commit row is removed by a prefix delete, which advances\n   *    `min(\"watermark\")` in the same SQLite transaction, so no state exposes\n   *    the boundary as gone while the minimum still points at it. Normally that\n   *    is {@link SQLiteChangeLogPurger.purgeBatch}'s ordinary whole-transaction\n   *    delete, taken on the batch where what is left of the torn transaction\n   *    finally fits the budget; the branch below is the same delete for the\n   *    case where the minimum is not itself in the eligible set.\n   *\n   * A crash mid-tear needs no recovery state: the torn transaction is still the\n   * minimum and still eligible, so the next batch continues it. Reconciliation\n   * is unaffected — it tests the head, and a tear is at the tail.\n   *\n   * Tearing the minimum is safe even in the unusual case where it is not itself\n   * in the eligible set (a writer whose clock went backwards can put a later\n   * watermark at an earlier `writeTimeMs`). Both watermark bounds are\n   * prefix-closed — a non-empty eligible set implies `min < externalFloor` and\n   * `min < head` — and the prefix delete this batch would otherwise run removes\n   * the minimum anyway. Only the time floor, which is a minimum-retention\n   * courtesy rather than a safety property, can be crossed that way.\n   */\n  #tear(opts: PurgeBatchOptions, budget: number): PurgeBatchResult {\n    const {minWatermark} = this.#stmts.minWatermark.get<{\n      minWatermark: string | null;\n    }>();\n    assert(\n      minWatermark !== null,\n      'change-log purge found eligible transactions in an empty log',\n    );\n\n    const boundary = this.#stmts.tearBoundary.get<{pos: number} | undefined>({\n      watermark: minWatermark,\n      offset: budget,\n    });\n    if (boundary === undefined) {\n      // At most `budget` rows are left at the minimum, which is reachable only\n      // when the minimum is not the transaction that failed to fit — i.e. when\n      // it is not itself eligible. The prefix delete removes it whole and\n      // advances `min(\"watermark\")`.\n      const deletedRows = this.#timed(\n        opts,\n        budget,\n        () => this.#stmts.deleteThrough.run(minWatermark).changes,\n      );\n      return {\n        deletedRows,\n        deletedThrough: minWatermark,\n        moreEligible: this.#anyEligible(opts),\n      };\n    }\n    // `boundary.pos` is the `budget`-th row by `pos`, so this removes exactly\n    // `budget` rows and always makes progress. The commit row is the highest\n    // `pos`, so it can only be reached by the branch above.\n    const deletedRows = this.#timed(\n      opts,\n      budget,\n      () =>\n        this.#stmts.deleteBelowPos.run({\n          watermark: minWatermark,\n          pos: boundary.pos,\n        }).changes,\n    );\n    return {deletedRows, deletedThrough: undefined, moreEligible: true};\n  }\n\n  #anyEligible({externalFloor, retentionCutoffMs}: PurgeBatchOptions): boolean {\n    return (\n      this.#stmts.eligibleCommits.get<{watermark: string} | undefined>({\n        retentionCutoffMs,\n        externalFloor,\n        limit: 1,\n      }) !== undefined\n    );\n  }\n\n  #budget(maxRows: number): number {\n    return Math.max(1, Math.min(maxRows, this.#budgetRows ?? maxRows));\n  }\n\n  /**\n   * Runs a delete and feeds its measured duration back into the row budget.\n   * Growth is gated on the budget having been the binding constraint: a batch\n   * that finished quickly because only four rows were eligible is no evidence\n   * that a larger statement would stay under the target.\n   */\n  #timed(\n    {maxRows, maxDurationMs}: PurgeBatchOptions,\n    budget: number,\n    run: () => number,\n  ): number {\n    const start = this.#now();\n    const deletedRows = run();\n    const elapsedMs = this.#now() - start;\n\n    if (elapsedMs > maxDurationMs) {\n      this.#budgetRows = Math.max(1, Math.floor(budget / 2));\n    } else if (deletedRows >= budget && elapsedMs * 2 <= maxDurationMs) {\n      this.#budgetRows = Math.min(maxRows, budget * 2);\n    } else {\n      this.#budgetRows = budget;\n    }\n    return deletedRows;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8CA,IAAa,yCAAiD;;UAEpD,wBAAwB;;;;uDAIqB,wBAAwB;;;;;;;;;;;;;;;;;AAkB/E,IAAa,0CAAkD;;;YAGnD,wBAAwB;;;;;;;;;;AAWpC,IAAM,qBAA6B;iBAClB,wBAAwB;;AAGzC,IAAM,oBAA4B;oDACkB,wBAAwB;;;;;;;AAQ5E,IAAM,oBAA4B;;UAExB,wBAAwB;;;;;;;;;;;AAYlC,IAAM,uBAA+B;iBACpB,wBAAwB;;;AA0DzC,IAAa,wBAAb,MAAmC;CACjC;CACA;CACA;;;;;;;CAQA;;;;;;;;CASA,YACE,IACA,MAAoB,YAAY,IAAI,KAAK,WAAW,GACpD;EACA,KAAK,MAAM;EACX,KAAK,OAAO;EACZ,KAAK,SAAS;GACZ,iBAAiB,GAAG,QAAQ,sCAAsC;GAClE,kBAAkB,GAAG,QAAQ,uCAAuC;GACpE,eAAe,GAAG,QAAQ,kBAAkB;GAC5C,cAAc,GAAG,QAAQ,iBAAiB;GAC1C,cAAc,GAAG,QAAQ,iBAAiB;GAC1C,gBAAgB,GAAG,QAAQ,oBAAoB;EACjD;CACF;;CAGA,IAAI,aAAiC;EACnC,OAAO,KAAK;CACd;;;;;;CAOA,WAAW,MAA2C;EACpD,MAAM,EAAC,SAAS,kBAAiB;EACjC,OACE,OAAO,cAAc,OAAO,KAAK,UAAU,GAC3C,0DACF;EACA,OACE,gBAAgB,GAChB,iDACF;EAEA,KAAK,IAAI,KAAK,iBAAiB;EAC/B,IAAI;GACF,MAAM,SAAS,KAAK,OAAO,IAAI;GAC/B,KAAK,IAAI,KAAK,QAAQ;GACtB,OAAO;EACT,SAAS,GAAG;GACV,IAAI,KAAK,IAAI,eACX,KAAK,IAAI,KAAK,UAAU;GAE1B,MAAM;EACR;CACF;CAEA,OAAO,MAA2C;EAChD,MAAM,EAAC,eAAe,sBAAqB;EAC3C,MAAM,SAAS,KAAK,QAAQ,KAAK,OAAO;EAGxC,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,KAAK,SAAS,CAAC,CAAC;EAC/C,MAAM,aAAa,KAAK,OAAO,gBAAgB,IAAyB;GACtE;GACA;GACA;EACF,CAAC;EACD,IAAI,WAAW,WAAW,GACxB,OAAO;GAAC,aAAa;GAAG,gBAAgB,KAAA;GAAW,cAAc;EAAK;EAWxE,IAAI,OAAO;EACX,IAAI;EACJ,KAAK,MAAM,EAAC,eAAc,YAAY;GAIpC,MAAM,YAAY,SAAS;GAC3B,MAAM,EAAC,MAAM,MAAK,KAAK,OAAO,iBAAiB,IAAoB;IACjE;IACA,OAAO,YAAY;GACrB,CAAC;GACD,IAAI,IAAI,WACN;GAEF,QAAQ;GACR,iBAAiB;EACnB;EAEA,IAAI,mBAAmB,KAAA,GACrB,OAAO,KAAK,MAAM,MAAM,MAAM;EAOhC,OAAO;GACL,aANkB,KAAK,OACvB,MACA,cACM,KAAK,OAAO,cAAc,IAAI,cAAc,CAAC,CAAC,OAGpD;GACA;GACA,cAAc,KAAK,aAAa,IAAI;EACtC;CACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwCA,MAAM,MAAyB,QAAkC;EAC/D,MAAM,EAAC,iBAAgB,KAAK,OAAO,aAAa,IAE7C;EACH,OACE,iBAAiB,MACjB,8DACF;EAEA,MAAM,WAAW,KAAK,OAAO,aAAa,IAA+B;GACvE,WAAW;GACX,QAAQ;EACV,CAAC;EACD,IAAI,aAAa,KAAA,GAUf,OAAO;GACL,aANkB,KAAK,OACvB,MACA,cACM,KAAK,OAAO,cAAc,IAAI,YAAY,CAAC,CAAC,OAGlD;GACA,gBAAgB;GAChB,cAAc,KAAK,aAAa,IAAI;EACtC;EAcF,OAAO;GAAC,aATY,KAAK,OACvB,MACA,cAEE,KAAK,OAAO,eAAe,IAAI;IAC7B,WAAW;IACX,KAAK,SAAS;GAChB,CAAC,CAAC,CAAC,OAEC;GAAa,gBAAgB,KAAA;GAAW,cAAc;EAAI;CACpE;CAEA,aAAa,EAAC,eAAe,qBAAgD;EAC3E,OACE,KAAK,OAAO,gBAAgB,IAAqC;GAC/D;GACA;GACA,OAAO;EACT,CAAC,MAAM,KAAA;CAEX;CAEA,QAAQ,SAAyB;EAC/B,OAAO,KAAK,IAAI,GAAG,KAAK,IAAI,SAAS,KAAK,eAAe,OAAO,CAAC;CACnE;;;;;;;CAQA,OACE,EAAC,SAAS,iBACV,QACA,KACQ;EACR,MAAM,QAAQ,KAAK,KAAK;EACxB,MAAM,cAAc,IAAI;EACxB,MAAM,YAAY,KAAK,KAAK,IAAI;EAEhC,IAAI,YAAY,eACd,KAAK,cAAc,KAAK,IAAI,GAAG,KAAK,MAAM,SAAS,CAAC,CAAC;OAChD,IAAI,eAAe,UAAU,YAAY,KAAK,eACnD,KAAK,cAAc,KAAK,IAAI,SAAS,SAAS,CAAC;OAE/C,KAAK,cAAc;EAErB,OAAO;CACT;AACF"}

package/out/zero-cache/src/services/replicator/write-worker-client.js

@@ -1,6 +1,6 @@
 import { assert } from "../../../../shared/src/asserts.js";
+import { resolver } from "../../../../shared/src/resolver.js";
 import { WRITE_WORKER_URL } from "../../server/worker-urls.js";
-import { resolver } from "@rocicorp/resolver";
 import { Worker } from "node:worker_threads";
 //#region ../zero-cache/src/services/replicator/write-worker-client.ts
 function serializeError(err) {
@@ -121,6 +121,6 @@ var ThreadWriteWorkerClient = class {
 	}
 };
 //#endregion
-export { ThreadWriteWorkerClient, applyPragmas, serializeError };
+export { ThreadWriteWorkerClient, applyPragmas, deserializeError, serializeError };
 
 //# sourceMappingURL=write-worker-client.js.map

package/out/zero-cache/src/services/replicator/write-worker-client.js.map

@@ -1 +1 @@
-{"version":3,"file":"write-worker-client.js","names":["#worker","#rejectAll","#errorHandler","#pending","#terminated","#call"],"sources":["../../../../../../zero-cache/src/services/replicator/write-worker-client.ts"],"sourcesContent":["import {Worker} from 'node:worker_threads';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport type {LogConfig} from '../../../../shared/src/logging.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {WRITE_WORKER_URL} from '../../server/worker-urls.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {BackfillRequest} from '../change-source/protocol/current/upstream.ts';\nimport type {ChangeProcessorMode, CommitResult} from './change-processor.ts';\nimport type {SubscriptionState} from './schema/replication-state.ts';\n\nexport type PragmaConfig = {\n  busyTimeout: number;\n  analysisLimit: number;\n  walAutocheckpoint?: number | undefined;\n};\n\n/**\n * Enables write-path checkpoint backpressure in the write worker. At each\n * commit boundary, if the WAL has grown to at least `checkpointThresholdPages`,\n * the worker forces an immediate litestream sync (sealing the WAL to LTX and\n * running litestream's checkpoint).\n *\n * The optional `walMaxPages` is an emergency break that pauses until the WAL drains,\n * rather than letting it exceed available disk space. This replaces litestream's\n * analogous `truncate-page-n` emergency break, which is more disruptive because of\n * https://github.com/benbjohnson/litestream/issues/1332. Note, however, this should\n * never been needed if non-initial snapshots are disabled (which is the default).\n *\n * Set only for the backup replicator when replicating with litestream v5; `null`\n * disables the feature entirely (no per-commit overhead).\n */\nexport type ForceCheckpointConfig = {\n  checkpointThresholdPages: number;\n  maxWalPages?: number | undefined;\n};\n\ntype ErrorHandler = (err: Error) => void;\n\n/**\n * Interface for a write worker that processes replication messages.\n */\n/**\n * The replica's {@link SubscriptionState}, along with its pending backfills,\n * read in the same snapshot.\n */\nexport type SubscriptionStateWithBackfills = SubscriptionState & {\n  backfills: BackfillRequest[];\n};\n\nexport interface WriteWorkerClient {\n  getSubscriptionState(): Promise<SubscriptionStateWithBackfills>;\n  processMessages(\n    downstream: readonly ChangeStreamData[],\n  ): Promise<CommitResult | null>;\n  abort(): void;\n  stop(): Promise<void>;\n  onError(handler: ErrorHandler): void;\n}\n\nexport type SerializedError = {\n  name: string;\n  message: string;\n  stack?: string | undefined;\n  cause?: SerializedError | string | undefined;\n  details?: Record<string, unknown> | undefined;\n};\n\nexport function serializeError(err: unknown): SerializedError {\n  if (!(err instanceof Error)) {\n    return {\n      name: 'Error',\n      message: String(err),\n      details: err && typeof err === 'object' ? {...err} : undefined,\n    };\n  }\n\n  // Error fields such as message, stack, and some native error details are\n  // non-enumerable, so JSON.stringify(err) would usually return \"{}\".\n  const details = Object.fromEntries(\n    Object.getOwnPropertyNames(err)\n      .filter(key => !['name', 'message', 'stack', 'cause'].includes(key))\n      .map(key => [key, (err as unknown as Record<string, unknown>)[key]]),\n  );\n  const cause =\n    err.cause instanceof Error\n      ? serializeError(err.cause)\n      : err.cause === undefined\n        ? undefined\n        : String(err.cause);\n\n  return {\n    name: err.name,\n    message: err.message,\n    stack: err.stack,\n    cause,\n    details: Object.keys(details).length ? details : undefined,\n  };\n}\n\nexport function deserializeError(serialized: SerializedError): Error {\n  const err = new Error(serialized.message);\n  err.name = serialized.name;\n  if (serialized.stack !== undefined) {\n    err.stack = serialized.stack;\n  }\n  if (serialized.cause !== undefined) {\n    err.cause =\n      typeof serialized.cause === 'string'\n        ? serialized.cause\n        : deserializeError(serialized.cause);\n  }\n  if (serialized.details) {\n    Object.assign(err, serialized.details);\n  }\n  return err;\n}\n\n// Wire protocol types.\nexport type ArgsMap = {\n  init: [\n    string,\n    ChangeProcessorMode,\n    PragmaConfig,\n    LogConfig,\n    ForceCheckpointConfig | null,\n  ];\n  getSubscriptionState: [];\n  processMessages: [readonly ChangeStreamData[]];\n  abort: [];\n  stop: [];\n};\n\nexport type Method = keyof ArgsMap;\n\nexport type Request<M extends Method = Method> = {method: M; args: ArgsMap[M]};\n\nexport type ResultMap = {\n  init: void;\n  getSubscriptionState: SubscriptionStateWithBackfills;\n  processMessages: CommitResult | null;\n  abort: void;\n  stop: void;\n};\n\nexport type Response<M extends Method = Method> =\n  | {method: M; result: ResultMap[M]; error?: undefined}\n  | {method: M; error: SerializedError; result?: undefined};\n\nexport type WriteError = {writeError: SerializedError};\n\nexport function applyPragmas(db: Database, pragmas: PragmaConfig) {\n  db.pragma(`busy_timeout = ${pragmas.busyTimeout}`);\n  db.pragma(`analysis_limit = ${pragmas.analysisLimit}`);\n  if (pragmas.walAutocheckpoint !== undefined) {\n    db.pragma(`wal_autocheckpoint = ${pragmas.walAutocheckpoint}`);\n  }\n}\n\n/**\n * Delegates SQLite writes to a worker_thread,\n * keeping the main event loop free for WebSocket heartbeats and IPC.\n */\nexport class ThreadWriteWorkerClient implements WriteWorkerClient {\n  readonly #worker: Worker;\n  #pending: Resolver<unknown, Error> | null = null;\n  #errorHandler: ErrorHandler = () => {};\n  #terminated = false;\n\n  constructor() {\n    this.#worker = new Worker(WRITE_WORKER_URL);\n\n    this.#worker.on('message', (msg: Response | WriteError) => {\n      if ('writeError' in msg) {\n        const error = deserializeError(msg.writeError);\n        this.#rejectAll(error);\n        this.#errorHandler(error);\n        return;\n      }\n      const r = this.#pending;\n      if (!r) return; // stale abort response\n      this.#pending = null;\n      if (msg.error !== undefined) {\n        r.reject(deserializeError(msg.error));\n      } else {\n        r.resolve(msg.result);\n      }\n    });\n\n    this.#worker.on('error', (err: Error) => {\n      this.#rejectAll(err);\n      this.#errorHandler(err);\n    });\n\n    this.#worker.on('exit', (code: number) => {\n      this.#terminated = true;\n      if (code !== 0) {\n        const err = new Error(`Worker exited with code ${code}`);\n        this.#rejectAll(err);\n        this.#errorHandler(err);\n      }\n    });\n  }\n\n  #rejectAll(err: Error) {\n    const r = this.#pending;\n    if (r) {\n      this.#pending = null;\n      r.reject(err);\n    }\n  }\n\n  #call<M extends Method>(method: M, args: ArgsMap[M]): Promise<ResultMap[M]> {\n    assert(this.#pending === null, `concurrent call: ${method}`);\n    const r = resolver<ResultMap[M]>();\n    this.#pending = r as Resolver<unknown, Error>;\n    this.#worker.postMessage({method, args} satisfies Request);\n    return r.promise;\n  }\n\n  init(\n    dbPath: string,\n    mode: ChangeProcessorMode,\n    pragmas: PragmaConfig,\n    logConfig: LogConfig,\n    checkpoint: ForceCheckpointConfig | null = null,\n  ): Promise<void> {\n    return this.#call('init', [dbPath, mode, pragmas, logConfig, checkpoint]);\n  }\n\n  getSubscriptionState(): Promise<SubscriptionStateWithBackfills> {\n    return this.#call('getSubscriptionState', []);\n  }\n\n  processMessages(\n    downstream: readonly ChangeStreamData[],\n  ): Promise<CommitResult | null> {\n    return this.#call('processMessages', [downstream]);\n  }\n\n  abort(): void {\n    if (!this.#terminated) {\n      this.#worker.postMessage({method: 'abort', args: []} satisfies Request);\n    }\n  }\n\n  async stop(): Promise<void> {\n    await this.#call('stop', []);\n    if (!this.#terminated) {\n      await this.#worker.terminate();\n    }\n  }\n\n  onError(handler: ErrorHandler): void {\n    this.#errorHandler = handler;\n  }\n}\n"],"mappings":";;;;;AAoEA,SAAgB,eAAe,KAA+B;CAC5D,IAAI,EAAE,eAAe,QACnB,OAAO;EACL,MAAM;EACN,SAAS,OAAO,GAAG;EACnB,SAAS,OAAO,OAAO,QAAQ,WAAW,EAAC,GAAG,IAAG,IAAI,KAAA;CACvD;CAKF,MAAM,UAAU,OAAO,YACrB,OAAO,oBAAoB,GAAG,EAC3B,QAAO,QAAO,CAAC;EAAC;EAAQ;EAAW;EAAS;CAAO,EAAE,SAAS,GAAG,CAAC,EAClE,KAAI,QAAO,CAAC,KAAM,IAA2C,IAAI,CAAC,CACvE;CACA,MAAM,QACJ,IAAI,iBAAiB,QACjB,eAAe,IAAI,KAAK,IACxB,IAAI,UAAU,KAAA,IACZ,KAAA,IACA,OAAO,IAAI,KAAK;CAExB,OAAO;EACL,MAAM,IAAI;EACV,SAAS,IAAI;EACb,OAAO,IAAI;EACX;EACA,SAAS,OAAO,KAAK,OAAO,EAAE,SAAS,UAAU,KAAA;CACnD;AACF;AAEA,SAAgB,iBAAiB,YAAoC;CACnE,MAAM,MAAM,IAAI,MAAM,WAAW,OAAO;CACxC,IAAI,OAAO,WAAW;CACtB,IAAI,WAAW,UAAU,KAAA,GACvB,IAAI,QAAQ,WAAW;CAEzB,IAAI,WAAW,UAAU,KAAA,GACvB,IAAI,QACF,OAAO,WAAW,UAAU,WACxB,WAAW,QACX,iBAAiB,WAAW,KAAK;CAEzC,IAAI,WAAW,SACb,OAAO,OAAO,KAAK,WAAW,OAAO;CAEvC,OAAO;AACT;AAmCA,SAAgB,aAAa,IAAc,SAAuB;CAChE,GAAG,OAAO,kBAAkB,QAAQ,aAAa;CACjD,GAAG,OAAO,oBAAoB,QAAQ,eAAe;CACrD,IAAI,QAAQ,sBAAsB,KAAA,GAChC,GAAG,OAAO,wBAAwB,QAAQ,mBAAmB;AAEjE;;;;;AAMA,IAAa,0BAAb,MAAkE;CAChE;CACA,WAA4C;CAC5C,sBAAoC,CAAC;CACrC,cAAc;CAEd,cAAc;EACZ,KAAKA,UAAU,IAAI,OAAO,gBAAgB;EAE1C,KAAKA,QAAQ,GAAG,YAAY,QAA+B;GACzD,IAAI,gBAAgB,KAAK;IACvB,MAAM,QAAQ,iBAAiB,IAAI,UAAU;IAC7C,KAAKC,WAAW,KAAK;IACrB,KAAKC,cAAc,KAAK;IACxB;GACF;GACA,MAAM,IAAI,KAAKC;GACf,IAAI,CAAC,GAAG;GACR,KAAKA,WAAW;GAChB,IAAI,IAAI,UAAU,KAAA,GAChB,EAAE,OAAO,iBAAiB,IAAI,KAAK,CAAC;QAEpC,EAAE,QAAQ,IAAI,MAAM;EAExB,CAAC;EAED,KAAKH,QAAQ,GAAG,UAAU,QAAe;GACvC,KAAKC,WAAW,GAAG;GACnB,KAAKC,cAAc,GAAG;EACxB,CAAC;EAED,KAAKF,QAAQ,GAAG,SAAS,SAAiB;GACxC,KAAKI,cAAc;GACnB,IAAI,SAAS,GAAG;IACd,MAAM,sBAAM,IAAI,MAAM,2BAA2B,MAAM;IACvD,KAAKH,WAAW,GAAG;IACnB,KAAKC,cAAc,GAAG;GACxB;EACF,CAAC;CACH;CAEA,WAAW,KAAY;EACrB,MAAM,IAAI,KAAKC;EACf,IAAI,GAAG;GACL,KAAKA,WAAW;GAChB,EAAE,OAAO,GAAG;EACd;CACF;CAEA,MAAwB,QAAW,MAAyC;EAC1E,OAAO,KAAKA,aAAa,MAAM,oBAAoB,QAAQ;EAC3D,MAAM,IAAI,SAAuB;EACjC,KAAKA,WAAW;EAChB,KAAKH,QAAQ,YAAY;GAAC;GAAQ;EAAI,CAAmB;EACzD,OAAO,EAAE;CACX;CAEA,KACE,QACA,MACA,SACA,WACA,aAA2C,MAC5B;EACf,OAAO,KAAKK,MAAM,QAAQ;GAAC;GAAQ;GAAM;GAAS;GAAW;EAAU,CAAC;CAC1E;CAEA,uBAAgE;EAC9D,OAAO,KAAKA,MAAM,wBAAwB,CAAC,CAAC;CAC9C;CAEA,gBACE,YAC8B;EAC9B,OAAO,KAAKA,MAAM,mBAAmB,CAAC,UAAU,CAAC;CACnD;CAEA,QAAc;EACZ,IAAI,CAAC,KAAKD,aACR,KAAKJ,QAAQ,YAAY;GAAC,QAAQ;GAAS,MAAM,CAAC;EAAC,CAAmB;CAE1E;CAEA,MAAM,OAAsB;EAC1B,MAAM,KAAKK,MAAM,QAAQ,CAAC,CAAC;EAC3B,IAAI,CAAC,KAAKD,aACR,MAAM,KAAKJ,QAAQ,UAAU;CAEjC;CAEA,QAAQ,SAA6B;EACnC,KAAKE,gBAAgB;CACvB;AACF"}
+{"version":3,"file":"write-worker-client.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/write-worker-client.ts"],"sourcesContent":["import {Worker} from 'node:worker_threads';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport type {LogConfig} from '../../../../shared/src/logging.ts';\nimport {resolver, type Resolver} from '../../../../shared/src/resolver.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {WRITE_WORKER_URL} from '../../server/worker-urls.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {BackfillRequest} from '../change-source/protocol/current/upstream.ts';\nimport type {ChangeProcessorMode, CommitResult} from './change-processor.ts';\nimport type {SubscriptionState} from './schema/replication-state.ts';\n\nexport type PragmaConfig = {\n  busyTimeout: number;\n  analysisLimit: number;\n  walAutocheckpoint?: number | undefined;\n};\n\n/**\n * Enables write-path checkpoint backpressure in the write worker. At each\n * commit boundary, if the WAL has grown to at least `checkpointThresholdPages`,\n * the worker forces an immediate litestream sync (sealing the WAL to LTX and\n * running litestream's checkpoint).\n *\n * The optional `walMaxPages` is an emergency break that pauses until the WAL drains,\n * rather than letting it exceed available disk space. This replaces litestream's\n * analogous `truncate-page-n` emergency break, which is more disruptive because of\n * https://github.com/benbjohnson/litestream/issues/1332. Note, however, this should\n * never been needed if non-initial snapshots are disabled (which is the default).\n *\n * Set only for the backup replicator when replicating with litestream v5; `null`\n * disables the feature entirely (no per-commit overhead).\n */\nexport type ForceCheckpointConfig = {\n  checkpointThresholdPages: number;\n  maxWalPages?: number | undefined;\n};\n\ntype ErrorHandler = (err: Error) => void;\n\n/**\n * Interface for a write worker that processes replication messages.\n */\n/**\n * The replica's {@link SubscriptionState}, along with its pending backfills,\n * read in the same snapshot.\n */\nexport type SubscriptionStateWithBackfills = SubscriptionState & {\n  backfills: BackfillRequest[];\n};\n\nexport interface WriteWorkerClient {\n  getSubscriptionState(): Promise<SubscriptionStateWithBackfills>;\n  processMessages(\n    downstream: readonly ChangeStreamData[],\n  ): Promise<CommitResult | null>;\n  abort(): void;\n  stop(): Promise<void>;\n  onError(handler: ErrorHandler): void;\n}\n\nexport type SerializedError = {\n  name: string;\n  message: string;\n  stack?: string | undefined;\n  cause?: SerializedError | string | undefined;\n  details?: Record<string, unknown> | undefined;\n};\n\nexport function serializeError(err: unknown): SerializedError {\n  if (!(err instanceof Error)) {\n    return {\n      name: 'Error',\n      message: String(err),\n      details: err && typeof err === 'object' ? {...err} : undefined,\n    };\n  }\n\n  // Error fields such as message, stack, and some native error details are\n  // non-enumerable, so JSON.stringify(err) would usually return \"{}\".\n  const details = Object.fromEntries(\n    Object.getOwnPropertyNames(err)\n      .filter(key => !['name', 'message', 'stack', 'cause'].includes(key))\n      .map(key => [key, (err as unknown as Record<string, unknown>)[key]]),\n  );\n  const cause =\n    err.cause instanceof Error\n      ? serializeError(err.cause)\n      : err.cause === undefined\n        ? undefined\n        : String(err.cause);\n\n  return {\n    name: err.name,\n    message: err.message,\n    stack: err.stack,\n    cause,\n    details: Object.keys(details).length ? details : undefined,\n  };\n}\n\nexport function deserializeError(serialized: SerializedError): Error {\n  const err = new Error(serialized.message);\n  err.name = serialized.name;\n  if (serialized.stack !== undefined) {\n    err.stack = serialized.stack;\n  }\n  if (serialized.cause !== undefined) {\n    err.cause =\n      typeof serialized.cause === 'string'\n        ? serialized.cause\n        : deserializeError(serialized.cause);\n  }\n  if (serialized.details) {\n    Object.assign(err, serialized.details);\n  }\n  return err;\n}\n\n// Wire protocol types.\nexport type ArgsMap = {\n  init: [\n    string,\n    ChangeProcessorMode,\n    PragmaConfig,\n    LogConfig,\n    ForceCheckpointConfig | null,\n  ];\n  getSubscriptionState: [];\n  processMessages: [readonly ChangeStreamData[]];\n  abort: [];\n  stop: [];\n};\n\nexport type Method = keyof ArgsMap;\n\nexport type Request<M extends Method = Method> = {method: M; args: ArgsMap[M]};\n\nexport type ResultMap = {\n  init: void;\n  getSubscriptionState: SubscriptionStateWithBackfills;\n  processMessages: CommitResult | null;\n  abort: void;\n  stop: void;\n};\n\nexport type Response<M extends Method = Method> =\n  | {method: M; result: ResultMap[M]; error?: undefined}\n  | {method: M; error: SerializedError; result?: undefined};\n\nexport type WriteError = {writeError: SerializedError};\n\nexport function applyPragmas(db: Database, pragmas: PragmaConfig) {\n  db.pragma(`busy_timeout = ${pragmas.busyTimeout}`);\n  db.pragma(`analysis_limit = ${pragmas.analysisLimit}`);\n  if (pragmas.walAutocheckpoint !== undefined) {\n    db.pragma(`wal_autocheckpoint = ${pragmas.walAutocheckpoint}`);\n  }\n}\n\n/**\n * Delegates SQLite writes to a worker_thread,\n * keeping the main event loop free for WebSocket heartbeats and IPC.\n */\nexport class ThreadWriteWorkerClient implements WriteWorkerClient {\n  readonly #worker: Worker;\n  #pending: Resolver<unknown, Error> | null = null;\n  #errorHandler: ErrorHandler = () => {};\n  #terminated = false;\n\n  constructor() {\n    this.#worker = new Worker(WRITE_WORKER_URL);\n\n    this.#worker.on('message', (msg: Response | WriteError) => {\n      if ('writeError' in msg) {\n        const error = deserializeError(msg.writeError);\n        this.#rejectAll(error);\n        this.#errorHandler(error);\n        return;\n      }\n      const r = this.#pending;\n      if (!r) return; // stale abort response\n      this.#pending = null;\n      if (msg.error !== undefined) {\n        r.reject(deserializeError(msg.error));\n      } else {\n        r.resolve(msg.result);\n      }\n    });\n\n    this.#worker.on('error', (err: Error) => {\n      this.#rejectAll(err);\n      this.#errorHandler(err);\n    });\n\n    this.#worker.on('exit', (code: number) => {\n      this.#terminated = true;\n      if (code !== 0) {\n        const err = new Error(`Worker exited with code ${code}`);\n        this.#rejectAll(err);\n        this.#errorHandler(err);\n      }\n    });\n  }\n\n  #rejectAll(err: Error) {\n    const r = this.#pending;\n    if (r) {\n      this.#pending = null;\n      r.reject(err);\n    }\n  }\n\n  #call<M extends Method>(method: M, args: ArgsMap[M]): Promise<ResultMap[M]> {\n    assert(this.#pending === null, `concurrent call: ${method}`);\n    const r = resolver<ResultMap[M]>();\n    this.#pending = r as Resolver<unknown, Error>;\n    this.#worker.postMessage({method, args} satisfies Request);\n    return r.promise;\n  }\n\n  init(\n    dbPath: string,\n    mode: ChangeProcessorMode,\n    pragmas: PragmaConfig,\n    logConfig: LogConfig,\n    checkpoint: ForceCheckpointConfig | null = null,\n  ): Promise<void> {\n    return this.#call('init', [dbPath, mode, pragmas, logConfig, checkpoint]);\n  }\n\n  getSubscriptionState(): Promise<SubscriptionStateWithBackfills> {\n    return this.#call('getSubscriptionState', []);\n  }\n\n  processMessages(\n    downstream: readonly ChangeStreamData[],\n  ): Promise<CommitResult | null> {\n    return this.#call('processMessages', [downstream]);\n  }\n\n  abort(): void {\n    if (!this.#terminated) {\n      this.#worker.postMessage({method: 'abort', args: []} satisfies Request);\n    }\n  }\n\n  async stop(): Promise<void> {\n    await this.#call('stop', []);\n    if (!this.#terminated) {\n      await this.#worker.terminate();\n    }\n  }\n\n  onError(handler: ErrorHandler): void {\n    this.#errorHandler = handler;\n  }\n}\n"],"mappings":";;;;;AAoEA,SAAgB,eAAe,KAA+B;CAC5D,IAAI,EAAE,eAAe,QACnB,OAAO;EACL,MAAM;EACN,SAAS,OAAO,GAAG;EACnB,SAAS,OAAO,OAAO,QAAQ,WAAW,EAAC,GAAG,IAAG,IAAI,KAAA;CACvD;CAKF,MAAM,UAAU,OAAO,YACrB,OAAO,oBAAoB,GAAG,CAAC,CAC5B,QAAO,QAAO,CAAC;EAAC;EAAQ;EAAW;EAAS;CAAO,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CACnE,KAAI,QAAO,CAAC,KAAM,IAA2C,IAAI,CAAC,CACvE;CACA,MAAM,QACJ,IAAI,iBAAiB,QACjB,eAAe,IAAI,KAAK,IACxB,IAAI,UAAU,KAAA,IACZ,KAAA,IACA,OAAO,IAAI,KAAK;CAExB,OAAO;EACL,MAAM,IAAI;EACV,SAAS,IAAI;EACb,OAAO,IAAI;EACX;EACA,SAAS,OAAO,KAAK,OAAO,CAAC,CAAC,SAAS,UAAU,KAAA;CACnD;AACF;AAEA,SAAgB,iBAAiB,YAAoC;CACnE,MAAM,MAAM,IAAI,MAAM,WAAW,OAAO;CACxC,IAAI,OAAO,WAAW;CACtB,IAAI,WAAW,UAAU,KAAA,GACvB,IAAI,QAAQ,WAAW;CAEzB,IAAI,WAAW,UAAU,KAAA,GACvB,IAAI,QACF,OAAO,WAAW,UAAU,WACxB,WAAW,QACX,iBAAiB,WAAW,KAAK;CAEzC,IAAI,WAAW,SACb,OAAO,OAAO,KAAK,WAAW,OAAO;CAEvC,OAAO;AACT;AAmCA,SAAgB,aAAa,IAAc,SAAuB;CAChE,GAAG,OAAO,kBAAkB,QAAQ,aAAa;CACjD,GAAG,OAAO,oBAAoB,QAAQ,eAAe;CACrD,IAAI,QAAQ,sBAAsB,KAAA,GAChC,GAAG,OAAO,wBAAwB,QAAQ,mBAAmB;AAEjE;;;;;AAMA,IAAa,0BAAb,MAAkE;CAChE;CACA,WAA4C;CAC5C,sBAAoC,CAAC;CACrC,cAAc;CAEd,cAAc;EACZ,KAAK,UAAU,IAAI,OAAO,gBAAgB;EAE1C,KAAK,QAAQ,GAAG,YAAY,QAA+B;GACzD,IAAI,gBAAgB,KAAK;IACvB,MAAM,QAAQ,iBAAiB,IAAI,UAAU;IAC7C,KAAK,WAAW,KAAK;IACrB,KAAK,cAAc,KAAK;IACxB;GACF;GACA,MAAM,IAAI,KAAK;GACf,IAAI,CAAC,GAAG;GACR,KAAK,WAAW;GAChB,IAAI,IAAI,UAAU,KAAA,GAChB,EAAE,OAAO,iBAAiB,IAAI,KAAK,CAAC;QAEpC,EAAE,QAAQ,IAAI,MAAM;EAExB,CAAC;EAED,KAAK,QAAQ,GAAG,UAAU,QAAe;GACvC,KAAK,WAAW,GAAG;GACnB,KAAK,cAAc,GAAG;EACxB,CAAC;EAED,KAAK,QAAQ,GAAG,SAAS,SAAiB;GACxC,KAAK,cAAc;GACnB,IAAI,SAAS,GAAG;IACd,MAAM,sBAAM,IAAI,MAAM,2BAA2B,MAAM;IACvD,KAAK,WAAW,GAAG;IACnB,KAAK,cAAc,GAAG;GACxB;EACF,CAAC;CACH;CAEA,WAAW,KAAY;EACrB,MAAM,IAAI,KAAK;EACf,IAAI,GAAG;GACL,KAAK,WAAW;GAChB,EAAE,OAAO,GAAG;EACd;CACF;CAEA,MAAwB,QAAW,MAAyC;EAC1E,OAAO,KAAK,aAAa,MAAM,oBAAoB,QAAQ;EAC3D,MAAM,IAAI,SAAuB;EACjC,KAAK,WAAW;EAChB,KAAK,QAAQ,YAAY;GAAC;GAAQ;EAAI,CAAmB;EACzD,OAAO,EAAE;CACX;CAEA,KACE,QACA,MACA,SACA,WACA,aAA2C,MAC5B;EACf,OAAO,KAAK,MAAM,QAAQ;GAAC;GAAQ;GAAM;GAAS;GAAW;EAAU,CAAC;CAC1E;CAEA,uBAAgE;EAC9D,OAAO,KAAK,MAAM,wBAAwB,CAAC,CAAC;CAC9C;CAEA,gBACE,YAC8B;EAC9B,OAAO,KAAK,MAAM,mBAAmB,CAAC,UAAU,CAAC;CACnD;CAEA,QAAc;EACZ,IAAI,CAAC,KAAK,aACR,KAAK,QAAQ,YAAY;GAAC,QAAQ;GAAS,MAAM,CAAC;EAAC,CAAmB;CAE1E;CAEA,MAAM,OAAsB;EAC1B,MAAM,KAAK,MAAM,QAAQ,CAAC,CAAC;EAC3B,IAAI,CAAC,KAAK,aACR,MAAM,KAAK,QAAQ,UAAU;CAEjC;CAEA,QAAQ,SAA6B;EACnC,KAAK,gBAAgB;CACvB;AACF"}

package/out/zero-cache/src/services/replicator/write-worker.js

@@ -1,11 +1,11 @@
 import { assert } from "../../../../shared/src/asserts.js";
 import { must } from "../../../../shared/src/must.js";
 import { Database } from "../../../../zqlite/src/db.js";
-import { readReplicaBackfills } from "./schema/backfilling.js";
-import { getSubscriptionState } from "./schema/replication-state.js";
-import { StatementRunner } from "../../db/statements.js";
 import { isSQLiteCorruption, logSQLiteCorruptionDiagnostics, registerSQLiteCorruptionDiagnosticTarget } from "../../db/sqlite-corruption.js";
 import { createLogContext } from "../../server/logging.js";
+import { StatementRunner } from "../../db/statements.js";
+import { readReplicaBackfills } from "./schema/backfilling.js";
+import { getSubscriptionState } from "./schema/replication-state.js";
 import { deleteLiteDB } from "../../db/delete-lite-db.js";
 import { ChangeProcessor } from "./change-processor.js";
 import { LitestreamController } from "../litestream/litestream-controller.js";