@rocicorp/zero 0.26.0-canary.2 → 0.26.0-canary.3

package/out/zero-cache/src/services/replicator/schema/column-metadata.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"column-metadata.js","sources":["../../../../../../../zero-cache/src/services/replicator/schema/column-metadata.ts"],"sourcesContent":["/**\n * Column metadata table for storing upstream PostgreSQL schema information.\n *\n * Previously, upstream type metadata was embedded in SQLite column type strings\n * using pipe-delimited notation (e.g., \"int8|NOT_NULL|TEXT_ENUM\"). This caused\n * issues with SQLite type affinity and made schema inspection difficult.\n *\n * This table stores that metadata separately, allowing SQLite columns to use\n * plain type names while preserving all necessary upstream type information.\n */\n\nimport type {Database, Statement} from '../../../../../zqlite/src/db.ts';\nimport {isArrayColumn, isEnumColumn} from '../../../db/pg-to-lite.ts';\nimport type {ColumnSpec, LiteTableSpec} from '../../../db/specs.ts';\nimport {\n  isArray as checkIsArray,\n  isEnum as checkIsEnum,\n  liteTypeString,\n  nullableUpstream,\n  upstreamDataType,\n} from '../../../types/lite.ts';\n\n/**\n * Structured column metadata, replacing the old pipe-delimited string format.\n */\nexport interface ColumnMetadata {\n  /** PostgreSQL type name, e.g., 'int8', 'varchar', 'text[]', 'user_role' */\n  upstreamType: string;\n  isNotNull: boolean;\n  isEnum: boolean;\n  isArray: boolean;\n  /** Maximum character length for varchar/char types */\n  characterMaxLength?: number | null;\n}\n\ntype ColumnMetadataRow = {\n  upstream_type: string;\n  is_not_null: number;\n  is_enum: number;\n  is_array: number;\n  character_max_length: number | null;\n};\n\nexport const CREATE_COLUMN_METADATA_TABLE = `\n  CREATE TABLE \"_zero.column_metadata\" (\n    table_name TEXT NOT NULL,\n    column_name TEXT NOT NULL,\n    upstream_type TEXT NOT NULL,\n    is_not_null INTEGER NOT NULL,\n    is_enum INTEGER NOT NULL,\n    is_array INTEGER NOT NULL,\n    character_max_length INTEGER,\n    PRIMARY KEY (table_name, column_name)\n  );\n`;\n\n/**\n * Efficient column metadata store that prepares all statements upfront.\n * Use this class to avoid re-preparing statements on every operation.\n *\n * Access via `ColumnMetadataStore.getInstance(db)`, which returns `undefined`\n * if the metadata table doesn't exist yet.\n */\nexport class ColumnMetadataStore {\n  static #instances = new WeakMap<Database, ColumnMetadataStore>();\n\n  readonly #insertStmt: Statement;\n  readonly #updateStmt: Statement;\n  readonly #deleteColumnStmt: Statement;\n  readonly #deleteTableStmt: Statement;\n  readonly #renameTableStmt: Statement;\n  readonly #getColumnStmt: Statement;\n  readonly #getTableStmt: Statement;\n  readonly #hasTableStmt: Statement;\n\n  private constructor(db: Database) {\n    this.#insertStmt = db.prepare(`\n      INSERT INTO \"_zero.column_metadata\"\n        (table_name, column_name, upstream_type, is_not_null, is_enum, is_array, character_max_length)\n      VALUES (?, ?, ?, ?, ?, ?, ?)\n    `);\n\n    this.#updateStmt = db.prepare(`\n      UPDATE \"_zero.column_metadata\"\n      SET column_name = ?,\n          upstream_type = ?,\n          is_not_null = ?,\n          is_enum = ?,\n          is_array = ?,\n          character_max_length = ?\n      WHERE table_name = ? AND column_name = ?\n    `);\n\n    this.#deleteColumnStmt = db.prepare(`\n      DELETE FROM \"_zero.column_metadata\"\n      WHERE table_name = ? AND column_name = ?\n    `);\n\n    this.#deleteTableStmt = db.prepare(`\n      DELETE FROM \"_zero.column_metadata\"\n      WHERE table_name = ?\n    `);\n\n    this.#renameTableStmt = db.prepare(`\n      UPDATE \"_zero.column_metadata\"\n      SET table_name = ?\n      WHERE table_name = ?\n    `);\n\n    this.#getColumnStmt = db.prepare(`\n      SELECT upstream_type, is_not_null, is_enum, is_array, character_max_length\n      FROM \"_zero.column_metadata\"\n      WHERE table_name = ? AND column_name = ?\n    `);\n\n    this.#getTableStmt = db.prepare(`\n      SELECT column_name, upstream_type, is_not_null, is_enum, is_array, character_max_length\n      FROM \"_zero.column_metadata\"\n      WHERE table_name = ?\n      ORDER BY column_name\n    `);\n\n    this.#hasTableStmt = db.prepare(`\n      SELECT 1 FROM sqlite_master\n      WHERE type = 'table' AND name = '_zero.column_metadata'\n    `);\n  }\n\n  /**\n   * Gets the singleton instance of ColumnMetadataStore for the given database.\n   * Returns `undefined` if the metadata table doesn't exist yet.\n   */\n  static getInstance(db: Database): ColumnMetadataStore | undefined {\n    // Check if table exists\n    const tableExists = db\n      .prepare(\n        `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_zero.column_metadata'`,\n      )\n      .get();\n\n    if (!tableExists) {\n      return undefined;\n    }\n\n    let instance = ColumnMetadataStore.#instances.get(db);\n    if (!instance) {\n      instance = new ColumnMetadataStore(db);\n      ColumnMetadataStore.#instances.set(db, instance);\n    }\n    return instance;\n  }\n\n  insert(tableName: string, columnName: string, spec: ColumnSpec): void {\n    const metadata = pgColumnSpecToMetadata(spec);\n    this.#insertMetadata(tableName, columnName, metadata);\n  }\n\n  #insertMetadata(\n    tableName: string,\n    columnName: string,\n    metadata: ColumnMetadata,\n  ): void {\n    this.#insertStmt.run(\n      tableName,\n      columnName,\n      metadata.upstreamType,\n      metadata.isNotNull ? 1 : 0,\n      metadata.isEnum ? 1 : 0,\n      metadata.isArray ? 1 : 0,\n      metadata.characterMaxLength ?? null,\n    );\n  }\n\n  update(\n    tableName: string,\n    oldColumnName: string,\n    newColumnName: string,\n    spec: ColumnSpec,\n  ): void {\n    const metadata = pgColumnSpecToMetadata(spec);\n    this.#updateStmt.run(\n      newColumnName,\n      metadata.upstreamType,\n      metadata.isNotNull ? 1 : 0,\n      metadata.isEnum ? 1 : 0,\n      metadata.isArray ? 1 : 0,\n      metadata.characterMaxLength ?? null,\n      tableName,\n      oldColumnName,\n    );\n  }\n\n  deleteColumn(tableName: string, columnName: string): void {\n    this.#deleteColumnStmt.run(tableName, columnName);\n  }\n\n  deleteTable(tableName: string): void {\n    this.#deleteTableStmt.run(tableName);\n  }\n\n  renameTable(oldTableName: string, newTableName: string): void {\n    this.#renameTableStmt.run(newTableName, oldTableName);\n  }\n\n  getColumn(tableName: string, columnName: string): ColumnMetadata | undefined {\n    const row = this.#getColumnStmt.get(tableName, columnName) as\n      | ColumnMetadataRow\n      | undefined;\n\n    if (!row) {\n      return undefined;\n    }\n\n    return {\n      upstreamType: row.upstream_type,\n      isNotNull: row.is_not_null !== 0,\n      isEnum: row.is_enum !== 0,\n      isArray: row.is_array !== 0,\n      characterMaxLength: row.character_max_length,\n    };\n  }\n\n  getTable(tableName: string): Map<string, ColumnMetadata> {\n    const rows = this.#getTableStmt.all(tableName) as Array<\n      ColumnMetadataRow & {column_name: string}\n    >;\n\n    const metadata = new Map<string, ColumnMetadata>();\n    for (const row of rows) {\n      metadata.set(row.column_name, {\n        upstreamType: row.upstream_type,\n        isNotNull: row.is_not_null !== 0,\n        isEnum: row.is_enum !== 0,\n        isArray: row.is_array !== 0,\n        characterMaxLength: row.character_max_length,\n      });\n    }\n\n    return metadata;\n  }\n\n  hasTable(): boolean {\n    const result = this.#hasTableStmt.get();\n    return result !== undefined;\n  }\n\n  /**\n   * Populates metadata table from existing tables that use pipe notation.\n   * This is used during migration v8 to backfill the metadata table.\n   */\n  populateFromExistingTables(tables: LiteTableSpec[]): void {\n    for (const table of tables) {\n      for (const [columnName, columnSpec] of Object.entries(table.columns)) {\n        const metadata = liteTypeStringToMetadata(\n          columnSpec.dataType,\n          columnSpec.characterMaximumLength,\n        );\n        this.#insertMetadata(table.name, columnName, metadata);\n      }\n    }\n  }\n}\n\n/**\n * Converts pipe-delimited LiteTypeString to structured ColumnMetadata.\n * This is a compatibility helper for the migration period.\n */\nexport function liteTypeStringToMetadata(\n  liteTypeString: string,\n  characterMaxLength?: number | null,\n): ColumnMetadata {\n  const baseType = upstreamDataType(liteTypeString);\n  const isArrayType = checkIsArray(liteTypeString);\n\n  // Reconstruct the full upstream type including array notation\n  // For new-style arrays like 'text[]', upstreamDataType returns 'text[]'\n  // For old-style arrays like 'int4|NOT_NULL[]', upstreamDataType returns 'int4', so we append '[]'\n  const fullUpstreamType =\n    isArrayType && !baseType.includes('[]') ? `${baseType}[]` : baseType;\n\n  return {\n    upstreamType: fullUpstreamType,\n    isNotNull: !nullableUpstream(liteTypeString),\n    isEnum: checkIsEnum(liteTypeString),\n    isArray: isArrayType,\n    characterMaxLength: characterMaxLength ?? null,\n  };\n}\n\n/**\n * Converts structured ColumnMetadata back to pipe-delimited LiteTypeString.\n * This is a compatibility helper for the migration period.\n */\nexport function metadataToLiteTypeString(metadata: ColumnMetadata): string {\n  return liteTypeString(\n    metadata.upstreamType,\n    metadata.isNotNull,\n    metadata.isEnum,\n    metadata.isArray,\n  );\n}\n\n/**\n * Converts PostgreSQL ColumnSpec to structured ColumnMetadata.\n * Used during replication to populate the metadata table from upstream schema.\n *\n * Uses the same logic as liteTypeString() and mapPostgresToLiteColumn() via shared helpers.\n */\nexport function pgColumnSpecToMetadata(spec: ColumnSpec): ColumnMetadata {\n  return {\n    upstreamType: spec.dataType,\n    isNotNull: spec.notNull ?? false,\n    isEnum: isEnumColumn(spec),\n    isArray: isArrayColumn(spec),\n    characterMaxLength: spec.characterMaximumLength ?? null,\n  };\n}\n"],"names":["liteTypeString","checkIsArray","checkIsEnum"],"mappings":";;AA2CO,MAAM,+BAA+B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAoBrC,MAAM,oBAAoB;AAAA,EAC/B,OAAO,aAAa,oBAAI,QAAA;AAAA,EAEf;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAED,YAAY,IAAc;AAChC,SAAK,cAAc,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA,KAI7B;AAED,SAAK,cAAc,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAS7B;AAED,SAAK,oBAAoB,GAAG,QAAQ;AAAA;AAAA;AAAA,KAGnC;AAED,SAAK,mBAAmB,GAAG,QAAQ;AAAA;AAAA;AAAA,KAGlC;AAED,SAAK,mBAAmB,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA,KAIlC;AAED,SAAK,iBAAiB,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA,KAIhC;AAED,SAAK,gBAAgB,GAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,KAK/B;AAED,SAAK,gBAAgB,GAAG,QAAQ;AAAA;AAAA;AAAA,KAG/B;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,OAAO,YAAY,IAA+C;AAEhE,UAAM,cAAc,GACjB;AAAA,MACC;AAAA,IAAA,EAED,IAAA;AAEH,QAAI,CAAC,aAAa;AAChB,aAAO;AAAA,IACT;AAEA,QAAI,WAAW,oBAAoB,WAAW,IAAI,EAAE;AACpD,QAAI,CAAC,UAAU;AACb,iBAAW,IAAI,oBAAoB,EAAE;AACrC,0BAAoB,WAAW,IAAI,IAAI,QAAQ;AAAA,IACjD;AACA,WAAO;AAAA,EACT;AAAA,EAEA,OAAO,WAAmB,YAAoB,MAAwB;AACpE,UAAM,WAAW,uBAAuB,IAAI;AAC5C,SAAK,gBAAgB,WAAW,YAAY,QAAQ;AAAA,EACtD;AAAA,EAEA,gBACE,WACA,YACA,UACM;AACN,SAAK,YAAY;AAAA,MACf;AAAA,MACA;AAAA,MACA,SAAS;AAAA,MACT,SAAS,YAAY,IAAI;AAAA,MACzB,SAAS,SAAS,IAAI;AAAA,MACtB,SAAS,UAAU,IAAI;AAAA,MACvB,SAAS,sBAAsB;AAAA,IAAA;AAAA,EAEnC;AAAA,EAEA,OACE,WACA,eACA,eACA,MACM;AACN,UAAM,WAAW,uBAAuB,IAAI;AAC5C,SAAK,YAAY;AAAA,MACf;AAAA,MACA,SAAS;AAAA,MACT,SAAS,YAAY,IAAI;AAAA,MACzB,SAAS,SAAS,IAAI;AAAA,MACtB,SAAS,UAAU,IAAI;AAAA,MACvB,SAAS,sBAAsB;AAAA,MAC/B;AAAA,MACA;AAAA,IAAA;AAAA,EAEJ;AAAA,EAEA,aAAa,WAAmB,YAA0B;AACxD,SAAK,kBAAkB,IAAI,WAAW,UAAU;AAAA,EAClD;AAAA,EAEA,YAAY,WAAyB;AACnC,SAAK,iBAAiB,IAAI,SAAS;AAAA,EACrC;AAAA,EAEA,YAAY,cAAsB,cAA4B;AAC5D,SAAK,iBAAiB,IAAI,cAAc,YAAY;AAAA,EACtD;AAAA,EAEA,UAAU,WAAmB,YAAgD;AAC3E,UAAM,MAAM,KAAK,eAAe,IAAI,WAAW,UAAU;AAIzD,QAAI,CAAC,KAAK;AACR,aAAO;AAAA,IACT;AAEA,WAAO;AAAA,MACL,cAAc,IAAI;AAAA,MAClB,WAAW,IAAI,gBAAgB;AAAA,MAC/B,QAAQ,IAAI,YAAY;AAAA,MACxB,SAAS,IAAI,aAAa;AAAA,MAC1B,oBAAoB,IAAI;AAAA,IAAA;AAAA,EAE5B;AAAA,EAEA,SAAS,WAAgD;AACvD,UAAM,OAAO,KAAK,cAAc,IAAI,SAAS;AAI7C,UAAM,+BAAe,IAAA;AACrB,eAAW,OAAO,MAAM;AACtB,eAAS,IAAI,IAAI,aAAa;AAAA,QAC5B,cAAc,IAAI;AAAA,QAClB,WAAW,IAAI,gBAAgB;AAAA,QAC/B,QAAQ,IAAI,YAAY;AAAA,QACxB,SAAS,IAAI,aAAa;AAAA,QAC1B,oBAAoB,IAAI;AAAA,MAAA,CACzB;AAAA,IACH;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,WAAoB;AAClB,UAAM,SAAS,KAAK,cAAc,IAAA;AAClC,WAAO,WAAW;AAAA,EACpB;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,2BAA2B,QAA+B;AACxD,eAAW,SAAS,QAAQ;AAC1B,iBAAW,CAAC,YAAY,UAAU,KAAK,OAAO,QAAQ,MAAM,OAAO,GAAG;AACpE,cAAM,WAAW;AAAA,UACf,WAAW;AAAA,UACX,WAAW;AAAA,QAAA;AAEb,aAAK,gBAAgB,MAAM,MAAM,YAAY,QAAQ;AAAA,MACvD;AAAA,IACF;AAAA,EACF;AACF;AAMO,SAAS,yBACdA,iBACA,oBACgB;AAChB,QAAM,WAAW,iBAAiBA,eAAc;AAChD,QAAM,cAAcC,QAAaD,eAAc;AAK/C,QAAM,mBACJ,eAAe,CAAC,SAAS,SAAS,IAAI,IAAI,GAAG,QAAQ,OAAO;AAE9D,SAAO;AAAA,IACL,cAAc;AAAA,IACd,WAAW,CAAC,iBAAiBA,eAAc;AAAA,IAC3C,QAAQE,OAAYF,eAAc;AAAA,IAClC,SAAS;AAAA,IACT,oBAAoB,sBAAsB;AAAA,EAAA;AAE9C;AAMO,SAAS,yBAAyB,UAAkC;AACzE,SAAO;AAAA,IACL,SAAS;AAAA,IACT,SAAS;AAAA,IACT,SAAS;AAAA,IACT,SAAS;AAAA,EAAA;AAEb;AAQO,SAAS,uBAAuB,MAAkC;AACvE,SAAO;AAAA,IACL,cAAc,KAAK;AAAA,IACnB,WAAW,KAAK,WAAW;AAAA,IAC3B,QAAQ,aAAa,IAAI;AAAA,IACzB,SAAS,cAAc,IAAI;AAAA,IAC3B,oBAAoB,KAAK,0BAA0B;AAAA,EAAA;AAEvD;"}

package/out/zero-cache/src/services/replicator/schema/replication-state.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"replication-state.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/replicator/schema/replication-state.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AACzD,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAC9D,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,2BAA2B,CAAC;AAC/D,OAAO,EAAC,wBAAwB,EAAC,MAAM,gBAAgB,CAAC;AAExD,OAAO,EAAC,wBAAwB,EAAC,CAAC;AAElC,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,SAAS,GAAG,QAAQ,CAAC;AAMzD,eAAO,MAAM,2BAA2B,iKAKvC,CAAC;AA+BF,QAAA,MAAM,uBAAuB;;;;EASxB,CAAC;AAEN,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,QAAA,MAAM,sBAAsB;;aAE1B,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,wBAAgB,oBAAoB,CAClC,EAAE,EAAE,QAAQ,EACZ,YAAY,EAAE,MAAM,EAAE,EACtB,SAAS,EAAE,MAAM,QAelB;AAED,wBAAgB,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,QAM5D;AAED,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,QAAQ;;;IAY9C;AAED,wBAAgB,oBAAoB,CAAC,EAAE,EAAE,eAAe,GAAG,iBAAiB,CAU3E;AAED,wBAAgB,0BAA0B,CACxC,EAAE,EAAE,eAAe,EACnB,SAAS,EAAE,MAAM,QAGlB;AAED,wBAAgB,mBAAmB,CAAC,EAAE,EAAE,eAAe,GAAG,gBAAgB,CAGzE"}
+{"version":3,"file":"replication-state.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/replicator/schema/replication-state.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AACzD,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAC9D,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,2BAA2B,CAAC;AAG/D,OAAO,EAAC,wBAAwB,EAAC,MAAM,gBAAgB,CAAC;AAExD,OAAO,EAAC,wBAAwB,EAAC,CAAC;AAElC,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,SAAS,GAAG,QAAQ,CAAC;AAMzD,eAAO,MAAM,2BAA2B,iKAKvC,CAAC;AAiCF,QAAA,MAAM,uBAAuB;;;;EASxB,CAAC;AAEN,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,QAAA,MAAM,sBAAsB;;aAE1B,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,wBAAgB,oBAAoB,CAClC,EAAE,EAAE,QAAQ,EACZ,YAAY,EAAE,MAAM,EAAE,EACtB,SAAS,EAAE,MAAM,QAelB;AAED,wBAAgB,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,QAM5D;AAED,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,QAAQ;;;IAY9C;AAED,wBAAgB,oBAAoB,CAAC,EAAE,EAAE,eAAe,GAAG,iBAAiB,CAU3E;AAED,wBAAgB,0BAA0B,CACxC,EAAE,EAAE,eAAe,EACnB,SAAS,EAAE,MAAM,QAGlB;AAED,wBAAgB,mBAAmB,CAAC,EAAE,EAAE,eAAe,GAAG,gBAAgB,CAGzE"}

package/out/zero-cache/src/services/replicator/schema/replication-state.js

@@ -1,4 +1,6 @@
 import { parse } from "../../../../../shared/src/valita.js";
+import { CREATE_CHANGELOG_SCHEMA } from "./change-log.js";
+import { CREATE_COLUMN_METADATA_TABLE } from "./column-metadata.js";
 import { array, string, object } from "@badrap/valita";
 const CREATE_RUNTIME_EVENTS_TABLE = `
   CREATE TABLE "_zero.runtimeEvents" (
@@ -24,7 +26,7 @@ const CREATE_REPLICATION_STATE_SCHEMA = (
     stateVersion TEXT NOT NULL,
     lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)
   );
-  ` + CREATE_RUNTIME_EVENTS_TABLE
+  ` + CREATE_CHANGELOG_SCHEMA + CREATE_RUNTIME_EVENTS_TABLE + CREATE_COLUMN_METADATA_TABLE
 );
 const stringArray = array(string());
 const subscriptionStateSchema = object({

package/out/zero-cache/src/services/replicator/schema/replication-state.js.map

@@ -1 +1 @@
-{"version":3,"file":"replication-state.js","sources":["../../../../../../../zero-cache/src/services/replicator/schema/replication-state.ts"],"sourcesContent":["/**\n * Replication metadata, used for incremental view maintenance and catchup.\n *\n * These tables are created atomically in {@link setupReplicationTables}\n * after the logical replication handoff when initial data synchronization has completed.\n */\n\nimport * as v from '../../../../../shared/src/valita.ts';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport type {StatementRunner} from '../../../db/statements.ts';\nimport {ZERO_VERSION_COLUMN_NAME} from './constants.ts';\n\nexport {ZERO_VERSION_COLUMN_NAME};\n\nexport type RuntimeEvent = 'sync' | 'upgrade' | 'vacuum';\n\n// event     : The RuntimeEvent. Only one row per event is tracked.\n//             Inserting an event will REPLACE any row for the same event.\n// timestamp : SQLite timestamp string, e.g. \"2024-04-12 11:37:46\".\n//             Append a `Z` when parsing with `new Date(...)`;\nexport const CREATE_RUNTIME_EVENTS_TABLE = `\n  CREATE TABLE \"_zero.runtimeEvents\" (\n    event TEXT PRIMARY KEY ON CONFLICT REPLACE,\n    timestamp TEXT NOT NULL DEFAULT (current_timestamp)\n  );\n`;\n\nconst CREATE_REPLICATION_STATE_SCHEMA =\n  // replicaVersion   : A value identifying the version at which the initial sync happened, i.e.\n  //                    the version at which all rows were copied, and to `_0_version` was set.\n  //                    This value is used to distinguish data from other replicas (e.g. if a\n  //                    replica is reset or if there are ever multiple replicas).\n  // publications     : JSON stringified array of publication names\n  // lock             : Auto-magic column for enforcing single-row semantics.\n  `\n  CREATE TABLE \"_zero.replicationConfig\" (\n    replicaVersion TEXT NOT NULL,\n    publications TEXT NOT NULL,\n    lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n  ` +\n  // stateVersion     : The latest version replicated from upstream, starting with the initial\n  //                    `replicaVersion` and moving forward to each subsequent commit watermark\n  //                    (e.g. corresponding to a Postgres LSN). Versions are represented as\n  //                    lexicographically sortable watermarks (e.g. LexiVersions).\n  //\n  `\n  CREATE TABLE \"_zero.replicationState\" (\n    stateVersion TEXT NOT NULL,\n    lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n  ` +\n  CREATE_RUNTIME_EVENTS_TABLE;\n\nconst stringArray = v.array(v.string());\n\nconst subscriptionStateSchema = v\n  .object({\n    replicaVersion: v.string(),\n    publications: v.string(),\n    watermark: v.string(),\n  })\n  .map(s => ({\n    ...s,\n    publications: v.parse(JSON.parse(s.publications), stringArray),\n  }));\n\nexport type SubscriptionState = v.Infer<typeof subscriptionStateSchema>;\n\nconst replicationStateSchema = v.object({\n  stateVersion: v.string(),\n});\n\nexport type ReplicationState = v.Infer<typeof replicationStateSchema>;\n\nexport function initReplicationState(\n  db: Database,\n  publications: string[],\n  watermark: string,\n) {\n  db.exec(CREATE_REPLICATION_STATE_SCHEMA);\n  db.prepare(\n    `\n    INSERT INTO \"_zero.replicationConfig\" \n       (replicaVersion, publications) VALUES (?, ?)\n    `,\n  ).run(watermark, JSON.stringify(publications.sort()));\n  db.prepare(\n    `\n    INSERT INTO \"_zero.replicationState\" (stateVersion) VALUES (?)\n    `,\n  ).run(watermark);\n  recordEvent(db, 'sync');\n}\n\nexport function recordEvent(db: Database, event: RuntimeEvent) {\n  db.prepare(\n    `\n    INSERT INTO \"_zero.runtimeEvents\" (event) VALUES (?) \n    `,\n  ).run(event);\n}\n\nexport function getAscendingEvents(db: Database) {\n  const result = db\n    .prepare(\n      `SELECT event, timestamp FROM \"_zero.runtimeEvents\" \n         ORDER BY timestamp ASC\n    `,\n    )\n    .all<{event: string; timestamp: string}>();\n  return result.map(({event, timestamp}) => ({\n    event,\n    timestamp: new Date(timestamp + 'Z'),\n  }));\n}\n\nexport function getSubscriptionState(db: StatementRunner): SubscriptionState {\n  const result = db.get(\n    `\n      SELECT c.replicaVersion, c.publications, s.stateVersion as watermark\n        FROM \"_zero.replicationConfig\" as c\n        JOIN \"_zero.replicationState\" as s\n        ON c.lock = s.lock\n    `,\n  );\n  return v.parse(result, subscriptionStateSchema);\n}\n\nexport function updateReplicationWatermark(\n  db: StatementRunner,\n  watermark: string,\n) {\n  db.run(`UPDATE \"_zero.replicationState\" SET stateVersion=?`, watermark);\n}\n\nexport function getReplicationState(db: StatementRunner): ReplicationState {\n  const result = db.get(`SELECT stateVersion FROM \"_zero.replicationState\"`);\n  return v.parse(result, replicationStateSchema);\n}\n"],"names":["v.array","v.string","v.object","v.parse"],"mappings":";;AAoBO,MAAM,8BAA8B;AAAA;AAAA;AAAA;AAAA;AAAA;AAO3C,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOJ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAkBA;AAAA;AAEF,MAAM,cAAcA,MAAQC,QAAU;AAEtC,MAAM,0BAA0BC,OACtB;AAAA,EACN,gBAAgBD,OAAE;AAAA,EAClB,cAAcA,OAAE;AAAA,EAChB,WAAWA,OAAE;AACf,CAAC,EACA,IAAI,CAAA,OAAM;AAAA,EACT,GAAG;AAAA,EACH,cAAcE,MAAQ,KAAK,MAAM,EAAE,YAAY,GAAG,WAAW;AAC/D,EAAE;AAIJ,MAAM,yBAAyBD,OAAS;AAAA,EACtC,cAAcD,OAAE;AAClB,CAAC;AAIM,SAAS,qBACd,IACA,cACA,WACA;AACA,KAAG,KAAK,+BAA+B;AACvC,KAAG;AAAA,IACD;AAAA;AAAA;AAAA;AAAA,EAAA,EAIA,IAAI,WAAW,KAAK,UAAU,aAAa,KAAA,CAAM,CAAC;AACpD,KAAG;AAAA,IACD;AAAA;AAAA;AAAA,EAAA,EAGA,IAAI,SAAS;AACf,cAAY,IAAI,MAAM;AACxB;AAEO,SAAS,YAAY,IAAc,OAAqB;AAC7D,KAAG;AAAA,IACD;AAAA;AAAA;AAAA,EAAA,EAGA,IAAI,KAAK;AACb;AAEO,SAAS,mBAAmB,IAAc;AAC/C,QAAM,SAAS,GACZ;AAAA,IACC;AAAA;AAAA;AAAA,EAAA,EAID,IAAA;AACH,SAAO,OAAO,IAAI,CAAC,EAAC,OAAO,iBAAgB;AAAA,IACzC;AAAA,IACA,WAAW,oBAAI,KAAK,YAAY,GAAG;AAAA,EAAA,EACnC;AACJ;AAEO,SAAS,qBAAqB,IAAwC;AAC3E,QAAM,SAAS,GAAG;AAAA,IAChB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAAA;AAOF,SAAOE,MAAQ,QAAQ,uBAAuB;AAChD;AAEO,SAAS,2BACd,IACA,WACA;AACA,KAAG,IAAI,sDAAsD,SAAS;AACxE;AAEO,SAAS,oBAAoB,IAAuC;AACzE,QAAM,SAAS,GAAG,IAAI,mDAAmD;AACzE,SAAOA,MAAQ,QAAQ,sBAAsB;AAC/C;"}
+{"version":3,"file":"replication-state.js","sources":["../../../../../../../zero-cache/src/services/replicator/schema/replication-state.ts"],"sourcesContent":["/**\n * Replication metadata, used for incremental view maintenance and catchup.\n *\n * These tables are created atomically in {@link setupReplicationTables}\n * after the logical replication handoff when initial data synchronization has completed.\n */\n\nimport * as v from '../../../../../shared/src/valita.ts';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport type {StatementRunner} from '../../../db/statements.ts';\nimport {CREATE_CHANGELOG_SCHEMA} from './change-log.ts';\nimport {CREATE_COLUMN_METADATA_TABLE} from './column-metadata.ts';\nimport {ZERO_VERSION_COLUMN_NAME} from './constants.ts';\n\nexport {ZERO_VERSION_COLUMN_NAME};\n\nexport type RuntimeEvent = 'sync' | 'upgrade' | 'vacuum';\n\n// event     : The RuntimeEvent. Only one row per event is tracked.\n//             Inserting an event will REPLACE any row for the same event.\n// timestamp : SQLite timestamp string, e.g. \"2024-04-12 11:37:46\".\n//             Append a `Z` when parsing with `new Date(...)`;\nexport const CREATE_RUNTIME_EVENTS_TABLE = `\n  CREATE TABLE \"_zero.runtimeEvents\" (\n    event TEXT PRIMARY KEY ON CONFLICT REPLACE,\n    timestamp TEXT NOT NULL DEFAULT (current_timestamp)\n  );\n`;\n\nconst CREATE_REPLICATION_STATE_SCHEMA =\n  // replicaVersion   : A value identifying the version at which the initial sync happened, i.e.\n  //                    the version at which all rows were copied, and to `_0_version` was set.\n  //                    This value is used to distinguish data from other replicas (e.g. if a\n  //                    replica is reset or if there are ever multiple replicas).\n  // publications     : JSON stringified array of publication names\n  // lock             : Auto-magic column for enforcing single-row semantics.\n  `\n  CREATE TABLE \"_zero.replicationConfig\" (\n    replicaVersion TEXT NOT NULL,\n    publications TEXT NOT NULL,\n    lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n  ` +\n  // stateVersion     : The latest version replicated from upstream, starting with the initial\n  //                    `replicaVersion` and moving forward to each subsequent commit watermark\n  //                    (e.g. corresponding to a Postgres LSN). Versions are represented as\n  //                    lexicographically sortable watermarks (e.g. LexiVersions).\n  //\n  `\n  CREATE TABLE \"_zero.replicationState\" (\n    stateVersion TEXT NOT NULL,\n    lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n  ` +\n  CREATE_CHANGELOG_SCHEMA +\n  CREATE_RUNTIME_EVENTS_TABLE +\n  CREATE_COLUMN_METADATA_TABLE;\n\nconst stringArray = v.array(v.string());\n\nconst subscriptionStateSchema = v\n  .object({\n    replicaVersion: v.string(),\n    publications: v.string(),\n    watermark: v.string(),\n  })\n  .map(s => ({\n    ...s,\n    publications: v.parse(JSON.parse(s.publications), stringArray),\n  }));\n\nexport type SubscriptionState = v.Infer<typeof subscriptionStateSchema>;\n\nconst replicationStateSchema = v.object({\n  stateVersion: v.string(),\n});\n\nexport type ReplicationState = v.Infer<typeof replicationStateSchema>;\n\nexport function initReplicationState(\n  db: Database,\n  publications: string[],\n  watermark: string,\n) {\n  db.exec(CREATE_REPLICATION_STATE_SCHEMA);\n  db.prepare(\n    `\n    INSERT INTO \"_zero.replicationConfig\" \n       (replicaVersion, publications) VALUES (?, ?)\n    `,\n  ).run(watermark, JSON.stringify(publications.sort()));\n  db.prepare(\n    `\n    INSERT INTO \"_zero.replicationState\" (stateVersion) VALUES (?)\n    `,\n  ).run(watermark);\n  recordEvent(db, 'sync');\n}\n\nexport function recordEvent(db: Database, event: RuntimeEvent) {\n  db.prepare(\n    `\n    INSERT INTO \"_zero.runtimeEvents\" (event) VALUES (?) \n    `,\n  ).run(event);\n}\n\nexport function getAscendingEvents(db: Database) {\n  const result = db\n    .prepare(\n      `SELECT event, timestamp FROM \"_zero.runtimeEvents\" \n         ORDER BY timestamp ASC\n    `,\n    )\n    .all<{event: string; timestamp: string}>();\n  return result.map(({event, timestamp}) => ({\n    event,\n    timestamp: new Date(timestamp + 'Z'),\n  }));\n}\n\nexport function getSubscriptionState(db: StatementRunner): SubscriptionState {\n  const result = db.get(\n    `\n      SELECT c.replicaVersion, c.publications, s.stateVersion as watermark\n        FROM \"_zero.replicationConfig\" as c\n        JOIN \"_zero.replicationState\" as s\n        ON c.lock = s.lock\n    `,\n  );\n  return v.parse(result, subscriptionStateSchema);\n}\n\nexport function updateReplicationWatermark(\n  db: StatementRunner,\n  watermark: string,\n) {\n  db.run(`UPDATE \"_zero.replicationState\" SET stateVersion=?`, watermark);\n}\n\nexport function getReplicationState(db: StatementRunner): ReplicationState {\n  const result = db.get(`SELECT stateVersion FROM \"_zero.replicationState\"`);\n  return v.parse(result, replicationStateSchema);\n}\n"],"names":["v.array","v.string","v.object","v.parse"],"mappings":";;;;AAsBO,MAAM,8BAA8B;AAAA;AAAA;AAAA;AAAA;AAAA;AAO3C,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOJ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAkBA,0BACA,8BACA;AAAA;AAEF,MAAM,cAAcA,MAAQC,QAAU;AAEtC,MAAM,0BAA0BC,OACtB;AAAA,EACN,gBAAgBD,OAAE;AAAA,EAClB,cAAcA,OAAE;AAAA,EAChB,WAAWA,OAAE;AACf,CAAC,EACA,IAAI,CAAA,OAAM;AAAA,EACT,GAAG;AAAA,EACH,cAAcE,MAAQ,KAAK,MAAM,EAAE,YAAY,GAAG,WAAW;AAC/D,EAAE;AAIJ,MAAM,yBAAyBD,OAAS;AAAA,EACtC,cAAcD,OAAE;AAClB,CAAC;AAIM,SAAS,qBACd,IACA,cACA,WACA;AACA,KAAG,KAAK,+BAA+B;AACvC,KAAG;AAAA,IACD;AAAA;AAAA;AAAA;AAAA,EAAA,EAIA,IAAI,WAAW,KAAK,UAAU,aAAa,KAAA,CAAM,CAAC;AACpD,KAAG;AAAA,IACD;AAAA;AAAA;AAAA,EAAA,EAGA,IAAI,SAAS;AACf,cAAY,IAAI,MAAM;AACxB;AAEO,SAAS,YAAY,IAAc,OAAqB;AAC7D,KAAG;AAAA,IACD;AAAA;AAAA;AAAA,EAAA,EAGA,IAAI,KAAK;AACb;AAEO,SAAS,mBAAmB,IAAc;AAC/C,QAAM,SAAS,GACZ;AAAA,IACC;AAAA;AAAA;AAAA,EAAA,EAID,IAAA;AACH,SAAO,OAAO,IAAI,CAAC,EAAC,OAAO,iBAAgB;AAAA,IACzC;AAAA,IACA,WAAW,oBAAI,KAAK,YAAY,GAAG;AAAA,EAAA,EACnC;AACJ;AAEO,SAAS,qBAAqB,IAAwC;AAC3E,QAAM,SAAS,GAAG;AAAA,IAChB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAAA;AAOF,SAAOE,MAAQ,QAAQ,uBAAuB;AAChD;AAEO,SAAS,2BACd,IACA,WACA;AACA,KAAG,IAAI,sDAAsD,SAAS;AACxE;AAEO,SAAS,oBAAoB,IAAuC;AACzE,QAAM,SAAS,GAAG,IAAI,mDAAmD;AACzE,SAAOA,MAAQ,QAAQ,sBAAsB;AAC/C;"}

package/out/zero-cache/src/services/run-ast.js

@@ -56,7 +56,7 @@ async function runAst(lc, clientSchema, ast, isTransformed, options, yieldProces
       await yieldProcess();
       continue;
     }
-    assert(rowChange.type === "add");
+    assert(rowChange.type === "add", "Hydration only handles add row changes");
     if (syncedRowCount % 10 === 0) {
       await Promise.resolve();
     }

package/out/zero-cache/src/services/run-ast.js.map

@@ -1 +1 @@
-{"version":3,"file":"run-ast.js","sources":["../../../../../zero-cache/src/services/run-ast.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\n// @circular-dep-ignore\nimport {astToZQL} from '../../../ast-to-zql/src/ast-to-zql.ts';\n// @circular-dep-ignore\nimport {formatOutput} from '../../../ast-to-zql/src/format.ts';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport type {AnalyzeQueryResult} from '../../../zero-protocol/src/analyze-query-result.ts';\nimport type {AST} from '../../../zero-protocol/src/ast.ts';\nimport {mapAST} from '../../../zero-protocol/src/ast.ts';\nimport type {Row} from '../../../zero-protocol/src/data.ts';\nimport {hashOfAST} from '../../../zero-protocol/src/query-hash.ts';\nimport type {PermissionsConfig} from '../../../zero-schema/src/compiled-permissions.ts';\nimport type {NameMapper} from '../../../zero-schema/src/name-mapper.ts';\nimport {\n  buildPipeline,\n  type BuilderDelegate,\n} from '../../../zql/src/builder/builder.ts';\nimport type {PlanDebugger} from '../../../zql/src/planner/planner-debug.ts';\nimport type {ConnectionCostModel} from '../../../zql/src/planner/planner-connection.ts';\nimport type {Database} from '../../../zqlite/src/db.ts';\nimport {transformAndHashQuery} from '../auth/read-authorizer.ts';\nimport type {LiteAndZqlSpec} from '../db/specs.ts';\nimport {hydrate} from './view-syncer/pipeline-driver.ts';\nimport type {TokenData} from './view-syncer/view-syncer.ts';\nimport type {ClientSchema} from '../../../zero-protocol/src/client-schema.ts';\n\nexport type RunAstOptions = {\n  applyPermissions?: boolean | undefined;\n  authData?: TokenData | undefined;\n  clientToServerMapper?: NameMapper | undefined;\n  costModel?: ConnectionCostModel | undefined;\n  db: Database;\n  host: BuilderDelegate;\n  permissions?: PermissionsConfig | undefined;\n  planDebugger?: PlanDebugger | undefined;\n  syncedRows?: boolean | undefined;\n  tableSpecs: Map<string, LiteAndZqlSpec>;\n  vendedRows?: boolean | undefined;\n};\n\nexport async function runAst(\n  lc: LogContext,\n  clientSchema: ClientSchema,\n  ast: AST,\n  isTransformed: boolean,\n  options: RunAstOptions,\n  yieldProcess: () => Promise<void>,\n): Promise<AnalyzeQueryResult> {\n  const {clientToServerMapper, permissions, host} = options;\n  const result: AnalyzeQueryResult = {\n    warnings: [],\n    syncedRows: undefined,\n    syncedRowCount: 0,\n    start: 0,\n    end: 0,\n    elapsed: 0,\n    afterPermissions: undefined,\n    readRows: undefined,\n    readRowCountsByQuery: {},\n    readRowCount: undefined,\n  };\n\n  if (!isTransformed) {\n    // map the AST to server names if not already transformed\n    ast = mapAST(ast, must(clientToServerMapper));\n  }\n  if (options.applyPermissions) {\n    const authData = options.authData?.decoded;\n    if (!authData) {\n      result.warnings.push(\n        'No auth data provided. Permission rules will compare to `NULL` wherever an auth data field is referenced.',\n      );\n    }\n    ast = transformAndHashQuery(\n      lc,\n      'clientGroupIDForAnalyze',\n      ast,\n      must(permissions),\n      authData,\n      false,\n    ).transformedAst;\n    result.afterPermissions = await formatOutput(ast.table + astToZQL(ast));\n  }\n\n  const pipeline = buildPipeline(\n    ast,\n    host,\n    'query-id',\n    options.costModel,\n    lc,\n    options.planDebugger,\n  );\n\n  const start = performance.now();\n\n  let syncedRowCount = 0;\n  const rowsByTable: Record<string, Row[]> = {};\n  const seenByTable: Set<string> = new Set();\n  for (const rowChange of hydrate(pipeline, hashOfAST(ast), clientSchema)) {\n    if (rowChange === 'yield') {\n      await yieldProcess();\n      continue;\n    }\n    assert(rowChange.type === 'add');\n\n    // yield to other tasks to avoid blocking for too long\n    if (syncedRowCount % 10 === 0) {\n      await Promise.resolve();\n    }\n    if (syncedRowCount % 100 === 0) {\n      await sleep(1);\n    }\n\n    let rows: Row[] = rowsByTable[rowChange.table];\n    const s = rowChange.table + '.' + JSON.stringify(rowChange.row);\n    if (seenByTable.has(s)) {\n      continue; // skip duplicates\n    }\n    syncedRowCount++;\n    seenByTable.add(s);\n    if (options.syncedRows) {\n      if (!rows) {\n        rows = [];\n        rowsByTable[rowChange.table] = rows;\n      }\n      rows.push(rowChange.row);\n    }\n  }\n\n  const end = performance.now();\n  if (options.syncedRows) {\n    result.syncedRows = rowsByTable;\n  }\n  result.start = start;\n  result.end = end;\n  result.elapsed = end - start;\n\n  // Always include the count of synced and vended rows.\n  result.syncedRowCount = syncedRowCount;\n  result.readRowCountsByQuery = host.debug?.getVendedRowCounts() ?? {};\n  let readRowCount = 0;\n  for (const c of Object.values(result.readRowCountsByQuery)) {\n    for (const v of Object.values(c)) {\n      readRowCount += v;\n    }\n  }\n  result.readRowCount = readRowCount;\n  result.dbScansByQuery = host.debug?.getNVisitCounts() ?? {};\n\n  if (options.vendedRows) {\n    result.readRows = host.debug?.getVendedRows();\n  }\n  return result;\n}\n"],"names":[],"mappings":";;;;;;;;;;AA0CA,eAAsB,OACpB,IACA,cACA,KACA,eACA,SACA,cAC6B;AAC7B,QAAM,EAAuB,aAAa,KAAA,IAAQ;AAClD,QAAM,SAA6B;AAAA,IACjC,UAAU,CAAA;AAAA,IACV,YAAY;AAAA,IACZ,gBAAgB;AAAA,IAChB,OAAO;AAAA,IACP,KAAK;AAAA,IACL,SAAS;AAAA,IACT,kBAAkB;AAAA,IAClB,UAAU;AAAA,IACV,sBAAsB,CAAA;AAAA,IACtB,cAAc;AAAA,EAAA;AAOhB,MAAI,QAAQ,kBAAkB;AAC5B,UAAM,WAAW,QAAQ,UAAU;AACnC,QAAI,CAAC,UAAU;AACb,aAAO,SAAS;AAAA,QACd;AAAA,MAAA;AAAA,IAEJ;AACA,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA,KAAK,WAAW;AAAA,MAChB;AAAA,MACA;AAAA,IAAA,EACA;AACF,WAAO,mBAAmB,MAAM,aAAa,IAAI,QAAQ,SAAS,GAAG,CAAC;AAAA,EACxE;AAEA,QAAM,WAAW;AAAA,IACf;AAAA,IACA;AAAA,IACA;AAAA,IACA,QAAQ;AAAA,IACR;AAAA,IACA,QAAQ;AAAA,EAAA;AAGV,QAAM,QAAQ,YAAY,IAAA;AAE1B,MAAI,iBAAiB;AACrB,QAAM,cAAqC,CAAA;AAC3C,QAAM,kCAA+B,IAAA;AACrC,aAAW,aAAa,QAAQ,UAAU,UAAU,GAAG,GAAG,YAAY,GAAG;AACvE,QAAI,cAAc,SAAS;AACzB,YAAM,aAAA;AACN;AAAA,IACF;AACA,WAAO,UAAU,SAAS,KAAK;AAG/B,QAAI,iBAAiB,OAAO,GAAG;AAC7B,YAAM,QAAQ,QAAA;AAAA,IAChB;AACA,QAAI,iBAAiB,QAAQ,GAAG;AAC9B,YAAM,MAAM,CAAC;AAAA,IACf;AAEA,QAAI,OAAc,YAAY,UAAU,KAAK;AAC7C,UAAM,IAAI,UAAU,QAAQ,MAAM,KAAK,UAAU,UAAU,GAAG;AAC9D,QAAI,YAAY,IAAI,CAAC,GAAG;AACtB;AAAA,IACF;AACA;AACA,gBAAY,IAAI,CAAC;AACjB,QAAI,QAAQ,YAAY;AACtB,UAAI,CAAC,MAAM;AACT,eAAO,CAAA;AACP,oBAAY,UAAU,KAAK,IAAI;AAAA,MACjC;AACA,WAAK,KAAK,UAAU,GAAG;AAAA,IACzB;AAAA,EACF;AAEA,QAAM,MAAM,YAAY,IAAA;AACxB,MAAI,QAAQ,YAAY;AACtB,WAAO,aAAa;AAAA,EACtB;AACA,SAAO,QAAQ;AACf,SAAO,MAAM;AACb,SAAO,UAAU,MAAM;AAGvB,SAAO,iBAAiB;AACxB,SAAO,uBAAuB,KAAK,OAAO,mBAAA,KAAwB,CAAA;AAClE,MAAI,eAAe;AACnB,aAAW,KAAK,OAAO,OAAO,OAAO,oBAAoB,GAAG;AAC1D,eAAW,KAAK,OAAO,OAAO,CAAC,GAAG;AAChC,sBAAgB;AAAA,IAClB;AAAA,EACF;AACA,SAAO,eAAe;AACtB,SAAO,iBAAiB,KAAK,OAAO,gBAAA,KAAqB,CAAA;AAEzD,MAAI,QAAQ,YAAY;AACtB,WAAO,WAAW,KAAK,OAAO,cAAA;AAAA,EAChC;AACA,SAAO;AACT;"}
+{"version":3,"file":"run-ast.js","sources":["../../../../../zero-cache/src/services/run-ast.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\n// @circular-dep-ignore\nimport {astToZQL} from '../../../ast-to-zql/src/ast-to-zql.ts';\n// @circular-dep-ignore\nimport {formatOutput} from '../../../ast-to-zql/src/format.ts';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport type {AnalyzeQueryResult} from '../../../zero-protocol/src/analyze-query-result.ts';\nimport type {AST} from '../../../zero-protocol/src/ast.ts';\nimport {mapAST} from '../../../zero-protocol/src/ast.ts';\nimport type {Row} from '../../../zero-protocol/src/data.ts';\nimport {hashOfAST} from '../../../zero-protocol/src/query-hash.ts';\nimport type {PermissionsConfig} from '../../../zero-schema/src/compiled-permissions.ts';\nimport type {NameMapper} from '../../../zero-schema/src/name-mapper.ts';\nimport {\n  buildPipeline,\n  type BuilderDelegate,\n} from '../../../zql/src/builder/builder.ts';\nimport type {PlanDebugger} from '../../../zql/src/planner/planner-debug.ts';\nimport type {ConnectionCostModel} from '../../../zql/src/planner/planner-connection.ts';\nimport type {Database} from '../../../zqlite/src/db.ts';\nimport {transformAndHashQuery} from '../auth/read-authorizer.ts';\nimport type {LiteAndZqlSpec} from '../db/specs.ts';\nimport {hydrate} from './view-syncer/pipeline-driver.ts';\nimport type {TokenData} from './view-syncer/view-syncer.ts';\nimport type {ClientSchema} from '../../../zero-protocol/src/client-schema.ts';\n\nexport type RunAstOptions = {\n  applyPermissions?: boolean | undefined;\n  authData?: TokenData | undefined;\n  clientToServerMapper?: NameMapper | undefined;\n  costModel?: ConnectionCostModel | undefined;\n  db: Database;\n  host: BuilderDelegate;\n  permissions?: PermissionsConfig | undefined;\n  planDebugger?: PlanDebugger | undefined;\n  syncedRows?: boolean | undefined;\n  tableSpecs: Map<string, LiteAndZqlSpec>;\n  vendedRows?: boolean | undefined;\n};\n\nexport async function runAst(\n  lc: LogContext,\n  clientSchema: ClientSchema,\n  ast: AST,\n  isTransformed: boolean,\n  options: RunAstOptions,\n  yieldProcess: () => Promise<void>,\n): Promise<AnalyzeQueryResult> {\n  const {clientToServerMapper, permissions, host} = options;\n  const result: AnalyzeQueryResult = {\n    warnings: [],\n    syncedRows: undefined,\n    syncedRowCount: 0,\n    start: 0,\n    end: 0,\n    elapsed: 0,\n    afterPermissions: undefined,\n    readRows: undefined,\n    readRowCountsByQuery: {},\n    readRowCount: undefined,\n  };\n\n  if (!isTransformed) {\n    // map the AST to server names if not already transformed\n    ast = mapAST(ast, must(clientToServerMapper));\n  }\n  if (options.applyPermissions) {\n    const authData = options.authData?.decoded;\n    if (!authData) {\n      result.warnings.push(\n        'No auth data provided. Permission rules will compare to `NULL` wherever an auth data field is referenced.',\n      );\n    }\n    ast = transformAndHashQuery(\n      lc,\n      'clientGroupIDForAnalyze',\n      ast,\n      must(permissions),\n      authData,\n      false,\n    ).transformedAst;\n    result.afterPermissions = await formatOutput(ast.table + astToZQL(ast));\n  }\n\n  const pipeline = buildPipeline(\n    ast,\n    host,\n    'query-id',\n    options.costModel,\n    lc,\n    options.planDebugger,\n  );\n\n  const start = performance.now();\n\n  let syncedRowCount = 0;\n  const rowsByTable: Record<string, Row[]> = {};\n  const seenByTable: Set<string> = new Set();\n  for (const rowChange of hydrate(pipeline, hashOfAST(ast), clientSchema)) {\n    if (rowChange === 'yield') {\n      await yieldProcess();\n      continue;\n    }\n    assert(rowChange.type === 'add', 'Hydration only handles add row changes');\n\n    // yield to other tasks to avoid blocking for too long\n    if (syncedRowCount % 10 === 0) {\n      await Promise.resolve();\n    }\n    if (syncedRowCount % 100 === 0) {\n      await sleep(1);\n    }\n\n    let rows: Row[] = rowsByTable[rowChange.table];\n    const s = rowChange.table + '.' + JSON.stringify(rowChange.row);\n    if (seenByTable.has(s)) {\n      continue; // skip duplicates\n    }\n    syncedRowCount++;\n    seenByTable.add(s);\n    if (options.syncedRows) {\n      if (!rows) {\n        rows = [];\n        rowsByTable[rowChange.table] = rows;\n      }\n      rows.push(rowChange.row);\n    }\n  }\n\n  const end = performance.now();\n  if (options.syncedRows) {\n    result.syncedRows = rowsByTable;\n  }\n  result.start = start;\n  result.end = end;\n  result.elapsed = end - start;\n\n  // Always include the count of synced and vended rows.\n  result.syncedRowCount = syncedRowCount;\n  result.readRowCountsByQuery = host.debug?.getVendedRowCounts() ?? {};\n  let readRowCount = 0;\n  for (const c of Object.values(result.readRowCountsByQuery)) {\n    for (const v of Object.values(c)) {\n      readRowCount += v;\n    }\n  }\n  result.readRowCount = readRowCount;\n  result.dbScansByQuery = host.debug?.getNVisitCounts() ?? {};\n\n  if (options.vendedRows) {\n    result.readRows = host.debug?.getVendedRows();\n  }\n  return result;\n}\n"],"names":[],"mappings":";;;;;;;;;;AA0CA,eAAsB,OACpB,IACA,cACA,KACA,eACA,SACA,cAC6B;AAC7B,QAAM,EAAuB,aAAa,KAAA,IAAQ;AAClD,QAAM,SAA6B;AAAA,IACjC,UAAU,CAAA;AAAA,IACV,YAAY;AAAA,IACZ,gBAAgB;AAAA,IAChB,OAAO;AAAA,IACP,KAAK;AAAA,IACL,SAAS;AAAA,IACT,kBAAkB;AAAA,IAClB,UAAU;AAAA,IACV,sBAAsB,CAAA;AAAA,IACtB,cAAc;AAAA,EAAA;AAOhB,MAAI,QAAQ,kBAAkB;AAC5B,UAAM,WAAW,QAAQ,UAAU;AACnC,QAAI,CAAC,UAAU;AACb,aAAO,SAAS;AAAA,QACd;AAAA,MAAA;AAAA,IAEJ;AACA,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA,KAAK,WAAW;AAAA,MAChB;AAAA,MACA;AAAA,IAAA,EACA;AACF,WAAO,mBAAmB,MAAM,aAAa,IAAI,QAAQ,SAAS,GAAG,CAAC;AAAA,EACxE;AAEA,QAAM,WAAW;AAAA,IACf;AAAA,IACA;AAAA,IACA;AAAA,IACA,QAAQ;AAAA,IACR;AAAA,IACA,QAAQ;AAAA,EAAA;AAGV,QAAM,QAAQ,YAAY,IAAA;AAE1B,MAAI,iBAAiB;AACrB,QAAM,cAAqC,CAAA;AAC3C,QAAM,kCAA+B,IAAA;AACrC,aAAW,aAAa,QAAQ,UAAU,UAAU,GAAG,GAAG,YAAY,GAAG;AACvE,QAAI,cAAc,SAAS;AACzB,YAAM,aAAA;AACN;AAAA,IACF;AACA,WAAO,UAAU,SAAS,OAAO,wCAAwC;AAGzE,QAAI,iBAAiB,OAAO,GAAG;AAC7B,YAAM,QAAQ,QAAA;AAAA,IAChB;AACA,QAAI,iBAAiB,QAAQ,GAAG;AAC9B,YAAM,MAAM,CAAC;AAAA,IACf;AAEA,QAAI,OAAc,YAAY,UAAU,KAAK;AAC7C,UAAM,IAAI,UAAU,QAAQ,MAAM,KAAK,UAAU,UAAU,GAAG;AAC9D,QAAI,YAAY,IAAI,CAAC,GAAG;AACtB;AAAA,IACF;AACA;AACA,gBAAY,IAAI,CAAC;AACjB,QAAI,QAAQ,YAAY;AACtB,UAAI,CAAC,MAAM;AACT,eAAO,CAAA;AACP,oBAAY,UAAU,KAAK,IAAI;AAAA,MACjC;AACA,WAAK,KAAK,UAAU,GAAG;AAAA,IACzB;AAAA,EACF;AAEA,QAAM,MAAM,YAAY,IAAA;AACxB,MAAI,QAAQ,YAAY;AACtB,WAAO,aAAa;AAAA,EACtB;AACA,SAAO,QAAQ;AACf,SAAO,MAAM;AACb,SAAO,UAAU,MAAM;AAGvB,SAAO,iBAAiB;AACxB,SAAO,uBAAuB,KAAK,OAAO,mBAAA,KAAwB,CAAA;AAClE,MAAI,eAAe;AACnB,aAAW,KAAK,OAAO,OAAO,OAAO,oBAAoB,GAAG;AAC1D,eAAW,KAAK,OAAO,OAAO,CAAC,GAAG;AAChC,sBAAgB;AAAA,IAClB;AAAA,EACF;AACA,SAAO,eAAe;AACtB,SAAO,iBAAiB,KAAK,OAAO,gBAAA,KAAqB,CAAA;AAEzD,MAAI,QAAQ,YAAY;AACtB,WAAO,WAAW,KAAK,OAAO,cAAA;AAAA,EAChC;AACA,SAAO;AACT;"}

package/out/zero-cache/src/services/statz.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"statz.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/statz.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,SAAS,CAAC;AAM1D,OAAO,KAAK,EAAC,oBAAoB,IAAI,UAAU,EAAC,MAAM,wBAAwB,CAAC;AAsS/E,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,GAAG,EAAE,cAAc,EACnB,GAAG,EAAE,YAAY,iBAqBlB"}
+{"version":3,"file":"statz.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/statz.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,SAAS,CAAC;AAM1D,OAAO,KAAK,EAAC,oBAAoB,IAAI,UAAU,EAAC,MAAM,wBAAwB,CAAC;AAuS/E,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,GAAG,EAAE,cAAc,EACnB,GAAG,EAAE,YAAY,iBAqBlB"}

package/out/zero-cache/src/services/statz.js

@@ -230,6 +230,7 @@ function osStats(out) {
       ["total mem", os.totalmem()],
       ["free mem", os.freemem()],
       ["cpus", os.cpus().length],
+      ["available parallelism", os.availableParallelism()],
       ["platform", os.platform()],
       ["arch", os.arch()],
       ["release", os.release()],

package/out/zero-cache/src/services/statz.js.map

@@ -1 +1 @@
-{"version":3,"file":"statz.js","sources":["../../../../../zero-cache/src/services/statz.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport auth from 'basic-auth';\nimport type {FastifyReply, FastifyRequest} from 'fastify';\nimport fs from 'fs';\nimport os from 'os';\nimport type {Writable} from 'stream';\nimport {BigIntJSON} from '../../../shared/src/bigint-json.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport type {NormalizedZeroConfig as ZeroConfig} from '../config/normalize.ts';\nimport {isAdminPasswordValid} from '../config/zero-config.ts';\nimport {pgClient} from '../types/pg.ts';\nimport {getShardID, upstreamSchema} from '../types/shards.ts';\n\nasync function upstreamStats(\n  lc: LogContext,\n  config: ZeroConfig,\n  out: Writable,\n) {\n  const schema = upstreamSchema(getShardID(config));\n  const sql = pgClient(lc, config.upstream.db);\n\n  out.write(header('Upstream'));\n\n  await printPgStats(\n    [\n      [\n        'num replicas',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"replicas\"`,\n      ],\n      [\n        'num clients with mutations',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"clients\"`,\n      ],\n      [\n        'num mutations processed',\n        sql`SELECT SUM(\"lastMutationID\") as \"c\" FROM ${sql(schema)}.\"clients\"`,\n      ],\n    ],\n    out,\n  );\n\n  await sql.end();\n}\n\nasync function cvrStats(lc: LogContext, config: ZeroConfig, out: Writable) {\n  out.write(header('CVR'));\n\n  const schema = upstreamSchema(getShardID(config)) + '/cvr';\n  const sql = pgClient(lc, config.cvr.db);\n\n  function numQueriesPerClientGroup(\n    active: boolean,\n  ): ReturnType<ReturnType<typeof pgClient>> {\n    const filter = active\n      ? sql`WHERE \"inactivatedAt\" IS NULL AND deleted = false`\n      : sql`WHERE \"inactivatedAt\" IS NOT NULL AND (\"inactivatedAt\" + \"ttl\") > NOW()`;\n    return sql`WITH\n    group_counts AS (\n      SELECT\n        \"clientGroupID\",\n        COUNT(*) AS num_queries\n      FROM ${sql(schema)}.\"desires\"\n      ${filter}\n      GROUP BY \"clientGroupID\"\n    ),\n    -- Count distinct clientIDs per clientGroupID\n    client_per_group_counts AS (\n      SELECT\n        \"clientGroupID\",\n        COUNT(DISTINCT \"clientID\") AS num_clients\n      FROM ${sql(schema)}.\"desires\"\n      ${filter}\n      GROUP BY \"clientGroupID\"\n    )\n    -- Combine all the information\n    SELECT\n      g.\"clientGroupID\",\n      cpg.num_clients,\n      g.num_queries\n    FROM group_counts g\n    JOIN client_per_group_counts cpg ON g.\"clientGroupID\" = cpg.\"clientGroupID\"\n    ORDER BY g.num_queries DESC;`;\n  }\n\n  await printPgStats(\n    [\n      [\n        'total num queries',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"desires\"`,\n      ],\n      [\n        'num unique query hashes',\n        sql`SELECT COUNT(DISTINCT \"queryHash\") as \"c\" FROM ${sql(\n          schema,\n        )}.\"desires\"`,\n      ],\n      [\n        'num active queries',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"desires\" WHERE \"inactivatedAt\" IS NULL AND \"deleted\" = false`,\n      ],\n      [\n        'num inactive queries',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"desires\" WHERE \"inactivatedAt\" IS NOT NULL AND (\"inactivatedAt\" + \"ttl\") > NOW()`,\n      ],\n      [\n        'num deleted queries',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"desires\" WHERE \"deleted\" = true`,\n      ],\n      [\n        'fresh queries percentiles',\n        sql`WITH client_group_counts AS (\n        -- Count inactive desires per clientGroupID\n        SELECT\n          \"clientGroupID\",\n          COUNT(*) AS fresh_count\n        FROM ${sql(schema)}.\"desires\"\n        WHERE\n          (\"inactivatedAt\" IS NOT NULL\n          AND (\"inactivatedAt\" + \"ttl\") > NOW()) OR (\"inactivatedAt\" IS NULL\n          AND deleted = false)\n        GROUP BY \"clientGroupID\"\n      )\n\n      SELECT\n        percentile_cont(0.50) WITHIN GROUP (ORDER BY fresh_count) AS \"p50\",\n        percentile_cont(0.75) WITHIN GROUP (ORDER BY fresh_count) AS \"p75\",\n        percentile_cont(0.90) WITHIN GROUP (ORDER BY fresh_count) AS \"p90\",\n        percentile_cont(0.95) WITHIN GROUP (ORDER BY fresh_count) AS \"p95\",\n        percentile_cont(0.99) WITHIN GROUP (ORDER BY fresh_count) AS \"p99\",\n        MIN(fresh_count) AS \"min\",\n        MAX(fresh_count) AS \"max\",\n        AVG(fresh_count) AS \"avg\"\n      FROM client_group_counts;`,\n      ],\n      [\n        'rows per client group percentiles',\n        sql`WITH client_group_counts AS (\n        -- Count inactive desires per clientGroupID\n        SELECT\n          \"clientGroupID\",\n          COUNT(*) AS row_count\n        FROM ${sql(schema)}.\"rows\"\n        GROUP BY \"clientGroupID\"\n      )\n      SELECT\n        percentile_cont(0.50) WITHIN GROUP (ORDER BY row_count) AS \"p50\",\n        percentile_cont(0.75) WITHIN GROUP (ORDER BY row_count) AS \"p75\",\n        percentile_cont(0.90) WITHIN GROUP (ORDER BY row_count) AS \"p90\",\n        percentile_cont(0.95) WITHIN GROUP (ORDER BY row_count) AS \"p95\",\n        percentile_cont(0.99) WITHIN GROUP (ORDER BY row_count) AS \"p99\",\n        MIN(row_count) AS \"min\",\n        MAX(row_count) AS \"max\",\n        AVG(row_count) AS \"avg\"\n      FROM client_group_counts;`,\n      ],\n      [\n        // check for AST blowup due to DNF conversion.\n        'ast sizes',\n        sql`SELECT\n        percentile_cont(0.25) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"25th_percentile\",\n        percentile_cont(0.5) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"50th_percentile\",\n        percentile_cont(0.75) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"75th_percentile\",\n        percentile_cont(0.9) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"90th_percentile\",\n        percentile_cont(0.95) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"95th_percentile\",\n        percentile_cont(0.99) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"99th_percentile\",\n        MIN(length(\"clientAST\"::text)) AS \"minimum_length\",\n        MAX(length(\"clientAST\"::text)) AS \"maximum_length\",\n        AVG(length(\"clientAST\"::text))::integer AS \"average_length\",\n        COUNT(*) AS \"total_records\"\n      FROM ${sql(schema)}.\"queries\";`,\n      ],\n      [\n        // output the hash of the largest AST\n        'biggest ast hash',\n        sql`SELECT \"queryHash\", length(\"clientAST\"::text) AS \"ast_length\"\n      FROM ${sql(schema)}.\"queries\"\n      ORDER BY length(\"clientAST\"::text) DESC\n      LIMIT 1;`,\n      ],\n      [\n        'total active queries per client and client group',\n        numQueriesPerClientGroup(true),\n      ],\n      [\n        'total inactive queries per client and client group',\n        numQueriesPerClientGroup(false),\n      ],\n      [\n        'total rows per client group',\n        sql`SELECT \"clientGroupID\", COUNT(*) as \"c\" FROM ${sql(\n          schema,\n        )}.\"rows\" GROUP BY \"clientGroupID\" ORDER BY \"c\" DESC`,\n      ],\n      [\n        'num rows per query',\n        sql`SELECT\n        k.key AS \"queryHash\",\n        COUNT(*) AS row_count\n      FROM ${sql(schema)}.\"rows\" r,\n      LATERAL jsonb_each(r.\"refCounts\") k\n      GROUP BY k.key\n      ORDER BY row_count DESC;`,\n      ],\n    ] satisfies [\n      name: string,\n      query: ReturnType<ReturnType<typeof pgClient>>,\n    ][],\n    out,\n  );\n\n  await sql.end();\n}\n\nasync function changelogStats(\n  lc: LogContext,\n  config: ZeroConfig,\n  out: Writable,\n) {\n  out.write(header('Change DB'));\n  const schema = upstreamSchema(getShardID(config)) + '/cdc';\n  const sql = pgClient(lc, config.change.db);\n\n  await printPgStats(\n    [\n      [\n        'change log size',\n        sql`SELECT COUNT(*) as \"change_log_size\" FROM ${sql(schema)}.\"changeLog\"`,\n      ],\n    ],\n    out,\n  );\n  await sql.end();\n}\n\nfunction replicaStats(lc: LogContext, config: ZeroConfig, out: Writable) {\n  out.write(header('Replica'));\n  const db = new Database(lc, config.replica.file);\n  printStats(\n    'replica',\n    [\n      ['wal checkpoint', pick(first(db.pragma('WAL_CHECKPOINT')))],\n      ['page count', pick(first(db.pragma('PAGE_COUNT')))],\n      ['page size', pick(first(db.pragma('PAGE_SIZE')))],\n      ['journal mode', pick(first(db.pragma('JOURNAL_MODE')))],\n      ['synchronous', pick(first(db.pragma('SYNCHRONOUS')))],\n      ['cache size', pick(first(db.pragma('CACHE_SIZE')))],\n      ['auto vacuum', pick(first(db.pragma('AUTO_VACUUM')))],\n      ['freelist count', pick(first(db.pragma('FREELIST_COUNT')))],\n      ['wal autocheckpoint', pick(first(db.pragma('WAL_AUTOCHECKPOINT')))],\n      ['db file stats', fs.statSync(config.replica.file)],\n    ] as const,\n    out,\n  );\n}\n\nfunction osStats(out: Writable) {\n  printStats(\n    'os',\n    [\n      ['load avg', os.loadavg()],\n      ['uptime', os.uptime()],\n      ['total mem', os.totalmem()],\n      ['free mem', os.freemem()],\n      ['cpus', os.cpus().length],\n      ['platform', os.platform()],\n      ['arch', os.arch()],\n      ['release', os.release()],\n      ['uptime', os.uptime()],\n    ] as const,\n    out,\n  );\n}\n\nasync function printPgStats(\n  pendingQueries: [\n    name: string,\n    query: ReturnType<ReturnType<typeof pgClient>>,\n  ][],\n  out: Writable,\n) {\n  const results = await Promise.all(\n    pendingQueries.map(async ([name, query]) => [name, await query]),\n  );\n  for (const [name, data] of results) {\n    out.write('\\n');\n    out.write(name);\n    out.write('\\n');\n    out.write(BigIntJSON.stringify(data, null, 2));\n  }\n}\n\nfunction printStats(\n  group: string,\n  queries: readonly [name: string, result: unknown][],\n  out: Writable,\n) {\n  out.write('\\n' + header(group));\n  for (const [name, result] of queries) {\n    out.write('\\n' + name + BigIntJSON.stringify(result, null, 2));\n  }\n}\n\nexport async function handleStatzRequest(\n  lc: LogContext,\n  config: ZeroConfig,\n  req: FastifyRequest,\n  res: FastifyReply,\n) {\n  const credentials = auth(req);\n  if (!isAdminPasswordValid(lc, config, credentials?.pass)) {\n    void res\n      .code(401)\n      .header('WWW-Authenticate', 'Basic realm=\"Statz Protected Area\"')\n      .send('Unauthorized');\n    return;\n  }\n\n  await upstreamStats(lc, config, res.raw);\n  res.raw.write('\\n\\n');\n  await cvrStats(lc, config, res.raw);\n  res.raw.write('\\n\\n');\n  await changelogStats(lc, config, res.raw);\n  res.raw.write('\\n\\n');\n  replicaStats(lc, config, res.raw);\n  res.raw.write('\\n\\n');\n  osStats(res.raw);\n  res.raw.end();\n}\n\nfunction first(x: object[]): object {\n  return x[0];\n}\n\nfunction pick(x: object): unknown {\n  return Object.values(x)[0];\n}\n\nfunction header(name: string): string {\n  return `=== ${name} ===\\n`;\n}\n"],"names":[],"mappings":";;;;;;;;AAaA,eAAe,cACb,IACA,QACA,KACA;AACA,QAAM,SAAS,eAAe,WAAW,MAAM,CAAC;AAChD,QAAM,MAAM,SAAS,IAAI,OAAO,SAAS,EAAE;AAE3C,MAAI,MAAM,OAAO,UAAU,CAAC;AAE5B,QAAM;AAAA,IACJ;AAAA,MACE;AAAA,QACE;AAAA,QACA,kCAAkC,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAE/C;AAAA,QACE;AAAA,QACA,kCAAkC,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAE/C;AAAA,QACE;AAAA,QACA,+CAA+C,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,IAC5D;AAAA,IAEF;AAAA,EAAA;AAGF,QAAM,IAAI,IAAA;AACZ;AAEA,eAAe,SAAS,IAAgB,QAAoB,KAAe;AACzE,MAAI,MAAM,OAAO,KAAK,CAAC;AAEvB,QAAM,SAAS,eAAe,WAAW,MAAM,CAAC,IAAI;AACpD,QAAM,MAAM,SAAS,IAAI,OAAO,IAAI,EAAE;AAEtC,WAAS,yBACP,QACyC;AACzC,UAAM,SAAS,SACX,yDACA;AACJ,WAAO;AAAA;AAAA;AAAA;AAAA;AAAA,aAKE,IAAI,MAAM,CAAC;AAAA,QAChB,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,aAQD,IAAI,MAAM,CAAC;AAAA,QAChB,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWZ;AAEA,QAAM;AAAA,IACJ;AAAA,MACE;AAAA,QACE;AAAA,QACA,kCAAkC,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAE/C;AAAA,QACE;AAAA,QACA,qDAAqD;AAAA,UACnD;AAAA,QAAA,CACD;AAAA,MAAA;AAAA,MAEH;AAAA,QACE;AAAA,QACA,kCAAkC,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAE/C;AAAA,QACE;AAAA,QACA,kCAAkC,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAE/C;AAAA,QACE;AAAA,QACA,kCAAkC,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAE/C;AAAA,QACE;AAAA,QACA;AAAA;AAAA;AAAA;AAAA;AAAA,eAKO,IAAI,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAAA;AAAA,MAmBpB;AAAA,QACE;AAAA,QACA;AAAA;AAAA;AAAA;AAAA;AAAA,eAKO,IAAI,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAAA;AAAA,MAcpB;AAAA;AAAA,QAEE;AAAA,QACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,aAWK,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAElB;AAAA;AAAA,QAEE;AAAA,QACA;AAAA,aACK,IAAI,MAAM,CAAC;AAAA;AAAA;AAAA,MAAA;AAAA,MAIlB;AAAA,QACE;AAAA,QACA,yBAAyB,IAAI;AAAA,MAAA;AAAA,MAE/B;AAAA,QACE;AAAA,QACA,yBAAyB,KAAK;AAAA,MAAA;AAAA,MAEhC;AAAA,QACE;AAAA,QACA,mDAAmD;AAAA,UACjD;AAAA,QAAA,CACD;AAAA,MAAA;AAAA,MAEH;AAAA,QACE;AAAA,QACA;AAAA;AAAA;AAAA,aAGK,IAAI,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA,MAAA;AAAA,IAIlB;AAAA,IAKF;AAAA,EAAA;AAGF,QAAM,IAAI,IAAA;AACZ;AAEA,eAAe,eACb,IACA,QACA,KACA;AACA,MAAI,MAAM,OAAO,WAAW,CAAC;AAC7B,QAAM,SAAS,eAAe,WAAW,MAAM,CAAC,IAAI;AACpD,QAAM,MAAM,SAAS,IAAI,OAAO,OAAO,EAAE;AAEzC,QAAM;AAAA,IACJ;AAAA,MACE;AAAA,QACE;AAAA,QACA,gDAAgD,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,IAC7D;AAAA,IAEF;AAAA,EAAA;AAEF,QAAM,IAAI,IAAA;AACZ;AAEA,SAAS,aAAa,IAAgB,QAAoB,KAAe;AACvE,MAAI,MAAM,OAAO,SAAS,CAAC;AAC3B,QAAM,KAAK,IAAI,SAAS,IAAI,OAAO,QAAQ,IAAI;AAC/C;AAAA,IACE;AAAA,IACA;AAAA,MACE,CAAC,kBAAkB,KAAK,MAAM,GAAG,OAAO,gBAAgB,CAAC,CAAC,CAAC;AAAA,MAC3D,CAAC,cAAc,KAAK,MAAM,GAAG,OAAO,YAAY,CAAC,CAAC,CAAC;AAAA,MACnD,CAAC,aAAa,KAAK,MAAM,GAAG,OAAO,WAAW,CAAC,CAAC,CAAC;AAAA,MACjD,CAAC,gBAAgB,KAAK,MAAM,GAAG,OAAO,cAAc,CAAC,CAAC,CAAC;AAAA,MACvD,CAAC,eAAe,KAAK,MAAM,GAAG,OAAO,aAAa,CAAC,CAAC,CAAC;AAAA,MACrD,CAAC,cAAc,KAAK,MAAM,GAAG,OAAO,YAAY,CAAC,CAAC,CAAC;AAAA,MACnD,CAAC,eAAe,KAAK,MAAM,GAAG,OAAO,aAAa,CAAC,CAAC,CAAC;AAAA,MACrD,CAAC,kBAAkB,KAAK,MAAM,GAAG,OAAO,gBAAgB,CAAC,CAAC,CAAC;AAAA,MAC3D,CAAC,sBAAsB,KAAK,MAAM,GAAG,OAAO,oBAAoB,CAAC,CAAC,CAAC;AAAA,MACnE,CAAC,iBAAiB,GAAG,SAAS,OAAO,QAAQ,IAAI,CAAC;AAAA,IAAA;AAAA,IAEpD;AAAA,EAAA;AAEJ;AAEA,SAAS,QAAQ,KAAe;AAC9B;AAAA,IACE;AAAA,IACA;AAAA,MACE,CAAC,YAAY,GAAG,SAAS;AAAA,MACzB,CAAC,UAAU,GAAG,QAAQ;AAAA,MACtB,CAAC,aAAa,GAAG,UAAU;AAAA,MAC3B,CAAC,YAAY,GAAG,SAAS;AAAA,MACzB,CAAC,QAAQ,GAAG,KAAA,EAAO,MAAM;AAAA,MACzB,CAAC,YAAY,GAAG,UAAU;AAAA,MAC1B,CAAC,QAAQ,GAAG,MAAM;AAAA,MAClB,CAAC,WAAW,GAAG,SAAS;AAAA,MACxB,CAAC,UAAU,GAAG,OAAA,CAAQ;AAAA,IAAA;AAAA,IAExB;AAAA,EAAA;AAEJ;AAEA,eAAe,aACb,gBAIA,KACA;AACA,QAAM,UAAU,MAAM,QAAQ;AAAA,IAC5B,eAAe,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,MAAM,KAAK,CAAC;AAAA,EAAA;AAEjE,aAAW,CAAC,MAAM,IAAI,KAAK,SAAS;AAClC,QAAI,MAAM,IAAI;AACd,QAAI,MAAM,IAAI;AACd,QAAI,MAAM,IAAI;AACd,QAAI,MAAM,WAAW,UAAU,MAAM,MAAM,CAAC,CAAC;AAAA,EAC/C;AACF;AAEA,SAAS,WACP,OACA,SACA,KACA;AACA,MAAI,MAAM,OAAO,OAAO,KAAK,CAAC;AAC9B,aAAW,CAAC,MAAM,MAAM,KAAK,SAAS;AACpC,QAAI,MAAM,OAAO,OAAO,WAAW,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,EAC/D;AACF;AAEA,eAAsB,mBACpB,IACA,QACA,KACA,KACA;AACA,QAAM,cAAc,KAAK,GAAG;AAC5B,MAAI,CAAC,qBAAqB,IAAI,QAAQ,aAAa,IAAI,GAAG;AACxD,SAAK,IACF,KAAK,GAAG,EACR,OAAO,oBAAoB,oCAAoC,EAC/D,KAAK,cAAc;AACtB;AAAA,EACF;AAEA,QAAM,cAAc,IAAI,QAAQ,IAAI,GAAG;AACvC,MAAI,IAAI,MAAM,MAAM;AACpB,QAAM,SAAS,IAAI,QAAQ,IAAI,GAAG;AAClC,MAAI,IAAI,MAAM,MAAM;AACpB,QAAM,eAAe,IAAI,QAAQ,IAAI,GAAG;AACxC,MAAI,IAAI,MAAM,MAAM;AACpB,eAAa,IAAI,QAAQ,IAAI,GAAG;AAChC,MAAI,IAAI,MAAM,MAAM;AACpB,UAAQ,IAAI,GAAG;AACf,MAAI,IAAI,IAAA;AACV;AAEA,SAAS,MAAM,GAAqB;AAClC,SAAO,EAAE,CAAC;AACZ;AAEA,SAAS,KAAK,GAAoB;AAChC,SAAO,OAAO,OAAO,CAAC,EAAE,CAAC;AAC3B;AAEA,SAAS,OAAO,MAAsB;AACpC,SAAO,OAAO,IAAI;AAAA;AACpB;"}
+{"version":3,"file":"statz.js","sources":["../../../../../zero-cache/src/services/statz.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport auth from 'basic-auth';\nimport type {FastifyReply, FastifyRequest} from 'fastify';\nimport fs from 'fs';\nimport os from 'os';\nimport type {Writable} from 'stream';\nimport {BigIntJSON} from '../../../shared/src/bigint-json.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport type {NormalizedZeroConfig as ZeroConfig} from '../config/normalize.ts';\nimport {isAdminPasswordValid} from '../config/zero-config.ts';\nimport {pgClient} from '../types/pg.ts';\nimport {getShardID, upstreamSchema} from '../types/shards.ts';\n\nasync function upstreamStats(\n  lc: LogContext,\n  config: ZeroConfig,\n  out: Writable,\n) {\n  const schema = upstreamSchema(getShardID(config));\n  const sql = pgClient(lc, config.upstream.db);\n\n  out.write(header('Upstream'));\n\n  await printPgStats(\n    [\n      [\n        'num replicas',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"replicas\"`,\n      ],\n      [\n        'num clients with mutations',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"clients\"`,\n      ],\n      [\n        'num mutations processed',\n        sql`SELECT SUM(\"lastMutationID\") as \"c\" FROM ${sql(schema)}.\"clients\"`,\n      ],\n    ],\n    out,\n  );\n\n  await sql.end();\n}\n\nasync function cvrStats(lc: LogContext, config: ZeroConfig, out: Writable) {\n  out.write(header('CVR'));\n\n  const schema = upstreamSchema(getShardID(config)) + '/cvr';\n  const sql = pgClient(lc, config.cvr.db);\n\n  function numQueriesPerClientGroup(\n    active: boolean,\n  ): ReturnType<ReturnType<typeof pgClient>> {\n    const filter = active\n      ? sql`WHERE \"inactivatedAt\" IS NULL AND deleted = false`\n      : sql`WHERE \"inactivatedAt\" IS NOT NULL AND (\"inactivatedAt\" + \"ttl\") > NOW()`;\n    return sql`WITH\n    group_counts AS (\n      SELECT\n        \"clientGroupID\",\n        COUNT(*) AS num_queries\n      FROM ${sql(schema)}.\"desires\"\n      ${filter}\n      GROUP BY \"clientGroupID\"\n    ),\n    -- Count distinct clientIDs per clientGroupID\n    client_per_group_counts AS (\n      SELECT\n        \"clientGroupID\",\n        COUNT(DISTINCT \"clientID\") AS num_clients\n      FROM ${sql(schema)}.\"desires\"\n      ${filter}\n      GROUP BY \"clientGroupID\"\n    )\n    -- Combine all the information\n    SELECT\n      g.\"clientGroupID\",\n      cpg.num_clients,\n      g.num_queries\n    FROM group_counts g\n    JOIN client_per_group_counts cpg ON g.\"clientGroupID\" = cpg.\"clientGroupID\"\n    ORDER BY g.num_queries DESC;`;\n  }\n\n  await printPgStats(\n    [\n      [\n        'total num queries',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"desires\"`,\n      ],\n      [\n        'num unique query hashes',\n        sql`SELECT COUNT(DISTINCT \"queryHash\") as \"c\" FROM ${sql(\n          schema,\n        )}.\"desires\"`,\n      ],\n      [\n        'num active queries',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"desires\" WHERE \"inactivatedAt\" IS NULL AND \"deleted\" = false`,\n      ],\n      [\n        'num inactive queries',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"desires\" WHERE \"inactivatedAt\" IS NOT NULL AND (\"inactivatedAt\" + \"ttl\") > NOW()`,\n      ],\n      [\n        'num deleted queries',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"desires\" WHERE \"deleted\" = true`,\n      ],\n      [\n        'fresh queries percentiles',\n        sql`WITH client_group_counts AS (\n        -- Count inactive desires per clientGroupID\n        SELECT\n          \"clientGroupID\",\n          COUNT(*) AS fresh_count\n        FROM ${sql(schema)}.\"desires\"\n        WHERE\n          (\"inactivatedAt\" IS NOT NULL\n          AND (\"inactivatedAt\" + \"ttl\") > NOW()) OR (\"inactivatedAt\" IS NULL\n          AND deleted = false)\n        GROUP BY \"clientGroupID\"\n      )\n\n      SELECT\n        percentile_cont(0.50) WITHIN GROUP (ORDER BY fresh_count) AS \"p50\",\n        percentile_cont(0.75) WITHIN GROUP (ORDER BY fresh_count) AS \"p75\",\n        percentile_cont(0.90) WITHIN GROUP (ORDER BY fresh_count) AS \"p90\",\n        percentile_cont(0.95) WITHIN GROUP (ORDER BY fresh_count) AS \"p95\",\n        percentile_cont(0.99) WITHIN GROUP (ORDER BY fresh_count) AS \"p99\",\n        MIN(fresh_count) AS \"min\",\n        MAX(fresh_count) AS \"max\",\n        AVG(fresh_count) AS \"avg\"\n      FROM client_group_counts;`,\n      ],\n      [\n        'rows per client group percentiles',\n        sql`WITH client_group_counts AS (\n        -- Count inactive desires per clientGroupID\n        SELECT\n          \"clientGroupID\",\n          COUNT(*) AS row_count\n        FROM ${sql(schema)}.\"rows\"\n        GROUP BY \"clientGroupID\"\n      )\n      SELECT\n        percentile_cont(0.50) WITHIN GROUP (ORDER BY row_count) AS \"p50\",\n        percentile_cont(0.75) WITHIN GROUP (ORDER BY row_count) AS \"p75\",\n        percentile_cont(0.90) WITHIN GROUP (ORDER BY row_count) AS \"p90\",\n        percentile_cont(0.95) WITHIN GROUP (ORDER BY row_count) AS \"p95\",\n        percentile_cont(0.99) WITHIN GROUP (ORDER BY row_count) AS \"p99\",\n        MIN(row_count) AS \"min\",\n        MAX(row_count) AS \"max\",\n        AVG(row_count) AS \"avg\"\n      FROM client_group_counts;`,\n      ],\n      [\n        // check for AST blowup due to DNF conversion.\n        'ast sizes',\n        sql`SELECT\n        percentile_cont(0.25) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"25th_percentile\",\n        percentile_cont(0.5) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"50th_percentile\",\n        percentile_cont(0.75) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"75th_percentile\",\n        percentile_cont(0.9) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"90th_percentile\",\n        percentile_cont(0.95) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"95th_percentile\",\n        percentile_cont(0.99) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"99th_percentile\",\n        MIN(length(\"clientAST\"::text)) AS \"minimum_length\",\n        MAX(length(\"clientAST\"::text)) AS \"maximum_length\",\n        AVG(length(\"clientAST\"::text))::integer AS \"average_length\",\n        COUNT(*) AS \"total_records\"\n      FROM ${sql(schema)}.\"queries\";`,\n      ],\n      [\n        // output the hash of the largest AST\n        'biggest ast hash',\n        sql`SELECT \"queryHash\", length(\"clientAST\"::text) AS \"ast_length\"\n      FROM ${sql(schema)}.\"queries\"\n      ORDER BY length(\"clientAST\"::text) DESC\n      LIMIT 1;`,\n      ],\n      [\n        'total active queries per client and client group',\n        numQueriesPerClientGroup(true),\n      ],\n      [\n        'total inactive queries per client and client group',\n        numQueriesPerClientGroup(false),\n      ],\n      [\n        'total rows per client group',\n        sql`SELECT \"clientGroupID\", COUNT(*) as \"c\" FROM ${sql(\n          schema,\n        )}.\"rows\" GROUP BY \"clientGroupID\" ORDER BY \"c\" DESC`,\n      ],\n      [\n        'num rows per query',\n        sql`SELECT\n        k.key AS \"queryHash\",\n        COUNT(*) AS row_count\n      FROM ${sql(schema)}.\"rows\" r,\n      LATERAL jsonb_each(r.\"refCounts\") k\n      GROUP BY k.key\n      ORDER BY row_count DESC;`,\n      ],\n    ] satisfies [\n      name: string,\n      query: ReturnType<ReturnType<typeof pgClient>>,\n    ][],\n    out,\n  );\n\n  await sql.end();\n}\n\nasync function changelogStats(\n  lc: LogContext,\n  config: ZeroConfig,\n  out: Writable,\n) {\n  out.write(header('Change DB'));\n  const schema = upstreamSchema(getShardID(config)) + '/cdc';\n  const sql = pgClient(lc, config.change.db);\n\n  await printPgStats(\n    [\n      [\n        'change log size',\n        sql`SELECT COUNT(*) as \"change_log_size\" FROM ${sql(schema)}.\"changeLog\"`,\n      ],\n    ],\n    out,\n  );\n  await sql.end();\n}\n\nfunction replicaStats(lc: LogContext, config: ZeroConfig, out: Writable) {\n  out.write(header('Replica'));\n  const db = new Database(lc, config.replica.file);\n  printStats(\n    'replica',\n    [\n      ['wal checkpoint', pick(first(db.pragma('WAL_CHECKPOINT')))],\n      ['page count', pick(first(db.pragma('PAGE_COUNT')))],\n      ['page size', pick(first(db.pragma('PAGE_SIZE')))],\n      ['journal mode', pick(first(db.pragma('JOURNAL_MODE')))],\n      ['synchronous', pick(first(db.pragma('SYNCHRONOUS')))],\n      ['cache size', pick(first(db.pragma('CACHE_SIZE')))],\n      ['auto vacuum', pick(first(db.pragma('AUTO_VACUUM')))],\n      ['freelist count', pick(first(db.pragma('FREELIST_COUNT')))],\n      ['wal autocheckpoint', pick(first(db.pragma('WAL_AUTOCHECKPOINT')))],\n      ['db file stats', fs.statSync(config.replica.file)],\n    ] as const,\n    out,\n  );\n}\n\nfunction osStats(out: Writable) {\n  printStats(\n    'os',\n    [\n      ['load avg', os.loadavg()],\n      ['uptime', os.uptime()],\n      ['total mem', os.totalmem()],\n      ['free mem', os.freemem()],\n      ['cpus', os.cpus().length],\n      ['available parallelism', os.availableParallelism()],\n      ['platform', os.platform()],\n      ['arch', os.arch()],\n      ['release', os.release()],\n      ['uptime', os.uptime()],\n    ] as const,\n    out,\n  );\n}\n\nasync function printPgStats(\n  pendingQueries: [\n    name: string,\n    query: ReturnType<ReturnType<typeof pgClient>>,\n  ][],\n  out: Writable,\n) {\n  const results = await Promise.all(\n    pendingQueries.map(async ([name, query]) => [name, await query]),\n  );\n  for (const [name, data] of results) {\n    out.write('\\n');\n    out.write(name);\n    out.write('\\n');\n    out.write(BigIntJSON.stringify(data, null, 2));\n  }\n}\n\nfunction printStats(\n  group: string,\n  queries: readonly [name: string, result: unknown][],\n  out: Writable,\n) {\n  out.write('\\n' + header(group));\n  for (const [name, result] of queries) {\n    out.write('\\n' + name + BigIntJSON.stringify(result, null, 2));\n  }\n}\n\nexport async function handleStatzRequest(\n  lc: LogContext,\n  config: ZeroConfig,\n  req: FastifyRequest,\n  res: FastifyReply,\n) {\n  const credentials = auth(req);\n  if (!isAdminPasswordValid(lc, config, credentials?.pass)) {\n    void res\n      .code(401)\n      .header('WWW-Authenticate', 'Basic realm=\"Statz Protected Area\"')\n      .send('Unauthorized');\n    return;\n  }\n\n  await upstreamStats(lc, config, res.raw);\n  res.raw.write('\\n\\n');\n  await cvrStats(lc, config, res.raw);\n  res.raw.write('\\n\\n');\n  await changelogStats(lc, config, res.raw);\n  res.raw.write('\\n\\n');\n  replicaStats(lc, config, res.raw);\n  res.raw.write('\\n\\n');\n  osStats(res.raw);\n  res.raw.end();\n}\n\nfunction first(x: object[]): object {\n  return x[0];\n}\n\nfunction pick(x: object): unknown {\n  return Object.values(x)[0];\n}\n\nfunction header(name: string): string {\n  return `=== ${name} ===\\n`;\n}\n"],"names":[],"mappings":";;;;;;;;AAaA,eAAe,cACb,IACA,QACA,KACA;AACA,QAAM,SAAS,eAAe,WAAW,MAAM,CAAC;AAChD,QAAM,MAAM,SAAS,IAAI,OAAO,SAAS,EAAE;AAE3C,MAAI,MAAM,OAAO,UAAU,CAAC;AAE5B,QAAM;AAAA,IACJ;AAAA,MACE;AAAA,QACE;AAAA,QACA,kCAAkC,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAE/C;AAAA,QACE;AAAA,QACA,kCAAkC,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAE/C;AAAA,QACE;AAAA,QACA,+CAA+C,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,IAC5D;AAAA,IAEF;AAAA,EAAA;AAGF,QAAM,IAAI,IAAA;AACZ;AAEA,eAAe,SAAS,IAAgB,QAAoB,KAAe;AACzE,MAAI,MAAM,OAAO,KAAK,CAAC;AAEvB,QAAM,SAAS,eAAe,WAAW,MAAM,CAAC,IAAI;AACpD,QAAM,MAAM,SAAS,IAAI,OAAO,IAAI,EAAE;AAEtC,WAAS,yBACP,QACyC;AACzC,UAAM,SAAS,SACX,yDACA;AACJ,WAAO;AAAA;AAAA;AAAA;AAAA;AAAA,aAKE,IAAI,MAAM,CAAC;AAAA,QAChB,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,aAQD,IAAI,MAAM,CAAC;AAAA,QAChB,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWZ;AAEA,QAAM;AAAA,IACJ;AAAA,MACE;AAAA,QACE;AAAA,QACA,kCAAkC,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAE/C;AAAA,QACE;AAAA,QACA,qDAAqD;AAAA,UACnD;AAAA,QAAA,CACD;AAAA,MAAA;AAAA,MAEH;AAAA,QACE;AAAA,QACA,kCAAkC,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAE/C;AAAA,QACE;AAAA,QACA,kCAAkC,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAE/C;AAAA,QACE;AAAA,QACA,kCAAkC,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAE/C;AAAA,QACE;AAAA,QACA;AAAA;AAAA;AAAA;AAAA;AAAA,eAKO,IAAI,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAAA;AAAA,MAmBpB;AAAA,QACE;AAAA,QACA;AAAA;AAAA;AAAA;AAAA;AAAA,eAKO,IAAI,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAAA;AAAA,MAcpB;AAAA;AAAA,QAEE;AAAA,QACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,aAWK,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAElB;AAAA;AAAA,QAEE;AAAA,QACA;AAAA,aACK,IAAI,MAAM,CAAC;AAAA;AAAA;AAAA,MAAA;AAAA,MAIlB;AAAA,QACE;AAAA,QACA,yBAAyB,IAAI;AAAA,MAAA;AAAA,MAE/B;AAAA,QACE;AAAA,QACA,yBAAyB,KAAK;AAAA,MAAA;AAAA,MAEhC;AAAA,QACE;AAAA,QACA,mDAAmD;AAAA,UACjD;AAAA,QAAA,CACD;AAAA,MAAA;AAAA,MAEH;AAAA,QACE;AAAA,QACA;AAAA;AAAA;AAAA,aAGK,IAAI,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA,MAAA;AAAA,IAIlB;AAAA,IAKF;AAAA,EAAA;AAGF,QAAM,IAAI,IAAA;AACZ;AAEA,eAAe,eACb,IACA,QACA,KACA;AACA,MAAI,MAAM,OAAO,WAAW,CAAC;AAC7B,QAAM,SAAS,eAAe,WAAW,MAAM,CAAC,IAAI;AACpD,QAAM,MAAM,SAAS,IAAI,OAAO,OAAO,EAAE;AAEzC,QAAM;AAAA,IACJ;AAAA,MACE;AAAA,QACE;AAAA,QACA,gDAAgD,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,IAC7D;AAAA,IAEF;AAAA,EAAA;AAEF,QAAM,IAAI,IAAA;AACZ;AAEA,SAAS,aAAa,IAAgB,QAAoB,KAAe;AACvE,MAAI,MAAM,OAAO,SAAS,CAAC;AAC3B,QAAM,KAAK,IAAI,SAAS,IAAI,OAAO,QAAQ,IAAI;AAC/C;AAAA,IACE;AAAA,IACA;AAAA,MACE,CAAC,kBAAkB,KAAK,MAAM,GAAG,OAAO,gBAAgB,CAAC,CAAC,CAAC;AAAA,MAC3D,CAAC,cAAc,KAAK,MAAM,GAAG,OAAO,YAAY,CAAC,CAAC,CAAC;AAAA,MACnD,CAAC,aAAa,KAAK,MAAM,GAAG,OAAO,WAAW,CAAC,CAAC,CAAC;AAAA,MACjD,CAAC,gBAAgB,KAAK,MAAM,GAAG,OAAO,cAAc,CAAC,CAAC,CAAC;AAAA,MACvD,CAAC,eAAe,KAAK,MAAM,GAAG,OAAO,aAAa,CAAC,CAAC,CAAC;AAAA,MACrD,CAAC,cAAc,KAAK,MAAM,GAAG,OAAO,YAAY,CAAC,CAAC,CAAC;AAAA,MACnD,CAAC,eAAe,KAAK,MAAM,GAAG,OAAO,aAAa,CAAC,CAAC,CAAC;AAAA,MACrD,CAAC,kBAAkB,KAAK,MAAM,GAAG,OAAO,gBAAgB,CAAC,CAAC,CAAC;AAAA,MAC3D,CAAC,sBAAsB,KAAK,MAAM,GAAG,OAAO,oBAAoB,CAAC,CAAC,CAAC;AAAA,MACnE,CAAC,iBAAiB,GAAG,SAAS,OAAO,QAAQ,IAAI,CAAC;AAAA,IAAA;AAAA,IAEpD;AAAA,EAAA;AAEJ;AAEA,SAAS,QAAQ,KAAe;AAC9B;AAAA,IACE;AAAA,IACA;AAAA,MACE,CAAC,YAAY,GAAG,SAAS;AAAA,MACzB,CAAC,UAAU,GAAG,QAAQ;AAAA,MACtB,CAAC,aAAa,GAAG,UAAU;AAAA,MAC3B,CAAC,YAAY,GAAG,SAAS;AAAA,MACzB,CAAC,QAAQ,GAAG,KAAA,EAAO,MAAM;AAAA,MACzB,CAAC,yBAAyB,GAAG,sBAAsB;AAAA,MACnD,CAAC,YAAY,GAAG,UAAU;AAAA,MAC1B,CAAC,QAAQ,GAAG,MAAM;AAAA,MAClB,CAAC,WAAW,GAAG,SAAS;AAAA,MACxB,CAAC,UAAU,GAAG,OAAA,CAAQ;AAAA,IAAA;AAAA,IAExB;AAAA,EAAA;AAEJ;AAEA,eAAe,aACb,gBAIA,KACA;AACA,QAAM,UAAU,MAAM,QAAQ;AAAA,IAC5B,eAAe,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,MAAM,KAAK,CAAC;AAAA,EAAA;AAEjE,aAAW,CAAC,MAAM,IAAI,KAAK,SAAS;AAClC,QAAI,MAAM,IAAI;AACd,QAAI,MAAM,IAAI;AACd,QAAI,MAAM,IAAI;AACd,QAAI,MAAM,WAAW,UAAU,MAAM,MAAM,CAAC,CAAC;AAAA,EAC/C;AACF;AAEA,SAAS,WACP,OACA,SACA,KACA;AACA,MAAI,MAAM,OAAO,OAAO,KAAK,CAAC;AAC9B,aAAW,CAAC,MAAM,MAAM,KAAK,SAAS;AACpC,QAAI,MAAM,OAAO,OAAO,WAAW,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,EAC/D;AACF;AAEA,eAAsB,mBACpB,IACA,QACA,KACA,KACA;AACA,QAAM,cAAc,KAAK,GAAG;AAC5B,MAAI,CAAC,qBAAqB,IAAI,QAAQ,aAAa,IAAI,GAAG;AACxD,SAAK,IACF,KAAK,GAAG,EACR,OAAO,oBAAoB,oCAAoC,EAC/D,KAAK,cAAc;AACtB;AAAA,EACF;AAEA,QAAM,cAAc,IAAI,QAAQ,IAAI,GAAG;AACvC,MAAI,IAAI,MAAM,MAAM;AACpB,QAAM,SAAS,IAAI,QAAQ,IAAI,GAAG;AAClC,MAAI,IAAI,MAAM,MAAM;AACpB,QAAM,eAAe,IAAI,QAAQ,IAAI,GAAG;AACxC,MAAI,IAAI,MAAM,MAAM;AACpB,eAAa,IAAI,QAAQ,IAAI,GAAG;AAChC,MAAI,IAAI,MAAM,MAAM;AACpB,UAAQ,IAAI,GAAG;AACf,MAAI,IAAI,IAAA;AACV;AAEA,SAAS,MAAM,GAAqB;AAClC,SAAO,EAAE,CAAC;AACZ;AAEA,SAAS,KAAK,GAAoB;AAChC,SAAO,OAAO,OAAO,CAAC,EAAE,CAAC;AAC3B;AAEA,SAAS,OAAO,MAAsB;AACpC,SAAO,OAAO,IAAI;AAAA;AACpB;"}

package/out/zero-cache/src/services/view-syncer/cvr-store.d.ts

@@ -19,7 +19,7 @@ export type CVRFlushStats = {
 };
 export declare class CVRStore {
     #private;
-    constructor(lc: LogContext, cvrDb: PostgresDB, upstreamDb: PostgresDB | undefined, shard: ShardID, taskID: string, cvrID: string, failService: (e: unknown) => void, loadAttemptIntervalMs?: number, maxLoadAttempts?: number, deferredRowFlushThreshold?: number, // somewhat arbitrary
+    constructor(lc: LogContext, cvrDb: PostgresDB, shard: ShardID, taskID: string, cvrID: string, failService: (e: unknown) => void, loadAttemptIntervalMs?: number, maxLoadAttempts?: number, deferredRowFlushThreshold?: number, // somewhat arbitrary
     setTimeoutFn?: typeof setTimeout);
     load(lc: LogContext, lastConnectTime: number): Promise<CVR>;
     getRowRecords(): Promise<ReadonlyMap<RowID, RowRecord>>;

package/out/zero-cache/src/services/view-syncer/cvr-store.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cvr-store.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/cvr-store.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAiBjD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,+CAA+C,CAAC;AAKnF,OAAO,EAAC,sBAAsB,EAAC,MAAM,iCAAiC,CAAC;AACvE,OAAO,KAAK,EAAC,UAAU,EAAE,mBAAmB,EAAC,MAAM,mBAAmB,CAAC;AAEvE,OAAO,EAAY,KAAK,OAAO,EAAiB,MAAM,uBAAuB,CAAC;AAC9E,OAAO,KAAK,EAAQ,cAAc,EAAC,MAAM,qBAAqB,CAAC;AAC/D,OAAO,KAAK,EAAC,GAAG,EAAE,WAAW,EAAC,MAAM,UAAU,CAAC;AAE/C,OAAO,EAKL,KAAK,OAAO,EACb,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAEL,KAAK,YAAY,EAGjB,KAAK,UAAU,EAGf,KAAK,kBAAkB,EACvB,KAAK,UAAU,EACf,KAAK,WAAW,EAEhB,KAAK,KAAK,EACV,KAAK,SAAS,EAGf,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,KAAK,QAAQ,EAGd,MAAM,gBAAgB,CAAC;AAExB,MAAM,MAAM,aAAa,GAAG;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAwDF,qBAAa,QAAQ;;gBA4BjB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,UAAU,EAKjB,UAAU,EAAE,UAAU,GAAG,SAAS,EAClC,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,CAAC,CAAC,EAAE,OAAO,KAAK,IAAI,EACjC,qBAAqB,SAA2B,EAChD,eAAe,SAAoB,EACnC,yBAAyB,SAAM,EAAE,qBAAqB;IACtD,YAAY,oBAAa;IA0B3B,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IA4M3D,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAIvD,YAAY,CAAC,GAAG,EAAE,SAAS,GAAG,IAAI;IAIlC;;;;;;;OAOG;IACH,YAAY,CAAC,EAAE,EAAE,KAAK,GAAG,IAAI;IAI7B;;;;OAIG;IACH,YAAY,CAAC,GAAG,GAAG,EAAE,KAAK,EAAE;IAM5B;;;;OAIG;IACG,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAO3E;;;;;OAKG;IACG,WAAW,IAAI,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC;IAYlD,WAAW,CAAC,EACV,OAAO,EACP,cAAc,EACd,UAAU,EACV,YAAY,EACZ,SAAS,EACT,QAAQ,GACT,EAAE,IAAI,CACL,WAAW,EACT,SAAS,GACT,gBAAgB,GAChB,YAAY,GACZ,cAAc,GACd,WAAW,GACX,UAAU,CACb,GAAG,IAAI;IAsBR,kBAAkB,CAAC,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,GAAG,IAAI;IAarE,QAAQ,CAAC,KAAK,EAAE,WAAW,GAAG,IAAI;IA4ClC,WAAW,CAAC,KAAK,EAAE,WAAW;IA2B9B,YAAY,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAYxC,YAAY,CAAC,QAAQ,EAAE,MAAM;IAgB7B,eAAe,CACb,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE;QAAC,EAAE,EAAE,MAAM,CAAA;KAAC,EACnB,MAAM,EAAE;QAAC,EAAE,EAAE,MAAM,CAAA;KAAC,EACpB,OAAO,EAAE,OAAO,EAChB,aAAa,EAAE,QAAQ,GAAG,SAAS,EACnC,GAAG,EAAE,MAAM,GACV,IAAI;IA+CP,iBAAiB,CACf,EAAE,EAAE,UAAU,EACd,YAAY,EAAE,kBAAkB,EAChC,OAAO,EAAE,WAAW,EACpB,OAAO,EAAE,UAAU,EACnB,kBAAkB,GAAE,MAAM,EAAO,GAChC,cAAc,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC;IAUvC,oBAAoB,CACxB,EAAE,EAAE,UAAU,EACd,YAAY,EAAE,kBAAkB,EAChC,OAAO,EAAE,WAAW,EACpB,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC,cAAc,EAAE,CAAC;IAgM5B,IAAI,QAAQ,IAAI,MAAM,CAErB;IAEK,KAAK,CACT,EAAE,EAAE,UAAU,EACd,sBAAsB,EAAE,UAAU,EAClC,GAAG,EAAE,WAAW,EAChB,eAAe,EAAE,MAAM,GACtB,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IA6BhC,iBAAiB,IAAI,OAAO;IAI5B,qDAAqD;IACrD,OAAO,CAAC,EAAE,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAIhC,cAAc,CAClB,EAAE,EAAE,UAAU,EACd,QAAQ,EAAE,QAAQ,EAClB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,eAAe,EAAE,CAAC;CAsC9B;AAED;;;;GAIG;AACH,wBAAsB,YAAY,CAChC,EAAE,EAAE,mBAAmB,EACvB,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,MAAM,EACrB,sBAAsB,EAAE,UAAU,GACjC,OAAO,CAAC,IAAI,CAAC,CAUf;AAED,qBAAa,mBAAoB,SAAQ,sBAAsB;gBACjD,OAAO,EAAE,MAAM;CAO5B;AAED,qBAAa,+BAAgC,SAAQ,sBAAsB;IACzE,QAAQ,CAAC,IAAI,qCAAqC;gBAEtC,eAAe,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM;CAU3D;AAED,qBAAa,cAAe,SAAQ,sBAAsB;IACxD,QAAQ,CAAC,IAAI,oBAAoB;gBAG/B,KAAK,EAAE,MAAM,GAAG,IAAI,EACpB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,eAAe,EAAE,MAAM;CAe1B;AAED,qBAAa,wBAAyB,SAAQ,sBAAsB;IAClE,QAAQ,CAAC,IAAI,8BAA8B;gBAE/B,KAAK,EAAE,OAAO;CAW3B;AAED,qBAAa,sBAAuB,SAAQ,KAAK;IAC/C,QAAQ,CAAC,IAAI,4BAA4B;IACzC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;gBAExB,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,IAAI;CAK3D"}
+{"version":3,"file":"cvr-store.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/cvr-store.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAiBjD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,+CAA+C,CAAC;AAKnF,OAAO,EAAC,sBAAsB,EAAC,MAAM,iCAAiC,CAAC;AACvE,OAAO,KAAK,EAAC,UAAU,EAAE,mBAAmB,EAAC,MAAM,mBAAmB,CAAC;AAEvE,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAQ,cAAc,EAAC,MAAM,qBAAqB,CAAC;AAC/D,OAAO,KAAK,EAAC,GAAG,EAAE,WAAW,EAAC,MAAM,UAAU,CAAC;AAE/C,OAAO,EAKL,KAAK,OAAO,EACb,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAEL,KAAK,YAAY,EAGjB,KAAK,UAAU,EAGf,KAAK,kBAAkB,EACvB,KAAK,UAAU,EACf,KAAK,WAAW,EAEhB,KAAK,KAAK,EACV,KAAK,SAAS,EAGf,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,KAAK,QAAQ,EAGd,MAAM,gBAAgB,CAAC;AAExB,MAAM,MAAM,aAAa,GAAG;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AA0DF,qBAAa,QAAQ;;gBAuBjB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,UAAU,EACjB,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,CAAC,CAAC,EAAE,OAAO,KAAK,IAAI,EACjC,qBAAqB,SAA2B,EAChD,eAAe,SAAoB,EACnC,yBAAyB,SAAM,EAAE,qBAAqB;IACtD,YAAY,oBAAa;IAwB3B,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAsN3D,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAIvD,YAAY,CAAC,GAAG,EAAE,SAAS,GAAG,IAAI;IAIlC;;;;;;;OAOG;IACH,YAAY,CAAC,EAAE,EAAE,KAAK,GAAG,IAAI;IAI7B;;;;OAIG;IACH,YAAY,CAAC,GAAG,GAAG,EAAE,KAAK,EAAE;IAM5B;;;;OAIG;IACG,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAO3E;;;;;OAKG;IACG,WAAW,IAAI,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC;IAelD,WAAW,CAAC,EACV,OAAO,EACP,cAAc,EACd,UAAU,EACV,YAAY,EACZ,SAAS,EACT,QAAQ,GACT,EAAE,IAAI,CACL,WAAW,EACT,SAAS,GACT,gBAAgB,GAChB,YAAY,GACZ,cAAc,GACd,WAAW,GACX,UAAU,CACb,GAAG,IAAI;IAsBR,kBAAkB,CAAC,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,GAAG,IAAI;IAarE,QAAQ,CAAC,KAAK,EAAE,WAAW,GAAG,IAAI;IA4ClC,WAAW,CAAC,KAAK,EAAE,WAAW;IA2B9B,YAAY,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAYxC,YAAY,CAAC,QAAQ,EAAE,MAAM;IAU7B,eAAe,CACb,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE;QAAC,EAAE,EAAE,MAAM,CAAA;KAAC,EACnB,MAAM,EAAE;QAAC,EAAE,EAAE,MAAM,CAAA;KAAC,EACpB,OAAO,EAAE,OAAO,EAChB,aAAa,EAAE,QAAQ,GAAG,SAAS,EACnC,GAAG,EAAE,MAAM,GACV,IAAI;IA+CP,iBAAiB,CACf,EAAE,EAAE,UAAU,EACd,YAAY,EAAE,kBAAkB,EAChC,OAAO,EAAE,WAAW,EACpB,OAAO,EAAE,UAAU,EACnB,kBAAkB,GAAE,MAAM,EAAO,GAChC,cAAc,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC;IAUvC,oBAAoB,CACxB,EAAE,EAAE,UAAU,EACd,YAAY,EAAE,kBAAkB,EAChC,OAAO,EAAE,WAAW,EACpB,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC,cAAc,EAAE,CAAC;IAkN5B,IAAI,QAAQ,IAAI,MAAM,CAErB;IAEK,KAAK,CACT,EAAE,EAAE,UAAU,EACd,sBAAsB,EAAE,UAAU,EAClC,GAAG,EAAE,WAAW,EAChB,eAAe,EAAE,MAAM,GACtB,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IA8BhC,iBAAiB,IAAI,OAAO;IAI5B,qDAAqD;IACrD,OAAO,CAAC,EAAE,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAIhC,cAAc,CAClB,EAAE,EAAE,UAAU,EACd,QAAQ,EAAE,QAAQ,EAClB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,eAAe,EAAE,CAAC;CAsC9B;AAED;;;;GAIG;AACH,wBAAsB,YAAY,CAChC,EAAE,EAAE,mBAAmB,EACvB,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,MAAM,EACrB,sBAAsB,EAAE,UAAU,GACjC,OAAO,CAAC,IAAI,CAAC,CAUf;AAED,qBAAa,mBAAoB,SAAQ,sBAAsB;gBACjD,OAAO,EAAE,MAAM;CAO5B;AAED,qBAAa,+BAAgC,SAAQ,sBAAsB;IACzE,QAAQ,CAAC,IAAI,qCAAqC;gBAEtC,eAAe,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM;CAU3D;AAED,qBAAa,cAAe,SAAQ,sBAAsB;IACxD,QAAQ,CAAC,IAAI,oBAAoB;gBAG/B,KAAK,EAAE,MAAM,GAAG,IAAI,EACpB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,eAAe,EAAE,MAAM;CAe1B;AAED,qBAAa,wBAAyB,SAAQ,sBAAsB;IAClE,QAAQ,CAAC,IAAI,8BAA8B;gBAE/B,KAAK,EAAE,OAAO;CAW3B;AAED,qBAAa,sBAAuB,SAAQ,KAAK;IAC/C,QAAQ,CAAC,IAAI,4BAA4B;IACzC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;gBAExB,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,IAAI;CAK3D"}

package/out/zero-cache/src/services/view-syncer/cvr-store.js

@@ -17,12 +17,13 @@ import { TransactionPool } from "../../db/transaction-pool.js";
 import { recordRowsSynced } from "../../server/anonymous-otel-start.js";
 import { ProtocolErrorWithLevel } from "../../types/error-with-level.js";
 import { rowIDString } from "../../types/row-key.js";
-import { cvrSchema, upstreamSchema } from "../../types/shards.js";
+import { cvrSchema } from "../../types/shards.js";
 import { RowRecordCache } from "./row-record-cache.js";
 import "pg-format";
 import "../../../../shared/src/bigint-json.js";
 import { EMPTY_CVR_VERSION, versionFromString, versionString, queryRecordToQueryRow, cmpVersions } from "./schema/types.js";
 import { ttlClockFromNumber, ttlClockAsNumber } from "./ttl-clock.js";
+let flushCounter = 0;
 const tracer = trace.getTracer("cvr-store", version);
 function asQuery(row) {
   const maybeVersion = (s) => s === null ? void 0 : versionFromString(s);
@@ -67,9 +68,7 @@ class CVRStore {
   #id;
   #failService;
   #db;
-  #upstreamDb;
   #writes = /* @__PURE__ */ new Set();
-  #upstreamWrites = [];
   #pendingRowRecordUpdates = new CustomKeyMap(
     rowIDString
   );
@@ -77,12 +76,10 @@ class CVRStore {
   #rowCache;
   #loadAttemptIntervalMs;
   #maxLoadAttempts;
-  #upstreamSchemaName;
   #rowCount = 0;
-  constructor(lc, cvrDb, upstreamDb, shard, taskID, cvrID, failService, loadAttemptIntervalMs = LOAD_ATTEMPT_INTERVAL_MS, maxLoadAttempts = MAX_LOAD_ATTEMPTS, deferredRowFlushThreshold = 100, setTimeoutFn = setTimeout) {
+  constructor(lc, cvrDb, shard, taskID, cvrID, failService, loadAttemptIntervalMs = LOAD_ATTEMPT_INTERVAL_MS, maxLoadAttempts = MAX_LOAD_ATTEMPTS, deferredRowFlushThreshold = 100, setTimeoutFn = setTimeout) {
     this.#failService = failService;
     this.#db = cvrDb;
-    this.#upstreamDb = upstreamDb;
     this.#schema = cvrSchema(shard);
     this.#taskID = taskID;
     this.#id = cvrID;
@@ -97,7 +94,6 @@ class CVRStore {
     );
     this.#loadAttemptIntervalMs = loadAttemptIntervalMs;
     this.#maxLoadAttempts = maxLoadAttempts;
-    this.#upstreamSchemaName = upstreamSchema(shard);
   }
   #cvr(table) {
     return this.#db(`${this.#schema}.${table}`);
@@ -117,7 +113,7 @@ class CVRStore {
         }
         return result;
       }
-      assert(err);
+      assert(err, "Expected error to be set after retry loop exhausted");
       throw new ClientNotFoundError(
         `max attempts exceeded waiting for CVR@${err.cvrVersion} to catch up from ${err.rowsVersion}`
       );
@@ -138,28 +134,30 @@ class CVRStore {
       clientSchema: null,
       profileID: null
     };
-    const [instance, clientsRows, queryRows, desiresRows] = await this.#db.begin(READONLY, (tx) => [
-      tx`SELECT cvr."version", 
+    const [instance, clientsRows, queryRows, desiresRows] = await this.#db.begin(READONLY, (tx) => {
+      lc.debug?.(`CVR tx started after ${Date.now() - start} ms`);
+      return [
+        tx`SELECT cvr."version",
                  "lastActive",
                  "ttlClock",
-                 "replicaVersion", 
-                 "owner", 
+                 "replicaVersion",
+                 "owner",
                  "grantedAt",
-                 "clientSchema", 
+                 "clientSchema",
                  "profileID",
                  "deleted",
                  rows."version" as "rowsVersion"
             FROM ${this.#cvr("instances")} AS cvr
-            LEFT JOIN ${this.#cvr("rowsVersion")} AS rows 
+            LEFT JOIN ${this.#cvr("rowsVersion")} AS rows
             ON cvr."clientGroupID" = rows."clientGroupID"
             WHERE cvr."clientGroupID" = ${id}`,
-      tx`SELECT "clientID" FROM ${this.#cvr(
-        "clients"
-      )}
+        tx`SELECT "clientID" FROM ${this.#cvr(
+          "clients"
+        )}
            WHERE "clientGroupID" = ${id}`,
-      tx`SELECT * FROM ${this.#cvr("queries")} 
+        tx`SELECT * FROM ${this.#cvr("queries")}
           WHERE "clientGroupID" = ${id} AND deleted IS DISTINCT FROM true`,
-      tx`SELECT 
+        tx`SELECT
           "clientGroupID",
           "clientID",
           "queryHash",
@@ -169,7 +167,11 @@ class CVRStore {
           "inactivatedAtMs" AS "inactivatedAt"
           FROM ${this.#cvr("desires")}
           WHERE "clientGroupID" = ${id}`
-    ]);
+      ];
+    });
+    lc.debug?.(
+      `CVR tx completed after ${Date.now() - start} ms (${clientsRows.length} clients, ${queryRows.length} queries, ${desiresRows.length} desires)`
+    );
     if (instance.length === 0) {
       this.putInstance({
         version: cvr.version,
@@ -181,7 +183,10 @@ class CVRStore {
         profileID: null
       });
     } else {
-      assert(instance.length === 1);
+      assert(
+        instance.length === 1,
+        () => `Expected exactly one CVR instance, got ${instance.length}`
+      );
       const {
         version: version2,
         lastActive,
@@ -313,7 +318,10 @@ class CVRStore {
     if (result.length === 0) {
       return void 0;
     }
-    assert(result.length === 1);
+    assert(
+      result.length === 1,
+      () => `Expected exactly one rowsVersion result, got ${result.length}`
+    );
     return result[0][0];
   }
   putInstance({
@@ -422,15 +430,10 @@ class CVRStore {
   deleteClient(clientID) {
     this.#writes.add({
       stats: { clients: 1 },
-      write: (sql) => sql`DELETE FROM ${this.#cvr("clients")} 
-            WHERE "clientGroupID" = ${this.#id} 
+      write: (sql) => sql`DELETE FROM ${this.#cvr("clients")}
+            WHERE "clientGroupID" = ${this.#id}
               AND "clientID" = ${clientID}`
     });
-    this.#upstreamWrites.push(
-      (sql) => sql`DELETE FROM ${sql(this.#upstreamSchemaName)}."mutations" 
-            WHERE "clientGroupID" = ${this.#id} 
-              AND "clientID" = ${clientID}`
-    );
   }
   putDesiredQuery(newVersion, query, client, deleted, inactivatedAt, ttl) {
     const change = {
@@ -507,7 +510,7 @@ class CVRStore {
         const { queryHash: id } = row;
         const patch = row.deleted ? { type: "query", op: "del", id } : { type: "query", op: "put", id };
         const v2 = row.patchVersion;
-        assert(v2);
+        assert(v2, "patchVersion must be set for query patches");
         patches.push({ patch, toVersion: versionFromString(v2) });
       }
       for (const row of allDesires) {
@@ -523,7 +526,9 @@ class CVRStore {
       reader.setDone();
     }
   }
-  async #checkVersionAndOwnership(tx, expectedCurrentVersion, lastConnectTime) {
+  async #checkVersionAndOwnership(lc, tx, expectedCurrentVersion, lastConnectTime) {
+    const start = Date.now();
+    lc.debug?.("checking cvr version and ownership");
     const expected = versionString(expectedCurrentVersion);
     const result = await tx`SELECT "version", "owner", "grantedAt" FROM ${this.#cvr("instances")}
         WHERE "clientGroupID" = ${this.#id}
@@ -533,6 +538,9 @@ class CVRStore {
       owner: null,
       grantedAt: null
     };
+    lc.debug?.(
+      "checked cvr version and ownership in " + (Date.now() - start) + " ms"
+    );
     if (owner !== this.#taskID && (grantedAt ?? 0) > lastConnectTime) {
       throw new OwnershipError(owner, grantedAt, lastConnectTime);
     }
@@ -540,7 +548,7 @@ class CVRStore {
       throw new ConcurrentModificationException(expected, version2);
     }
   }
-  async #flush(expectedCurrentVersion, cvr, lastConnectTime) {
+  async #flush(lc, expectedCurrentVersion, cvr, lastConnectTime) {
     const stats = {
       instances: 0,
       queries: 0,
@@ -574,7 +582,10 @@ class CVRStore {
       return null;
     }
     this.putInstance(cvr);
+    const start = Date.now();
+    lc.debug?.("flush tx beginning");
     const rowsFlushed = await this.#db.begin(READ_COMMITTED, async (tx) => {
+      lc.debug?.(`flush tx begun after ${Date.now() - start} ms`);
       const pipelined = [
         // #checkVersionAndOwnership() executes a `SELECT ... FOR UPDATE`
         // query to acquire a row-level lock so that version-updating
@@ -584,29 +595,41 @@ class CVRStore {
         // to this lock and can thus commit / be-committed independently of
         // cvr.instances.
         this.#checkVersionAndOwnership(
+          lc,
           tx,
           expectedCurrentVersion,
           lastConnectTime
         )
       ];
+      let i = 0;
       for (const write of this.#writes) {
         stats.instances += write.stats.instances ?? 0;
         stats.queries += write.stats.queries ?? 0;
         stats.desires += write.stats.desires ?? 0;
         stats.clients += write.stats.clients ?? 0;
         stats.rows += write.stats.rows ?? 0;
-        pipelined.push(write.write(tx, lastConnectTime).execute());
+        const writeIndex = i++;
+        const writeStart = Date.now();
+        pipelined.push(
+          write.write(tx, lastConnectTime).execute().then(() => {
+            lc.debug?.(
+              `write ${writeIndex}/${this.#writes.size} completed in ${Date.now() - writeStart} ms`
+            );
+          })
+        );
         stats.statements++;
       }
       const rowUpdates = this.#rowCache.executeRowUpdates(
         tx,
         cvr.version,
         this.#pendingRowRecordUpdates,
-        "allow-defer"
+        "allow-defer",
+        lc
       );
       pipelined.push(...rowUpdates);
       stats.statements += rowUpdates.length;
       await Promise.all(pipelined);
+      lc.debug?.(`flush tx returning after ${Date.now() - start} ms`);
       if (rowUpdates.length === 0) {
         stats.rowsDeferred = this.#pendingRowRecordUpdates.size;
         return false;
@@ -620,11 +643,6 @@ class CVRStore {
       rowsFlushed
     );
     recordRowsSynced(this.#rowCount);
-    if (this.#upstreamDb) {
-      await this.#upstreamDb.begin(READ_COMMITTED, async (tx) => {
-        await Promise.all(this.#upstreamWrites.map((write) => write(tx)));
-      });
-    }
     return stats;
   }
   get rowCount() {
@@ -632,8 +650,10 @@ class CVRStore {
   }
   async flush(lc, expectedCurrentVersion, cvr, lastConnectTime) {
     const start = performance.now();
+    lc = lc.withContext("cvrFlushID", flushCounter++);
     try {
       const stats = await this.#flush(
+        lc,
         expectedCurrentVersion,
         cvr,
         lastConnectTime
@@ -651,7 +671,6 @@ class CVRStore {
       throw e;
     } finally {
       this.#writes.clear();
-      this.#upstreamWrites.length = 0;
       this.#pendingRowRecordUpdates.clear();
       this.#forceUpdates.clear();
     }