@rocicorp/zero 1.12.0-canary.8 → 1.12.0-canary.9

package/out/zero-cache/src/services/mutagen/mutagen.js

@@ -1,4 +1,5 @@
 import { assert, unreachable } from "../../../../shared/src/asserts.js";
+import { resolver } from "../../../../shared/src/resolver.js";
 import { parse } from "../../../../shared/src/valita.js";
 import { InvalidPush, MutationFailed, MutationRateLimited } from "../../../../zero-protocol/src/error-kind-enum.js";
 import { ZeroCache } from "../../../../zero-protocol/src/error-origin-enum.js";
@@ -16,7 +17,6 @@ import { runTx } from "../../db/run-transaction.js";
 import { recordMutation } from "../../server/anonymous-otel-start.js";
 import { WriteAuthorizerImpl } from "../../auth/write-authorizer.js";
 import { SlidingWindowLimiter } from "../limiter/sliding-window-limiter.js";
-import { resolver } from "@rocicorp/resolver";
 import postgres from "postgres";
 import { PG_SERIALIZATION_FAILURE } from "@drdgvhbh/postgres-error-codes";
 //#region ../zero-cache/src/services/mutagen/mutagen.ts
@@ -207,6 +207,6 @@ async function checkSchemaVersionAndIncrementLastMutationID(tx, shard, clientGro
 	});
 }
 //#endregion
-export { MutagenService, getInsertSQL, getUpsertSQL, processMutation, processMutationWithTx };
+export { MutagenService, processMutation };
 
 //# sourceMappingURL=mutagen.js.map

package/out/zero-cache/src/services/mutagen/mutagen.js.map

@@ -1 +1 @@
-{"version":3,"file":"mutagen.js","names":["ErrorKind.MutationRateLimited","MutationType.CRUD","Mode.SERIALIZABLE","ErrorKind.MutationFailed","v.parse","ErrorKind.InvalidPush","ErrorOrigin.ZeroCache"],"sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"sourcesContent":["import {PG_SERIALIZATION_FAILURE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {JWTPayload} from 'jose';\nimport postgres from 'postgres';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {ErrorKind} from '../../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../../zero-protocol/src/error-origin.ts';\nimport {\n  isProtocolError,\n  ProtocolError,\n} from '../../../../zero-protocol/src/error.ts';\nimport * as MutationType from '../../../../zero-protocol/src/mutation-type-enum.ts';\nimport {\n  type CRUDMutation,\n  type DeleteOp,\n  type InsertOp,\n  type Mutation,\n  type UpdateOp,\n  type UpsertOp,\n} from '../../../../zero-protocol/src/mutation.ts';\nimport {\n  primaryKeyValueSchema,\n  type PrimaryKeyValue,\n} from '../../../../zero-protocol/src/primary-key.ts';\nimport type {DatabaseStorage} from '../../../../zqlite/src/database-storage.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  WriteAuthorizerImpl,\n  type WriteAuthorizer,\n} from '../../auth/write-authorizer.ts';\nimport {type ZeroConfig} from '../../config/zero-config.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {recordMutation} from '../../server/anonymous-otel-start.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../types/shards.ts';\nimport {SlidingWindowLimiter} from '../limiter/sliding-window-limiter.ts';\nimport type {RefCountedService, Service} from '../service.ts';\nimport {MutationAlreadyProcessedError} from './error.ts';\n\n// An error encountered processing a mutation.\n// Returned back to application for display to user.\nexport type MutationError = [\n  kind: ErrorKind.MutationFailed | ErrorKind.MutationRateLimited,\n  desc: string,\n];\n\nexport interface Mutagen extends RefCountedService {\n  /**\n   * Callers must hold a {@link ref} for the duration of a push (i.e. across\n   * all of its mutations). The service stops, closing its replica handle,\n   * when its ref count drops to zero.\n   */\n  processMutation(\n    mutation: Mutation,\n    authData: JWTPayload | undefined,\n    customMutatorsEnabled: boolean,\n  ): Promise<MutationError | undefined>;\n}\n\nexport class MutagenService implements Mutagen, Service {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #upstream: PostgresDB;\n  readonly #shard: ShardID;\n  readonly #stopped = resolver();\n  readonly #replica: Database;\n  readonly #writeAuthorizer: WriteAuthorizerImpl;\n  readonly #limiter: SlidingWindowLimiter | undefined;\n  #refCount = 0;\n  #isStopped = false;\n\n  readonly #crudMutations = getOrCreateCounter(\n    'mutation',\n    'crud',\n    'Number of CRUD mutations processed',\n  );\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    clientGroupID: string,\n    upstream: PostgresDB,\n    config: ZeroConfig,\n    writeAuthzStorage: DatabaseStorage,\n  ) {\n    this.id = clientGroupID;\n    this.#lc = lc;\n    this.#upstream = upstream;\n    this.#shard = shard;\n    this.#replica = new Database(this.#lc, config.replica.file, {\n      fileMustExist: true,\n    });\n    this.#writeAuthorizer = new WriteAuthorizerImpl(\n      this.#lc,\n      config,\n      this.#replica,\n      shard.appID,\n      clientGroupID,\n      writeAuthzStorage,\n    );\n\n    if (config.perUserMutationLimit.max !== undefined) {\n      this.#limiter = new SlidingWindowLimiter(\n        config.perUserMutationLimit.windowMs,\n        config.perUserMutationLimit.max,\n      );\n    }\n  }\n\n  ref() {\n    assert(!this.#isStopped, 'MutagenService is already stopped');\n    ++this.#refCount;\n  }\n\n  unref() {\n    assert(!this.#isStopped, 'MutagenService is already stopped');\n    --this.#refCount;\n    if (this.#refCount <= 0) {\n      void this.stop();\n    }\n  }\n\n  hasRefs(): boolean {\n    return this.#refCount > 0;\n  }\n\n  processMutation(\n    mutation: Mutation,\n    authData: JWTPayload | undefined,\n    customMutatorsEnabled = false,\n  ): Promise<MutationError | undefined> {\n    if (this.#limiter?.canDo() === false) {\n      return Promise.resolve([\n        ErrorKind.MutationRateLimited,\n        'Rate limit exceeded',\n      ]);\n    }\n    this.#crudMutations.add(1, {\n      clientGroupID: this.id,\n    });\n    return processMutation(\n      this.#lc,\n      authData,\n      this.#upstream,\n      this.#shard,\n      this.id,\n      mutation,\n      this.#writeAuthorizer,\n      undefined,\n      customMutatorsEnabled,\n    );\n  }\n\n  run(): Promise<void> {\n    return this.#stopped.promise;\n  }\n\n  stop(): Promise<void> {\n    if (this.#isStopped) {\n      return this.#stopped.promise;\n    }\n    this.#isStopped = true;\n    try {\n      this.#writeAuthorizer.destroy();\n    } catch (e) {\n      this.#lc.error?.('error destroying write authorizer storage', e);\n    } finally {\n      // The replica's statistics are maintained by the replicator, so skip\n      // the `PRAGMA optimize` that closing a writable handle would otherwise\n      // run on every client group churn.\n      this.#replica.close({optimize: false});\n    }\n    this.#stopped.resolve();\n    return this.#stopped.promise;\n  }\n}\n\nconst MAX_SERIALIZATION_ATTEMPTS = 10;\n\nexport async function processMutation(\n  lc: LogContext,\n  authData: JWTPayload | undefined,\n  db: PostgresDB,\n  shard: ShardID,\n  clientGroupID: string,\n  mutation: Mutation,\n  writeAuthorizer: WriteAuthorizer,\n  onTxStart?: () => void | Promise<void>, // for testing\n  customMutatorsEnabled = false,\n): Promise<MutationError | undefined> {\n  assert(\n    mutation.type === MutationType.CRUD,\n    'Only CRUD mutations are supported',\n  );\n  // log-leak-ignore -- the mutation id, not row data\n  lc = lc.withContext('mutationID', mutation.id);\n  lc = lc.withContext('processMutation');\n  // The mutationID context set above identifies it; the mutation itself\n  // carries the row values being written.\n  lc.debug?.('Process mutation start');\n\n  // Record mutation processing attempt for telemetry (regardless of success/failure)\n  recordMutation('crud');\n\n  let result: MutationError | undefined;\n\n  const start = Date.now();\n  try {\n    // Mutations can fail for a variety of reasons:\n    //\n    // - application error\n    // - network/db error\n    // - zero bug\n    //\n    // For application errors what we want is to re-run the mutation in\n    // \"error mode\", which skips the actual mutation and just updates the\n    // lastMutationID. Then return the error to the app.\n    //\n    // However, it's hard to tell the difference between application errors\n    // and the other types.\n    //\n    // A reasonable policy ends up being to just retry every mutation once\n    // in error mode. If the error mode mutation succeeds then we assume it\n    // was an application error and return the error to the app. Otherwise,\n    // we know it was something internal and we log it.\n    //\n    // This is not 100% correct - there are theoretical cases where we\n    // return an internal error to the app that shouldn't have been. But it\n    // would have to be a crazy coincidence: we'd have to have a network\n    // error on the first attempt that resolves by the second attempt.\n    //\n    // One might ask why not try/catch just the calls to the mutators and\n    // consider those application errors. That is actually what we do in\n    // Replicache:\n    //\n    // https://github.com/rocicorp/todo-row-versioning/blob/9a0a79dc2d2de32c4fac61b5d1634bd9a9e66b7c/server/src/push.ts#L131\n    //\n    // We don't do it here because:\n    //\n    // 1. It's still not perfect. It's hard to isolate SQL errors in\n    //    mutators due to app developer mistakes from SQL errors due to\n    //    Zero mistakes.\n    // 2. It's not possible to do this with the pg library we're using in\n    //    Zero anyway: https://github.com/porsager/postgres/issues/455.\n    //\n    // Personally I think this simple retry policy is nice.\n    let errorMode = false;\n    for (let i = 0; i < MAX_SERIALIZATION_ATTEMPTS; i++) {\n      try {\n        await runTx(\n          db,\n          async tx => {\n            // Simulates a concurrent request for testing. In production this is a noop.\n            const done = onTxStart?.();\n            try {\n              return await processMutationWithTx(\n                lc,\n                tx,\n                authData,\n                shard,\n                clientGroupID,\n                mutation,\n                errorMode,\n                writeAuthorizer,\n              );\n            } finally {\n              await done;\n            }\n          },\n          {mode: Mode.SERIALIZABLE},\n        );\n        if (errorMode) {\n          lc.debug?.('Ran mutation successfully in error mode');\n        }\n        break;\n      } catch (e) {\n        if (e instanceof MutationAlreadyProcessedError) {\n          lc.debug?.(e.message);\n          // Don't double-count already processed mutations, but they were counted above\n          return undefined;\n        }\n        if (\n          isProtocolError(e) &&\n          !errorMode &&\n          e.kind === ErrorKind.InvalidPush &&\n          customMutatorsEnabled &&\n          i < 2\n        ) {\n          // We're temporarily supporting custom mutators AND CRUD mutators at the same time.\n          // This can create a lot of OOO mutation errors since we do not know when the API server\n          // has applied a custom mutation before moving on to process CRUD mutations.\n          // The temporary workaround (since CRUD is being deprecated) is to retry the mutation\n          // after a small delay. Users are not expected to be running both CRUD and Custom mutators.\n          // They should migrate completely to custom mutators.\n          lc.info?.(\n            'Both CRUD and Custom mutators are being used at once. This is supported for now but IS NOT RECOMMENDED. Migrate completely to custom mutators.',\n            e,\n          );\n          await new Promise(resolve => setTimeout(resolve, 100));\n          continue;\n        }\n        if (isProtocolError(e) || errorMode) {\n          lc.error?.('Process mutation error', e);\n          throw e;\n        }\n        if (\n          e instanceof postgres.PostgresError &&\n          e.code === PG_SERIALIZATION_FAILURE\n        ) {\n          lc.info?.(`attempt ${i + 1}: ${String(e)}`, e);\n          continue; // Retry up to MAX_SERIALIZATION_ATTEMPTS.\n        }\n        result = [ErrorKind.MutationFailed, String(e)];\n        if (errorMode) {\n          break;\n        }\n        lc.error?.('Got error running mutation, re-running in error mode', e);\n        errorMode = true;\n        i--;\n      }\n    }\n  } finally {\n    lc.debug?.('Process mutation complete in', Date.now() - start);\n  }\n  return result;\n}\n\nexport async function processMutationWithTx(\n  lc: LogContext,\n  tx: PostgresTransaction,\n  authData: JWTPayload | undefined,\n  shard: ShardID,\n  clientGroupID: string,\n  mutation: CRUDMutation,\n  errorMode: boolean,\n  authorizer: WriteAuthorizer,\n) {\n  const tasks: (() => Promise<unknown>)[] = [];\n\n  async function execute(stmt: postgres.PendingQuery<postgres.Row[]>) {\n    try {\n      return await stmt.execute();\n    } finally {\n      const q = stmt as unknown as Query;\n      // `q.parameters` are the bound row values -- log the parameterized\n      // statement and the parameter count, never the values themselves.\n      lc.debug?.(`${q.string} (${q.parameters.length} params)`);\n    }\n  }\n\n  authorizer.reloadPermissions();\n\n  if (!errorMode) {\n    // Validates table names and keys every op by the replica's primary key,\n    // which is what the authorizer checks.\n    const ops = authorizer.validateOps(mutation.args[0].ops);\n\n    const normalizedOps = authorizer.normalizeOps(ops);\n    const [canPre, canPost] = await Promise.all([\n      authorizer.canPreMutation(authData, normalizedOps),\n      authorizer.canPostMutation(authData, normalizedOps),\n    ]);\n    if (canPre && canPost) {\n      for (const op of ops) {\n        switch (op.op) {\n          case 'insert':\n            tasks.push(() => execute(getInsertSQL(tx, op)));\n            break;\n          case 'upsert':\n            tasks.push(() => execute(getUpsertSQL(tx, op)));\n            break;\n          case 'update':\n            tasks.push(() => execute(getUpdateSQL(tx, op)));\n            break;\n          case 'delete':\n            tasks.push(() => execute(getDeleteSQL(tx, op)));\n            break;\n          default:\n            unreachable(op);\n        }\n      }\n    }\n  }\n\n  // Confirm the mutation even though it may have been blocked by the authorizer.\n  // Authorizer blocking a mutation is not an error but the correct result of the mutation.\n  tasks.unshift(() =>\n    checkSchemaVersionAndIncrementLastMutationID(\n      tx,\n      shard,\n      clientGroupID,\n      mutation.clientID,\n      mutation.id,\n    ),\n  );\n\n  // Note: An error thrown from any Promise aborts the entire transaction.\n  await Promise.all(tasks.map(task => task()));\n}\n\nexport function getInsertSQL(\n  tx: postgres.TransactionSql,\n  create: InsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  return tx`\n    INSERT INTO ${tx(create.tableName)} ${tx(create.value)}\n    ON CONFLICT (${tx(create.primaryKey)}) DO NOTHING\n  `;\n}\n\nexport function getUpsertSQL(\n  tx: postgres.TransactionSql,\n  set: UpsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const {tableName, primaryKey, value} = set;\n  return tx`\n    INSERT INTO ${tx(tableName)} ${tx(value)}\n    ON CONFLICT (${tx(primaryKey)})\n    DO UPDATE SET ${tx(value)}\n  `;\n}\n\nfunction getUpdateSQL(\n  tx: postgres.TransactionSql,\n  update: UpdateOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const table = update.tableName;\n  const {primaryKey, value} = update;\n  const id: Record<string, PrimaryKeyValue> = {};\n  for (const key of primaryKey) {\n    id[key] = v.parse(value[key], primaryKeyValueSchema);\n  }\n  return tx`UPDATE ${tx(table)} SET ${tx(value)} WHERE ${Object.entries(\n    id,\n  ).flatMap(([key, value], i) =>\n    i ? [tx`AND`, tx`${tx(key)} = ${value}`] : tx`${tx(key)} = ${value}`,\n  )}`;\n}\n\nfunction getDeleteSQL(\n  tx: postgres.TransactionSql,\n  deleteOp: DeleteOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const {tableName, primaryKey, value} = deleteOp;\n\n  const conditions = [];\n  for (const key of primaryKey) {\n    if (conditions.length > 0) {\n      conditions.push(tx`AND`);\n    }\n    conditions.push(tx`${tx(key)} = ${value[key]}`);\n  }\n\n  return tx`DELETE FROM ${tx(tableName)} WHERE ${conditions}`;\n}\n\nasync function checkSchemaVersionAndIncrementLastMutationID(\n  tx: PostgresTransaction,\n  shard: ShardID,\n  clientGroupID: string,\n  clientID: string,\n  receivedMutationID: number,\n) {\n  const [{lastMutationID}] = await tx<{lastMutationID: bigint}[]>`\n    INSERT INTO ${tx(upstreamSchema(shard))}.clients \n      as current (\"clientGroupID\", \"clientID\", \"lastMutationID\")\n          VALUES (${clientGroupID}, ${clientID}, ${1})\n      ON CONFLICT (\"clientGroupID\", \"clientID\")\n      DO UPDATE SET \"lastMutationID\" = current.\"lastMutationID\" + 1\n      RETURNING \"lastMutationID\"\n  `;\n\n  // ABORT if the resulting lastMutationID is not equal to the receivedMutationID.\n  if (receivedMutationID < lastMutationID) {\n    throw new MutationAlreadyProcessedError(\n      clientID,\n      receivedMutationID,\n      lastMutationID,\n    );\n  } else if (receivedMutationID > lastMutationID) {\n    throw new ProtocolError({\n      kind: ErrorKind.InvalidPush,\n      message: `Push contains unexpected mutation id ${receivedMutationID} for client ${clientID}. Expected mutation id ${lastMutationID.toString()}.`,\n      origin: ErrorOrigin.ZeroCache,\n    });\n  }\n}\n\n// The slice of information from the Query object in Postgres.js that gets logged for debugging.\n// https://github.com/porsager/postgres/blob/f58cd4f3affd3e8ce8f53e42799672d86cd2c70b/src/connection.js#L219\ntype Query = {string: string; parameters: object[]};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA+DA,IAAa,iBAAb,MAAwD;CACtD;CACA;CACA;CACA;CACA,WAAoB,SAAS;CAC7B;CACA;CACA;CACA,YAAY;CACZ,aAAa;CAEb,iBAA0B,mBACxB,YACA,QACA,oCACF;CAEA,YACE,IACA,OACA,eACA,UACA,QACA,mBACA;EACA,KAAK,KAAK;EACV,KAAK,MAAM;EACX,KAAK,YAAY;EACjB,KAAK,SAAS;EACd,KAAK,WAAW,IAAI,SAAS,KAAK,KAAK,OAAO,QAAQ,MAAM,EAC1D,eAAe,KACjB,CAAC;EACD,KAAK,mBAAmB,IAAI,oBAC1B,KAAK,KACL,QACA,KAAK,UACL,MAAM,OACN,eACA,iBACF;EAEA,IAAI,OAAO,qBAAqB,QAAQ,KAAA,GACtC,KAAK,WAAW,IAAI,qBAClB,OAAO,qBAAqB,UAC5B,OAAO,qBAAqB,GAC9B;CAEJ;CAEA,MAAM;EACJ,OAAO,CAAC,KAAK,YAAY,mCAAmC;EAC5D,EAAE,KAAK;CACT;CAEA,QAAQ;EACN,OAAO,CAAC,KAAK,YAAY,mCAAmC;EAC5D,EAAE,KAAK;EACP,IAAI,KAAK,aAAa,GACpB,KAAU,KAAK;CAEnB;CAEA,UAAmB;EACjB,OAAO,KAAK,YAAY;CAC1B;CAEA,gBACE,UACA,UACA,wBAAwB,OACY;EACpC,IAAI,KAAK,UAAU,MAAM,MAAM,OAC7B,OAAO,QAAQ,QAAQ,CACrBA,qBACA,qBACF,CAAC;EAEH,KAAK,eAAe,IAAI,GAAG,EACzB,eAAe,KAAK,GACtB,CAAC;EACD,OAAO,gBACL,KAAK,KACL,UACA,KAAK,WACL,KAAK,QACL,KAAK,IACL,UACA,KAAK,kBACL,KAAA,GACA,qBACF;CACF;CAEA,MAAqB;EACnB,OAAO,KAAK,SAAS;CACvB;CAEA,OAAsB;EACpB,IAAI,KAAK,YACP,OAAO,KAAK,SAAS;EAEvB,KAAK,aAAa;EAClB,IAAI;GACF,KAAK,iBAAiB,QAAQ;EAChC,SAAS,GAAG;GACV,KAAK,IAAI,QAAQ,6CAA6C,CAAC;EACjE,UAAU;GAIR,KAAK,SAAS,MAAM,EAAC,UAAU,MAAK,CAAC;EACvC;EACA,KAAK,SAAS,QAAQ;EACtB,OAAO,KAAK,SAAS;CACvB;AACF;AAEA,IAAM,6BAA6B;AAEnC,eAAsB,gBACpB,IACA,UACA,IACA,OACA,eACA,UACA,iBACA,WACA,wBAAwB,OACY;CACpC,OACE,SAAS,SAASC,MAClB,mCACF;CAEA,KAAK,GAAG,YAAY,cAAc,SAAS,EAAE;CAC7C,KAAK,GAAG,YAAY,iBAAiB;CAGrC,GAAG,QAAQ,wBAAwB;CAGnC,eAAe,MAAM;CAErB,IAAI;CAEJ,MAAM,QAAQ,KAAK,IAAI;CACvB,IAAI;EAuCF,IAAI,YAAY;EAChB,KAAK,IAAI,IAAI,GAAG,IAAI,4BAA4B,KAC9C,IAAI;GACF,MAAM,MACJ,IACA,OAAM,OAAM;IAEV,MAAM,OAAO,YAAY;IACzB,IAAI;KACF,OAAO,MAAM,sBACX,IACA,IACA,UACA,OACA,eACA,UACA,WACA,eACF;IACF,UAAU;KACR,MAAM;IACR;GACF,GACA,EAAC,MAAMC,aAAiB,CAC1B;GACA,IAAI,WACF,GAAG,QAAQ,yCAAyC;GAEtD;EACF,SAAS,GAAG;GACV,IAAI,aAAa,+BAA+B;IAC9C,GAAG,QAAQ,EAAE,OAAO;IAEpB;GACF;GACA,IACE,gBAAgB,CAAC,KACjB,CAAC,aACD,EAAE,SAAS,iBACX,yBACA,IAAI,GACJ;IAOA,GAAG,OACD,kJACA,CACF;IACA,MAAM,IAAI,SAAQ,YAAW,WAAW,SAAS,GAAG,CAAC;IACrD;GACF;GACA,IAAI,gBAAgB,CAAC,KAAK,WAAW;IACnC,GAAG,QAAQ,0BAA0B,CAAC;IACtC,MAAM;GACR;GACA,IACE,aAAa,SAAS,iBACtB,EAAE,SAAS,0BACX;IACA,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C;GACF;GACA,SAAS,CAACC,gBAA0B,OAAO,CAAC,CAAC;GAC7C,IAAI,WACF;GAEF,GAAG,QAAQ,wDAAwD,CAAC;GACpE,YAAY;GACZ;EACF;CAEJ,UAAU;EACR,GAAG,QAAQ,gCAAgC,KAAK,IAAI,IAAI,KAAK;CAC/D;CACA,OAAO;AACT;AAEA,eAAsB,sBACpB,IACA,IACA,UACA,OACA,eACA,UACA,WACA,YACA;CACA,MAAM,QAAoC,CAAC;CAE3C,eAAe,QAAQ,MAA6C;EAClE,IAAI;GACF,OAAO,MAAM,KAAK,QAAQ;EAC5B,UAAU;GACR,MAAM,IAAI;GAGV,GAAG,QAAQ,GAAG,EAAE,OAAO,IAAI,EAAE,WAAW,OAAO,SAAS;EAC1D;CACF;CAEA,WAAW,kBAAkB;CAE7B,IAAI,CAAC,WAAW;EAGd,MAAM,MAAM,WAAW,YAAY,SAAS,KAAK,EAAE,CAAC,GAAG;EAEvD,MAAM,gBAAgB,WAAW,aAAa,GAAG;EACjD,MAAM,CAAC,QAAQ,WAAW,MAAM,QAAQ,IAAI,CAC1C,WAAW,eAAe,UAAU,aAAa,GACjD,WAAW,gBAAgB,UAAU,aAAa,CACpD,CAAC;EACD,IAAI,UAAU,SACZ,KAAK,MAAM,MAAM,KACf,QAAQ,GAAG,IAAX;GACE,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,SACE,YAAY,EAAE;EAClB;CAGN;CAIA,MAAM,cACJ,6CACE,IACA,OACA,eACA,SAAS,UACT,SAAS,EACX,CACF;CAGA,MAAM,QAAQ,IAAI,MAAM,KAAI,SAAQ,KAAK,CAAC,CAAC;AAC7C;AAEA,SAAgB,aACd,IACA,QACuC;CACvC,OAAO,EAAE;kBACO,GAAG,OAAO,SAAS,EAAE,GAAG,GAAG,OAAO,KAAK,EAAE;mBACxC,GAAG,OAAO,UAAU,EAAE;;AAEzC;AAEA,SAAgB,aACd,IACA,KACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CACvC,OAAO,EAAE;kBACO,GAAG,SAAS,EAAE,GAAG,GAAG,KAAK,EAAE;mBAC1B,GAAG,UAAU,EAAE;oBACd,GAAG,KAAK,EAAE;;AAE9B;AAEA,SAAS,aACP,IACA,QACuC;CACvC,MAAM,QAAQ,OAAO;CACrB,MAAM,EAAC,YAAY,UAAS;CAC5B,MAAM,KAAsC,CAAC;CAC7C,KAAK,MAAM,OAAO,YAChB,GAAG,OAAOC,MAAQ,MAAM,MAAM,qBAAqB;CAErD,OAAO,EAAE,UAAU,GAAG,KAAK,EAAE,OAAO,GAAG,KAAK,EAAE,SAAS,OAAO,QAC5D,EACF,CAAC,CAAC,SAAS,CAAC,KAAK,QAAQ,MACvB,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,OAAO,IAAI,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,OAC/D;AACF;AAEA,SAAS,aACP,IACA,UACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CAEvC,MAAM,aAAa,CAAC;CACpB,KAAK,MAAM,OAAO,YAAY;EAC5B,IAAI,WAAW,SAAS,GACtB,WAAW,KAAK,EAAE,KAAK;EAEzB,WAAW,KAAK,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,MAAM,MAAM;CAChD;CAEA,OAAO,EAAE,eAAe,GAAG,SAAS,EAAE,SAAS;AACjD;AAEA,eAAe,6CACb,IACA,OACA,eACA,UACA,oBACA;CACA,MAAM,CAAC,EAAC,oBAAmB,MAAM,EAA8B;kBAC/C,GAAG,eAAe,KAAK,CAAC,EAAE;;oBAExB,cAAc,IAAI,SAAS,IAAI,EAAE;;;;;CAOnD,IAAI,qBAAqB,gBACvB,MAAM,IAAI,8BACR,UACA,oBACA,cACF;MACK,IAAI,qBAAqB,gBAC9B,MAAM,IAAI,cAAc;EACtB,MAAMC;EACN,SAAS,wCAAwC,mBAAmB,cAAc,SAAS,yBAAyB,eAAe,SAAS,EAAE;EAC9I,QAAQC;CACV,CAAC;AAEL"}
+{"version":3,"file":"mutagen.js","names":["ErrorKind.MutationRateLimited","MutationType.CRUD","Mode.SERIALIZABLE","ErrorKind.MutationFailed","v.parse","ErrorKind.InvalidPush","ErrorOrigin.ZeroCache"],"sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"sourcesContent":["import {PG_SERIALIZATION_FAILURE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {JWTPayload} from 'jose';\nimport postgres from 'postgres';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport {resolver} from '../../../../shared/src/resolver.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {ErrorKind} from '../../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../../zero-protocol/src/error-origin.ts';\nimport {\n  isProtocolError,\n  ProtocolError,\n} from '../../../../zero-protocol/src/error.ts';\nimport * as MutationType from '../../../../zero-protocol/src/mutation-type-enum.ts';\nimport {\n  type CRUDMutation,\n  type DeleteOp,\n  type InsertOp,\n  type Mutation,\n  type UpdateOp,\n  type UpsertOp,\n} from '../../../../zero-protocol/src/mutation.ts';\nimport {\n  primaryKeyValueSchema,\n  type PrimaryKeyValue,\n} from '../../../../zero-protocol/src/primary-key.ts';\nimport type {DatabaseStorage} from '../../../../zqlite/src/database-storage.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  WriteAuthorizerImpl,\n  type WriteAuthorizer,\n} from '../../auth/write-authorizer.ts';\nimport {type ZeroConfig} from '../../config/zero-config.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {recordMutation} from '../../server/anonymous-otel-start.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../types/shards.ts';\nimport {SlidingWindowLimiter} from '../limiter/sliding-window-limiter.ts';\nimport type {RefCountedService, Service} from '../service.ts';\nimport {MutationAlreadyProcessedError} from './error.ts';\n\n// An error encountered processing a mutation.\n// Returned back to application for display to user.\nexport type MutationError = [\n  kind: ErrorKind.MutationFailed | ErrorKind.MutationRateLimited,\n  desc: string,\n];\n\nexport interface Mutagen extends RefCountedService {\n  /**\n   * Callers must hold a {@link ref} for the duration of a push (i.e. across\n   * all of its mutations). The service stops, closing its replica handle,\n   * when its ref count drops to zero.\n   */\n  processMutation(\n    mutation: Mutation,\n    authData: JWTPayload | undefined,\n    customMutatorsEnabled: boolean,\n  ): Promise<MutationError | undefined>;\n}\n\nexport class MutagenService implements Mutagen, Service {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #upstream: PostgresDB;\n  readonly #shard: ShardID;\n  readonly #stopped = resolver();\n  readonly #replica: Database;\n  readonly #writeAuthorizer: WriteAuthorizerImpl;\n  readonly #limiter: SlidingWindowLimiter | undefined;\n  #refCount = 0;\n  #isStopped = false;\n\n  readonly #crudMutations = getOrCreateCounter(\n    'mutation',\n    'crud',\n    'Number of CRUD mutations processed',\n  );\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    clientGroupID: string,\n    upstream: PostgresDB,\n    config: ZeroConfig,\n    writeAuthzStorage: DatabaseStorage,\n  ) {\n    this.id = clientGroupID;\n    this.#lc = lc;\n    this.#upstream = upstream;\n    this.#shard = shard;\n    this.#replica = new Database(this.#lc, config.replica.file, {\n      fileMustExist: true,\n    });\n    this.#writeAuthorizer = new WriteAuthorizerImpl(\n      this.#lc,\n      config,\n      this.#replica,\n      shard.appID,\n      clientGroupID,\n      writeAuthzStorage,\n    );\n\n    if (config.perUserMutationLimit.max !== undefined) {\n      this.#limiter = new SlidingWindowLimiter(\n        config.perUserMutationLimit.windowMs,\n        config.perUserMutationLimit.max,\n      );\n    }\n  }\n\n  ref() {\n    assert(!this.#isStopped, 'MutagenService is already stopped');\n    ++this.#refCount;\n  }\n\n  unref() {\n    assert(!this.#isStopped, 'MutagenService is already stopped');\n    --this.#refCount;\n    if (this.#refCount <= 0) {\n      void this.stop();\n    }\n  }\n\n  hasRefs(): boolean {\n    return this.#refCount > 0;\n  }\n\n  processMutation(\n    mutation: Mutation,\n    authData: JWTPayload | undefined,\n    customMutatorsEnabled = false,\n  ): Promise<MutationError | undefined> {\n    if (this.#limiter?.canDo() === false) {\n      return Promise.resolve([\n        ErrorKind.MutationRateLimited,\n        'Rate limit exceeded',\n      ]);\n    }\n    this.#crudMutations.add(1, {\n      clientGroupID: this.id,\n    });\n    return processMutation(\n      this.#lc,\n      authData,\n      this.#upstream,\n      this.#shard,\n      this.id,\n      mutation,\n      this.#writeAuthorizer,\n      undefined,\n      customMutatorsEnabled,\n    );\n  }\n\n  run(): Promise<void> {\n    return this.#stopped.promise;\n  }\n\n  stop(): Promise<void> {\n    if (this.#isStopped) {\n      return this.#stopped.promise;\n    }\n    this.#isStopped = true;\n    try {\n      this.#writeAuthorizer.destroy();\n    } catch (e) {\n      this.#lc.error?.('error destroying write authorizer storage', e);\n    } finally {\n      // The replica's statistics are maintained by the replicator, so skip\n      // the `PRAGMA optimize` that closing a writable handle would otherwise\n      // run on every client group churn.\n      this.#replica.close({optimize: false});\n    }\n    this.#stopped.resolve();\n    return this.#stopped.promise;\n  }\n}\n\nconst MAX_SERIALIZATION_ATTEMPTS = 10;\n\nexport async function processMutation(\n  lc: LogContext,\n  authData: JWTPayload | undefined,\n  db: PostgresDB,\n  shard: ShardID,\n  clientGroupID: string,\n  mutation: Mutation,\n  writeAuthorizer: WriteAuthorizer,\n  onTxStart?: () => void | Promise<void>, // for testing\n  customMutatorsEnabled = false,\n): Promise<MutationError | undefined> {\n  assert(\n    mutation.type === MutationType.CRUD,\n    'Only CRUD mutations are supported',\n  );\n  // log-leak-ignore -- the mutation id, not row data\n  lc = lc.withContext('mutationID', mutation.id);\n  lc = lc.withContext('processMutation');\n  // The mutationID context set above identifies it; the mutation itself\n  // carries the row values being written.\n  lc.debug?.('Process mutation start');\n\n  // Record mutation processing attempt for telemetry (regardless of success/failure)\n  recordMutation('crud');\n\n  let result: MutationError | undefined;\n\n  const start = Date.now();\n  try {\n    // Mutations can fail for a variety of reasons:\n    //\n    // - application error\n    // - network/db error\n    // - zero bug\n    //\n    // For application errors what we want is to re-run the mutation in\n    // \"error mode\", which skips the actual mutation and just updates the\n    // lastMutationID. Then return the error to the app.\n    //\n    // However, it's hard to tell the difference between application errors\n    // and the other types.\n    //\n    // A reasonable policy ends up being to just retry every mutation once\n    // in error mode. If the error mode mutation succeeds then we assume it\n    // was an application error and return the error to the app. Otherwise,\n    // we know it was something internal and we log it.\n    //\n    // This is not 100% correct - there are theoretical cases where we\n    // return an internal error to the app that shouldn't have been. But it\n    // would have to be a crazy coincidence: we'd have to have a network\n    // error on the first attempt that resolves by the second attempt.\n    //\n    // One might ask why not try/catch just the calls to the mutators and\n    // consider those application errors. That is actually what we do in\n    // Replicache:\n    //\n    // https://github.com/rocicorp/todo-row-versioning/blob/9a0a79dc2d2de32c4fac61b5d1634bd9a9e66b7c/server/src/push.ts#L131\n    //\n    // We don't do it here because:\n    //\n    // 1. It's still not perfect. It's hard to isolate SQL errors in\n    //    mutators due to app developer mistakes from SQL errors due to\n    //    Zero mistakes.\n    // 2. It's not possible to do this with the pg library we're using in\n    //    Zero anyway: https://github.com/porsager/postgres/issues/455.\n    //\n    // Personally I think this simple retry policy is nice.\n    let errorMode = false;\n    for (let i = 0; i < MAX_SERIALIZATION_ATTEMPTS; i++) {\n      try {\n        await runTx(\n          db,\n          async tx => {\n            // Simulates a concurrent request for testing. In production this is a noop.\n            const done = onTxStart?.();\n            try {\n              return await processMutationWithTx(\n                lc,\n                tx,\n                authData,\n                shard,\n                clientGroupID,\n                mutation,\n                errorMode,\n                writeAuthorizer,\n              );\n            } finally {\n              await done;\n            }\n          },\n          {mode: Mode.SERIALIZABLE},\n        );\n        if (errorMode) {\n          lc.debug?.('Ran mutation successfully in error mode');\n        }\n        break;\n      } catch (e) {\n        if (e instanceof MutationAlreadyProcessedError) {\n          lc.debug?.(e.message);\n          // Don't double-count already processed mutations, but they were counted above\n          return undefined;\n        }\n        if (\n          isProtocolError(e) &&\n          !errorMode &&\n          e.kind === ErrorKind.InvalidPush &&\n          customMutatorsEnabled &&\n          i < 2\n        ) {\n          // We're temporarily supporting custom mutators AND CRUD mutators at the same time.\n          // This can create a lot of OOO mutation errors since we do not know when the API server\n          // has applied a custom mutation before moving on to process CRUD mutations.\n          // The temporary workaround (since CRUD is being deprecated) is to retry the mutation\n          // after a small delay. Users are not expected to be running both CRUD and Custom mutators.\n          // They should migrate completely to custom mutators.\n          lc.info?.(\n            'Both CRUD and Custom mutators are being used at once. This is supported for now but IS NOT RECOMMENDED. Migrate completely to custom mutators.',\n            e,\n          );\n          await new Promise(resolve => setTimeout(resolve, 100));\n          continue;\n        }\n        if (isProtocolError(e) || errorMode) {\n          lc.error?.('Process mutation error', e);\n          throw e;\n        }\n        if (\n          e instanceof postgres.PostgresError &&\n          e.code === PG_SERIALIZATION_FAILURE\n        ) {\n          lc.info?.(`attempt ${i + 1}: ${String(e)}`, e);\n          continue; // Retry up to MAX_SERIALIZATION_ATTEMPTS.\n        }\n        result = [ErrorKind.MutationFailed, String(e)];\n        if (errorMode) {\n          break;\n        }\n        lc.error?.('Got error running mutation, re-running in error mode', e);\n        errorMode = true;\n        i--;\n      }\n    }\n  } finally {\n    lc.debug?.('Process mutation complete in', Date.now() - start);\n  }\n  return result;\n}\n\nasync function processMutationWithTx(\n  lc: LogContext,\n  tx: PostgresTransaction,\n  authData: JWTPayload | undefined,\n  shard: ShardID,\n  clientGroupID: string,\n  mutation: CRUDMutation,\n  errorMode: boolean,\n  authorizer: WriteAuthorizer,\n) {\n  const tasks: (() => Promise<unknown>)[] = [];\n\n  async function execute(stmt: postgres.PendingQuery<postgres.Row[]>) {\n    try {\n      return await stmt.execute();\n    } finally {\n      const q = stmt as unknown as Query;\n      // `q.parameters` are the bound row values -- log the parameterized\n      // statement and the parameter count, never the values themselves.\n      lc.debug?.(`${q.string} (${q.parameters.length} params)`);\n    }\n  }\n\n  authorizer.reloadPermissions();\n\n  if (!errorMode) {\n    // Validates table names and keys every op by the replica's primary key,\n    // which is what the authorizer checks.\n    const ops = authorizer.validateOps(mutation.args[0].ops);\n\n    const normalizedOps = authorizer.normalizeOps(ops);\n    const [canPre, canPost] = await Promise.all([\n      authorizer.canPreMutation(authData, normalizedOps),\n      authorizer.canPostMutation(authData, normalizedOps),\n    ]);\n    if (canPre && canPost) {\n      for (const op of ops) {\n        switch (op.op) {\n          case 'insert':\n            tasks.push(() => execute(getInsertSQL(tx, op)));\n            break;\n          case 'upsert':\n            tasks.push(() => execute(getUpsertSQL(tx, op)));\n            break;\n          case 'update':\n            tasks.push(() => execute(getUpdateSQL(tx, op)));\n            break;\n          case 'delete':\n            tasks.push(() => execute(getDeleteSQL(tx, op)));\n            break;\n          default:\n            unreachable(op);\n        }\n      }\n    }\n  }\n\n  // Confirm the mutation even though it may have been blocked by the authorizer.\n  // Authorizer blocking a mutation is not an error but the correct result of the mutation.\n  tasks.unshift(() =>\n    checkSchemaVersionAndIncrementLastMutationID(\n      tx,\n      shard,\n      clientGroupID,\n      mutation.clientID,\n      mutation.id,\n    ),\n  );\n\n  // Note: An error thrown from any Promise aborts the entire transaction.\n  await Promise.all(tasks.map(task => task()));\n}\n\nfunction getInsertSQL(\n  tx: postgres.TransactionSql,\n  create: InsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  return tx`\n    INSERT INTO ${tx(create.tableName)} ${tx(create.value)}\n    ON CONFLICT (${tx(create.primaryKey)}) DO NOTHING\n  `;\n}\n\nfunction getUpsertSQL(\n  tx: postgres.TransactionSql,\n  set: UpsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const {tableName, primaryKey, value} = set;\n  return tx`\n    INSERT INTO ${tx(tableName)} ${tx(value)}\n    ON CONFLICT (${tx(primaryKey)})\n    DO UPDATE SET ${tx(value)}\n  `;\n}\n\nfunction getUpdateSQL(\n  tx: postgres.TransactionSql,\n  update: UpdateOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const table = update.tableName;\n  const {primaryKey, value} = update;\n  const id: Record<string, PrimaryKeyValue> = {};\n  for (const key of primaryKey) {\n    id[key] = v.parse(value[key], primaryKeyValueSchema);\n  }\n  return tx`UPDATE ${tx(table)} SET ${tx(value)} WHERE ${Object.entries(\n    id,\n  ).flatMap(([key, value], i) =>\n    i ? [tx`AND`, tx`${tx(key)} = ${value}`] : tx`${tx(key)} = ${value}`,\n  )}`;\n}\n\nfunction getDeleteSQL(\n  tx: postgres.TransactionSql,\n  deleteOp: DeleteOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const {tableName, primaryKey, value} = deleteOp;\n\n  const conditions = [];\n  for (const key of primaryKey) {\n    if (conditions.length > 0) {\n      conditions.push(tx`AND`);\n    }\n    conditions.push(tx`${tx(key)} = ${value[key]}`);\n  }\n\n  return tx`DELETE FROM ${tx(tableName)} WHERE ${conditions}`;\n}\n\nasync function checkSchemaVersionAndIncrementLastMutationID(\n  tx: PostgresTransaction,\n  shard: ShardID,\n  clientGroupID: string,\n  clientID: string,\n  receivedMutationID: number,\n) {\n  const [{lastMutationID}] = await tx<{lastMutationID: bigint}[]>`\n    INSERT INTO ${tx(upstreamSchema(shard))}.clients \n      as current (\"clientGroupID\", \"clientID\", \"lastMutationID\")\n          VALUES (${clientGroupID}, ${clientID}, ${1})\n      ON CONFLICT (\"clientGroupID\", \"clientID\")\n      DO UPDATE SET \"lastMutationID\" = current.\"lastMutationID\" + 1\n      RETURNING \"lastMutationID\"\n  `;\n\n  // ABORT if the resulting lastMutationID is not equal to the receivedMutationID.\n  if (receivedMutationID < lastMutationID) {\n    throw new MutationAlreadyProcessedError(\n      clientID,\n      receivedMutationID,\n      lastMutationID,\n    );\n  } else if (receivedMutationID > lastMutationID) {\n    throw new ProtocolError({\n      kind: ErrorKind.InvalidPush,\n      message: `Push contains unexpected mutation id ${receivedMutationID} for client ${clientID}. Expected mutation id ${lastMutationID.toString()}.`,\n      origin: ErrorOrigin.ZeroCache,\n    });\n  }\n}\n\n// The slice of information from the Query object in Postgres.js that gets logged for debugging.\n// https://github.com/porsager/postgres/blob/f58cd4f3affd3e8ce8f53e42799672d86cd2c70b/src/connection.js#L219\ntype Query = {string: string; parameters: object[]};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA+DA,IAAa,iBAAb,MAAwD;CACtD;CACA;CACA;CACA;CACA,WAAoB,SAAS;CAC7B;CACA;CACA;CACA,YAAY;CACZ,aAAa;CAEb,iBAA0B,mBACxB,YACA,QACA,oCACF;CAEA,YACE,IACA,OACA,eACA,UACA,QACA,mBACA;EACA,KAAK,KAAK;EACV,KAAK,MAAM;EACX,KAAK,YAAY;EACjB,KAAK,SAAS;EACd,KAAK,WAAW,IAAI,SAAS,KAAK,KAAK,OAAO,QAAQ,MAAM,EAC1D,eAAe,KACjB,CAAC;EACD,KAAK,mBAAmB,IAAI,oBAC1B,KAAK,KACL,QACA,KAAK,UACL,MAAM,OACN,eACA,iBACF;EAEA,IAAI,OAAO,qBAAqB,QAAQ,KAAA,GACtC,KAAK,WAAW,IAAI,qBAClB,OAAO,qBAAqB,UAC5B,OAAO,qBAAqB,GAC9B;CAEJ;CAEA,MAAM;EACJ,OAAO,CAAC,KAAK,YAAY,mCAAmC;EAC5D,EAAE,KAAK;CACT;CAEA,QAAQ;EACN,OAAO,CAAC,KAAK,YAAY,mCAAmC;EAC5D,EAAE,KAAK;EACP,IAAI,KAAK,aAAa,GACpB,KAAU,KAAK;CAEnB;CAEA,UAAmB;EACjB,OAAO,KAAK,YAAY;CAC1B;CAEA,gBACE,UACA,UACA,wBAAwB,OACY;EACpC,IAAI,KAAK,UAAU,MAAM,MAAM,OAC7B,OAAO,QAAQ,QAAQ,CACrBA,qBACA,qBACF,CAAC;EAEH,KAAK,eAAe,IAAI,GAAG,EACzB,eAAe,KAAK,GACtB,CAAC;EACD,OAAO,gBACL,KAAK,KACL,UACA,KAAK,WACL,KAAK,QACL,KAAK,IACL,UACA,KAAK,kBACL,KAAA,GACA,qBACF;CACF;CAEA,MAAqB;EACnB,OAAO,KAAK,SAAS;CACvB;CAEA,OAAsB;EACpB,IAAI,KAAK,YACP,OAAO,KAAK,SAAS;EAEvB,KAAK,aAAa;EAClB,IAAI;GACF,KAAK,iBAAiB,QAAQ;EAChC,SAAS,GAAG;GACV,KAAK,IAAI,QAAQ,6CAA6C,CAAC;EACjE,UAAU;GAIR,KAAK,SAAS,MAAM,EAAC,UAAU,MAAK,CAAC;EACvC;EACA,KAAK,SAAS,QAAQ;EACtB,OAAO,KAAK,SAAS;CACvB;AACF;AAEA,IAAM,6BAA6B;AAEnC,eAAsB,gBACpB,IACA,UACA,IACA,OACA,eACA,UACA,iBACA,WACA,wBAAwB,OACY;CACpC,OACE,SAAS,SAASC,MAClB,mCACF;CAEA,KAAK,GAAG,YAAY,cAAc,SAAS,EAAE;CAC7C,KAAK,GAAG,YAAY,iBAAiB;CAGrC,GAAG,QAAQ,wBAAwB;CAGnC,eAAe,MAAM;CAErB,IAAI;CAEJ,MAAM,QAAQ,KAAK,IAAI;CACvB,IAAI;EAuCF,IAAI,YAAY;EAChB,KAAK,IAAI,IAAI,GAAG,IAAI,4BAA4B,KAC9C,IAAI;GACF,MAAM,MACJ,IACA,OAAM,OAAM;IAEV,MAAM,OAAO,YAAY;IACzB,IAAI;KACF,OAAO,MAAM,sBACX,IACA,IACA,UACA,OACA,eACA,UACA,WACA,eACF;IACF,UAAU;KACR,MAAM;IACR;GACF,GACA,EAAC,MAAMC,aAAiB,CAC1B;GACA,IAAI,WACF,GAAG,QAAQ,yCAAyC;GAEtD;EACF,SAAS,GAAG;GACV,IAAI,aAAa,+BAA+B;IAC9C,GAAG,QAAQ,EAAE,OAAO;IAEpB;GACF;GACA,IACE,gBAAgB,CAAC,KACjB,CAAC,aACD,EAAE,SAAS,iBACX,yBACA,IAAI,GACJ;IAOA,GAAG,OACD,kJACA,CACF;IACA,MAAM,IAAI,SAAQ,YAAW,WAAW,SAAS,GAAG,CAAC;IACrD;GACF;GACA,IAAI,gBAAgB,CAAC,KAAK,WAAW;IACnC,GAAG,QAAQ,0BAA0B,CAAC;IACtC,MAAM;GACR;GACA,IACE,aAAa,SAAS,iBACtB,EAAE,SAAS,0BACX;IACA,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C;GACF;GACA,SAAS,CAACC,gBAA0B,OAAO,CAAC,CAAC;GAC7C,IAAI,WACF;GAEF,GAAG,QAAQ,wDAAwD,CAAC;GACpE,YAAY;GACZ;EACF;CAEJ,UAAU;EACR,GAAG,QAAQ,gCAAgC,KAAK,IAAI,IAAI,KAAK;CAC/D;CACA,OAAO;AACT;AAEA,eAAe,sBACb,IACA,IACA,UACA,OACA,eACA,UACA,WACA,YACA;CACA,MAAM,QAAoC,CAAC;CAE3C,eAAe,QAAQ,MAA6C;EAClE,IAAI;GACF,OAAO,MAAM,KAAK,QAAQ;EAC5B,UAAU;GACR,MAAM,IAAI;GAGV,GAAG,QAAQ,GAAG,EAAE,OAAO,IAAI,EAAE,WAAW,OAAO,SAAS;EAC1D;CACF;CAEA,WAAW,kBAAkB;CAE7B,IAAI,CAAC,WAAW;EAGd,MAAM,MAAM,WAAW,YAAY,SAAS,KAAK,EAAE,CAAC,GAAG;EAEvD,MAAM,gBAAgB,WAAW,aAAa,GAAG;EACjD,MAAM,CAAC,QAAQ,WAAW,MAAM,QAAQ,IAAI,CAC1C,WAAW,eAAe,UAAU,aAAa,GACjD,WAAW,gBAAgB,UAAU,aAAa,CACpD,CAAC;EACD,IAAI,UAAU,SACZ,KAAK,MAAM,MAAM,KACf,QAAQ,GAAG,IAAX;GACE,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,SACE,YAAY,EAAE;EAClB;CAGN;CAIA,MAAM,cACJ,6CACE,IACA,OACA,eACA,SAAS,UACT,SAAS,EACX,CACF;CAGA,MAAM,QAAQ,IAAI,MAAM,KAAI,SAAQ,KAAK,CAAC,CAAC;AAC7C;AAEA,SAAS,aACP,IACA,QACuC;CACvC,OAAO,EAAE;kBACO,GAAG,OAAO,SAAS,EAAE,GAAG,GAAG,OAAO,KAAK,EAAE;mBACxC,GAAG,OAAO,UAAU,EAAE;;AAEzC;AAEA,SAAS,aACP,IACA,KACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CACvC,OAAO,EAAE;kBACO,GAAG,SAAS,EAAE,GAAG,GAAG,KAAK,EAAE;mBAC1B,GAAG,UAAU,EAAE;oBACd,GAAG,KAAK,EAAE;;AAE9B;AAEA,SAAS,aACP,IACA,QACuC;CACvC,MAAM,QAAQ,OAAO;CACrB,MAAM,EAAC,YAAY,UAAS;CAC5B,MAAM,KAAsC,CAAC;CAC7C,KAAK,MAAM,OAAO,YAChB,GAAG,OAAOC,MAAQ,MAAM,MAAM,qBAAqB;CAErD,OAAO,EAAE,UAAU,GAAG,KAAK,EAAE,OAAO,GAAG,KAAK,EAAE,SAAS,OAAO,QAC5D,EACF,CAAC,CAAC,SAAS,CAAC,KAAK,QAAQ,MACvB,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,OAAO,IAAI,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,OAC/D;AACF;AAEA,SAAS,aACP,IACA,UACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CAEvC,MAAM,aAAa,CAAC;CACpB,KAAK,MAAM,OAAO,YAAY;EAC5B,IAAI,WAAW,SAAS,GACtB,WAAW,KAAK,EAAE,KAAK;EAEzB,WAAW,KAAK,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,MAAM,MAAM;CAChD;CAEA,OAAO,EAAE,eAAe,GAAG,SAAS,EAAE,SAAS;AACjD;AAEA,eAAe,6CACb,IACA,OACA,eACA,UACA,oBACA;CACA,MAAM,CAAC,EAAC,oBAAmB,MAAM,EAA8B;kBAC/C,GAAG,eAAe,KAAK,CAAC,EAAE;;oBAExB,cAAc,IAAI,SAAS,IAAI,EAAE;;;;;CAOnD,IAAI,qBAAqB,gBACvB,MAAM,IAAI,8BACR,UACA,oBACA,cACF;MACK,IAAI,qBAAqB,gBAC9B,MAAM,IAAI,cAAc;EACtB,MAAMC;EACN,SAAS,wCAAwC,mBAAmB,cAAc,SAAS,yBAAyB,eAAe,SAAS,EAAE;EAC9I,QAAQC;CACV,CAAC;AAEL"}

package/out/zero-cache/src/services/plannerz.d.ts

@@ -43,7 +43,7 @@ export type Stat4Sample = {
  * hundreds of samples per index, which is a lot of context to spend on data an
  * LLM would only summarize anyway.
  */
-export type Stat4PrefixSummary = {
+type Stat4PrefixSummary = {
     /** Rows equal to the most common sampled key at this depth. */
     maxRowsPerKey: number;
     /** Median rows per sampled key at this depth. */
@@ -61,7 +61,7 @@ export type Stat4Summary = {
     /** One entry per key prefix depth: 1 column, then 2, and so on. */
     perPrefix: Stat4PrefixSummary[];
 };
-export type IndexStats = {
+type IndexStats = {
     name: string;
     columns: {
         name: string;
@@ -74,7 +74,7 @@ export type IndexStats = {
     /** The full histogram, only with `?stat4=full`. */
     stat4Samples?: Stat4Sample[] | undefined;
 };
-export type ColumnStats = {
+type ColumnStats = {
     name: string;
     dataType: string;
     nullable: boolean;
@@ -82,7 +82,7 @@ export type ColumnStats = {
     zqlType: string | null;
     backfilling?: true | undefined;
 };
-export type TableStats = {
+type TableStats = {
     name: string;
     columns: ColumnStats[];
     primaryKey: string[] | undefined;
@@ -117,14 +117,6 @@ export type PlannerzBundle = {
  * @visibleForTesting
  */
 export declare function parseStat1(raw: string): Stat1;
-/**
- * Builds the `/plannerz` bundle from the replica's catalog, `sqlite_stat*`
- * tables and the server config. No user table is read, so the cost does not
- * depend on how much data the replica holds.
- *
- * @visibleForTesting
- */
-export declare function buildPlannerzBundle(lc: LogContext, config: ZeroConfig, opts?: BundleOptions): PlannerzBundle;
 export type BundleOptions = {
     /** Include the full `sqlite_stat4` histogram, not just the digest. */
     fullStat4?: boolean | undefined;

package/out/zero-cache/src/services/plannerz.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"plannerz.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/plannerz.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,SAAS,CAAC;AAC1D,OAAO,EAAC,QAAQ,EAAC,MAAM,2BAA2B,CAAC;AACnD,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,4CAA4C,CAAC;AACpD,OAAO,KAAK,EAAC,oBAAoB,IAAI,UAAU,EAAC,MAAM,wBAAwB,CAAC;AAuB/E;;;;GAIG;AACH,QAAA,MAAM,KAAK;aACT,UAAU;aAIV,WAAW;aAMX,WAAW;aAOX,qBAAqB;aAUrB,gBAAgB,oBAGd,+CAA+C,UAG/C,wCAAwC;aAE1C,OAAO;CASC,CAAC;AAEX,MAAM,MAAM,KAAK,GAAG;IAClB,2EAA2E;IAC3E,GAAG,EAAE,MAAM,CAAC;IACZ,mCAAmC;IACnC,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,kEAAkE;IAClE,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB,CAAC;AAEF,uEAAuE;AACvE,MAAM,MAAM,WAAW,GAAG;IACxB,GAAG,EAAE,MAAM,EAAE,CAAC;IACd,GAAG,EAAE,MAAM,EAAE,CAAC;IACd,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,yDAAyD;IACzD,MAAM,EAAE,eAAe,EAAE,CAAC;CAC3B,CAAC;AAEF;;;;;GAKG;AACH,MAAM,MAAM,kBAAkB,GAAG;IAC/B,+DAA+D;IAC/D,aAAa,EAAE,MAAM,CAAC;IACtB,iDAAiD;IACjD,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,qBAAqB,EAAE,MAAM,CAAC;CAC/B,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,8DAA8D;IAC9D,WAAW,EAAE,eAAe,EAAE,CAAC;IAC/B,mEAAmE;IACnE,SAAS,EAAE,kBAAkB,EAAE,CAAC;CACjC,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,KAAK,GAAG,MAAM,CAAA;KAAC,EAAE,CAAC;IAC/C,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,KAAK,GAAG,SAAS,CAAC;IAC1B,KAAK,CAAC,EAAE,YAAY,GAAG,SAAS,CAAC;IACjC,mDAAmD;IACnD,YAAY,CAAC,EAAE,WAAW,EAAE,GAAG,SAAS,CAAC;CAC1C,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,OAAO,CAAC;IAClB,+DAA+D;IAC/D,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,WAAW,CAAC,EAAE,IAAI,GAAG,SAAS,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,UAAU,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IACjC,aAAa,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,QAAQ,EAAE,OAAO,CAAC;IAClB,OAAO,EAAE,UAAU,EAAE,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,KAAK,EAAE,OAAO,KAAK,CAAC;IACpB,MAAM,EAAE;QACN,WAAW,EAAE,MAAM,CAAC;QACpB,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,MAAM,GAAG,SAAS,CAAC;QACrC,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE;YACP,kBAAkB,EAAE,OAAO,CAAC;YAC5B,iCAAiC,EAAE,OAAO,CAAC;YAC3C,0BAA0B,EAAE,OAAO,CAAC;SACrC,CAAC;KACH,CAAC;IACF,YAAY,EAAE;QACZ,YAAY,EAAE,OAAO,CAAC;QACtB,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;IACF,MAAM,EAAE,UAAU,EAAE,CAAC;CACtB,CAAC;AAEF;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,KAAK,CAmB7C;AAmCD;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CACjC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,IAAI,GAAE,aAAkB,GACvB,cAAc,CAGhB;AAED,MAAM,MAAM,aAAa,GAAG;IAC1B,sEAAsE;IACtE,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACjC,CAAC;AAEF,yBAAyB;AACzB,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,EACZ,MAAM,EAAE,UAAU,EAClB,IAAI,GAAE,aAAkB,GACvB,cAAc,CAsHhB;AAiBD;;;;;;;;;;GAUG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,YAAY,CAuBnE;AAmBD;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,GAAG,EAAE,cAAc,EACnB,GAAG,EAAE,YAAY,iBA0BlB"}
+{"version":3,"file":"plannerz.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/plannerz.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,SAAS,CAAC;AAC1D,OAAO,EAAC,QAAQ,EAAC,MAAM,2BAA2B,CAAC;AACnD,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,4CAA4C,CAAC;AACpD,OAAO,KAAK,EAAC,oBAAoB,IAAI,UAAU,EAAC,MAAM,wBAAwB,CAAC;AAuB/E;;;;GAIG;AACH,QAAA,MAAM,KAAK;aACT,UAAU;aAIV,WAAW;aAMX,WAAW;aAOX,qBAAqB;aAUrB,gBAAgB,oBAGd,+CAA+C,UAG/C,wCAAwC;aAE1C,OAAO;CASC,CAAC;AAEX,MAAM,MAAM,KAAK,GAAG;IAClB,2EAA2E;IAC3E,GAAG,EAAE,MAAM,CAAC;IACZ,mCAAmC;IACnC,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,kEAAkE;IAClE,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB,CAAC;AAEF,uEAAuE;AACvE,MAAM,MAAM,WAAW,GAAG;IACxB,GAAG,EAAE,MAAM,EAAE,CAAC;IACd,GAAG,EAAE,MAAM,EAAE,CAAC;IACd,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,yDAAyD;IACzD,MAAM,EAAE,eAAe,EAAE,CAAC;CAC3B,CAAC;AAEF;;;;;GAKG;AACH,KAAK,kBAAkB,GAAG;IACxB,+DAA+D;IAC/D,aAAa,EAAE,MAAM,CAAC;IACtB,iDAAiD;IACjD,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,qBAAqB,EAAE,MAAM,CAAC;CAC/B,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,8DAA8D;IAC9D,WAAW,EAAE,eAAe,EAAE,CAAC;IAC/B,mEAAmE;IACnE,SAAS,EAAE,kBAAkB,EAAE,CAAC;CACjC,CAAC;AAEF,KAAK,UAAU,GAAG;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,KAAK,GAAG,MAAM,CAAA;KAAC,EAAE,CAAC;IAC/C,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,KAAK,GAAG,SAAS,CAAC;IAC1B,KAAK,CAAC,EAAE,YAAY,GAAG,SAAS,CAAC;IACjC,mDAAmD;IACnD,YAAY,CAAC,EAAE,WAAW,EAAE,GAAG,SAAS,CAAC;CAC1C,CAAC;AAEF,KAAK,WAAW,GAAG;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,OAAO,CAAC;IAClB,+DAA+D;IAC/D,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,WAAW,CAAC,EAAE,IAAI,GAAG,SAAS,CAAC;CAChC,CAAC;AAEF,KAAK,UAAU,GAAG;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,UAAU,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IACjC,aAAa,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,QAAQ,EAAE,OAAO,CAAC;IAClB,OAAO,EAAE,UAAU,EAAE,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,KAAK,EAAE,OAAO,KAAK,CAAC;IACpB,MAAM,EAAE;QACN,WAAW,EAAE,MAAM,CAAC;QACpB,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,MAAM,GAAG,SAAS,CAAC;QACrC,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE;YACP,kBAAkB,EAAE,OAAO,CAAC;YAC5B,iCAAiC,EAAE,OAAO,CAAC;YAC3C,0BAA0B,EAAE,OAAO,CAAC;SACrC,CAAC;KACH,CAAC;IACF,YAAY,EAAE;QACZ,YAAY,EAAE,OAAO,CAAC;QACtB,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;IACF,MAAM,EAAE,UAAU,EAAE,CAAC;CACtB,CAAC;AAEF;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,KAAK,CAmB7C;AAiDD,MAAM,MAAM,aAAa,GAAG;IAC1B,sEAAsE;IACtE,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACjC,CAAC;AAEF,yBAAyB;AACzB,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,EACZ,MAAM,EAAE,UAAU,EAClB,IAAI,GAAE,aAAkB,GACvB,cAAc,CAsHhB;AAiBD;;;;;;;;;;GAUG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,YAAY,CAuBnE;AAmBD;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,GAAG,EAAE,cAAc,EACnB,GAAG,EAAE,YAAY,iBA0BlB"}

package/out/zero-cache/src/services/plannerz.js

@@ -73,8 +73,6 @@ function parseCounts(raw) {
 * Builds the `/plannerz` bundle from the replica's catalog, `sqlite_stat*`
 * tables and the server config. No user table is read, so the cost does not
 * depend on how much data the replica holds.
-*
-* @visibleForTesting
 */
 function buildPlannerzBundle(lc, config, opts = {}) {
 	try {
@@ -263,6 +261,6 @@ async function handlePlannerzRequest(lc, config, req, res) {
 	await res.header("Content-Type", "application/json").send(JSON.stringify(bundle, null, indent));
 }
 //#endregion
-export { buildPlannerzBundle, buildPlannerzBundleFromDB, handlePlannerzRequest, parseStat1, summarizeStat4 };
+export { buildPlannerzBundleFromDB, handlePlannerzRequest, parseStat1, summarizeStat4 };
 
 //# sourceMappingURL=plannerz.js.map

package/out/zero-cache/src/services/plannerz.js.map

@@ -1 +1 @@
-{"version":3,"file":"plannerz.js","names":[],"sources":["../../../../../zero-cache/src/services/plannerz.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport auth from 'basic-auth';\nimport type {FastifyReply, FastifyRequest} from 'fastify';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport {\n  decodeSampleKinds,\n  type SampleValueKind,\n} from '../../../zqlite/src/sqlite-stat4-sample.ts';\nimport type {NormalizedZeroConfig as ZeroConfig} from '../config/normalize.ts';\nimport {getOperatorAccess, getServerVersion} from '../config/zero-config.ts';\nimport {\n  computeZqlSpecsFromLiteSpecs,\n  listIndexes,\n  listTables,\n} from '../db/lite-tables.ts';\nimport {StatementRunner} from '../db/statements.ts';\nimport {getReplicationState} from './replicator/schema/replication-state.ts';\n\n/**\n * How the statistics in the replica were gathered. zero-cache only ever runs\n * `PRAGMA optimize`, and always under an `analysis_limit`, which samples rather\n * than scanning. A replica that someone ran `ANALYZE` on by hand has better\n * statistics, which is visible in `sqlite_stat4` having rows at all.\n */\nconst APPROXIMATE_METHOD =\n  'PRAGMA optimize under an analysis_limit. Row counts and per-key averages ' +\n  'are sampled estimates, and sqlite_stat4 is not collected.';\nconst FULL_METHOD =\n  'ANALYZE, with no analysis limit. sqlite_stat4 has rows, so a full ANALYZE ' +\n  'ran against this replica at some point.';\n\n/**\n * A fixed glossary, sent with every bundle. The audience is a language model\n * that has the user's codebase but has never seen these statistics, and would\n * otherwise read a sampled estimate as an exact count.\n */\nconst ABOUT = {\n  whatThisIs:\n    'Schema, indexes and SQLite statistics for the SQLite replica that ' +\n    'zero-cache queries. Use it to decide how to rewrite ZQL queries and ' +\n    'which indexes to add. It contains no row data.',\n  stat1Format:\n    'sqlite_stat1.stat is \"<rows> <avg rows per distinct 1-column prefix> ' +\n    '<... per 2-column prefix> ...\", parsed here into `rows` and ' +\n    '`avgRowsPerPrefix`. NULLs count as ordinary values, so a sparse column ' +\n    'looks less selective than it is. Trailing flags such as \"unordered\", ' +\n    '\"noskipscan\" and \"sz=N\" are kept in `flags`.',\n  stat4Format:\n    'sqlite_stat4 is a per-index histogram. For each sample, `nEq` is the ' +\n    'rows equal to it, `nLt` the rows less than it, and `nDLt` the distinct ' +\n    'values less than it, one entry per key column. A large `nEq` next to a ' +\n    'small `nDLt` means a skewed column, where the average in stat1 hides ' +\n    'the common values. Sample values are never returned: `sample` reports ' +\n    'only each key column’s type.',\n  howThePlannerUsesThis: [\n    'Cost comes from SQLite scanstatus estimates for the SQL each source ' +\n      'connection would run, so the statistics here are what the planner sees.',\n    'Join fanout comes from sqlite_stat4, else sqlite_stat1, else a default ' +\n      'of 3. With no stat4 (the usual case) fanout on a nullable foreign key ' +\n      'is overestimated, because stat1 counts NULLs.',\n    'The planner chooses per join whether to run it as a semi-join or to flip ' +\n      'it so the child drives. An index that matches the child’s ' +\n      'constraint plus ordering is what makes a flip cheap.',\n  ],\n  whatYouCanChange: [\n    'The ZQL query: `whereExists` vs `related`, the order of conditions, and ' +\n      'ordering that matches an existing index.',\n    'A `limit`, which lets the planner stop early.',\n    'Indexes: add them in Postgres, not on the replica. The replica copies ' +\n      'upstream indexes, so a new index reaches it through replication.',\n    'The planner flags in `server.planner`.',\n  ],\n  caveats: [\n    'Row counts are estimates from the statistics, not COUNT(*). This ' +\n      'endpoint never scans a table.',\n    'Statistics are as of the last time they were gathered, which SQLite does ' +\n      'not record. A table whose size changed a lot since then can be far off.',\n    'Tables in `tables` with `syncable: false` cannot be queried by clients: ' +\n      'they lack a primary key or unique index, or every column has a type ' +\n      'ZQL does not support.',\n  ],\n} as const;\n\nexport type Stat1 = {\n  /** The raw `sqlite_stat1.stat` text, in case a flag is not parsed here. */\n  raw: string;\n  /** Estimated rows in the table. */\n  rows: number;\n  /**\n   * Average rows per distinct key prefix: the first entry is for a 1-column\n   * prefix, the second for 2 columns, and so on.\n   */\n  avgRowsPerPrefix: number[];\n  /** Non-numeric tokens, e.g. `unordered`, `noskipscan`, `sz=N`. */\n  flags: string[];\n};\n\n/** A `sqlite_stat4` sample with its values replaced by their types. */\nexport type Stat4Sample = {\n  nEq: number[];\n  nLt: number[];\n  nDLt: number[];\n  /** One entry per key column, plus the trailing rowid. */\n  sample: SampleValueKind[];\n};\n\n/**\n * A digest of an index's `sqlite_stat4` histogram, one entry per key prefix\n * depth. This is what the default response carries: a full histogram runs to\n * hundreds of samples per index, which is a lot of context to spend on data an\n * LLM would only summarize anyway.\n */\nexport type Stat4PrefixSummary = {\n  /** Rows equal to the most common sampled key at this depth. */\n  maxRowsPerKey: number;\n  /** Median rows per sampled key at this depth. */\n  medianRowsPerKey: number;\n  /**\n   * Lower bound on distinct keys at this depth, from the last sample's count of\n   * distinct values less than it.\n   */\n  estimatedDistinctKeys: number;\n};\n\nexport type Stat4Summary = {\n  samples: number;\n  /** The type of each key column, and of the trailing rowid. */\n  sampleKinds: SampleValueKind[];\n  /** One entry per key prefix depth: 1 column, then 2, and so on. */\n  perPrefix: Stat4PrefixSummary[];\n};\n\nexport type IndexStats = {\n  name: string;\n  columns: {name: string; dir: 'ASC' | 'DESC'}[];\n  unique: boolean;\n  partial: boolean;\n  stat1?: Stat1 | undefined;\n  stat4?: Stat4Summary | undefined;\n  /** The full histogram, only with `?stat4=full`. */\n  stat4Samples?: Stat4Sample[] | undefined;\n};\n\nexport type ColumnStats = {\n  name: string;\n  dataType: string;\n  nullable: boolean;\n  /** The ZQL type, or `null` for a column clients cannot see. */\n  zqlType: string | null;\n  backfilling?: true | undefined;\n};\n\nexport type TableStats = {\n  name: string;\n  columns: ColumnStats[];\n  primaryKey: string[] | undefined;\n  estimatedRows: number | undefined;\n  syncable: boolean;\n  indexes: IndexStats[];\n};\n\nexport type PlannerzBundle = {\n  about: typeof ABOUT;\n  server: {\n    zeroVersion: string;\n    sqliteVersion: string;\n    replicaWatermark: string | undefined;\n    generatedAt: string;\n    planner: {\n      enableQueryPlanner: boolean;\n      enableCorrelatedPredicatePushdown: boolean;\n      enablePlannerAwarePushdown: boolean;\n    };\n  };\n  statsQuality: {\n    stat1Present: boolean;\n    stat4Rows: number;\n    method: string;\n    tablesWithoutStats: string[];\n  };\n  tables: TableStats[];\n};\n\n/**\n * Parses a `sqlite_stat1.stat` value.\n *\n * @visibleForTesting\n */\nexport function parseStat1(raw: string): Stat1 {\n  const tokens = raw.trim().split(SEPARATOR).filter(Boolean);\n  const numbers: number[] = [];\n  const flags: string[] = [];\n  for (const token of tokens) {\n    // Only leading tokens are counts; anything else is a flag, even if it\n    // parses as a number (e.g. a future \"sz=N\" spelled differently).\n    if (flags.length === 0 && COUNT.test(token)) {\n      numbers.push(parseInt(token, 10));\n    } else {\n      flags.push(token);\n    }\n  }\n  return {\n    raw,\n    rows: numbers[0] ?? 0,\n    avgRowsPerPrefix: numbers.slice(1),\n    flags,\n  };\n}\n\n/** Whitespace between the counts in a `sqlite_stat1` or `sqlite_stat4` value. */\nconst SEPARATOR = /\\s+/;\n/** A count, as opposed to a trailing flag such as `unordered` or `sz=N`. */\nconst COUNT = /^\\d+$/;\n\ntype Stat1Row = {tbl: string; idx: string | null; stat: string};\ntype Stat4Row = {\n  tbl: string;\n  idx: string;\n  neq: string;\n  nlt: string;\n  ndlt: string;\n  sample: Buffer;\n};\n\nfunction tableExists(db: Database, name: string): boolean {\n  return (\n    db\n      .prepare(\n        `SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?`,\n      )\n      .all(name).length > 0\n  );\n}\n\nfunction parseCounts(raw: string): number[] {\n  return raw\n    .trim()\n    .split(SEPARATOR)\n    .filter(Boolean)\n    .map(n => parseInt(n, 10));\n}\n\n/**\n * Builds the `/plannerz` bundle from the replica's catalog, `sqlite_stat*`\n * tables and the server config. No user table is read, so the cost does not\n * depend on how much data the replica holds.\n *\n * @visibleForTesting\n */\nexport function buildPlannerzBundle(\n  lc: LogContext,\n  config: ZeroConfig,\n  opts: BundleOptions = {},\n): PlannerzBundle {\n  using db = new Database(lc, config.replica.file, {readonly: true});\n  return buildPlannerzBundleFromDB(lc, db, config, opts);\n}\n\nexport type BundleOptions = {\n  /** Include the full `sqlite_stat4` histogram, not just the digest. */\n  fullStat4?: boolean | undefined;\n};\n\n/** @visibleForTesting */\nexport function buildPlannerzBundleFromDB(\n  lc: LogContext,\n  db: Database,\n  config: ZeroConfig,\n  opts: BundleOptions = {},\n): PlannerzBundle {\n  const tables = listReplicaTables(lc, db);\n  const indexes = listIndexes(db);\n  const zqlSpecs = computeZqlSpecsFromLiteSpecs(\n    tables,\n    indexes,\n    {includeBackfillingColumns: false},\n    undefined,\n    undefined,\n    lc,\n  );\n\n  const stat1Present = tableExists(db, 'sqlite_stat1');\n  const stat1ByTableAndIndex = new Map<string, Stat1>();\n  const statsByTable = new Map<string, Stat1>();\n  if (stat1Present) {\n    for (const {tbl, idx, stat} of db\n      .prepare(`SELECT tbl, idx, stat FROM sqlite_stat1`)\n      .all() as Stat1Row[]) {\n      const parsed = parseStat1(stat);\n      if (idx !== null) {\n        stat1ByTableAndIndex.set(`${tbl}.${idx}`, parsed);\n      }\n      // Every row for a table starts with the table's row count. A row with a\n      // null `idx` is written for a table with no indexes, and is preferred\n      // because it is about the table itself.\n      if (idx === null || !statsByTable.has(tbl)) {\n        statsByTable.set(tbl, parsed);\n      }\n    }\n  }\n\n  const stat4Present = tableExists(db, 'sqlite_stat4');\n  const stat4ByTableAndIndex = new Map<string, Stat4Sample[]>();\n  let stat4Rows = 0;\n  if (stat4Present) {\n    for (const {tbl, idx, neq, nlt, ndlt, sample} of db\n      .prepare(`SELECT tbl, idx, neq, nlt, ndlt, sample FROM sqlite_stat4`)\n      .all() as Stat4Row[]) {\n      stat4Rows++;\n      const samples = stat4ByTableAndIndex.get(`${tbl}.${idx}`) ?? [];\n      samples.push({\n        nEq: parseCounts(neq),\n        nLt: parseCounts(nlt),\n        nDLt: parseCounts(ndlt),\n        // Values are deliberately dropped here: only the types are reported.\n        sample: decodeSampleKinds(sample),\n      });\n      stat4ByTableAndIndex.set(`${tbl}.${idx}`, samples);\n    }\n  }\n\n  const indexesByTable = new Map<string, IndexStats[]>();\n  for (const index of indexes) {\n    const key = `${index.tableName}.${index.name}`;\n    const stat4 = stat4ByTableAndIndex.get(key);\n    const forTable = indexesByTable.get(index.tableName) ?? [];\n    forTable.push({\n      name: index.name,\n      columns: Object.entries(index.columns).map(([name, dir]) => ({\n        name,\n        dir,\n      })),\n      unique: index.unique,\n      partial: index.partial === true,\n      stat1: stat1ByTableAndIndex.get(key),\n      stat4: stat4?.length ? summarizeStat4(stat4) : undefined,\n      stat4Samples: opts.fullStat4 && stat4?.length ? stat4 : undefined,\n    });\n    indexesByTable.set(index.tableName, forTable);\n  }\n\n  const tablesWithoutStats: string[] = [];\n  const tableStats = tables.map(table => {\n    const zqlSpec = zqlSpecs.get(table.name)?.zqlSpec;\n    if (!statsByTable.has(table.name)) {\n      tablesWithoutStats.push(table.name);\n    }\n    const backfilling = new Set(table.backfilling ?? []);\n    return {\n      name: table.name,\n      columns: Object.entries(table.columns).map(([name, spec]) => ({\n        name,\n        dataType: spec.dataType,\n        nullable: !spec.notNull,\n        zqlType: zqlSpec?.[name]?.type ?? null,\n        ...(backfilling.has(name) ? {backfilling: true as const} : {}),\n      })),\n      primaryKey: table.primaryKey ? [...table.primaryKey] : undefined,\n      estimatedRows: statsByTable.get(table.name)?.rows,\n      syncable: zqlSpec !== undefined,\n      indexes: indexesByTable.get(table.name) ?? [],\n    };\n  });\n\n  return {\n    about: ABOUT,\n    server: {\n      zeroVersion: getServerVersion(config),\n      sqliteVersion: sqliteVersion(db),\n      replicaWatermark: replicaWatermark(lc, db),\n      generatedAt: new Date().toISOString(),\n      planner: {\n        enableQueryPlanner: config.enableQueryPlanner,\n        // Both default to on, matching services/analyze.ts.\n        enableCorrelatedPredicatePushdown:\n          config.enableCorrelatedPredicatePushdown !== false,\n        enablePlannerAwarePushdown: config.enablePlannerAwarePushdown !== false,\n      },\n    },\n    statsQuality: {\n      stat1Present,\n      stat4Rows,\n      method: stat4Rows > 0 ? FULL_METHOD : APPROXIMATE_METHOD,\n      tablesWithoutStats,\n    },\n    tables: tableStats,\n  };\n}\n\n/**\n * Lists the replica's tables, tolerating a replica that has no metadata tables\n * yet. `listTables` reads `_zero.tableMetadata` for backfill status, which does\n * not exist until initial sync has created it. Reporting the schema without\n * backfill status beats failing the whole request.\n */\nfunction listReplicaTables(lc: LogContext, db: Database) {\n  try {\n    return listTables(db);\n  } catch (e) {\n    lc.debug?.('reading table metadata failed; listing tables without it', e);\n    return listTables(db, true, false);\n  }\n}\n\n/**\n * Digests an index's stat4 samples.\n *\n * Note that `nLt` and `nDLt` are stored as text, so SQL cannot order the\n * samples by them numerically. This works off the whole set rather than\n * relying on the samples' order: the highest `nDLt` is a lower bound on the\n * distinct keys, since the sample holding it has that many distinct keys below\n * it plus itself.\n *\n * @visibleForTesting\n */\nexport function summarizeStat4(samples: Stat4Sample[]): Stat4Summary {\n  const depth = Math.max(...samples.map(s => s.nEq.length));\n  const perPrefix: Stat4PrefixSummary[] = [];\n  for (let d = 0; d < depth; d++) {\n    const counts = samples\n      .map(s => s.nEq[d])\n      .filter(n => n !== undefined)\n      .sort((a, b) => a - b);\n    const maxDistinctBelow = Math.max(\n      ...samples.map(s => s.nDLt[d]).filter(n => n !== undefined),\n    );\n    perPrefix.push({\n      maxRowsPerKey: counts.at(-1)!,\n      medianRowsPerKey: counts[Math.floor((counts.length - 1) / 2)],\n      estimatedDistinctKeys: maxDistinctBelow + 1,\n    });\n  }\n  return {\n    samples: samples.length,\n    // Every sample of an index describes the same columns.\n    sampleKinds: samples[0].sample,\n    perPrefix,\n  };\n}\n\nfunction sqliteVersion(db: Database): string {\n  const [{version}] = db\n    .prepare(`SELECT sqlite_version() AS version`)\n    .all() as {version: string}[];\n  return version;\n}\n\nfunction replicaWatermark(lc: LogContext, db: Database): string | undefined {\n  try {\n    return getReplicationState(new StatementRunner(db)).stateVersion;\n  } catch (e) {\n    // A replica that has not finished initial sync has no replication state.\n    lc.debug?.('could not read replication state', e);\n    return undefined;\n  }\n}\n\n/**\n * Serves the replica's schema, indexes and SQLite statistics for consumption by\n * a language model, so that it can advise on query rewrites and indexes.\n *\n * Row data is never returned. See `designs/003_replica_stats_endpoint.md`.\n *\n * This is available to the operator password as well as the admin password\n * (see `--operator-password`). That does not extend to `/plannerz/analyze`,\n * which runs caller-supplied queries against the replica and so must require\n * the admin password (`isAdminPasswordValid`).\n *\n * HTTP query parameters:\n * * `pretty`: indents the JSON\n * * `stat4=full`: includes the full sqlite_stat4 histogram, which is large\n */\nexport async function handlePlannerzRequest(\n  lc: LogContext,\n  config: ZeroConfig,\n  req: FastifyRequest,\n  res: FastifyReply,\n) {\n  const credentials = auth(req);\n  if (getOperatorAccess(lc, config, credentials?.pass) === undefined) {\n    void res\n      .code(401)\n      .header('WWW-Authenticate', 'Basic realm=\"Plannerz Protected Area\"')\n      .send('Unauthorized');\n    return;\n  }\n\n  const query = req.query as Record<string, unknown>;\n  const indent = query.pretty !== undefined ? 2 : undefined;\n  let bundle: PlannerzBundle;\n  try {\n    bundle = buildPlannerzBundle(lc, config, {\n      fullStat4: query.stat4 === 'full',\n    });\n  } catch (e) {\n    lc.error?.('error building plannerz bundle', e);\n    await res.code(500).send({error: String(e)});\n    return;\n  }\n  await res\n    .header('Content-Type', 'application/json')\n    .send(JSON.stringify(bundle, null, indent));\n}\n"],"mappings":";;;;;;;;;;;;;;;AAwBA,IAAM,qBACJ;AAEF,IAAM,cACJ;;;;;;AAQF,IAAM,QAAQ;CACZ,YACE;CAGF,aACE;CAKF,aACE;CAMF,uBAAuB;EACrB;EAEA;EAGA;CAGF;CACA,kBAAkB;EAChB;EAEA;EACA;EAEA;CACF;CACA,SAAS;EACP;EAEA;EAEA;CAGF;AACF;;;;;;AA2GA,SAAgB,WAAW,KAAoB;CAC7C,MAAM,SAAS,IAAI,KAAK,CAAC,CAAC,MAAM,SAAS,CAAC,CAAC,OAAO,OAAO;CACzD,MAAM,UAAoB,CAAC;CAC3B,MAAM,QAAkB,CAAC;CACzB,KAAK,MAAM,SAAS,QAGlB,IAAI,MAAM,WAAW,KAAK,MAAM,KAAK,KAAK,GACxC,QAAQ,KAAK,SAAS,OAAO,EAAE,CAAC;MAEhC,MAAM,KAAK,KAAK;CAGpB,OAAO;EACL;EACA,MAAM,QAAQ,MAAM;EACpB,kBAAkB,QAAQ,MAAM,CAAC;EACjC;CACF;AACF;;AAGA,IAAM,YAAY;;AAElB,IAAM,QAAQ;AAYd,SAAS,YAAY,IAAc,MAAuB;CACxD,OACE,GACG,QACC,kEACF,CAAC,CACA,IAAI,IAAI,CAAC,CAAC,SAAS;AAE1B;AAEA,SAAS,YAAY,KAAuB;CAC1C,OAAO,IACJ,KAAK,CAAC,CACN,MAAM,SAAS,CAAC,CAChB,OAAO,OAAO,CAAC,CACf,KAAI,MAAK,SAAS,GAAG,EAAE,CAAC;AAC7B;;;;;;;;AASA,SAAgB,oBACd,IACA,QACA,OAAsB,CAAC,GACP;;;EAEhB,OAAO,0BAA0B,IAAA,YAAA,EADtB,IAAI,SAAS,IAAI,OAAO,QAAQ,MAAM,EAAC,UAAU,KAAI,CAAC,CAC5B,GAAI,QAAQ,IAAI;;;;;;AACvD;;AAQA,SAAgB,0BACd,IACA,IACA,QACA,OAAsB,CAAC,GACP;CAChB,MAAM,SAAS,kBAAkB,IAAI,EAAE;CACvC,MAAM,UAAU,YAAY,EAAE;CAC9B,MAAM,WAAW,6BACf,QACA,SACA,EAAC,2BAA2B,MAAK,GACjC,KAAA,GACA,KAAA,GACA,EACF;CAEA,MAAM,eAAe,YAAY,IAAI,cAAc;CACnD,MAAM,uCAAuB,IAAI,IAAmB;CACpD,MAAM,+BAAe,IAAI,IAAmB;CAC5C,IAAI,cACF,KAAK,MAAM,EAAC,KAAK,KAAK,UAAS,GAC5B,QAAQ,yCAAyC,CAAC,CAClD,IAAI,GAAiB;EACtB,MAAM,SAAS,WAAW,IAAI;EAC9B,IAAI,QAAQ,MACV,qBAAqB,IAAI,GAAG,IAAI,GAAG,OAAO,MAAM;EAKlD,IAAI,QAAQ,QAAQ,CAAC,aAAa,IAAI,GAAG,GACvC,aAAa,IAAI,KAAK,MAAM;CAEhC;CAGF,MAAM,eAAe,YAAY,IAAI,cAAc;CACnD,MAAM,uCAAuB,IAAI,IAA2B;CAC5D,IAAI,YAAY;CAChB,IAAI,cACF,KAAK,MAAM,EAAC,KAAK,KAAK,KAAK,KAAK,MAAM,YAAW,GAC9C,QAAQ,2DAA2D,CAAC,CACpE,IAAI,GAAiB;EACtB;EACA,MAAM,UAAU,qBAAqB,IAAI,GAAG,IAAI,GAAG,KAAK,KAAK,CAAC;EAC9D,QAAQ,KAAK;GACX,KAAK,YAAY,GAAG;GACpB,KAAK,YAAY,GAAG;GACpB,MAAM,YAAY,IAAI;GAEtB,QAAQ,kBAAkB,MAAM;EAClC,CAAC;EACD,qBAAqB,IAAI,GAAG,IAAI,GAAG,OAAO,OAAO;CACnD;CAGF,MAAM,iCAAiB,IAAI,IAA0B;CACrD,KAAK,MAAM,SAAS,SAAS;EAC3B,MAAM,MAAM,GAAG,MAAM,UAAU,GAAG,MAAM;EACxC,MAAM,QAAQ,qBAAqB,IAAI,GAAG;EAC1C,MAAM,WAAW,eAAe,IAAI,MAAM,SAAS,KAAK,CAAC;EACzD,SAAS,KAAK;GACZ,MAAM,MAAM;GACZ,SAAS,OAAO,QAAQ,MAAM,OAAO,CAAC,CAAC,KAAK,CAAC,MAAM,UAAU;IAC3D;IACA;GACF,EAAE;GACF,QAAQ,MAAM;GACd,SAAS,MAAM,YAAY;GAC3B,OAAO,qBAAqB,IAAI,GAAG;GACnC,OAAO,OAAO,SAAS,eAAe,KAAK,IAAI,KAAA;GAC/C,cAAc,KAAK,aAAa,OAAO,SAAS,QAAQ,KAAA;EAC1D,CAAC;EACD,eAAe,IAAI,MAAM,WAAW,QAAQ;CAC9C;CAEA,MAAM,qBAA+B,CAAC;CACtC,MAAM,aAAa,OAAO,KAAI,UAAS;EACrC,MAAM,UAAU,SAAS,IAAI,MAAM,IAAI,CAAC,EAAE;EAC1C,IAAI,CAAC,aAAa,IAAI,MAAM,IAAI,GAC9B,mBAAmB,KAAK,MAAM,IAAI;EAEpC,MAAM,cAAc,IAAI,IAAI,MAAM,eAAe,CAAC,CAAC;EACnD,OAAO;GACL,MAAM,MAAM;GACZ,SAAS,OAAO,QAAQ,MAAM,OAAO,CAAC,CAAC,KAAK,CAAC,MAAM,WAAW;IAC5D;IACA,UAAU,KAAK;IACf,UAAU,CAAC,KAAK;IAChB,SAAS,UAAU,KAAK,EAAE,QAAQ;IAClC,GAAI,YAAY,IAAI,IAAI,IAAI,EAAC,aAAa,KAAa,IAAI,CAAC;GAC9D,EAAE;GACF,YAAY,MAAM,aAAa,CAAC,GAAG,MAAM,UAAU,IAAI,KAAA;GACvD,eAAe,aAAa,IAAI,MAAM,IAAI,CAAC,EAAE;GAC7C,UAAU,YAAY,KAAA;GACtB,SAAS,eAAe,IAAI,MAAM,IAAI,KAAK,CAAC;EAC9C;CACF,CAAC;CAED,OAAO;EACL,OAAO;EACP,QAAQ;GACN,aAAa,iBAAiB,MAAM;GACpC,eAAe,cAAc,EAAE;GAC/B,kBAAkB,iBAAiB,IAAI,EAAE;GACzC,8BAAa,IAAI,KAAK,EAAA,CAAE,YAAY;GACpC,SAAS;IACP,oBAAoB,OAAO;IAE3B,mCACE,OAAO,sCAAsC;IAC/C,4BAA4B,OAAO,+BAA+B;GACpE;EACF;EACA,cAAc;GACZ;GACA;GACA,QAAQ,YAAY,IAAI,cAAc;GACtC;EACF;EACA,QAAQ;CACV;AACF;;;;;;;AAQA,SAAS,kBAAkB,IAAgB,IAAc;CACvD,IAAI;EACF,OAAO,WAAW,EAAE;CACtB,SAAS,GAAG;EACV,GAAG,QAAQ,4DAA4D,CAAC;EACxE,OAAO,WAAW,IAAI,MAAM,KAAK;CACnC;AACF;;;;;;;;;;;;AAaA,SAAgB,eAAe,SAAsC;CACnE,MAAM,QAAQ,KAAK,IAAI,GAAG,QAAQ,KAAI,MAAK,EAAE,IAAI,MAAM,CAAC;CACxD,MAAM,YAAkC,CAAC;CACzC,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,KAAK;EAC9B,MAAM,SAAS,QACZ,KAAI,MAAK,EAAE,IAAI,EAAE,CAAC,CAClB,QAAO,MAAK,MAAM,KAAA,CAAS,CAAC,CAC5B,MAAM,GAAG,MAAM,IAAI,CAAC;EACvB,MAAM,mBAAmB,KAAK,IAC5B,GAAG,QAAQ,KAAI,MAAK,EAAE,KAAK,EAAE,CAAC,CAAC,QAAO,MAAK,MAAM,KAAA,CAAS,CAC5D;EACA,UAAU,KAAK;GACb,eAAe,OAAO,GAAG,EAAE;GAC3B,kBAAkB,OAAO,KAAK,OAAO,OAAO,SAAS,KAAK,CAAC;GAC3D,uBAAuB,mBAAmB;EAC5C,CAAC;CACH;CACA,OAAO;EACL,SAAS,QAAQ;EAEjB,aAAa,QAAQ,EAAE,CAAC;EACxB;CACF;AACF;AAEA,SAAS,cAAc,IAAsB;CAC3C,MAAM,CAAC,EAAC,aAAY,GACjB,QAAQ,oCAAoC,CAAC,CAC7C,IAAI;CACP,OAAO;AACT;AAEA,SAAS,iBAAiB,IAAgB,IAAkC;CAC1E,IAAI;EACF,OAAO,oBAAoB,IAAI,gBAAgB,EAAE,CAAC,CAAC,CAAC;CACtD,SAAS,GAAG;EAEV,GAAG,QAAQ,oCAAoC,CAAC;EAChD;CACF;AACF;;;;;;;;;;;;;;;;AAiBA,eAAsB,sBACpB,IACA,QACA,KACA,KACA;CACA,MAAM,cAAc,KAAK,GAAG;CAC5B,IAAI,kBAAkB,IAAI,QAAQ,aAAa,IAAI,MAAM,KAAA,GAAW;EAClE,IACG,KAAK,GAAG,CAAC,CACT,OAAO,oBAAoB,yCAAuC,CAAC,CACnE,KAAK,cAAc;EACtB;CACF;CAEA,MAAM,QAAQ,IAAI;CAClB,MAAM,SAAS,MAAM,WAAW,KAAA,IAAY,IAAI,KAAA;CAChD,IAAI;CACJ,IAAI;EACF,SAAS,oBAAoB,IAAI,QAAQ,EACvC,WAAW,MAAM,UAAU,OAC7B,CAAC;CACH,SAAS,GAAG;EACV,GAAG,QAAQ,kCAAkC,CAAC;EAC9C,MAAM,IAAI,KAAK,GAAG,CAAC,CAAC,KAAK,EAAC,OAAO,OAAO,CAAC,EAAC,CAAC;EAC3C;CACF;CACA,MAAM,IACH,OAAO,gBAAgB,kBAAkB,CAAC,CAC1C,KAAK,KAAK,UAAU,QAAQ,MAAM,MAAM,CAAC;AAC9C"}
+{"version":3,"file":"plannerz.js","names":[],"sources":["../../../../../zero-cache/src/services/plannerz.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport auth from 'basic-auth';\nimport type {FastifyReply, FastifyRequest} from 'fastify';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport {\n  decodeSampleKinds,\n  type SampleValueKind,\n} from '../../../zqlite/src/sqlite-stat4-sample.ts';\nimport type {NormalizedZeroConfig as ZeroConfig} from '../config/normalize.ts';\nimport {getOperatorAccess, getServerVersion} from '../config/zero-config.ts';\nimport {\n  computeZqlSpecsFromLiteSpecs,\n  listIndexes,\n  listTables,\n} from '../db/lite-tables.ts';\nimport {StatementRunner} from '../db/statements.ts';\nimport {getReplicationState} from './replicator/schema/replication-state.ts';\n\n/**\n * How the statistics in the replica were gathered. zero-cache only ever runs\n * `PRAGMA optimize`, and always under an `analysis_limit`, which samples rather\n * than scanning. A replica that someone ran `ANALYZE` on by hand has better\n * statistics, which is visible in `sqlite_stat4` having rows at all.\n */\nconst APPROXIMATE_METHOD =\n  'PRAGMA optimize under an analysis_limit. Row counts and per-key averages ' +\n  'are sampled estimates, and sqlite_stat4 is not collected.';\nconst FULL_METHOD =\n  'ANALYZE, with no analysis limit. sqlite_stat4 has rows, so a full ANALYZE ' +\n  'ran against this replica at some point.';\n\n/**\n * A fixed glossary, sent with every bundle. The audience is a language model\n * that has the user's codebase but has never seen these statistics, and would\n * otherwise read a sampled estimate as an exact count.\n */\nconst ABOUT = {\n  whatThisIs:\n    'Schema, indexes and SQLite statistics for the SQLite replica that ' +\n    'zero-cache queries. Use it to decide how to rewrite ZQL queries and ' +\n    'which indexes to add. It contains no row data.',\n  stat1Format:\n    'sqlite_stat1.stat is \"<rows> <avg rows per distinct 1-column prefix> ' +\n    '<... per 2-column prefix> ...\", parsed here into `rows` and ' +\n    '`avgRowsPerPrefix`. NULLs count as ordinary values, so a sparse column ' +\n    'looks less selective than it is. Trailing flags such as \"unordered\", ' +\n    '\"noskipscan\" and \"sz=N\" are kept in `flags`.',\n  stat4Format:\n    'sqlite_stat4 is a per-index histogram. For each sample, `nEq` is the ' +\n    'rows equal to it, `nLt` the rows less than it, and `nDLt` the distinct ' +\n    'values less than it, one entry per key column. A large `nEq` next to a ' +\n    'small `nDLt` means a skewed column, where the average in stat1 hides ' +\n    'the common values. Sample values are never returned: `sample` reports ' +\n    'only each key column’s type.',\n  howThePlannerUsesThis: [\n    'Cost comes from SQLite scanstatus estimates for the SQL each source ' +\n      'connection would run, so the statistics here are what the planner sees.',\n    'Join fanout comes from sqlite_stat4, else sqlite_stat1, else a default ' +\n      'of 3. With no stat4 (the usual case) fanout on a nullable foreign key ' +\n      'is overestimated, because stat1 counts NULLs.',\n    'The planner chooses per join whether to run it as a semi-join or to flip ' +\n      'it so the child drives. An index that matches the child’s ' +\n      'constraint plus ordering is what makes a flip cheap.',\n  ],\n  whatYouCanChange: [\n    'The ZQL query: `whereExists` vs `related`, the order of conditions, and ' +\n      'ordering that matches an existing index.',\n    'A `limit`, which lets the planner stop early.',\n    'Indexes: add them in Postgres, not on the replica. The replica copies ' +\n      'upstream indexes, so a new index reaches it through replication.',\n    'The planner flags in `server.planner`.',\n  ],\n  caveats: [\n    'Row counts are estimates from the statistics, not COUNT(*). This ' +\n      'endpoint never scans a table.',\n    'Statistics are as of the last time they were gathered, which SQLite does ' +\n      'not record. A table whose size changed a lot since then can be far off.',\n    'Tables in `tables` with `syncable: false` cannot be queried by clients: ' +\n      'they lack a primary key or unique index, or every column has a type ' +\n      'ZQL does not support.',\n  ],\n} as const;\n\nexport type Stat1 = {\n  /** The raw `sqlite_stat1.stat` text, in case a flag is not parsed here. */\n  raw: string;\n  /** Estimated rows in the table. */\n  rows: number;\n  /**\n   * Average rows per distinct key prefix: the first entry is for a 1-column\n   * prefix, the second for 2 columns, and so on.\n   */\n  avgRowsPerPrefix: number[];\n  /** Non-numeric tokens, e.g. `unordered`, `noskipscan`, `sz=N`. */\n  flags: string[];\n};\n\n/** A `sqlite_stat4` sample with its values replaced by their types. */\nexport type Stat4Sample = {\n  nEq: number[];\n  nLt: number[];\n  nDLt: number[];\n  /** One entry per key column, plus the trailing rowid. */\n  sample: SampleValueKind[];\n};\n\n/**\n * A digest of an index's `sqlite_stat4` histogram, one entry per key prefix\n * depth. This is what the default response carries: a full histogram runs to\n * hundreds of samples per index, which is a lot of context to spend on data an\n * LLM would only summarize anyway.\n */\ntype Stat4PrefixSummary = {\n  /** Rows equal to the most common sampled key at this depth. */\n  maxRowsPerKey: number;\n  /** Median rows per sampled key at this depth. */\n  medianRowsPerKey: number;\n  /**\n   * Lower bound on distinct keys at this depth, from the last sample's count of\n   * distinct values less than it.\n   */\n  estimatedDistinctKeys: number;\n};\n\nexport type Stat4Summary = {\n  samples: number;\n  /** The type of each key column, and of the trailing rowid. */\n  sampleKinds: SampleValueKind[];\n  /** One entry per key prefix depth: 1 column, then 2, and so on. */\n  perPrefix: Stat4PrefixSummary[];\n};\n\ntype IndexStats = {\n  name: string;\n  columns: {name: string; dir: 'ASC' | 'DESC'}[];\n  unique: boolean;\n  partial: boolean;\n  stat1?: Stat1 | undefined;\n  stat4?: Stat4Summary | undefined;\n  /** The full histogram, only with `?stat4=full`. */\n  stat4Samples?: Stat4Sample[] | undefined;\n};\n\ntype ColumnStats = {\n  name: string;\n  dataType: string;\n  nullable: boolean;\n  /** The ZQL type, or `null` for a column clients cannot see. */\n  zqlType: string | null;\n  backfilling?: true | undefined;\n};\n\ntype TableStats = {\n  name: string;\n  columns: ColumnStats[];\n  primaryKey: string[] | undefined;\n  estimatedRows: number | undefined;\n  syncable: boolean;\n  indexes: IndexStats[];\n};\n\nexport type PlannerzBundle = {\n  about: typeof ABOUT;\n  server: {\n    zeroVersion: string;\n    sqliteVersion: string;\n    replicaWatermark: string | undefined;\n    generatedAt: string;\n    planner: {\n      enableQueryPlanner: boolean;\n      enableCorrelatedPredicatePushdown: boolean;\n      enablePlannerAwarePushdown: boolean;\n    };\n  };\n  statsQuality: {\n    stat1Present: boolean;\n    stat4Rows: number;\n    method: string;\n    tablesWithoutStats: string[];\n  };\n  tables: TableStats[];\n};\n\n/**\n * Parses a `sqlite_stat1.stat` value.\n *\n * @visibleForTesting\n */\nexport function parseStat1(raw: string): Stat1 {\n  const tokens = raw.trim().split(SEPARATOR).filter(Boolean);\n  const numbers: number[] = [];\n  const flags: string[] = [];\n  for (const token of tokens) {\n    // Only leading tokens are counts; anything else is a flag, even if it\n    // parses as a number (e.g. a future \"sz=N\" spelled differently).\n    if (flags.length === 0 && COUNT.test(token)) {\n      numbers.push(parseInt(token, 10));\n    } else {\n      flags.push(token);\n    }\n  }\n  return {\n    raw,\n    rows: numbers[0] ?? 0,\n    avgRowsPerPrefix: numbers.slice(1),\n    flags,\n  };\n}\n\n/** Whitespace between the counts in a `sqlite_stat1` or `sqlite_stat4` value. */\nconst SEPARATOR = /\\s+/;\n/** A count, as opposed to a trailing flag such as `unordered` or `sz=N`. */\nconst COUNT = /^\\d+$/;\n\ntype Stat1Row = {tbl: string; idx: string | null; stat: string};\ntype Stat4Row = {\n  tbl: string;\n  idx: string;\n  neq: string;\n  nlt: string;\n  ndlt: string;\n  sample: Buffer;\n};\n\nfunction tableExists(db: Database, name: string): boolean {\n  return (\n    db\n      .prepare(\n        `SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?`,\n      )\n      .all(name).length > 0\n  );\n}\n\nfunction parseCounts(raw: string): number[] {\n  return raw\n    .trim()\n    .split(SEPARATOR)\n    .filter(Boolean)\n    .map(n => parseInt(n, 10));\n}\n\n/**\n * Builds the `/plannerz` bundle from the replica's catalog, `sqlite_stat*`\n * tables and the server config. No user table is read, so the cost does not\n * depend on how much data the replica holds.\n */\nfunction buildPlannerzBundle(\n  lc: LogContext,\n  config: ZeroConfig,\n  opts: BundleOptions = {},\n): PlannerzBundle {\n  using db = new Database(lc, config.replica.file, {readonly: true});\n  return buildPlannerzBundleFromDB(lc, db, config, opts);\n}\n\nexport type BundleOptions = {\n  /** Include the full `sqlite_stat4` histogram, not just the digest. */\n  fullStat4?: boolean | undefined;\n};\n\n/** @visibleForTesting */\nexport function buildPlannerzBundleFromDB(\n  lc: LogContext,\n  db: Database,\n  config: ZeroConfig,\n  opts: BundleOptions = {},\n): PlannerzBundle {\n  const tables = listReplicaTables(lc, db);\n  const indexes = listIndexes(db);\n  const zqlSpecs = computeZqlSpecsFromLiteSpecs(\n    tables,\n    indexes,\n    {includeBackfillingColumns: false},\n    undefined,\n    undefined,\n    lc,\n  );\n\n  const stat1Present = tableExists(db, 'sqlite_stat1');\n  const stat1ByTableAndIndex = new Map<string, Stat1>();\n  const statsByTable = new Map<string, Stat1>();\n  if (stat1Present) {\n    for (const {tbl, idx, stat} of db\n      .prepare(`SELECT tbl, idx, stat FROM sqlite_stat1`)\n      .all() as Stat1Row[]) {\n      const parsed = parseStat1(stat);\n      if (idx !== null) {\n        stat1ByTableAndIndex.set(`${tbl}.${idx}`, parsed);\n      }\n      // Every row for a table starts with the table's row count. A row with a\n      // null `idx` is written for a table with no indexes, and is preferred\n      // because it is about the table itself.\n      if (idx === null || !statsByTable.has(tbl)) {\n        statsByTable.set(tbl, parsed);\n      }\n    }\n  }\n\n  const stat4Present = tableExists(db, 'sqlite_stat4');\n  const stat4ByTableAndIndex = new Map<string, Stat4Sample[]>();\n  let stat4Rows = 0;\n  if (stat4Present) {\n    for (const {tbl, idx, neq, nlt, ndlt, sample} of db\n      .prepare(`SELECT tbl, idx, neq, nlt, ndlt, sample FROM sqlite_stat4`)\n      .all() as Stat4Row[]) {\n      stat4Rows++;\n      const samples = stat4ByTableAndIndex.get(`${tbl}.${idx}`) ?? [];\n      samples.push({\n        nEq: parseCounts(neq),\n        nLt: parseCounts(nlt),\n        nDLt: parseCounts(ndlt),\n        // Values are deliberately dropped here: only the types are reported.\n        sample: decodeSampleKinds(sample),\n      });\n      stat4ByTableAndIndex.set(`${tbl}.${idx}`, samples);\n    }\n  }\n\n  const indexesByTable = new Map<string, IndexStats[]>();\n  for (const index of indexes) {\n    const key = `${index.tableName}.${index.name}`;\n    const stat4 = stat4ByTableAndIndex.get(key);\n    const forTable = indexesByTable.get(index.tableName) ?? [];\n    forTable.push({\n      name: index.name,\n      columns: Object.entries(index.columns).map(([name, dir]) => ({\n        name,\n        dir,\n      })),\n      unique: index.unique,\n      partial: index.partial === true,\n      stat1: stat1ByTableAndIndex.get(key),\n      stat4: stat4?.length ? summarizeStat4(stat4) : undefined,\n      stat4Samples: opts.fullStat4 && stat4?.length ? stat4 : undefined,\n    });\n    indexesByTable.set(index.tableName, forTable);\n  }\n\n  const tablesWithoutStats: string[] = [];\n  const tableStats = tables.map(table => {\n    const zqlSpec = zqlSpecs.get(table.name)?.zqlSpec;\n    if (!statsByTable.has(table.name)) {\n      tablesWithoutStats.push(table.name);\n    }\n    const backfilling = new Set(table.backfilling ?? []);\n    return {\n      name: table.name,\n      columns: Object.entries(table.columns).map(([name, spec]) => ({\n        name,\n        dataType: spec.dataType,\n        nullable: !spec.notNull,\n        zqlType: zqlSpec?.[name]?.type ?? null,\n        ...(backfilling.has(name) ? {backfilling: true as const} : {}),\n      })),\n      primaryKey: table.primaryKey ? [...table.primaryKey] : undefined,\n      estimatedRows: statsByTable.get(table.name)?.rows,\n      syncable: zqlSpec !== undefined,\n      indexes: indexesByTable.get(table.name) ?? [],\n    };\n  });\n\n  return {\n    about: ABOUT,\n    server: {\n      zeroVersion: getServerVersion(config),\n      sqliteVersion: sqliteVersion(db),\n      replicaWatermark: replicaWatermark(lc, db),\n      generatedAt: new Date().toISOString(),\n      planner: {\n        enableQueryPlanner: config.enableQueryPlanner,\n        // Both default to on, matching services/analyze.ts.\n        enableCorrelatedPredicatePushdown:\n          config.enableCorrelatedPredicatePushdown !== false,\n        enablePlannerAwarePushdown: config.enablePlannerAwarePushdown !== false,\n      },\n    },\n    statsQuality: {\n      stat1Present,\n      stat4Rows,\n      method: stat4Rows > 0 ? FULL_METHOD : APPROXIMATE_METHOD,\n      tablesWithoutStats,\n    },\n    tables: tableStats,\n  };\n}\n\n/**\n * Lists the replica's tables, tolerating a replica that has no metadata tables\n * yet. `listTables` reads `_zero.tableMetadata` for backfill status, which does\n * not exist until initial sync has created it. Reporting the schema without\n * backfill status beats failing the whole request.\n */\nfunction listReplicaTables(lc: LogContext, db: Database) {\n  try {\n    return listTables(db);\n  } catch (e) {\n    lc.debug?.('reading table metadata failed; listing tables without it', e);\n    return listTables(db, true, false);\n  }\n}\n\n/**\n * Digests an index's stat4 samples.\n *\n * Note that `nLt` and `nDLt` are stored as text, so SQL cannot order the\n * samples by them numerically. This works off the whole set rather than\n * relying on the samples' order: the highest `nDLt` is a lower bound on the\n * distinct keys, since the sample holding it has that many distinct keys below\n * it plus itself.\n *\n * @visibleForTesting\n */\nexport function summarizeStat4(samples: Stat4Sample[]): Stat4Summary {\n  const depth = Math.max(...samples.map(s => s.nEq.length));\n  const perPrefix: Stat4PrefixSummary[] = [];\n  for (let d = 0; d < depth; d++) {\n    const counts = samples\n      .map(s => s.nEq[d])\n      .filter(n => n !== undefined)\n      .sort((a, b) => a - b);\n    const maxDistinctBelow = Math.max(\n      ...samples.map(s => s.nDLt[d]).filter(n => n !== undefined),\n    );\n    perPrefix.push({\n      maxRowsPerKey: counts.at(-1)!,\n      medianRowsPerKey: counts[Math.floor((counts.length - 1) / 2)],\n      estimatedDistinctKeys: maxDistinctBelow + 1,\n    });\n  }\n  return {\n    samples: samples.length,\n    // Every sample of an index describes the same columns.\n    sampleKinds: samples[0].sample,\n    perPrefix,\n  };\n}\n\nfunction sqliteVersion(db: Database): string {\n  const [{version}] = db\n    .prepare(`SELECT sqlite_version() AS version`)\n    .all() as {version: string}[];\n  return version;\n}\n\nfunction replicaWatermark(lc: LogContext, db: Database): string | undefined {\n  try {\n    return getReplicationState(new StatementRunner(db)).stateVersion;\n  } catch (e) {\n    // A replica that has not finished initial sync has no replication state.\n    lc.debug?.('could not read replication state', e);\n    return undefined;\n  }\n}\n\n/**\n * Serves the replica's schema, indexes and SQLite statistics for consumption by\n * a language model, so that it can advise on query rewrites and indexes.\n *\n * Row data is never returned. See `designs/003_replica_stats_endpoint.md`.\n *\n * This is available to the operator password as well as the admin password\n * (see `--operator-password`). That does not extend to `/plannerz/analyze`,\n * which runs caller-supplied queries against the replica and so must require\n * the admin password (`isAdminPasswordValid`).\n *\n * HTTP query parameters:\n * * `pretty`: indents the JSON\n * * `stat4=full`: includes the full sqlite_stat4 histogram, which is large\n */\nexport async function handlePlannerzRequest(\n  lc: LogContext,\n  config: ZeroConfig,\n  req: FastifyRequest,\n  res: FastifyReply,\n) {\n  const credentials = auth(req);\n  if (getOperatorAccess(lc, config, credentials?.pass) === undefined) {\n    void res\n      .code(401)\n      .header('WWW-Authenticate', 'Basic realm=\"Plannerz Protected Area\"')\n      .send('Unauthorized');\n    return;\n  }\n\n  const query = req.query as Record<string, unknown>;\n  const indent = query.pretty !== undefined ? 2 : undefined;\n  let bundle: PlannerzBundle;\n  try {\n    bundle = buildPlannerzBundle(lc, config, {\n      fullStat4: query.stat4 === 'full',\n    });\n  } catch (e) {\n    lc.error?.('error building plannerz bundle', e);\n    await res.code(500).send({error: String(e)});\n    return;\n  }\n  await res\n    .header('Content-Type', 'application/json')\n    .send(JSON.stringify(bundle, null, indent));\n}\n"],"mappings":";;;;;;;;;;;;;;;AAwBA,IAAM,qBACJ;AAEF,IAAM,cACJ;;;;;;AAQF,IAAM,QAAQ;CACZ,YACE;CAGF,aACE;CAKF,aACE;CAMF,uBAAuB;EACrB;EAEA;EAGA;CAGF;CACA,kBAAkB;EAChB;EAEA;EACA;EAEA;CACF;CACA,SAAS;EACP;EAEA;EAEA;CAGF;AACF;;;;;;AA2GA,SAAgB,WAAW,KAAoB;CAC7C,MAAM,SAAS,IAAI,KAAK,CAAC,CAAC,MAAM,SAAS,CAAC,CAAC,OAAO,OAAO;CACzD,MAAM,UAAoB,CAAC;CAC3B,MAAM,QAAkB,CAAC;CACzB,KAAK,MAAM,SAAS,QAGlB,IAAI,MAAM,WAAW,KAAK,MAAM,KAAK,KAAK,GACxC,QAAQ,KAAK,SAAS,OAAO,EAAE,CAAC;MAEhC,MAAM,KAAK,KAAK;CAGpB,OAAO;EACL;EACA,MAAM,QAAQ,MAAM;EACpB,kBAAkB,QAAQ,MAAM,CAAC;EACjC;CACF;AACF;;AAGA,IAAM,YAAY;;AAElB,IAAM,QAAQ;AAYd,SAAS,YAAY,IAAc,MAAuB;CACxD,OACE,GACG,QACC,kEACF,CAAC,CACA,IAAI,IAAI,CAAC,CAAC,SAAS;AAE1B;AAEA,SAAS,YAAY,KAAuB;CAC1C,OAAO,IACJ,KAAK,CAAC,CACN,MAAM,SAAS,CAAC,CAChB,OAAO,OAAO,CAAC,CACf,KAAI,MAAK,SAAS,GAAG,EAAE,CAAC;AAC7B;;;;;;AAOA,SAAS,oBACP,IACA,QACA,OAAsB,CAAC,GACP;;;EAEhB,OAAO,0BAA0B,IAAA,YAAA,EADtB,IAAI,SAAS,IAAI,OAAO,QAAQ,MAAM,EAAC,UAAU,KAAI,CAAC,CAC5B,GAAI,QAAQ,IAAI;;;;;;AACvD;;AAQA,SAAgB,0BACd,IACA,IACA,QACA,OAAsB,CAAC,GACP;CAChB,MAAM,SAAS,kBAAkB,IAAI,EAAE;CACvC,MAAM,UAAU,YAAY,EAAE;CAC9B,MAAM,WAAW,6BACf,QACA,SACA,EAAC,2BAA2B,MAAK,GACjC,KAAA,GACA,KAAA,GACA,EACF;CAEA,MAAM,eAAe,YAAY,IAAI,cAAc;CACnD,MAAM,uCAAuB,IAAI,IAAmB;CACpD,MAAM,+BAAe,IAAI,IAAmB;CAC5C,IAAI,cACF,KAAK,MAAM,EAAC,KAAK,KAAK,UAAS,GAC5B,QAAQ,yCAAyC,CAAC,CAClD,IAAI,GAAiB;EACtB,MAAM,SAAS,WAAW,IAAI;EAC9B,IAAI,QAAQ,MACV,qBAAqB,IAAI,GAAG,IAAI,GAAG,OAAO,MAAM;EAKlD,IAAI,QAAQ,QAAQ,CAAC,aAAa,IAAI,GAAG,GACvC,aAAa,IAAI,KAAK,MAAM;CAEhC;CAGF,MAAM,eAAe,YAAY,IAAI,cAAc;CACnD,MAAM,uCAAuB,IAAI,IAA2B;CAC5D,IAAI,YAAY;CAChB,IAAI,cACF,KAAK,MAAM,EAAC,KAAK,KAAK,KAAK,KAAK,MAAM,YAAW,GAC9C,QAAQ,2DAA2D,CAAC,CACpE,IAAI,GAAiB;EACtB;EACA,MAAM,UAAU,qBAAqB,IAAI,GAAG,IAAI,GAAG,KAAK,KAAK,CAAC;EAC9D,QAAQ,KAAK;GACX,KAAK,YAAY,GAAG;GACpB,KAAK,YAAY,GAAG;GACpB,MAAM,YAAY,IAAI;GAEtB,QAAQ,kBAAkB,MAAM;EAClC,CAAC;EACD,qBAAqB,IAAI,GAAG,IAAI,GAAG,OAAO,OAAO;CACnD;CAGF,MAAM,iCAAiB,IAAI,IAA0B;CACrD,KAAK,MAAM,SAAS,SAAS;EAC3B,MAAM,MAAM,GAAG,MAAM,UAAU,GAAG,MAAM;EACxC,MAAM,QAAQ,qBAAqB,IAAI,GAAG;EAC1C,MAAM,WAAW,eAAe,IAAI,MAAM,SAAS,KAAK,CAAC;EACzD,SAAS,KAAK;GACZ,MAAM,MAAM;GACZ,SAAS,OAAO,QAAQ,MAAM,OAAO,CAAC,CAAC,KAAK,CAAC,MAAM,UAAU;IAC3D;IACA;GACF,EAAE;GACF,QAAQ,MAAM;GACd,SAAS,MAAM,YAAY;GAC3B,OAAO,qBAAqB,IAAI,GAAG;GACnC,OAAO,OAAO,SAAS,eAAe,KAAK,IAAI,KAAA;GAC/C,cAAc,KAAK,aAAa,OAAO,SAAS,QAAQ,KAAA;EAC1D,CAAC;EACD,eAAe,IAAI,MAAM,WAAW,QAAQ;CAC9C;CAEA,MAAM,qBAA+B,CAAC;CACtC,MAAM,aAAa,OAAO,KAAI,UAAS;EACrC,MAAM,UAAU,SAAS,IAAI,MAAM,IAAI,CAAC,EAAE;EAC1C,IAAI,CAAC,aAAa,IAAI,MAAM,IAAI,GAC9B,mBAAmB,KAAK,MAAM,IAAI;EAEpC,MAAM,cAAc,IAAI,IAAI,MAAM,eAAe,CAAC,CAAC;EACnD,OAAO;GACL,MAAM,MAAM;GACZ,SAAS,OAAO,QAAQ,MAAM,OAAO,CAAC,CAAC,KAAK,CAAC,MAAM,WAAW;IAC5D;IACA,UAAU,KAAK;IACf,UAAU,CAAC,KAAK;IAChB,SAAS,UAAU,KAAK,EAAE,QAAQ;IAClC,GAAI,YAAY,IAAI,IAAI,IAAI,EAAC,aAAa,KAAa,IAAI,CAAC;GAC9D,EAAE;GACF,YAAY,MAAM,aAAa,CAAC,GAAG,MAAM,UAAU,IAAI,KAAA;GACvD,eAAe,aAAa,IAAI,MAAM,IAAI,CAAC,EAAE;GAC7C,UAAU,YAAY,KAAA;GACtB,SAAS,eAAe,IAAI,MAAM,IAAI,KAAK,CAAC;EAC9C;CACF,CAAC;CAED,OAAO;EACL,OAAO;EACP,QAAQ;GACN,aAAa,iBAAiB,MAAM;GACpC,eAAe,cAAc,EAAE;GAC/B,kBAAkB,iBAAiB,IAAI,EAAE;GACzC,8BAAa,IAAI,KAAK,EAAA,CAAE,YAAY;GACpC,SAAS;IACP,oBAAoB,OAAO;IAE3B,mCACE,OAAO,sCAAsC;IAC/C,4BAA4B,OAAO,+BAA+B;GACpE;EACF;EACA,cAAc;GACZ;GACA;GACA,QAAQ,YAAY,IAAI,cAAc;GACtC;EACF;EACA,QAAQ;CACV;AACF;;;;;;;AAQA,SAAS,kBAAkB,IAAgB,IAAc;CACvD,IAAI;EACF,OAAO,WAAW,EAAE;CACtB,SAAS,GAAG;EACV,GAAG,QAAQ,4DAA4D,CAAC;EACxE,OAAO,WAAW,IAAI,MAAM,KAAK;CACnC;AACF;;;;;;;;;;;;AAaA,SAAgB,eAAe,SAAsC;CACnE,MAAM,QAAQ,KAAK,IAAI,GAAG,QAAQ,KAAI,MAAK,EAAE,IAAI,MAAM,CAAC;CACxD,MAAM,YAAkC,CAAC;CACzC,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,KAAK;EAC9B,MAAM,SAAS,QACZ,KAAI,MAAK,EAAE,IAAI,EAAE,CAAC,CAClB,QAAO,MAAK,MAAM,KAAA,CAAS,CAAC,CAC5B,MAAM,GAAG,MAAM,IAAI,CAAC;EACvB,MAAM,mBAAmB,KAAK,IAC5B,GAAG,QAAQ,KAAI,MAAK,EAAE,KAAK,EAAE,CAAC,CAAC,QAAO,MAAK,MAAM,KAAA,CAAS,CAC5D;EACA,UAAU,KAAK;GACb,eAAe,OAAO,GAAG,EAAE;GAC3B,kBAAkB,OAAO,KAAK,OAAO,OAAO,SAAS,KAAK,CAAC;GAC3D,uBAAuB,mBAAmB;EAC5C,CAAC;CACH;CACA,OAAO;EACL,SAAS,QAAQ;EAEjB,aAAa,QAAQ,EAAE,CAAC;EACxB;CACF;AACF;AAEA,SAAS,cAAc,IAAsB;CAC3C,MAAM,CAAC,EAAC,aAAY,GACjB,QAAQ,oCAAoC,CAAC,CAC7C,IAAI;CACP,OAAO;AACT;AAEA,SAAS,iBAAiB,IAAgB,IAAkC;CAC1E,IAAI;EACF,OAAO,oBAAoB,IAAI,gBAAgB,EAAE,CAAC,CAAC,CAAC;CACtD,SAAS,GAAG;EAEV,GAAG,QAAQ,oCAAoC,CAAC;EAChD;CACF;AACF;;;;;;;;;;;;;;;;AAiBA,eAAsB,sBACpB,IACA,QACA,KACA,KACA;CACA,MAAM,cAAc,KAAK,GAAG;CAC5B,IAAI,kBAAkB,IAAI,QAAQ,aAAa,IAAI,MAAM,KAAA,GAAW;EAClE,IACG,KAAK,GAAG,CAAC,CACT,OAAO,oBAAoB,yCAAuC,CAAC,CACnE,KAAK,cAAc;EACtB;CACF;CAEA,MAAM,QAAQ,IAAI;CAClB,MAAM,SAAS,MAAM,WAAW,KAAA,IAAY,IAAI,KAAA;CAChD,IAAI;CACJ,IAAI;EACF,SAAS,oBAAoB,IAAI,QAAQ,EACvC,WAAW,MAAM,UAAU,OAC7B,CAAC;CACH,SAAS,GAAG;EACV,GAAG,QAAQ,kCAAkC,CAAC;EAC9C,MAAM,IAAI,KAAK,GAAG,CAAC,CAAC,KAAK,EAAC,OAAO,OAAO,CAAC,EAAC,CAAC;EAC3C;CACF;CACA,MAAM,IACH,OAAO,gBAAgB,kBAAkB,CAAC,CAC1C,KAAK,KAAK,UAAU,QAAQ,MAAM,MAAM,CAAC;AAC9C"}

package/out/zero-cache/src/services/replicator/change-log-db.js

@@ -487,6 +487,6 @@ function tableExists(db, table) {
       `).get(table) !== void 0;
 }
 //#endregion
-export { CHANGE_LOG_META_TABLE, CHANGE_LOG_STREAM_TABLE, CHANGE_LOG_STREAM_WRITE_TIME_INDEX, CREATE_CHANGE_LOG_STREAM_SCHEMA, ChangeLogRebuildRequired, MAX_RECONCILE_TRUNCATE_BYTES, MAX_RECONCILE_TRUNCATE_ROWS, applyChangeLogPragmas, changeLogFileName, changeLogWipeReason, deleteChangeLogDB, estimateChangeLogStreamRowBytes, openChangeLogDB, openChangeLogDBForWriting, readChangeLogBounds, readChangeLogHead, readChangeLogMeta, rebuildChangeLogDBForWriting, reconcileChangeLog, seedChangeLogStream };
+export { CHANGE_LOG_META_TABLE, CHANGE_LOG_STREAM_TABLE, CHANGE_LOG_STREAM_WRITE_TIME_INDEX, CREATE_CHANGE_LOG_STREAM_SCHEMA, ChangeLogRebuildRequired, MAX_RECONCILE_TRUNCATE_BYTES, MAX_RECONCILE_TRUNCATE_ROWS, applyChangeLogPragmas, changeLogFileName, changeLogWipeReason, deleteChangeLogDB, estimateChangeLogStreamRowBytes, openChangeLogDB, openChangeLogDBForWriting, readChangeLogBounds, readChangeLogHead, readChangeLogMeta, rebuildChangeLogDBForWriting, reconcileChangeLog };
 
 //# sourceMappingURL=change-log-db.js.map

package/out/zero-cache/src/services/replicator/change-log-db.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-log-db.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/change-log-db.ts"],"sourcesContent":["/**\n * The change log lives in its own SQLite database, `${replicaFile}-change-log`,\n * rather than in the replica file. It is a catchup cache: everything it holds is\n * either already durable in the replica's litestream backup or re-derivable from\n * the upstream replication slot, so it is deliberately excluded from the backup\n * and is never a source of truth.\n *\n * The change-streamer writes it, from the stream loop that fans changes out to\n * subscribers, and commits it before forwarding each transaction's `commit`\n * message. Its head is therefore always at or above the watermark the stream\n * would resume from, which is what {@link reconcileChangeLog} anchors on.\n *\n * Beside the buffer it holds the log's *cookie jar* — the backfill progress\n * state folded up to the same head. That is `change-log-cookies.ts`; this file\n * owns its tables' lifecycle, because a cookie set is only meaningful paired\n * with the watermark it was folded to, and so must be created, dropped, and —\n * whenever reconciliation moves the head — replaced with the anchor's, all with\n * the buffer rather than beside it.\n *\n * Because it is disposable, it has no migration framework. Any inconsistency —\n * a schema change, a leftover file from a different replica, a gap left by a\n * crash or by running with the writer disabled, a corrupt file — is resolved by\n * {@link reconcileChangeLog} and {@link openChangeLogDBForWriting}, which\n * truncate a bounded suffix or replace and reseed the log at the resume\n * watermark. The worst outcome is a `too-old` for a lagging subscriber; a\n * silently delivered gap is not reachable.\n *\n * Disk sizing: a task that writes the log needs room for the replica *plus* the\n * retained change stream — the log's main file, its wal2 sidecars, and pages a\n * purge has freed but not yet returned to the OS. The change-streamer schedules\n * purges against the same floor as the PG change log, so retention is bounded\n * by the confirmed backup watermark ANDed with the minimum retention window —\n * a function of the backup interval, not a constant: 15–30+ minutes behind an\n * RMv1 sync interval, ~60 seconds once litestream v5 backs up every 15–30 s.\n * A held snapshot reservation pins retention unboundedly for its duration, so\n * peak retained bytes is set by the slowest concurrent restore rather than by\n * the retention setting — size disk and alerts on that. The log is excluded\n * from the litestream backup, so this is local disk only and never S3.\n */\n\nimport {existsSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {\n  CHANGE_LOG_BACKFILLING_TABLE,\n  CHANGE_LOG_TABLE_METADATA_TABLE,\n  CREATE_CHANGE_LOG_COOKIE_SCHEMA,\n  DROP_CHANGE_LOG_COOKIE_TABLES,\n  replaceCookies,\n  type CookieSet,\n} from './change-log-cookies.ts';\n\n/**\n * Bumped whenever the change-log database's schema changes. Since the file is a\n * cache, a mismatch costs one reseed rather than a migration.\n *\n * v2 re-homed the writer to the change-streamer: the log anchors on the\n * watermark its stream connection resumes from rather than on the replica's\n * state version, and its meta row carries the replica's identity triple plus\n * the seed point.\n *\n * v3 added the cookie tables (see `change-log-cookies.ts`), i.e. the log's\n * second job. The reseed it costs on upgrade buys one retention window of\n * catchup reach on a log nothing reads yet.\n *\n * v4 stores each row's tag and estimated retained bytes alongside its JSON.\n * Readers can now filter without parsing arbitrary row payloads, and\n * reconciliation can bound truncation by bytes as well as rows.\n *\n * `auto_vacuum = INCREMENTAL` arrived within v2, deliberately without a bump:\n * it is a file-header property rather than a schema change, and the reseed a\n * version mismatch triggers cannot enable it (see\n * {@link applyChangeLogPragmas}), so {@link openChangeLogDBForWriting} guards\n * on the pragma itself instead of on this number.\n */\nexport const CHANGE_LOG_DB_SCHEMA_VERSION = 4;\n\n/** https://www.sqlite.org/pragma.html#pragma_auto_vacuum */\nconst AUTO_VACUUM_INCREMENTAL = 2;\n\nexport const CHANGE_LOG_STREAM_TABLE = '_zero.changeLogStream';\nexport const CHANGE_LOG_STREAM_WRITE_TIME_INDEX =\n  '_zero.changeLogStream_writeTimeMs';\n\nconst INTEGER_BYTES = 8;\n\n/**\n * Estimates the retained payload size of a stream row. This deliberately does\n * not claim to measure SQLite b-tree/page overhead; it is stable across the\n * write path, reconciliation, and the startup scan used by observability.\n */\nexport function estimateChangeLogStreamRowBytes(\n  watermark: string,\n  tag: string,\n  change: string,\n  precommit?: string,\n  hasWriteTime = false,\n): number {\n  return (\n    Buffer.byteLength(watermark) +\n    INTEGER_BYTES +\n    Buffer.byteLength(tag) +\n    Buffer.byteLength(change) +\n    (precommit === undefined ? 0 : Buffer.byteLength(precommit)) +\n    (hasWriteTime ? INTEGER_BYTES : 0)\n  );\n}\n\n// The index is partial because only commit rows carry a \"writeTimeMs\", so it\n// holds one entry per transaction rather than one per change. It is what the\n// purger scans to find transactions older than the retention window.\n//\n// Pre-release builds briefly carried this table in the replica; see the test\n// fixture `CREATE_V14_CHANGE_LOG_STREAM` in\n// `change-source/common/replica-schema.ts`. Its old shape must not be kept in\n// sync with this one.\nexport const CREATE_CHANGE_LOG_STREAM_SCHEMA = /*sql*/ `\n  CREATE TABLE \"${CHANGE_LOG_STREAM_TABLE}\" (\n    \"watermark\"      TEXT NOT NULL,\n    \"pos\"            INTEGER NOT NULL,\n    \"tag\"            TEXT NOT NULL,\n    \"estimatedBytes\"  INTEGER NOT NULL,\n    \"change\"          TEXT NOT NULL,\n    \"precommit\"       TEXT,\n    \"writeTimeMs\"     INTEGER,\n    PRIMARY KEY (\"watermark\", \"pos\")\n  );\n\n  CREATE INDEX \"${CHANGE_LOG_STREAM_WRITE_TIME_INDEX}\"\n    ON \"${CHANGE_LOG_STREAM_TABLE}\" (\"writeTimeMs\", \"watermark\")\n    WHERE \"writeTimeMs\" IS NOT NULL;\n`;\n\n/**\n * Inserts the synthetic begin/commit pair for the seed watermark.\n *\n * Both rows are inserted by one statement so a caller outside a larger\n * transaction cannot leave a partial seed.\n */\nconst SEED_CHANGE_LOG_STREAM_SQL = /*sql*/ `\n  INSERT INTO \"${CHANGE_LOG_STREAM_TABLE}\"\n    (\"watermark\", \"pos\", \"tag\", \"estimatedBytes\", \"change\", \"precommit\",\n     \"writeTimeMs\")\n  VALUES\n    (@watermark, 0, 'begin', @beginBytes, @beginChange, NULL, NULL),\n    (@watermark, 1, 'commit', @commitBytes, @commitChange, @watermark,\n     @writeTimeMs)\n  ON CONFLICT (\"watermark\", \"pos\") DO NOTHING\n`;\n\nexport const CHANGE_LOG_META_TABLE = '_zero.changeLogMeta';\n\n// The identity triple (see ChangeLogIdentity) detects a file left behind by a\n// different replica -- most concretely, a leftover log on a reused volume.\n//\n// \"seededAtMs\"    : when the log was seeded, i.e. how far its retention window\n//                   can possibly reach back. Slice 11 uses it as the warm-up\n//                   signal for canary reads.\n// \"seedWatermark\" : where the log was seeded. `min(\"watermark\")` cannot answer\n//                   this once purge has advanced the minimum, and the\n//                   difference is what distinguishes a log that purged past the\n//                   retention floor (an invariant violation) from one that\n//                   never held that history at all (routine at every fork).\n// \"lock\"          : enforces single-row semantics, as in\n//                   \"_zero.replicationConfig\".\nconst CREATE_CHANGE_LOG_META_SCHEMA = /*sql*/ `\n  CREATE TABLE \"${CHANGE_LOG_META_TABLE}\" (\n    \"epoch\"         TEXT,\n    \"generation\"    TEXT NOT NULL,\n    \"replicaID\"     TEXT,\n    \"schemaVersion\" INTEGER NOT NULL,\n    \"seededAtMs\"    INTEGER NOT NULL,\n    \"seedWatermark\" TEXT NOT NULL,\n    \"lock\"          INTEGER PRIMARY KEY DEFAULT 1 CHECK (\"lock\" = 1)\n  );\n`;\n\n/**\n * `${replicaFile}-change-log`, matching the `-serving-copy` convention.\n *\n * The name outlives the writer's move into the change-streamer: the\n * change-streamer derives the path from the replica file it owns, so renaming\n * would save no plumbing and would orphan the old file on every existing\n * volume, where nothing would delete it.\n *\n * Its `-wal` / `-wal2` / `-shm` sidecars are suffixes of *this* name, so they\n * never collide with the replica's own sidecars.\n */\nexport function changeLogFileName(replicaFile: string): string {\n  return `${replicaFile}-change-log`;\n}\n\n/**\n * Removes the change-log database and its `-wal` / `-wal2` / `-shm` sidecars.\n * Safe to call when the file does not exist.\n */\nexport function deleteChangeLogDB(replicaFile: string): void {\n  deleteLiteDB(changeLogFileName(replicaFile));\n}\n\n/**\n * Which replica the log's contents belong to.\n *\n * `generation` is RMv1's `replicaVersion`: the version at which initial sync\n * copied every row. It is shared by every sibling of a forked replica, which is\n * why it cannot be the whole of the identity under RMv2 — `epoch` and\n * `replicaID` are what distinguish two siblings' logs. Both are `null` until\n * RMv2 supplies them, and a `null` matches only another `null`, so a log\n * written before they exist is still recognized as this replica's.\n */\nexport type ChangeLogIdentity = {\n  readonly epoch: string | null;\n  readonly generation: string;\n  readonly replicaID: string | null;\n};\n\n/**\n * Where the stream connection resumes, and everything the log's state at that\n * position consists of.\n *\n * The two fields travel together because a cookie set is only meaningful paired\n * with the watermark it was folded up to (invariant 15). They are read from one\n * store, in one snapshot: `Storer.getStartStreamInitializationParameters()`\n * reads both from Postgres in a single REPEATABLE READ transaction. Pairing one\n * store's cookies with another's watermark would silently drop every backfill\n * that completed in the interval, and those rows are unrecoverable — backfill\n * transactions are synthesized by the change source rather than replayed from\n * the slot.\n */\nexport type ChangeLogResumePoint = {\n  /**\n   * The watermark this stream connection resumes from. The log's own head is\n   * measured against it, and a reseed seeds a synthetic transaction at it.\n   *\n   * Deliberately *not* \"the replica's state version\": after the writer moved to\n   * the change-streamer, the canonical replicator is a subscriber downstream of\n   * the write loop, so its state version is a consequence of this log rather\n   * than a constraint on it.\n   */\n  readonly resumeWatermark: string;\n  /**\n   * The cookie set that belongs to {@link resumeWatermark}. Any reconciliation\n   * that moves the head invalidates the set the log was holding — the cookies\n   * are unversioned point-in-time state, and truncating the stream above `W`\n   * does not roll them back to `W` — so the same transaction installs this one\n   * in its place (invariant 17).\n   *\n   * Postgres supplies it while it remains the store the stream resumes from. At\n   * the C5 flip the log's own head *is* the resume point, phantoms cannot\n   * exist, and both this field and the truncate-above path become dead code.\n   */\n  readonly cookies: CookieSet;\n};\n\n/**\n * What the log is reconciled against. Reconciliation runs once per stream\n * connection, since every reconnect can re-read a different resume point.\n */\nexport type ChangeLogAnchor = ChangeLogResumePoint & {\n  readonly identity: ChangeLogIdentity;\n  /**\n   * The change-streamer's clock, at reconciliation time. Supplied rather than\n   * read inline so that tests control it. Used for `seededAtMs` and for the\n   * seed transaction's `writeTimeMs`, which is what the purger's time floor\n   * measures.\n   */\n  readonly nowMs: number;\n};\n\n/** The meta row, i.e. everything the log records about itself. */\nexport type ChangeLogMeta = ChangeLogIdentity & {\n  readonly schemaVersion: number;\n  readonly seededAtMs: number;\n  readonly seedWatermark: string;\n};\n\n/**\n * Opens the change-log database beside `replicaFile`.\n *\n * A writable handle creates the file if it is absent; a `readonly` handle\n * throws, which is the normal state for a change-streamer whose writer is\n * disabled. Pragmas are the caller's responsibility, since only the writer sets\n * the persistent ones.\n */\nexport function openChangeLogDB(\n  lc: LogContext,\n  replicaFile: string,\n  opts: {readonly: boolean},\n): Database {\n  return new Database(\n    lc.withContext('component', 'change-log-db'),\n    changeLogFileName(replicaFile),\n    {readonly: opts.readonly},\n  );\n}\n\n/**\n * Configures the change-log database, which the change-streamer's writer owns\n * exclusively. These are the settings the `separate-files` mode of\n * `sqlite-change-log-ceiling.bench.ts` measured.\n *\n * `wal2` because the log is a continuous append that catchup reads\n * continuously, which is the case wal2 exists for: it avoids checkpoint\n * starvation without needing an autocheckpoint. `wal_autocheckpoint` therefore\n * keeps its default, unlike the backup replica's 0, which hands checkpointing\n * to litestream — nothing else owns this file.\n *\n * `synchronous = NORMAL` because the commit sits on the forward path, once per\n * upstream transaction, and NORMAL keeps that from costing an fsync. It is not\n * a durability compromise here: in WAL mode NORMAL still writes every commit\n * through to the OS, so it survives a process crash, an OOM kill, or a SIGKILL.\n * It is lost only to kernel panic or power loss, and those destroy the node —\n * the task restarts elsewhere, restores the replica from S3, and has no\n * change-log file at all. Whatever a power loss does take is detected by\n * {@link reconcileChangeLog} and reseeded.\n *\n * `auto_vacuum = INCREMENTAL` so that purge can return freed pages to the OS\n * rather than leaving the file at its high-water mark. **It must be the first\n * statement here.** SQLite honours a `none` -> `incremental` transition only\n * while the file is still empty, and `journal_mode = wal2` materializes page 1;\n * appending the pragma instead of prepending it silently leaves the file at\n * mode 0 forever, since neither `DROP TABLE` nor a reseed makes a file empty\n * again. {@link openChangeLogDBForWriting} verifies the result.\n */\nexport function applyChangeLogPragmas(db: Database): void {\n  db.pragma('auto_vacuum = INCREMENTAL');\n  db.pragma('busy_timeout = 30000');\n  db.pragma('analysis_limit = 1000');\n  db.pragma('journal_mode = wal2');\n  db.pragma('synchronous = NORMAL');\n}\n\nfunction readAutoVacuum(db: Database): number {\n  const [{auto_vacuum: mode}] = db.pragma<{auto_vacuum: number}>('auto_vacuum');\n  return mode;\n}\n\n/**\n * Opens the change-log database for the writer, applies its pragmas, and\n * reconciles it against `anchor`, returning the handle and what reconciliation\n * had to do.\n *\n * Corruption is answered with a rebuild rather than an escalation, which is a\n * deliberate divergence from how the replica's own corruption is handled. A\n * corrupt replica is a lost source of truth, so it ends in a restore or an\n * auto-reset; the change log holds nothing that is not either already in the\n * replica's litestream backup or re-derivable from the upstream slot, so the\n * entire response is to delete the file and seed a new one at the resume\n * watermark. The cost is one retention window of catchup — reconnecting\n * subscribers below the head get `too-old` — which is what every other\n * {@link ReseedReason} costs.\n *\n * Only corruption is rebuilt through. Anything else (a bad path, a permissions\n * error, a full disk) is a problem with the environment rather than with the\n * file's contents, and deleting a database in response would destroy state\n * without fixing it. Those propagate to the caller, which disables the writer\n * and keeps replicating.\n *\n * A file that predates `auto_vacuum = INCREMENTAL` takes the same rebuild, for\n * the same reason and at the same cost: the pragma only takes on an empty file,\n * so a reseed cannot enable it. Unlike corruption, a *second* failure there is\n * tolerated rather than thrown — see {@link ensureIncrementalAutoVacuum}.\n */\n/**\n * Returns an anchor after the database opens.\n *\n * The resolver can inspect the existing file before it selects the anchor. A\n * valid log can use its own head. A new or invalid log uses the replica seed.\n * The rebuild path calls the resolver again for the new file.\n */\nexport type AnchorResolver = (db: Database) => ChangeLogAnchor;\n\nexport function openChangeLogDBForWriting(\n  lc: LogContext,\n  replicaFile: string,\n  anchor: AnchorResolver,\n): {db: Database; result: ReconcileResult; replacedFile: boolean} {\n  const existed = existsSync(changeLogFileName(replicaFile));\n  const opened = openOrRebuildCorrupt(lc, replicaFile, anchor, existed);\n  const {db, result, replacedFile} = ensureIncrementalAutoVacuum(\n    lc,\n    replicaFile,\n    anchor,\n    opened,\n  );\n  return {db, result, replacedFile};\n}\n\nfunction openOrRebuildCorrupt(\n  lc: LogContext,\n  replicaFile: string,\n  anchor: AnchorResolver,\n  boundReconciliation: boolean,\n): OpenedChangeLog {\n  try {\n    return openAndReconcile(lc, replicaFile, anchor, boundReconciliation);\n  } catch (e) {\n    if (e instanceof ChangeLogRebuildRequired) {\n      return rebuildChangeLog(lc, replicaFile, anchor, e);\n    }\n    if (!isSQLiteCorruption(e)) {\n      throw e;\n    }\n    const file = changeLogFileName(replicaFile);\n    logSQLiteCorruptionDiagnostics(lc, 'change-log', file, e);\n    lc.error?.('rebuilding the corrupt SQLite change log', {\n      sqliteChangeLog: {file},\n    });\n    return rebuildChangeLog(lc, replicaFile, anchor);\n  }\n}\n\n/**\n * Rebuilds a log whose `auto_vacuum` mode is not `INCREMENTAL`, which is every\n * log created before the pragma shipped — including every v2 file the writer\n * wrote in the meantime, which is why the guard is on the pragma rather than\n * on the schema version. Deleting the file is the only way to enable it:\n * `DROP TABLE` leaves the header's mode intact, and the in-place alternative —\n * a full `VACUUM` — is an unbounded blocking statement on a file the writer\n * needs.\n *\n * A mode that is still wrong after the rebuild is logged and tolerated, which\n * is the one place this diverges from the corruption path. The two failures are\n * not alike: a file that stays corrupt after a rebuild is an environment\n * problem, but a pragma that stays wrong on a brand-new file can only mean\n * {@link applyChangeLogPragmas}' statement order is wrong — a code bug, present\n * on every task, on every start. Throwing there turns a lost optimization into\n * a crash loop, and rebuilding again turns it into a reseed per start, which is\n * worse than the freelist growth it was trying to avoid. `Database.compact()`\n * already warns and no-ops when the mode is not `INCREMENTAL`, so degrading is\n * the accepted fallback.\n */\nfunction ensureIncrementalAutoVacuum(\n  lc: LogContext,\n  replicaFile: string,\n  anchor: AnchorResolver,\n  opened: OpenedChangeLog,\n): OpenedChangeLog {\n  if (opened.autoVacuum === AUTO_VACUUM_INCREMENTAL) {\n    return opened;\n  }\n  const file = changeLogFileName(replicaFile);\n  lc.info?.('rebuilding the SQLite change log to enable incremental vacuum', {\n    sqliteChangeLog: {file, autoVacuum: opened.autoVacuum},\n  });\n  opened.db.close();\n  const rebuilt = rebuildChangeLog(lc, replicaFile, anchor);\n  if (rebuilt.autoVacuum !== AUTO_VACUUM_INCREMENTAL) {\n    lc.error?.(\n      'SQLite change log does not support incremental vacuum; ' +\n        'it will plateau at its high-water mark',\n      {sqliteChangeLog: {file, autoVacuum: rebuilt.autoVacuum}},\n    );\n  }\n  return rebuilt;\n}\n\n/** The shared \"delete the file and open a new one\" body. */\nfunction rebuildChangeLog(\n  lc: LogContext,\n  replicaFile: string,\n  anchor: AnchorResolver,\n  required?: ChangeLogRebuildRequired | undefined,\n): OpenedChangeLog {\n  if (required) {\n    lc.warn?.(\n      'rebuilding the SQLite change log instead of running an unbounded reconciliation',\n      {\n        sqliteChangeLogRebuild: {\n          reason: required.reason,\n          rows: required.rows,\n          estimatedBytes: required.estimatedBytes,\n        },\n      },\n    );\n  }\n  deleteChangeLogDB(replicaFile);\n  // The file is absent, so the reseed only drops empty/nonexistent tables. A\n  // second failure is the environment's, not the old file's.\n  const opened = openAndReconcile(\n    lc,\n    replicaFile,\n    anchor,\n    false,\n    required?.reason,\n  );\n  return {...opened, replacedFile: true};\n}\n\n/**\n * Replaces an already-closed change-log file after bounded reconciliation\n * declined its work. The caller closes cached readers before calling this so\n * none can continue serving the unlinked inode.\n */\nexport function rebuildChangeLogDBForWriting(\n  lc: LogContext,\n  replicaFile: string,\n  anchor: ChangeLogAnchor,\n  required: ChangeLogRebuildRequired,\n): {db: Database; result: ReconcileResult; replacedFile: true} {\n  const {db, result} = rebuildChangeLog(\n    lc,\n    replicaFile,\n    () => anchor,\n    required,\n  );\n  return {db, result, replacedFile: true};\n}\n\ntype OpenedChangeLog = {\n  db: Database;\n  result: ReconcileResult;\n  replacedFile: boolean;\n  /** The file's persistent `auto_vacuum` mode, read back after the pragmas. */\n  autoVacuum: number;\n};\n\nfunction openAndReconcile(\n  lc: LogContext,\n  replicaFile: string,\n  anchor: AnchorResolver,\n  boundReconciliation: boolean,\n  reseedReason?: ReseedReason | undefined,\n): OpenedChangeLog {\n  const db = openChangeLogDB(lc, replicaFile, {readonly: false});\n  try {\n    applyChangeLogPragmas(db);\n    const autoVacuum = readAutoVacuum(db);\n    return {\n      db,\n      result: reconcileChangeLog(lc, db, anchor(db), {\n        rebuildInsteadOfUnboundedWork: boundReconciliation,\n        reseedReason,\n      }),\n      replacedFile: false,\n      autoVacuum,\n    };\n  } catch (e) {\n    // The caller never sees this handle, so it closes here or leaks — and it\n    // must be closed before the corruption path deletes the file.\n    db.close();\n    throw e;\n  }\n}\n\nexport type ReseedReason =\n  | 'created' // file or table absent\n  | 'schema-mismatch'\n  | 'identity-mismatch'\n  | 'gap' // head does not land on the resume watermark after truncation\n  | 'oversized-truncate'; // preserving the prefix would require too much work\n\n/**\n * Reconciliation runs synchronously on the change-streamer's event loop. A\n * normal phantom is one transaction, so work above either limit is answered by\n * unlinking and reseeding this disposable cache rather than deleting an\n * arbitrarily large suffix in place.\n */\nexport const MAX_RECONCILE_TRUNCATE_ROWS = 10_000;\nexport const MAX_RECONCILE_TRUNCATE_BYTES = 16 * 1024 * 1024;\n\nexport class ChangeLogRebuildRequired extends Error {\n  override readonly name = 'ChangeLogRebuildRequired';\n  readonly reason: ReseedReason;\n  readonly rows?: number | undefined;\n  readonly estimatedBytes?: number | undefined;\n\n  constructor(\n    reason: ReseedReason,\n    rows?: number | undefined,\n    estimatedBytes?: number | undefined,\n  ) {\n    super(`SQLite change log requires a ${reason} rebuild`);\n    this.reason = reason;\n    this.rows = rows;\n    this.estimatedBytes = estimatedBytes;\n  }\n}\n\ntype ReconcileOptions = {\n  /** Throw before an unbounded delete/drop so the file owner can unlink it. */\n  readonly rebuildInsteadOfUnboundedWork?: boolean | undefined;\n  /** Preserve the reason that caused an old file to be unlinked. */\n  readonly reseedReason?: ReseedReason | undefined;\n};\n\n/**\n * `cookiesStale` records that the cookie set the log held did not survive\n * reconciliation. The cookies are unversioned point-in-time state, so any\n * reconciliation that moves the head invalidates them, and the same transaction\n * replaces them with the anchor's — the set that belongs to the resume\n * watermark (see `change-log-cookies.ts`). It is `action !== 'none'` today; it\n * is a field rather than a derivation because the replacement reads as intent\n * at the call site, and because it is what the `cookie_reseed` counter is keyed\n * on.\n */\nexport type ReconcileResult =\n  | {action: 'none'; head: string; cookiesStale: false}\n  | {action: 'truncated'; head: string; rows: number; cookiesStale: true}\n  | {\n      action: 'reseeded';\n      head: string;\n      reason: ReseedReason;\n      cookiesStale: true;\n    };\n\n/**\n * Brings the change log into agreement with the watermark its stream connection\n * resumes from, in one transaction on the change-log database:\n *\n * ```\n * head === resumeWatermark  → keep appending\n * head  >  resumeWatermark  → truncate above resumeWatermark\n * otherwise                 → wipe, seed at resumeWatermark, record the reason\n * ```\n *\n * The log commits before anything that can advance the resume watermark, so a\n * *phantom* — a transaction the log holds whose watermark the resumed stream\n * re-delivers — is the normal state, and truncate-above is the whole of\n * reconciliation rather than a special case. A phantom's rows all carry the\n * transaction's commit watermark, which is strictly greater than the resume\n * watermark, so deleting on that predicate removes phantoms whole and retains\n * every transaction at or below the resume point.\n *\n * It must run per stream connection, not once per process: the stream loop\n * re-reads its resume watermark on every reconnect, and the writer inserts with\n * a plain `INSERT` against `PRIMARY KEY (\"watermark\",\"pos\")`, so an\n * un-truncated overlap is a constraint violation on the first re-delivered\n * transaction.\n *\n * Anything truncation cannot resolve — the file was deleted, the writer was\n * disabled for a while, the replica was restored, power was lost with\n * `synchronous=NORMAL` — surfaces as a head that does not land on the resume\n * watermark, and is resolved by wiping and reseeding there.\n *\n * Whichever of the two moves the head also replaces the cookie set with the one\n * the anchor carries, in the same transaction: the head and the cookies are one\n * position, and nothing may observe them at two.\n */\nexport function reconcileChangeLog(\n  lc: LogContext,\n  db: Database,\n  anchor: ChangeLogAnchor,\n  opts: ReconcileOptions = {},\n): ReconcileResult {\n  db.exec('BEGIN IMMEDIATE');\n  try {\n    if (opts.rebuildInsteadOfUnboundedWork) {\n      const required = rebuildRequired(db, anchor);\n      if (required) {\n        throw required;\n      }\n    }\n    const result = opts.reseedReason\n      ? reseed(lc, db, anchor, opts.reseedReason)\n      : reconcile(lc, db, anchor);\n    db.exec('COMMIT');\n    return result;\n  } catch (e) {\n    if (db.inTransaction) {\n      db.exec('ROLLBACK');\n    }\n    throw e;\n  }\n}\n\n/**\n * Decides whether reconciliation can be performed within its synchronous work\n * budget. Every query is either an index seek or a scan capped at one row past\n * the delete limit. No state changes before the decision is complete.\n */\nfunction rebuildRequired(\n  db: Database,\n  anchor: ChangeLogAnchor,\n): ChangeLogRebuildRequired | undefined {\n  const wipe = changeLogWipeReason(db, anchor.identity);\n  if (wipe !== undefined) {\n    return new ChangeLogRebuildRequired(wipe);\n  }\n\n  const head = readChangeLogHead(db);\n  const {resumeWatermark} = anchor;\n  if (head === null || head < resumeWatermark) {\n    return new ChangeLogRebuildRequired('gap');\n  }\n  if (head === resumeWatermark) {\n    return undefined;\n  }\n\n  const boundary = db\n    .prepare(/*sql*/ `\n      SELECT 1 FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n        WHERE \"watermark\" = ?\n        LIMIT 1\n    `)\n    .get<{1: number} | undefined>(resumeWatermark);\n  if (boundary === undefined) {\n    return new ChangeLogRebuildRequired('gap');\n  }\n\n  const {rows, estimatedBytes} = db\n    .prepare(/*sql*/ `\n      SELECT count(*) AS \"rows\",\n             coalesce(sum(\"estimatedBytes\"), 0) AS \"estimatedBytes\"\n        FROM (\n          SELECT \"estimatedBytes\" FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n            WHERE \"watermark\" > ?\n            LIMIT ${MAX_RECONCILE_TRUNCATE_ROWS + 1}\n        )\n    `)\n    .get<{rows: number; estimatedBytes: number}>(resumeWatermark);\n  return rows > MAX_RECONCILE_TRUNCATE_ROWS ||\n    estimatedBytes > MAX_RECONCILE_TRUNCATE_BYTES\n    ? new ChangeLogRebuildRequired('oversized-truncate', rows, estimatedBytes)\n    : undefined;\n}\n\nfunction reconcile(\n  lc: LogContext,\n  db: Database,\n  anchor: ChangeLogAnchor,\n): ReconcileResult {\n  const wipe = changeLogWipeReason(db, anchor.identity);\n  if (wipe !== undefined) {\n    return reseed(lc, db, anchor, wipe);\n  }\n\n  const {resumeWatermark} = anchor;\n  let head = readChangeLogHead(db);\n  let rows = 0;\n  if (head !== null && head > resumeWatermark) {\n    rows = db\n      .prepare(/*sql*/ `\n        DELETE FROM \"${CHANGE_LOG_STREAM_TABLE}\" WHERE \"watermark\" > ?\n      `)\n      .run(resumeWatermark).changes;\n    head = readChangeLogHead(db);\n  }\n\n  if (head !== resumeWatermark) {\n    return reseed(lc, db, anchor, 'gap');\n  }\n  if (rows > 0) {\n    // The truncated transactions can have carried `create-table`,\n    // `add-column`, and `backfill-completed`, none of which the truncate rolls\n    // back, so the cookie set no longer belongs to the head. The anchor's set\n    // does, and it lands in this same transaction: invariant 17 holds at every\n    // point another connection could observe the log.\n    replaceCookies(db, anchor.cookies);\n    lc.info?.('truncated phantom transactions from the SQLite change log', {\n      sqliteChangeLogReconcile: {head, rows, cookies: cookieCounts(anchor)},\n    });\n    return {action: 'truncated', head, rows, cookiesStale: true};\n  }\n  lc.debug?.('SQLite change log is at the resume watermark', {\n    sqliteChangeLogReconcile: {head},\n  });\n  return {action: 'none', head, cookiesStale: false};\n}\n\n/**\n * Returns the reason that the log must be recreated. Returns `undefined` when\n * the writer can continue to use the log.\n *\n * This function takes only the expected identity because callers use the result\n * to select the anchor.\n */\nexport function changeLogWipeReason(\n  db: Database,\n  identity: ChangeLogIdentity,\n): ReseedReason | undefined {\n  if (\n    !tableExists(db, CHANGE_LOG_STREAM_TABLE) ||\n    !tableExists(db, CHANGE_LOG_META_TABLE)\n  ) {\n    return 'created';\n  }\n  // The schema version is read on its own because it is the one column every\n  // version of this table has had. Reading the rest of a v1 row would fail on\n  // its missing columns instead of reporting the mismatch that explains it.\n  const version = db\n    .prepare(/*sql*/ `\n      SELECT \"schemaVersion\" FROM \"${CHANGE_LOG_META_TABLE}\"\n    `)\n    .get<{schemaVersion: number} | undefined>();\n  if (version === undefined) {\n    return 'created';\n  }\n  if (version.schemaVersion !== CHANGE_LOG_DB_SCHEMA_VERSION) {\n    return 'schema-mismatch';\n  }\n  // Checked *after* the version, so that every pre-v4 file — which is every\n  // file in the fleet at the upgrade — reports the `schema-mismatch` that\n  // explains it rather than the `created` its absent cookie tables would\n  // otherwise look like.\n  if (\n    !tableExists(db, CHANGE_LOG_TABLE_METADATA_TABLE) ||\n    !tableExists(db, CHANGE_LOG_BACKFILLING_TABLE)\n  ) {\n    return 'created';\n  }\n  const {epoch, generation, replicaID} = readChangeLogMeta(db);\n  if (\n    epoch !== identity.epoch ||\n    generation !== identity.generation ||\n    replicaID !== identity.replicaID\n  ) {\n    return 'identity-mismatch';\n  }\n  return undefined;\n}\n\nfunction reseed(\n  lc: LogContext,\n  db: Database,\n  anchor: ChangeLogAnchor,\n  reason: ReseedReason,\n): ReconcileResult {\n  const {identity, resumeWatermark, nowMs} = anchor;\n  // Dropping the table drops its partial index with it. The cookie tables go\n  // with the buffer: a cookie set without the transactions it was folded from\n  // is not a smaller cookie set, it is a wrong one.\n  db.exec(/*sql*/ `\n    DROP TABLE IF EXISTS \"${CHANGE_LOG_STREAM_TABLE}\";\n    DROP TABLE IF EXISTS \"${CHANGE_LOG_META_TABLE}\";\n    ${DROP_CHANGE_LOG_COOKIE_TABLES}\n    ${CREATE_CHANGE_LOG_STREAM_SCHEMA}\n    ${CREATE_CHANGE_LOG_META_SCHEMA}\n    ${CREATE_CHANGE_LOG_COOKIE_SCHEMA}\n  `);\n  db.prepare(/*sql*/ `\n    INSERT INTO \"${CHANGE_LOG_META_TABLE}\"\n      (\"epoch\", \"generation\", \"replicaID\", \"schemaVersion\", \"seededAtMs\",\n       \"seedWatermark\")\n      VALUES (@epoch, @generation, @replicaID, @schemaVersion, @seededAtMs,\n              @seedWatermark)\n  `).run({\n    epoch: identity.epoch,\n    generation: identity.generation,\n    replicaID: identity.replicaID,\n    schemaVersion: CHANGE_LOG_DB_SCHEMA_VERSION,\n    seededAtMs: nowMs,\n    seedWatermark: resumeWatermark,\n  });\n  seedChangeLogStream(db, anchor);\n  // Into tables this statement just created, so the wholesale replacement is\n  // all insert. It is still the same call the truncate path makes: a reseed is\n  // the case where the log's cookie set was not merely moved but discarded, and\n  // the anchor's is equally what belongs at the head either way.\n  replaceCookies(db, anchor.cookies);\n\n  lc.info?.('reseeded the SQLite change log', {\n    sqliteChangeLogReconcile: {\n      reason,\n      head: resumeWatermark,\n      ...identity,\n      schemaVersion: CHANGE_LOG_DB_SCHEMA_VERSION,\n      seededAtMs: nowMs,\n      cookies: cookieCounts(anchor),\n    },\n  });\n  return {\n    action: 'reseeded',\n    head: resumeWatermark,\n    reason,\n    cookiesStale: true,\n  };\n}\n\n/**\n * Seeds a valid synthetic transaction at the anchor's resume watermark, making\n * an otherwise empty log serviceable as a catchup boundary for a subscriber\n * that is exactly at that watermark.\n */\nexport function seedChangeLogStream(\n  db: Database,\n  anchor: ChangeLogAnchor,\n): void {\n  const beginChange = '{\"tag\":\"begin\"}';\n  const commitChange = '{\"tag\":\"commit\"}';\n  db.prepare(SEED_CHANGE_LOG_STREAM_SQL).run({\n    watermark: anchor.resumeWatermark,\n    beginBytes: estimateChangeLogStreamRowBytes(\n      anchor.resumeWatermark,\n      'begin',\n      beginChange,\n    ),\n    beginChange,\n    commitBytes: estimateChangeLogStreamRowBytes(\n      anchor.resumeWatermark,\n      'commit',\n      commitChange,\n      anchor.resumeWatermark,\n      true,\n    ),\n    commitChange,\n    writeTimeMs: anchor.nowMs,\n  });\n}\n\n/** Reads the whole meta row. Throws if the log has not been reconciled. */\nexport function readChangeLogMeta(db: Database): ChangeLogMeta {\n  const meta = db\n    .prepare(/*sql*/ `\n      SELECT \"epoch\", \"generation\", \"replicaID\", \"schemaVersion\",\n             \"seededAtMs\", \"seedWatermark\"\n        FROM \"${CHANGE_LOG_META_TABLE}\"\n    `)\n    .get<ChangeLogMeta | undefined>();\n  if (meta === undefined) {\n    throw new Error('the SQLite change log has no meta row');\n  }\n  return meta;\n}\n\n/**\n * `max()` on the primary key's leading column, i.e. an index seek. Null when\n * the log is empty.\n */\nexport function readChangeLogHead(db: Database): string | null {\n  const {head} = db\n    .prepare(/*sql*/ `\n      SELECT max(\"watermark\") AS \"head\" FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n    `)\n    .get<{head: string | null}>();\n  return head;\n}\n\n/**\n * The covered watermark range. Both scalar subqueries seek the watermark\n * index, so unlike a `count()` this does not scan the log. Both bounds are\n * null when it is empty.\n */\nexport function readChangeLogBounds(db: Database): ChangeLogBounds {\n  return db\n    .prepare(/*sql*/ `\n      SELECT\n        (SELECT min(\"watermark\") FROM \"${CHANGE_LOG_STREAM_TABLE}\")\n          AS \"minWatermark\",\n        (SELECT max(\"watermark\") FROM \"${CHANGE_LOG_STREAM_TABLE}\")\n          AS \"headWatermark\"\n    `)\n    .get<ChangeLogBounds>();\n}\n\nexport type ChangeLogBounds = {\n  readonly minWatermark: string | null;\n  readonly headWatermark: string | null;\n};\n\n/**\n * What was installed, for the reconcile log line. Normally `{0, 0}`: a nonzero\n * `backfilling` means the resumed stream is being handed backfills to re-request.\n */\nfunction cookieCounts({cookies}: ChangeLogAnchor) {\n  return {\n    tableMetadata: cookies.tableMetadata.length,\n    backfilling: cookies.backfilling.length,\n  };\n}\n\nfunction tableExists(db: Database, table: string): boolean {\n  return (\n    db\n      .prepare(/*sql*/ `\n        SELECT 1 FROM \"sqlite_master\" WHERE \"type\" = 'table' AND \"name\" = ?\n      `)\n      .get<{1: number} | undefined>(table) !== undefined\n  );\n}\n"],"mappings":";;;;;;AAmFA,IAAM,0BAA0B;AAEhC,IAAa,0BAA0B;AACvC,IAAa,qCACX;AAEF,IAAM,gBAAgB;;;;;;AAOtB,SAAgB,gCACd,WACA,KACA,QACA,WACA,eAAe,OACP;CACR,OACE,OAAO,WAAW,SAAS,IAC3B,gBACA,OAAO,WAAW,GAAG,IACrB,OAAO,WAAW,MAAM,KACvB,cAAc,KAAA,IAAY,IAAI,OAAO,WAAW,SAAS,MACzD,eAAe,gBAAgB;AAEpC;AAUA,IAAa,kCAA0C;kBACrC,wBAAwB;;;;;;;;;;;kBAWxB,mCAAmC;UAC3C,wBAAwB;;;;;;;;;AAUlC,IAAM,6BAAqC;iBAC1B,wBAAwB;;;;;;;;;AAUzC,IAAa,wBAAwB;AAerC,IAAM,gCAAwC;kBAC5B,sBAAsB;;;;;;;;;;;;;;;;;;;;;AAsBxC,SAAgB,kBAAkB,aAA6B;CAC7D,OAAO,GAAG,YAAY;AACxB;;;;;AAMA,SAAgB,kBAAkB,aAA2B;CAC3D,aAAa,kBAAkB,WAAW,CAAC;AAC7C;;;;;;;;;AAsFA,SAAgB,gBACd,IACA,aACA,MACU;CACV,OAAO,IAAI,SACT,GAAG,YAAY,aAAa,eAAe,GAC3C,kBAAkB,WAAW,GAC7B,EAAC,UAAU,KAAK,SAAQ,CAC1B;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,SAAgB,sBAAsB,IAAoB;CACxD,GAAG,OAAO,2BAA2B;CACrC,GAAG,OAAO,sBAAsB;CAChC,GAAG,OAAO,uBAAuB;CACjC,GAAG,OAAO,qBAAqB;CAC/B,GAAG,OAAO,sBAAsB;AAClC;AAEA,SAAS,eAAe,IAAsB;CAC5C,MAAM,CAAC,EAAC,aAAa,UAAS,GAAG,OAA8B,aAAa;CAC5E,OAAO;AACT;AAqCA,SAAgB,0BACd,IACA,aACA,QACgE;CAGhE,MAAM,EAAC,IAAI,QAAQ,iBAAgB,4BACjC,IACA,aACA,QAJa,qBAAqB,IAAI,aAAa,QADrC,WAAW,kBAAkB,WAAW,CACK,CAK3D,CACF;CACA,OAAO;EAAC;EAAI;EAAQ;CAAY;AAClC;AAEA,SAAS,qBACP,IACA,aACA,QACA,qBACiB;CACjB,IAAI;EACF,OAAO,iBAAiB,IAAI,aAAa,QAAQ,mBAAmB;CACtE,SAAS,GAAG;EACV,IAAI,aAAa,0BACf,OAAO,iBAAiB,IAAI,aAAa,QAAQ,CAAC;EAEpD,IAAI,CAAC,mBAAmB,CAAC,GACvB,MAAM;EAER,MAAM,OAAO,kBAAkB,WAAW;EAC1C,+BAA+B,IAAI,cAAc,MAAM,CAAC;EACxD,GAAG,QAAQ,4CAA4C,EACrD,iBAAiB,EAAC,KAAI,EACxB,CAAC;EACD,OAAO,iBAAiB,IAAI,aAAa,MAAM;CACjD;AACF;;;;;;;;;;;;;;;;;;;;;AAsBA,SAAS,4BACP,IACA,aACA,QACA,QACiB;CACjB,IAAI,OAAO,eAAe,yBACxB,OAAO;CAET,MAAM,OAAO,kBAAkB,WAAW;CAC1C,GAAG,OAAO,iEAAiE,EACzE,iBAAiB;EAAC;EAAM,YAAY,OAAO;CAAU,EACvD,CAAC;CACD,OAAO,GAAG,MAAM;CAChB,MAAM,UAAU,iBAAiB,IAAI,aAAa,MAAM;CACxD,IAAI,QAAQ,eAAe,yBACzB,GAAG,QACD,iGAEA,EAAC,iBAAiB;EAAC;EAAM,YAAY,QAAQ;CAAU,EAAC,CAC1D;CAEF,OAAO;AACT;;AAGA,SAAS,iBACP,IACA,aACA,QACA,UACiB;CACjB,IAAI,UACF,GAAG,OACD,mFACA,EACE,wBAAwB;EACtB,QAAQ,SAAS;EACjB,MAAM,SAAS;EACf,gBAAgB,SAAS;CAC3B,EACF,CACF;CAEF,kBAAkB,WAAW;CAU7B,OAAO;EAAC,GAPO,iBACb,IACA,aACA,QACA,OACA,UAAU,MAED;EAAQ,cAAc;CAAI;AACvC;;;;;;AAOA,SAAgB,6BACd,IACA,aACA,QACA,UAC6D;CAC7D,MAAM,EAAC,IAAI,WAAU,iBACnB,IACA,mBACM,QACN,QACF;CACA,OAAO;EAAC;EAAI;EAAQ,cAAc;CAAI;AACxC;AAUA,SAAS,iBACP,IACA,aACA,QACA,qBACA,cACiB;CACjB,MAAM,KAAK,gBAAgB,IAAI,aAAa,EAAC,UAAU,MAAK,CAAC;CAC7D,IAAI;EACF,sBAAsB,EAAE;EACxB,MAAM,aAAa,eAAe,EAAE;EACpC,OAAO;GACL;GACA,QAAQ,mBAAmB,IAAI,IAAI,OAAO,EAAE,GAAG;IAC7C,+BAA+B;IAC/B;GACF,CAAC;GACD,cAAc;GACd;EACF;CACF,SAAS,GAAG;EAGV,GAAG,MAAM;EACT,MAAM;CACR;AACF;;;;;;;AAeA,IAAa,8BAA8B;AAC3C,IAAa,+BAA+B;AAE5C,IAAa,2BAAb,cAA8C,MAAM;CAClD,OAAyB;CACzB;CACA;CACA;CAEA,YACE,QACA,MACA,gBACA;EACA,MAAM,gCAAgC,OAAO,SAAS;EACtD,KAAK,SAAS;EACd,KAAK,OAAO;EACZ,KAAK,iBAAiB;CACxB;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8DA,SAAgB,mBACd,IACA,IACA,QACA,OAAyB,CAAC,GACT;CACjB,GAAG,KAAK,iBAAiB;CACzB,IAAI;EACF,IAAI,KAAK,+BAA+B;GACtC,MAAM,WAAW,gBAAgB,IAAI,MAAM;GAC3C,IAAI,UACF,MAAM;EAEV;EACA,MAAM,SAAS,KAAK,eAChB,OAAO,IAAI,IAAI,QAAQ,KAAK,YAAY,IACxC,UAAU,IAAI,IAAI,MAAM;EAC5B,GAAG,KAAK,QAAQ;EAChB,OAAO;CACT,SAAS,GAAG;EACV,IAAI,GAAG,eACL,GAAG,KAAK,UAAU;EAEpB,MAAM;CACR;AACF;;;;;;AAOA,SAAS,gBACP,IACA,QACsC;CACtC,MAAM,OAAO,oBAAoB,IAAI,OAAO,QAAQ;CACpD,IAAI,SAAS,KAAA,GACX,OAAO,IAAI,yBAAyB,IAAI;CAG1C,MAAM,OAAO,kBAAkB,EAAE;CACjC,MAAM,EAAC,oBAAmB;CAC1B,IAAI,SAAS,QAAQ,OAAO,iBAC1B,OAAO,IAAI,yBAAyB,KAAK;CAE3C,IAAI,SAAS,iBACX;CAUF,IAPiB,GACd,QAAgB;;;;KAIhB,CAAC,CACD,IAA6B,eAC5B,MAAa,KAAA,GACf,OAAO,IAAI,yBAAyB,KAAK;CAG3C,MAAM,EAAC,MAAM,mBAAkB,GAC5B,QAAgB;;;;0CAIqB,wBAAwB;;;;KAI7D,CAAC,CACD,IAA4C,eAAe;CAC9D,OAAO,OAAA,OACL,iBAAA,WACE,IAAI,yBAAyB,sBAAsB,MAAM,cAAc,IACvE,KAAA;AACN;AAEA,SAAS,UACP,IACA,IACA,QACiB;CACjB,MAAM,OAAO,oBAAoB,IAAI,OAAO,QAAQ;CACpD,IAAI,SAAS,KAAA,GACX,OAAO,OAAO,IAAI,IAAI,QAAQ,IAAI;CAGpC,MAAM,EAAC,oBAAmB;CAC1B,IAAI,OAAO,kBAAkB,EAAE;CAC/B,IAAI,OAAO;CACX,IAAI,SAAS,QAAQ,OAAO,iBAAiB;EAC3C,OAAO,GACJ,QAAgB;uBACA,wBAAwB;OACxC,CAAC,CACD,IAAI,eAAe,CAAC,CAAC;EACxB,OAAO,kBAAkB,EAAE;CAC7B;CAEA,IAAI,SAAS,iBACX,OAAO,OAAO,IAAI,IAAI,QAAQ,KAAK;CAErC,IAAI,OAAO,GAAG;EAMZ,eAAe,IAAI,OAAO,OAAO;EACjC,GAAG,OAAO,6DAA6D,EACrE,0BAA0B;GAAC;GAAM;GAAM,SAAS,aAAa,MAAM;EAAC,EACtE,CAAC;EACD,OAAO;GAAC,QAAQ;GAAa;GAAM;GAAM,cAAc;EAAI;CAC7D;CACA,GAAG,QAAQ,gDAAgD,EACzD,0BAA0B,EAAC,KAAI,EACjC,CAAC;CACD,OAAO;EAAC,QAAQ;EAAQ;EAAM,cAAc;CAAK;AACnD;;;;;;;;AASA,SAAgB,oBACd,IACA,UAC0B;CAC1B,IACE,CAAC,YAAY,IAAA,uBAA2B,KACxC,CAAC,YAAY,IAAA,qBAAyB,GAEtC,OAAO;CAKT,MAAM,UAAU,GACb,QAAgB;qCACgB,sBAAsB;KACtD,CAAC,CACD,IAAyC;CAC5C,IAAI,YAAY,KAAA,GACd,OAAO;CAET,IAAI,QAAQ,kBAAA,GACV,OAAO;CAMT,IACE,CAAC,YAAY,IAAA,8BAAmC,KAChD,CAAC,YAAY,IAAA,4BAAgC,GAE7C,OAAO;CAET,MAAM,EAAC,OAAO,YAAY,cAAa,kBAAkB,EAAE;CAC3D,IACE,UAAU,SAAS,SACnB,eAAe,SAAS,cACxB,cAAc,SAAS,WAEvB,OAAO;AAGX;AAEA,SAAS,OACP,IACA,IACA,QACA,QACiB;CACjB,MAAM,EAAC,UAAU,iBAAiB,UAAS;CAI3C,GAAG,KAAa;4BACU,wBAAwB;4BACxB,sBAAsB;MAC5C,8BAA8B;MAC9B,gCAAgC;MAChC,8BAA8B;MAC9B,gCAAgC;GACnC;CACD,GAAG,QAAgB;mBACF,sBAAsB;;;;;GAKtC,CAAC,CAAC,IAAI;EACL,OAAO,SAAS;EAChB,YAAY,SAAS;EACrB,WAAW,SAAS;EACpB,eAAA;EACA,YAAY;EACZ,eAAe;CACjB,CAAC;CACD,oBAAoB,IAAI,MAAM;CAK9B,eAAe,IAAI,OAAO,OAAO;CAEjC,GAAG,OAAO,kCAAkC,EAC1C,0BAA0B;EACxB;EACA,MAAM;EACN,GAAG;EACH,eAAA;EACA,YAAY;EACZ,SAAS,aAAa,MAAM;CAC9B,EACF,CAAC;CACD,OAAO;EACL,QAAQ;EACR,MAAM;EACN;EACA,cAAc;CAChB;AACF;;;;;;AAOA,SAAgB,oBACd,IACA,QACM;CACN,MAAM,cAAc;CACpB,MAAM,eAAe;CACrB,GAAG,QAAQ,0BAA0B,CAAC,CAAC,IAAI;EACzC,WAAW,OAAO;EAClB,YAAY,gCACV,OAAO,iBACP,SACA,WACF;EACA;EACA,aAAa,gCACX,OAAO,iBACP,UACA,cACA,OAAO,iBACP,IACF;EACA;EACA,aAAa,OAAO;CACtB,CAAC;AACH;;AAGA,SAAgB,kBAAkB,IAA6B;CAC7D,MAAM,OAAO,GACV,QAAgB;;;gBAGL,sBAAsB;KACjC,CAAC,CACD,IAA+B;CAClC,IAAI,SAAS,KAAA,GACX,MAAM,IAAI,MAAM,uCAAuC;CAEzD,OAAO;AACT;;;;;AAMA,SAAgB,kBAAkB,IAA6B;CAC7D,MAAM,EAAC,SAAQ,GACZ,QAAgB;gDAC2B,wBAAwB;KACnE,CAAC,CACD,IAA2B;CAC9B,OAAO;AACT;;;;;;AAOA,SAAgB,oBAAoB,IAA+B;CACjE,OAAO,GACJ,QAAgB;;yCAEoB,wBAAwB;;yCAExB,wBAAwB;;KAE5D,CAAC,CACD,IAAqB;AAC1B;;;;;AAWA,SAAS,aAAa,EAAC,WAA2B;CAChD,OAAO;EACL,eAAe,QAAQ,cAAc;EACrC,aAAa,QAAQ,YAAY;CACnC;AACF;AAEA,SAAS,YAAY,IAAc,OAAwB;CACzD,OACE,GACG,QAAgB;;OAEhB,CAAC,CACD,IAA6B,KAAK,MAAM,KAAA;AAE/C"}
+{"version":3,"file":"change-log-db.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/change-log-db.ts"],"sourcesContent":["/**\n * The change log lives in its own SQLite database, `${replicaFile}-change-log`,\n * rather than in the replica file. It is a catchup cache: everything it holds is\n * either already durable in the replica's litestream backup or re-derivable from\n * the upstream replication slot, so it is deliberately excluded from the backup\n * and is never a source of truth.\n *\n * The change-streamer writes it, from the stream loop that fans changes out to\n * subscribers, and commits it before forwarding each transaction's `commit`\n * message. Its head is therefore always at or above the watermark the stream\n * would resume from, which is what {@link reconcileChangeLog} anchors on.\n *\n * Beside the buffer it holds the log's *cookie jar* — the backfill progress\n * state folded up to the same head. That is `change-log-cookies.ts`; this file\n * owns its tables' lifecycle, because a cookie set is only meaningful paired\n * with the watermark it was folded to, and so must be created, dropped, and —\n * whenever reconciliation moves the head — replaced with the anchor's, all with\n * the buffer rather than beside it.\n *\n * Because it is disposable, it has no migration framework. Any inconsistency —\n * a schema change, a leftover file from a different replica, a gap left by a\n * crash or by running with the writer disabled, a corrupt file — is resolved by\n * {@link reconcileChangeLog} and {@link openChangeLogDBForWriting}, which\n * truncate a bounded suffix or replace and reseed the log at the resume\n * watermark. The worst outcome is a `too-old` for a lagging subscriber; a\n * silently delivered gap is not reachable.\n *\n * Disk sizing: a task that writes the log needs room for the replica *plus* the\n * retained change stream — the log's main file, its wal2 sidecars, and pages a\n * purge has freed but not yet returned to the OS. The change-streamer schedules\n * purges against the same floor as the PG change log, so retention is bounded\n * by the confirmed backup watermark ANDed with the minimum retention window —\n * a function of the backup interval, not a constant: 15–30+ minutes behind an\n * RMv1 sync interval, ~60 seconds once litestream v5 backs up every 15–30 s.\n * A held snapshot reservation pins retention unboundedly for its duration, so\n * peak retained bytes is set by the slowest concurrent restore rather than by\n * the retention setting — size disk and alerts on that. The log is excluded\n * from the litestream backup, so this is local disk only and never S3.\n */\n\nimport {existsSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {\n  CHANGE_LOG_BACKFILLING_TABLE,\n  CHANGE_LOG_TABLE_METADATA_TABLE,\n  CREATE_CHANGE_LOG_COOKIE_SCHEMA,\n  DROP_CHANGE_LOG_COOKIE_TABLES,\n  replaceCookies,\n  type CookieSet,\n} from './change-log-cookies.ts';\n\n/**\n * Bumped whenever the change-log database's schema changes. Since the file is a\n * cache, a mismatch costs one reseed rather than a migration.\n *\n * v2 re-homed the writer to the change-streamer: the log anchors on the\n * watermark its stream connection resumes from rather than on the replica's\n * state version, and its meta row carries the replica's identity triple plus\n * the seed point.\n *\n * v3 added the cookie tables (see `change-log-cookies.ts`), i.e. the log's\n * second job. The reseed it costs on upgrade buys one retention window of\n * catchup reach on a log nothing reads yet.\n *\n * v4 stores each row's tag and estimated retained bytes alongside its JSON.\n * Readers can now filter without parsing arbitrary row payloads, and\n * reconciliation can bound truncation by bytes as well as rows.\n *\n * `auto_vacuum = INCREMENTAL` arrived within v2, deliberately without a bump:\n * it is a file-header property rather than a schema change, and the reseed a\n * version mismatch triggers cannot enable it (see\n * {@link applyChangeLogPragmas}), so {@link openChangeLogDBForWriting} guards\n * on the pragma itself instead of on this number.\n */\nexport const CHANGE_LOG_DB_SCHEMA_VERSION = 4;\n\n/** https://www.sqlite.org/pragma.html#pragma_auto_vacuum */\nconst AUTO_VACUUM_INCREMENTAL = 2;\n\nexport const CHANGE_LOG_STREAM_TABLE = '_zero.changeLogStream';\nexport const CHANGE_LOG_STREAM_WRITE_TIME_INDEX =\n  '_zero.changeLogStream_writeTimeMs';\n\nconst INTEGER_BYTES = 8;\n\n/**\n * Estimates the retained payload size of a stream row. This deliberately does\n * not claim to measure SQLite b-tree/page overhead; it is stable across the\n * write path, reconciliation, and the startup scan used by observability.\n */\nexport function estimateChangeLogStreamRowBytes(\n  watermark: string,\n  tag: string,\n  change: string,\n  precommit?: string,\n  hasWriteTime = false,\n): number {\n  return (\n    Buffer.byteLength(watermark) +\n    INTEGER_BYTES +\n    Buffer.byteLength(tag) +\n    Buffer.byteLength(change) +\n    (precommit === undefined ? 0 : Buffer.byteLength(precommit)) +\n    (hasWriteTime ? INTEGER_BYTES : 0)\n  );\n}\n\n// The index is partial because only commit rows carry a \"writeTimeMs\", so it\n// holds one entry per transaction rather than one per change. It is what the\n// purger scans to find transactions older than the retention window.\n//\n// Pre-release builds briefly carried this table in the replica; see the test\n// fixture `CREATE_V14_CHANGE_LOG_STREAM` in\n// `change-source/common/replica-schema.ts`. Its old shape must not be kept in\n// sync with this one.\nexport const CREATE_CHANGE_LOG_STREAM_SCHEMA = /*sql*/ `\n  CREATE TABLE \"${CHANGE_LOG_STREAM_TABLE}\" (\n    \"watermark\"      TEXT NOT NULL,\n    \"pos\"            INTEGER NOT NULL,\n    \"tag\"            TEXT NOT NULL,\n    \"estimatedBytes\"  INTEGER NOT NULL,\n    \"change\"          TEXT NOT NULL,\n    \"precommit\"       TEXT,\n    \"writeTimeMs\"     INTEGER,\n    PRIMARY KEY (\"watermark\", \"pos\")\n  );\n\n  CREATE INDEX \"${CHANGE_LOG_STREAM_WRITE_TIME_INDEX}\"\n    ON \"${CHANGE_LOG_STREAM_TABLE}\" (\"writeTimeMs\", \"watermark\")\n    WHERE \"writeTimeMs\" IS NOT NULL;\n`;\n\n/**\n * Inserts the synthetic begin/commit pair for the seed watermark.\n *\n * Both rows are inserted by one statement so a caller outside a larger\n * transaction cannot leave a partial seed.\n */\nconst SEED_CHANGE_LOG_STREAM_SQL = /*sql*/ `\n  INSERT INTO \"${CHANGE_LOG_STREAM_TABLE}\"\n    (\"watermark\", \"pos\", \"tag\", \"estimatedBytes\", \"change\", \"precommit\",\n     \"writeTimeMs\")\n  VALUES\n    (@watermark, 0, 'begin', @beginBytes, @beginChange, NULL, NULL),\n    (@watermark, 1, 'commit', @commitBytes, @commitChange, @watermark,\n     @writeTimeMs)\n  ON CONFLICT (\"watermark\", \"pos\") DO NOTHING\n`;\n\nexport const CHANGE_LOG_META_TABLE = '_zero.changeLogMeta';\n\n// The identity triple (see ChangeLogIdentity) detects a file left behind by a\n// different replica -- most concretely, a leftover log on a reused volume.\n//\n// \"seededAtMs\"    : when the log was seeded, i.e. how far its retention window\n//                   can possibly reach back. Slice 11 uses it as the warm-up\n//                   signal for canary reads.\n// \"seedWatermark\" : where the log was seeded. `min(\"watermark\")` cannot answer\n//                   this once purge has advanced the minimum, and the\n//                   difference is what distinguishes a log that purged past the\n//                   retention floor (an invariant violation) from one that\n//                   never held that history at all (routine at every fork).\n// \"lock\"          : enforces single-row semantics, as in\n//                   \"_zero.replicationConfig\".\nconst CREATE_CHANGE_LOG_META_SCHEMA = /*sql*/ `\n  CREATE TABLE \"${CHANGE_LOG_META_TABLE}\" (\n    \"epoch\"         TEXT,\n    \"generation\"    TEXT NOT NULL,\n    \"replicaID\"     TEXT,\n    \"schemaVersion\" INTEGER NOT NULL,\n    \"seededAtMs\"    INTEGER NOT NULL,\n    \"seedWatermark\" TEXT NOT NULL,\n    \"lock\"          INTEGER PRIMARY KEY DEFAULT 1 CHECK (\"lock\" = 1)\n  );\n`;\n\n/**\n * `${replicaFile}-change-log`, matching the `-serving-copy` convention.\n *\n * The name outlives the writer's move into the change-streamer: the\n * change-streamer derives the path from the replica file it owns, so renaming\n * would save no plumbing and would orphan the old file on every existing\n * volume, where nothing would delete it.\n *\n * Its `-wal` / `-wal2` / `-shm` sidecars are suffixes of *this* name, so they\n * never collide with the replica's own sidecars.\n */\nexport function changeLogFileName(replicaFile: string): string {\n  return `${replicaFile}-change-log`;\n}\n\n/**\n * Removes the change-log database and its `-wal` / `-wal2` / `-shm` sidecars.\n * Safe to call when the file does not exist.\n */\nexport function deleteChangeLogDB(replicaFile: string): void {\n  deleteLiteDB(changeLogFileName(replicaFile));\n}\n\n/**\n * Which replica the log's contents belong to.\n *\n * `generation` is RMv1's `replicaVersion`: the version at which initial sync\n * copied every row. It is shared by every sibling of a forked replica, which is\n * why it cannot be the whole of the identity under RMv2 — `epoch` and\n * `replicaID` are what distinguish two siblings' logs. Both are `null` until\n * RMv2 supplies them, and a `null` matches only another `null`, so a log\n * written before they exist is still recognized as this replica's.\n */\nexport type ChangeLogIdentity = {\n  readonly epoch: string | null;\n  readonly generation: string;\n  readonly replicaID: string | null;\n};\n\n/**\n * Where the stream connection resumes, and everything the log's state at that\n * position consists of.\n *\n * The two fields travel together because a cookie set is only meaningful paired\n * with the watermark it was folded up to (invariant 15). They are read from one\n * store, in one snapshot: `Storer.getStartStreamInitializationParameters()`\n * reads both from Postgres in a single REPEATABLE READ transaction. Pairing one\n * store's cookies with another's watermark would silently drop every backfill\n * that completed in the interval, and those rows are unrecoverable — backfill\n * transactions are synthesized by the change source rather than replayed from\n * the slot.\n */\nexport type ChangeLogResumePoint = {\n  /**\n   * The watermark this stream connection resumes from. The log's own head is\n   * measured against it, and a reseed seeds a synthetic transaction at it.\n   *\n   * Deliberately *not* \"the replica's state version\": after the writer moved to\n   * the change-streamer, the canonical replicator is a subscriber downstream of\n   * the write loop, so its state version is a consequence of this log rather\n   * than a constraint on it.\n   */\n  readonly resumeWatermark: string;\n  /**\n   * The cookie set that belongs to {@link resumeWatermark}. Any reconciliation\n   * that moves the head invalidates the set the log was holding — the cookies\n   * are unversioned point-in-time state, and truncating the stream above `W`\n   * does not roll them back to `W` — so the same transaction installs this one\n   * in its place (invariant 17).\n   *\n   * Postgres supplies it while it remains the store the stream resumes from. At\n   * the C5 flip the log's own head *is* the resume point, phantoms cannot\n   * exist, and both this field and the truncate-above path become dead code.\n   */\n  readonly cookies: CookieSet;\n};\n\n/**\n * What the log is reconciled against. Reconciliation runs once per stream\n * connection, since every reconnect can re-read a different resume point.\n */\nexport type ChangeLogAnchor = ChangeLogResumePoint & {\n  readonly identity: ChangeLogIdentity;\n  /**\n   * The change-streamer's clock, at reconciliation time. Supplied rather than\n   * read inline so that tests control it. Used for `seededAtMs` and for the\n   * seed transaction's `writeTimeMs`, which is what the purger's time floor\n   * measures.\n   */\n  readonly nowMs: number;\n};\n\n/** The meta row, i.e. everything the log records about itself. */\nexport type ChangeLogMeta = ChangeLogIdentity & {\n  readonly schemaVersion: number;\n  readonly seededAtMs: number;\n  readonly seedWatermark: string;\n};\n\n/**\n * Opens the change-log database beside `replicaFile`.\n *\n * A writable handle creates the file if it is absent; a `readonly` handle\n * throws, which is the normal state for a change-streamer whose writer is\n * disabled. Pragmas are the caller's responsibility, since only the writer sets\n * the persistent ones.\n */\nexport function openChangeLogDB(\n  lc: LogContext,\n  replicaFile: string,\n  opts: {readonly: boolean},\n): Database {\n  return new Database(\n    lc.withContext('component', 'change-log-db'),\n    changeLogFileName(replicaFile),\n    {readonly: opts.readonly},\n  );\n}\n\n/**\n * Configures the change-log database, which the change-streamer's writer owns\n * exclusively. These are the settings the `separate-files` mode of\n * `sqlite-change-log-ceiling.bench.ts` measured.\n *\n * `wal2` because the log is a continuous append that catchup reads\n * continuously, which is the case wal2 exists for: it avoids checkpoint\n * starvation without needing an autocheckpoint. `wal_autocheckpoint` therefore\n * keeps its default, unlike the backup replica's 0, which hands checkpointing\n * to litestream — nothing else owns this file.\n *\n * `synchronous = NORMAL` because the commit sits on the forward path, once per\n * upstream transaction, and NORMAL keeps that from costing an fsync. It is not\n * a durability compromise here: in WAL mode NORMAL still writes every commit\n * through to the OS, so it survives a process crash, an OOM kill, or a SIGKILL.\n * It is lost only to kernel panic or power loss, and those destroy the node —\n * the task restarts elsewhere, restores the replica from S3, and has no\n * change-log file at all. Whatever a power loss does take is detected by\n * {@link reconcileChangeLog} and reseeded.\n *\n * `auto_vacuum = INCREMENTAL` so that purge can return freed pages to the OS\n * rather than leaving the file at its high-water mark. **It must be the first\n * statement here.** SQLite honours a `none` -> `incremental` transition only\n * while the file is still empty, and `journal_mode = wal2` materializes page 1;\n * appending the pragma instead of prepending it silently leaves the file at\n * mode 0 forever, since neither `DROP TABLE` nor a reseed makes a file empty\n * again. {@link openChangeLogDBForWriting} verifies the result.\n */\nexport function applyChangeLogPragmas(db: Database): void {\n  db.pragma('auto_vacuum = INCREMENTAL');\n  db.pragma('busy_timeout = 30000');\n  db.pragma('analysis_limit = 1000');\n  db.pragma('journal_mode = wal2');\n  db.pragma('synchronous = NORMAL');\n}\n\nfunction readAutoVacuum(db: Database): number {\n  const [{auto_vacuum: mode}] = db.pragma<{auto_vacuum: number}>('auto_vacuum');\n  return mode;\n}\n\n/**\n * Opens the change-log database for the writer, applies its pragmas, and\n * reconciles it against `anchor`, returning the handle and what reconciliation\n * had to do.\n *\n * Corruption is answered with a rebuild rather than an escalation, which is a\n * deliberate divergence from how the replica's own corruption is handled. A\n * corrupt replica is a lost source of truth, so it ends in a restore or an\n * auto-reset; the change log holds nothing that is not either already in the\n * replica's litestream backup or re-derivable from the upstream slot, so the\n * entire response is to delete the file and seed a new one at the resume\n * watermark. The cost is one retention window of catchup — reconnecting\n * subscribers below the head get `too-old` — which is what every other\n * {@link ReseedReason} costs.\n *\n * Only corruption is rebuilt through. Anything else (a bad path, a permissions\n * error, a full disk) is a problem with the environment rather than with the\n * file's contents, and deleting a database in response would destroy state\n * without fixing it. Those propagate to the caller, which disables the writer\n * and keeps replicating.\n *\n * A file that predates `auto_vacuum = INCREMENTAL` takes the same rebuild, for\n * the same reason and at the same cost: the pragma only takes on an empty file,\n * so a reseed cannot enable it. Unlike corruption, a *second* failure there is\n * tolerated rather than thrown — see {@link ensureIncrementalAutoVacuum}.\n */\n/**\n * Returns an anchor after the database opens.\n *\n * The resolver can inspect the existing file before it selects the anchor. A\n * valid log can use its own head. A new or invalid log uses the replica seed.\n * The rebuild path calls the resolver again for the new file.\n */\nexport type AnchorResolver = (db: Database) => ChangeLogAnchor;\n\nexport function openChangeLogDBForWriting(\n  lc: LogContext,\n  replicaFile: string,\n  anchor: AnchorResolver,\n): {db: Database; result: ReconcileResult; replacedFile: boolean} {\n  const existed = existsSync(changeLogFileName(replicaFile));\n  const opened = openOrRebuildCorrupt(lc, replicaFile, anchor, existed);\n  const {db, result, replacedFile} = ensureIncrementalAutoVacuum(\n    lc,\n    replicaFile,\n    anchor,\n    opened,\n  );\n  return {db, result, replacedFile};\n}\n\nfunction openOrRebuildCorrupt(\n  lc: LogContext,\n  replicaFile: string,\n  anchor: AnchorResolver,\n  boundReconciliation: boolean,\n): OpenedChangeLog {\n  try {\n    return openAndReconcile(lc, replicaFile, anchor, boundReconciliation);\n  } catch (e) {\n    if (e instanceof ChangeLogRebuildRequired) {\n      return rebuildChangeLog(lc, replicaFile, anchor, e);\n    }\n    if (!isSQLiteCorruption(e)) {\n      throw e;\n    }\n    const file = changeLogFileName(replicaFile);\n    logSQLiteCorruptionDiagnostics(lc, 'change-log', file, e);\n    lc.error?.('rebuilding the corrupt SQLite change log', {\n      sqliteChangeLog: {file},\n    });\n    return rebuildChangeLog(lc, replicaFile, anchor);\n  }\n}\n\n/**\n * Rebuilds a log whose `auto_vacuum` mode is not `INCREMENTAL`, which is every\n * log created before the pragma shipped — including every v2 file the writer\n * wrote in the meantime, which is why the guard is on the pragma rather than\n * on the schema version. Deleting the file is the only way to enable it:\n * `DROP TABLE` leaves the header's mode intact, and the in-place alternative —\n * a full `VACUUM` — is an unbounded blocking statement on a file the writer\n * needs.\n *\n * A mode that is still wrong after the rebuild is logged and tolerated, which\n * is the one place this diverges from the corruption path. The two failures are\n * not alike: a file that stays corrupt after a rebuild is an environment\n * problem, but a pragma that stays wrong on a brand-new file can only mean\n * {@link applyChangeLogPragmas}' statement order is wrong — a code bug, present\n * on every task, on every start. Throwing there turns a lost optimization into\n * a crash loop, and rebuilding again turns it into a reseed per start, which is\n * worse than the freelist growth it was trying to avoid. `Database.compact()`\n * already warns and no-ops when the mode is not `INCREMENTAL`, so degrading is\n * the accepted fallback.\n */\nfunction ensureIncrementalAutoVacuum(\n  lc: LogContext,\n  replicaFile: string,\n  anchor: AnchorResolver,\n  opened: OpenedChangeLog,\n): OpenedChangeLog {\n  if (opened.autoVacuum === AUTO_VACUUM_INCREMENTAL) {\n    return opened;\n  }\n  const file = changeLogFileName(replicaFile);\n  lc.info?.('rebuilding the SQLite change log to enable incremental vacuum', {\n    sqliteChangeLog: {file, autoVacuum: opened.autoVacuum},\n  });\n  opened.db.close();\n  const rebuilt = rebuildChangeLog(lc, replicaFile, anchor);\n  if (rebuilt.autoVacuum !== AUTO_VACUUM_INCREMENTAL) {\n    lc.error?.(\n      'SQLite change log does not support incremental vacuum; ' +\n        'it will plateau at its high-water mark',\n      {sqliteChangeLog: {file, autoVacuum: rebuilt.autoVacuum}},\n    );\n  }\n  return rebuilt;\n}\n\n/** The shared \"delete the file and open a new one\" body. */\nfunction rebuildChangeLog(\n  lc: LogContext,\n  replicaFile: string,\n  anchor: AnchorResolver,\n  required?: ChangeLogRebuildRequired | undefined,\n): OpenedChangeLog {\n  if (required) {\n    lc.warn?.(\n      'rebuilding the SQLite change log instead of running an unbounded reconciliation',\n      {\n        sqliteChangeLogRebuild: {\n          reason: required.reason,\n          rows: required.rows,\n          estimatedBytes: required.estimatedBytes,\n        },\n      },\n    );\n  }\n  deleteChangeLogDB(replicaFile);\n  // The file is absent, so the reseed only drops empty/nonexistent tables. A\n  // second failure is the environment's, not the old file's.\n  const opened = openAndReconcile(\n    lc,\n    replicaFile,\n    anchor,\n    false,\n    required?.reason,\n  );\n  return {...opened, replacedFile: true};\n}\n\n/**\n * Replaces an already-closed change-log file after bounded reconciliation\n * declined its work. The caller closes cached readers before calling this so\n * none can continue serving the unlinked inode.\n */\nexport function rebuildChangeLogDBForWriting(\n  lc: LogContext,\n  replicaFile: string,\n  anchor: ChangeLogAnchor,\n  required: ChangeLogRebuildRequired,\n): {db: Database; result: ReconcileResult; replacedFile: true} {\n  const {db, result} = rebuildChangeLog(\n    lc,\n    replicaFile,\n    () => anchor,\n    required,\n  );\n  return {db, result, replacedFile: true};\n}\n\ntype OpenedChangeLog = {\n  db: Database;\n  result: ReconcileResult;\n  replacedFile: boolean;\n  /** The file's persistent `auto_vacuum` mode, read back after the pragmas. */\n  autoVacuum: number;\n};\n\nfunction openAndReconcile(\n  lc: LogContext,\n  replicaFile: string,\n  anchor: AnchorResolver,\n  boundReconciliation: boolean,\n  reseedReason?: ReseedReason | undefined,\n): OpenedChangeLog {\n  const db = openChangeLogDB(lc, replicaFile, {readonly: false});\n  try {\n    applyChangeLogPragmas(db);\n    const autoVacuum = readAutoVacuum(db);\n    return {\n      db,\n      result: reconcileChangeLog(lc, db, anchor(db), {\n        rebuildInsteadOfUnboundedWork: boundReconciliation,\n        reseedReason,\n      }),\n      replacedFile: false,\n      autoVacuum,\n    };\n  } catch (e) {\n    // The caller never sees this handle, so it closes here or leaks — and it\n    // must be closed before the corruption path deletes the file.\n    db.close();\n    throw e;\n  }\n}\n\nexport type ReseedReason =\n  | 'created' // file or table absent\n  | 'schema-mismatch'\n  | 'identity-mismatch'\n  | 'gap' // head does not land on the resume watermark after truncation\n  | 'oversized-truncate'; // preserving the prefix would require too much work\n\n/**\n * Reconciliation runs synchronously on the change-streamer's event loop. A\n * normal phantom is one transaction, so work above either limit is answered by\n * unlinking and reseeding this disposable cache rather than deleting an\n * arbitrarily large suffix in place.\n */\nexport const MAX_RECONCILE_TRUNCATE_ROWS = 10_000;\nexport const MAX_RECONCILE_TRUNCATE_BYTES = 16 * 1024 * 1024;\n\nexport class ChangeLogRebuildRequired extends Error {\n  override readonly name = 'ChangeLogRebuildRequired';\n  readonly reason: ReseedReason;\n  readonly rows?: number | undefined;\n  readonly estimatedBytes?: number | undefined;\n\n  constructor(\n    reason: ReseedReason,\n    rows?: number | undefined,\n    estimatedBytes?: number | undefined,\n  ) {\n    super(`SQLite change log requires a ${reason} rebuild`);\n    this.reason = reason;\n    this.rows = rows;\n    this.estimatedBytes = estimatedBytes;\n  }\n}\n\ntype ReconcileOptions = {\n  /** Throw before an unbounded delete/drop so the file owner can unlink it. */\n  readonly rebuildInsteadOfUnboundedWork?: boolean | undefined;\n  /** Preserve the reason that caused an old file to be unlinked. */\n  readonly reseedReason?: ReseedReason | undefined;\n};\n\n/**\n * `cookiesStale` records that the cookie set the log held did not survive\n * reconciliation. The cookies are unversioned point-in-time state, so any\n * reconciliation that moves the head invalidates them, and the same transaction\n * replaces them with the anchor's — the set that belongs to the resume\n * watermark (see `change-log-cookies.ts`). It is `action !== 'none'` today; it\n * is a field rather than a derivation because the replacement reads as intent\n * at the call site, and because it is what the `cookie_reseed` counter is keyed\n * on.\n */\nexport type ReconcileResult =\n  | {action: 'none'; head: string; cookiesStale: false}\n  | {action: 'truncated'; head: string; rows: number; cookiesStale: true}\n  | {\n      action: 'reseeded';\n      head: string;\n      reason: ReseedReason;\n      cookiesStale: true;\n    };\n\n/**\n * Brings the change log into agreement with the watermark its stream connection\n * resumes from, in one transaction on the change-log database:\n *\n * ```\n * head === resumeWatermark  → keep appending\n * head  >  resumeWatermark  → truncate above resumeWatermark\n * otherwise                 → wipe, seed at resumeWatermark, record the reason\n * ```\n *\n * The log commits before anything that can advance the resume watermark, so a\n * *phantom* — a transaction the log holds whose watermark the resumed stream\n * re-delivers — is the normal state, and truncate-above is the whole of\n * reconciliation rather than a special case. A phantom's rows all carry the\n * transaction's commit watermark, which is strictly greater than the resume\n * watermark, so deleting on that predicate removes phantoms whole and retains\n * every transaction at or below the resume point.\n *\n * It must run per stream connection, not once per process: the stream loop\n * re-reads its resume watermark on every reconnect, and the writer inserts with\n * a plain `INSERT` against `PRIMARY KEY (\"watermark\",\"pos\")`, so an\n * un-truncated overlap is a constraint violation on the first re-delivered\n * transaction.\n *\n * Anything truncation cannot resolve — the file was deleted, the writer was\n * disabled for a while, the replica was restored, power was lost with\n * `synchronous=NORMAL` — surfaces as a head that does not land on the resume\n * watermark, and is resolved by wiping and reseeding there.\n *\n * Whichever of the two moves the head also replaces the cookie set with the one\n * the anchor carries, in the same transaction: the head and the cookies are one\n * position, and nothing may observe them at two.\n */\nexport function reconcileChangeLog(\n  lc: LogContext,\n  db: Database,\n  anchor: ChangeLogAnchor,\n  opts: ReconcileOptions = {},\n): ReconcileResult {\n  db.exec('BEGIN IMMEDIATE');\n  try {\n    if (opts.rebuildInsteadOfUnboundedWork) {\n      const required = rebuildRequired(db, anchor);\n      if (required) {\n        throw required;\n      }\n    }\n    const result = opts.reseedReason\n      ? reseed(lc, db, anchor, opts.reseedReason)\n      : reconcile(lc, db, anchor);\n    db.exec('COMMIT');\n    return result;\n  } catch (e) {\n    if (db.inTransaction) {\n      db.exec('ROLLBACK');\n    }\n    throw e;\n  }\n}\n\n/**\n * Decides whether reconciliation can be performed within its synchronous work\n * budget. Every query is either an index seek or a scan capped at one row past\n * the delete limit. No state changes before the decision is complete.\n */\nfunction rebuildRequired(\n  db: Database,\n  anchor: ChangeLogAnchor,\n): ChangeLogRebuildRequired | undefined {\n  const wipe = changeLogWipeReason(db, anchor.identity);\n  if (wipe !== undefined) {\n    return new ChangeLogRebuildRequired(wipe);\n  }\n\n  const head = readChangeLogHead(db);\n  const {resumeWatermark} = anchor;\n  if (head === null || head < resumeWatermark) {\n    return new ChangeLogRebuildRequired('gap');\n  }\n  if (head === resumeWatermark) {\n    return undefined;\n  }\n\n  const boundary = db\n    .prepare(/*sql*/ `\n      SELECT 1 FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n        WHERE \"watermark\" = ?\n        LIMIT 1\n    `)\n    .get<{1: number} | undefined>(resumeWatermark);\n  if (boundary === undefined) {\n    return new ChangeLogRebuildRequired('gap');\n  }\n\n  const {rows, estimatedBytes} = db\n    .prepare(/*sql*/ `\n      SELECT count(*) AS \"rows\",\n             coalesce(sum(\"estimatedBytes\"), 0) AS \"estimatedBytes\"\n        FROM (\n          SELECT \"estimatedBytes\" FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n            WHERE \"watermark\" > ?\n            LIMIT ${MAX_RECONCILE_TRUNCATE_ROWS + 1}\n        )\n    `)\n    .get<{rows: number; estimatedBytes: number}>(resumeWatermark);\n  return rows > MAX_RECONCILE_TRUNCATE_ROWS ||\n    estimatedBytes > MAX_RECONCILE_TRUNCATE_BYTES\n    ? new ChangeLogRebuildRequired('oversized-truncate', rows, estimatedBytes)\n    : undefined;\n}\n\nfunction reconcile(\n  lc: LogContext,\n  db: Database,\n  anchor: ChangeLogAnchor,\n): ReconcileResult {\n  const wipe = changeLogWipeReason(db, anchor.identity);\n  if (wipe !== undefined) {\n    return reseed(lc, db, anchor, wipe);\n  }\n\n  const {resumeWatermark} = anchor;\n  let head = readChangeLogHead(db);\n  let rows = 0;\n  if (head !== null && head > resumeWatermark) {\n    rows = db\n      .prepare(/*sql*/ `\n        DELETE FROM \"${CHANGE_LOG_STREAM_TABLE}\" WHERE \"watermark\" > ?\n      `)\n      .run(resumeWatermark).changes;\n    head = readChangeLogHead(db);\n  }\n\n  if (head !== resumeWatermark) {\n    return reseed(lc, db, anchor, 'gap');\n  }\n  if (rows > 0) {\n    // The truncated transactions can have carried `create-table`,\n    // `add-column`, and `backfill-completed`, none of which the truncate rolls\n    // back, so the cookie set no longer belongs to the head. The anchor's set\n    // does, and it lands in this same transaction: invariant 17 holds at every\n    // point another connection could observe the log.\n    replaceCookies(db, anchor.cookies);\n    lc.info?.('truncated phantom transactions from the SQLite change log', {\n      sqliteChangeLogReconcile: {head, rows, cookies: cookieCounts(anchor)},\n    });\n    return {action: 'truncated', head, rows, cookiesStale: true};\n  }\n  lc.debug?.('SQLite change log is at the resume watermark', {\n    sqliteChangeLogReconcile: {head},\n  });\n  return {action: 'none', head, cookiesStale: false};\n}\n\n/**\n * Returns the reason that the log must be recreated. Returns `undefined` when\n * the writer can continue to use the log.\n *\n * This function takes only the expected identity because callers use the result\n * to select the anchor.\n */\nexport function changeLogWipeReason(\n  db: Database,\n  identity: ChangeLogIdentity,\n): ReseedReason | undefined {\n  if (\n    !tableExists(db, CHANGE_LOG_STREAM_TABLE) ||\n    !tableExists(db, CHANGE_LOG_META_TABLE)\n  ) {\n    return 'created';\n  }\n  // The schema version is read on its own because it is the one column every\n  // version of this table has had. Reading the rest of a v1 row would fail on\n  // its missing columns instead of reporting the mismatch that explains it.\n  const version = db\n    .prepare(/*sql*/ `\n      SELECT \"schemaVersion\" FROM \"${CHANGE_LOG_META_TABLE}\"\n    `)\n    .get<{schemaVersion: number} | undefined>();\n  if (version === undefined) {\n    return 'created';\n  }\n  if (version.schemaVersion !== CHANGE_LOG_DB_SCHEMA_VERSION) {\n    return 'schema-mismatch';\n  }\n  // Checked *after* the version, so that every pre-v4 file — which is every\n  // file in the fleet at the upgrade — reports the `schema-mismatch` that\n  // explains it rather than the `created` its absent cookie tables would\n  // otherwise look like.\n  if (\n    !tableExists(db, CHANGE_LOG_TABLE_METADATA_TABLE) ||\n    !tableExists(db, CHANGE_LOG_BACKFILLING_TABLE)\n  ) {\n    return 'created';\n  }\n  const {epoch, generation, replicaID} = readChangeLogMeta(db);\n  if (\n    epoch !== identity.epoch ||\n    generation !== identity.generation ||\n    replicaID !== identity.replicaID\n  ) {\n    return 'identity-mismatch';\n  }\n  return undefined;\n}\n\nfunction reseed(\n  lc: LogContext,\n  db: Database,\n  anchor: ChangeLogAnchor,\n  reason: ReseedReason,\n): ReconcileResult {\n  const {identity, resumeWatermark, nowMs} = anchor;\n  // Dropping the table drops its partial index with it. The cookie tables go\n  // with the buffer: a cookie set without the transactions it was folded from\n  // is not a smaller cookie set, it is a wrong one.\n  db.exec(/*sql*/ `\n    DROP TABLE IF EXISTS \"${CHANGE_LOG_STREAM_TABLE}\";\n    DROP TABLE IF EXISTS \"${CHANGE_LOG_META_TABLE}\";\n    ${DROP_CHANGE_LOG_COOKIE_TABLES}\n    ${CREATE_CHANGE_LOG_STREAM_SCHEMA}\n    ${CREATE_CHANGE_LOG_META_SCHEMA}\n    ${CREATE_CHANGE_LOG_COOKIE_SCHEMA}\n  `);\n  db.prepare(/*sql*/ `\n    INSERT INTO \"${CHANGE_LOG_META_TABLE}\"\n      (\"epoch\", \"generation\", \"replicaID\", \"schemaVersion\", \"seededAtMs\",\n       \"seedWatermark\")\n      VALUES (@epoch, @generation, @replicaID, @schemaVersion, @seededAtMs,\n              @seedWatermark)\n  `).run({\n    epoch: identity.epoch,\n    generation: identity.generation,\n    replicaID: identity.replicaID,\n    schemaVersion: CHANGE_LOG_DB_SCHEMA_VERSION,\n    seededAtMs: nowMs,\n    seedWatermark: resumeWatermark,\n  });\n  seedChangeLogStream(db, anchor);\n  // Into tables this statement just created, so the wholesale replacement is\n  // all insert. It is still the same call the truncate path makes: a reseed is\n  // the case where the log's cookie set was not merely moved but discarded, and\n  // the anchor's is equally what belongs at the head either way.\n  replaceCookies(db, anchor.cookies);\n\n  lc.info?.('reseeded the SQLite change log', {\n    sqliteChangeLogReconcile: {\n      reason,\n      head: resumeWatermark,\n      ...identity,\n      schemaVersion: CHANGE_LOG_DB_SCHEMA_VERSION,\n      seededAtMs: nowMs,\n      cookies: cookieCounts(anchor),\n    },\n  });\n  return {\n    action: 'reseeded',\n    head: resumeWatermark,\n    reason,\n    cookiesStale: true,\n  };\n}\n\n/**\n * Seeds a valid synthetic transaction at the anchor's resume watermark, making\n * an otherwise empty log serviceable as a catchup boundary for a subscriber\n * that is exactly at that watermark.\n */\nfunction seedChangeLogStream(db: Database, anchor: ChangeLogAnchor): void {\n  const beginChange = '{\"tag\":\"begin\"}';\n  const commitChange = '{\"tag\":\"commit\"}';\n  db.prepare(SEED_CHANGE_LOG_STREAM_SQL).run({\n    watermark: anchor.resumeWatermark,\n    beginBytes: estimateChangeLogStreamRowBytes(\n      anchor.resumeWatermark,\n      'begin',\n      beginChange,\n    ),\n    beginChange,\n    commitBytes: estimateChangeLogStreamRowBytes(\n      anchor.resumeWatermark,\n      'commit',\n      commitChange,\n      anchor.resumeWatermark,\n      true,\n    ),\n    commitChange,\n    writeTimeMs: anchor.nowMs,\n  });\n}\n\n/** Reads the whole meta row. Throws if the log has not been reconciled. */\nexport function readChangeLogMeta(db: Database): ChangeLogMeta {\n  const meta = db\n    .prepare(/*sql*/ `\n      SELECT \"epoch\", \"generation\", \"replicaID\", \"schemaVersion\",\n             \"seededAtMs\", \"seedWatermark\"\n        FROM \"${CHANGE_LOG_META_TABLE}\"\n    `)\n    .get<ChangeLogMeta | undefined>();\n  if (meta === undefined) {\n    throw new Error('the SQLite change log has no meta row');\n  }\n  return meta;\n}\n\n/**\n * `max()` on the primary key's leading column, i.e. an index seek. Null when\n * the log is empty.\n */\nexport function readChangeLogHead(db: Database): string | null {\n  const {head} = db\n    .prepare(/*sql*/ `\n      SELECT max(\"watermark\") AS \"head\" FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n    `)\n    .get<{head: string | null}>();\n  return head;\n}\n\n/**\n * The covered watermark range. Both scalar subqueries seek the watermark\n * index, so unlike a `count()` this does not scan the log. Both bounds are\n * null when it is empty.\n */\nexport function readChangeLogBounds(db: Database): ChangeLogBounds {\n  return db\n    .prepare(/*sql*/ `\n      SELECT\n        (SELECT min(\"watermark\") FROM \"${CHANGE_LOG_STREAM_TABLE}\")\n          AS \"minWatermark\",\n        (SELECT max(\"watermark\") FROM \"${CHANGE_LOG_STREAM_TABLE}\")\n          AS \"headWatermark\"\n    `)\n    .get<ChangeLogBounds>();\n}\n\nexport type ChangeLogBounds = {\n  readonly minWatermark: string | null;\n  readonly headWatermark: string | null;\n};\n\n/**\n * What was installed, for the reconcile log line. Normally `{0, 0}`: a nonzero\n * `backfilling` means the resumed stream is being handed backfills to re-request.\n */\nfunction cookieCounts({cookies}: ChangeLogAnchor) {\n  return {\n    tableMetadata: cookies.tableMetadata.length,\n    backfilling: cookies.backfilling.length,\n  };\n}\n\nfunction tableExists(db: Database, table: string): boolean {\n  return (\n    db\n      .prepare(/*sql*/ `\n        SELECT 1 FROM \"sqlite_master\" WHERE \"type\" = 'table' AND \"name\" = ?\n      `)\n      .get<{1: number} | undefined>(table) !== undefined\n  );\n}\n"],"mappings":";;;;;;AAmFA,IAAM,0BAA0B;AAEhC,IAAa,0BAA0B;AACvC,IAAa,qCACX;AAEF,IAAM,gBAAgB;;;;;;AAOtB,SAAgB,gCACd,WACA,KACA,QACA,WACA,eAAe,OACP;CACR,OACE,OAAO,WAAW,SAAS,IAC3B,gBACA,OAAO,WAAW,GAAG,IACrB,OAAO,WAAW,MAAM,KACvB,cAAc,KAAA,IAAY,IAAI,OAAO,WAAW,SAAS,MACzD,eAAe,gBAAgB;AAEpC;AAUA,IAAa,kCAA0C;kBACrC,wBAAwB;;;;;;;;;;;kBAWxB,mCAAmC;UAC3C,wBAAwB;;;;;;;;;AAUlC,IAAM,6BAAqC;iBAC1B,wBAAwB;;;;;;;;;AAUzC,IAAa,wBAAwB;AAerC,IAAM,gCAAwC;kBAC5B,sBAAsB;;;;;;;;;;;;;;;;;;;;;AAsBxC,SAAgB,kBAAkB,aAA6B;CAC7D,OAAO,GAAG,YAAY;AACxB;;;;;AAMA,SAAgB,kBAAkB,aAA2B;CAC3D,aAAa,kBAAkB,WAAW,CAAC;AAC7C;;;;;;;;;AAsFA,SAAgB,gBACd,IACA,aACA,MACU;CACV,OAAO,IAAI,SACT,GAAG,YAAY,aAAa,eAAe,GAC3C,kBAAkB,WAAW,GAC7B,EAAC,UAAU,KAAK,SAAQ,CAC1B;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,SAAgB,sBAAsB,IAAoB;CACxD,GAAG,OAAO,2BAA2B;CACrC,GAAG,OAAO,sBAAsB;CAChC,GAAG,OAAO,uBAAuB;CACjC,GAAG,OAAO,qBAAqB;CAC/B,GAAG,OAAO,sBAAsB;AAClC;AAEA,SAAS,eAAe,IAAsB;CAC5C,MAAM,CAAC,EAAC,aAAa,UAAS,GAAG,OAA8B,aAAa;CAC5E,OAAO;AACT;AAqCA,SAAgB,0BACd,IACA,aACA,QACgE;CAGhE,MAAM,EAAC,IAAI,QAAQ,iBAAgB,4BACjC,IACA,aACA,QAJa,qBAAqB,IAAI,aAAa,QADrC,WAAW,kBAAkB,WAAW,CACK,CAK3D,CACF;CACA,OAAO;EAAC;EAAI;EAAQ;CAAY;AAClC;AAEA,SAAS,qBACP,IACA,aACA,QACA,qBACiB;CACjB,IAAI;EACF,OAAO,iBAAiB,IAAI,aAAa,QAAQ,mBAAmB;CACtE,SAAS,GAAG;EACV,IAAI,aAAa,0BACf,OAAO,iBAAiB,IAAI,aAAa,QAAQ,CAAC;EAEpD,IAAI,CAAC,mBAAmB,CAAC,GACvB,MAAM;EAER,MAAM,OAAO,kBAAkB,WAAW;EAC1C,+BAA+B,IAAI,cAAc,MAAM,CAAC;EACxD,GAAG,QAAQ,4CAA4C,EACrD,iBAAiB,EAAC,KAAI,EACxB,CAAC;EACD,OAAO,iBAAiB,IAAI,aAAa,MAAM;CACjD;AACF;;;;;;;;;;;;;;;;;;;;;AAsBA,SAAS,4BACP,IACA,aACA,QACA,QACiB;CACjB,IAAI,OAAO,eAAe,yBACxB,OAAO;CAET,MAAM,OAAO,kBAAkB,WAAW;CAC1C,GAAG,OAAO,iEAAiE,EACzE,iBAAiB;EAAC;EAAM,YAAY,OAAO;CAAU,EACvD,CAAC;CACD,OAAO,GAAG,MAAM;CAChB,MAAM,UAAU,iBAAiB,IAAI,aAAa,MAAM;CACxD,IAAI,QAAQ,eAAe,yBACzB,GAAG,QACD,iGAEA,EAAC,iBAAiB;EAAC;EAAM,YAAY,QAAQ;CAAU,EAAC,CAC1D;CAEF,OAAO;AACT;;AAGA,SAAS,iBACP,IACA,aACA,QACA,UACiB;CACjB,IAAI,UACF,GAAG,OACD,mFACA,EACE,wBAAwB;EACtB,QAAQ,SAAS;EACjB,MAAM,SAAS;EACf,gBAAgB,SAAS;CAC3B,EACF,CACF;CAEF,kBAAkB,WAAW;CAU7B,OAAO;EAAC,GAPO,iBACb,IACA,aACA,QACA,OACA,UAAU,MAED;EAAQ,cAAc;CAAI;AACvC;;;;;;AAOA,SAAgB,6BACd,IACA,aACA,QACA,UAC6D;CAC7D,MAAM,EAAC,IAAI,WAAU,iBACnB,IACA,mBACM,QACN,QACF;CACA,OAAO;EAAC;EAAI;EAAQ,cAAc;CAAI;AACxC;AAUA,SAAS,iBACP,IACA,aACA,QACA,qBACA,cACiB;CACjB,MAAM,KAAK,gBAAgB,IAAI,aAAa,EAAC,UAAU,MAAK,CAAC;CAC7D,IAAI;EACF,sBAAsB,EAAE;EACxB,MAAM,aAAa,eAAe,EAAE;EACpC,OAAO;GACL;GACA,QAAQ,mBAAmB,IAAI,IAAI,OAAO,EAAE,GAAG;IAC7C,+BAA+B;IAC/B;GACF,CAAC;GACD,cAAc;GACd;EACF;CACF,SAAS,GAAG;EAGV,GAAG,MAAM;EACT,MAAM;CACR;AACF;;;;;;;AAeA,IAAa,8BAA8B;AAC3C,IAAa,+BAA+B;AAE5C,IAAa,2BAAb,cAA8C,MAAM;CAClD,OAAyB;CACzB;CACA;CACA;CAEA,YACE,QACA,MACA,gBACA;EACA,MAAM,gCAAgC,OAAO,SAAS;EACtD,KAAK,SAAS;EACd,KAAK,OAAO;EACZ,KAAK,iBAAiB;CACxB;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8DA,SAAgB,mBACd,IACA,IACA,QACA,OAAyB,CAAC,GACT;CACjB,GAAG,KAAK,iBAAiB;CACzB,IAAI;EACF,IAAI,KAAK,+BAA+B;GACtC,MAAM,WAAW,gBAAgB,IAAI,MAAM;GAC3C,IAAI,UACF,MAAM;EAEV;EACA,MAAM,SAAS,KAAK,eAChB,OAAO,IAAI,IAAI,QAAQ,KAAK,YAAY,IACxC,UAAU,IAAI,IAAI,MAAM;EAC5B,GAAG,KAAK,QAAQ;EAChB,OAAO;CACT,SAAS,GAAG;EACV,IAAI,GAAG,eACL,GAAG,KAAK,UAAU;EAEpB,MAAM;CACR;AACF;;;;;;AAOA,SAAS,gBACP,IACA,QACsC;CACtC,MAAM,OAAO,oBAAoB,IAAI,OAAO,QAAQ;CACpD,IAAI,SAAS,KAAA,GACX,OAAO,IAAI,yBAAyB,IAAI;CAG1C,MAAM,OAAO,kBAAkB,EAAE;CACjC,MAAM,EAAC,oBAAmB;CAC1B,IAAI,SAAS,QAAQ,OAAO,iBAC1B,OAAO,IAAI,yBAAyB,KAAK;CAE3C,IAAI,SAAS,iBACX;CAUF,IAPiB,GACd,QAAgB;;;;KAIhB,CAAC,CACD,IAA6B,eAC5B,MAAa,KAAA,GACf,OAAO,IAAI,yBAAyB,KAAK;CAG3C,MAAM,EAAC,MAAM,mBAAkB,GAC5B,QAAgB;;;;0CAIqB,wBAAwB;;;;KAI7D,CAAC,CACD,IAA4C,eAAe;CAC9D,OAAO,OAAA,OACL,iBAAA,WACE,IAAI,yBAAyB,sBAAsB,MAAM,cAAc,IACvE,KAAA;AACN;AAEA,SAAS,UACP,IACA,IACA,QACiB;CACjB,MAAM,OAAO,oBAAoB,IAAI,OAAO,QAAQ;CACpD,IAAI,SAAS,KAAA,GACX,OAAO,OAAO,IAAI,IAAI,QAAQ,IAAI;CAGpC,MAAM,EAAC,oBAAmB;CAC1B,IAAI,OAAO,kBAAkB,EAAE;CAC/B,IAAI,OAAO;CACX,IAAI,SAAS,QAAQ,OAAO,iBAAiB;EAC3C,OAAO,GACJ,QAAgB;uBACA,wBAAwB;OACxC,CAAC,CACD,IAAI,eAAe,CAAC,CAAC;EACxB,OAAO,kBAAkB,EAAE;CAC7B;CAEA,IAAI,SAAS,iBACX,OAAO,OAAO,IAAI,IAAI,QAAQ,KAAK;CAErC,IAAI,OAAO,GAAG;EAMZ,eAAe,IAAI,OAAO,OAAO;EACjC,GAAG,OAAO,6DAA6D,EACrE,0BAA0B;GAAC;GAAM;GAAM,SAAS,aAAa,MAAM;EAAC,EACtE,CAAC;EACD,OAAO;GAAC,QAAQ;GAAa;GAAM;GAAM,cAAc;EAAI;CAC7D;CACA,GAAG,QAAQ,gDAAgD,EACzD,0BAA0B,EAAC,KAAI,EACjC,CAAC;CACD,OAAO;EAAC,QAAQ;EAAQ;EAAM,cAAc;CAAK;AACnD;;;;;;;;AASA,SAAgB,oBACd,IACA,UAC0B;CAC1B,IACE,CAAC,YAAY,IAAA,uBAA2B,KACxC,CAAC,YAAY,IAAA,qBAAyB,GAEtC,OAAO;CAKT,MAAM,UAAU,GACb,QAAgB;qCACgB,sBAAsB;KACtD,CAAC,CACD,IAAyC;CAC5C,IAAI,YAAY,KAAA,GACd,OAAO;CAET,IAAI,QAAQ,kBAAA,GACV,OAAO;CAMT,IACE,CAAC,YAAY,IAAA,8BAAmC,KAChD,CAAC,YAAY,IAAA,4BAAgC,GAE7C,OAAO;CAET,MAAM,EAAC,OAAO,YAAY,cAAa,kBAAkB,EAAE;CAC3D,IACE,UAAU,SAAS,SACnB,eAAe,SAAS,cACxB,cAAc,SAAS,WAEvB,OAAO;AAGX;AAEA,SAAS,OACP,IACA,IACA,QACA,QACiB;CACjB,MAAM,EAAC,UAAU,iBAAiB,UAAS;CAI3C,GAAG,KAAa;4BACU,wBAAwB;4BACxB,sBAAsB;MAC5C,8BAA8B;MAC9B,gCAAgC;MAChC,8BAA8B;MAC9B,gCAAgC;GACnC;CACD,GAAG,QAAgB;mBACF,sBAAsB;;;;;GAKtC,CAAC,CAAC,IAAI;EACL,OAAO,SAAS;EAChB,YAAY,SAAS;EACrB,WAAW,SAAS;EACpB,eAAA;EACA,YAAY;EACZ,eAAe;CACjB,CAAC;CACD,oBAAoB,IAAI,MAAM;CAK9B,eAAe,IAAI,OAAO,OAAO;CAEjC,GAAG,OAAO,kCAAkC,EAC1C,0BAA0B;EACxB;EACA,MAAM;EACN,GAAG;EACH,eAAA;EACA,YAAY;EACZ,SAAS,aAAa,MAAM;CAC9B,EACF,CAAC;CACD,OAAO;EACL,QAAQ;EACR,MAAM;EACN;EACA,cAAc;CAChB;AACF;;;;;;AAOA,SAAS,oBAAoB,IAAc,QAA+B;CACxE,MAAM,cAAc;CACpB,MAAM,eAAe;CACrB,GAAG,QAAQ,0BAA0B,CAAC,CAAC,IAAI;EACzC,WAAW,OAAO;EAClB,YAAY,gCACV,OAAO,iBACP,SACA,WACF;EACA;EACA,aAAa,gCACX,OAAO,iBACP,UACA,cACA,OAAO,iBACP,IACF;EACA;EACA,aAAa,OAAO;CACtB,CAAC;AACH;;AAGA,SAAgB,kBAAkB,IAA6B;CAC7D,MAAM,OAAO,GACV,QAAgB;;;gBAGL,sBAAsB;KACjC,CAAC,CACD,IAA+B;CAClC,IAAI,SAAS,KAAA,GACX,MAAM,IAAI,MAAM,uCAAuC;CAEzD,OAAO;AACT;;;;;AAMA,SAAgB,kBAAkB,IAA6B;CAC7D,MAAM,EAAC,SAAQ,GACZ,QAAgB;gDAC2B,wBAAwB;KACnE,CAAC,CACD,IAA2B;CAC9B,OAAO;AACT;;;;;;AAOA,SAAgB,oBAAoB,IAA+B;CACjE,OAAO,GACJ,QAAgB;;yCAEoB,wBAAwB;;yCAExB,wBAAwB;;KAE5D,CAAC,CACD,IAAqB;AAC1B;;;;;AAWA,SAAS,aAAa,EAAC,WAA2B;CAChD,OAAO;EACL,eAAe,QAAQ,cAAc;EACrC,aAAa,QAAQ,YAAY;CACnC;AACF;AAEA,SAAS,YAAY,IAAc,OAAwB;CACzD,OACE,GACG,QAAgB;;OAEhB,CAAC,CACD,IAA6B,KAAK,MAAM,KAAA;AAE/C"}

package/out/zero-cache/src/services/replicator/change-log-stream-writer.js

@@ -143,6 +143,6 @@ var ChangeLogStreamWriter = class {
 	}
 };
 //#endregion
-export { ChangeLogStreamInvariantError, ChangeLogStreamWriter };
+export { ChangeLogStreamWriter };
 
 //# sourceMappingURL=change-log-stream-writer.js.map