@rocicorp/zero 1.11.0-canary.4 → 1.11.0-canary.41

package/out/zero-cache/src/services/replicator/replication-status.js.map

@@ -1 +1 @@
-{"version":3,"file":"replication-status.js","names":["#dbRunner","#publish","#timer"],"sources":["../../../../../../zero-cache/src/services/replicator/replication-status.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {createSilentLogContext} from '../../../../shared/src/logging-test-utils.ts';\nimport type {JSONObject} from '../../../../zero-events/src/json.ts';\nimport type {\n  ReplicatedIndex,\n  ReplicatedTable,\n  ReplicationStage,\n  ReplicationState,\n  ReplicationStatusEvent,\n  Status,\n} from '../../../../zero-events/src/status.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs, listIndexes} from '../../db/lite-tables.ts';\nimport type {LiteTableSpec} from '../../db/specs.ts';\nimport {\n  makeErrorDetails,\n  publishCriticalEvent,\n} from '../../observability/events.ts';\n\nconst byKeys = (a: [string, unknown], b: [string, unknown]) =>\n  a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0;\n\nexport class ReplicationStatusPublisher {\n  readonly #dbRunner: <T>(lc: LogContext, fn: (db: Database) => T) => T;\n  readonly #publish: typeof publishCriticalEvent;\n  #timer: NodeJS.Timeout | undefined;\n\n  static forTesting() {\n    return ReplicationStatusPublisher.forReplicaFile(':memory:');\n  }\n\n  static forRunningTransaction(tx: Database, publishFn = publishCriticalEvent) {\n    return new ReplicationStatusPublisher((_lc, fn) => fn(tx), publishFn);\n  }\n\n  static forReplicaFile(file: string, publishFn = publishCriticalEvent) {\n    return new ReplicationStatusPublisher((lc, fn) => {\n      const db = new Database(lc, file, {readonly: true});\n      try {\n        return fn(db);\n      } finally {\n        db.close();\n      }\n    }, publishFn);\n  }\n\n  constructor(\n    dbRunner: <T>(lc: LogContext, fn: (db: Database) => T) => T,\n    publishFn = publishCriticalEvent,\n  ) {\n    this.#dbRunner = dbRunner;\n    this.#publish = publishFn;\n  }\n\n  publish(\n    lc: LogContext,\n    stage: ReplicationStage,\n    description?: string,\n    interval = 0,\n    extraState?: () => Partial<ReplicationState>,\n    now = new Date(),\n  ): this {\n    this.stop();\n\n    const event = this.#dbRunner(lc, db =>\n      replicationStatusEvent(lc, db, stage, 'OK', description, now),\n    );\n    if (event.state) {\n      event.state = {\n        ...event.state,\n        ...extraState?.(),\n      };\n    }\n    void this.#publish(lc, event);\n\n    if (interval) {\n      this.#timer = setInterval(\n        () => this.publish(lc, stage, description, interval, extraState),\n        interval,\n      );\n    }\n    return this;\n  }\n\n  async publishAndThrowError(\n    lc: LogContext,\n    stage: ReplicationStage,\n    e: unknown,\n  ): Promise<never> {\n    this.stop();\n    const event = this.#dbRunner(lc, db =>\n      replicationStatusError(lc, stage, e, db),\n    );\n    await this.#publish(lc, event);\n    throw e;\n  }\n\n  stop(): this {\n    clearInterval(this.#timer);\n    return this;\n  }\n}\n\nexport async function publishReplicationError(\n  lc: LogContext,\n  stage: ReplicationStage,\n  description: string,\n  errorDetails?: JSONObject,\n  now = new Date(),\n) {\n  const event: ReplicationStatusEvent = {\n    type: 'zero/events/status/replication/v1',\n    component: 'replication',\n    status: 'ERROR',\n    stage,\n    description,\n    errorDetails,\n    time: now.toISOString(),\n  };\n  await publishCriticalEvent(lc, event);\n}\n\nexport function replicationStatusError(\n  lc: LogContext,\n  stage: ReplicationStage,\n  e: unknown,\n  db?: Database,\n  now = new Date(),\n) {\n  const event = replicationStatusEvent(lc, db, stage, 'ERROR', String(e), now);\n  event.errorDetails = makeErrorDetails(e);\n  return event;\n}\n\n// Exported for testing.\nexport function replicationStatusEvent(\n  lc: LogContext,\n  db: Database | undefined,\n  stage: ReplicationStage,\n  status: Status,\n  description?: string,\n  now = new Date(),\n): ReplicationStatusEvent {\n  const start = performance.now();\n  try {\n    return {\n      type: 'zero/events/status/replication/v1',\n      component: 'replication',\n      status,\n      stage,\n      description,\n      time: now.toISOString(),\n      state: {\n        tables: db ? getReplicatedTables(db) : [],\n        indexes: db ? getReplicatedIndexes(db) : [],\n        replicaSize: db ? getReplicaSize(db) : undefined,\n      },\n    };\n  } catch (e) {\n    lc.warn?.(`Unable to create full ReplicationStatusEvent`, e);\n    return {\n      type: 'zero/events/status/replication/v1',\n      component: 'replication',\n      status,\n      stage,\n      description,\n      time: now.toISOString(),\n      state: {\n        tables: [],\n        indexes: [],\n        replicaSize: 0,\n      },\n    };\n  } finally {\n    const elapsed = (performance.now() - start).toFixed(3);\n    lc.debug?.(`computed schema for replication event (${elapsed} ms)`);\n  }\n}\n\nfunction getReplicatedTables(db: Database): ReplicatedTable[] {\n  const fullTables = new Map<string, LiteTableSpec>();\n  const clientSchema = computeZqlSpecs(\n    createSilentLogContext(), // avoid logging warnings about indexes\n    db,\n    {includeBackfillingColumns: false},\n    new Map(),\n    fullTables,\n  );\n\n  // oxlint-disable-next-line e18e/prefer-array-to-sorted\n  return [...fullTables.entries()].sort(byKeys).map(([table, spec]) => ({\n    table,\n    columns: Object.entries(spec.columns)\n      .sort(byKeys)\n      .map(([column, spec]) => ({\n        column,\n        upstreamType: spec.dataType.split('|')[0],\n        clientType: clientSchema.get(table)?.zqlSpec[column]?.type ?? null,\n      })),\n  }));\n}\n\nfunction getReplicatedIndexes(db: Database): ReplicatedIndex[] {\n  return listIndexes(db).map(({tableName: table, columns, unique}) => ({\n    table,\n    unique,\n    columns: Object.entries(columns)\n      .sort(byKeys)\n      .map(([column, dir]) => ({column, dir})),\n  }));\n}\n\nfunction getReplicaSize(db: Database) {\n  const [{page_count: pageCount}] = db.pragma<{page_count: number}>(\n    'page_count',\n  );\n  const [{page_size: pageSize}] = db.pragma<{page_size: number}>('page_size');\n  return pageCount * pageSize;\n}\n"],"mappings":";;;;;AAmBA,IAAM,UAAU,GAAsB,MACpC,EAAE,KAAK,EAAE,KAAK,KAAK,EAAE,KAAK,EAAE,KAAK,IAAI;AAEvC,IAAa,6BAAb,MAAa,2BAA2B;CACtC;CACA;CACA;CAEA,OAAO,aAAa;EAClB,OAAO,2BAA2B,eAAe,UAAU;CAC7D;CAEA,OAAO,sBAAsB,IAAc,YAAY,sBAAsB;EAC3E,OAAO,IAAI,4BAA4B,KAAK,OAAO,GAAG,EAAE,GAAG,SAAS;CACtE;CAEA,OAAO,eAAe,MAAc,YAAY,sBAAsB;EACpE,OAAO,IAAI,4BAA4B,IAAI,OAAO;GAChD,MAAM,KAAK,IAAI,SAAS,IAAI,MAAM,EAAC,UAAU,KAAI,CAAC;GAClD,IAAI;IACF,OAAO,GAAG,EAAE;GACd,UAAU;IACR,GAAG,MAAM;GACX;EACF,GAAG,SAAS;CACd;CAEA,YACE,UACA,YAAY,sBACZ;EACA,KAAKA,YAAY;EACjB,KAAKC,WAAW;CAClB;CAEA,QACE,IACA,OACA,aACA,WAAW,GACX,YACA,sBAAM,IAAI,KAAK,GACT;EACN,KAAK,KAAK;EAEV,MAAM,QAAQ,KAAKD,UAAU,KAAI,OAC/B,uBAAuB,IAAI,IAAI,OAAO,MAAM,aAAa,GAAG,CAC9D;EACA,IAAI,MAAM,OACR,MAAM,QAAQ;GACZ,GAAG,MAAM;GACT,GAAG,aAAa;EAClB;EAEF,KAAUC,SAAS,IAAI,KAAK;EAE5B,IAAI,UACF,KAAKC,SAAS,kBACN,KAAK,QAAQ,IAAI,OAAO,aAAa,UAAU,UAAU,GAC/D,QACF;EAEF,OAAO;CACT;CAEA,MAAM,qBACJ,IACA,OACA,GACgB;EAChB,KAAK,KAAK;EACV,MAAM,QAAQ,KAAKF,UAAU,KAAI,OAC/B,uBAAuB,IAAI,OAAO,GAAG,EAAE,CACzC;EACA,MAAM,KAAKC,SAAS,IAAI,KAAK;EAC7B,MAAM;CACR;CAEA,OAAa;EACX,cAAc,KAAKC,MAAM;EACzB,OAAO;CACT;AACF;AAEA,eAAsB,wBACpB,IACA,OACA,aACA,cACA,sBAAM,IAAI,KAAK,GACf;CAUA,MAAM,qBAAqB,IAAI;EAR7B,MAAM;EACN,WAAW;EACX,QAAQ;EACR;EACA;EACA;EACA,MAAM,IAAI,YAAY;CAEO,CAAK;AACtC;AAEA,SAAgB,uBACd,IACA,OACA,GACA,IACA,sBAAM,IAAI,KAAK,GACf;CACA,MAAM,QAAQ,uBAAuB,IAAI,IAAI,OAAO,SAAS,OAAO,CAAC,GAAG,GAAG;CAC3E,MAAM,eAAe,iBAAiB,CAAC;CACvC,OAAO;AACT;AAGA,SAAgB,uBACd,IACA,IACA,OACA,QACA,aACA,sBAAM,IAAI,KAAK,GACS;CACxB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;EACF,OAAO;GACL,MAAM;GACN,WAAW;GACX;GACA;GACA;GACA,MAAM,IAAI,YAAY;GACtB,OAAO;IACL,QAAQ,KAAK,oBAAoB,EAAE,IAAI,CAAC;IACxC,SAAS,KAAK,qBAAqB,EAAE,IAAI,CAAC;IAC1C,aAAa,KAAK,eAAe,EAAE,IAAI,KAAA;GACzC;EACF;CACF,SAAS,GAAG;EACV,GAAG,OAAO,gDAAgD,CAAC;EAC3D,OAAO;GACL,MAAM;GACN,WAAW;GACX;GACA;GACA;GACA,MAAM,IAAI,YAAY;GACtB,OAAO;IACL,QAAQ,CAAC;IACT,SAAS,CAAC;IACV,aAAa;GACf;EACF;CACF,UAAU;EACR,MAAM,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;EACrD,GAAG,QAAQ,0CAA0C,QAAQ,KAAK;CACpE;AACF;AAEA,SAAS,oBAAoB,IAAiC;CAC5D,MAAM,6BAAa,IAAI,IAA2B;CAClD,MAAM,eAAe,gBACnB,uBAAuB,GACvB,IACA,EAAC,2BAA2B,MAAK,mBACjC,IAAI,IAAI,GACR,UACF;CAGA,OAAO,CAAC,GAAG,WAAW,QAAQ,CAAC,EAAE,KAAK,MAAM,EAAE,KAAK,CAAC,OAAO,WAAW;EACpE;EACA,SAAS,OAAO,QAAQ,KAAK,OAAO,EACjC,KAAK,MAAM,EACX,KAAK,CAAC,QAAQ,WAAW;GACxB;GACA,cAAc,KAAK,SAAS,MAAM,GAAG,EAAE;GACvC,YAAY,aAAa,IAAI,KAAK,GAAG,QAAQ,SAAS,QAAQ;EAChE,EAAE;CACN,EAAE;AACJ;AAEA,SAAS,qBAAqB,IAAiC;CAC7D,OAAO,YAAY,EAAE,EAAE,KAAK,EAAC,WAAW,OAAO,SAAS,cAAa;EACnE;EACA;EACA,SAAS,OAAO,QAAQ,OAAO,EAC5B,KAAK,MAAM,EACX,KAAK,CAAC,QAAQ,UAAU;GAAC;GAAQ;EAAG,EAAE;CAC3C,EAAE;AACJ;AAEA,SAAS,eAAe,IAAc;CACpC,MAAM,CAAC,EAAC,YAAY,eAAc,GAAG,OACnC,YACF;CACA,MAAM,CAAC,EAAC,WAAW,cAAa,GAAG,OAA4B,WAAW;CAC1E,OAAO,YAAY;AACrB"}
+{"version":3,"file":"replication-status.js","names":["#dbRunner","#publishFn","#publish","#timer","#totalIndexes","#now","#count","#current","#completedMs"],"sources":["../../../../../../zero-cache/src/services/replicator/replication-status.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {createSilentLogContext} from '../../../../shared/src/logging-test-utils.ts';\nimport type {JSONObject} from '../../../../zero-events/src/json.ts';\nimport type {\n  IndexingStatus,\n  ReplicatedIndex,\n  ReplicatedTable,\n  ReplicationStage,\n  ReplicationState,\n  ReplicationStatusEvent,\n  Status,\n} from '../../../../zero-events/src/status.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs, listIndexes} from '../../db/lite-tables.ts';\nimport type {LiteIndexSpec, LiteTableSpec} from '../../db/specs.ts';\nimport {\n  makeErrorDetails,\n  publishCriticalEvent,\n} from '../../observability/events.ts';\n\nconst byKeys = (a: [string, unknown], b: [string, unknown]) =>\n  a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0;\n\nexport class ReplicationStatusPublisher {\n  readonly #dbRunner: <T>(lc: LogContext, fn: (db: Database) => T) => T;\n  readonly #publishFn: typeof publishCriticalEvent;\n  #timer: NodeJS.Timeout | undefined;\n\n  static forTesting() {\n    return ReplicationStatusPublisher.forReplicaFile(':memory:');\n  }\n\n  static forRunningTransaction(tx: Database, publishFn = publishCriticalEvent) {\n    return new ReplicationStatusPublisher((_lc, fn) => fn(tx), publishFn);\n  }\n\n  static forReplicaFile(file: string, publishFn = publishCriticalEvent) {\n    return new ReplicationStatusPublisher((lc, fn) => {\n      const db = new Database(lc, file, {readonly: true});\n      try {\n        return fn(db);\n      } finally {\n        db.close();\n      }\n    }, publishFn);\n  }\n\n  constructor(\n    dbRunner: <T>(lc: LogContext, fn: (db: Database) => T) => T,\n    publishFn = publishCriticalEvent,\n  ) {\n    this.#dbRunner = dbRunner;\n    this.#publishFn = publishFn;\n  }\n\n  publish(\n    lc: LogContext,\n    stage: ReplicationStage,\n    description?: string,\n    interval = 0,\n    extraState?: () => Partial<ReplicationState>,\n    now = new Date(),\n  ): this {\n    void this.#publish(lc, stage, description, interval, extraState, now);\n    return this;\n  }\n\n  /**\n   * Like {@link publish}, but waits (for at most `maxWaitMs`) for the event\n   * to be sent. Use this before a long synchronous operation (e.g. creating\n   * an index) that blocks the event loop and would otherwise prevent the\n   * event from being sent until the operation completes.\n   */\n  async publishAndFlush(\n    lc: LogContext,\n    stage: ReplicationStage,\n    description?: string,\n    interval = 0,\n    extraState?: () => Partial<ReplicationState>,\n    maxWaitMs = 1000,\n  ): Promise<void> {\n    const published = this.#publish(\n      lc,\n      stage,\n      description,\n      interval,\n      extraState,\n    );\n    let timer: NodeJS.Timeout | undefined;\n    try {\n      await Promise.race([\n        published,\n        new Promise<void>(resolve => {\n          timer = setTimeout(resolve, maxWaitMs);\n        }),\n      ]);\n    } finally {\n      clearTimeout(timer);\n    }\n  }\n\n  #publish(\n    lc: LogContext,\n    stage: ReplicationStage,\n    description: string | undefined,\n    interval: number,\n    extraState: (() => Partial<ReplicationState>) | undefined,\n    now = new Date(),\n  ): Promise<void> {\n    this.stop();\n\n    const event = this.#dbRunner(lc, db =>\n      replicationStatusEvent(lc, db, stage, 'OK', description, now),\n    );\n    if (event.state) {\n      event.state = {\n        ...event.state,\n        ...extraState?.(),\n      };\n    }\n    const published = this.#publishFn(lc, event);\n\n    if (interval) {\n      this.#timer = setInterval(\n        () => this.publish(lc, stage, description, interval, extraState),\n        interval,\n      );\n    }\n    return published;\n  }\n\n  async publishAndThrowError(\n    lc: LogContext,\n    stage: ReplicationStage,\n    e: unknown,\n  ): Promise<never> {\n    this.stop();\n    const event = this.#dbRunner(lc, db =>\n      replicationStatusError(lc, stage, e, db),\n    );\n    await this.#publishFn(lc, event);\n    throw e;\n  }\n\n  stop(): this {\n    clearInterval(this.#timer);\n    return this;\n  }\n}\n\n/**\n * Tracks the progress of a sequence of index creations for reporting in\n * the `indexingStatus` field of {@link ReplicationState}.\n */\nexport class IndexingProgress {\n  readonly #totalIndexes: number | undefined;\n  readonly #now: () => number;\n  #count = 0;\n  #completedMs = 0;\n  #current:\n    | {index: LiteIndexSpec; start: number; elapsedMs: number | undefined}\n    | undefined;\n\n  /**\n   * @param totalIndexes The total number of indexes to be created, if known.\n   */\n  constructor(totalIndexes?: number, now = () => performance.now()) {\n    this.#totalIndexes = totalIndexes;\n    this.#now = now;\n  }\n\n  /** Marks the start of creating the `index`. */\n  start(index: LiteIndexSpec) {\n    this.#count++;\n    this.#current = {index, start: this.#now(), elapsedMs: undefined};\n  }\n\n  /**\n   * Restarts the timer of the current index, e.g. to exclude time spent\n   * reporting its start from its creation time.\n   */\n  restartTimer() {\n    if (this.#current) {\n      this.#current.start = this.#now();\n    }\n  }\n\n  /**\n   * Marks the completion of the index passed to the last call to\n   * {@link start}, returning the milliseconds it took to create.\n   */\n  finish(): number {\n    const current = this.#current;\n    if (!current) {\n      return 0;\n    }\n    if (current.elapsedMs === undefined) {\n      current.elapsedMs = this.#now() - current.start;\n      this.#completedMs += current.elapsedMs;\n    }\n    return current.elapsedMs;\n  }\n\n  /** The `extraState` for {@link ReplicationStatusPublisher.publish}. */\n  readonly state = (): Partial<ReplicationState> => {\n    const status = this.status();\n    return status ? {indexingStatus: status} : {};\n  };\n\n  status(): IndexingStatus | undefined {\n    const current = this.#current;\n    if (!current) {\n      return undefined;\n    }\n    const {index, start, elapsedMs} = current;\n    const done = elapsedMs !== undefined;\n    return {\n      name: index.name,\n      table: index.tableName,\n      columns: Object.keys(index.columns),\n      unique: index.unique,\n      index: this.#count,\n      totalIndexes: this.#totalIndexes,\n      elapsedMs: Math.round(done ? elapsedMs : this.#now() - start),\n      completedMs: Math.round(\n        done ? this.#completedMs - elapsedMs : this.#completedMs,\n      ),\n      done,\n    };\n  }\n}\n\nexport async function publishReplicationError(\n  lc: LogContext,\n  stage: ReplicationStage,\n  description: string,\n  errorDetails?: JSONObject,\n  now = new Date(),\n) {\n  const event: ReplicationStatusEvent = {\n    type: 'zero/events/status/replication/v1',\n    component: 'replication',\n    status: 'ERROR',\n    stage,\n    description,\n    errorDetails,\n    time: now.toISOString(),\n  };\n  await publishCriticalEvent(lc, event);\n}\n\nexport function replicationStatusError(\n  lc: LogContext,\n  stage: ReplicationStage,\n  e: unknown,\n  db?: Database,\n  now = new Date(),\n) {\n  const event = replicationStatusEvent(lc, db, stage, 'ERROR', String(e), now);\n  event.errorDetails = makeErrorDetails(e);\n  return event;\n}\n\n// Exported for testing.\nexport function replicationStatusEvent(\n  lc: LogContext,\n  db: Database | undefined,\n  stage: ReplicationStage,\n  status: Status,\n  description?: string,\n  now = new Date(),\n): ReplicationStatusEvent {\n  const start = performance.now();\n  try {\n    return {\n      type: 'zero/events/status/replication/v1',\n      component: 'replication',\n      status,\n      stage,\n      description,\n      time: now.toISOString(),\n      state: {\n        tables: db ? getReplicatedTables(db) : [],\n        indexes: db ? getReplicatedIndexes(db) : [],\n        replicaSize: db ? getReplicaSize(db) : undefined,\n      },\n    };\n  } catch (e) {\n    lc.warn?.(`Unable to create full ReplicationStatusEvent`, e);\n    return {\n      type: 'zero/events/status/replication/v1',\n      component: 'replication',\n      status,\n      stage,\n      description,\n      time: now.toISOString(),\n      state: {\n        tables: [],\n        indexes: [],\n        replicaSize: 0,\n      },\n    };\n  } finally {\n    const elapsed = (performance.now() - start).toFixed(3);\n    lc.debug?.(`computed schema for replication event (${elapsed} ms)`);\n  }\n}\n\nfunction getReplicatedTables(db: Database): ReplicatedTable[] {\n  const fullTables = new Map<string, LiteTableSpec>();\n  const clientSchema = computeZqlSpecs(\n    createSilentLogContext(), // avoid logging warnings about indexes\n    db,\n    {includeBackfillingColumns: false},\n    new Map(),\n    fullTables,\n  );\n\n  // oxlint-disable-next-line e18e/prefer-array-to-sorted\n  return [...fullTables.entries()].sort(byKeys).map(([table, spec]) => ({\n    table,\n    columns: Object.entries(spec.columns)\n      .sort(byKeys)\n      .map(([column, spec]) => ({\n        column,\n        upstreamType: spec.dataType.split('|')[0],\n        clientType: clientSchema.get(table)?.zqlSpec[column]?.type ?? null,\n      })),\n  }));\n}\n\nfunction getReplicatedIndexes(db: Database): ReplicatedIndex[] {\n  return listIndexes(db).map(({name, tableName: table, columns, unique}) => ({\n    name,\n    table,\n    unique,\n    columns: Object.entries(columns).map(([column, dir]) => ({column, dir})),\n  }));\n}\n\nfunction getReplicaSize(db: Database) {\n  const [{page_count: pageCount}] = db.pragma<{page_count: number}>(\n    'page_count',\n  );\n  const [{page_size: pageSize}] = db.pragma<{page_size: number}>('page_size');\n  return pageCount * pageSize;\n}\n"],"mappings":";;;;;AAoBA,IAAM,UAAU,GAAsB,MACpC,EAAE,KAAK,EAAE,KAAK,KAAK,EAAE,KAAK,EAAE,KAAK,IAAI;AAEvC,IAAa,6BAAb,MAAa,2BAA2B;CACtC;CACA;CACA;CAEA,OAAO,aAAa;EAClB,OAAO,2BAA2B,eAAe,UAAU;CAC7D;CAEA,OAAO,sBAAsB,IAAc,YAAY,sBAAsB;EAC3E,OAAO,IAAI,4BAA4B,KAAK,OAAO,GAAG,EAAE,GAAG,SAAS;CACtE;CAEA,OAAO,eAAe,MAAc,YAAY,sBAAsB;EACpE,OAAO,IAAI,4BAA4B,IAAI,OAAO;GAChD,MAAM,KAAK,IAAI,SAAS,IAAI,MAAM,EAAC,UAAU,KAAI,CAAC;GAClD,IAAI;IACF,OAAO,GAAG,EAAE;GACd,UAAU;IACR,GAAG,MAAM;GACX;EACF,GAAG,SAAS;CACd;CAEA,YACE,UACA,YAAY,sBACZ;EACA,KAAKA,YAAY;EACjB,KAAKC,aAAa;CACpB;CAEA,QACE,IACA,OACA,aACA,WAAW,GACX,YACA,sBAAM,IAAI,KAAK,GACT;EACN,KAAUC,SAAS,IAAI,OAAO,aAAa,UAAU,YAAY,GAAG;EACpE,OAAO;CACT;;;;;;;CAQA,MAAM,gBACJ,IACA,OACA,aACA,WAAW,GACX,YACA,YAAY,KACG;EACf,MAAM,YAAY,KAAKA,SACrB,IACA,OACA,aACA,UACA,UACF;EACA,IAAI;EACJ,IAAI;GACF,MAAM,QAAQ,KAAK,CACjB,WACA,IAAI,SAAc,YAAW;IAC3B,QAAQ,WAAW,SAAS,SAAS;GACvC,CAAC,CACH,CAAC;EACH,UAAU;GACR,aAAa,KAAK;EACpB;CACF;CAEA,SACE,IACA,OACA,aACA,UACA,YACA,sBAAM,IAAI,KAAK,GACA;EACf,KAAK,KAAK;EAEV,MAAM,QAAQ,KAAKF,UAAU,KAAI,OAC/B,uBAAuB,IAAI,IAAI,OAAO,MAAM,aAAa,GAAG,CAC9D;EACA,IAAI,MAAM,OACR,MAAM,QAAQ;GACZ,GAAG,MAAM;GACT,GAAG,aAAa;EAClB;EAEF,MAAM,YAAY,KAAKC,WAAW,IAAI,KAAK;EAE3C,IAAI,UACF,KAAKE,SAAS,kBACN,KAAK,QAAQ,IAAI,OAAO,aAAa,UAAU,UAAU,GAC/D,QACF;EAEF,OAAO;CACT;CAEA,MAAM,qBACJ,IACA,OACA,GACgB;EAChB,KAAK,KAAK;EACV,MAAM,QAAQ,KAAKH,UAAU,KAAI,OAC/B,uBAAuB,IAAI,OAAO,GAAG,EAAE,CACzC;EACA,MAAM,KAAKC,WAAW,IAAI,KAAK;EAC/B,MAAM;CACR;CAEA,OAAa;EACX,cAAc,KAAKE,MAAM;EACzB,OAAO;CACT;AACF;;;;;AAMA,IAAa,mBAAb,MAA8B;CAC5B;CACA;CACA,SAAS;CACT,eAAe;CACf;;;;CAOA,YAAY,cAAuB,YAAY,YAAY,IAAI,GAAG;EAChE,KAAKC,gBAAgB;EACrB,KAAKC,OAAO;CACd;;CAGA,MAAM,OAAsB;EAC1B,KAAKC;EACL,KAAKC,WAAW;GAAC;GAAO,OAAO,KAAKF,KAAK;GAAG,WAAW,KAAA;EAAS;CAClE;;;;;CAMA,eAAe;EACb,IAAI,KAAKE,UACP,KAAKA,SAAS,QAAQ,KAAKF,KAAK;CAEpC;;;;;CAMA,SAAiB;EACf,MAAM,UAAU,KAAKE;EACrB,IAAI,CAAC,SACH,OAAO;EAET,IAAI,QAAQ,cAAc,KAAA,GAAW;GACnC,QAAQ,YAAY,KAAKF,KAAK,IAAI,QAAQ;GAC1C,KAAKG,gBAAgB,QAAQ;EAC/B;EACA,OAAO,QAAQ;CACjB;;CAGA,cAAkD;EAChD,MAAM,SAAS,KAAK,OAAO;EAC3B,OAAO,SAAS,EAAC,gBAAgB,OAAM,IAAI,CAAC;CAC9C;CAEA,SAAqC;EACnC,MAAM,UAAU,KAAKD;EACrB,IAAI,CAAC,SACH;EAEF,MAAM,EAAC,OAAO,OAAO,cAAa;EAClC,MAAM,OAAO,cAAc,KAAA;EAC3B,OAAO;GACL,MAAM,MAAM;GACZ,OAAO,MAAM;GACb,SAAS,OAAO,KAAK,MAAM,OAAO;GAClC,QAAQ,MAAM;GACd,OAAO,KAAKD;GACZ,cAAc,KAAKF;GACnB,WAAW,KAAK,MAAM,OAAO,YAAY,KAAKC,KAAK,IAAI,KAAK;GAC5D,aAAa,KAAK,MAChB,OAAO,KAAKG,eAAe,YAAY,KAAKA,YAC9C;GACA;EACF;CACF;AACF;AAEA,eAAsB,wBACpB,IACA,OACA,aACA,cACA,sBAAM,IAAI,KAAK,GACf;CAUA,MAAM,qBAAqB,IAAI;EAR7B,MAAM;EACN,WAAW;EACX,QAAQ;EACR;EACA;EACA;EACA,MAAM,IAAI,YAAY;CAEO,CAAK;AACtC;AAEA,SAAgB,uBACd,IACA,OACA,GACA,IACA,sBAAM,IAAI,KAAK,GACf;CACA,MAAM,QAAQ,uBAAuB,IAAI,IAAI,OAAO,SAAS,OAAO,CAAC,GAAG,GAAG;CAC3E,MAAM,eAAe,iBAAiB,CAAC;CACvC,OAAO;AACT;AAGA,SAAgB,uBACd,IACA,IACA,OACA,QACA,aACA,sBAAM,IAAI,KAAK,GACS;CACxB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;EACF,OAAO;GACL,MAAM;GACN,WAAW;GACX;GACA;GACA;GACA,MAAM,IAAI,YAAY;GACtB,OAAO;IACL,QAAQ,KAAK,oBAAoB,EAAE,IAAI,CAAC;IACxC,SAAS,KAAK,qBAAqB,EAAE,IAAI,CAAC;IAC1C,aAAa,KAAK,eAAe,EAAE,IAAI,KAAA;GACzC;EACF;CACF,SAAS,GAAG;EACV,GAAG,OAAO,gDAAgD,CAAC;EAC3D,OAAO;GACL,MAAM;GACN,WAAW;GACX;GACA;GACA;GACA,MAAM,IAAI,YAAY;GACtB,OAAO;IACL,QAAQ,CAAC;IACT,SAAS,CAAC;IACV,aAAa;GACf;EACF;CACF,UAAU;EACR,MAAM,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;EACrD,GAAG,QAAQ,0CAA0C,QAAQ,KAAK;CACpE;AACF;AAEA,SAAS,oBAAoB,IAAiC;CAC5D,MAAM,6BAAa,IAAI,IAA2B;CAClD,MAAM,eAAe,gBACnB,uBAAuB,GACvB,IACA,EAAC,2BAA2B,MAAK,mBACjC,IAAI,IAAI,GACR,UACF;CAGA,OAAO,CAAC,GAAG,WAAW,QAAQ,CAAC,EAAE,KAAK,MAAM,EAAE,KAAK,CAAC,OAAO,WAAW;EACpE;EACA,SAAS,OAAO,QAAQ,KAAK,OAAO,EACjC,KAAK,MAAM,EACX,KAAK,CAAC,QAAQ,WAAW;GACxB;GACA,cAAc,KAAK,SAAS,MAAM,GAAG,EAAE;GACvC,YAAY,aAAa,IAAI,KAAK,GAAG,QAAQ,SAAS,QAAQ;EAChE,EAAE;CACN,EAAE;AACJ;AAEA,SAAS,qBAAqB,IAAiC;CAC7D,OAAO,YAAY,EAAE,EAAE,KAAK,EAAC,MAAM,WAAW,OAAO,SAAS,cAAa;EACzE;EACA;EACA;EACA,SAAS,OAAO,QAAQ,OAAO,EAAE,KAAK,CAAC,QAAQ,UAAU;GAAC;GAAQ;EAAG,EAAE;CACzE,EAAE;AACJ;AAEA,SAAS,eAAe,IAAc;CACpC,MAAM,CAAC,EAAC,YAAY,eAAc,GAAG,OACnC,YACF;CACA,MAAM,CAAC,EAAC,WAAW,cAAa,GAAG,OAA4B,WAAW;CAC1E,OAAO,YAAY;AACrB"}

package/out/zero-cache/src/services/replicator/replicator.d.ts

@@ -1,9 +1,12 @@
 import type { LogContext } from '@rocicorp/logger';
 import type { ReadonlyJSONObject } from '../../../../shared/src/json.ts';
 import type { Source } from '../../types/streams.ts';
+import type { ReservationFollowup } from '../change-streamer/change-streamer-http.ts';
 import type { ChangeStreamer } from '../change-streamer/change-streamer.ts';
 import type { Service } from '../service.ts';
+import type { SwitchoverOptions } from './drain-handler.ts';
 import { type ReplicationStatusPublisher } from './replication-status.ts';
+import type { ReplicationLagReadinessOptions } from './reporter/recorder.ts';
 import type { WriteWorkerClient } from './write-worker-client.ts';
 /** See {@link ReplicaStateNotifier.subscribe()}. */
 export type ReplicaState = {
@@ -71,10 +74,11 @@ export type ReplicatorMode = 'backup' | 'serving';
 export declare class ReplicatorService implements Replicator, Service {
     #private;
     readonly id: string;
-    constructor(lc: LogContext, taskID: string, id: string, mode: ReplicatorMode, changeStreamer: ChangeStreamer, worker: WriteWorkerClient, statusPublisher: ReplicationStatusPublisher | null);
+    constructor(lc: LogContext, taskID: string, id: string, mode: ReplicatorMode, changeStreamer: ChangeStreamer, worker: WriteWorkerClient, statusPublisher: ReplicationStatusPublisher | null, replicationLagReadinessOptions?: ReplicationLagReadinessOptions, initialConnection?: ReservationFollowup, switchover?: SwitchoverOptions);
     status(): Promise<{
         status: string;
     }>;
+    ready(): Promise<void>;
     run(): Promise<void>;
     subscribe(): Source<ReplicaState>;
     stop(): Promise<void>;

package/out/zero-cache/src/services/replicator/replicator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"replicator.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/replicator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,gCAAgC,CAAC;AACvE,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,uCAAuC,CAAC;AAC1E,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAE3C,OAAO,EAEL,KAAK,0BAA0B,EAChC,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAEhE,oDAAoD;AACpD,MAAM,MAAM,YAAY,GAAG;IACzB,QAAQ,CAAC,KAAK,EAAE,eAAe,CAAC;IAChC;;;;OAIG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC;;OAEG;IACH,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAEjD;;;;;;;;;;;;;OAaG;IACH,QAAQ,CAAC,oBAAoB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAInD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1C,CAAC;AAEF,MAAM,WAAW,oBAAoB;IACnC;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,UAAW,SAAQ,oBAAoB;IACtD;;;;OAIG;IACH,MAAM,IAAI,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACvC;AAED,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,SAAS,CAAC;AAElD,qBAAa,iBAAkB,YAAW,UAAU,EAAE,OAAO;;IAC3D,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAOpB,YACE,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,MAAM,EACV,IAAI,EAAE,cAAc,EACpB,cAAc,EAAE,cAAc,EAC9B,MAAM,EAAE,iBAAiB,EACzB,eAAe,EAAE,0BAA0B,GAAG,IAAI,EAkBnD;IAED,MAAM;;OAEL;IAED,GAAG,kBAYF;IAED,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC,CAEhC;IAEK,IAAI,kBAOT;CACF"}
+{"version":3,"file":"replicator.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/replicator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,gCAAgC,CAAC;AACvE,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,4CAA4C,CAAC;AACpF,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,uCAAuC,CAAC;AAC1E,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,oBAAoB,CAAC;AAE1D,OAAO,EAEL,KAAK,0BAA0B,EAChC,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAC,8BAA8B,EAAC,MAAM,wBAAwB,CAAC;AAC3E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAEhE,oDAAoD;AACpD,MAAM,MAAM,YAAY,GAAG;IACzB,QAAQ,CAAC,KAAK,EAAE,eAAe,CAAC;IAChC;;;;OAIG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC;;OAEG;IACH,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAEjD;;;;;;;;;;;;;OAaG;IACH,QAAQ,CAAC,oBAAoB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAInD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1C,CAAC;AAEF,MAAM,WAAW,oBAAoB;IACnC;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,UAAW,SAAQ,oBAAoB;IACtD;;;;OAIG;IACH,MAAM,IAAI,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACvC;AAED,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,SAAS,CAAC;AAElD,qBAAa,iBAAkB,YAAW,UAAU,EAAE,OAAO;;IAC3D,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAOpB,YACE,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,MAAM,EACV,IAAI,EAAE,cAAc,EACpB,cAAc,EAAE,cAAc,EAC9B,MAAM,EAAE,iBAAiB,EACzB,eAAe,EAAE,0BAA0B,GAAG,IAAI,EAClD,8BAA8B,GAAE,8BAAmC,EAKnE,iBAAiB,CAAC,EAAE,mBAAmB,EACvC,UAAU,CAAC,EAAE,iBAAiB,EAqB/B;IAED,MAAM;;OAEL;IAED,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAErB;IAED,GAAG,kBAYF;IAED,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC,CAEhC;IAEK,IAAI,kBAOT;CACF"}

package/out/zero-cache/src/services/replicator/replicator.js

@@ -8,16 +8,19 @@ var ReplicatorService = class {
 	#shouldPublishErrors;
 	#worker;
 	#runPromise;
-	constructor(lc, taskID, id, mode, changeStreamer, worker, statusPublisher) {
+	constructor(lc, taskID, id, mode, changeStreamer, worker, statusPublisher, replicationLagReadinessOptions = {}, initialConnection, switchover) {
 		this.id = id;
 		this.#lc = lc.withContext("component", "replicator").withContext("serviceID", this.id);
 		this.#worker = worker;
 		this.#shouldPublishErrors = statusPublisher !== null;
-		this.#incrementalSyncer = new IncrementalSyncer(this.#lc, taskID, `${taskID}/${id}`, changeStreamer, worker, mode, statusPublisher);
+		this.#incrementalSyncer = new IncrementalSyncer(this.#lc, taskID, `${taskID}/${id}`, changeStreamer, worker, mode, statusPublisher, replicationLagReadinessOptions, initialConnection, switchover);
 	}
 	status() {
 		return Promise.resolve({ status: "ok" });
 	}
+	ready() {
+		return this.#incrementalSyncer.ready();
+	}
 	run() {
 		this.#runPromise = this.#incrementalSyncer.run().catch(async (error) => {
 			if (this.#shouldPublishErrors) await publishReplicationError(this.#lc, "Replicating", "Replication stopped because the replica writer failed");

package/out/zero-cache/src/services/replicator/replicator.js.map

@@ -1 +1 @@
-{"version":3,"file":"replicator.js","names":["#lc","#incrementalSyncer","#shouldPublishErrors","#worker","#runPromise"],"sources":["../../../../../../zero-cache/src/services/replicator/replicator.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {ReadonlyJSONObject} from '../../../../shared/src/json.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {ChangeStreamer} from '../change-streamer/change-streamer.ts';\nimport type {Service} from '../service.ts';\nimport {IncrementalSyncer} from './incremental-sync.ts';\nimport {\n  publishReplicationError,\n  type ReplicationStatusPublisher,\n} from './replication-status.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\n/** See {@link ReplicaStateNotifier.subscribe()}. */\nexport type ReplicaState = {\n  readonly state: 'version-ready';\n  /**\n   * The replica watermark that became ready. Omitted for the initial\n   * \"current state is ready\" notification, which should not count as newly\n   * unserved work for a freshly-started ViewSyncer.\n   */\n  readonly watermark?: string | undefined;\n  /**\n   * Millisecond epoch when `watermark` became ready to read from the replica.\n   */\n  readonly replicaReadyTimeMs?: number | undefined;\n\n  /**\n   * Millisecond epoch at which the transaction behind `watermark` committed\n   * *upstream*, as reported by the ChangeSource. Unlike `replicaReadyTimeMs`,\n   * which is stamped locally when the change lands in the replica, this is the\n   * origin of the whole pipeline, so `now - upstreamCommitTimeMs` at the point\n   * the change is poked to clients is the end-to-end serving lag.\n   *\n   * Note that this crosses a clock domain: it comes from the upstream\n   * database, whereas it is subtracted from the local clock in the ViewSyncer.\n   * The existing `zero.replication.total_lag` metric spans the same boundary.\n   *\n   * Undefined when the ChangeSource does not report a commit time, and for the\n   * initial \"current state is ready\" notification.\n   */\n  readonly upstreamCommitTimeMs?: number | undefined;\n\n  // Used in tests to verify behavior when additional information\n  // is ferried in the future. Not set in production.\n  readonly testSeqNum?: number | undefined;\n};\n\nexport interface ReplicaStateNotifier {\n  /**\n   * Creates a cancelable subscription of changes in the replica state.\n   *\n   * A `version-ready` message indicates that the replica is ready to be\n   * read, and henceforth that a _new_ version is ready, i.e. whenever a\n   * change is committed to the replica. The `version-ready` message itself\n   * otherwise contains no other information; the subscriber queries the\n   * replica for the current data.\n   *\n   * A `maintenance` state indicates that the replica should not be read from.\n   * If a subscriber is holding any transaction locks, it should release them\n   * until the next `version-ready` signal.\n   *\n   * Upon subscription, the current state of the replica is sent immediately\n   * if known. If multiple notifications occur before the subscriber\n   * can consume them, all but the last notification are discarded by the\n   * Subscription object (i.e. not buffered). Thus, a subscriber only\n   * ever consumes the current (i.e. known) state of the replica. This avoids\n   * a buildup of \"work\" if a subscriber is too busy to consume all\n   * notifications.\n   */\n  subscribe(): Source<ReplicaState>;\n}\n\nexport interface Replicator extends ReplicaStateNotifier {\n  /**\n   * Returns an opaque message for human-readable consumption. This is\n   * purely for ensuring that the Replicator has started at least once to\n   * bootstrap a new replica.\n   */\n  status(): Promise<ReadonlyJSONObject>;\n}\n\nexport type ReplicatorMode = 'backup' | 'serving';\n\nexport class ReplicatorService implements Replicator, Service {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #incrementalSyncer: IncrementalSyncer;\n  readonly #shouldPublishErrors: boolean;\n  readonly #worker: WriteWorkerClient;\n  #runPromise: Promise<void> | undefined;\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    id: string,\n    mode: ReplicatorMode,\n    changeStreamer: ChangeStreamer,\n    worker: WriteWorkerClient,\n    statusPublisher: ReplicationStatusPublisher | null,\n  ) {\n    this.id = id;\n    this.#lc = lc\n      .withContext('component', 'replicator')\n      .withContext('serviceID', this.id);\n    this.#worker = worker;\n    this.#shouldPublishErrors = statusPublisher !== null;\n\n    this.#incrementalSyncer = new IncrementalSyncer(\n      this.#lc,\n      taskID,\n      `${taskID}/${id}`,\n      changeStreamer,\n      worker,\n      mode,\n      statusPublisher,\n    );\n  }\n\n  status() {\n    return Promise.resolve({status: 'ok'});\n  }\n\n  run() {\n    this.#runPromise = this.#incrementalSyncer.run().catch(async error => {\n      if (this.#shouldPublishErrors) {\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          'Replication stopped because the replica writer failed',\n        );\n      }\n      throw error;\n    });\n    return this.#runPromise;\n  }\n\n  subscribe(): Source<ReplicaState> {\n    return this.#incrementalSyncer.subscribe();\n  }\n\n  async stop() {\n    this.#incrementalSyncer.stop(this.#lc);\n    // Wait for the syncer's run loop to finish so that any in-flight\n    // worker.processMessages() call completes and clears #pending\n    // before we send the 'stop' message to the worker.\n    await this.#runPromise;\n    await this.#worker.stop();\n  }\n}\n"],"mappings":";;;AAmFA,IAAa,oBAAb,MAA8D;CAC5D;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,IACA,QACA,IACA,MACA,gBACA,QACA,iBACA;EACA,KAAK,KAAK;EACV,KAAKA,MAAM,GACR,YAAY,aAAa,YAAY,EACrC,YAAY,aAAa,KAAK,EAAE;EACnC,KAAKG,UAAU;EACf,KAAKD,uBAAuB,oBAAoB;EAEhD,KAAKD,qBAAqB,IAAI,kBAC5B,KAAKD,KACL,QACA,GAAG,OAAO,GAAG,MACb,gBACA,QACA,MACA,eACF;CACF;CAEA,SAAS;EACP,OAAO,QAAQ,QAAQ,EAAC,QAAQ,KAAI,CAAC;CACvC;CAEA,MAAM;EACJ,KAAKI,cAAc,KAAKH,mBAAmB,IAAI,EAAE,MAAM,OAAM,UAAS;GACpE,IAAI,KAAKC,sBACP,MAAM,wBACJ,KAAKF,KACL,eACA,uDACF;GAEF,MAAM;EACR,CAAC;EACD,OAAO,KAAKI;CACd;CAEA,YAAkC;EAChC,OAAO,KAAKH,mBAAmB,UAAU;CAC3C;CAEA,MAAM,OAAO;EACX,KAAKA,mBAAmB,KAAK,KAAKD,GAAG;EAIrC,MAAM,KAAKI;EACX,MAAM,KAAKD,QAAQ,KAAK;CAC1B;AACF"}
+{"version":3,"file":"replicator.js","names":["#lc","#incrementalSyncer","#shouldPublishErrors","#worker","#runPromise"],"sources":["../../../../../../zero-cache/src/services/replicator/replicator.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {ReadonlyJSONObject} from '../../../../shared/src/json.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {ReservationFollowup} from '../change-streamer/change-streamer-http.ts';\nimport type {ChangeStreamer} from '../change-streamer/change-streamer.ts';\nimport type {Service} from '../service.ts';\nimport type {SwitchoverOptions} from './drain-handler.ts';\nimport {IncrementalSyncer} from './incremental-sync.ts';\nimport {\n  publishReplicationError,\n  type ReplicationStatusPublisher,\n} from './replication-status.ts';\nimport type {ReplicationLagReadinessOptions} from './reporter/recorder.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\n/** See {@link ReplicaStateNotifier.subscribe()}. */\nexport type ReplicaState = {\n  readonly state: 'version-ready';\n  /**\n   * The replica watermark that became ready. Omitted for the initial\n   * \"current state is ready\" notification, which should not count as newly\n   * unserved work for a freshly-started ViewSyncer.\n   */\n  readonly watermark?: string | undefined;\n  /**\n   * Millisecond epoch when `watermark` became ready to read from the replica.\n   */\n  readonly replicaReadyTimeMs?: number | undefined;\n\n  /**\n   * Millisecond epoch at which the transaction behind `watermark` committed\n   * *upstream*, as reported by the ChangeSource. Unlike `replicaReadyTimeMs`,\n   * which is stamped locally when the change lands in the replica, this is the\n   * origin of the whole pipeline, so `now - upstreamCommitTimeMs` at the point\n   * the change is poked to clients is the end-to-end serving lag.\n   *\n   * Note that this crosses a clock domain: it comes from the upstream\n   * database, whereas it is subtracted from the local clock in the ViewSyncer.\n   * The existing `zero.replication.total_lag` metric spans the same boundary.\n   *\n   * Undefined when the ChangeSource does not report a commit time, and for the\n   * initial \"current state is ready\" notification.\n   */\n  readonly upstreamCommitTimeMs?: number | undefined;\n\n  // Used in tests to verify behavior when additional information\n  // is ferried in the future. Not set in production.\n  readonly testSeqNum?: number | undefined;\n};\n\nexport interface ReplicaStateNotifier {\n  /**\n   * Creates a cancelable subscription of changes in the replica state.\n   *\n   * A `version-ready` message indicates that the replica is ready to be\n   * read, and henceforth that a _new_ version is ready, i.e. whenever a\n   * change is committed to the replica. The `version-ready` message itself\n   * otherwise contains no other information; the subscriber queries the\n   * replica for the current data.\n   *\n   * A `maintenance` state indicates that the replica should not be read from.\n   * If a subscriber is holding any transaction locks, it should release them\n   * until the next `version-ready` signal.\n   *\n   * Upon subscription, the current state of the replica is sent immediately\n   * if known. If multiple notifications occur before the subscriber\n   * can consume them, all but the last notification are discarded by the\n   * Subscription object (i.e. not buffered). Thus, a subscriber only\n   * ever consumes the current (i.e. known) state of the replica. This avoids\n   * a buildup of \"work\" if a subscriber is too busy to consume all\n   * notifications.\n   */\n  subscribe(): Source<ReplicaState>;\n}\n\nexport interface Replicator extends ReplicaStateNotifier {\n  /**\n   * Returns an opaque message for human-readable consumption. This is\n   * purely for ensuring that the Replicator has started at least once to\n   * bootstrap a new replica.\n   */\n  status(): Promise<ReadonlyJSONObject>;\n}\n\nexport type ReplicatorMode = 'backup' | 'serving';\n\nexport class ReplicatorService implements Replicator, Service {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #incrementalSyncer: IncrementalSyncer;\n  readonly #shouldPublishErrors: boolean;\n  readonly #worker: WriteWorkerClient;\n  #runPromise: Promise<void> | undefined;\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    id: string,\n    mode: ReplicatorMode,\n    changeStreamer: ChangeStreamer,\n    worker: WriteWorkerClient,\n    statusPublisher: ReplicationStatusPublisher | null,\n    replicationLagReadinessOptions: ReplicationLagReadinessOptions = {},\n    // The still-open connection from the snapshot reservation that produced\n    // the replica this process just restored (serving mode only). Used for\n    // IncrementalSyncer's first subscribe() so the replication-manager that\n    // reserved the change log is the one that serves the subscription.\n    initialConnection?: ReservationFollowup,\n    switchover?: SwitchoverOptions,\n  ) {\n    this.id = id;\n    this.#lc = lc\n      .withContext('component', 'replicator')\n      .withContext('serviceID', this.id);\n    this.#worker = worker;\n    this.#shouldPublishErrors = statusPublisher !== null;\n\n    this.#incrementalSyncer = new IncrementalSyncer(\n      this.#lc,\n      taskID,\n      `${taskID}/${id}`,\n      changeStreamer,\n      worker,\n      mode,\n      statusPublisher,\n      replicationLagReadinessOptions,\n      initialConnection,\n      switchover,\n    );\n  }\n\n  status() {\n    return Promise.resolve({status: 'ok'});\n  }\n\n  ready(): Promise<void> {\n    return this.#incrementalSyncer.ready();\n  }\n\n  run() {\n    this.#runPromise = this.#incrementalSyncer.run().catch(async error => {\n      if (this.#shouldPublishErrors) {\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          'Replication stopped because the replica writer failed',\n        );\n      }\n      throw error;\n    });\n    return this.#runPromise;\n  }\n\n  subscribe(): Source<ReplicaState> {\n    return this.#incrementalSyncer.subscribe();\n  }\n\n  async stop() {\n    this.#incrementalSyncer.stop(this.#lc);\n    // Wait for the syncer's run loop to finish so that any in-flight\n    // worker.processMessages() call completes and clears #pending\n    // before we send the 'stop' message to the worker.\n    await this.#runPromise;\n    await this.#worker.stop();\n  }\n}\n"],"mappings":";;;AAsFA,IAAa,oBAAb,MAA8D;CAC5D;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,IACA,QACA,IACA,MACA,gBACA,QACA,iBACA,iCAAiE,CAAC,GAKlE,mBACA,YACA;EACA,KAAK,KAAK;EACV,KAAKA,MAAM,GACR,YAAY,aAAa,YAAY,EACrC,YAAY,aAAa,KAAK,EAAE;EACnC,KAAKG,UAAU;EACf,KAAKD,uBAAuB,oBAAoB;EAEhD,KAAKD,qBAAqB,IAAI,kBAC5B,KAAKD,KACL,QACA,GAAG,OAAO,GAAG,MACb,gBACA,QACA,MACA,iBACA,gCACA,mBACA,UACF;CACF;CAEA,SAAS;EACP,OAAO,QAAQ,QAAQ,EAAC,QAAQ,KAAI,CAAC;CACvC;CAEA,QAAuB;EACrB,OAAO,KAAKC,mBAAmB,MAAM;CACvC;CAEA,MAAM;EACJ,KAAKG,cAAc,KAAKH,mBAAmB,IAAI,EAAE,MAAM,OAAM,UAAS;GACpE,IAAI,KAAKC,sBACP,MAAM,wBACJ,KAAKF,KACL,eACA,uDACF;GAEF,MAAM;EACR,CAAC;EACD,OAAO,KAAKI;CACd;CAEA,YAAkC;EAChC,OAAO,KAAKH,mBAAmB,UAAU;CAC3C;CAEA,MAAM,OAAO;EACX,KAAKA,mBAAmB,KAAK,KAAKD,GAAG;EAIrC,MAAM,KAAKI;EACX,MAAM,KAAKD,QAAQ,KAAK;CAC1B;AACF"}

package/out/zero-cache/src/services/replicator/reporter/recorder.d.ts

@@ -1,9 +1,16 @@
 import type { ObservableResult } from '@opentelemetry/api';
 import type { LogContext } from '@rocicorp/logger';
 import type { ReplicationReport } from './report-schema.ts';
+export type ReplicationLagReadinessOptions = {
+    readinessThresholdMs?: number | undefined;
+    maxReadinessDelayMs?: number | undefined;
+    setTimeoutFn?: typeof setTimeout | undefined;
+    clearTimeoutFn?: typeof clearTimeout | undefined;
+};
 export declare class ReplicationReportRecorder {
     #private;
-    constructor(lc: LogContext);
+    constructor(lc: LogContext, { readinessThresholdMs, maxReadinessDelayMs, setTimeoutFn, clearTimeoutFn, }?: ReplicationLagReadinessOptions);
+    ready(): Promise<void>;
     record(report: ReplicationReport): void;
     readonly reportUpstreamLag: (o: ObservableResult) => void;
     readonly reportReplicaLag: (o: ObservableResult) => void;
@@ -37,4 +44,12 @@ export declare function estimateUpstreamClockSkewMs({ sendTimeMs, commitTimeMs,
     commitTimeMs: number;
     receiveTimeMs: number;
 }): number;
+/**
+ * Returns whether the `report` is the signal sent by the change-streamer,
+ * once the subscriber is caught up, to indicate that lag reporting is
+ * disabled upstream (and thus readiness cannot be gated on lag reports).
+ *
+ * This is a lag report with a `nextSendTimeMs` of 0 and no `lastTimings`.
+ */
+export declare function isLagReportingDisabledSignal(report: ReplicationReport): boolean;
 //# sourceMappingURL=recorder.d.ts.map

package/out/zero-cache/src/services/replicator/reporter/recorder.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"recorder.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/replicator/reporter/recorder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,oBAAoB,CAAC;AACzD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,oBAAoB,CAAC;AAM1D,qBAAa,yBAAyB;;IAIpC,YAAY,EAAE,EAAE,UAAU,EAEzB;IAED,MAAM,CAAC,MAAM,EAAE,iBAAiB,QAgE/B;IAED,QAAQ,CAAC,iBAAiB,MAAO,gBAAgB,UAK/C;IAEF,QAAQ,CAAC,gBAAgB,MAAO,gBAAgB,UAK9C;IAEF,QAAQ,CAAC,cAAc,MAAO,gBAAgB,UAK5C;IAEF,QAAQ,CAAC,kBAAkB,MAAO,gBAAgB,UAKhD;IAEF,QAAQ,CAAC,uBAAuB,MAAO,gBAAgB,UAKrD;CACH;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,2BAA2B,CAAC,EAC1C,UAAU,EACV,YAAY,EACZ,aAAa,GACd,EAAE;IACD,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;CACvB,GAAG,MAAM,CAET"}
+{"version":3,"file":"recorder.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/replicator/reporter/recorder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,oBAAoB,CAAC;AACzD,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAG3D,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,oBAAoB,CAAC;AAM1D,MAAM,MAAM,8BAA8B,GAAG;IAC3C,oBAAoB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1C,mBAAmB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC,YAAY,CAAC,EAAE,OAAO,UAAU,GAAG,SAAS,CAAC;IAC7C,cAAc,CAAC,EAAE,OAAO,YAAY,GAAG,SAAS,CAAC;CAClD,CAAC;AAEF,qBAAa,yBAAyB;;IASpC,YACE,EAAE,EAAE,UAAU,EACd,EACE,oBAA2B,EAC3B,mBAA6B,EAC7B,YAAyB,EACzB,cAA6B,GAC9B,GAAE,8BAAmC,EAgBvC;IAED,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAErB;IAED,MAAM,CAAC,MAAM,EAAE,iBAAiB,QAsC/B;IA0DD,QAAQ,CAAC,iBAAiB,MAAO,gBAAgB,UAK/C;IAEF,QAAQ,CAAC,gBAAgB,MAAO,gBAAgB,UAK9C;IAEF,QAAQ,CAAC,cAAc,MAAO,gBAAgB,UAK5C;IAEF,QAAQ,CAAC,kBAAkB,MAAO,gBAAgB,UAKhD;IAEF,QAAQ,CAAC,uBAAuB,MAAO,gBAAgB,UAKrD;CACH;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,2BAA2B,CAAC,EAC1C,UAAU,EACV,YAAY,EACZ,aAAa,GACd,EAAE;IACD,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;CACvB,GAAG,MAAM,CAET;AAED;;;;;;GAMG;AACH,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,iBAAiB,GACxB,OAAO,CAET"}

package/out/zero-cache/src/services/replicator/reporter/recorder.js

@@ -1,45 +1,78 @@
 import { getOrCreateGauge } from "../../../observability/metrics.js";
+import { resolver } from "@rocicorp/resolver";
 //#region ../zero-cache/src/services/replicator/reporter/recorder.ts
 var LOG_ALL_REPLICATION_REPORTS_AT_DEBUG = process.env.ZERO_LOG_ALL_REPLICATION_REPORTS_AT_DEBUG === "1";
 var ReplicationReportRecorder = class {
 	#lc;
+	#readinessThresholdMs;
+	#readyGate = resolver();
+	#clearTimeout;
+	#readinessTimer;
+	#ready = false;
 	#last = null;
-	constructor(lc) {
+	constructor(lc, { readinessThresholdMs = 1e3, maxReadinessDelayMs = 6e5, setTimeoutFn = setTimeout, clearTimeoutFn = clearTimeout } = {}) {
 		this.#lc = lc;
+		this.#readinessThresholdMs = readinessThresholdMs;
+		this.#clearTimeout = clearTimeoutFn;
+		this.#readinessTimer = setTimeoutFn(() => this.#setReady("warn", `timed out waiting for replication lag readiness threshold of ${this.#readinessThresholdMs} ms`), maxReadinessDelayMs);
+		this.#readinessTimer?.unref?.();
+	}
+	ready() {
+		return this.#readyGate.promise;
 	}
 	record(report) {
-		const first = this.#last === null;
+		if (isLagReportingDisabledSignal(report)) {
+			this.#setReady("info", `replica is caught up (replication lag reporting is disabled upstream)`);
+			return;
+		}
 		this.#last = report;
 		const { lastTimings } = report;
 		if (lastTimings) {
 			const total = lastTimings.replicateTimeMs - lastTimings.sendTimeMs;
-			if (total > 1e4) this.#lc.warn?.(`high replication lag: ${total} ms`, report);
-			else if (total > 1e3) this.#lc.info?.(`replication lag: ${total} ms`, report);
-			if (LOG_ALL_REPLICATION_REPORTS_AT_DEBUG) this.#lc.debug?.(`replication lag ${total} ms`, report);
-		}
-		if (first) {
-			getOrCreateGauge("replication", "upstream_lag", {
-				description: "Latency from sending an upstream replication report to receiving it in the replication stream",
-				unit: "millisecond"
-			}).addCallback(this.reportUpstreamLag);
-			getOrCreateGauge("replication", "replica_lag", {
-				description: "Latency from receiving an upstream replication report to its reaching the replica",
-				unit: "millisecond"
-			}).addCallback(this.reportReplicaLag);
-			getOrCreateGauge("replication", "total_lag", {
-				description: "Latency from sending an upstream replication report to its reaching the replica. This reflects the actual measured round-trip of the most recently received report and does not grow when reports stop arriving.",
-				unit: "millisecond"
-			}).addCallback(this.reportTotalLag);
-			getOrCreateGauge("replication", "last_total_lag", {
-				description: "Latency from sending the most recently received upstream replication report to its reaching the replica. This is an alias of replication.total_lag retained for dashboards that explicitly want the non-extrapolated value.",
-				unit: "millisecond"
-			}).addCallback(this.reportLastTotalLag);
-			getOrCreateGauge("replication", "upstream_clock_skew", {
-				description: "Estimated offset of the upstream database clock relative to zero-cache's. Positive means upstream is running ahead. Derived from the lag report round-trip, whose send and receive times are stamped on our clock either side of the upstream commit timestamp. This matters because sync.e2e_serving_lag subtracts an upstream commit timestamp from a local clock, so a skewed upstream clock biases it directly -- and an upstream clock running ahead biases it *low*, which looks like healthy serving rather than a broken measurement.",
-				unit: "millisecond"
-			}).addCallback(this.reportUpstreamClockSkew);
+			if (total > 1e4) {
+				const log = this.#ready ? "warn" : "info";
+				this.#lc[log]?.(`high replication lag: ${total} ms`, {
+					ready: this.#ready,
+					report
+				});
+			} else if (total > 1e3) this.#lc.info?.(`replication lag: ${total} ms`, {
+				ready: this.#ready,
+				report
+			});
+			if (LOG_ALL_REPLICATION_REPORTS_AT_DEBUG) this.#lc.debug?.(`replication lag ${total} ms`, {
+				ready: this.#ready,
+				report
+			});
+			if (!this.#ready && total <= this.#readinessThresholdMs) this.#setReady("info", `replicator is now ready with replication lag (${total} ms) less than ${this.#readinessThresholdMs} ms readiness threshold`);
 		}
 	}
+	#setReady(log, message) {
+		if (this.#ready) return;
+		this.#lc[log]?.(message, { lastReport: this.#last });
+		this.#ready = true;
+		this.#clearTimeout(this.#readinessTimer);
+		this.#readyGate.resolve();
+		getOrCreateGauge("replication", "upstream_lag", {
+			description: "Latency from sending an upstream replication report to receiving it in the replication stream",
+			unit: "millisecond"
+		}).addCallback(this.reportUpstreamLag);
+		getOrCreateGauge("replication", "replica_lag", {
+			description: "Latency from receiving an upstream replication report to its reaching the replica",
+			unit: "millisecond"
+		}).addCallback(this.reportReplicaLag);
+		getOrCreateGauge("replication", "total_lag", {
+			description: "Latency from sending an upstream replication report to its reaching the replica. This reflects the actual measured round-trip of the most recently received report and does not grow when reports stop arriving.",
+			unit: "millisecond"
+		}).addCallback(this.reportTotalLag);
+		getOrCreateGauge("replication", "last_total_lag", {
+			description: "Latency from sending the most recently received upstream replication report to its reaching the replica. This is an alias of replication.total_lag retained for dashboards that explicitly want the non-extrapolated value.",
+			unit: "millisecond"
+		}).addCallback(this.reportLastTotalLag);
+		getOrCreateGauge("replication", "upstream_clock_skew", {
+			description: "Estimated offset of the upstream database clock relative to zero-cache's. Positive means upstream is running ahead. Derived from the lag report round-trip, whose send and receive times are stamped on our clock either side of the upstream commit timestamp. This matters because sync.e2e_serving_lag subtracts an upstream commit timestamp from a local clock, so a skewed upstream clock biases it directly -- and an upstream clock running ahead biases it *low*, which looks like healthy serving rather than a broken measurement.",
+			unit: "millisecond"
+		}).addCallback(this.reportUpstreamClockSkew);
+	}
 	reportUpstreamLag = (o) => {
 		const last = this.#last?.lastTimings;
 		if (last) o.observe(last.receiveTimeMs - last.sendTimeMs);
@@ -85,6 +118,16 @@ var ReplicationReportRecorder = class {
 function estimateUpstreamClockSkewMs({ sendTimeMs, commitTimeMs, receiveTimeMs }) {
 	return commitTimeMs - (sendTimeMs + receiveTimeMs) / 2;
 }
+/**
+* Returns whether the `report` is the signal sent by the change-streamer,
+* once the subscriber is caught up, to indicate that lag reporting is
+* disabled upstream (and thus readiness cannot be gated on lag reports).
+*
+* This is a lag report with a `nextSendTimeMs` of 0 and no `lastTimings`.
+*/
+function isLagReportingDisabledSignal(report) {
+	return report.nextSendTimeMs === 0 && report.lastTimings === void 0;
+}
 //#endregion
 export { ReplicationReportRecorder };
 

package/out/zero-cache/src/services/replicator/reporter/recorder.js.map

@@ -1 +1 @@
-{"version":3,"file":"recorder.js","names":["#lc","#last"],"sources":["../../../../../../../zero-cache/src/services/replicator/reporter/recorder.ts"],"sourcesContent":["import type {ObservableResult} from '@opentelemetry/api';\nimport type {LogContext} from '@rocicorp/logger';\nimport {getOrCreateGauge} from '../../../observability/metrics.ts';\nimport type {ReplicationReport} from './report-schema.ts';\n\n// Hook for sanity checking lag reports in development.\nconst LOG_ALL_REPLICATION_REPORTS_AT_DEBUG =\n  process.env.ZERO_LOG_ALL_REPLICATION_REPORTS_AT_DEBUG === '1';\n\nexport class ReplicationReportRecorder {\n  readonly #lc: LogContext;\n  #last: ReplicationReport | null = null;\n\n  constructor(lc: LogContext) {\n    this.#lc = lc;\n  }\n\n  record(report: ReplicationReport) {\n    const first = this.#last === null;\n    this.#last = report;\n\n    const {lastTimings} = report;\n    if (lastTimings) {\n      const total = lastTimings.replicateTimeMs - lastTimings.sendTimeMs;\n      if (total > 10_000) {\n        this.#lc.warn?.(`high replication lag: ${total} ms`, report);\n      } else if (total > 1_000) {\n        this.#lc.info?.(`replication lag: ${total} ms`, report);\n      }\n      if (LOG_ALL_REPLICATION_REPORTS_AT_DEBUG) {\n        this.#lc.debug?.(`replication lag ${total} ms`, report);\n      }\n    }\n\n    if (first) {\n      getOrCreateGauge('replication', 'upstream_lag', {\n        description:\n          'Latency from sending an upstream replication report ' +\n          'to receiving it in the replication stream',\n        unit: 'millisecond',\n      }).addCallback(this.reportUpstreamLag);\n\n      getOrCreateGauge('replication', 'replica_lag', {\n        description:\n          'Latency from receiving an upstream replication report ' +\n          'to its reaching the replica',\n        unit: 'millisecond',\n      }).addCallback(this.reportReplicaLag);\n\n      getOrCreateGauge('replication', 'total_lag', {\n        description:\n          'Latency from sending an upstream replication report to its ' +\n          'reaching the replica. This reflects the actual measured ' +\n          'round-trip of the most recently received report and does not ' +\n          'grow when reports stop arriving.',\n        unit: 'millisecond',\n      }).addCallback(this.reportTotalLag);\n\n      getOrCreateGauge('replication', 'last_total_lag', {\n        description:\n          'Latency from sending the most recently received upstream ' +\n          'replication report to its reaching the replica. This is an alias ' +\n          'of replication.total_lag retained for dashboards that explicitly ' +\n          'want the non-extrapolated value.',\n        unit: 'millisecond',\n      }).addCallback(this.reportLastTotalLag);\n\n      getOrCreateGauge('replication', 'upstream_clock_skew', {\n        description:\n          'Estimated offset of the upstream database clock relative to ' +\n          \"zero-cache's. Positive means upstream is running ahead. \" +\n          'Derived from the lag report round-trip, whose send and receive ' +\n          'times are stamped on our clock either side of the upstream ' +\n          'commit timestamp. This matters because ' +\n          'sync.e2e_serving_lag subtracts an upstream commit timestamp from ' +\n          'a local clock, so a skewed upstream clock biases it directly -- ' +\n          'and an upstream clock running ahead biases it *low*, which looks ' +\n          'like healthy serving rather than a broken measurement.',\n        unit: 'millisecond',\n      }).addCallback(this.reportUpstreamClockSkew);\n    }\n  }\n\n  readonly reportUpstreamLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.receiveTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportReplicaLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.receiveTimeMs);\n    }\n  };\n\n  readonly reportTotalLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportLastTotalLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportUpstreamClockSkew = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(estimateUpstreamClockSkewMs(last));\n    }\n  };\n}\n\n/**\n * Estimates how far the upstream database's clock is from ours, in\n * milliseconds, positive meaning upstream is ahead.\n *\n * `sendTimeMs` and `receiveTimeMs` are both stamped in the\n * replication-manager, either side of `commitTimeMs`, which upstream stamps.\n * Assuming the upstream commit fell midway through that round trip, the\n * difference between where it actually landed and that midpoint is the clock\n * offset -- the same round-trip estimate NTP uses.\n *\n * The estimate is biased by any asymmetry between the two legs: the outbound\n * leg is a SQL round trip, while the return leg additionally waits on a WAL\n * flush and the walsender (which the pre-PG-17 path notes can add 50~100ms on\n * an idle database). So treat this as a detector of gross skew -- the kind\n * that invalidates sync.e2e_serving_lag -- and not as a sub-100ms correction.\n *\n * The rigorous bound, if a tighter reading is ever needed: the commit\n * genuinely happened between send and receive, so a `commitTimeMs` outside\n * that window proves skew of at least the amount by which it falls outside,\n * independent of any latency assumption.\n */\nexport function estimateUpstreamClockSkewMs({\n  sendTimeMs,\n  commitTimeMs,\n  receiveTimeMs,\n}: {\n  sendTimeMs: number;\n  commitTimeMs: number;\n  receiveTimeMs: number;\n}): number {\n  return commitTimeMs - (sendTimeMs + receiveTimeMs) / 2;\n}\n"],"mappings":";;AAMA,IAAM,uCACJ,QAAQ,IAAI,8CAA8C;AAE5D,IAAa,4BAAb,MAAuC;CACrC;CACA,QAAkC;CAElC,YAAY,IAAgB;EAC1B,KAAKA,MAAM;CACb;CAEA,OAAO,QAA2B;EAChC,MAAM,QAAQ,KAAKC,UAAU;EAC7B,KAAKA,QAAQ;EAEb,MAAM,EAAC,gBAAe;EACtB,IAAI,aAAa;GACf,MAAM,QAAQ,YAAY,kBAAkB,YAAY;GACxD,IAAI,QAAQ,KACV,KAAKD,IAAI,OAAO,yBAAyB,MAAM,MAAM,MAAM;QACtD,IAAI,QAAQ,KACjB,KAAKA,IAAI,OAAO,oBAAoB,MAAM,MAAM,MAAM;GAExD,IAAI,sCACF,KAAKA,IAAI,QAAQ,mBAAmB,MAAM,MAAM,MAAM;EAE1D;EAEA,IAAI,OAAO;GACT,iBAAiB,eAAe,gBAAgB;IAC9C,aACE;IAEF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,iBAAiB;GAErC,iBAAiB,eAAe,eAAe;IAC7C,aACE;IAEF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,gBAAgB;GAEpC,iBAAiB,eAAe,aAAa;IAC3C,aACE;IAIF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,cAAc;GAElC,iBAAiB,eAAe,kBAAkB;IAChD,aACE;IAIF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,kBAAkB;GAEtC,iBAAiB,eAAe,uBAAuB;IACrD,aACE;IASF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,uBAAuB;EAC7C;CACF;CAEA,qBAA8B,MAAwB;EACpD,MAAM,OAAO,KAAKC,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,gBAAgB,KAAK,UAAU;CAElD;CAEA,oBAA6B,MAAwB;EACnD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,aAAa;CAEvD;CAEA,kBAA2B,MAAwB;EACjD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,UAAU;CAEpD;CAEA,sBAA+B,MAAwB;EACrD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,UAAU;CAEpD;CAEA,2BAAoC,MAAwB;EAC1D,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,4BAA4B,IAAI,CAAC;CAE/C;AACF;;;;;;;;;;;;;;;;;;;;;;AAuBA,SAAgB,4BAA4B,EAC1C,YACA,cACA,iBAKS;CACT,OAAO,gBAAgB,aAAa,iBAAiB;AACvD"}
+{"version":3,"file":"recorder.js","names":["#lc","#readinessThresholdMs","#readyGate","#clearTimeout","#readinessTimer","#setReady","#last","#ready"],"sources":["../../../../../../../zero-cache/src/services/replicator/reporter/recorder.ts"],"sourcesContent":["import type {ObservableResult} from '@opentelemetry/api';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {getOrCreateGauge} from '../../../observability/metrics.ts';\nimport type {ReplicationReport} from './report-schema.ts';\n\n// Hook for sanity checking lag reports in development.\nconst LOG_ALL_REPLICATION_REPORTS_AT_DEBUG =\n  process.env.ZERO_LOG_ALL_REPLICATION_REPORTS_AT_DEBUG === '1';\n\nexport type ReplicationLagReadinessOptions = {\n  readinessThresholdMs?: number | undefined;\n  maxReadinessDelayMs?: number | undefined;\n  setTimeoutFn?: typeof setTimeout | undefined;\n  clearTimeoutFn?: typeof clearTimeout | undefined;\n};\n\nexport class ReplicationReportRecorder {\n  readonly #lc: LogContext;\n  readonly #readinessThresholdMs: number;\n  readonly #readyGate = resolver();\n  readonly #clearTimeout: typeof clearTimeout;\n  readonly #readinessTimer: ReturnType<typeof setTimeout>;\n  #ready = false;\n  #last: ReplicationReport | null = null;\n\n  constructor(\n    lc: LogContext,\n    {\n      readinessThresholdMs = 1000,\n      maxReadinessDelayMs = 600_000,\n      setTimeoutFn = setTimeout,\n      clearTimeoutFn = clearTimeout,\n    }: ReplicationLagReadinessOptions = {},\n  ) {\n    this.#lc = lc;\n    this.#readinessThresholdMs = readinessThresholdMs;\n    this.#clearTimeout = clearTimeoutFn;\n\n    this.#readinessTimer = setTimeoutFn(\n      () =>\n        this.#setReady(\n          'warn',\n          `timed out waiting for replication lag readiness threshold of ${this.#readinessThresholdMs} ms`,\n        ),\n      maxReadinessDelayMs,\n    );\n    // The timer should not keep the process alive.\n    this.#readinessTimer?.unref?.();\n  }\n\n  ready(): Promise<void> {\n    return this.#readyGate.promise;\n  }\n\n  record(report: ReplicationReport) {\n    if (isLagReportingDisabledSignal(report)) {\n      this.#setReady(\n        'info',\n        `replica is caught up (replication lag reporting is disabled upstream)`,\n      );\n      return;\n    }\n    this.#last = report;\n\n    const {lastTimings} = report;\n    if (lastTimings) {\n      const total = lastTimings.replicateTimeMs - lastTimings.sendTimeMs;\n      if (total > 10_000) {\n        const log = this.#ready ? 'warn' : 'info';\n        this.#lc[log]?.(`high replication lag: ${total} ms`, {\n          ready: this.#ready,\n          report,\n        });\n      } else if (total > 1_000) {\n        this.#lc.info?.(`replication lag: ${total} ms`, {\n          ready: this.#ready,\n          report,\n        });\n      }\n      if (LOG_ALL_REPLICATION_REPORTS_AT_DEBUG) {\n        this.#lc.debug?.(`replication lag ${total} ms`, {\n          ready: this.#ready,\n          report,\n        });\n      }\n      if (!this.#ready && total <= this.#readinessThresholdMs) {\n        this.#setReady(\n          'info',\n          `replicator is now ready with replication lag (${total} ms) less than ${this.#readinessThresholdMs} ms readiness threshold`,\n        );\n      }\n    }\n  }\n\n  #setReady(log: LogLevel, message: string) {\n    if (this.#ready) {\n      return;\n    }\n    this.#lc[log]?.(message, {lastReport: this.#last});\n    this.#ready = true;\n    this.#clearTimeout(this.#readinessTimer);\n    this.#readyGate.resolve();\n\n    getOrCreateGauge('replication', 'upstream_lag', {\n      description:\n        'Latency from sending an upstream replication report ' +\n        'to receiving it in the replication stream',\n      unit: 'millisecond',\n    }).addCallback(this.reportUpstreamLag);\n\n    getOrCreateGauge('replication', 'replica_lag', {\n      description:\n        'Latency from receiving an upstream replication report ' +\n        'to its reaching the replica',\n      unit: 'millisecond',\n    }).addCallback(this.reportReplicaLag);\n\n    getOrCreateGauge('replication', 'total_lag', {\n      description:\n        'Latency from sending an upstream replication report to its ' +\n        'reaching the replica. This reflects the actual measured ' +\n        'round-trip of the most recently received report and does not ' +\n        'grow when reports stop arriving.',\n      unit: 'millisecond',\n    }).addCallback(this.reportTotalLag);\n\n    getOrCreateGauge('replication', 'last_total_lag', {\n      description:\n        'Latency from sending the most recently received upstream ' +\n        'replication report to its reaching the replica. This is an alias ' +\n        'of replication.total_lag retained for dashboards that explicitly ' +\n        'want the non-extrapolated value.',\n      unit: 'millisecond',\n    }).addCallback(this.reportLastTotalLag);\n\n    getOrCreateGauge('replication', 'upstream_clock_skew', {\n      description:\n        'Estimated offset of the upstream database clock relative to ' +\n        \"zero-cache's. Positive means upstream is running ahead. \" +\n        'Derived from the lag report round-trip, whose send and receive ' +\n        'times are stamped on our clock either side of the upstream ' +\n        'commit timestamp. This matters because ' +\n        'sync.e2e_serving_lag subtracts an upstream commit timestamp from ' +\n        'a local clock, so a skewed upstream clock biases it directly -- ' +\n        'and an upstream clock running ahead biases it *low*, which looks ' +\n        'like healthy serving rather than a broken measurement.',\n      unit: 'millisecond',\n    }).addCallback(this.reportUpstreamClockSkew);\n  }\n\n  readonly reportUpstreamLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.receiveTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportReplicaLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.receiveTimeMs);\n    }\n  };\n\n  readonly reportTotalLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportLastTotalLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportUpstreamClockSkew = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(estimateUpstreamClockSkewMs(last));\n    }\n  };\n}\n\n/**\n * Estimates how far the upstream database's clock is from ours, in\n * milliseconds, positive meaning upstream is ahead.\n *\n * `sendTimeMs` and `receiveTimeMs` are both stamped in the\n * replication-manager, either side of `commitTimeMs`, which upstream stamps.\n * Assuming the upstream commit fell midway through that round trip, the\n * difference between where it actually landed and that midpoint is the clock\n * offset -- the same round-trip estimate NTP uses.\n *\n * The estimate is biased by any asymmetry between the two legs: the outbound\n * leg is a SQL round trip, while the return leg additionally waits on a WAL\n * flush and the walsender (which the pre-PG-17 path notes can add 50~100ms on\n * an idle database). So treat this as a detector of gross skew -- the kind\n * that invalidates sync.e2e_serving_lag -- and not as a sub-100ms correction.\n *\n * The rigorous bound, if a tighter reading is ever needed: the commit\n * genuinely happened between send and receive, so a `commitTimeMs` outside\n * that window proves skew of at least the amount by which it falls outside,\n * independent of any latency assumption.\n */\nexport function estimateUpstreamClockSkewMs({\n  sendTimeMs,\n  commitTimeMs,\n  receiveTimeMs,\n}: {\n  sendTimeMs: number;\n  commitTimeMs: number;\n  receiveTimeMs: number;\n}): number {\n  return commitTimeMs - (sendTimeMs + receiveTimeMs) / 2;\n}\n\n/**\n * Returns whether the `report` is the signal sent by the change-streamer,\n * once the subscriber is caught up, to indicate that lag reporting is\n * disabled upstream (and thus readiness cannot be gated on lag reports).\n *\n * This is a lag report with a `nextSendTimeMs` of 0 and no `lastTimings`.\n */\nexport function isLagReportingDisabledSignal(\n  report: ReplicationReport,\n): boolean {\n  return report.nextSendTimeMs === 0 && report.lastTimings === undefined;\n}\n"],"mappings":";;;AAOA,IAAM,uCACJ,QAAQ,IAAI,8CAA8C;AAS5D,IAAa,4BAAb,MAAuC;CACrC;CACA;CACA,aAAsB,SAAS;CAC/B;CACA;CACA,SAAS;CACT,QAAkC;CAElC,YACE,IACA,EACE,uBAAuB,KACvB,sBAAsB,KACtB,eAAe,YACf,iBAAiB,iBACiB,CAAC,GACrC;EACA,KAAKA,MAAM;EACX,KAAKC,wBAAwB;EAC7B,KAAKE,gBAAgB;EAErB,KAAKC,kBAAkB,mBAEnB,KAAKC,UACH,QACA,gEAAgE,KAAKJ,sBAAsB,IAC7F,GACF,mBACF;EAEA,KAAKG,iBAAiB,QAAQ;CAChC;CAEA,QAAuB;EACrB,OAAO,KAAKF,WAAW;CACzB;CAEA,OAAO,QAA2B;EAChC,IAAI,6BAA6B,MAAM,GAAG;GACxC,KAAKG,UACH,QACA,uEACF;GACA;EACF;EACA,KAAKC,QAAQ;EAEb,MAAM,EAAC,gBAAe;EACtB,IAAI,aAAa;GACf,MAAM,QAAQ,YAAY,kBAAkB,YAAY;GACxD,IAAI,QAAQ,KAAQ;IAClB,MAAM,MAAM,KAAKC,SAAS,SAAS;IACnC,KAAKP,IAAI,OAAO,yBAAyB,MAAM,MAAM;KACnD,OAAO,KAAKO;KACZ;IACF,CAAC;GACH,OAAO,IAAI,QAAQ,KACjB,KAAKP,IAAI,OAAO,oBAAoB,MAAM,MAAM;IAC9C,OAAO,KAAKO;IACZ;GACF,CAAC;GAEH,IAAI,sCACF,KAAKP,IAAI,QAAQ,mBAAmB,MAAM,MAAM;IAC9C,OAAO,KAAKO;IACZ;GACF,CAAC;GAEH,IAAI,CAAC,KAAKA,UAAU,SAAS,KAAKN,uBAChC,KAAKI,UACH,QACA,iDAAiD,MAAM,iBAAiB,KAAKJ,sBAAsB,wBACrG;EAEJ;CACF;CAEA,UAAU,KAAe,SAAiB;EACxC,IAAI,KAAKM,QACP;EAEF,KAAKP,IAAI,OAAO,SAAS,EAAC,YAAY,KAAKM,MAAK,CAAC;EACjD,KAAKC,SAAS;EACd,KAAKJ,cAAc,KAAKC,eAAe;EACvC,KAAKF,WAAW,QAAQ;EAExB,iBAAiB,eAAe,gBAAgB;GAC9C,aACE;GAEF,MAAM;EACR,CAAC,EAAE,YAAY,KAAK,iBAAiB;EAErC,iBAAiB,eAAe,eAAe;GAC7C,aACE;GAEF,MAAM;EACR,CAAC,EAAE,YAAY,KAAK,gBAAgB;EAEpC,iBAAiB,eAAe,aAAa;GAC3C,aACE;GAIF,MAAM;EACR,CAAC,EAAE,YAAY,KAAK,cAAc;EAElC,iBAAiB,eAAe,kBAAkB;GAChD,aACE;GAIF,MAAM;EACR,CAAC,EAAE,YAAY,KAAK,kBAAkB;EAEtC,iBAAiB,eAAe,uBAAuB;GACrD,aACE;GASF,MAAM;EACR,CAAC,EAAE,YAAY,KAAK,uBAAuB;CAC7C;CAEA,qBAA8B,MAAwB;EACpD,MAAM,OAAO,KAAKI,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,gBAAgB,KAAK,UAAU;CAElD;CAEA,oBAA6B,MAAwB;EACnD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,aAAa;CAEvD;CAEA,kBAA2B,MAAwB;EACjD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,UAAU;CAEpD;CAEA,sBAA+B,MAAwB;EACrD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,UAAU;CAEpD;CAEA,2BAAoC,MAAwB;EAC1D,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,4BAA4B,IAAI,CAAC;CAE/C;AACF;;;;;;;;;;;;;;;;;;;;;;AAuBA,SAAgB,4BAA4B,EAC1C,YACA,cACA,iBAKS;CACT,OAAO,gBAAgB,aAAa,iBAAiB;AACvD;;;;;;;;AASA,SAAgB,6BACd,QACS;CACT,OAAO,OAAO,mBAAmB,KAAK,OAAO,gBAAgB,KAAA;AAC/D"}

package/out/zero-cache/src/services/replicator/schema/backfilling.d.ts

@@ -36,11 +36,11 @@
  */
 import type { LogContext } from '@rocicorp/logger';
 import type { Database } from '../../../../../zqlite/src/db.ts';
-import type { SchemaChange } from '../../change-source/protocol/current/data.ts';
+import type { BackfillProgressMark, Identifier, SchemaChange } from '../../change-source/protocol/current/data.ts';
 import type { BackfillRequest } from '../../change-source/protocol/current/upstream.ts';
 import { type CookieOp, type CookieSet } from '../change-log-cookies.ts';
 export declare const BACKFILLING_TABLE = "_zero.backfilling";
-export declare const CREATE_BACKFILLING_TABLE = "\n  CREATE TABLE \"_zero.backfilling\" (\n    \"schema\"   TEXT NOT NULL,\n    \"table\"    TEXT NOT NULL,\n    \"column\"   TEXT NOT NULL,\n    \"backfill\" TEXT NOT NULL,\n    PRIMARY KEY (\"schema\", \"table\", \"column\")\n  );\n";
+export declare const CREATE_BACKFILLING_TABLE = "\n  CREATE TABLE \"_zero.backfilling\" (\n    \"schema\"   TEXT NOT NULL,\n    \"table\"    TEXT NOT NULL,\n    \"column\"   TEXT NOT NULL,\n    \"backfill\" TEXT NOT NULL,\n    \"progress\" TEXT,\n    PRIMARY KEY (\"schema\", \"table\", \"column\")\n  );\n";
 /**
  * The replica's interpreter of the cookie fold in
  * `replicator/change-log-cookies.ts`.
@@ -70,6 +70,19 @@ export declare class BackfillingTracker {
      * that does not support backfill — apply nothing.
      */
     apply(change: SchemaChange): CookieOp[];
+    /**
+     * Returns the tracked progress of the backfilling columns of the `table`,
+     * keyed by column name. Columns that are not backfilling are absent.
+     */
+    getProgress(table: Identifier): Map<string, BackfillProgressMark | undefined>;
+    /** Sets (or clears) the tracked progress of a backfilling column. */
+    setProgress(table: Identifier, column: string, progress: BackfillProgressMark | undefined): void;
+    /**
+     * Marks the backfill of the `columns` as complete. This is the subset of
+     * a `backfill-completed` message's columns (including row key columns)
+     * whose progress is contiguous with the message.
+     */
+    complete(table: Identifier, columns: readonly string[]): void;
 }
 /**
  * Returns the {@link BackfillRequest}s from the current replica cookies. The
@@ -80,6 +93,14 @@ export declare class BackfillingTracker {
  * changes before it compares the results.
  */
 export declare function readBackfillRequests(db: Database): BackfillRequest[];
+/**
+ * Returns the {@link BackfillRequest}s of the replica's pending backfills,
+ * including the tracked progress of each column. These are the backfills that
+ * a subscriber reports to the change-streamer when starting a subscription.
+ *
+ * The caller must pair this snapshot with the replica state version.
+ */
+export declare function readReplicaBackfills(db: Database): BackfillRequest[];
 /**
  * The replica's whole cookie set, in the shape the change log holds it — the
  * seed a change log would be initialized from, and the replica-derived half of

package/out/zero-cache/src/services/replicator/schema/backfilling.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backfilling.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/replicator/schema/backfilling.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAEH,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,KAAK,EAAC,QAAQ,EAAY,MAAM,iCAAiC,CAAC;AAGzE,OAAO,KAAK,EAGV,YAAY,EAEb,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,kDAAkD,CAAC;AACtF,OAAO,EAGL,KAAK,QAAQ,EACb,KAAK,SAAS,EACf,MAAM,0BAA0B,CAAC;AAElC,eAAO,MAAM,iBAAiB,sBAAsB,CAAC;AAKrD,eAAO,MAAM,wBAAwB,8OAQpC,CAAC;AAEF;;;;;;;;;;;;;;;;;;GAkBG;AACH,qBAAa,kBAAkB;;IAS7B,YAAY,EAAE,EAAE,QAAQ,EAEvB;IAED;;;;;OAKG;IACH,KAAK,CAAC,MAAM,EAAE,YAAY,GAAG,QAAQ,EAAE,CAMtC;CAmEF;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,EAAE,EAAE,QAAQ,GAAG,eAAe,EAAE,CAEpE;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,QAAQ,GAAG,SAAS,CA8B1D;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,qCAAqC,CACnD,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,GACX,IAAI,CA4DN"}
+{"version":3,"file":"backfilling.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/replicator/schema/backfilling.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAEH,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,KAAK,EAAC,QAAQ,EAAY,MAAM,iCAAiC,CAAC;AAGzE,OAAO,KAAK,EAEV,oBAAoB,EACpB,UAAU,EACV,YAAY,EAEb,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,kDAAkD,CAAC;AACtF,OAAO,EAGL,KAAK,QAAQ,EACb,KAAK,SAAS,EACf,MAAM,0BAA0B,CAAC;AAElC,eAAO,MAAM,iBAAiB,sBAAsB,CAAC;AAYrD,eAAO,MAAM,wBAAwB,sQASpC,CAAC;AAEF;;;;;;;;;;;;;;;;;;GAkBG;AACH,qBAAa,kBAAkB;;IAW7B,YAAY,EAAE,EAAE,QAAQ,EAEvB;IAED;;;;;OAKG;IACH,KAAK,CAAC,MAAM,EAAE,YAAY,GAAG,QAAQ,EAAE,CAMtC;IA6DD;;;OAGG;IACH,WAAW,CACT,KAAK,EAAE,UAAU,GAChB,GAAG,CAAC,MAAM,EAAE,oBAAoB,GAAG,SAAS,CAAC,CAgB/C;IAED,qEAAqE;IACrE,WAAW,CACT,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,oBAAoB,GAAG,SAAS,GACzC,IAAI,CAUN;IAED;;;;OAIG;IACH,QAAQ,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI,CAE5D;CAQF;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,EAAE,EAAE,QAAQ,GAAG,eAAe,EAAE,CAEpE;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,EAAE,EAAE,QAAQ,GAAG,eAAe,EAAE,CAkBpE;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,QAAQ,GAAG,SAAS,CA8B1D;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,qCAAqC,CACnD,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,GACX,IAAI,CA4DN"}

package/out/zero-cache/src/services/replicator/schema/backfilling.js

@@ -11,6 +11,7 @@ var CREATE_BACKFILLING_TABLE = `
     "table"    TEXT NOT NULL,
     "column"   TEXT NOT NULL,
     "backfill" TEXT NOT NULL,
+    "progress" TEXT,
     PRIMARY KEY ("schema", "table", "column")
   );
 `;
@@ -40,6 +41,8 @@ var BackfillingTracker = class {
 	#dropTable;
 	#renameColumn;
 	#dropColumn;
+	#getProgress;
+	#setProgress;
 	constructor(db) {
 		this.#db = db;
 	}
@@ -62,7 +65,7 @@ var BackfillingTracker = class {
           INSERT INTO "${BACKFILLING_TABLE}"
             ("schema", "table", "column", "backfill") VALUES (?, ?, ?, ?)
             ON CONFLICT ("schema", "table", "column")
-            DO UPDATE SET "backfill" = excluded."backfill"
+            DO UPDATE SET "backfill" = excluded."backfill", "progress" = NULL
         `)).run(op.table.schema, op.table.name, op.column, BigIntJSON.stringify(op.backfill));
 				break;
 			case "rename-table":
@@ -92,6 +95,36 @@ var BackfillingTracker = class {
 			default: unreachable(op);
 		}
 	}
+	/**
+	* Returns the tracked progress of the backfilling columns of the `table`,
+	* keyed by column name. Columns that are not backfilling are absent.
+	*/
+	getProgress(table) {
+		const rows = (this.#getProgress ??= this.#db.prepare(`
+      SELECT "column", "progress" FROM "${BACKFILLING_TABLE}"
+        WHERE "schema" = ? AND "table" = ?
+    `)).all(table.schema, table.name);
+		return new Map(rows.map(({ column, progress }) => [column, progress === null ? void 0 : BigIntJSON.parse(progress)]));
+	}
+	/** Sets (or clears) the tracked progress of a backfilling column. */
+	setProgress(table, column, progress) {
+		(this.#setProgress ??= this.#db.prepare(`
+      UPDATE "${BACKFILLING_TABLE}" SET "progress" = ?
+        WHERE "schema" = ? AND "table" = ? AND "column" = ?
+    `)).run(progress === void 0 ? null : BigIntJSON.stringify(progress), table.schema, table.name, column);
+	}
+	/**
+	* Marks the backfill of the `columns` as complete. This is the subset of
+	* a `backfill-completed` message's columns (including row key columns)
+	* whose progress is contiguous with the message.
+	*/
+	complete(table, columns) {
+		this.#run({
+			op: "complete-backfill",
+			table,
+			columns
+		});
+	}
 	#deleteColumn(table, column) {
 		(this.#dropColumn ??= this.#db.prepare(`
       DELETE FROM "${BACKFILLING_TABLE}"
@@ -111,6 +144,25 @@ function readBackfillRequests(db) {
 	return backfillRequestsFrom(readReplicaCookies(db));
 }
 /**
+* Returns the {@link BackfillRequest}s of the replica's pending backfills,
+* including the tracked progress of each column. These are the backfills that
+* a subscriber reports to the change-streamer when starting a subscription.
+*
+* The caller must pair this snapshot with the replica state version.
+*/
+function readReplicaBackfills(db) {
+	const requests = readBackfillRequests(db);
+	const progress = db.prepare(`
+      SELECT "schema", "table", "column", "progress"
+        FROM "${BACKFILLING_TABLE}" WHERE "progress" IS NOT NULL
+    `).all();
+	for (const { schema, table, column, progress: mark } of progress) {
+		const col = requests.find((r) => r.table.schema === schema && r.table.name === table)?.columns[column];
+		if (col) col.progress = BigIntJSON.parse(mark);
+	}
+	return requests;
+}
+/**
 * The replica's whole cookie set, in the shape the change log holds it — the
 * seed a change log would be initialized from, and the replica-derived half of
 * the initialization comparison.
@@ -209,6 +261,6 @@ function populateBackfillingFromColumnMetadata(lc, db) {
 	lc.info?.(`seeded "${BACKFILLING_TABLE}" with ${backfilling.length} in-flight backfill(s), ${unresolved} of which could not be resolved`);
 }
 //#endregion
-export { BackfillingTracker, CREATE_BACKFILLING_TABLE, populateBackfillingFromColumnMetadata, readBackfillRequests, readReplicaCookies };
+export { BACKFILLING_TABLE, BackfillingTracker, CREATE_BACKFILLING_TABLE, populateBackfillingFromColumnMetadata, readBackfillRequests, readReplicaBackfills, readReplicaCookies };
 
 //# sourceMappingURL=backfilling.js.map