@rocicorp/zero 1.11.0-canary.13 → 1.11.0-canary.14

package/out/zero-cache/src/services/replicator/incremental-sync.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"incremental-sync.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,KAAK,EAAQ,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAG1D,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,4CAA4C,CAAC;AACpF,OAAO,EAGL,KAAK,cAAc,EACpB,MAAM,uCAAuC,CAAC;AAM/C,OAAO,EAEL,KAAK,0BAA0B,EAChC,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,iBAAiB,CAAC;AAGlE,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAOhE;;;;;;GAMG;AACH,qBAAa,iBAAiB;;IAsB5B,YACE,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,MAAM,EACV,cAAc,EAAE,cAAc,EAC9B,MAAM,EAAE,iBAAiB,EACzB,IAAI,EAAE,cAAc,EACpB,eAAe,EAAE,0BAA0B,GAAG,IAAI,EAClD,iBAAiB,CAAC,EAAE,mBAAmB,EAYxC;IAEK,GAAG,kBA0OR;IA6BD,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC,CAEhC;IAED,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO,QAIjC;CACF"}
+{"version":3,"file":"incremental-sync.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,KAAK,EAAQ,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAG1D,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,4CAA4C,CAAC;AACpF,OAAO,EAGL,KAAK,cAAc,EACpB,MAAM,uCAAuC,CAAC;AAM/C,OAAO,EAEL,KAAK,0BAA0B,EAChC,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,iBAAiB,CAAC;AAGlE,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAOhE;;;;;;GAMG;AACH,qBAAa,iBAAiB;;IAsB5B,YACE,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,MAAM,EACV,cAAc,EAAE,cAAc,EAC9B,MAAM,EAAE,iBAAiB,EACzB,IAAI,EAAE,cAAc,EACpB,eAAe,EAAE,0BAA0B,GAAG,IAAI,EAClD,iBAAiB,CAAC,EAAE,mBAAmB,EAYxC;IAEK,GAAG,kBA2OR;IA6BD,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC,CAEhC;IAED,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO,QAIjC;CACF"}

package/out/zero-cache/src/services/replicator/incremental-sync.js

@@ -55,18 +55,19 @@ var IncrementalSyncer = class {
 			watermark: initialWatermark
 		});
 		while (this.#state.shouldRun()) {
-			const { replicaVersion, watermark } = await this.#worker.getSubscriptionState();
+			const { replicaVersion, watermark, backfills } = await this.#worker.getSubscriptionState();
 			let downstream;
 			let unregister = () => {};
 			let err;
 			try {
 				const ctx = {
-					protocolVersion: 7,
+					protocolVersion: 8,
 					taskID: this.#taskID,
 					id: this.#id,
 					mode: this.#mode,
 					watermark,
-					replicaVersion
+					replicaVersion,
+					backfills
 				};
 				const initialConnection = this.#initialConnection;
 				this.#initialConnection = void 0;

package/out/zero-cache/src/services/replicator/incremental-sync.js.map

@@ -1 +1 @@
-{"version":3,"file":"incremental-sync.js","names":["#lc","#taskID","#id","#changeStreamer","#worker","#mode","#statusPublisher","#notifier","#reporter","#state","#replicationEvents","#initialConnection","#handleResult"],"sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {mapPostgresToLiteIndex} from '../../db/pg-to-lite.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Sized, Source} from '../../types/streams.ts';\nimport type {DownloadStatus} from '../change-source/protocol/current.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {ReservationFollowup} from '../change-streamer/change-streamer-http.ts';\nimport {\n  errorTypeToReadableName,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n} from '../change-streamer/change-streamer.ts';\nimport type * as ErrorType from '../change-streamer/error-type-enum.ts';\nimport type {SubscribeDownstream} from '../change-streamer/subscribe.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {CommitResult} from './change-processor.ts';\nimport {Notifier} from './notifier.ts';\nimport {\n  IndexingProgress,\n  type ReplicationStatusPublisher,\n} from './replication-status.ts';\nimport type {ReplicaState, ReplicatorMode} from './replicator.ts';\nimport {ReplicationReportRecorder} from './reporter/recorder.ts';\nimport type {ReplicationReport} from './reporter/report-schema.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst MAX_WORKER_BATCH_MESSAGES = 256;\nconst MAX_WORKER_BATCH_SIZE = 1024 * 1024;\n\n/**\n * The {@link IncrementalSyncer} manages a logical replication stream from upstream,\n * handling application lifecycle events (start, stop) and retrying the\n * connection with exponential backoff. The actual handling of the logical\n * replication messages is done by the {@link ChangeProcessor}, which runs\n * in a worker thread via the {@link WriteWorkerClient}.\n */\nexport class IncrementalSyncer {\n  readonly #lc: LogContext;\n  readonly #taskID: string;\n  readonly #id: string;\n  readonly #changeStreamer: ChangeStreamer;\n  readonly #worker: WriteWorkerClient;\n  readonly #mode: ReplicatorMode;\n  readonly #statusPublisher: ReplicationStatusPublisher | null;\n  readonly #notifier: Notifier;\n  readonly #reporter: ReplicationReportRecorder;\n  // Consumed (and cleared) by the first subscribe() in run()'s loop; every\n  // subsequent (re)connection goes through #changeStreamer.subscribe().\n  #initialConnection: ReservationFollowup | undefined;\n\n  readonly #state = new RunningState('IncrementalSyncer');\n\n  readonly #replicationEvents = getOrCreateCounter(\n    'replication',\n    'events',\n    'Number of replication events processed',\n  );\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    id: string,\n    changeStreamer: ChangeStreamer,\n    worker: WriteWorkerClient,\n    mode: ReplicatorMode,\n    statusPublisher: ReplicationStatusPublisher | null,\n    initialConnection?: ReservationFollowup,\n  ) {\n    this.#lc = lc;\n    this.#taskID = taskID;\n    this.#id = id;\n    this.#changeStreamer = changeStreamer;\n    this.#worker = worker;\n    this.#mode = mode;\n    this.#statusPublisher = statusPublisher;\n    this.#notifier = new Notifier();\n    this.#reporter = new ReplicationReportRecorder(lc);\n    this.#initialConnection = initialConnection;\n  }\n\n  async run() {\n    const lc = this.#lc;\n    let workerError: Error | undefined;\n    this.#worker.onError(err => {\n      workerError ??= err;\n      this.#state.stop(lc, err);\n    });\n    lc.info?.(`Starting IncrementalSyncer`);\n    const {watermark: initialWatermark} =\n      await this.#worker.getSubscriptionState();\n\n    // Notify any waiting subscribers that the replica is ready to be read.\n    // This initial notification intentionally omits replicaReadyTimeMs because\n    // it represents already-current state, not newly-unserved work.\n    void this.#notifier.notifySubscribers({\n      state: 'version-ready',\n      watermark: initialWatermark,\n    });\n\n    while (this.#state.shouldRun()) {\n      const {replicaVersion, watermark} =\n        await this.#worker.getSubscriptionState();\n\n      let downstream: Source<Sized<SubscribeDownstream>> | undefined;\n      let unregister = () => {};\n      let err: unknown | undefined;\n\n      try {\n        const ctx = {\n          protocolVersion: PROTOCOL_VERSION,\n          taskID: this.#taskID,\n          id: this.#id,\n          mode: this.#mode,\n          watermark,\n          replicaVersion,\n        };\n        // The first connection reuses the still-open reservation connection\n        // (if any) from the backup restore, so the replication-manager that\n        // reserved the change log is the one that serves the subscription.\n        // Every reconnect after that opens a fresh connection.\n        const initialConnection = this.#initialConnection;\n        this.#initialConnection = undefined;\n        downstream = initialConnection\n          ? await initialConnection.subscribe(ctx)\n          : await this.#changeStreamer.subscribe(ctx);\n        unregister = this.#state.cancelOnStop(downstream);\n        this.#statusPublisher?.publish(\n          lc,\n          'Replicating',\n          `Replicating from ${watermark}`,\n        );\n\n        let backfill:\n          | {status: DownloadStatus; table: string; columns: string[]}\n          | undefined;\n        let writeBatch: ChangeStreamData[] = [];\n        let writeBatchSize = 0;\n        let inTransaction = false;\n        // Indexes created in the current run of consecutive index creations.\n        let indexing: IndexingProgress | undefined;\n\n        const publishBackfillStatus = (table: string) =>\n          this.#statusPublisher?.publish(\n            lc,\n            'Replicating',\n            `Backfilling ${table} table`,\n            3000,\n            () =>\n              backfill\n                ? {\n                    downloadStatus: [\n                      {\n                        ...backfill.status,\n                        table: backfill.table,\n                        columns: backfill.columns,\n                      },\n                    ],\n                  }\n                : {},\n          );\n\n        const flushWrites = async () => {\n          if (writeBatch.length === 0) {\n            return;\n          }\n          const batch = writeBatch;\n          writeBatch = [];\n          writeBatchSize = 0;\n\n          const result = await this.#worker.processMessages(batch);\n          this.#handleResult(lc, result);\n          if (result?.completedBackfill) {\n            backfill = undefined;\n          }\n        };\n\n        for await (const {data: message, size} of downstream) {\n          // Only reset backoff after receiving an actual message.\n          this.#state.resetBackoff();\n\n          this.#replicationEvents.add(1);\n          switch (message[0]) {\n            case 'reserved':\n              lc.warn?.(`unexpected snapshot reservation message received`, {\n                message: message[0],\n              });\n              break;\n            case 'status': {\n              const {lagReport} = message[1];\n              if (lagReport) {\n                const report: ReplicationReport = {\n                  nextSendTimeMs: lagReport.nextSendTimeMs,\n                };\n                if (lagReport.lastTimings) {\n                  report.lastTimings = {\n                    ...lagReport.lastTimings,\n                    replicateTimeMs: Date.now(),\n                  };\n                }\n                this.#reporter.record(report);\n              }\n              break;\n            }\n            case 'error': {\n              // Signal from the replication-manager that the view-syncer must\n              // shut down and restore a new backup from litestream.\n              const {type, message: msg} = message[1];\n              this.stop(\n                lc,\n                // Note: The AbortError indicates a clean / intentional shutdown.\n                new AbortError(\n                  `${errorTypeToReadableName(type as ErrorType)}: ${msg}`,\n                ),\n              );\n              break;\n            }\n            default: {\n              const msg = message[1];\n              if (msg.tag === 'backfill' && msg.status) {\n                const {status, relation} = msg;\n                const first = !backfill;\n                backfill = {\n                  status, // Update the current status\n                  table: relation.name,\n                  columns: [...relation.rowKey.columns, ...msg.columns],\n                };\n                if (first) {\n                  // Start publishing the status every 3 seconds.\n                  publishBackfillStatus(relation.name);\n                }\n              }\n\n              if (msg.tag === 'create-index' && this.#statusPublisher) {\n                // Creating an index on an existing table can take a long\n                // time. Flush preceding changes so that the index creation\n                // is processed (and timed) on its own.\n                await flushWrites();\n                const index = mapPostgresToLiteIndex(msg.spec);\n                indexing ??= new IndexingProgress();\n                indexing.start(index);\n                this.#statusPublisher.publish(\n                  lc,\n                  'Replicating',\n                  `Creating index ${index.name} on ${index.tableName}`,\n                  3000,\n                  indexing.state,\n                );\n                // Exclude the time spent publishing the status.\n                indexing.restartTimer();\n                writeBatch.push(message as ChangeStreamData);\n                writeBatchSize += size;\n                await flushWrites();\n                const elapsed = indexing.finish();\n                lc.info?.(\n                  `Created index ${index.name} (${elapsed.toFixed(3)} ms)`,\n                );\n                this.#statusPublisher.publish(\n                  lc,\n                  'Replicating',\n                  `Created index ${index.name} on ${index.tableName}`,\n                  0,\n                  indexing.state,\n                );\n                if (backfill) {\n                  // Resume reporting the progress of the ongoing backfill.\n                  publishBackfillStatus(backfill.table);\n                }\n                break;\n              }\n              indexing = undefined;\n\n              const type = message[0];\n              const invalidTransactionSequence =\n                type === 'begin' ? inTransaction : !inTransaction;\n              if (type === 'begin') {\n                inTransaction = true;\n              } else if (type === 'commit' || type === 'rollback') {\n                inTransaction = false;\n              }\n\n              writeBatch.push(message as ChangeStreamData);\n              writeBatchSize += size;\n              if (\n                // Waiting here ensures that consuming the commit, which ACKs\n                // it upstream, only happens after the SQLite commit finishes.\n                type === 'commit' ||\n                type === 'rollback' ||\n                // Promptly surface malformed transaction sequences instead\n                // of leaving a partial batch waiting for another message.\n                invalidTransactionSequence ||\n                writeBatch.length >= MAX_WORKER_BATCH_MESSAGES ||\n                writeBatchSize >= MAX_WORKER_BATCH_SIZE\n              ) {\n                await flushWrites();\n              }\n              break;\n            }\n          }\n        }\n        this.#worker.abort();\n      } catch (e) {\n        err = e;\n        this.#worker.abort();\n      } finally {\n        downstream?.cancel();\n        unregister();\n        this.#statusPublisher?.stop();\n      }\n      await this.#state.backoff(lc, err);\n    }\n    lc.info?.('IncrementalSyncer stopped');\n    if (workerError) {\n      throw workerError;\n    }\n  }\n\n  #handleResult(lc: LogContext, result: CommitResult | null) {\n    if (!result) {\n      return;\n    }\n    if (result.completedBackfill) {\n      // Publish the final status\n      const status = result.completedBackfill;\n      this.#statusPublisher?.publish(\n        lc,\n        'Replicating',\n        `Backfilled ${status.table} table`,\n        0,\n        () => ({downloadStatus: [status]}),\n      );\n    } else if (result.schemaUpdated) {\n      this.#statusPublisher?.publish(lc, 'Replicating', 'Schema updated');\n    }\n    if (result.watermark && result.changeLogUpdated) {\n      void this.#notifier.notifySubscribers({\n        state: 'version-ready',\n        watermark: result.watermark,\n        replicaReadyTimeMs: Date.now(),\n        upstreamCommitTimeMs: result.upstreamCommitTimeMs,\n      });\n    }\n  }\n\n  subscribe(): Source<ReplicaState> {\n    return this.#notifier.subscribe();\n  }\n\n  stop(lc: LogContext, err?: unknown) {\n    this.#state.stop(lc, err);\n    // Abort any polling loop that the write worker may be in.\n    this.#worker.abort();\n  }\n}\n"],"mappings":";;;;;;;;;AA8BA,IAAM,4BAA4B;AAClC,IAAM,wBAAwB,OAAO;;;;;;;;AASrC,IAAa,oBAAb,MAA+B;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAGA;CAEA,SAAkB,IAAI,aAAa,mBAAmB;CAEtD,qBAA8B,mBAC5B,eACA,UACA,wCACF;CAEA,YACE,IACA,QACA,IACA,gBACA,QACA,MACA,iBACA,mBACA;EACA,KAAKA,MAAM;EACX,KAAKC,UAAU;EACf,KAAKC,MAAM;EACX,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,QAAQ;EACb,KAAKC,mBAAmB;EACxB,KAAKC,YAAY,IAAI,SAAS;EAC9B,KAAKC,YAAY,IAAI,0BAA0B,EAAE;EACjD,KAAKG,qBAAqB;CAC5B;CAEA,MAAM,MAAM;EACV,MAAM,KAAK,KAAKX;EAChB,IAAI;EACJ,KAAKI,QAAQ,SAAQ,QAAO;GAC1B,gBAAgB;GAChB,KAAKK,OAAO,KAAK,IAAI,GAAG;EAC1B,CAAC;EACD,GAAG,OAAO,4BAA4B;EACtC,MAAM,EAAC,WAAW,qBAChB,MAAM,KAAKL,QAAQ,qBAAqB;EAK1C,KAAUG,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW;EACb,CAAC;EAED,OAAO,KAAKE,OAAO,UAAU,GAAG;GAC9B,MAAM,EAAC,gBAAgB,cACrB,MAAM,KAAKL,QAAQ,qBAAqB;GAE1C,IAAI;GACJ,IAAI,mBAAmB,CAAC;GACxB,IAAI;GAEJ,IAAI;IACF,MAAM,MAAM;KACV,iBAAA;KACA,QAAQ,KAAKH;KACb,IAAI,KAAKC;KACT,MAAM,KAAKG;KACX;KACA;IACF;IAKA,MAAM,oBAAoB,KAAKM;IAC/B,KAAKA,qBAAqB,KAAA;IAC1B,aAAa,oBACT,MAAM,kBAAkB,UAAU,GAAG,IACrC,MAAM,KAAKR,gBAAgB,UAAU,GAAG;IAC5C,aAAa,KAAKM,OAAO,aAAa,UAAU;IAChD,KAAKH,kBAAkB,QACrB,IACA,eACA,oBAAoB,WACtB;IAEA,IAAI;IAGJ,IAAI,aAAiC,CAAC;IACtC,IAAI,iBAAiB;IACrB,IAAI,gBAAgB;IAEpB,IAAI;IAEJ,MAAM,yBAAyB,UAC7B,KAAKA,kBAAkB,QACrB,IACA,eACA,eAAe,MAAM,SACrB,WAEE,WACI,EACE,gBAAgB,CACd;KACE,GAAG,SAAS;KACZ,OAAO,SAAS;KAChB,SAAS,SAAS;IACpB,CACF,EACF,IACA,CAAC,CACT;IAEF,MAAM,cAAc,YAAY;KAC9B,IAAI,WAAW,WAAW,GACxB;KAEF,MAAM,QAAQ;KACd,aAAa,CAAC;KACd,iBAAiB;KAEjB,MAAM,SAAS,MAAM,KAAKF,QAAQ,gBAAgB,KAAK;KACvD,KAAKQ,cAAc,IAAI,MAAM;KAC7B,IAAI,QAAQ,mBACV,WAAW,KAAA;IAEf;IAEA,WAAW,MAAM,EAAC,MAAM,SAAS,UAAS,YAAY;KAEpD,KAAKH,OAAO,aAAa;KAEzB,KAAKC,mBAAmB,IAAI,CAAC;KAC7B,QAAQ,QAAQ,IAAhB;MACE,KAAK;OACH,GAAG,OAAO,oDAAoD,EAC5D,SAAS,QAAQ,GACnB,CAAC;OACD;MACF,KAAK,UAAU;OACb,MAAM,EAAC,cAAa,QAAQ;OAC5B,IAAI,WAAW;QACb,MAAM,SAA4B,EAChC,gBAAgB,UAAU,eAC5B;QACA,IAAI,UAAU,aACZ,OAAO,cAAc;SACnB,GAAG,UAAU;SACb,iBAAiB,KAAK,IAAI;QAC5B;QAEF,KAAKF,UAAU,OAAO,MAAM;OAC9B;OACA;MACF;MACA,KAAK,SAAS;OAGZ,MAAM,EAAC,MAAM,SAAS,QAAO,QAAQ;OACrC,KAAK,KACH,IAEA,IAAI,WACF,GAAG,wBAAwB,IAAiB,EAAE,IAAI,KACpD,CACF;OACA;MACF;MACA,SAAS;OACP,MAAM,MAAM,QAAQ;OACpB,IAAI,IAAI,QAAQ,cAAc,IAAI,QAAQ;QACxC,MAAM,EAAC,QAAQ,aAAY;QAC3B,MAAM,QAAQ,CAAC;QACf,WAAW;SACT;SACA,OAAO,SAAS;SAChB,SAAS,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,IAAI,OAAO;QACtD;QACA,IAAI,OAEF,sBAAsB,SAAS,IAAI;OAEvC;OAEA,IAAI,IAAI,QAAQ,kBAAkB,KAAKF,kBAAkB;QAIvD,MAAM,YAAY;QAClB,MAAM,QAAQ,uBAAuB,IAAI,IAAI;QAC7C,aAAa,IAAI,iBAAiB;QAClC,SAAS,MAAM,KAAK;QACpB,KAAKA,iBAAiB,QACpB,IACA,eACA,kBAAkB,MAAM,KAAK,MAAM,MAAM,aACzC,KACA,SAAS,KACX;QAEA,SAAS,aAAa;QACtB,WAAW,KAAK,OAA2B;QAC3C,kBAAkB;QAClB,MAAM,YAAY;QAClB,MAAM,UAAU,SAAS,OAAO;QAChC,GAAG,OACD,iBAAiB,MAAM,KAAK,IAAI,QAAQ,QAAQ,CAAC,EAAE,KACrD;QACA,KAAKA,iBAAiB,QACpB,IACA,eACA,iBAAiB,MAAM,KAAK,MAAM,MAAM,aACxC,GACA,SAAS,KACX;QACA,IAAI,UAEF,sBAAsB,SAAS,KAAK;QAEtC;OACF;OACA,WAAW,KAAA;OAEX,MAAM,OAAO,QAAQ;OACrB,MAAM,6BACJ,SAAS,UAAU,gBAAgB,CAAC;OACtC,IAAI,SAAS,SACX,gBAAgB;YACX,IAAI,SAAS,YAAY,SAAS,YACvC,gBAAgB;OAGlB,WAAW,KAAK,OAA2B;OAC3C,kBAAkB;OAClB,IAGE,SAAS,YACT,SAAS,cAGT,8BACA,WAAW,UAAU,6BACrB,kBAAkB,uBAElB,MAAM,YAAY;OAEpB;MACF;KACF;IACF;IACA,KAAKF,QAAQ,MAAM;GACrB,SAAS,GAAG;IACV,MAAM;IACN,KAAKA,QAAQ,MAAM;GACrB,UAAU;IACR,YAAY,OAAO;IACnB,WAAW;IACX,KAAKE,kBAAkB,KAAK;GAC9B;GACA,MAAM,KAAKG,OAAO,QAAQ,IAAI,GAAG;EACnC;EACA,GAAG,OAAO,2BAA2B;EACrC,IAAI,aACF,MAAM;CAEV;CAEA,cAAc,IAAgB,QAA6B;EACzD,IAAI,CAAC,QACH;EAEF,IAAI,OAAO,mBAAmB;GAE5B,MAAM,SAAS,OAAO;GACtB,KAAKH,kBAAkB,QACrB,IACA,eACA,cAAc,OAAO,MAAM,SAC3B,UACO,EAAC,gBAAgB,CAAC,MAAM,EAAC,EAClC;EACF,OAAO,IAAI,OAAO,eAChB,KAAKA,kBAAkB,QAAQ,IAAI,eAAe,gBAAgB;EAEpE,IAAI,OAAO,aAAa,OAAO,kBAC7B,KAAUC,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW,OAAO;GAClB,oBAAoB,KAAK,IAAI;GAC7B,sBAAsB,OAAO;EAC/B,CAAC;CAEL;CAEA,YAAkC;EAChC,OAAO,KAAKA,UAAU,UAAU;CAClC;CAEA,KAAK,IAAgB,KAAe;EAClC,KAAKE,OAAO,KAAK,IAAI,GAAG;EAExB,KAAKL,QAAQ,MAAM;CACrB;AACF"}
+{"version":3,"file":"incremental-sync.js","names":["#lc","#taskID","#id","#changeStreamer","#worker","#mode","#statusPublisher","#notifier","#reporter","#state","#replicationEvents","#initialConnection","#handleResult"],"sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {mapPostgresToLiteIndex} from '../../db/pg-to-lite.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Sized, Source} from '../../types/streams.ts';\nimport type {DownloadStatus} from '../change-source/protocol/current.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {ReservationFollowup} from '../change-streamer/change-streamer-http.ts';\nimport {\n  errorTypeToReadableName,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n} from '../change-streamer/change-streamer.ts';\nimport type * as ErrorType from '../change-streamer/error-type-enum.ts';\nimport type {SubscribeDownstream} from '../change-streamer/subscribe.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {CommitResult} from './change-processor.ts';\nimport {Notifier} from './notifier.ts';\nimport {\n  IndexingProgress,\n  type ReplicationStatusPublisher,\n} from './replication-status.ts';\nimport type {ReplicaState, ReplicatorMode} from './replicator.ts';\nimport {ReplicationReportRecorder} from './reporter/recorder.ts';\nimport type {ReplicationReport} from './reporter/report-schema.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst MAX_WORKER_BATCH_MESSAGES = 256;\nconst MAX_WORKER_BATCH_SIZE = 1024 * 1024;\n\n/**\n * The {@link IncrementalSyncer} manages a logical replication stream from upstream,\n * handling application lifecycle events (start, stop) and retrying the\n * connection with exponential backoff. The actual handling of the logical\n * replication messages is done by the {@link ChangeProcessor}, which runs\n * in a worker thread via the {@link WriteWorkerClient}.\n */\nexport class IncrementalSyncer {\n  readonly #lc: LogContext;\n  readonly #taskID: string;\n  readonly #id: string;\n  readonly #changeStreamer: ChangeStreamer;\n  readonly #worker: WriteWorkerClient;\n  readonly #mode: ReplicatorMode;\n  readonly #statusPublisher: ReplicationStatusPublisher | null;\n  readonly #notifier: Notifier;\n  readonly #reporter: ReplicationReportRecorder;\n  // Consumed (and cleared) by the first subscribe() in run()'s loop; every\n  // subsequent (re)connection goes through #changeStreamer.subscribe().\n  #initialConnection: ReservationFollowup | undefined;\n\n  readonly #state = new RunningState('IncrementalSyncer');\n\n  readonly #replicationEvents = getOrCreateCounter(\n    'replication',\n    'events',\n    'Number of replication events processed',\n  );\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    id: string,\n    changeStreamer: ChangeStreamer,\n    worker: WriteWorkerClient,\n    mode: ReplicatorMode,\n    statusPublisher: ReplicationStatusPublisher | null,\n    initialConnection?: ReservationFollowup,\n  ) {\n    this.#lc = lc;\n    this.#taskID = taskID;\n    this.#id = id;\n    this.#changeStreamer = changeStreamer;\n    this.#worker = worker;\n    this.#mode = mode;\n    this.#statusPublisher = statusPublisher;\n    this.#notifier = new Notifier();\n    this.#reporter = new ReplicationReportRecorder(lc);\n    this.#initialConnection = initialConnection;\n  }\n\n  async run() {\n    const lc = this.#lc;\n    let workerError: Error | undefined;\n    this.#worker.onError(err => {\n      workerError ??= err;\n      this.#state.stop(lc, err);\n    });\n    lc.info?.(`Starting IncrementalSyncer`);\n    const {watermark: initialWatermark} =\n      await this.#worker.getSubscriptionState();\n\n    // Notify any waiting subscribers that the replica is ready to be read.\n    // This initial notification intentionally omits replicaReadyTimeMs because\n    // it represents already-current state, not newly-unserved work.\n    void this.#notifier.notifySubscribers({\n      state: 'version-ready',\n      watermark: initialWatermark,\n    });\n\n    while (this.#state.shouldRun()) {\n      const {replicaVersion, watermark, backfills} =\n        await this.#worker.getSubscriptionState();\n\n      let downstream: Source<Sized<SubscribeDownstream>> | undefined;\n      let unregister = () => {};\n      let err: unknown | undefined;\n\n      try {\n        const ctx = {\n          protocolVersion: PROTOCOL_VERSION,\n          taskID: this.#taskID,\n          id: this.#id,\n          mode: this.#mode,\n          watermark,\n          replicaVersion,\n          backfills,\n        };\n        // The first connection reuses the still-open reservation connection\n        // (if any) from the backup restore, so the replication-manager that\n        // reserved the change log is the one that serves the subscription.\n        // Every reconnect after that opens a fresh connection.\n        const initialConnection = this.#initialConnection;\n        this.#initialConnection = undefined;\n        downstream = initialConnection\n          ? await initialConnection.subscribe(ctx)\n          : await this.#changeStreamer.subscribe(ctx);\n        unregister = this.#state.cancelOnStop(downstream);\n        this.#statusPublisher?.publish(\n          lc,\n          'Replicating',\n          `Replicating from ${watermark}`,\n        );\n\n        let backfill:\n          | {status: DownloadStatus; table: string; columns: string[]}\n          | undefined;\n        let writeBatch: ChangeStreamData[] = [];\n        let writeBatchSize = 0;\n        let inTransaction = false;\n        // Indexes created in the current run of consecutive index creations.\n        let indexing: IndexingProgress | undefined;\n\n        const publishBackfillStatus = (table: string) =>\n          this.#statusPublisher?.publish(\n            lc,\n            'Replicating',\n            `Backfilling ${table} table`,\n            3000,\n            () =>\n              backfill\n                ? {\n                    downloadStatus: [\n                      {\n                        ...backfill.status,\n                        table: backfill.table,\n                        columns: backfill.columns,\n                      },\n                    ],\n                  }\n                : {},\n          );\n\n        const flushWrites = async () => {\n          if (writeBatch.length === 0) {\n            return;\n          }\n          const batch = writeBatch;\n          writeBatch = [];\n          writeBatchSize = 0;\n\n          const result = await this.#worker.processMessages(batch);\n          this.#handleResult(lc, result);\n          if (result?.completedBackfill) {\n            backfill = undefined;\n          }\n        };\n\n        for await (const {data: message, size} of downstream) {\n          // Only reset backoff after receiving an actual message.\n          this.#state.resetBackoff();\n\n          this.#replicationEvents.add(1);\n          switch (message[0]) {\n            case 'reserved':\n              lc.warn?.(`unexpected snapshot reservation message received`, {\n                message: message[0],\n              });\n              break;\n            case 'status': {\n              const {lagReport} = message[1];\n              if (lagReport) {\n                const report: ReplicationReport = {\n                  nextSendTimeMs: lagReport.nextSendTimeMs,\n                };\n                if (lagReport.lastTimings) {\n                  report.lastTimings = {\n                    ...lagReport.lastTimings,\n                    replicateTimeMs: Date.now(),\n                  };\n                }\n                this.#reporter.record(report);\n              }\n              break;\n            }\n            case 'error': {\n              // Signal from the replication-manager that the view-syncer must\n              // shut down and restore a new backup from litestream.\n              const {type, message: msg} = message[1];\n              this.stop(\n                lc,\n                // Note: The AbortError indicates a clean / intentional shutdown.\n                new AbortError(\n                  `${errorTypeToReadableName(type as ErrorType)}: ${msg}`,\n                ),\n              );\n              break;\n            }\n            default: {\n              const msg = message[1];\n              if (msg.tag === 'backfill' && msg.status) {\n                const {status, relation} = msg;\n                const first = !backfill;\n                backfill = {\n                  status, // Update the current status\n                  table: relation.name,\n                  columns: [...relation.rowKey.columns, ...msg.columns],\n                };\n                if (first) {\n                  // Start publishing the status every 3 seconds.\n                  publishBackfillStatus(relation.name);\n                }\n              }\n\n              if (msg.tag === 'create-index' && this.#statusPublisher) {\n                // Creating an index on an existing table can take a long\n                // time. Flush preceding changes so that the index creation\n                // is processed (and timed) on its own.\n                await flushWrites();\n                const index = mapPostgresToLiteIndex(msg.spec);\n                indexing ??= new IndexingProgress();\n                indexing.start(index);\n                this.#statusPublisher.publish(\n                  lc,\n                  'Replicating',\n                  `Creating index ${index.name} on ${index.tableName}`,\n                  3000,\n                  indexing.state,\n                );\n                // Exclude the time spent publishing the status.\n                indexing.restartTimer();\n                writeBatch.push(message as ChangeStreamData);\n                writeBatchSize += size;\n                await flushWrites();\n                const elapsed = indexing.finish();\n                lc.info?.(\n                  `Created index ${index.name} (${elapsed.toFixed(3)} ms)`,\n                );\n                this.#statusPublisher.publish(\n                  lc,\n                  'Replicating',\n                  `Created index ${index.name} on ${index.tableName}`,\n                  0,\n                  indexing.state,\n                );\n                if (backfill) {\n                  // Resume reporting the progress of the ongoing backfill.\n                  publishBackfillStatus(backfill.table);\n                }\n                break;\n              }\n              indexing = undefined;\n\n              const type = message[0];\n              const invalidTransactionSequence =\n                type === 'begin' ? inTransaction : !inTransaction;\n              if (type === 'begin') {\n                inTransaction = true;\n              } else if (type === 'commit' || type === 'rollback') {\n                inTransaction = false;\n              }\n\n              writeBatch.push(message as ChangeStreamData);\n              writeBatchSize += size;\n              if (\n                // Waiting here ensures that consuming the commit, which ACKs\n                // it upstream, only happens after the SQLite commit finishes.\n                type === 'commit' ||\n                type === 'rollback' ||\n                // Promptly surface malformed transaction sequences instead\n                // of leaving a partial batch waiting for another message.\n                invalidTransactionSequence ||\n                writeBatch.length >= MAX_WORKER_BATCH_MESSAGES ||\n                writeBatchSize >= MAX_WORKER_BATCH_SIZE\n              ) {\n                await flushWrites();\n              }\n              break;\n            }\n          }\n        }\n        this.#worker.abort();\n      } catch (e) {\n        err = e;\n        this.#worker.abort();\n      } finally {\n        downstream?.cancel();\n        unregister();\n        this.#statusPublisher?.stop();\n      }\n      await this.#state.backoff(lc, err);\n    }\n    lc.info?.('IncrementalSyncer stopped');\n    if (workerError) {\n      throw workerError;\n    }\n  }\n\n  #handleResult(lc: LogContext, result: CommitResult | null) {\n    if (!result) {\n      return;\n    }\n    if (result.completedBackfill) {\n      // Publish the final status\n      const status = result.completedBackfill;\n      this.#statusPublisher?.publish(\n        lc,\n        'Replicating',\n        `Backfilled ${status.table} table`,\n        0,\n        () => ({downloadStatus: [status]}),\n      );\n    } else if (result.schemaUpdated) {\n      this.#statusPublisher?.publish(lc, 'Replicating', 'Schema updated');\n    }\n    if (result.watermark && result.changeLogUpdated) {\n      void this.#notifier.notifySubscribers({\n        state: 'version-ready',\n        watermark: result.watermark,\n        replicaReadyTimeMs: Date.now(),\n        upstreamCommitTimeMs: result.upstreamCommitTimeMs,\n      });\n    }\n  }\n\n  subscribe(): Source<ReplicaState> {\n    return this.#notifier.subscribe();\n  }\n\n  stop(lc: LogContext, err?: unknown) {\n    this.#state.stop(lc, err);\n    // Abort any polling loop that the write worker may be in.\n    this.#worker.abort();\n  }\n}\n"],"mappings":";;;;;;;;;AA8BA,IAAM,4BAA4B;AAClC,IAAM,wBAAwB,OAAO;;;;;;;;AASrC,IAAa,oBAAb,MAA+B;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAGA;CAEA,SAAkB,IAAI,aAAa,mBAAmB;CAEtD,qBAA8B,mBAC5B,eACA,UACA,wCACF;CAEA,YACE,IACA,QACA,IACA,gBACA,QACA,MACA,iBACA,mBACA;EACA,KAAKA,MAAM;EACX,KAAKC,UAAU;EACf,KAAKC,MAAM;EACX,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,QAAQ;EACb,KAAKC,mBAAmB;EACxB,KAAKC,YAAY,IAAI,SAAS;EAC9B,KAAKC,YAAY,IAAI,0BAA0B,EAAE;EACjD,KAAKG,qBAAqB;CAC5B;CAEA,MAAM,MAAM;EACV,MAAM,KAAK,KAAKX;EAChB,IAAI;EACJ,KAAKI,QAAQ,SAAQ,QAAO;GAC1B,gBAAgB;GAChB,KAAKK,OAAO,KAAK,IAAI,GAAG;EAC1B,CAAC;EACD,GAAG,OAAO,4BAA4B;EACtC,MAAM,EAAC,WAAW,qBAChB,MAAM,KAAKL,QAAQ,qBAAqB;EAK1C,KAAUG,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW;EACb,CAAC;EAED,OAAO,KAAKE,OAAO,UAAU,GAAG;GAC9B,MAAM,EAAC,gBAAgB,WAAW,cAChC,MAAM,KAAKL,QAAQ,qBAAqB;GAE1C,IAAI;GACJ,IAAI,mBAAmB,CAAC;GACxB,IAAI;GAEJ,IAAI;IACF,MAAM,MAAM;KACV,iBAAA;KACA,QAAQ,KAAKH;KACb,IAAI,KAAKC;KACT,MAAM,KAAKG;KACX;KACA;KACA;IACF;IAKA,MAAM,oBAAoB,KAAKM;IAC/B,KAAKA,qBAAqB,KAAA;IAC1B,aAAa,oBACT,MAAM,kBAAkB,UAAU,GAAG,IACrC,MAAM,KAAKR,gBAAgB,UAAU,GAAG;IAC5C,aAAa,KAAKM,OAAO,aAAa,UAAU;IAChD,KAAKH,kBAAkB,QACrB,IACA,eACA,oBAAoB,WACtB;IAEA,IAAI;IAGJ,IAAI,aAAiC,CAAC;IACtC,IAAI,iBAAiB;IACrB,IAAI,gBAAgB;IAEpB,IAAI;IAEJ,MAAM,yBAAyB,UAC7B,KAAKA,kBAAkB,QACrB,IACA,eACA,eAAe,MAAM,SACrB,WAEE,WACI,EACE,gBAAgB,CACd;KACE,GAAG,SAAS;KACZ,OAAO,SAAS;KAChB,SAAS,SAAS;IACpB,CACF,EACF,IACA,CAAC,CACT;IAEF,MAAM,cAAc,YAAY;KAC9B,IAAI,WAAW,WAAW,GACxB;KAEF,MAAM,QAAQ;KACd,aAAa,CAAC;KACd,iBAAiB;KAEjB,MAAM,SAAS,MAAM,KAAKF,QAAQ,gBAAgB,KAAK;KACvD,KAAKQ,cAAc,IAAI,MAAM;KAC7B,IAAI,QAAQ,mBACV,WAAW,KAAA;IAEf;IAEA,WAAW,MAAM,EAAC,MAAM,SAAS,UAAS,YAAY;KAEpD,KAAKH,OAAO,aAAa;KAEzB,KAAKC,mBAAmB,IAAI,CAAC;KAC7B,QAAQ,QAAQ,IAAhB;MACE,KAAK;OACH,GAAG,OAAO,oDAAoD,EAC5D,SAAS,QAAQ,GACnB,CAAC;OACD;MACF,KAAK,UAAU;OACb,MAAM,EAAC,cAAa,QAAQ;OAC5B,IAAI,WAAW;QACb,MAAM,SAA4B,EAChC,gBAAgB,UAAU,eAC5B;QACA,IAAI,UAAU,aACZ,OAAO,cAAc;SACnB,GAAG,UAAU;SACb,iBAAiB,KAAK,IAAI;QAC5B;QAEF,KAAKF,UAAU,OAAO,MAAM;OAC9B;OACA;MACF;MACA,KAAK,SAAS;OAGZ,MAAM,EAAC,MAAM,SAAS,QAAO,QAAQ;OACrC,KAAK,KACH,IAEA,IAAI,WACF,GAAG,wBAAwB,IAAiB,EAAE,IAAI,KACpD,CACF;OACA;MACF;MACA,SAAS;OACP,MAAM,MAAM,QAAQ;OACpB,IAAI,IAAI,QAAQ,cAAc,IAAI,QAAQ;QACxC,MAAM,EAAC,QAAQ,aAAY;QAC3B,MAAM,QAAQ,CAAC;QACf,WAAW;SACT;SACA,OAAO,SAAS;SAChB,SAAS,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,IAAI,OAAO;QACtD;QACA,IAAI,OAEF,sBAAsB,SAAS,IAAI;OAEvC;OAEA,IAAI,IAAI,QAAQ,kBAAkB,KAAKF,kBAAkB;QAIvD,MAAM,YAAY;QAClB,MAAM,QAAQ,uBAAuB,IAAI,IAAI;QAC7C,aAAa,IAAI,iBAAiB;QAClC,SAAS,MAAM,KAAK;QACpB,KAAKA,iBAAiB,QACpB,IACA,eACA,kBAAkB,MAAM,KAAK,MAAM,MAAM,aACzC,KACA,SAAS,KACX;QAEA,SAAS,aAAa;QACtB,WAAW,KAAK,OAA2B;QAC3C,kBAAkB;QAClB,MAAM,YAAY;QAClB,MAAM,UAAU,SAAS,OAAO;QAChC,GAAG,OACD,iBAAiB,MAAM,KAAK,IAAI,QAAQ,QAAQ,CAAC,EAAE,KACrD;QACA,KAAKA,iBAAiB,QACpB,IACA,eACA,iBAAiB,MAAM,KAAK,MAAM,MAAM,aACxC,GACA,SAAS,KACX;QACA,IAAI,UAEF,sBAAsB,SAAS,KAAK;QAEtC;OACF;OACA,WAAW,KAAA;OAEX,MAAM,OAAO,QAAQ;OACrB,MAAM,6BACJ,SAAS,UAAU,gBAAgB,CAAC;OACtC,IAAI,SAAS,SACX,gBAAgB;YACX,IAAI,SAAS,YAAY,SAAS,YACvC,gBAAgB;OAGlB,WAAW,KAAK,OAA2B;OAC3C,kBAAkB;OAClB,IAGE,SAAS,YACT,SAAS,cAGT,8BACA,WAAW,UAAU,6BACrB,kBAAkB,uBAElB,MAAM,YAAY;OAEpB;MACF;KACF;IACF;IACA,KAAKF,QAAQ,MAAM;GACrB,SAAS,GAAG;IACV,MAAM;IACN,KAAKA,QAAQ,MAAM;GACrB,UAAU;IACR,YAAY,OAAO;IACnB,WAAW;IACX,KAAKE,kBAAkB,KAAK;GAC9B;GACA,MAAM,KAAKG,OAAO,QAAQ,IAAI,GAAG;EACnC;EACA,GAAG,OAAO,2BAA2B;EACrC,IAAI,aACF,MAAM;CAEV;CAEA,cAAc,IAAgB,QAA6B;EACzD,IAAI,CAAC,QACH;EAEF,IAAI,OAAO,mBAAmB;GAE5B,MAAM,SAAS,OAAO;GACtB,KAAKH,kBAAkB,QACrB,IACA,eACA,cAAc,OAAO,MAAM,SAC3B,UACO,EAAC,gBAAgB,CAAC,MAAM,EAAC,EAClC;EACF,OAAO,IAAI,OAAO,eAChB,KAAKA,kBAAkB,QAAQ,IAAI,eAAe,gBAAgB;EAEpE,IAAI,OAAO,aAAa,OAAO,kBAC7B,KAAUC,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW,OAAO;GAClB,oBAAoB,KAAK,IAAI;GAC7B,sBAAsB,OAAO;EAC/B,CAAC;CAEL;CAEA,YAAkC;EAChC,OAAO,KAAKA,UAAU,UAAU;CAClC;CAEA,KAAK,IAAgB,KAAe;EAClC,KAAKE,OAAO,KAAK,IAAI,GAAG;EAExB,KAAKL,QAAQ,MAAM;CACrB;AACF"}

package/out/zero-cache/src/services/replicator/schema/backfilling.d.ts

@@ -36,11 +36,11 @@
  */
 import type { LogContext } from '@rocicorp/logger';
 import type { Database } from '../../../../../zqlite/src/db.ts';
-import type { SchemaChange } from '../../change-source/protocol/current/data.ts';
+import type { BackfillProgressMark, Identifier, SchemaChange } from '../../change-source/protocol/current/data.ts';
 import type { BackfillRequest } from '../../change-source/protocol/current/upstream.ts';
 import { type CookieOp, type CookieSet } from '../change-log-cookies.ts';
 export declare const BACKFILLING_TABLE = "_zero.backfilling";
-export declare const CREATE_BACKFILLING_TABLE = "\n  CREATE TABLE \"_zero.backfilling\" (\n    \"schema\"   TEXT NOT NULL,\n    \"table\"    TEXT NOT NULL,\n    \"column\"   TEXT NOT NULL,\n    \"backfill\" TEXT NOT NULL,\n    PRIMARY KEY (\"schema\", \"table\", \"column\")\n  );\n";
+export declare const CREATE_BACKFILLING_TABLE = "\n  CREATE TABLE \"_zero.backfilling\" (\n    \"schema\"   TEXT NOT NULL,\n    \"table\"    TEXT NOT NULL,\n    \"column\"   TEXT NOT NULL,\n    \"backfill\" TEXT NOT NULL,\n    \"progress\" TEXT,\n    PRIMARY KEY (\"schema\", \"table\", \"column\")\n  );\n";
 /**
  * The replica's interpreter of the cookie fold in
  * `replicator/change-log-cookies.ts`.
@@ -70,6 +70,19 @@ export declare class BackfillingTracker {
      * that does not support backfill — apply nothing.
      */
     apply(change: SchemaChange): CookieOp[];
+    /**
+     * Returns the tracked progress of the backfilling columns of the `table`,
+     * keyed by column name. Columns that are not backfilling are absent.
+     */
+    getProgress(table: Identifier): Map<string, BackfillProgressMark | undefined>;
+    /** Sets (or clears) the tracked progress of a backfilling column. */
+    setProgress(table: Identifier, column: string, progress: BackfillProgressMark | undefined): void;
+    /**
+     * Marks the backfill of the `columns` as complete. This is the subset of
+     * a `backfill-completed` message's columns (including row key columns)
+     * whose progress is contiguous with the message.
+     */
+    complete(table: Identifier, columns: readonly string[]): void;
 }
 /**
  * Returns the {@link BackfillRequest}s from the current replica cookies. The
@@ -80,6 +93,14 @@ export declare class BackfillingTracker {
  * changes before it compares the results.
  */
 export declare function readBackfillRequests(db: Database): BackfillRequest[];
+/**
+ * Returns the {@link BackfillRequest}s of the replica's pending backfills,
+ * including the tracked progress of each column. These are the backfills that
+ * a subscriber reports to the change-streamer when starting a subscription.
+ *
+ * The caller must pair this snapshot with the replica state version.
+ */
+export declare function readReplicaBackfills(db: Database): BackfillRequest[];
 /**
  * The replica's whole cookie set, in the shape the change log holds it — the
  * seed a change log would be initialized from, and the replica-derived half of

package/out/zero-cache/src/services/replicator/schema/backfilling.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backfilling.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/replicator/schema/backfilling.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAEH,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,KAAK,EAAC,QAAQ,EAAY,MAAM,iCAAiC,CAAC;AAGzE,OAAO,KAAK,EAGV,YAAY,EAEb,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,kDAAkD,CAAC;AACtF,OAAO,EAGL,KAAK,QAAQ,EACb,KAAK,SAAS,EACf,MAAM,0BAA0B,CAAC;AAElC,eAAO,MAAM,iBAAiB,sBAAsB,CAAC;AAKrD,eAAO,MAAM,wBAAwB,8OAQpC,CAAC;AAEF;;;;;;;;;;;;;;;;;;GAkBG;AACH,qBAAa,kBAAkB;;IAS7B,YAAY,EAAE,EAAE,QAAQ,EAEvB;IAED;;;;;OAKG;IACH,KAAK,CAAC,MAAM,EAAE,YAAY,GAAG,QAAQ,EAAE,CAMtC;CAmEF;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,EAAE,EAAE,QAAQ,GAAG,eAAe,EAAE,CAEpE;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,QAAQ,GAAG,SAAS,CA8B1D;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,qCAAqC,CACnD,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,GACX,IAAI,CA4DN"}
+{"version":3,"file":"backfilling.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/replicator/schema/backfilling.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAEH,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,KAAK,EAAC,QAAQ,EAAY,MAAM,iCAAiC,CAAC;AAGzE,OAAO,KAAK,EAEV,oBAAoB,EACpB,UAAU,EACV,YAAY,EAEb,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,kDAAkD,CAAC;AACtF,OAAO,EAGL,KAAK,QAAQ,EACb,KAAK,SAAS,EACf,MAAM,0BAA0B,CAAC;AAElC,eAAO,MAAM,iBAAiB,sBAAsB,CAAC;AAYrD,eAAO,MAAM,wBAAwB,sQASpC,CAAC;AAEF;;;;;;;;;;;;;;;;;;GAkBG;AACH,qBAAa,kBAAkB;;IAW7B,YAAY,EAAE,EAAE,QAAQ,EAEvB;IAED;;;;;OAKG;IACH,KAAK,CAAC,MAAM,EAAE,YAAY,GAAG,QAAQ,EAAE,CAMtC;IA6DD;;;OAGG;IACH,WAAW,CACT,KAAK,EAAE,UAAU,GAChB,GAAG,CAAC,MAAM,EAAE,oBAAoB,GAAG,SAAS,CAAC,CAgB/C;IAED,qEAAqE;IACrE,WAAW,CACT,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,oBAAoB,GAAG,SAAS,GACzC,IAAI,CAUN;IAED;;;;OAIG;IACH,QAAQ,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI,CAE5D;CAQF;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,EAAE,EAAE,QAAQ,GAAG,eAAe,EAAE,CAEpE;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,EAAE,EAAE,QAAQ,GAAG,eAAe,EAAE,CAkBpE;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,QAAQ,GAAG,SAAS,CA8B1D;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,qCAAqC,CACnD,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,GACX,IAAI,CA4DN"}

package/out/zero-cache/src/services/replicator/schema/backfilling.js

@@ -11,6 +11,7 @@ var CREATE_BACKFILLING_TABLE = `
     "table"    TEXT NOT NULL,
     "column"   TEXT NOT NULL,
     "backfill" TEXT NOT NULL,
+    "progress" TEXT,
     PRIMARY KEY ("schema", "table", "column")
   );
 `;
@@ -40,6 +41,8 @@ var BackfillingTracker = class {
 	#dropTable;
 	#renameColumn;
 	#dropColumn;
+	#getProgress;
+	#setProgress;
 	constructor(db) {
 		this.#db = db;
 	}
@@ -62,7 +65,7 @@ var BackfillingTracker = class {
           INSERT INTO "${BACKFILLING_TABLE}"
             ("schema", "table", "column", "backfill") VALUES (?, ?, ?, ?)
             ON CONFLICT ("schema", "table", "column")
-            DO UPDATE SET "backfill" = excluded."backfill"
+            DO UPDATE SET "backfill" = excluded."backfill", "progress" = NULL
         `)).run(op.table.schema, op.table.name, op.column, BigIntJSON.stringify(op.backfill));
 				break;
 			case "rename-table":
@@ -92,6 +95,36 @@ var BackfillingTracker = class {
 			default: unreachable(op);
 		}
 	}
+	/**
+	* Returns the tracked progress of the backfilling columns of the `table`,
+	* keyed by column name. Columns that are not backfilling are absent.
+	*/
+	getProgress(table) {
+		const rows = (this.#getProgress ??= this.#db.prepare(`
+      SELECT "column", "progress" FROM "${BACKFILLING_TABLE}"
+        WHERE "schema" = ? AND "table" = ?
+    `)).all(table.schema, table.name);
+		return new Map(rows.map(({ column, progress }) => [column, progress === null ? void 0 : BigIntJSON.parse(progress)]));
+	}
+	/** Sets (or clears) the tracked progress of a backfilling column. */
+	setProgress(table, column, progress) {
+		(this.#setProgress ??= this.#db.prepare(`
+      UPDATE "${BACKFILLING_TABLE}" SET "progress" = ?
+        WHERE "schema" = ? AND "table" = ? AND "column" = ?
+    `)).run(progress === void 0 ? null : BigIntJSON.stringify(progress), table.schema, table.name, column);
+	}
+	/**
+	* Marks the backfill of the `columns` as complete. This is the subset of
+	* a `backfill-completed` message's columns (including row key columns)
+	* whose progress is contiguous with the message.
+	*/
+	complete(table, columns) {
+		this.#run({
+			op: "complete-backfill",
+			table,
+			columns
+		});
+	}
 	#deleteColumn(table, column) {
 		(this.#dropColumn ??= this.#db.prepare(`
       DELETE FROM "${BACKFILLING_TABLE}"
@@ -111,6 +144,25 @@ function readBackfillRequests(db) {
 	return backfillRequestsFrom(readReplicaCookies(db));
 }
 /**
+* Returns the {@link BackfillRequest}s of the replica's pending backfills,
+* including the tracked progress of each column. These are the backfills that
+* a subscriber reports to the change-streamer when starting a subscription.
+*
+* The caller must pair this snapshot with the replica state version.
+*/
+function readReplicaBackfills(db) {
+	const requests = readBackfillRequests(db);
+	const progress = db.prepare(`
+      SELECT "schema", "table", "column", "progress"
+        FROM "${BACKFILLING_TABLE}" WHERE "progress" IS NOT NULL
+    `).all();
+	for (const { schema, table, column, progress: mark } of progress) {
+		const col = requests.find((r) => r.table.schema === schema && r.table.name === table)?.columns[column];
+		if (col) col.progress = BigIntJSON.parse(mark);
+	}
+	return requests;
+}
+/**
 * The replica's whole cookie set, in the shape the change log holds it — the
 * seed a change log would be initialized from, and the replica-derived half of
 * the initialization comparison.
@@ -209,6 +261,6 @@ function populateBackfillingFromColumnMetadata(lc, db) {
 	lc.info?.(`seeded "${BACKFILLING_TABLE}" with ${backfilling.length} in-flight backfill(s), ${unresolved} of which could not be resolved`);
 }
 //#endregion
-export { BackfillingTracker, CREATE_BACKFILLING_TABLE, populateBackfillingFromColumnMetadata, readBackfillRequests, readReplicaCookies };
+export { BACKFILLING_TABLE, BackfillingTracker, CREATE_BACKFILLING_TABLE, populateBackfillingFromColumnMetadata, readBackfillRequests, readReplicaBackfills, readReplicaCookies };
 
 //# sourceMappingURL=backfilling.js.map

package/out/zero-cache/src/services/replicator/schema/backfilling.js.map

@@ -1 +1 @@
-{"version":3,"file":"backfilling.js","names":["#db","#run","#upsert","#renameTable","#dropTable","#renameColumn","#deleteColumn","#dropColumn"],"sources":["../../../../../../../zero-cache/src/services/replicator/schema/backfilling.ts"],"sourcesContent":["/**\n * The replica's backfill cookie, keyed by upstream identity.\n *\n * `_zero.column_metadata.backfill` already records which columns are being\n * backfilled, and it stays authoritative for `isBackfilling`, which is what the\n * replica's own read paths use. What it cannot do is name the table an upstream\n * would recognize: it is keyed by *lite* table name, and `liteTableName()`\n * collapses the `public` schema and has no inverse anywhere in the tree. A lite\n * name containing a dot is ambiguous — `{public, \"a.b\"}` and `{a, b}` produce\n * the same string — and a table with backfilling columns and no\n * `_zero.tableMetadata` row (legal, since its metadata is optional) cannot be\n * resolved at all.\n *\n * That is the whole of what blocks a change log from being initialized out of a\n * restored replica, so this table carries the identity the reconstruction needs:\n * a straight row copy into the change log's own cookie tables, with nothing to\n * reconstruct and nothing to guess. The replica's copy is the *seed* source; the\n * change log's copy (`replicator/change-log-cookies.ts`) is the *working* set,\n * folded forward from the seed as the log's head advances.\n *\n * The cost is a second copy of `backfill` inside the replica, taken\n * deliberately — though not to save writes: `_zero.column_metadata` is written\n * only by initial sync, the DDL handlers, and backfill completion, so two more\n * columns on it would be free. It is a matter of shape. That table is keyed by\n * lite table name, so `schema`/`table` would be non-key duplicates of its own\n * key, to be held in sync with it across every rename with no answer to which\n * encoding is authoritative. This one is keyed exactly as `cdc.backfilling` and\n * `_zero.changeLogBackfilling` are, which is what makes the change log's\n * initialization comparison a row-set diff rather than a translation, and what\n * lets all three stores interpret the same fold. It also holds in-flight backfills and nothing else,\n * where the same query against `column_metadata` is an unindexed scan over a row\n * per column of every table.\n *\n * This table is write-only from the replica's perspective. Nothing in the\n * replicator reads it; it exists to be read by the change-streamer.\n */\n\nimport type {LogContext} from '@rocicorp/logger';\nimport {unreachable} from '../../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../../shared/src/bigint-json.ts';\nimport type {Database, Statement} from '../../../../../zqlite/src/db.ts';\nimport {getOrCreateCounter} from '../../../observability/metrics.ts';\nimport {liteTableName} from '../../../types/names.ts';\nimport type {\n  BackfillID,\n  Identifier,\n  SchemaChange,\n  TableMetadata,\n} from '../../change-source/protocol/current/data.ts';\nimport type {BackfillRequest} from '../../change-source/protocol/current/upstream.ts';\nimport {\n  backfillRequestsFrom,\n  cookieOps,\n  type CookieOp,\n  type CookieSet,\n} from '../change-log-cookies.ts';\n\nexport const BACKFILLING_TABLE = '_zero.backfilling';\n\n// `backfill` holds the JSON that `cdc.backfilling` holds as JSONB. SQLite has\n// no JSONB, and nothing here queries into the document: it is stored to be\n// handed back to the change source verbatim.\nexport const CREATE_BACKFILLING_TABLE = /*sql*/ `\n  CREATE TABLE \"${BACKFILLING_TABLE}\" (\n    \"schema\"   TEXT NOT NULL,\n    \"table\"    TEXT NOT NULL,\n    \"column\"   TEXT NOT NULL,\n    \"backfill\" TEXT NOT NULL,\n    PRIMARY KEY (\"schema\", \"table\", \"column\")\n  );\n`;\n\n/**\n * The replica's interpreter of the cookie fold in\n * `replicator/change-log-cookies.ts`.\n *\n * The fold is shared rather than re-implemented for the same reason the\n * Postgres and SQLite change logs share it: the three stores must agree on every\n * transition forever, and the failure mode when they drift is a backfill that is\n * silently never re-requested, on a column that is then silently never\n * populated. Sharing it makes the replica-versus-Postgres comparison a\n * regression test on the interpreters rather than on three implementations of\n * the same switch.\n *\n * Only the backfill half of the fold is interpreted here. The replica's metadata\n * cookie is `_zero.tableMetadata.upstreamMetadata`, which `TableMetadataTracker`\n * already maintains at the same sites, so an `upsert-metadata` op is inert.\n *\n * Statements are lazily prepared, matching `TableMetadataTracker`: a replica\n * that never sees a schema change never prepares any of them.\n */\nexport class BackfillingTracker {\n  readonly #db: Database;\n\n  #upsert: Statement | undefined;\n  #renameTable: Statement | undefined;\n  #dropTable: Statement | undefined;\n  #renameColumn: Statement | undefined;\n  #dropColumn: Statement | undefined;\n\n  constructor(db: Database) {\n    this.#db = db;\n  }\n\n  /**\n   * Applies the change's backfill-cookie ops, returning the ops that were\n   * applied. Changes that carry no backfill state — index changes, metadata-only\n   * updates, and the `create-table` / `add-column` variants from a change source\n   * that does not support backfill — apply nothing.\n   */\n  apply(change: SchemaChange): CookieOp[] {\n    const ops = cookieOps(change);\n    for (const op of ops) {\n      this.#run(op);\n    }\n    return ops;\n  }\n\n  #run(op: CookieOp): void {\n    switch (op.op) {\n      case 'upsert-metadata':\n        // Maintained by TableMetadataTracker in \"_zero.tableMetadata\".\n        break;\n\n      case 'upsert-backfill':\n        (this.#upsert ??= this.#db.prepare(/*sql*/ `\n          INSERT INTO \"${BACKFILLING_TABLE}\"\n            (\"schema\", \"table\", \"column\", \"backfill\") VALUES (?, ?, ?, ?)\n            ON CONFLICT (\"schema\", \"table\", \"column\")\n            DO UPDATE SET \"backfill\" = excluded.\"backfill\"\n        `)).run(\n          op.table.schema,\n          op.table.name,\n          op.column,\n          BigIntJSON.stringify(op.backfill),\n        );\n        break;\n\n      case 'rename-table':\n        (this.#renameTable ??= this.#db.prepare(/*sql*/ `\n          UPDATE \"${BACKFILLING_TABLE}\"\n            SET \"schema\" = ?, \"table\" = ? WHERE \"schema\" = ? AND \"table\" = ?\n        `)).run(op.new.schema, op.new.name, op.old.schema, op.old.name);\n        break;\n\n      case 'drop-table':\n        (this.#dropTable ??= this.#db.prepare(/*sql*/ `\n          DELETE FROM \"${BACKFILLING_TABLE}\"\n            WHERE \"schema\" = ? AND \"table\" = ?\n        `)).run(op.table.schema, op.table.name);\n        break;\n\n      case 'rename-column':\n        (this.#renameColumn ??= this.#db.prepare(/*sql*/ `\n          UPDATE \"${BACKFILLING_TABLE}\" SET \"column\" = ?\n            WHERE \"schema\" = ? AND \"table\" = ? AND \"column\" = ?\n        `)).run(op.new, op.table.schema, op.table.name, op.old);\n        break;\n\n      case 'drop-column':\n        this.#deleteColumn(op.table, op.column);\n        break;\n\n      case 'complete-backfill':\n        // A per-column delete rather than an `IN` list, which cannot be a\n        // single prepared statement across arities. The columns of one\n        // completed backfill are few, and this runs only on schema changes.\n        for (const column of op.columns) {\n          this.#deleteColumn(op.table, column);\n        }\n        break;\n\n      default:\n        unreachable(op);\n    }\n  }\n\n  #deleteColumn(table: Identifier, column: string): void {\n    (this.#dropColumn ??= this.#db.prepare(/*sql*/ `\n      DELETE FROM \"${BACKFILLING_TABLE}\"\n        WHERE \"schema\" = ? AND \"table\" = ? AND \"column\" = ?\n    `)).run(table.schema, table.name, column);\n  }\n}\n\n/**\n * Returns the {@link BackfillRequest}s from the current replica cookies. The\n * result has the same shape as the Postgres initialization result.\n *\n * The caller must pair this snapshot with the replica state version. When the\n * replica trails the change log, the comparison applies the missing schema\n * changes before it compares the results.\n */\nexport function readBackfillRequests(db: Database): BackfillRequest[] {\n  return backfillRequestsFrom(readReplicaCookies(db));\n}\n\n/**\n * The replica's whole cookie set, in the shape the change log holds it — the\n * seed a change log would be initialized from, and the replica-derived half of\n * the initialization comparison.\n *\n * The metadata half is filtered on `upstreamMetadata IS NOT NULL` because\n * `_zero.tableMetadata` carries `minRowVersion` as well, so a row exists for\n * every table whose rows were ever force-re-downloaded, whether or not upstream\n * ever sent metadata for it. Neither cookie store can represent that row:\n * `cdc.tableMetadata.\"metadata\"` and `_zero.changeLogTableMetadata.\"metadata\"`\n * are both `NOT NULL`.\n *\n * Only meaningful paired with `_zero.replicationState.stateVersion`, read in\n * the same snapshot (invariant 15).\n */\nexport function readReplicaCookies(db: Database): CookieSet {\n  const tableMetadata = db\n    .prepare(/*sql*/ `\n      SELECT \"schema\", \"table\", \"upstreamMetadata\" AS \"metadata\"\n        FROM \"_zero.tableMetadata\"\n        WHERE \"upstreamMetadata\" IS NOT NULL\n        ORDER BY \"schema\", \"table\"\n    `)\n    .all<{schema: string; table: string; metadata: string}>()\n    .map(({schema, table, metadata}) => ({\n      schema,\n      table,\n      metadata: BigIntJSON.parse(metadata) as TableMetadata,\n    }));\n\n  const backfilling = db\n    .prepare(/*sql*/ `\n      SELECT \"schema\", \"table\", \"column\", \"backfill\"\n        FROM \"${BACKFILLING_TABLE}\"\n        ORDER BY \"schema\", \"table\", \"column\"\n    `)\n    .all<{schema: string; table: string; column: string; backfill: string}>()\n    .map(({schema, table, column, backfill}) => ({\n      schema,\n      table,\n      column,\n      backfill: BigIntJSON.parse(backfill) as BackfillID,\n    }));\n\n  return {tableMetadata, backfilling};\n}\n\n/**\n * Seeds {@link BACKFILLING_TABLE} from `_zero.column_metadata`, resolving each\n * lite table name back to its upstream identity. Used by the v17 migration.\n *\n * The resolution is exact for a table that has a `_zero.tableMetadata` row, and\n * exact for a lite name with no dot in it — `liteTableName()` only omits the\n * schema for `public`. What is left over is an in-flight backfill, on a\n * non-`public` schema, on a table with no metadata row, at the instant of\n * upgrade. Those get the best-effort first-dot split and are counted, because a\n * wrong identity produces a `BackfillRequest` the change source will reject\n * rather than a silently wrong one, and because the initialization comparison\n * surfaces it before anything depends on it. The expected count is zero.\n *\n * A full replace rather than an insert-if-absent: `migrateData` re-runs after a\n * rollback and roll-forward, during which a v16 zero-cache advanced the replica\n * without knowing this table existed. Rebuilding it from `column_metadata` —\n * which that zero-cache *did* maintain — is what makes the re-run both\n * idempotent and correct.\n */\nexport function populateBackfillingFromColumnMetadata(\n  lc: LogContext,\n  db: Database,\n): void {\n  db.prepare(/*sql*/ `DELETE FROM \"${BACKFILLING_TABLE}\"`).run();\n\n  const backfilling = db\n    .prepare(/*sql*/ `\n      SELECT table_name, column_name, backfill FROM \"_zero.column_metadata\"\n        WHERE backfill IS NOT NULL\n        ORDER BY table_name, column_name\n    `)\n    .all<{table_name: string; column_name: string; backfill: string}>();\n\n  const unresolvable = getOrCreateCounter(\n    'replica',\n    'backfilling_unresolvable_rows',\n    'In-flight backfills whose upstream table identity could not be recovered ' +\n      'from the replica when \"' +\n      BACKFILLING_TABLE +\n      '\" was created. A best-effort identity is stored; the expected count is 0.',\n  );\n  if (backfilling.length === 0) {\n    unresolvable.add(0); // the common case: publish the series anyway\n    return;\n  }\n\n  const byLiteName = new Map<string, Identifier>(\n    db\n      .prepare(/*sql*/ `SELECT \"schema\", \"table\" FROM \"_zero.tableMetadata\"`)\n      .all<{schema: string; table: string}>()\n      .map(({schema, table}) => [\n        liteTableName({schema, name: table}),\n        {schema, name: table},\n      ]),\n  );\n\n  const insert = db.prepare(/*sql*/ `\n    INSERT INTO \"${BACKFILLING_TABLE}\"\n      (\"schema\", \"table\", \"column\", \"backfill\") VALUES (?, ?, ?, ?)\n  `);\n  let unresolved = 0;\n  for (const {table_name: liteName, column_name, backfill} of backfilling) {\n    const known = byLiteName.get(liteName);\n    const dot = known ? -1 : liteName.indexOf('.');\n    const table = known ?? {\n      schema: dot < 0 ? 'public' : liteName.slice(0, dot),\n      name: dot < 0 ? liteName : liteName.slice(dot + 1),\n    };\n    if (!known && dot >= 0) {\n      unresolved++;\n      lc.warn?.(\n        `no metadata for backfilling table \"${liteName}\"; ` +\n          `assuming {schema: \"${table.schema}\", name: \"${table.name}\"}`,\n      );\n    }\n    insert.run(table.schema, table.name, column_name, backfill);\n  }\n  unresolvable.add(unresolved);\n  lc.info?.(\n    `seeded \"${BACKFILLING_TABLE}\" with ${backfilling.length} in-flight ` +\n      `backfill(s), ${unresolved} of which could not be resolved`,\n  );\n}\n"],"mappings":";;;;;;AAyDA,IAAa,oBAAoB;AAKjC,IAAa,2BAAmC;kBAC9B,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4BpC,IAAa,qBAAb,MAAgC;CAC9B;CAEA;CACA;CACA;CACA;CACA;CAEA,YAAY,IAAc;EACxB,KAAKA,MAAM;CACb;;;;;;;CAQA,MAAM,QAAkC;EACtC,MAAM,MAAM,UAAU,MAAM;EAC5B,KAAK,MAAM,MAAM,KACf,KAAKC,KAAK,EAAE;EAEd,OAAO;CACT;CAEA,KAAK,IAAoB;EACvB,QAAQ,GAAG,IAAX;GACE,KAAK,mBAEH;GAEF,KAAK;IACH,CAAC,KAAKC,YAAY,KAAKF,IAAI,QAAgB;yBAC1B,kBAAkB;;;;SAIlC,GAAG,IACF,GAAG,MAAM,QACT,GAAG,MAAM,MACT,GAAG,QACH,WAAW,UAAU,GAAG,QAAQ,CAClC;IACA;GAEF,KAAK;IACH,CAAC,KAAKG,iBAAiB,KAAKH,IAAI,QAAgB;oBACpC,kBAAkB;;SAE7B,GAAG,IAAI,GAAG,IAAI,QAAQ,GAAG,IAAI,MAAM,GAAG,IAAI,QAAQ,GAAG,IAAI,IAAI;IAC9D;GAEF,KAAK;IACH,CAAC,KAAKI,eAAe,KAAKJ,IAAI,QAAgB;yBAC7B,kBAAkB;;SAElC,GAAG,IAAI,GAAG,MAAM,QAAQ,GAAG,MAAM,IAAI;IACtC;GAEF,KAAK;IACH,CAAC,KAAKK,kBAAkB,KAAKL,IAAI,QAAgB;oBACrC,kBAAkB;;SAE7B,GAAG,IAAI,GAAG,KAAK,GAAG,MAAM,QAAQ,GAAG,MAAM,MAAM,GAAG,GAAG;IACtD;GAEF,KAAK;IACH,KAAKM,cAAc,GAAG,OAAO,GAAG,MAAM;IACtC;GAEF,KAAK;IAIH,KAAK,MAAM,UAAU,GAAG,SACtB,KAAKA,cAAc,GAAG,OAAO,MAAM;IAErC;GAEF,SACE,YAAY,EAAE;EAClB;CACF;CAEA,cAAc,OAAmB,QAAsB;EACrD,CAAC,KAAKC,gBAAgB,KAAKP,IAAI,QAAgB;qBAC9B,kBAAkB;;KAElC,GAAG,IAAI,MAAM,QAAQ,MAAM,MAAM,MAAM;CAC1C;AACF;;;;;;;;;AAUA,SAAgB,qBAAqB,IAAiC;CACpE,OAAO,qBAAqB,mBAAmB,EAAE,CAAC;AACpD;;;;;;;;;;;;;;;;AAiBA,SAAgB,mBAAmB,IAAyB;CA6B1D,OAAO;EAAC,eA5Bc,GACnB,QAAgB;;;;;KAKhB,EACA,IAAuD,EACvD,KAAK,EAAC,QAAQ,OAAO,gBAAe;GACnC;GACA;GACA,UAAU,WAAW,MAAM,QAAQ;EACrC,EAgBM;EAAe,aAdH,GACjB,QAAgB;;gBAEL,kBAAkB;;KAE7B,EACA,IAAuE,EACvE,KAAK,EAAC,QAAQ,OAAO,QAAQ,gBAAe;GAC3C;GACA;GACA;GACA,UAAU,WAAW,MAAM,QAAQ;EACrC,EAEqB;CAAW;AACpC;;;;;;;;;;;;;;;;;;;;AAqBA,SAAgB,sCACd,IACA,IACM;CACN,GAAG,QAAgB,gBAAgB,kBAAkB,EAAE,EAAE,IAAI;CAE7D,MAAM,cAAc,GACjB,QAAgB;;;;KAIhB,EACA,IAAiE;CAEpE,MAAM,eAAe,mBACnB,WACA,iCACA,sGAEE,oBACA,4EACJ;CACA,IAAI,YAAY,WAAW,GAAG;EAC5B,aAAa,IAAI,CAAC;EAClB;CACF;CAEA,MAAM,aAAa,IAAI,IACrB,GACG,QAAgB,qDAAqD,EACrE,IAAqC,EACrC,KAAK,EAAC,QAAQ,YAAW,CACxB,cAAc;EAAC;EAAQ,MAAM;CAAK,CAAC,GACnC;EAAC;EAAQ,MAAM;CAAK,CACtB,CAAC,CACL;CAEA,MAAM,SAAS,GAAG,QAAgB;mBACjB,kBAAkB;;GAElC;CACD,IAAI,aAAa;CACjB,KAAK,MAAM,EAAC,YAAY,UAAU,aAAa,cAAa,aAAa;EACvE,MAAM,QAAQ,WAAW,IAAI,QAAQ;EACrC,MAAM,MAAM,QAAQ,KAAK,SAAS,QAAQ,GAAG;EAC7C,MAAM,QAAQ,SAAS;GACrB,QAAQ,MAAM,IAAI,WAAW,SAAS,MAAM,GAAG,GAAG;GAClD,MAAM,MAAM,IAAI,WAAW,SAAS,MAAM,MAAM,CAAC;EACnD;EACA,IAAI,CAAC,SAAS,OAAO,GAAG;GACtB;GACA,GAAG,OACD,sCAAsC,SAAS,wBACvB,MAAM,OAAO,YAAY,MAAM,KAAK,GAC9D;EACF;EACA,OAAO,IAAI,MAAM,QAAQ,MAAM,MAAM,aAAa,QAAQ;CAC5D;CACA,aAAa,IAAI,UAAU;CAC3B,GAAG,OACD,WAAW,kBAAkB,SAAS,YAAY,OAAO,0BACvC,WAAW,gCAC/B;AACF"}
+{"version":3,"file":"backfilling.js","names":["#db","#run","#upsert","#renameTable","#dropTable","#renameColumn","#deleteColumn","#getProgress","#setProgress","#dropColumn"],"sources":["../../../../../../../zero-cache/src/services/replicator/schema/backfilling.ts"],"sourcesContent":["/**\n * The replica's backfill cookie, keyed by upstream identity.\n *\n * `_zero.column_metadata.backfill` already records which columns are being\n * backfilled, and it stays authoritative for `isBackfilling`, which is what the\n * replica's own read paths use. What it cannot do is name the table an upstream\n * would recognize: it is keyed by *lite* table name, and `liteTableName()`\n * collapses the `public` schema and has no inverse anywhere in the tree. A lite\n * name containing a dot is ambiguous — `{public, \"a.b\"}` and `{a, b}` produce\n * the same string — and a table with backfilling columns and no\n * `_zero.tableMetadata` row (legal, since its metadata is optional) cannot be\n * resolved at all.\n *\n * That is the whole of what blocks a change log from being initialized out of a\n * restored replica, so this table carries the identity the reconstruction needs:\n * a straight row copy into the change log's own cookie tables, with nothing to\n * reconstruct and nothing to guess. The replica's copy is the *seed* source; the\n * change log's copy (`replicator/change-log-cookies.ts`) is the *working* set,\n * folded forward from the seed as the log's head advances.\n *\n * The cost is a second copy of `backfill` inside the replica, taken\n * deliberately — though not to save writes: `_zero.column_metadata` is written\n * only by initial sync, the DDL handlers, and backfill completion, so two more\n * columns on it would be free. It is a matter of shape. That table is keyed by\n * lite table name, so `schema`/`table` would be non-key duplicates of its own\n * key, to be held in sync with it across every rename with no answer to which\n * encoding is authoritative. This one is keyed exactly as `cdc.backfilling` and\n * `_zero.changeLogBackfilling` are, which is what makes the change log's\n * initialization comparison a row-set diff rather than a translation, and what\n * lets all three stores interpret the same fold. It also holds in-flight backfills and nothing else,\n * where the same query against `column_metadata` is an unindexed scan over a row\n * per column of every table.\n *\n * This table is write-only from the replica's perspective. Nothing in the\n * replicator reads it; it exists to be read by the change-streamer.\n */\n\nimport type {LogContext} from '@rocicorp/logger';\nimport {unreachable} from '../../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../../shared/src/bigint-json.ts';\nimport type {Database, Statement} from '../../../../../zqlite/src/db.ts';\nimport {getOrCreateCounter} from '../../../observability/metrics.ts';\nimport {liteTableName} from '../../../types/names.ts';\nimport type {\n  BackfillID,\n  BackfillProgressMark,\n  Identifier,\n  SchemaChange,\n  TableMetadata,\n} from '../../change-source/protocol/current/data.ts';\nimport type {BackfillRequest} from '../../change-source/protocol/current/upstream.ts';\nimport {\n  backfillRequestsFrom,\n  cookieOps,\n  type CookieOp,\n  type CookieSet,\n} from '../change-log-cookies.ts';\n\nexport const BACKFILLING_TABLE = '_zero.backfilling';\n\n// `backfill` holds the JSON that `cdc.backfilling` holds as JSONB. SQLite has\n// no JSONB, and nothing here queries into the document: it is stored to be\n// handed back to the change source verbatim.\n//\n// `progress` holds the JSON of the BackfillProgressMark up to which the\n// column's backfill data has been received contiguously (see\n// `acceptBackfill()`), or NULL if no data has been received (i.e. the backfill\n// must start from scratch). It is deliberately a separate column from\n// `backfill` so that the cookie set (and its comparison with the change log's\n// cookie jars) is unaffected by it.\nexport const CREATE_BACKFILLING_TABLE = /*sql*/ `\n  CREATE TABLE \"${BACKFILLING_TABLE}\" (\n    \"schema\"   TEXT NOT NULL,\n    \"table\"    TEXT NOT NULL,\n    \"column\"   TEXT NOT NULL,\n    \"backfill\" TEXT NOT NULL,\n    \"progress\" TEXT,\n    PRIMARY KEY (\"schema\", \"table\", \"column\")\n  );\n`;\n\n/**\n * The replica's interpreter of the cookie fold in\n * `replicator/change-log-cookies.ts`.\n *\n * The fold is shared rather than re-implemented for the same reason the\n * Postgres and SQLite change logs share it: the three stores must agree on every\n * transition forever, and the failure mode when they drift is a backfill that is\n * silently never re-requested, on a column that is then silently never\n * populated. Sharing it makes the replica-versus-Postgres comparison a\n * regression test on the interpreters rather than on three implementations of\n * the same switch.\n *\n * Only the backfill half of the fold is interpreted here. The replica's metadata\n * cookie is `_zero.tableMetadata.upstreamMetadata`, which `TableMetadataTracker`\n * already maintains at the same sites, so an `upsert-metadata` op is inert.\n *\n * Statements are lazily prepared, matching `TableMetadataTracker`: a replica\n * that never sees a schema change never prepares any of them.\n */\nexport class BackfillingTracker {\n  readonly #db: Database;\n\n  #upsert: Statement | undefined;\n  #renameTable: Statement | undefined;\n  #dropTable: Statement | undefined;\n  #renameColumn: Statement | undefined;\n  #dropColumn: Statement | undefined;\n  #getProgress: Statement | undefined;\n  #setProgress: Statement | undefined;\n\n  constructor(db: Database) {\n    this.#db = db;\n  }\n\n  /**\n   * Applies the change's backfill-cookie ops, returning the ops that were\n   * applied. Changes that carry no backfill state — index changes, metadata-only\n   * updates, and the `create-table` / `add-column` variants from a change source\n   * that does not support backfill — apply nothing.\n   */\n  apply(change: SchemaChange): CookieOp[] {\n    const ops = cookieOps(change);\n    for (const op of ops) {\n      this.#run(op);\n    }\n    return ops;\n  }\n\n  #run(op: CookieOp): void {\n    switch (op.op) {\n      case 'upsert-metadata':\n        // Maintained by TableMetadataTracker in \"_zero.tableMetadata\".\n        break;\n\n      case 'upsert-backfill':\n        (this.#upsert ??= this.#db.prepare(/*sql*/ `\n          INSERT INTO \"${BACKFILLING_TABLE}\"\n            (\"schema\", \"table\", \"column\", \"backfill\") VALUES (?, ?, ?, ?)\n            ON CONFLICT (\"schema\", \"table\", \"column\")\n            DO UPDATE SET \"backfill\" = excluded.\"backfill\", \"progress\" = NULL\n        `)).run(\n          op.table.schema,\n          op.table.name,\n          op.column,\n          BigIntJSON.stringify(op.backfill),\n        );\n        break;\n\n      case 'rename-table':\n        (this.#renameTable ??= this.#db.prepare(/*sql*/ `\n          UPDATE \"${BACKFILLING_TABLE}\"\n            SET \"schema\" = ?, \"table\" = ? WHERE \"schema\" = ? AND \"table\" = ?\n        `)).run(op.new.schema, op.new.name, op.old.schema, op.old.name);\n        break;\n\n      case 'drop-table':\n        (this.#dropTable ??= this.#db.prepare(/*sql*/ `\n          DELETE FROM \"${BACKFILLING_TABLE}\"\n            WHERE \"schema\" = ? AND \"table\" = ?\n        `)).run(op.table.schema, op.table.name);\n        break;\n\n      case 'rename-column':\n        (this.#renameColumn ??= this.#db.prepare(/*sql*/ `\n          UPDATE \"${BACKFILLING_TABLE}\" SET \"column\" = ?\n            WHERE \"schema\" = ? AND \"table\" = ? AND \"column\" = ?\n        `)).run(op.new, op.table.schema, op.table.name, op.old);\n        break;\n\n      case 'drop-column':\n        this.#deleteColumn(op.table, op.column);\n        break;\n\n      case 'complete-backfill':\n        // A per-column delete rather than an `IN` list, which cannot be a\n        // single prepared statement across arities. The columns of one\n        // completed backfill are few, and this runs only on schema changes.\n        for (const column of op.columns) {\n          this.#deleteColumn(op.table, column);\n        }\n        break;\n\n      default:\n        unreachable(op);\n    }\n  }\n\n  /**\n   * Returns the tracked progress of the backfilling columns of the `table`,\n   * keyed by column name. Columns that are not backfilling are absent.\n   */\n  getProgress(\n    table: Identifier,\n  ): Map<string, BackfillProgressMark | undefined> {\n    const rows = (this.#getProgress ??= this.#db.prepare(/*sql*/ `\n      SELECT \"column\", \"progress\" FROM \"${BACKFILLING_TABLE}\"\n        WHERE \"schema\" = ? AND \"table\" = ?\n    `)).all<{column: string; progress: string | null}>(\n      table.schema,\n      table.name,\n    );\n    return new Map(\n      rows.map(({column, progress}) => [\n        column,\n        progress === null\n          ? undefined\n          : (BigIntJSON.parse(progress) as BackfillProgressMark),\n      ]),\n    );\n  }\n\n  /** Sets (or clears) the tracked progress of a backfilling column. */\n  setProgress(\n    table: Identifier,\n    column: string,\n    progress: BackfillProgressMark | undefined,\n  ): void {\n    (this.#setProgress ??= this.#db.prepare(/*sql*/ `\n      UPDATE \"${BACKFILLING_TABLE}\" SET \"progress\" = ?\n        WHERE \"schema\" = ? AND \"table\" = ? AND \"column\" = ?\n    `)).run(\n      progress === undefined ? null : BigIntJSON.stringify(progress),\n      table.schema,\n      table.name,\n      column,\n    );\n  }\n\n  /**\n   * Marks the backfill of the `columns` as complete. This is the subset of\n   * a `backfill-completed` message's columns (including row key columns)\n   * whose progress is contiguous with the message.\n   */\n  complete(table: Identifier, columns: readonly string[]): void {\n    this.#run({op: 'complete-backfill', table, columns});\n  }\n\n  #deleteColumn(table: Identifier, column: string): void {\n    (this.#dropColumn ??= this.#db.prepare(/*sql*/ `\n      DELETE FROM \"${BACKFILLING_TABLE}\"\n        WHERE \"schema\" = ? AND \"table\" = ? AND \"column\" = ?\n    `)).run(table.schema, table.name, column);\n  }\n}\n\n/**\n * Returns the {@link BackfillRequest}s from the current replica cookies. The\n * result has the same shape as the Postgres initialization result.\n *\n * The caller must pair this snapshot with the replica state version. When the\n * replica trails the change log, the comparison applies the missing schema\n * changes before it compares the results.\n */\nexport function readBackfillRequests(db: Database): BackfillRequest[] {\n  return backfillRequestsFrom(readReplicaCookies(db));\n}\n\n/**\n * Returns the {@link BackfillRequest}s of the replica's pending backfills,\n * including the tracked progress of each column. These are the backfills that\n * a subscriber reports to the change-streamer when starting a subscription.\n *\n * The caller must pair this snapshot with the replica state version.\n */\nexport function readReplicaBackfills(db: Database): BackfillRequest[] {\n  const requests = readBackfillRequests(db);\n  const progress = db\n    .prepare(/*sql*/ `\n      SELECT \"schema\", \"table\", \"column\", \"progress\"\n        FROM \"${BACKFILLING_TABLE}\" WHERE \"progress\" IS NOT NULL\n    `)\n    .all<{schema: string; table: string; column: string; progress: string}>();\n  for (const {schema, table, column, progress: mark} of progress) {\n    const req = requests.find(\n      r => r.table.schema === schema && r.table.name === table,\n    );\n    const col = req?.columns[column];\n    if (col) {\n      col.progress = BigIntJSON.parse(mark) as BackfillProgressMark;\n    }\n  }\n  return requests;\n}\n\n/**\n * The replica's whole cookie set, in the shape the change log holds it — the\n * seed a change log would be initialized from, and the replica-derived half of\n * the initialization comparison.\n *\n * The metadata half is filtered on `upstreamMetadata IS NOT NULL` because\n * `_zero.tableMetadata` carries `minRowVersion` as well, so a row exists for\n * every table whose rows were ever force-re-downloaded, whether or not upstream\n * ever sent metadata for it. Neither cookie store can represent that row:\n * `cdc.tableMetadata.\"metadata\"` and `_zero.changeLogTableMetadata.\"metadata\"`\n * are both `NOT NULL`.\n *\n * Only meaningful paired with `_zero.replicationState.stateVersion`, read in\n * the same snapshot (invariant 15).\n */\nexport function readReplicaCookies(db: Database): CookieSet {\n  const tableMetadata = db\n    .prepare(/*sql*/ `\n      SELECT \"schema\", \"table\", \"upstreamMetadata\" AS \"metadata\"\n        FROM \"_zero.tableMetadata\"\n        WHERE \"upstreamMetadata\" IS NOT NULL\n        ORDER BY \"schema\", \"table\"\n    `)\n    .all<{schema: string; table: string; metadata: string}>()\n    .map(({schema, table, metadata}) => ({\n      schema,\n      table,\n      metadata: BigIntJSON.parse(metadata) as TableMetadata,\n    }));\n\n  const backfilling = db\n    .prepare(/*sql*/ `\n      SELECT \"schema\", \"table\", \"column\", \"backfill\"\n        FROM \"${BACKFILLING_TABLE}\"\n        ORDER BY \"schema\", \"table\", \"column\"\n    `)\n    .all<{schema: string; table: string; column: string; backfill: string}>()\n    .map(({schema, table, column, backfill}) => ({\n      schema,\n      table,\n      column,\n      backfill: BigIntJSON.parse(backfill) as BackfillID,\n    }));\n\n  return {tableMetadata, backfilling};\n}\n\n/**\n * Seeds {@link BACKFILLING_TABLE} from `_zero.column_metadata`, resolving each\n * lite table name back to its upstream identity. Used by the v17 migration.\n *\n * The resolution is exact for a table that has a `_zero.tableMetadata` row, and\n * exact for a lite name with no dot in it — `liteTableName()` only omits the\n * schema for `public`. What is left over is an in-flight backfill, on a\n * non-`public` schema, on a table with no metadata row, at the instant of\n * upgrade. Those get the best-effort first-dot split and are counted, because a\n * wrong identity produces a `BackfillRequest` the change source will reject\n * rather than a silently wrong one, and because the initialization comparison\n * surfaces it before anything depends on it. The expected count is zero.\n *\n * A full replace rather than an insert-if-absent: `migrateData` re-runs after a\n * rollback and roll-forward, during which a v16 zero-cache advanced the replica\n * without knowing this table existed. Rebuilding it from `column_metadata` —\n * which that zero-cache *did* maintain — is what makes the re-run both\n * idempotent and correct.\n */\nexport function populateBackfillingFromColumnMetadata(\n  lc: LogContext,\n  db: Database,\n): void {\n  db.prepare(/*sql*/ `DELETE FROM \"${BACKFILLING_TABLE}\"`).run();\n\n  const backfilling = db\n    .prepare(/*sql*/ `\n      SELECT table_name, column_name, backfill FROM \"_zero.column_metadata\"\n        WHERE backfill IS NOT NULL\n        ORDER BY table_name, column_name\n    `)\n    .all<{table_name: string; column_name: string; backfill: string}>();\n\n  const unresolvable = getOrCreateCounter(\n    'replica',\n    'backfilling_unresolvable_rows',\n    'In-flight backfills whose upstream table identity could not be recovered ' +\n      'from the replica when \"' +\n      BACKFILLING_TABLE +\n      '\" was created. A best-effort identity is stored; the expected count is 0.',\n  );\n  if (backfilling.length === 0) {\n    unresolvable.add(0); // the common case: publish the series anyway\n    return;\n  }\n\n  const byLiteName = new Map<string, Identifier>(\n    db\n      .prepare(/*sql*/ `SELECT \"schema\", \"table\" FROM \"_zero.tableMetadata\"`)\n      .all<{schema: string; table: string}>()\n      .map(({schema, table}) => [\n        liteTableName({schema, name: table}),\n        {schema, name: table},\n      ]),\n  );\n\n  const insert = db.prepare(/*sql*/ `\n    INSERT INTO \"${BACKFILLING_TABLE}\"\n      (\"schema\", \"table\", \"column\", \"backfill\") VALUES (?, ?, ?, ?)\n  `);\n  let unresolved = 0;\n  for (const {table_name: liteName, column_name, backfill} of backfilling) {\n    const known = byLiteName.get(liteName);\n    const dot = known ? -1 : liteName.indexOf('.');\n    const table = known ?? {\n      schema: dot < 0 ? 'public' : liteName.slice(0, dot),\n      name: dot < 0 ? liteName : liteName.slice(dot + 1),\n    };\n    if (!known && dot >= 0) {\n      unresolved++;\n      lc.warn?.(\n        `no metadata for backfilling table \"${liteName}\"; ` +\n          `assuming {schema: \"${table.schema}\", name: \"${table.name}\"}`,\n      );\n    }\n    insert.run(table.schema, table.name, column_name, backfill);\n  }\n  unresolvable.add(unresolved);\n  lc.info?.(\n    `seeded \"${BACKFILLING_TABLE}\" with ${backfilling.length} in-flight ` +\n      `backfill(s), ${unresolved} of which could not be resolved`,\n  );\n}\n"],"mappings":";;;;;;AA0DA,IAAa,oBAAoB;AAYjC,IAAa,2BAAmC;kBAC9B,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6BpC,IAAa,qBAAb,MAAgC;CAC9B;CAEA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,IAAc;EACxB,KAAKA,MAAM;CACb;;;;;;;CAQA,MAAM,QAAkC;EACtC,MAAM,MAAM,UAAU,MAAM;EAC5B,KAAK,MAAM,MAAM,KACf,KAAKC,KAAK,EAAE;EAEd,OAAO;CACT;CAEA,KAAK,IAAoB;EACvB,QAAQ,GAAG,IAAX;GACE,KAAK,mBAEH;GAEF,KAAK;IACH,CAAC,KAAKC,YAAY,KAAKF,IAAI,QAAgB;yBAC1B,kBAAkB;;;;SAIlC,GAAG,IACF,GAAG,MAAM,QACT,GAAG,MAAM,MACT,GAAG,QACH,WAAW,UAAU,GAAG,QAAQ,CAClC;IACA;GAEF,KAAK;IACH,CAAC,KAAKG,iBAAiB,KAAKH,IAAI,QAAgB;oBACpC,kBAAkB;;SAE7B,GAAG,IAAI,GAAG,IAAI,QAAQ,GAAG,IAAI,MAAM,GAAG,IAAI,QAAQ,GAAG,IAAI,IAAI;IAC9D;GAEF,KAAK;IACH,CAAC,KAAKI,eAAe,KAAKJ,IAAI,QAAgB;yBAC7B,kBAAkB;;SAElC,GAAG,IAAI,GAAG,MAAM,QAAQ,GAAG,MAAM,IAAI;IACtC;GAEF,KAAK;IACH,CAAC,KAAKK,kBAAkB,KAAKL,IAAI,QAAgB;oBACrC,kBAAkB;;SAE7B,GAAG,IAAI,GAAG,KAAK,GAAG,MAAM,QAAQ,GAAG,MAAM,MAAM,GAAG,GAAG;IACtD;GAEF,KAAK;IACH,KAAKM,cAAc,GAAG,OAAO,GAAG,MAAM;IACtC;GAEF,KAAK;IAIH,KAAK,MAAM,UAAU,GAAG,SACtB,KAAKA,cAAc,GAAG,OAAO,MAAM;IAErC;GAEF,SACE,YAAY,EAAE;EAClB;CACF;;;;;CAMA,YACE,OAC+C;EAC/C,MAAM,QAAQ,KAAKC,iBAAiB,KAAKP,IAAI,QAAgB;0CACvB,kBAAkB;;KAEvD,GAAG,IACF,MAAM,QACN,MAAM,IACR;EACA,OAAO,IAAI,IACT,KAAK,KAAK,EAAC,QAAQ,eAAc,CAC/B,QACA,aAAa,OACT,KAAA,IACC,WAAW,MAAM,QAAQ,CAChC,CAAC,CACH;CACF;;CAGA,YACE,OACA,QACA,UACM;EACN,CAAC,KAAKQ,iBAAiB,KAAKR,IAAI,QAAgB;gBACpC,kBAAkB;;KAE7B,GAAG,IACF,aAAa,KAAA,IAAY,OAAO,WAAW,UAAU,QAAQ,GAC7D,MAAM,QACN,MAAM,MACN,MACF;CACF;;;;;;CAOA,SAAS,OAAmB,SAAkC;EAC5D,KAAKC,KAAK;GAAC,IAAI;GAAqB;GAAO;EAAO,CAAC;CACrD;CAEA,cAAc,OAAmB,QAAsB;EACrD,CAAC,KAAKQ,gBAAgB,KAAKT,IAAI,QAAgB;qBAC9B,kBAAkB;;KAElC,GAAG,IAAI,MAAM,QAAQ,MAAM,MAAM,MAAM;CAC1C;AACF;;;;;;;;;AAUA,SAAgB,qBAAqB,IAAiC;CACpE,OAAO,qBAAqB,mBAAmB,EAAE,CAAC;AACpD;;;;;;;;AASA,SAAgB,qBAAqB,IAAiC;CACpE,MAAM,WAAW,qBAAqB,EAAE;CACxC,MAAM,WAAW,GACd,QAAgB;;gBAEL,kBAAkB;KAC7B,EACA,IAAuE;CAC1E,KAAK,MAAM,EAAC,QAAQ,OAAO,QAAQ,UAAU,UAAS,UAAU;EAI9D,MAAM,MAHM,SAAS,MACnB,MAAK,EAAE,MAAM,WAAW,UAAU,EAAE,MAAM,SAAS,KAEzC,GAAK,QAAQ;EACzB,IAAI,KACF,IAAI,WAAW,WAAW,MAAM,IAAI;CAExC;CACA,OAAO;AACT;;;;;;;;;;;;;;;;AAiBA,SAAgB,mBAAmB,IAAyB;CA6B1D,OAAO;EAAC,eA5Bc,GACnB,QAAgB;;;;;KAKhB,EACA,IAAuD,EACvD,KAAK,EAAC,QAAQ,OAAO,gBAAe;GACnC;GACA;GACA,UAAU,WAAW,MAAM,QAAQ;EACrC,EAgBM;EAAe,aAdH,GACjB,QAAgB;;gBAEL,kBAAkB;;KAE7B,EACA,IAAuE,EACvE,KAAK,EAAC,QAAQ,OAAO,QAAQ,gBAAe;GAC3C;GACA;GACA;GACA,UAAU,WAAW,MAAM,QAAQ;EACrC,EAEqB;CAAW;AACpC;;;;;;;;;;;;;;;;;;;;AAqBA,SAAgB,sCACd,IACA,IACM;CACN,GAAG,QAAgB,gBAAgB,kBAAkB,EAAE,EAAE,IAAI;CAE7D,MAAM,cAAc,GACjB,QAAgB;;;;KAIhB,EACA,IAAiE;CAEpE,MAAM,eAAe,mBACnB,WACA,iCACA,sGAEE,oBACA,4EACJ;CACA,IAAI,YAAY,WAAW,GAAG;EAC5B,aAAa,IAAI,CAAC;EAClB;CACF;CAEA,MAAM,aAAa,IAAI,IACrB,GACG,QAAgB,qDAAqD,EACrE,IAAqC,EACrC,KAAK,EAAC,QAAQ,YAAW,CACxB,cAAc;EAAC;EAAQ,MAAM;CAAK,CAAC,GACnC;EAAC;EAAQ,MAAM;CAAK,CACtB,CAAC,CACL;CAEA,MAAM,SAAS,GAAG,QAAgB;mBACjB,kBAAkB;;GAElC;CACD,IAAI,aAAa;CACjB,KAAK,MAAM,EAAC,YAAY,UAAU,aAAa,cAAa,aAAa;EACvE,MAAM,QAAQ,WAAW,IAAI,QAAQ;EACrC,MAAM,MAAM,QAAQ,KAAK,SAAS,QAAQ,GAAG;EAC7C,MAAM,QAAQ,SAAS;GACrB,QAAQ,MAAM,IAAI,WAAW,SAAS,MAAM,GAAG,GAAG;GAClD,MAAM,MAAM,IAAI,WAAW,SAAS,MAAM,MAAM,CAAC;EACnD;EACA,IAAI,CAAC,SAAS,OAAO,GAAG;GACtB;GACA,GAAG,OACD,sCAAsC,SAAS,wBACvB,MAAM,OAAO,YAAY,MAAM,KAAK,GAC9D;EACF;EACA,OAAO,IAAI,MAAM,QAAQ,MAAM,MAAM,aAAa,QAAQ;CAC5D;CACA,aAAa,IAAI,UAAU;CAC3B,GAAG,OACD,WAAW,kBAAkB,SAAS,YAAY,OAAO,0BACvC,WAAW,gCAC/B;AACF"}

package/out/zero-cache/src/services/replicator/write-worker-client.d.ts

@@ -1,6 +1,7 @@
 import type { LogConfig } from '../../../../shared/src/logging.ts';
 import type { Database } from '../../../../zqlite/src/db.ts';
 import type { ChangeStreamData } from '../change-source/protocol/current/downstream.ts';
+import type { BackfillRequest } from '../change-source/protocol/current/upstream.ts';
 import type { ChangeProcessorMode, CommitResult } from './change-processor.ts';
 import type { SubscriptionState } from './schema/replication-state.ts';
 export type PragmaConfig = {
@@ -31,8 +32,15 @@ type ErrorHandler = (err: Error) => void;
 /**
  * Interface for a write worker that processes replication messages.
  */
+/**
+ * The replica's {@link SubscriptionState}, along with its pending backfills,
+ * read in the same snapshot.
+ */
+export type SubscriptionStateWithBackfills = SubscriptionState & {
+    backfills: BackfillRequest[];
+};
 export interface WriteWorkerClient {
-    getSubscriptionState(): Promise<SubscriptionState>;
+    getSubscriptionState(): Promise<SubscriptionStateWithBackfills>;
     processMessages(downstream: readonly ChangeStreamData[]): Promise<CommitResult | null>;
     abort(): void;
     stop(): Promise<void>;
@@ -67,7 +75,7 @@ export type Request<M extends Method = Method> = {
 };
 export type ResultMap = {
     init: void;
-    getSubscriptionState: SubscriptionState;
+    getSubscriptionState: SubscriptionStateWithBackfills;
     processMessages: CommitResult | null;
     abort: void;
     stop: void;
@@ -93,7 +101,7 @@ export declare class ThreadWriteWorkerClient implements WriteWorkerClient {
     #private;
     constructor();
     init(dbPath: string, mode: ChangeProcessorMode, pragmas: PragmaConfig, logConfig: LogConfig, checkpoint?: ForceCheckpointConfig | null): Promise<void>;
-    getSubscriptionState(): Promise<SubscriptionState>;
+    getSubscriptionState(): Promise<SubscriptionStateWithBackfills>;
     processMessages(downstream: readonly ChangeStreamData[]): Promise<CommitResult | null>;
     abort(): void;
     stop(): Promise<void>;

package/out/zero-cache/src/services/replicator/write-worker-client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"write-worker-client.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/write-worker-client.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,mCAAmC,CAAC;AACjE,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAE3D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AACtF,OAAO,KAAK,EAAC,mBAAmB,EAAE,YAAY,EAAC,MAAM,uBAAuB,CAAC;AAC7E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,+BAA+B,CAAC;AAErE,MAAM,MAAM,YAAY,GAAG;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACxC,CAAC;AAEF;;;;;;;;;;;;;;GAcG;AACH,MAAM,MAAM,qBAAqB,GAAG;IAClC,wBAAwB,EAAE,MAAM,CAAC;IACjC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAClC,CAAC;AAEF,KAAK,YAAY,GAAG,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,CAAC;AAEzC;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,oBAAoB,IAAI,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACnD,eAAe,CACb,UAAU,EAAE,SAAS,gBAAgB,EAAE,GACtC,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;IAChC,KAAK,IAAI,IAAI,CAAC;IACd,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI,CAAC;CACtC;AAED,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,KAAK,CAAC,EAAE,eAAe,GAAG,MAAM,GAAG,SAAS,CAAC;IAC7C,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;CAC/C,CAAC;AAEF,wBAAgB,cAAc,CAAC,GAAG,EAAE,OAAO,GAAG,eAAe,CA8B5D;AAED,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,eAAe,GAAG,KAAK,CAgBnE;AAGD,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE;QACJ,MAAM;QACN,mBAAmB;QACnB,YAAY;QACZ,SAAS;QACT,qBAAqB,GAAG,IAAI;KAC7B,CAAC;IACF,oBAAoB,EAAE,EAAE,CAAC;IACzB,eAAe,EAAE,CAAC,SAAS,gBAAgB,EAAE,CAAC,CAAC;IAC/C,KAAK,EAAE,EAAE,CAAC;IACV,IAAI,EAAE,EAAE,CAAC;CACV,CAAC;AAEF,MAAM,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC;AAEnC,MAAM,MAAM,OAAO,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,IAAI;IAAC,MAAM,EAAE,CAAC,CAAC;IAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAA;CAAC,CAAC;AAE/E,MAAM,MAAM,SAAS,GAAG;IACtB,IAAI,EAAE,IAAI,CAAC;IACX,oBAAoB,EAAE,iBAAiB,CAAC;IACxC,eAAe,EAAE,YAAY,GAAG,IAAI,CAAC;IACrC,KAAK,EAAE,IAAI,CAAC;IACZ,IAAI,EAAE,IAAI,CAAC;CACZ,CAAC;AAEF,MAAM,MAAM,QAAQ,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,IAC1C;IAAC,MAAM,EAAE,CAAC,CAAC;IAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,SAAS,CAAA;CAAC,GACpD;IAAC,MAAM,EAAE,CAAC,CAAC;IAAC,KAAK,EAAE,eAAe,CAAC;IAAC,MAAM,CAAC,EAAE,SAAS,CAAA;CAAC,CAAC;AAE5D,MAAM,MAAM,UAAU,GAAG;IAAC,UAAU,EAAE,eAAe,CAAA;CAAC,CAAC;AAEvD,wBAAgB,YAAY,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,QAM/D;AAED;;;GAGG;AACH,qBAAa,uBAAwB,YAAW,iBAAiB;;IAM/D,cAiCC;IAkBD,IAAI,CACF,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,mBAAmB,EACzB,OAAO,EAAE,YAAY,EACrB,SAAS,EAAE,SAAS,EACpB,UAAU,GAAE,qBAAqB,GAAG,IAAW,GAC9C,OAAO,CAAC,IAAI,CAAC,CAEf;IAED,oBAAoB,IAAI,OAAO,CAAC,iBAAiB,CAAC,CAEjD;IAED,eAAe,CACb,UAAU,EAAE,SAAS,gBAAgB,EAAE,GACtC,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAE9B;IAED,KAAK,IAAI,IAAI,CAIZ;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAK1B;IAED,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI,CAEnC;CACF"}
+{"version":3,"file":"write-worker-client.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/write-worker-client.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,mCAAmC,CAAC;AACjE,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAE3D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AACtF,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,+CAA+C,CAAC;AACnF,OAAO,KAAK,EAAC,mBAAmB,EAAE,YAAY,EAAC,MAAM,uBAAuB,CAAC;AAC7E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,+BAA+B,CAAC;AAErE,MAAM,MAAM,YAAY,GAAG;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACxC,CAAC;AAEF;;;;;;;;;;;;;;GAcG;AACH,MAAM,MAAM,qBAAqB,GAAG;IAClC,wBAAwB,EAAE,MAAM,CAAC;IACjC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAClC,CAAC;AAEF,KAAK,YAAY,GAAG,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,CAAC;AAEzC;;GAEG;AACH;;;GAGG;AACH,MAAM,MAAM,8BAA8B,GAAG,iBAAiB,GAAG;IAC/D,SAAS,EAAE,eAAe,EAAE,CAAC;CAC9B,CAAC;AAEF,MAAM,WAAW,iBAAiB;IAChC,oBAAoB,IAAI,OAAO,CAAC,8BAA8B,CAAC,CAAC;IAChE,eAAe,CACb,UAAU,EAAE,SAAS,gBAAgB,EAAE,GACtC,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;IAChC,KAAK,IAAI,IAAI,CAAC;IACd,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI,CAAC;CACtC;AAED,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,KAAK,CAAC,EAAE,eAAe,GAAG,MAAM,GAAG,SAAS,CAAC;IAC7C,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;CAC/C,CAAC;AAEF,wBAAgB,cAAc,CAAC,GAAG,EAAE,OAAO,GAAG,eAAe,CA8B5D;AAED,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,eAAe,GAAG,KAAK,CAgBnE;AAGD,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE;QACJ,MAAM;QACN,mBAAmB;QACnB,YAAY;QACZ,SAAS;QACT,qBAAqB,GAAG,IAAI;KAC7B,CAAC;IACF,oBAAoB,EAAE,EAAE,CAAC;IACzB,eAAe,EAAE,CAAC,SAAS,gBAAgB,EAAE,CAAC,CAAC;IAC/C,KAAK,EAAE,EAAE,CAAC;IACV,IAAI,EAAE,EAAE,CAAC;CACV,CAAC;AAEF,MAAM,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC;AAEnC,MAAM,MAAM,OAAO,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,IAAI;IAAC,MAAM,EAAE,CAAC,CAAC;IAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAA;CAAC,CAAC;AAE/E,MAAM,MAAM,SAAS,GAAG;IACtB,IAAI,EAAE,IAAI,CAAC;IACX,oBAAoB,EAAE,8BAA8B,CAAC;IACrD,eAAe,EAAE,YAAY,GAAG,IAAI,CAAC;IACrC,KAAK,EAAE,IAAI,CAAC;IACZ,IAAI,EAAE,IAAI,CAAC;CACZ,CAAC;AAEF,MAAM,MAAM,QAAQ,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,IAC1C;IAAC,MAAM,EAAE,CAAC,CAAC;IAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,SAAS,CAAA;CAAC,GACpD;IAAC,MAAM,EAAE,CAAC,CAAC;IAAC,KAAK,EAAE,eAAe,CAAC;IAAC,MAAM,CAAC,EAAE,SAAS,CAAA;CAAC,CAAC;AAE5D,MAAM,MAAM,UAAU,GAAG;IAAC,UAAU,EAAE,eAAe,CAAA;CAAC,CAAC;AAEvD,wBAAgB,YAAY,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,QAM/D;AAED;;;GAGG;AACH,qBAAa,uBAAwB,YAAW,iBAAiB;;IAM/D,cAiCC;IAkBD,IAAI,CACF,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,mBAAmB,EACzB,OAAO,EAAE,YAAY,EACrB,SAAS,EAAE,SAAS,EACpB,UAAU,GAAE,qBAAqB,GAAG,IAAW,GAC9C,OAAO,CAAC,IAAI,CAAC,CAEf;IAED,oBAAoB,IAAI,OAAO,CAAC,8BAA8B,CAAC,CAE9D;IAED,eAAe,CACb,UAAU,EAAE,SAAS,gBAAgB,EAAE,GACtC,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAE9B;IAED,KAAK,IAAI,IAAI,CAIZ;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAK1B;IAED,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI,CAEnC;CACF"}

package/out/zero-cache/src/services/replicator/write-worker-client.js.map

@@ -1 +1 @@
-{"version":3,"file":"write-worker-client.js","names":["#worker","#rejectAll","#errorHandler","#pending","#terminated","#call"],"sources":["../../../../../../zero-cache/src/services/replicator/write-worker-client.ts"],"sourcesContent":["import {Worker} from 'node:worker_threads';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport type {LogConfig} from '../../../../shared/src/logging.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {WRITE_WORKER_URL} from '../../server/worker-urls.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {ChangeProcessorMode, CommitResult} from './change-processor.ts';\nimport type {SubscriptionState} from './schema/replication-state.ts';\n\nexport type PragmaConfig = {\n  busyTimeout: number;\n  analysisLimit: number;\n  walAutocheckpoint?: number | undefined;\n};\n\n/**\n * Enables write-path checkpoint backpressure in the write worker. At each\n * commit boundary, if the WAL has grown to at least `checkpointThresholdPages`,\n * the worker forces an immediate litestream sync (sealing the WAL to LTX and\n * running litestream's checkpoint).\n *\n * The optional `walMaxPages` is an emergency break that pauses until the WAL drains,\n * rather than letting it exceed available disk space. This replaces litestream's\n * analogous `truncate-page-n` emergency break, which is more disruptive because of\n * https://github.com/benbjohnson/litestream/issues/1332. Note, however, this should\n * never been needed if non-initial snapshots are disabled (which is the default).\n *\n * Set only for the backup replicator when replicating with litestream v5; `null`\n * disables the feature entirely (no per-commit overhead).\n */\nexport type ForceCheckpointConfig = {\n  checkpointThresholdPages: number;\n  maxWalPages?: number | undefined;\n};\n\ntype ErrorHandler = (err: Error) => void;\n\n/**\n * Interface for a write worker that processes replication messages.\n */\nexport interface WriteWorkerClient {\n  getSubscriptionState(): Promise<SubscriptionState>;\n  processMessages(\n    downstream: readonly ChangeStreamData[],\n  ): Promise<CommitResult | null>;\n  abort(): void;\n  stop(): Promise<void>;\n  onError(handler: ErrorHandler): void;\n}\n\nexport type SerializedError = {\n  name: string;\n  message: string;\n  stack?: string | undefined;\n  cause?: SerializedError | string | undefined;\n  details?: Record<string, unknown> | undefined;\n};\n\nexport function serializeError(err: unknown): SerializedError {\n  if (!(err instanceof Error)) {\n    return {\n      name: 'Error',\n      message: String(err),\n      details: err && typeof err === 'object' ? {...err} : undefined,\n    };\n  }\n\n  // Error fields such as message, stack, and some native error details are\n  // non-enumerable, so JSON.stringify(err) would usually return \"{}\".\n  const details = Object.fromEntries(\n    Object.getOwnPropertyNames(err)\n      .filter(key => !['name', 'message', 'stack', 'cause'].includes(key))\n      .map(key => [key, (err as unknown as Record<string, unknown>)[key]]),\n  );\n  const cause =\n    err.cause instanceof Error\n      ? serializeError(err.cause)\n      : err.cause === undefined\n        ? undefined\n        : String(err.cause);\n\n  return {\n    name: err.name,\n    message: err.message,\n    stack: err.stack,\n    cause,\n    details: Object.keys(details).length ? details : undefined,\n  };\n}\n\nexport function deserializeError(serialized: SerializedError): Error {\n  const err = new Error(serialized.message);\n  err.name = serialized.name;\n  if (serialized.stack !== undefined) {\n    err.stack = serialized.stack;\n  }\n  if (serialized.cause !== undefined) {\n    err.cause =\n      typeof serialized.cause === 'string'\n        ? serialized.cause\n        : deserializeError(serialized.cause);\n  }\n  if (serialized.details) {\n    Object.assign(err, serialized.details);\n  }\n  return err;\n}\n\n// Wire protocol types.\nexport type ArgsMap = {\n  init: [\n    string,\n    ChangeProcessorMode,\n    PragmaConfig,\n    LogConfig,\n    ForceCheckpointConfig | null,\n  ];\n  getSubscriptionState: [];\n  processMessages: [readonly ChangeStreamData[]];\n  abort: [];\n  stop: [];\n};\n\nexport type Method = keyof ArgsMap;\n\nexport type Request<M extends Method = Method> = {method: M; args: ArgsMap[M]};\n\nexport type ResultMap = {\n  init: void;\n  getSubscriptionState: SubscriptionState;\n  processMessages: CommitResult | null;\n  abort: void;\n  stop: void;\n};\n\nexport type Response<M extends Method = Method> =\n  | {method: M; result: ResultMap[M]; error?: undefined}\n  | {method: M; error: SerializedError; result?: undefined};\n\nexport type WriteError = {writeError: SerializedError};\n\nexport function applyPragmas(db: Database, pragmas: PragmaConfig) {\n  db.pragma(`busy_timeout = ${pragmas.busyTimeout}`);\n  db.pragma(`analysis_limit = ${pragmas.analysisLimit}`);\n  if (pragmas.walAutocheckpoint !== undefined) {\n    db.pragma(`wal_autocheckpoint = ${pragmas.walAutocheckpoint}`);\n  }\n}\n\n/**\n * Delegates SQLite writes to a worker_thread,\n * keeping the main event loop free for WebSocket heartbeats and IPC.\n */\nexport class ThreadWriteWorkerClient implements WriteWorkerClient {\n  readonly #worker: Worker;\n  #pending: Resolver<unknown, Error> | null = null;\n  #errorHandler: ErrorHandler = () => {};\n  #terminated = false;\n\n  constructor() {\n    this.#worker = new Worker(WRITE_WORKER_URL);\n\n    this.#worker.on('message', (msg: Response | WriteError) => {\n      if ('writeError' in msg) {\n        const error = deserializeError(msg.writeError);\n        this.#rejectAll(error);\n        this.#errorHandler(error);\n        return;\n      }\n      const r = this.#pending;\n      if (!r) return; // stale abort response\n      this.#pending = null;\n      if (msg.error !== undefined) {\n        r.reject(deserializeError(msg.error));\n      } else {\n        r.resolve(msg.result);\n      }\n    });\n\n    this.#worker.on('error', (err: Error) => {\n      this.#rejectAll(err);\n      this.#errorHandler(err);\n    });\n\n    this.#worker.on('exit', (code: number) => {\n      this.#terminated = true;\n      if (code !== 0) {\n        const err = new Error(`Worker exited with code ${code}`);\n        this.#rejectAll(err);\n        this.#errorHandler(err);\n      }\n    });\n  }\n\n  #rejectAll(err: Error) {\n    const r = this.#pending;\n    if (r) {\n      this.#pending = null;\n      r.reject(err);\n    }\n  }\n\n  #call<M extends Method>(method: M, args: ArgsMap[M]): Promise<ResultMap[M]> {\n    assert(this.#pending === null, `concurrent call: ${method}`);\n    const r = resolver<ResultMap[M]>();\n    this.#pending = r as Resolver<unknown, Error>;\n    this.#worker.postMessage({method, args} satisfies Request);\n    return r.promise;\n  }\n\n  init(\n    dbPath: string,\n    mode: ChangeProcessorMode,\n    pragmas: PragmaConfig,\n    logConfig: LogConfig,\n    checkpoint: ForceCheckpointConfig | null = null,\n  ): Promise<void> {\n    return this.#call('init', [dbPath, mode, pragmas, logConfig, checkpoint]);\n  }\n\n  getSubscriptionState(): Promise<SubscriptionState> {\n    return this.#call('getSubscriptionState', []);\n  }\n\n  processMessages(\n    downstream: readonly ChangeStreamData[],\n  ): Promise<CommitResult | null> {\n    return this.#call('processMessages', [downstream]);\n  }\n\n  abort(): void {\n    if (!this.#terminated) {\n      this.#worker.postMessage({method: 'abort', args: []} satisfies Request);\n    }\n  }\n\n  async stop(): Promise<void> {\n    await this.#call('stop', []);\n    if (!this.#terminated) {\n      await this.#worker.terminate();\n    }\n  }\n\n  onError(handler: ErrorHandler): void {\n    this.#errorHandler = handler;\n  }\n}\n"],"mappings":";;;;;AA2DA,SAAgB,eAAe,KAA+B;CAC5D,IAAI,EAAE,eAAe,QACnB,OAAO;EACL,MAAM;EACN,SAAS,OAAO,GAAG;EACnB,SAAS,OAAO,OAAO,QAAQ,WAAW,EAAC,GAAG,IAAG,IAAI,KAAA;CACvD;CAKF,MAAM,UAAU,OAAO,YACrB,OAAO,oBAAoB,GAAG,EAC3B,QAAO,QAAO,CAAC;EAAC;EAAQ;EAAW;EAAS;CAAO,EAAE,SAAS,GAAG,CAAC,EAClE,KAAI,QAAO,CAAC,KAAM,IAA2C,IAAI,CAAC,CACvE;CACA,MAAM,QACJ,IAAI,iBAAiB,QACjB,eAAe,IAAI,KAAK,IACxB,IAAI,UAAU,KAAA,IACZ,KAAA,IACA,OAAO,IAAI,KAAK;CAExB,OAAO;EACL,MAAM,IAAI;EACV,SAAS,IAAI;EACb,OAAO,IAAI;EACX;EACA,SAAS,OAAO,KAAK,OAAO,EAAE,SAAS,UAAU,KAAA;CACnD;AACF;AAEA,SAAgB,iBAAiB,YAAoC;CACnE,MAAM,MAAM,IAAI,MAAM,WAAW,OAAO;CACxC,IAAI,OAAO,WAAW;CACtB,IAAI,WAAW,UAAU,KAAA,GACvB,IAAI,QAAQ,WAAW;CAEzB,IAAI,WAAW,UAAU,KAAA,GACvB,IAAI,QACF,OAAO,WAAW,UAAU,WACxB,WAAW,QACX,iBAAiB,WAAW,KAAK;CAEzC,IAAI,WAAW,SACb,OAAO,OAAO,KAAK,WAAW,OAAO;CAEvC,OAAO;AACT;AAmCA,SAAgB,aAAa,IAAc,SAAuB;CAChE,GAAG,OAAO,kBAAkB,QAAQ,aAAa;CACjD,GAAG,OAAO,oBAAoB,QAAQ,eAAe;CACrD,IAAI,QAAQ,sBAAsB,KAAA,GAChC,GAAG,OAAO,wBAAwB,QAAQ,mBAAmB;AAEjE;;;;;AAMA,IAAa,0BAAb,MAAkE;CAChE;CACA,WAA4C;CAC5C,sBAAoC,CAAC;CACrC,cAAc;CAEd,cAAc;EACZ,KAAKA,UAAU,IAAI,OAAO,gBAAgB;EAE1C,KAAKA,QAAQ,GAAG,YAAY,QAA+B;GACzD,IAAI,gBAAgB,KAAK;IACvB,MAAM,QAAQ,iBAAiB,IAAI,UAAU;IAC7C,KAAKC,WAAW,KAAK;IACrB,KAAKC,cAAc,KAAK;IACxB;GACF;GACA,MAAM,IAAI,KAAKC;GACf,IAAI,CAAC,GAAG;GACR,KAAKA,WAAW;GAChB,IAAI,IAAI,UAAU,KAAA,GAChB,EAAE,OAAO,iBAAiB,IAAI,KAAK,CAAC;QAEpC,EAAE,QAAQ,IAAI,MAAM;EAExB,CAAC;EAED,KAAKH,QAAQ,GAAG,UAAU,QAAe;GACvC,KAAKC,WAAW,GAAG;GACnB,KAAKC,cAAc,GAAG;EACxB,CAAC;EAED,KAAKF,QAAQ,GAAG,SAAS,SAAiB;GACxC,KAAKI,cAAc;GACnB,IAAI,SAAS,GAAG;IACd,MAAM,sBAAM,IAAI,MAAM,2BAA2B,MAAM;IACvD,KAAKH,WAAW,GAAG;IACnB,KAAKC,cAAc,GAAG;GACxB;EACF,CAAC;CACH;CAEA,WAAW,KAAY;EACrB,MAAM,IAAI,KAAKC;EACf,IAAI,GAAG;GACL,KAAKA,WAAW;GAChB,EAAE,OAAO,GAAG;EACd;CACF;CAEA,MAAwB,QAAW,MAAyC;EAC1E,OAAO,KAAKA,aAAa,MAAM,oBAAoB,QAAQ;EAC3D,MAAM,IAAI,SAAuB;EACjC,KAAKA,WAAW;EAChB,KAAKH,QAAQ,YAAY;GAAC;GAAQ;EAAI,CAAmB;EACzD,OAAO,EAAE;CACX;CAEA,KACE,QACA,MACA,SACA,WACA,aAA2C,MAC5B;EACf,OAAO,KAAKK,MAAM,QAAQ;GAAC;GAAQ;GAAM;GAAS;GAAW;EAAU,CAAC;CAC1E;CAEA,uBAAmD;EACjD,OAAO,KAAKA,MAAM,wBAAwB,CAAC,CAAC;CAC9C;CAEA,gBACE,YAC8B;EAC9B,OAAO,KAAKA,MAAM,mBAAmB,CAAC,UAAU,CAAC;CACnD;CAEA,QAAc;EACZ,IAAI,CAAC,KAAKD,aACR,KAAKJ,QAAQ,YAAY;GAAC,QAAQ;GAAS,MAAM,CAAC;EAAC,CAAmB;CAE1E;CAEA,MAAM,OAAsB;EAC1B,MAAM,KAAKK,MAAM,QAAQ,CAAC,CAAC;EAC3B,IAAI,CAAC,KAAKD,aACR,MAAM,KAAKJ,QAAQ,UAAU;CAEjC;CAEA,QAAQ,SAA6B;EACnC,KAAKE,gBAAgB;CACvB;AACF"}
+{"version":3,"file":"write-worker-client.js","names":["#worker","#rejectAll","#errorHandler","#pending","#terminated","#call"],"sources":["../../../../../../zero-cache/src/services/replicator/write-worker-client.ts"],"sourcesContent":["import {Worker} from 'node:worker_threads';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport type {LogConfig} from '../../../../shared/src/logging.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {WRITE_WORKER_URL} from '../../server/worker-urls.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {BackfillRequest} from '../change-source/protocol/current/upstream.ts';\nimport type {ChangeProcessorMode, CommitResult} from './change-processor.ts';\nimport type {SubscriptionState} from './schema/replication-state.ts';\n\nexport type PragmaConfig = {\n  busyTimeout: number;\n  analysisLimit: number;\n  walAutocheckpoint?: number | undefined;\n};\n\n/**\n * Enables write-path checkpoint backpressure in the write worker. At each\n * commit boundary, if the WAL has grown to at least `checkpointThresholdPages`,\n * the worker forces an immediate litestream sync (sealing the WAL to LTX and\n * running litestream's checkpoint).\n *\n * The optional `walMaxPages` is an emergency break that pauses until the WAL drains,\n * rather than letting it exceed available disk space. This replaces litestream's\n * analogous `truncate-page-n` emergency break, which is more disruptive because of\n * https://github.com/benbjohnson/litestream/issues/1332. Note, however, this should\n * never been needed if non-initial snapshots are disabled (which is the default).\n *\n * Set only for the backup replicator when replicating with litestream v5; `null`\n * disables the feature entirely (no per-commit overhead).\n */\nexport type ForceCheckpointConfig = {\n  checkpointThresholdPages: number;\n  maxWalPages?: number | undefined;\n};\n\ntype ErrorHandler = (err: Error) => void;\n\n/**\n * Interface for a write worker that processes replication messages.\n */\n/**\n * The replica's {@link SubscriptionState}, along with its pending backfills,\n * read in the same snapshot.\n */\nexport type SubscriptionStateWithBackfills = SubscriptionState & {\n  backfills: BackfillRequest[];\n};\n\nexport interface WriteWorkerClient {\n  getSubscriptionState(): Promise<SubscriptionStateWithBackfills>;\n  processMessages(\n    downstream: readonly ChangeStreamData[],\n  ): Promise<CommitResult | null>;\n  abort(): void;\n  stop(): Promise<void>;\n  onError(handler: ErrorHandler): void;\n}\n\nexport type SerializedError = {\n  name: string;\n  message: string;\n  stack?: string | undefined;\n  cause?: SerializedError | string | undefined;\n  details?: Record<string, unknown> | undefined;\n};\n\nexport function serializeError(err: unknown): SerializedError {\n  if (!(err instanceof Error)) {\n    return {\n      name: 'Error',\n      message: String(err),\n      details: err && typeof err === 'object' ? {...err} : undefined,\n    };\n  }\n\n  // Error fields such as message, stack, and some native error details are\n  // non-enumerable, so JSON.stringify(err) would usually return \"{}\".\n  const details = Object.fromEntries(\n    Object.getOwnPropertyNames(err)\n      .filter(key => !['name', 'message', 'stack', 'cause'].includes(key))\n      .map(key => [key, (err as unknown as Record<string, unknown>)[key]]),\n  );\n  const cause =\n    err.cause instanceof Error\n      ? serializeError(err.cause)\n      : err.cause === undefined\n        ? undefined\n        : String(err.cause);\n\n  return {\n    name: err.name,\n    message: err.message,\n    stack: err.stack,\n    cause,\n    details: Object.keys(details).length ? details : undefined,\n  };\n}\n\nexport function deserializeError(serialized: SerializedError): Error {\n  const err = new Error(serialized.message);\n  err.name = serialized.name;\n  if (serialized.stack !== undefined) {\n    err.stack = serialized.stack;\n  }\n  if (serialized.cause !== undefined) {\n    err.cause =\n      typeof serialized.cause === 'string'\n        ? serialized.cause\n        : deserializeError(serialized.cause);\n  }\n  if (serialized.details) {\n    Object.assign(err, serialized.details);\n  }\n  return err;\n}\n\n// Wire protocol types.\nexport type ArgsMap = {\n  init: [\n    string,\n    ChangeProcessorMode,\n    PragmaConfig,\n    LogConfig,\n    ForceCheckpointConfig | null,\n  ];\n  getSubscriptionState: [];\n  processMessages: [readonly ChangeStreamData[]];\n  abort: [];\n  stop: [];\n};\n\nexport type Method = keyof ArgsMap;\n\nexport type Request<M extends Method = Method> = {method: M; args: ArgsMap[M]};\n\nexport type ResultMap = {\n  init: void;\n  getSubscriptionState: SubscriptionStateWithBackfills;\n  processMessages: CommitResult | null;\n  abort: void;\n  stop: void;\n};\n\nexport type Response<M extends Method = Method> =\n  | {method: M; result: ResultMap[M]; error?: undefined}\n  | {method: M; error: SerializedError; result?: undefined};\n\nexport type WriteError = {writeError: SerializedError};\n\nexport function applyPragmas(db: Database, pragmas: PragmaConfig) {\n  db.pragma(`busy_timeout = ${pragmas.busyTimeout}`);\n  db.pragma(`analysis_limit = ${pragmas.analysisLimit}`);\n  if (pragmas.walAutocheckpoint !== undefined) {\n    db.pragma(`wal_autocheckpoint = ${pragmas.walAutocheckpoint}`);\n  }\n}\n\n/**\n * Delegates SQLite writes to a worker_thread,\n * keeping the main event loop free for WebSocket heartbeats and IPC.\n */\nexport class ThreadWriteWorkerClient implements WriteWorkerClient {\n  readonly #worker: Worker;\n  #pending: Resolver<unknown, Error> | null = null;\n  #errorHandler: ErrorHandler = () => {};\n  #terminated = false;\n\n  constructor() {\n    this.#worker = new Worker(WRITE_WORKER_URL);\n\n    this.#worker.on('message', (msg: Response | WriteError) => {\n      if ('writeError' in msg) {\n        const error = deserializeError(msg.writeError);\n        this.#rejectAll(error);\n        this.#errorHandler(error);\n        return;\n      }\n      const r = this.#pending;\n      if (!r) return; // stale abort response\n      this.#pending = null;\n      if (msg.error !== undefined) {\n        r.reject(deserializeError(msg.error));\n      } else {\n        r.resolve(msg.result);\n      }\n    });\n\n    this.#worker.on('error', (err: Error) => {\n      this.#rejectAll(err);\n      this.#errorHandler(err);\n    });\n\n    this.#worker.on('exit', (code: number) => {\n      this.#terminated = true;\n      if (code !== 0) {\n        const err = new Error(`Worker exited with code ${code}`);\n        this.#rejectAll(err);\n        this.#errorHandler(err);\n      }\n    });\n  }\n\n  #rejectAll(err: Error) {\n    const r = this.#pending;\n    if (r) {\n      this.#pending = null;\n      r.reject(err);\n    }\n  }\n\n  #call<M extends Method>(method: M, args: ArgsMap[M]): Promise<ResultMap[M]> {\n    assert(this.#pending === null, `concurrent call: ${method}`);\n    const r = resolver<ResultMap[M]>();\n    this.#pending = r as Resolver<unknown, Error>;\n    this.#worker.postMessage({method, args} satisfies Request);\n    return r.promise;\n  }\n\n  init(\n    dbPath: string,\n    mode: ChangeProcessorMode,\n    pragmas: PragmaConfig,\n    logConfig: LogConfig,\n    checkpoint: ForceCheckpointConfig | null = null,\n  ): Promise<void> {\n    return this.#call('init', [dbPath, mode, pragmas, logConfig, checkpoint]);\n  }\n\n  getSubscriptionState(): Promise<SubscriptionStateWithBackfills> {\n    return this.#call('getSubscriptionState', []);\n  }\n\n  processMessages(\n    downstream: readonly ChangeStreamData[],\n  ): Promise<CommitResult | null> {\n    return this.#call('processMessages', [downstream]);\n  }\n\n  abort(): void {\n    if (!this.#terminated) {\n      this.#worker.postMessage({method: 'abort', args: []} satisfies Request);\n    }\n  }\n\n  async stop(): Promise<void> {\n    await this.#call('stop', []);\n    if (!this.#terminated) {\n      await this.#worker.terminate();\n    }\n  }\n\n  onError(handler: ErrorHandler): void {\n    this.#errorHandler = handler;\n  }\n}\n"],"mappings":";;;;;AAoEA,SAAgB,eAAe,KAA+B;CAC5D,IAAI,EAAE,eAAe,QACnB,OAAO;EACL,MAAM;EACN,SAAS,OAAO,GAAG;EACnB,SAAS,OAAO,OAAO,QAAQ,WAAW,EAAC,GAAG,IAAG,IAAI,KAAA;CACvD;CAKF,MAAM,UAAU,OAAO,YACrB,OAAO,oBAAoB,GAAG,EAC3B,QAAO,QAAO,CAAC;EAAC;EAAQ;EAAW;EAAS;CAAO,EAAE,SAAS,GAAG,CAAC,EAClE,KAAI,QAAO,CAAC,KAAM,IAA2C,IAAI,CAAC,CACvE;CACA,MAAM,QACJ,IAAI,iBAAiB,QACjB,eAAe,IAAI,KAAK,IACxB,IAAI,UAAU,KAAA,IACZ,KAAA,IACA,OAAO,IAAI,KAAK;CAExB,OAAO;EACL,MAAM,IAAI;EACV,SAAS,IAAI;EACb,OAAO,IAAI;EACX;EACA,SAAS,OAAO,KAAK,OAAO,EAAE,SAAS,UAAU,KAAA;CACnD;AACF;AAEA,SAAgB,iBAAiB,YAAoC;CACnE,MAAM,MAAM,IAAI,MAAM,WAAW,OAAO;CACxC,IAAI,OAAO,WAAW;CACtB,IAAI,WAAW,UAAU,KAAA,GACvB,IAAI,QAAQ,WAAW;CAEzB,IAAI,WAAW,UAAU,KAAA,GACvB,IAAI,QACF,OAAO,WAAW,UAAU,WACxB,WAAW,QACX,iBAAiB,WAAW,KAAK;CAEzC,IAAI,WAAW,SACb,OAAO,OAAO,KAAK,WAAW,OAAO;CAEvC,OAAO;AACT;AAmCA,SAAgB,aAAa,IAAc,SAAuB;CAChE,GAAG,OAAO,kBAAkB,QAAQ,aAAa;CACjD,GAAG,OAAO,oBAAoB,QAAQ,eAAe;CACrD,IAAI,QAAQ,sBAAsB,KAAA,GAChC,GAAG,OAAO,wBAAwB,QAAQ,mBAAmB;AAEjE;;;;;AAMA,IAAa,0BAAb,MAAkE;CAChE;CACA,WAA4C;CAC5C,sBAAoC,CAAC;CACrC,cAAc;CAEd,cAAc;EACZ,KAAKA,UAAU,IAAI,OAAO,gBAAgB;EAE1C,KAAKA,QAAQ,GAAG,YAAY,QAA+B;GACzD,IAAI,gBAAgB,KAAK;IACvB,MAAM,QAAQ,iBAAiB,IAAI,UAAU;IAC7C,KAAKC,WAAW,KAAK;IACrB,KAAKC,cAAc,KAAK;IACxB;GACF;GACA,MAAM,IAAI,KAAKC;GACf,IAAI,CAAC,GAAG;GACR,KAAKA,WAAW;GAChB,IAAI,IAAI,UAAU,KAAA,GAChB,EAAE,OAAO,iBAAiB,IAAI,KAAK,CAAC;QAEpC,EAAE,QAAQ,IAAI,MAAM;EAExB,CAAC;EAED,KAAKH,QAAQ,GAAG,UAAU,QAAe;GACvC,KAAKC,WAAW,GAAG;GACnB,KAAKC,cAAc,GAAG;EACxB,CAAC;EAED,KAAKF,QAAQ,GAAG,SAAS,SAAiB;GACxC,KAAKI,cAAc;GACnB,IAAI,SAAS,GAAG;IACd,MAAM,sBAAM,IAAI,MAAM,2BAA2B,MAAM;IACvD,KAAKH,WAAW,GAAG;IACnB,KAAKC,cAAc,GAAG;GACxB;EACF,CAAC;CACH;CAEA,WAAW,KAAY;EACrB,MAAM,IAAI,KAAKC;EACf,IAAI,GAAG;GACL,KAAKA,WAAW;GAChB,EAAE,OAAO,GAAG;EACd;CACF;CAEA,MAAwB,QAAW,MAAyC;EAC1E,OAAO,KAAKA,aAAa,MAAM,oBAAoB,QAAQ;EAC3D,MAAM,IAAI,SAAuB;EACjC,KAAKA,WAAW;EAChB,KAAKH,QAAQ,YAAY;GAAC;GAAQ;EAAI,CAAmB;EACzD,OAAO,EAAE;CACX;CAEA,KACE,QACA,MACA,SACA,WACA,aAA2C,MAC5B;EACf,OAAO,KAAKK,MAAM,QAAQ;GAAC;GAAQ;GAAM;GAAS;GAAW;EAAU,CAAC;CAC1E;CAEA,uBAAgE;EAC9D,OAAO,KAAKA,MAAM,wBAAwB,CAAC,CAAC;CAC9C;CAEA,gBACE,YAC8B;EAC9B,OAAO,KAAKA,MAAM,mBAAmB,CAAC,UAAU,CAAC;CACnD;CAEA,QAAc;EACZ,IAAI,CAAC,KAAKD,aACR,KAAKJ,QAAQ,YAAY;GAAC,QAAQ;GAAS,MAAM,CAAC;EAAC,CAAmB;CAE1E;CAEA,MAAM,OAAsB;EAC1B,MAAM,KAAKK,MAAM,QAAQ,CAAC,CAAC;EAC3B,IAAI,CAAC,KAAKD,aACR,MAAM,KAAKJ,QAAQ,UAAU;CAEjC;CAEA,QAAQ,SAA6B;EACnC,KAAKE,gBAAgB;CACvB;AACF"}

package/out/zero-cache/src/services/replicator/write-worker.js

@@ -1,6 +1,7 @@
 import { assert } from "../../../../shared/src/asserts.js";
 import { must } from "../../../../shared/src/must.js";
 import { Database } from "../../../../zqlite/src/db.js";
+import { readReplicaBackfills } from "./schema/backfilling.js";
 import { getSubscriptionState } from "./schema/replication-state.js";
 import { StatementRunner } from "../../db/statements.js";
 import { isSQLiteCorruption, logSQLiteCorruptionDiagnostics, registerSQLiteCorruptionDiagnosticTarget } from "../../db/sqlite-corruption.js";
@@ -74,7 +75,10 @@ function createAPI() {
 		},
 		getSubscriptionState() {
 			try {
-				return getSubscriptionState(must(runner));
+				return {
+					...getSubscriptionState(must(runner)),
+					backfills: readReplicaBackfills(must(runner).db)
+				};
 			} catch (e) {
 				handleCorruptedDb(e);
 				throw e;

package/out/zero-cache/src/services/replicator/write-worker.js.map

@@ -1 +1 @@
-{"version":3,"file":"write-worker.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/write-worker.ts"],"sourcesContent":["import {parentPort} from 'node:worker_threads';\nimport type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport type {LogConfig} from '../../../../shared/src/logging.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n  registerSQLiteCorruptionDiagnosticTarget,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {createLogContext} from '../../server/logging.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {LitestreamCheckpointer} from '../litestream/litestream-checkpointer.ts';\nimport {LitestreamController} from '../litestream/litestream-controller.ts';\nimport {ChangeProcessor, type ChangeProcessorMode} from './change-processor.ts';\nimport {getSubscriptionState} from './schema/replication-state.ts';\nimport {\n  applyPragmas,\n  serializeError,\n  type ArgsMap,\n  type ForceCheckpointConfig,\n  type Method,\n  type PragmaConfig,\n  type Request,\n  type Response,\n  type ResultMap,\n  type WriteError,\n} from './write-worker-client.ts';\n\nif (!parentPort) {\n  throw new Error('write-worker must be run as a worker thread');\n}\n\nconst port = parentPort;\n\ntype API = {\n  [M in Method]: (...args: ArgsMap[M]) => ResultMap[M] | Promise<ResultMap[M]>;\n};\n\nfunction createAPI(): API {\n  let db: Database | undefined;\n  let runner: StatementRunner | undefined;\n  let processor: ChangeProcessor | undefined;\n  let mode: ChangeProcessorMode | undefined;\n  let lc: LogContext | undefined;\n  let replicaDbPath: string | undefined;\n  let unregisterCorruptionDiagnosticTargets: (() => void)[] = [];\n\n  // Set when write-path checkpoint backpressure is enabled (backup replicator\n  // on litestream v5). `undefined` disables the feature.\n  let checkpointerConfig: ForceCheckpointConfig | undefined;\n  let checkpointer: LitestreamCheckpointer | undefined;\n\n  function unregisterCorruptionDiagnostics() {\n    unregisterCorruptionDiagnosticTargets.forEach(unregister => unregister());\n    unregisterCorruptionDiagnosticTargets = [];\n  }\n\n  function handleCorruptedDb(err: unknown) {\n    if (!lc || !replicaDbPath || !isSQLiteCorruption(err)) {\n      return;\n    }\n    logSQLiteCorruptionDiagnostics(lc, 'write-worker', replicaDbPath, err);\n    try {\n      lc.warn?.(`deleting corrupted db at ${replicaDbPath}`);\n      deleteLiteDB(replicaDbPath);\n    } catch (e) {\n      lc.warn?.(`error deleting corrupted db at ${replicaDbPath}`, e);\n    }\n  }\n\n  function createProcessor() {\n    processor = new ChangeProcessor(must(runner), must(mode), (_lc, err) => {\n      handleCorruptedDb(err);\n      port.postMessage({\n        writeError: serializeError(err),\n      } satisfies WriteError);\n    });\n  }\n\n  function createCheckpointer() {\n    if (checkpointerConfig) {\n      assert(lc && db && replicaDbPath, `not initialized`);\n      checkpointer = new LitestreamCheckpointer(\n        lc,\n        db,\n        new LitestreamController(lc, replicaDbPath),\n        checkpointerConfig,\n      );\n    }\n  }\n\n  return {\n    init(\n      dbPath: string,\n      cpMode: ChangeProcessorMode,\n      pragmas: PragmaConfig,\n      logConfig: LogConfig,\n      checkpointConfig: ForceCheckpointConfig | null,\n    ): void {\n      replicaDbPath = dbPath;\n      lc = createLogContext({log: logConfig}, 'write-worker');\n      unregisterCorruptionDiagnostics();\n      unregisterCorruptionDiagnosticTargets.push(\n        registerSQLiteCorruptionDiagnosticTarget({\n          debugName: 'write-worker',\n          dbPath,\n        }),\n      );\n      try {\n        db = new Database(lc, dbPath);\n        applyPragmas(db, pragmas);\n        runner = new StatementRunner(db);\n        mode = cpMode;\n        checkpointerConfig = checkpointConfig ?? undefined;\n        createProcessor();\n        createCheckpointer();\n      } catch (e) {\n        handleCorruptedDb(e);\n        throw e;\n      }\n    },\n\n    getSubscriptionState() {\n      try {\n        return getSubscriptionState(must(runner));\n      } catch (e) {\n        handleCorruptedDb(e);\n        throw e;\n      }\n    },\n\n    async processMessages(downstream: readonly ChangeStreamData[]) {\n      try {\n        let commitResult = null;\n        for (const message of downstream) {\n          const committed = must(processor).processMessage(must(lc), message);\n          if (committed) {\n            commitResult = committed;\n            if (checkpointer) {\n              await checkpointer.maybeCheckpoint();\n            }\n          }\n        }\n        return commitResult;\n      } catch (e) {\n        handleCorruptedDb(e);\n        throw e;\n      }\n    },\n\n    abort() {\n      must(processor).abort(must(lc));\n      checkpointer?.close();\n      createProcessor();\n      createCheckpointer();\n    },\n\n    stop() {\n      checkpointer?.close();\n      checkpointer = undefined;\n      db?.close();\n      db = undefined;\n      runner = undefined;\n      processor = undefined;\n      replicaDbPath = undefined;\n      unregisterCorruptionDiagnostics();\n    },\n  };\n}\n\nconst api = createAPI();\n\nport.on('message', async (msg: Request) => {\n  try {\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any -- TS can't narrow msg.method + msg.args together\n    const result = await (api[msg.method] as (...args: any[]) => unknown)(\n      ...msg.args,\n    );\n    // abort is fire-and-forget — no pending slot on the client side.\n    if (msg.method !== 'abort') {\n      port.postMessage({method: msg.method, result} as Response);\n    }\n  } catch (e) {\n    if (msg.method !== 'abort') {\n      port.postMessage({\n        method: msg.method,\n        error: serializeError(e),\n      } as Response);\n    }\n  }\n});\n"],"mappings":";;;;;;;;;;;;;;AAgCA,IAAI,CAAC,YACH,MAAM,IAAI,MAAM,6CAA6C;AAG/D,IAAM,OAAO;AAMb,SAAS,YAAiB;CACxB,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI,wCAAwD,CAAC;CAI7D,IAAI;CACJ,IAAI;CAEJ,SAAS,kCAAkC;EACzC,sCAAsC,SAAQ,eAAc,WAAW,CAAC;EACxE,wCAAwC,CAAC;CAC3C;CAEA,SAAS,kBAAkB,KAAc;EACvC,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,mBAAmB,GAAG,GAClD;EAEF,+BAA+B,IAAI,gBAAgB,eAAe,GAAG;EACrE,IAAI;GACF,GAAG,OAAO,4BAA4B,eAAe;GACrD,aAAa,aAAa;EAC5B,SAAS,GAAG;GACV,GAAG,OAAO,kCAAkC,iBAAiB,CAAC;EAChE;CACF;CAEA,SAAS,kBAAkB;EACzB,YAAY,IAAI,gBAAgB,KAAK,MAAM,GAAG,KAAK,IAAI,IAAI,KAAK,QAAQ;GACtE,kBAAkB,GAAG;GACrB,KAAK,YAAY,EACf,YAAY,eAAe,GAAG,EAChC,CAAsB;EACxB,CAAC;CACH;CAEA,SAAS,qBAAqB;EAC5B,IAAI,oBAAoB;GACtB,OAAO,MAAM,MAAM,eAAe,iBAAiB;GACnD,eAAe,IAAI,uBACjB,IACA,IACA,IAAI,qBAAqB,IAAI,aAAa,GAC1C,kBACF;EACF;CACF;CAEA,OAAO;EACL,KACE,QACA,QACA,SACA,WACA,kBACM;GACN,gBAAgB;GAChB,KAAK,iBAAiB,EAAC,KAAK,UAAS,GAAG,cAAc;GACtD,gCAAgC;GAChC,sCAAsC,KACpC,yCAAyC;IACvC,WAAW;IACX;GACF,CAAC,CACH;GACA,IAAI;IACF,KAAK,IAAI,SAAS,IAAI,MAAM;IAC5B,aAAa,IAAI,OAAO;IACxB,SAAS,IAAI,gBAAgB,EAAE;IAC/B,OAAO;IACP,qBAAqB,oBAAoB,KAAA;IACzC,gBAAgB;IAChB,mBAAmB;GACrB,SAAS,GAAG;IACV,kBAAkB,CAAC;IACnB,MAAM;GACR;EACF;EAEA,uBAAuB;GACrB,IAAI;IACF,OAAO,qBAAqB,KAAK,MAAM,CAAC;GAC1C,SAAS,GAAG;IACV,kBAAkB,CAAC;IACnB,MAAM;GACR;EACF;EAEA,MAAM,gBAAgB,YAAyC;GAC7D,IAAI;IACF,IAAI,eAAe;IACnB,KAAK,MAAM,WAAW,YAAY;KAChC,MAAM,YAAY,KAAK,SAAS,EAAE,eAAe,KAAK,EAAE,GAAG,OAAO;KAClE,IAAI,WAAW;MACb,eAAe;MACf,IAAI,cACF,MAAM,aAAa,gBAAgB;KAEvC;IACF;IACA,OAAO;GACT,SAAS,GAAG;IACV,kBAAkB,CAAC;IACnB,MAAM;GACR;EACF;EAEA,QAAQ;GACN,KAAK,SAAS,EAAE,MAAM,KAAK,EAAE,CAAC;GAC9B,cAAc,MAAM;GACpB,gBAAgB;GAChB,mBAAmB;EACrB;EAEA,OAAO;GACL,cAAc,MAAM;GACpB,eAAe,KAAA;GACf,IAAI,MAAM;GACV,KAAK,KAAA;GACL,SAAS,KAAA;GACT,YAAY,KAAA;GACZ,gBAAgB,KAAA;GAChB,gCAAgC;EAClC;CACF;AACF;AAEA,IAAM,MAAM,UAAU;AAEtB,KAAK,GAAG,WAAW,OAAO,QAAiB;CACzC,IAAI;EAEF,MAAM,SAAS,MAAO,IAAI,IAAI,QAC5B,GAAG,IAAI,IACT;EAEA,IAAI,IAAI,WAAW,SACjB,KAAK,YAAY;GAAC,QAAQ,IAAI;GAAQ;EAAM,CAAa;CAE7D,SAAS,GAAG;EACV,IAAI,IAAI,WAAW,SACjB,KAAK,YAAY;GACf,QAAQ,IAAI;GACZ,OAAO,eAAe,CAAC;EACzB,CAAa;CAEjB;AACF,CAAC"}
+{"version":3,"file":"write-worker.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/write-worker.ts"],"sourcesContent":["import {parentPort} from 'node:worker_threads';\nimport type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport type {LogConfig} from '../../../../shared/src/logging.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n  registerSQLiteCorruptionDiagnosticTarget,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {createLogContext} from '../../server/logging.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {LitestreamCheckpointer} from '../litestream/litestream-checkpointer.ts';\nimport {LitestreamController} from '../litestream/litestream-controller.ts';\nimport {ChangeProcessor, type ChangeProcessorMode} from './change-processor.ts';\nimport {readReplicaBackfills} from './schema/backfilling.ts';\nimport {getSubscriptionState} from './schema/replication-state.ts';\nimport {\n  applyPragmas,\n  serializeError,\n  type ArgsMap,\n  type ForceCheckpointConfig,\n  type Method,\n  type PragmaConfig,\n  type Request,\n  type Response,\n  type ResultMap,\n  type WriteError,\n} from './write-worker-client.ts';\n\nif (!parentPort) {\n  throw new Error('write-worker must be run as a worker thread');\n}\n\nconst port = parentPort;\n\ntype API = {\n  [M in Method]: (...args: ArgsMap[M]) => ResultMap[M] | Promise<ResultMap[M]>;\n};\n\nfunction createAPI(): API {\n  let db: Database | undefined;\n  let runner: StatementRunner | undefined;\n  let processor: ChangeProcessor | undefined;\n  let mode: ChangeProcessorMode | undefined;\n  let lc: LogContext | undefined;\n  let replicaDbPath: string | undefined;\n  let unregisterCorruptionDiagnosticTargets: (() => void)[] = [];\n\n  // Set when write-path checkpoint backpressure is enabled (backup replicator\n  // on litestream v5). `undefined` disables the feature.\n  let checkpointerConfig: ForceCheckpointConfig | undefined;\n  let checkpointer: LitestreamCheckpointer | undefined;\n\n  function unregisterCorruptionDiagnostics() {\n    unregisterCorruptionDiagnosticTargets.forEach(unregister => unregister());\n    unregisterCorruptionDiagnosticTargets = [];\n  }\n\n  function handleCorruptedDb(err: unknown) {\n    if (!lc || !replicaDbPath || !isSQLiteCorruption(err)) {\n      return;\n    }\n    logSQLiteCorruptionDiagnostics(lc, 'write-worker', replicaDbPath, err);\n    try {\n      lc.warn?.(`deleting corrupted db at ${replicaDbPath}`);\n      deleteLiteDB(replicaDbPath);\n    } catch (e) {\n      lc.warn?.(`error deleting corrupted db at ${replicaDbPath}`, e);\n    }\n  }\n\n  function createProcessor() {\n    processor = new ChangeProcessor(must(runner), must(mode), (_lc, err) => {\n      handleCorruptedDb(err);\n      port.postMessage({\n        writeError: serializeError(err),\n      } satisfies WriteError);\n    });\n  }\n\n  function createCheckpointer() {\n    if (checkpointerConfig) {\n      assert(lc && db && replicaDbPath, `not initialized`);\n      checkpointer = new LitestreamCheckpointer(\n        lc,\n        db,\n        new LitestreamController(lc, replicaDbPath),\n        checkpointerConfig,\n      );\n    }\n  }\n\n  return {\n    init(\n      dbPath: string,\n      cpMode: ChangeProcessorMode,\n      pragmas: PragmaConfig,\n      logConfig: LogConfig,\n      checkpointConfig: ForceCheckpointConfig | null,\n    ): void {\n      replicaDbPath = dbPath;\n      lc = createLogContext({log: logConfig}, 'write-worker');\n      unregisterCorruptionDiagnostics();\n      unregisterCorruptionDiagnosticTargets.push(\n        registerSQLiteCorruptionDiagnosticTarget({\n          debugName: 'write-worker',\n          dbPath,\n        }),\n      );\n      try {\n        db = new Database(lc, dbPath);\n        applyPragmas(db, pragmas);\n        runner = new StatementRunner(db);\n        mode = cpMode;\n        checkpointerConfig = checkpointConfig ?? undefined;\n        createProcessor();\n        createCheckpointer();\n      } catch (e) {\n        handleCorruptedDb(e);\n        throw e;\n      }\n    },\n\n    getSubscriptionState() {\n      try {\n        // The write worker is the replica's only writer, so these reads are\n        // of the same snapshot.\n        return {\n          ...getSubscriptionState(must(runner)),\n          backfills: readReplicaBackfills(must(runner).db),\n        };\n      } catch (e) {\n        handleCorruptedDb(e);\n        throw e;\n      }\n    },\n\n    async processMessages(downstream: readonly ChangeStreamData[]) {\n      try {\n        let commitResult = null;\n        for (const message of downstream) {\n          const committed = must(processor).processMessage(must(lc), message);\n          if (committed) {\n            commitResult = committed;\n            if (checkpointer) {\n              await checkpointer.maybeCheckpoint();\n            }\n          }\n        }\n        return commitResult;\n      } catch (e) {\n        handleCorruptedDb(e);\n        throw e;\n      }\n    },\n\n    abort() {\n      must(processor).abort(must(lc));\n      checkpointer?.close();\n      createProcessor();\n      createCheckpointer();\n    },\n\n    stop() {\n      checkpointer?.close();\n      checkpointer = undefined;\n      db?.close();\n      db = undefined;\n      runner = undefined;\n      processor = undefined;\n      replicaDbPath = undefined;\n      unregisterCorruptionDiagnostics();\n    },\n  };\n}\n\nconst api = createAPI();\n\nport.on('message', async (msg: Request) => {\n  try {\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any -- TS can't narrow msg.method + msg.args together\n    const result = await (api[msg.method] as (...args: any[]) => unknown)(\n      ...msg.args,\n    );\n    // abort is fire-and-forget — no pending slot on the client side.\n    if (msg.method !== 'abort') {\n      port.postMessage({method: msg.method, result} as Response);\n    }\n  } catch (e) {\n    if (msg.method !== 'abort') {\n      port.postMessage({\n        method: msg.method,\n        error: serializeError(e),\n      } as Response);\n    }\n  }\n});\n"],"mappings":";;;;;;;;;;;;;;;AAiCA,IAAI,CAAC,YACH,MAAM,IAAI,MAAM,6CAA6C;AAG/D,IAAM,OAAO;AAMb,SAAS,YAAiB;CACxB,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI,wCAAwD,CAAC;CAI7D,IAAI;CACJ,IAAI;CAEJ,SAAS,kCAAkC;EACzC,sCAAsC,SAAQ,eAAc,WAAW,CAAC;EACxE,wCAAwC,CAAC;CAC3C;CAEA,SAAS,kBAAkB,KAAc;EACvC,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,mBAAmB,GAAG,GAClD;EAEF,+BAA+B,IAAI,gBAAgB,eAAe,GAAG;EACrE,IAAI;GACF,GAAG,OAAO,4BAA4B,eAAe;GACrD,aAAa,aAAa;EAC5B,SAAS,GAAG;GACV,GAAG,OAAO,kCAAkC,iBAAiB,CAAC;EAChE;CACF;CAEA,SAAS,kBAAkB;EACzB,YAAY,IAAI,gBAAgB,KAAK,MAAM,GAAG,KAAK,IAAI,IAAI,KAAK,QAAQ;GACtE,kBAAkB,GAAG;GACrB,KAAK,YAAY,EACf,YAAY,eAAe,GAAG,EAChC,CAAsB;EACxB,CAAC;CACH;CAEA,SAAS,qBAAqB;EAC5B,IAAI,oBAAoB;GACtB,OAAO,MAAM,MAAM,eAAe,iBAAiB;GACnD,eAAe,IAAI,uBACjB,IACA,IACA,IAAI,qBAAqB,IAAI,aAAa,GAC1C,kBACF;EACF;CACF;CAEA,OAAO;EACL,KACE,QACA,QACA,SACA,WACA,kBACM;GACN,gBAAgB;GAChB,KAAK,iBAAiB,EAAC,KAAK,UAAS,GAAG,cAAc;GACtD,gCAAgC;GAChC,sCAAsC,KACpC,yCAAyC;IACvC,WAAW;IACX;GACF,CAAC,CACH;GACA,IAAI;IACF,KAAK,IAAI,SAAS,IAAI,MAAM;IAC5B,aAAa,IAAI,OAAO;IACxB,SAAS,IAAI,gBAAgB,EAAE;IAC/B,OAAO;IACP,qBAAqB,oBAAoB,KAAA;IACzC,gBAAgB;IAChB,mBAAmB;GACrB,SAAS,GAAG;IACV,kBAAkB,CAAC;IACnB,MAAM;GACR;EACF;EAEA,uBAAuB;GACrB,IAAI;IAGF,OAAO;KACL,GAAG,qBAAqB,KAAK,MAAM,CAAC;KACpC,WAAW,qBAAqB,KAAK,MAAM,EAAE,EAAE;IACjD;GACF,SAAS,GAAG;IACV,kBAAkB,CAAC;IACnB,MAAM;GACR;EACF;EAEA,MAAM,gBAAgB,YAAyC;GAC7D,IAAI;IACF,IAAI,eAAe;IACnB,KAAK,MAAM,WAAW,YAAY;KAChC,MAAM,YAAY,KAAK,SAAS,EAAE,eAAe,KAAK,EAAE,GAAG,OAAO;KAClE,IAAI,WAAW;MACb,eAAe;MACf,IAAI,cACF,MAAM,aAAa,gBAAgB;KAEvC;IACF;IACA,OAAO;GACT,SAAS,GAAG;IACV,kBAAkB,CAAC;IACnB,MAAM;GACR;EACF;EAEA,QAAQ;GACN,KAAK,SAAS,EAAE,MAAM,KAAK,EAAE,CAAC;GAC9B,cAAc,MAAM;GACpB,gBAAgB;GAChB,mBAAmB;EACrB;EAEA,OAAO;GACL,cAAc,MAAM;GACpB,eAAe,KAAA;GACf,IAAI,MAAM;GACV,KAAK,KAAA;GACL,SAAS,KAAA;GACT,YAAY,KAAA;GACZ,gBAAgB,KAAA;GAChB,gCAAgC;EAClC;CACF;AACF;AAEA,IAAM,MAAM,UAAU;AAEtB,KAAK,GAAG,WAAW,OAAO,QAAiB;CACzC,IAAI;EAEF,MAAM,SAAS,MAAO,IAAI,IAAI,QAC5B,GAAG,IAAI,IACT;EAEA,IAAI,IAAI,WAAW,SACjB,KAAK,YAAY;GAAC,QAAQ,IAAI;GAAQ;EAAM,CAAa;CAE7D,SAAS,GAAG;EACV,IAAI,IAAI,WAAW,SACjB,KAAK,YAAY;GACf,QAAQ,IAAI;GACZ,OAAO,eAAe,CAAC;EACzB,CAAa;CAEjB;AACF,CAAC"}