@rocicorp/zero 1.6.0-canary.0 → 1.6.0-canary.10

package/out/zero-cache/src/services/analyze.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"analyze.js","names":[],"sources":["../../../../../zero-cache/src/services/analyze.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {AnalyzeQueryResult} from '../../../zero-protocol/src/analyze-query-result.ts';\nimport type {AST} from '../../../zero-protocol/src/ast.ts';\nimport type {ClientSchema} from '../../../zero-protocol/src/client-schema.ts';\nimport type {PermissionsConfig} from '../../../zero-schema/src/compiled-permissions.ts';\nimport {Debug} from '../../../zql/src/builder/debug-delegate.ts';\nimport {MemoryStorage} from '../../../zql/src/ivm/memory-storage.ts';\nimport {\n  AccumulatorDebugger,\n  serializePlanDebugEvents,\n} from '../../../zql/src/planner/planner-debug.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport {explainQueries} from '../../../zqlite/src/explain-queries.ts';\nimport {createSQLiteCostModel} from '../../../zqlite/src/sqlite-cost-model.ts';\nimport {TableSource} from '../../../zqlite/src/table-source.ts';\nimport type {JWTAuth} from '../auth/auth.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {computeZqlSpecs, mustGetTableSpec} from '../db/lite-tables.ts';\nimport type {LiteAndZqlSpec, LiteTableSpec} from '../db/specs.ts';\nimport {runAst} from './run-ast.ts';\nimport {TimeSliceTimer} from './view-syncer/view-syncer.ts';\n\nconst TIME_SLICE_LAP_THRESHOLD_MS = 200;\n\nexport async function analyzeQuery(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  clientSchema: ClientSchema,\n  ast: AST,\n  syncedRows = true,\n  vendedRows = false,\n  permissions?: PermissionsConfig,\n  auth?: JWTAuth,\n  joinPlans = false,\n): Promise<AnalyzeQueryResult> {\n  using db = new Database(lc, config.replica.file);\n  const fullTables = new Map<string, LiteTableSpec>();\n  const tableSpecs = new Map<string, LiteAndZqlSpec>();\n  const tables = new Map<string, TableSource>();\n\n  computeZqlSpecs(\n    lc,\n    db,\n    {includeBackfillingColumns: false},\n    tableSpecs,\n    fullTables,\n  );\n\n  // Mirror production: the planner runs iff ZERO_ENABLE_QUERY_PLANNER is set\n  // on the server, so the analysis reflects what actually executes. Diagnostic\n  // event collection is orthogonal and opt-in via `joinPlans`.\n  const costModel = config.enableQueryPlanner\n    ? createSQLiteCostModel(db, tableSpecs)\n    : undefined;\n  const planDebugger = joinPlans ? new AccumulatorDebugger() : undefined;\n  const timer = await new TimeSliceTimer(lc).start();\n  const shouldYield = () => timer.elapsedLap() > TIME_SLICE_LAP_THRESHOLD_MS;\n  const yieldProcess = () => timer.yieldProcess();\n  const result = await runAst(\n    lc,\n    clientSchema,\n    ast,\n    true,\n    {\n      applyPermissions: permissions !== undefined,\n      syncedRows,\n      vendedRows,\n      auth,\n      db,\n      tableSpecs,\n      permissions,\n      costModel,\n      planDebugger,\n      host: {\n        debug: new Debug(),\n        getSource(tableName: string) {\n          let source = tables.get(tableName);\n          if (source) {\n            return source;\n          }\n\n          const tableSpec = mustGetTableSpec(tableSpecs, tableName);\n          const {primaryKey} = tableSpec.tableSpec;\n\n          source = new TableSource(\n            lc,\n            config.log,\n            db,\n            tableName,\n            tableSpec.zqlSpec,\n            primaryKey,\n            shouldYield,\n          );\n          tables.set(tableName, source);\n          return source;\n        },\n        createStorage() {\n          return new MemoryStorage();\n        },\n        decorateSourceInput: input => input,\n        decorateInput: input => input,\n        addEdge() {},\n        decorateFilterInput: input => input,\n      },\n    },\n    yieldProcess,\n  );\n\n  result.sqlitePlans = explainQueries(result.readRowCountsByQuery ?? {}, db);\n\n  if (planDebugger) {\n    result.joinPlans = serializePlanDebugEvents(planDebugger.events);\n  }\n\n  return result;\n}\n"],"mappings":";;;;;;;;;;;;AAsBA,IAAM,8BAA8B;AAEpC,eAAsB,aACpB,IACA,QACA,cACA,KACA,aAAa,MACb,aAAa,OACb,aACA,MACA,YAAY,OACiB;;;EAC7B,MAAM,KAAA,YAAA,EAAK,IAAI,SAAS,IAAI,OAAO,QAAQ,KAAK,CAAA;EAChD,MAAM,6BAAa,IAAI,KAA4B;EACnD,MAAM,6BAAa,IAAI,KAA6B;EACpD,MAAM,yBAAS,IAAI,KAA0B;AAE7C,kBACE,IACA,IACA,EAAC,2BAA2B,OAAM,EAClC,YACA,WACD;EAKD,MAAM,YAAY,OAAO,qBACrB,sBAAsB,IAAI,WAAW,GACrC,KAAA;EACJ,MAAM,eAAe,YAAY,IAAI,qBAAqB,GAAG,KAAA;EAC7D,MAAM,QAAQ,MAAM,IAAI,eAAe,GAAG,CAAC,OAAO;EAClD,MAAM,oBAAoB,MAAM,YAAY,GAAG;EAC/C,MAAM,qBAAqB,MAAM,cAAc;EAC/C,MAAM,SAAS,MAAM,OACnB,IACA,cACA,KACA,MACA;GACE,kBAAkB,gBAAgB,KAAA;GAClC;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA,MAAM;IACJ,OAAO,IAAI,OAAO;IAClB,UAAU,WAAmB;KAC3B,IAAI,SAAS,OAAO,IAAI,UAAU;AAClC,SAAI,OACF,QAAO;KAGT,MAAM,YAAY,iBAAiB,YAAY,UAAU;KACzD,MAAM,EAAC,eAAc,UAAU;AAE/B,cAAS,IAAI,YACX,IACA,OAAO,KACP,IACA,WACA,UAAU,SACV,YACA,YACD;AACD,YAAO,IAAI,WAAW,OAAO;AAC7B,YAAO;;IAET,gBAAgB;AACd,YAAO,IAAI,eAAe;;IAE5B,sBAAqB,UAAS;IAC9B,gBAAe,UAAS;IACxB,UAAU;IACV,sBAAqB,UAAS;IAC/B;GACF,EACD,aACD;AAED,SAAO,cAAc,eAAe,OAAO,wBAAwB,EAAE,EAAE,GAAG;AAE1E,MAAI,aACF,QAAO,YAAY,yBAAyB,aAAa,OAAO;AAGlE,SAAO"}

package/out/zero-cache/src/services/change-source/common/backfill-manager.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"backfill-manager.js","names":["#lc","#requiredBackfills","#changeStreamer","#backfillStreamer","#jsonFormat","#awaitingStatusWatermarks","#minBackoffMs","#maxBackoffMs","#retryDelayMs","#lastStatusWatermark","#setRequiredBackfill","#checkAndStartBackfill","#backfillRetryTimer","#runningBackfill","#runBackfill","#stopRunningBackfill","#retryBackfillWithBackoff","#changeStreamReached","#currentTxWatermark","#setLastStatusWatermark","#backfillRunningFor","#deleteRequiredBackfill"],"sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {stringify} from '../../../../../shared/src/bigint-json.ts';\nimport {CustomKeyMap} from '../../../../../shared/src/custom-key-map.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {randInt} from '../../../../../shared/src/rand.ts';\nimport {JSON_STRINGIFIED, type JSONFormat} from '../../../types/lite.ts';\nimport {\n  stateVersionFromString,\n  stateVersionToString,\n} from '../../../types/state-version.ts';\nimport type {\n  BackfillCompleted,\n  BackfillRequest,\n  ChangeStreamMessage,\n  Identifier,\n  MessageBackfill,\n} from '../protocol/current.ts';\nimport type {\n  Cancelable,\n  ChangeStreamMultiplexer,\n  Listener,\n} from './change-stream-multiplexer.ts';\n\nfunction tableKey({schema, name}: Identifier) {\n  return `${schema}.${name}`;\n}\n\ntype BackfillStreamer = (\n  req: BackfillRequest,\n) => AsyncGenerator<MessageBackfill | BackfillCompleted>;\n\ntype RunningBackfillState = {\n  request: BackfillRequest;\n  canceledReason?: string | undefined;\n  minWatermark: string;\n};\n\nconst MIN_BACKOFF_INTERVAL_MS = 2_000;\nconst MAX_BACKOFF_INTERVAL_MS = 60_000;\n\ntype AwaitingStatusWatermark = {\n  watermark: string;\n  reached: () => void;\n};\n\n/**\n * The BackfillManager initiates backfills for BackfillRequests from the\n * change-streamer (i.e. unfinished backfills from previous sessions)\n * or for new backfills signaled by `create-table` or `add-column` messages\n * from the change-source.\n *\n * The BackfillManager registers itself as a change stream listener in order\n * to track necessary backfills, and potentially invalidate the in-progress\n * backfill (e.g. due to a schema change) so that it can be retried at a\n * new snapshot.\n *\n * The manager also handles low priority streaming of the backfill messages\n * using the {@link ChangeStreamMultiplexer}, implementing a policy of always\n * releasing its reservation if another producer (i.e. the main change stream)\n * has messages to stream.\n */\nexport class BackfillManager implements Cancelable, Listener {\n  readonly #lc: LogContext;\n\n  /**\n   * Tracks the metadata of required backfills based on schema changes\n   * and initial backfill requests.\n   */\n  readonly #requiredBackfills = new CustomKeyMap<Identifier, BackfillRequest>(\n    tableKey,\n  );\n  readonly #changeStreamer: ChangeStreamMultiplexer;\n  readonly #backfillStreamer: BackfillStreamer;\n  readonly #jsonFormat: JSONFormat;\n\n  /**\n   * The current running backfill. The backfill request is always also in\n   * `#requiredBackfills` (technically, it can be a subset of what's in\n   * `#requiredBackfills`); the request is removed from `#requiredBackfills`\n   * upon completion.\n   */\n  #runningBackfill: RunningBackfillState | null = null;\n\n  /** The last seen watermark in the change stream. */\n  #lastStatusWatermark: string | null = null;\n\n  readonly #awaitingStatusWatermarks: AwaitingStatusWatermark[] = [];\n\n  /** The watermark of the current transaction in the change stream. */\n  #currentTxWatermark: string | null = null;\n\n  constructor(\n    lc: LogContext,\n    changeStreamer: ChangeStreamMultiplexer,\n    backfillStreamer: BackfillStreamer,\n    jsonFormat: JSONFormat = JSON_STRINGIFIED,\n    minBackoffMs = MIN_BACKOFF_INTERVAL_MS,\n    maxBackoffMs = MAX_BACKOFF_INTERVAL_MS,\n  ) {\n    this.#lc = lc.withContext('component', 'backfill-manager');\n    this.#changeStreamer = changeStreamer;\n    this.#backfillStreamer = backfillStreamer;\n    this.#jsonFormat = jsonFormat;\n    this.#minBackoffMs = minBackoffMs;\n    this.#maxBackoffMs = maxBackoffMs;\n    this.#retryDelayMs = minBackoffMs;\n  }\n\n  run(lastWatermark: string, initialRequests: BackfillRequest[]) {\n    this.#lc.info?.(\n      `starting backfill manager with ${initialRequests.length} initial requests`,\n      {requests: initialRequests},\n    );\n    this.#lastStatusWatermark = lastWatermark;\n    initialRequests.forEach(req =>\n      this.#setRequiredBackfill('initial-request', req),\n    );\n    this.#checkAndStartBackfill();\n  }\n\n  #setLastStatusWatermark({watermark}: {watermark: string}) {\n    // Only allow the watermark to move forward. This prevents a backfill\n    // transaction (whose watermark is unrelated to change-stream state)\n    // from moving the watermark backwards.\n    if ((this.#lastStatusWatermark ?? '') < watermark) {\n      this.#lastStatusWatermark = watermark;\n      for (let i = this.#awaitingStatusWatermarks.length - 1; i >= 0; i--) {\n        const awaiting = this.#awaitingStatusWatermarks[i];\n        if (watermark >= awaiting.watermark) {\n          awaiting.reached();\n          this.#awaitingStatusWatermarks.splice(i, 1);\n        }\n      }\n    }\n  }\n\n  #changeStreamReached(\n    lc: LogContext,\n    watermark: string,\n  ): Promise<void> | null {\n    if ((this.#lastStatusWatermark ?? '') < watermark) {\n      const {promise, resolve: reached} = resolver();\n      this.#awaitingStatusWatermarks.push({watermark, reached});\n      lc.info?.(\n        `waiting for change stream (at ${this.#lastStatusWatermark}) to reach ${watermark}`,\n      );\n      return promise;\n    }\n    return null;\n  }\n\n  readonly #minBackoffMs: number;\n  readonly #maxBackoffMs: number;\n  #retryDelayMs: number;\n  #backfillRetryTimer: NodeJS.Timeout | undefined;\n\n  #checkAndStartBackfill() {\n    if (\n      !this.#backfillRetryTimer &&\n      !this.#runningBackfill &&\n      this.#requiredBackfills.size\n    ) {\n      // Pick a random backfill to avoid head-of-line blocking by a\n      // problematic backfill (e.g. awaiting a primary key). This is\n      // simpler that adding logic to classify (and declassify)\n      // problematic backfills.\n      const candidates = [...this.#requiredBackfills.values()];\n      const request = candidates[randInt(0, candidates.length - 1)];\n      const state = {request, minWatermark: ''};\n      const lc = this.#lc.withContext('table', request.table.name);\n\n      this.#runningBackfill = state;\n      void this.#runBackfill(lc, state)\n        .then(() => {\n          this.#stopRunningBackfill('backfill exited', state);\n          this.#retryDelayMs = this.#minBackoffMs; // reset on success\n        })\n        // For unexpected errors (e.g. upstream replication slot\n        // unavailability), retry with exponential backoff.\n        .catch(e => {\n          this.#stopRunningBackfill(String(e), state);\n          this.#retryBackfillWithBackoff(e);\n        });\n    }\n  }\n\n  #retryBackfillWithBackoff(e: unknown) {\n    const log = this.#retryDelayMs === this.#maxBackoffMs ? 'error' : 'warn';\n    this.#lc[log]?.(\n      `Error running backfill. Retrying in ${this.#retryDelayMs} ms`,\n      e,\n    );\n    this.#backfillRetryTimer = setTimeout(() => {\n      this.#backfillRetryTimer = undefined;\n      this.#checkAndStartBackfill();\n    }, this.#retryDelayMs);\n\n    this.#retryDelayMs = Math.min(this.#retryDelayMs * 2, this.#maxBackoffMs);\n  }\n\n  async #runBackfill(lc: LogContext, state: RunningBackfillState) {\n    const changeStream = this.#changeStreamer; // Purely for readability\n\n    // backfillTx is set if and only if a changeStreamer reservation has been\n    // acquired and the backfill stream is inside a transaction.\n    let backfillTx: string | null = null;\n\n    /**\n     * @returns the new tx watermark, or null if backfill was cancelled\n     */\n    const beginTxFor = async (\n      msg: MessageBackfill | BackfillCompleted,\n    ): Promise<string | null> => {\n      assert(backfillTx === null, 'Expected no active backfill transaction');\n      const lastWatermark = await changeStream.reserve('backfill');\n\n      // After obtaining the changeStream reservation, check if the stream\n      // had changes that resulted in invalidating / canceling this backfill.\n      if (\n        state.canceledReason ||\n        (msg.tag === 'backfill' && msg.watermark < state.minWatermark)\n      ) {\n        if (state.canceledReason === undefined) {\n          assert(msg.tag === 'backfill', 'Expected backfill message tag'); // TypeScript should have figured this out.\n          this.#stopRunningBackfill(\n            `row key change at ${state.minWatermark} ` +\n              `postdates backfill watermark at ${msg.watermark}`,\n            state,\n          );\n        }\n        changeStream.release(lastWatermark);\n        return null;\n      }\n\n      const {major, minor = 0n} = stateVersionFromString(lastWatermark);\n      let tx = stateVersionToString({\n        major,\n        minor: BigInt(minor) + 1n,\n      });\n\n      if (msg.tag === 'backfill-completed' && tx < msg.watermark) {\n        // At this point it must be the case that the #changeStreamReached() the\n        // backfill watermark. Given that guarantee, ensure that the version of the\n        // transaction containing the backfill-completed message is at least up\n        // to the backfill watermark, so that the final database state version is\n        // never earlier than the version of any backfilled rows.\n        tx = msg.watermark;\n      }\n\n      void changeStream.push([\n        'begin',\n        {tag: 'begin', json: this.#jsonFormat, skipAck: true},\n        {commitWatermark: tx},\n      ]);\n      return (backfillTx = tx);\n    };\n\n    const commitTx = () => {\n      if (backfillTx) {\n        void changeStream.push([\n          'commit',\n          {tag: 'commit'},\n          {watermark: backfillTx},\n        ]);\n        changeStream.release(backfillTx);\n      }\n      backfillTx = null;\n    };\n\n    for await (const msg of this.#backfillStreamer(state.request)) {\n      // Before sending `backfill-completed`, the main replication stream\n      // may need to catch up, and/or the current transaction may need to be\n      // committed to open a new transaction that's up to backfill watermark.\n      const mustWaitBeforeFlush =\n        msg.tag === 'backfill-completed' &&\n        (this.#changeStreamReached(lc, msg.watermark) ||\n          (backfillTx !== null && backfillTx < msg.watermark));\n\n      // If necessary, yield the reservation to the main stream.\n      if (\n        backfillTx &&\n        (changeStream.waiterDelay() > 0 || mustWaitBeforeFlush)\n      ) {\n        commitTx();\n      }\n\n      mustWaitBeforeFlush && (await mustWaitBeforeFlush);\n\n      if (\n        msg.tag === 'backfill' &&\n        msg.rowValues.length > 0 &&\n        msg.relation.rowKey.columns.length === 0\n      ) {\n        throw new MissingRowKeyError(state.request);\n      }\n\n      // Reserve the changeStreamer if not in a transaction.\n      if ((backfillTx ??= await beginTxFor(msg)) === null) {\n        lc.info?.(\n          `backfill stream canceled: ${state.canceledReason}`,\n          state.request,\n        );\n        this.#checkAndStartBackfill(); // start the next backfill if present\n        return; // this backfill is canceled\n      }\n\n      // `await` to allow the change streamer to exert back pressure\n      // on backfills.\n      await changeStream.push(['data', msg]);\n    }\n\n    // Flush any final tx and release the stream.\n    backfillTx && commitTx();\n    lc.debug?.(`backfill stream exited`, state.canceledReason ?? '');\n  }\n\n  #backfillRunningFor(table: Identifier): RunningBackfillState | null {\n    const state = this.#runningBackfill;\n    return state?.request.table.schema === table.schema &&\n      state.request.table.name === table.name\n      ? state\n      : null;\n  }\n\n  /**\n   * Stops the running backfill for the specified `reason`. If `instance` is\n   * specified, the running backfill is stopped only if it is that instance.\n   * This allows the running backfill itself to clear backfill state without\n   * accidentally stopping a different (e.g. subsequent) backfill.\n   */\n  #stopRunningBackfill(reason?: string, instance?: RunningBackfillState) {\n    const backfill = this.#runningBackfill;\n    if (backfill && backfill === (instance ?? backfill)) {\n      backfill.canceledReason = reason;\n      this.#runningBackfill = null;\n      reason && this.#lc.info?.(`canceling backfill:`, reason);\n    }\n  }\n\n  #setRequiredBackfill(source: string, req: BackfillRequest) {\n    const action = this.#requiredBackfills.has(req.table) ? 'updated' : 'added';\n    this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n    this.#requiredBackfills.set(req.table, req);\n  }\n\n  #deleteRequiredBackfill(source: string, id: Identifier) {\n    const req = this.#requiredBackfills.get(id);\n    if (req) {\n      const action = source === 'backfill-completed' ? 'completed' : 'dropped';\n      this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n      this.#requiredBackfills.delete(id);\n    }\n  }\n\n  /**\n   * Implements {@link Listener.onChange()}, invoked by the\n   * {@link ChangeStreamMultiplexer}.\n   */\n  onChange(message: ChangeStreamMessage): void {\n    if (message[0] === 'begin') {\n      this.#currentTxWatermark = message[2].commitWatermark;\n      return;\n    }\n    if (message[0] === 'commit') {\n      this.#currentTxWatermark = null;\n      this.#setLastStatusWatermark(message[2]);\n      // Every commit is a candidate for starting the next backfill\n      // (if one is not currently running).\n      this.#checkAndStartBackfill();\n      return;\n    }\n    if (message[0] === 'status') {\n      this.#setLastStatusWatermark(message[2]);\n      return;\n    }\n    if (message[0] !== 'data') {\n      return;\n    }\n    const change = message[1];\n    const {tag} = change;\n    switch (tag) {\n      case 'update-table-metadata': {\n        const {table, new: metadata} = change;\n        const backfillRequest = this.#requiredBackfills.get(table);\n        if (backfillRequest) {\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            table: {...backfillRequest.table, metadata},\n          });\n          if (this.#backfillRunningFor(table)) {\n            this.#stopRunningBackfill(`TableMetadata updated`);\n          }\n        }\n        break;\n      }\n      case 'create-table': {\n        const {\n          spec: {schema, name},\n          metadata = null,\n          backfill,\n        } = change;\n\n        if (backfill) {\n          this.#setRequiredBackfill(tag, {\n            table: {schema, name, metadata},\n            columns: backfill,\n          });\n        }\n        break;\n      }\n      case 'rename-table': {\n        const {old, new: newTable} = change;\n        const backfillRequest = this.#requiredBackfills.get(old);\n        if (backfillRequest) {\n          const {schema, name} = newTable;\n          this.#deleteRequiredBackfill(tag, old);\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            table: {...backfillRequest.table, schema, name},\n          });\n          if (this.#backfillRunningFor(old)) {\n            this.#stopRunningBackfill(`table renamed`);\n          }\n        }\n        break;\n      }\n      case 'drop-table': {\n        const {id} = change;\n        const backfillRequest = this.#requiredBackfills.get(id);\n        if (backfillRequest) {\n          this.#deleteRequiredBackfill(tag, id);\n          if (this.#backfillRunningFor(id)) {\n            this.#stopRunningBackfill(`table dropped`);\n          }\n        }\n        break;\n      }\n      case 'add-column': {\n        const {\n          table,\n          tableMetadata: metadata = null,\n          column,\n          backfill,\n        } = change;\n        if (backfill) {\n          const backfillRequest = this.#requiredBackfills.get(table);\n          if (!backfillRequest) {\n            this.#setRequiredBackfill(tag, {\n              table: {...table, metadata},\n              columns: {[column.name]: backfill},\n            });\n          } else {\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              table: {...backfillRequest.table, metadata},\n              columns: {\n                ...backfillRequest.columns,\n                [column.name]: backfill,\n              },\n            });\n            // Note: The running backfill need not be canceled if a\n            //   new column is added. The new column will be backfilled\n            //   by its own stream after the current backfill completes.\n          }\n        }\n        break;\n      }\n      case 'update-column': {\n        const {\n          table,\n          old: {name: oldName},\n          new: {name: newName},\n        } = change;\n        if (oldName !== newName) {\n          const backfillRequest = this.#requiredBackfills.get(table);\n          if (backfillRequest && oldName in backfillRequest.columns) {\n            const {[oldName]: colSpec, ...otherCols} = backfillRequest.columns;\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              columns: {...otherCols, [newName]: colSpec},\n            });\n            const backfill = this.#backfillRunningFor(table);\n            if (backfill && oldName in backfill.request.columns) {\n              this.#stopRunningBackfill(`column renamed`);\n            }\n          }\n        }\n        break;\n      }\n      case 'drop-column': {\n        const {table, column} = change;\n        const backfillRequest = this.#requiredBackfills.get(table);\n        if (backfillRequest && column in backfillRequest.columns) {\n          const {[column]: _excluded, ...remaining} = backfillRequest.columns;\n          if (Object.keys(remaining).length === 0) {\n            this.#deleteRequiredBackfill(tag, table);\n          } else {\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              columns: remaining,\n            });\n          }\n          const backfill = this.#backfillRunningFor(table);\n          if (backfill && column in backfill.request.columns) {\n            this.#stopRunningBackfill(`column dropped`);\n          }\n        }\n        break;\n      }\n      case 'update': {\n        const {relation, key, new: row} = change;\n        const backfill = this.#backfillRunningFor(relation);\n        const txWatermark = must(this.#currentTxWatermark, `not in a tx`);\n        if (backfill?.request.table.metadata && key !== null) {\n          // A corner case that backfill is unable to correctly handle is\n          // when a row's key changes; this is decomposed into a delete\n          // of the old key and a set of the new key in the replica change\n          // log, at which point the backfill algorithm assumes that the\n          // (old) row is deleted but does not know to backfill the new row.\n          // In these corner cases, the current backfill is canceled and\n          // retried if its version precedes this update.\n          for (const col of Object.keys(\n            backfill.request.table.metadata.rowKey,\n          )) {\n            if (key[col] !== row[col]) {\n              backfill.minWatermark = txWatermark;\n              this.#lc.info?.(\n                `key for row as changed (col: ${col}). ` +\n                  `backfill data must not predate ${backfill.minWatermark}`,\n              );\n              break;\n            }\n          }\n        }\n        break;\n      }\n      case 'backfill-completed': {\n        const {relation, columns} = change;\n        const backfillRequest = this.#requiredBackfills.get(relation);\n        assert(\n          backfillRequest,\n          () => `No BackfillRequest completed backfill ${stringify(change)}`,\n        );\n        const remaining = Object.entries(backfillRequest.columns).filter(\n          ([col]) =>\n            !(columns.includes(col) || relation.rowKey.columns.includes(col)),\n        );\n        if (remaining.length === 0) {\n          this.#deleteRequiredBackfill(tag, relation);\n        } else {\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            columns: Object.fromEntries(remaining),\n          });\n        }\n        // Technically the backfill is already stopping, but this method\n        // cleans up the state that tracks it.\n        this.#stopRunningBackfill();\n        break;\n      }\n    }\n  }\n\n  cancel(): void {\n    this.#stopRunningBackfill(`change stream canceled`);\n    clearTimeout(this.#backfillRetryTimer);\n  }\n}\n\nabstract class BackfillStreamError extends Error {\n  constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n    super(\n      `Cannot backfill ${bf.table.schema}.${bf.table.name}` +\n        `[${Object.keys(bf.columns).join(',')}]: ${msg}`,\n      {cause},\n    );\n  }\n}\n\n/**\n * Background: The zero-cache supports replication of tables without a\n * PRIMARY KEY to facilitate the onboarding process. These rows can be\n * INSERT'ed, but postgres will rightfully prohibit UPDATEs and DELETEs\n * on such tables because the rows cannot be identified by a key. Supporting\n * this mode of replication allows the user to \"fix\" the setup by adding the\n * primary key, after which the table can be published downstream without\n * requiring a resync of the data.\n *\n * In terms of backfill, however, non-empty tables without a row key **cannot**\n * be backfilled, because backfill retries would result in writing duplicating\n * rows. (Empty tables, on the other hand, are fine because there is no data\n * to be deduped.)\n *\n * The MissingRowKeyError is used to signal that the table cannot be backfilled\n * in its current state. For simplicity, it is handled like runtime errors and\n * retried with backoff, with which it can eventually succeed if (1) a primary\n * key is added or (2) the table is emptied, e.g. via a TRUNCATE.\n */\nclass MissingRowKeyError extends BackfillStreamError {\n  readonly name = 'MissingRowKeyError';\n\n  constructor(bf: BackfillRequest, cause?: unknown) {\n    super(bf, `\"${bf.table.name}\" is missing a PRIMARY KEY`, cause);\n  }\n}\n\n/**\n * Error type for backfill stream implementations to throw indicating that\n * the backfill request failed due to a schema incompatibility error. This\n * type of error does not need exponential backoff, as the retry happens\n * naturally once the invalidating schema change is processed and committed.\n */\nexport class SchemaIncompatibilityError extends BackfillStreamError {\n  readonly name = 'SchemaIncompatibilityError';\n\n  constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n    super(bf, msg, cause);\n  }\n}\n"],"mappings":";;;;;;;;;AAyBA,SAAS,SAAS,EAAC,QAAQ,QAAmB;AAC5C,QAAO,GAAG,OAAO,GAAG;;AAatB,IAAM,0BAA0B;AAChC,IAAM,0BAA0B;;;;;;;;;;;;;;;;;AAuBhC,IAAa,kBAAb,MAA6D;CAC3D;;;;;CAMA,qBAA8B,IAAI,aAChC,SACD;CACD;CACA;CACA;;;;;;;CAQA,mBAAgD;;CAGhD,uBAAsC;CAEtC,4BAAgE,EAAE;;CAGlE,sBAAqC;CAErC,YACE,IACA,gBACA,kBACA,aAAA,KACA,eAAe,yBACf,eAAe,yBACf;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,mBAAmB;AAC1D,QAAA,iBAAuB;AACvB,QAAA,mBAAyB;AACzB,QAAA,aAAmB;AACnB,QAAA,eAAqB;AACrB,QAAA,eAAqB;AACrB,QAAA,eAAqB;;CAGvB,IAAI,eAAuB,iBAAoC;AAC7D,QAAA,GAAS,OACP,kCAAkC,gBAAgB,OAAO,oBACzD,EAAC,UAAU,iBAAgB,CAC5B;AACD,QAAA,sBAA4B;AAC5B,kBAAgB,SAAQ,QACtB,MAAA,oBAA0B,mBAAmB,IAAI,CAClD;AACD,QAAA,uBAA6B;;CAG/B,wBAAwB,EAAC,aAAiC;AAIxD,OAAK,MAAA,uBAA6B,MAAM,WAAW;AACjD,SAAA,sBAA4B;AAC5B,QAAK,IAAI,IAAI,MAAA,yBAA+B,SAAS,GAAG,KAAK,GAAG,KAAK;IACnE,MAAM,WAAW,MAAA,yBAA+B;AAChD,QAAI,aAAa,SAAS,WAAW;AACnC,cAAS,SAAS;AAClB,WAAA,yBAA+B,OAAO,GAAG,EAAE;;;;;CAMnD,qBACE,IACA,WACsB;AACtB,OAAK,MAAA,uBAA6B,MAAM,WAAW;GACjD,MAAM,EAAC,SAAS,SAAS,YAAW,UAAU;AAC9C,SAAA,yBAA+B,KAAK;IAAC;IAAW;IAAQ,CAAC;AACzD,MAAG,OACD,iCAAiC,MAAA,oBAA0B,aAAa,YACzE;AACD,UAAO;;AAET,SAAO;;CAGT;CACA;CACA;CACA;CAEA,yBAAyB;AACvB,MACE,CAAC,MAAA,sBACD,CAAC,MAAA,mBACD,MAAA,kBAAwB,MACxB;GAKA,MAAM,aAAa,CAAC,GAAG,MAAA,kBAAwB,QAAQ,CAAC;GACxD,MAAM,UAAU,WAAW,QAAQ,GAAG,WAAW,SAAS,EAAE;GAC5D,MAAM,QAAQ;IAAC;IAAS,cAAc;IAAG;GACzC,MAAM,KAAK,MAAA,GAAS,YAAY,SAAS,QAAQ,MAAM,KAAK;AAE5D,SAAA,kBAAwB;AACnB,SAAA,YAAkB,IAAI,MAAM,CAC9B,WAAW;AACV,UAAA,oBAA0B,mBAAmB,MAAM;AACnD,UAAA,eAAqB,MAAA;KACrB,CAGD,OAAM,MAAK;AACV,UAAA,oBAA0B,OAAO,EAAE,EAAE,MAAM;AAC3C,UAAA,yBAA+B,EAAE;KACjC;;;CAIR,0BAA0B,GAAY;EACpC,MAAM,MAAM,MAAA,iBAAuB,MAAA,eAAqB,UAAU;AAClE,QAAA,GAAS,OACP,uCAAuC,MAAA,aAAmB,MAC1D,EACD;AACD,QAAA,qBAA2B,iBAAiB;AAC1C,SAAA,qBAA2B,KAAA;AAC3B,SAAA,uBAA6B;KAC5B,MAAA,aAAmB;AAEtB,QAAA,eAAqB,KAAK,IAAI,MAAA,eAAqB,GAAG,MAAA,aAAmB;;CAG3E,OAAA,YAAmB,IAAgB,OAA6B;EAC9D,MAAM,eAAe,MAAA;EAIrB,IAAI,aAA4B;;;;EAKhC,MAAM,aAAa,OACjB,QAC2B;AAC3B,UAAO,eAAe,MAAM,0CAA0C;GACtE,MAAM,gBAAgB,MAAM,aAAa,QAAQ,WAAW;AAI5D,OACE,MAAM,kBACL,IAAI,QAAQ,cAAc,IAAI,YAAY,MAAM,cACjD;AACA,QAAI,MAAM,mBAAmB,KAAA,GAAW;AACtC,YAAO,IAAI,QAAQ,YAAY,gCAAgC;AAC/D,WAAA,oBACE,qBAAqB,MAAM,aAAa,mCACH,IAAI,aACzC,MACD;;AAEH,iBAAa,QAAQ,cAAc;AACnC,WAAO;;GAGT,MAAM,EAAC,OAAO,QAAQ,OAAM,uBAAuB,cAAc;GACjE,IAAI,KAAK,qBAAqB;IAC5B;IACA,OAAO,OAAO,MAAM,GAAG;IACxB,CAAC;AAEF,OAAI,IAAI,QAAQ,wBAAwB,KAAK,IAAI,UAM/C,MAAK,IAAI;AAGN,gBAAa,KAAK;IACrB;IACA;KAAC,KAAK;KAAS,MAAM,MAAA;KAAkB,SAAS;KAAK;IACrD,EAAC,iBAAiB,IAAG;IACtB,CAAC;AACF,UAAQ,aAAa;;EAGvB,MAAM,iBAAiB;AACrB,OAAI,YAAY;AACT,iBAAa,KAAK;KACrB;KACA,EAAC,KAAK,UAAS;KACf,EAAC,WAAW,YAAW;KACxB,CAAC;AACF,iBAAa,QAAQ,WAAW;;AAElC,gBAAa;;AAGf,aAAW,MAAM,OAAO,MAAA,iBAAuB,MAAM,QAAQ,EAAE;GAI7D,MAAM,sBACJ,IAAI,QAAQ,yBACX,MAAA,oBAA0B,IAAI,IAAI,UAAU,IAC1C,eAAe,QAAQ,aAAa,IAAI;AAG7C,OACE,eACC,aAAa,aAAa,GAAG,KAAK,qBAEnC,WAAU;AAGZ,0BAAwB,MAAM;AAE9B,OACE,IAAI,QAAQ,cACZ,IAAI,UAAU,SAAS,KACvB,IAAI,SAAS,OAAO,QAAQ,WAAW,EAEvC,OAAM,IAAI,mBAAmB,MAAM,QAAQ;AAI7C,QAAK,eAAe,MAAM,WAAW,IAAI,MAAM,MAAM;AACnD,OAAG,OACD,6BAA6B,MAAM,kBACnC,MAAM,QACP;AACD,UAAA,uBAA6B;AAC7B;;AAKF,SAAM,aAAa,KAAK,CAAC,QAAQ,IAAI,CAAC;;AAIxC,gBAAc,UAAU;AACxB,KAAG,QAAQ,0BAA0B,MAAM,kBAAkB,GAAG;;CAGlE,oBAAoB,OAAgD;EAClE,MAAM,QAAQ,MAAA;AACd,SAAO,OAAO,QAAQ,MAAM,WAAW,MAAM,UAC3C,MAAM,QAAQ,MAAM,SAAS,MAAM,OACjC,QACA;;;;;;;;CASN,qBAAqB,QAAiB,UAAiC;EACrE,MAAM,WAAW,MAAA;AACjB,MAAI,YAAY,cAAc,YAAY,WAAW;AACnD,YAAS,iBAAiB;AAC1B,SAAA,kBAAwB;AACxB,aAAU,MAAA,GAAS,OAAO,uBAAuB,OAAO;;;CAI5D,qBAAqB,QAAgB,KAAsB;EACzD,MAAM,SAAS,MAAA,kBAAwB,IAAI,IAAI,MAAM,GAAG,YAAY;AACpE,QAAA,GAAS,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,KAAI,CAAC;AACjE,QAAA,kBAAwB,IAAI,IAAI,OAAO,IAAI;;CAG7C,wBAAwB,QAAgB,IAAgB;EACtD,MAAM,MAAM,MAAA,kBAAwB,IAAI,GAAG;AAC3C,MAAI,KAAK;GACP,MAAM,SAAS,WAAW,uBAAuB,cAAc;AAC/D,SAAA,GAAS,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,KAAI,CAAC;AACjE,SAAA,kBAAwB,OAAO,GAAG;;;;;;;CAQtC,SAAS,SAAoC;AAC3C,MAAI,QAAQ,OAAO,SAAS;AAC1B,SAAA,qBAA2B,QAAQ,GAAG;AACtC;;AAEF,MAAI,QAAQ,OAAO,UAAU;AAC3B,SAAA,qBAA2B;AAC3B,SAAA,uBAA6B,QAAQ,GAAG;AAGxC,SAAA,uBAA6B;AAC7B;;AAEF,MAAI,QAAQ,OAAO,UAAU;AAC3B,SAAA,uBAA6B,QAAQ,GAAG;AACxC;;AAEF,MAAI,QAAQ,OAAO,OACjB;EAEF,MAAM,SAAS,QAAQ;EACvB,MAAM,EAAC,QAAO;AACd,UAAQ,KAAR;GACE,KAAK,yBAAyB;IAC5B,MAAM,EAAC,OAAO,KAAK,aAAY;IAC/B,MAAM,kBAAkB,MAAA,kBAAwB,IAAI,MAAM;AAC1D,QAAI,iBAAiB;AACnB,WAAA,oBAA0B,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;OAAS;MAC5C,CAAC;AACF,SAAI,MAAA,mBAAyB,MAAM,CACjC,OAAA,oBAA0B,wBAAwB;;AAGtD;;GAEF,KAAK,gBAAgB;IACnB,MAAM,EACJ,MAAM,EAAC,QAAQ,QACf,WAAW,MACX,aACE;AAEJ,QAAI,SACF,OAAA,oBAA0B,KAAK;KAC7B,OAAO;MAAC;MAAQ;MAAM;MAAS;KAC/B,SAAS;KACV,CAAC;AAEJ;;GAEF,KAAK,gBAAgB;IACnB,MAAM,EAAC,KAAK,KAAK,aAAY;IAC7B,MAAM,kBAAkB,MAAA,kBAAwB,IAAI,IAAI;AACxD,QAAI,iBAAiB;KACnB,MAAM,EAAC,QAAQ,SAAQ;AACvB,WAAA,uBAA6B,KAAK,IAAI;AACtC,WAAA,oBAA0B,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;OAAQ;OAAK;MAChD,CAAC;AACF,SAAI,MAAA,mBAAyB,IAAI,CAC/B,OAAA,oBAA0B,gBAAgB;;AAG9C;;GAEF,KAAK,cAAc;IACjB,MAAM,EAAC,OAAM;AAEb,QADwB,MAAA,kBAAwB,IAAI,GAAG,EAClC;AACnB,WAAA,uBAA6B,KAAK,GAAG;AACrC,SAAI,MAAA,mBAAyB,GAAG,CAC9B,OAAA,oBAA0B,gBAAgB;;AAG9C;;GAEF,KAAK,cAAc;IACjB,MAAM,EACJ,OACA,eAAe,WAAW,MAC1B,QACA,aACE;AACJ,QAAI,UAAU;KACZ,MAAM,kBAAkB,MAAA,kBAAwB,IAAI,MAAM;AAC1D,SAAI,CAAC,gBACH,OAAA,oBAA0B,KAAK;MAC7B,OAAO;OAAC,GAAG;OAAO;OAAS;MAC3B,SAAS,GAAE,OAAO,OAAO,UAAS;MACnC,CAAC;SAEF,OAAA,oBAA0B,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;OAAS;MAC3C,SAAS;OACP,GAAG,gBAAgB;QAClB,OAAO,OAAO;OAChB;MACF,CAAC;;AAMN;;GAEF,KAAK,iBAAiB;IACpB,MAAM,EACJ,OACA,KAAK,EAAC,MAAM,WACZ,KAAK,EAAC,MAAM,cACV;AACJ,QAAI,YAAY,SAAS;KACvB,MAAM,kBAAkB,MAAA,kBAAwB,IAAI,MAAM;AAC1D,SAAI,mBAAmB,WAAW,gBAAgB,SAAS;MACzD,MAAM,GAAE,UAAU,SAAS,GAAG,cAAa,gBAAgB;AAC3D,YAAA,oBAA0B,KAAK;OAC7B,GAAG;OACH,SAAS;QAAC,GAAG;SAAY,UAAU;QAAQ;OAC5C,CAAC;MACF,MAAM,WAAW,MAAA,mBAAyB,MAAM;AAChD,UAAI,YAAY,WAAW,SAAS,QAAQ,QAC1C,OAAA,oBAA0B,iBAAiB;;;AAIjD;;GAEF,KAAK,eAAe;IAClB,MAAM,EAAC,OAAO,WAAU;IACxB,MAAM,kBAAkB,MAAA,kBAAwB,IAAI,MAAM;AAC1D,QAAI,mBAAmB,UAAU,gBAAgB,SAAS;KACxD,MAAM,GAAE,SAAS,WAAW,GAAG,cAAa,gBAAgB;AAC5D,SAAI,OAAO,KAAK,UAAU,CAAC,WAAW,EACpC,OAAA,uBAA6B,KAAK,MAAM;SAExC,OAAA,oBAA0B,KAAK;MAC7B,GAAG;MACH,SAAS;MACV,CAAC;KAEJ,MAAM,WAAW,MAAA,mBAAyB,MAAM;AAChD,SAAI,YAAY,UAAU,SAAS,QAAQ,QACzC,OAAA,oBAA0B,iBAAiB;;AAG/C;;GAEF,KAAK,UAAU;IACb,MAAM,EAAC,UAAU,KAAK,KAAK,QAAO;IAClC,MAAM,WAAW,MAAA,mBAAyB,SAAS;IACnD,MAAM,cAAc,KAAK,MAAA,oBAA0B,cAAc;AACjE,QAAI,UAAU,QAAQ,MAAM,YAAY,QAAQ;UAQzC,MAAM,OAAO,OAAO,KACvB,SAAS,QAAQ,MAAM,SAAS,OACjC,CACC,KAAI,IAAI,SAAS,IAAI,MAAM;AACzB,eAAS,eAAe;AACxB,YAAA,GAAS,OACP,gCAAgC,IAAI,oCACA,SAAS,eAC9C;AACD;;;AAIN;;GAEF,KAAK,sBAAsB;IACzB,MAAM,EAAC,UAAU,YAAW;IAC5B,MAAM,kBAAkB,MAAA,kBAAwB,IAAI,SAAS;AAC7D,WACE,uBACM,yCAAyC,UAAU,OAAO,GACjE;IACD,MAAM,YAAY,OAAO,QAAQ,gBAAgB,QAAQ,CAAC,QACvD,CAAC,SACA,EAAE,QAAQ,SAAS,IAAI,IAAI,SAAS,OAAO,QAAQ,SAAS,IAAI,EACnE;AACD,QAAI,UAAU,WAAW,EACvB,OAAA,uBAA6B,KAAK,SAAS;QAE3C,OAAA,oBAA0B,KAAK;KAC7B,GAAG;KACH,SAAS,OAAO,YAAY,UAAU;KACvC,CAAC;AAIJ,UAAA,qBAA2B;AAC3B;;;;CAKN,SAAe;AACb,QAAA,oBAA0B,yBAAyB;AACnD,eAAa,MAAA,mBAAyB;;;AAI1C,IAAe,sBAAf,cAA2C,MAAM;CAC/C,YAAY,IAAqB,KAAa,OAAiB;AAC7D,QACE,mBAAmB,GAAG,MAAM,OAAO,GAAG,GAAG,MAAM,KAAA,GACzC,OAAO,KAAK,GAAG,QAAQ,CAAC,KAAK,IAAI,CAAC,KAAK,OAC7C,EAAC,OAAM,CACR;;;;;;;;;;;;;;;;;;;;;;AAuBL,IAAM,qBAAN,cAAiC,oBAAoB;CACnD,OAAgB;CAEhB,YAAY,IAAqB,OAAiB;AAChD,QAAM,IAAI,IAAI,GAAG,MAAM,KAAK,6BAA6B,MAAM;;;;;;;;;AAUnE,IAAa,6BAAb,cAAgD,oBAAoB;CAClE,OAAgB;CAEhB,YAAY,IAAqB,KAAa,OAAiB;AAC7D,QAAM,IAAI,KAAK,MAAM"}

package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"change-stream-multiplexer.js","names":["#lc","#sub","#producers","#listeners","#waiters","#lastWatermark"],"sources":["../../../../../../../zero-cache/src/services/change-source/common/change-stream-multiplexer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport type {Source} from '../../../types/streams.ts';\nimport {Subscription} from '../../../types/subscription.ts';\nimport type {\n  ChangeStreamMessage,\n  DownstreamStatusMessage,\n} from '../protocol/current.ts';\n\nexport type Cancelable = {\n  cancel(): void;\n};\n\nexport type Listener = {\n  onChange(change: ChangeStreamMessage): void;\n};\n\ntype Waiter = {\n  producer: string;\n  startTime: number;\n  grantReservation: (watermark: string) => void;\n};\n\n/**\n * Facilitates cooperative multiplexing of transactions from\n * multiple producers into a single change stream.\n */\nexport class ChangeStreamMultiplexer {\n  readonly #lc: LogContext;\n  readonly #sub: Subscription<ChangeStreamMessage>;\n  readonly #producers: Cancelable[] = [];\n  readonly #listeners: Listener[] = [];\n\n  /**\n   * The `#lastWatermark` tracks the watermark of the last transaction\n   * committed to stream. It is set to null when a producer has reserved\n   * the stream, and set to the new watermark when the producer releases\n   * the reservation.\n   */\n  #lastWatermark: string | null;\n\n  /**\n   * Tracks the queue of producers waiting for a reservation.\n   */\n  readonly #waiters: Waiter[] = [];\n\n  constructor(lc: LogContext, lastWatermark: string) {\n    this.#lc = lc;\n    this.#sub = Subscription.create<ChangeStreamMessage>({\n      cleanup: () => this.#producers.forEach(p => p.cancel()),\n    });\n    this.#lastWatermark = lastWatermark;\n  }\n\n  addProducers(...p: Cancelable[]): this {\n    this.#producers.push(...p);\n    return this;\n  }\n\n  addListeners(...l: Listener[]): this {\n    this.#listeners.push(...l);\n    return this;\n  }\n\n  /**\n   * Called by a producer to \"reserve\" the exclusive right to push data\n   * changes to the stream. If the stream is currently reserved,\n   * the resulting Promise must be awaited.\n   *\n   * The producer must then {@link release()} the reservation when it sees\n   * fit.\n   *\n   * @param producer The name of the producer, purely for debugging output\n   */\n  reserve(producer: string): string | Promise<string> {\n    if (this.#lastWatermark !== null) {\n      // If the stream is not reserved, reserve it and return the\n      // watermark.\n      const lastWatermark = this.#lastWatermark;\n      this.#lastWatermark = null;\n      return lastWatermark;\n    }\n\n    // Otherwise, wait for the current reservation to be released.\n    const startTime = performance.now();\n    const {promise, resolve: grantReservation} = resolver<string>();\n    this.#waiters.push({producer, startTime, grantReservation});\n\n    return promise;\n  }\n\n  /**\n   * If there are producers currently awaiting a reservation, returns\n   * the duration (in milliseconds) of the oldest reservation request.\n   * Returns a negative number if there are no waiters.\n   */\n  waiterDelay(): number {\n    if (this.#waiters.length === 0) {\n      return -1;\n    }\n    return performance.now() - this.#waiters[0].startTime;\n  }\n\n  /**\n   * Called by a producer to release its reservation after committing its\n   * last transaction.\n   */\n  release(newWatermark: string) {\n    const waiter = this.#waiters.shift();\n    if (!waiter) {\n      this.#lastWatermark = newWatermark;\n    } else {\n      const {producer, startTime, grantReservation} = waiter;\n      grantReservation(newWatermark);\n      const elapsed = performance.now() - startTime;\n      this.#lc.info?.(\n        `${producer} waited ${elapsed.toFixed(3)} ms for stream reservation`,\n      );\n    }\n  }\n\n  /**\n   * `pushStatus()` can be called without a reservation, as it\n   * does not constitute a data change and can appear anywhere\n   * in the stream.\n   */\n  pushStatus(message: DownstreamStatusMessage) {\n    // Let listeners know about all status messages.\n    this.#listeners.forEach(l => l.onChange(message));\n    // The ChangeStreamer only cares about status messages requiring an ack\n    // or containing a lagReport. To reduce churn, avoid sending other status\n    // messages.\n    if (message[1].ack || message[1].lagReport) {\n      this.#sub.push(message);\n    }\n  }\n\n  /**\n   * `push()` must only be called by a producer after it has\n   * {@link reserve}d the stream.\n   */\n  push(message: ChangeStreamMessage): Promise<unknown> {\n    assert(\n      this.#lastWatermark === null,\n      `push() called without reserve()-ing the stream`,\n    );\n    this.#listeners.forEach(l => l.onChange(message));\n    return this.#sub.push(message).result;\n  }\n\n  fail(err: Error) {\n    this.#sub.fail(err);\n  }\n\n  asSource(): Source<ChangeStreamMessage> {\n    return this.#sub;\n  }\n}\n"],"mappings":";;;;;;;;AA4BA,IAAa,0BAAb,MAAqC;CACnC;CACA;CACA,aAAoC,EAAE;CACtC,aAAkC,EAAE;;;;;;;CAQpC;;;;CAKA,WAA8B,EAAE;CAEhC,YAAY,IAAgB,eAAuB;AACjD,QAAA,KAAW;AACX,QAAA,MAAY,aAAa,OAA4B,EACnD,eAAe,MAAA,UAAgB,SAAQ,MAAK,EAAE,QAAQ,CAAC,EACxD,CAAC;AACF,QAAA,gBAAsB;;CAGxB,aAAa,GAAG,GAAuB;AACrC,QAAA,UAAgB,KAAK,GAAG,EAAE;AAC1B,SAAO;;CAGT,aAAa,GAAG,GAAqB;AACnC,QAAA,UAAgB,KAAK,GAAG,EAAE;AAC1B,SAAO;;;;;;;;;;;;CAaT,QAAQ,UAA4C;AAClD,MAAI,MAAA,kBAAwB,MAAM;GAGhC,MAAM,gBAAgB,MAAA;AACtB,SAAA,gBAAsB;AACtB,UAAO;;EAIT,MAAM,YAAY,YAAY,KAAK;EACnC,MAAM,EAAC,SAAS,SAAS,qBAAoB,UAAkB;AAC/D,QAAA,QAAc,KAAK;GAAC;GAAU;GAAW;GAAiB,CAAC;AAE3D,SAAO;;;;;;;CAQT,cAAsB;AACpB,MAAI,MAAA,QAAc,WAAW,EAC3B,QAAO;AAET,SAAO,YAAY,KAAK,GAAG,MAAA,QAAc,GAAG;;;;;;CAO9C,QAAQ,cAAsB;EAC5B,MAAM,SAAS,MAAA,QAAc,OAAO;AACpC,MAAI,CAAC,OACH,OAAA,gBAAsB;OACjB;GACL,MAAM,EAAC,UAAU,WAAW,qBAAoB;AAChD,oBAAiB,aAAa;GAC9B,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,SAAA,GAAS,OACP,GAAG,SAAS,UAAU,QAAQ,QAAQ,EAAE,CAAC,4BAC1C;;;;;;;;CASL,WAAW,SAAkC;AAE3C,QAAA,UAAgB,SAAQ,MAAK,EAAE,SAAS,QAAQ,CAAC;AAIjD,MAAI,QAAQ,GAAG,OAAO,QAAQ,GAAG,UAC/B,OAAA,IAAU,KAAK,QAAQ;;;;;;CAQ3B,KAAK,SAAgD;AACnD,SACE,MAAA,kBAAwB,MACxB,iDACD;AACD,QAAA,UAAgB,SAAQ,MAAK,EAAE,SAAS,QAAQ,CAAC;AACjD,SAAO,MAAA,IAAU,KAAK,QAAQ,CAAC;;CAGjC,KAAK,KAAY;AACf,QAAA,IAAU,KAAK,IAAI;;CAGrB,WAAwC;AACtC,SAAO,MAAA"}

package/out/zero-cache/src/services/change-source/common/replica-schema.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"replica-schema.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-schema.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {SqliteError} from '@rocicorp/zero-sqlite3';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport {listTables} from '../../../db/lite-tables.ts';\nimport {\n  runSchemaMigrations,\n  type IncrementalMigrationMap,\n  type Migration,\n} from '../../../db/migration-lite.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {populateFromExistingTables} from '../../replicator/schema/column-metadata.ts';\nimport {\n  CREATE_RUNTIME_EVENTS_TABLE,\n  recordEvent,\n} from '../../replicator/schema/replication-state.ts';\n\nexport async function initReplica(\n  log: LogContext,\n  debugName: string,\n  dbPath: string,\n  initialSync: (lc: LogContext, tx: Database) => Promise<void>,\n): Promise<void> {\n  const setupMigration: Migration = {\n    migrateSchema: (log, tx) => initialSync(log, tx),\n    minSafeVersion: 1,\n  };\n\n  try {\n    await runSchemaMigrations(\n      log,\n      debugName,\n      dbPath,\n      setupMigration,\n      schemaVersionMigrationMap,\n    );\n  } catch (e) {\n    if (e instanceof SqliteError && e.code === 'SQLITE_CORRUPT') {\n      throw new AutoResetSignal(e.message);\n    }\n    throw e;\n  }\n}\n\nexport async function upgradeReplica(\n  log: LogContext,\n  debugName: string,\n  dbPath: string,\n) {\n  await runSchemaMigrations(\n    log,\n    debugName,\n    dbPath,\n    // setupMigration should never be invoked\n    {\n      migrateSchema: () => {\n        throw new Error(\n          'This should only be called for already synced replicas',\n        );\n      },\n    },\n    schemaVersionMigrationMap,\n  );\n}\n\nexport const CREATE_V6_COLUMN_METADATA_TABLE = /*sql*/ `\n  CREATE TABLE \"_zero.column_metadata\" (\n    table_name TEXT NOT NULL,\n    column_name TEXT NOT NULL,\n    upstream_type TEXT NOT NULL,\n    is_not_null INTEGER NOT NULL,\n    is_enum INTEGER NOT NULL,\n    is_array INTEGER NOT NULL,\n    character_max_length INTEGER,\n    PRIMARY KEY (table_name, column_name)\n  );\n`;\n\nexport const CREATE_V7_CHANGE_LOG = /*sql*/ `\n  CREATE TABLE \"_zero.changeLog2\" (\n    \"stateVersion\"              TEXT NOT NULL,\n    \"pos\"                       INT  NOT NULL,\n    \"table\"                     TEXT NOT NULL,\n    \"rowKey\"                    TEXT NOT NULL,\n    \"op\"                        TEXT NOT NULL,\n    PRIMARY KEY(\"stateVersion\", \"pos\"),\n    UNIQUE(\"table\", \"rowKey\")\n  );\n`;\n\nexport const CREATE_V9_TABLE_METADATA_TABLE = /*sql*/ `\n  CREATE TABLE \"_zero.tableMetadata\" (\n    \"schema\"    TEXT NOT NULL,\n    \"table\"     TEXT NOT NULL,\n    \"metadata\"  TEXT NOT NULL,\n    PRIMARY KEY (\"schema\", \"table\")\n  );\n`;\n\nexport const schemaVersionMigrationMap: IncrementalMigrationMap = {\n  // There's no incremental migration from v1. Just reset the replica.\n  4: {\n    migrateSchema: () => {\n      throw new AutoResetSignal('upgrading replica to new schema');\n    },\n    minSafeVersion: 3,\n  },\n\n  5: {\n    migrateSchema: (_, db) => {\n      db.exec(CREATE_RUNTIME_EVENTS_TABLE);\n    },\n    migrateData: (_, db) => {\n      recordEvent(db, 'upgrade');\n    },\n  },\n\n  // Revised in the migration to v8 because the v6 code was incomplete.\n  6: {},\n\n  7: {\n    migrateSchema: (_, db) => {\n      // Note: The original \"changeLog\" table is kept so that the replica file\n      // is compatible with older zero-caches. However, it is truncated for\n      // space savings (since historic changes were never read).\n      db.exec(`DELETE FROM \"_zero.changeLog\"`);\n      // First version of changeLog2\n      db.exec(CREATE_V7_CHANGE_LOG);\n    },\n  },\n\n  8: {\n    migrateSchema: (_, db) => {\n      const tableExists = db\n        .prepare(\n          `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_zero.column_metadata'`,\n        )\n        .get();\n\n      if (!tableExists) {\n        db.exec(CREATE_V6_COLUMN_METADATA_TABLE);\n      }\n    },\n    migrateData: (_, db) => {\n      // Re-populate the ColumnMetadataStore; the original migration\n      // at v6 was incomplete, as covered replicas migrated from earlier\n      // versions but did not initialize the table for new replicas.\n      db.exec(/*sql*/ `DELETE FROM \"_zero.column_metadata\"`);\n\n      const tables = listTables(db, false, false);\n      populateFromExistingTables(db, tables);\n    },\n  },\n\n  9: {\n    migrateSchema: (_, db) => {\n      db.exec(\n        /*sql*/ `\n        ALTER TABLE \"_zero.changeLog2\" \n          ADD COLUMN \"backfillingColumnVersions\" TEXT DEFAULT '{}';\n        ALTER TABLE \"_zero.column_metadata\"\n          ADD COLUMN backfill TEXT;\n      ` + CREATE_V9_TABLE_METADATA_TABLE,\n      );\n    },\n  },\n\n  10: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.replicationConfig\" \n          ADD COLUMN \"initialSyncContext\" TEXT DEFAULT '{}';\n      `);\n    },\n  },\n\n  11: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"minRowVersion\" TEXT NOT NULL DEFAULT '00';\n\n        -- Removing the NOT NULL constraint from \"metadata\" requires copying\n        -- the column. We piggyback the rename to \"upstreamMetadata\" here.\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"upstreamMetadata\" TEXT;\n        UPDATE \"_zero.tableMetadata\" SET \"upstreamMetadata\" = \"metadata\";\n        ALTER TABLE \"_zero.tableMetadata\" DROP \"metadata\";\n      `);\n    },\n  },\n\n  12: {\n    migrateSchema: (_, db) => {\n      // Bring back the \"metadata\" column removed in v11, but as a NULL-able column.\n      // It is needed for backwards compatibility.\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"metadata\" TEXT;\n      `);\n    },\n\n    migrateData: (_, db) => {\n      // For rollback then roll forward, re-copy anything written to metadata.\n      db.exec(/*sql*/ `\n        UPDATE \"_zero.tableMetadata\" \n          SET \"upstreamMetadata\" = COALESCE(\"metadata\", \"upstreamMetadata\"),\n              \"metadata\" = NULL;\n      `);\n    },\n  },\n\n  13: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.replicationState\" ADD COLUMN writeTimeMs INTEGER;\n      `);\n    },\n\n    migrateData: (_, db) => {\n      db.exec(/*sql*/ `\n        UPDATE \"_zero.replicationState\" \n          SET writeTimeMs = COALESCE(writeTimeMs, unixepoch('subsec') * 1000)`);\n    },\n  },\n};\n\n// Referenced in tests.\nexport const CURRENT_SCHEMA_VERSION = Object.keys(\n  schemaVersionMigrationMap,\n).reduce((prev, curr) => Math.max(prev, parseInt(curr)), 0);\n"],"mappings":";;;;;;;AAgBA,eAAsB,YACpB,KACA,WACA,QACA,aACe;CACf,MAAM,iBAA4B;EAChC,gBAAgB,KAAK,OAAO,YAAY,KAAK,GAAG;EAChD,gBAAgB;EACjB;AAED,KAAI;AACF,QAAM,oBACJ,KACA,WACA,QACA,gBACA,0BACD;UACM,GAAG;AACV,MAAI,aAAa,eAAe,EAAE,SAAS,iBACzC,OAAM,IAAI,gBAAgB,EAAE,QAAQ;AAEtC,QAAM;;;AAIV,eAAsB,eACpB,KACA,WACA,QACA;AACA,OAAM,oBACJ,KACA,WACA,QAEA,EACE,qBAAqB;AACnB,QAAM,IAAI,MACR,yDACD;IAEJ,EACD,0BACD;;AAGH,IAAa,kCAA0C;;;;;;;;;;;;AAavD,IAAa,uBAA+B;;;;;;;;;;;AAY5C,IAAa,iCAAyC;;;;;;;;AAStD,IAAa,4BAAqD;CAEhE,GAAG;EACD,qBAAqB;AACnB,SAAM,IAAI,gBAAgB,kCAAkC;;EAE9D,gBAAgB;EACjB;CAED,GAAG;EACD,gBAAgB,GAAG,OAAO;AACxB,MAAG,KAAK,4BAA4B;;EAEtC,cAAc,GAAG,OAAO;AACtB,eAAY,IAAI,UAAU;;EAE7B;CAGD,GAAG,EAAE;CAEL,GAAG,EACD,gBAAgB,GAAG,OAAO;AAIxB,KAAG,KAAK,gCAAgC;AAExC,KAAG,KAAK,qBAAqB;IAEhC;CAED,GAAG;EACD,gBAAgB,GAAG,OAAO;AAOxB,OAAI,CANgB,GACjB,QACC,sFACD,CACA,KAAK,CAGN,IAAG,KAAK,gCAAgC;;EAG5C,cAAc,GAAG,OAAO;AAItB,MAAG,KAAa,sCAAsC;AAGtD,8BAA2B,IADZ,WAAW,IAAI,OAAO,MAAM,CACL;;EAEzC;CAED,GAAG,EACD,gBAAgB,GAAG,OAAO;AACxB,KAAG,KACO;;;;;UAKN,+BACH;IAEJ;CAED,IAAI,EACF,gBAAgB,GAAG,OAAO;AACxB,KAAG,KAAa;;;QAGd;IAEL;CAED,IAAI,EACF,gBAAgB,GAAG,OAAO;AACxB,KAAG,KAAa;;;;;;;;;;QAUd;IAEL;CAED,IAAI;EACF,gBAAgB,GAAG,OAAO;AAGxB,MAAG,KAAa;;;QAGd;;EAGJ,cAAc,GAAG,OAAO;AAEtB,MAAG,KAAa;;;;QAId;;EAEL;CAED,IAAI;EACF,gBAAgB,GAAG,OAAO;AACxB,MAAG,KAAa;;QAEd;;EAGJ,cAAc,GAAG,OAAO;AACtB,MAAG,KAAa;;+EAEyD;;EAE5E;CACF;AAGqC,OAAO,KAC3C,0BACD,CAAC,QAAQ,MAAM,SAAS,KAAK,IAAI,MAAM,SAAS,KAAK,CAAC,EAAE,EAAE"}

package/out/zero-cache/src/services/change-source/custom/change-source.js

@@ -1,176 +0,0 @@
-import { assert, unreachable } from "../../../../../shared/src/asserts.js";
-import { deepEqual } from "../../../../../shared/src/json.js";
-import { stringify } from "../../../../../shared/src/bigint-json.js";
-import { changeStreamMessageSchema } from "../protocol/current/downstream.js";
-import "../protocol/current/upstream.js";
-import { Database } from "../../../../../zqlite/src/db.js";
-import { computeZqlSpecs } from "../../../db/lite-tables.js";
-import { createReplicationStateTables, getSubscriptionState, initReplicationState } from "../../replicator/schema/replication-state.js";
-import { StatementRunner } from "../../../db/statements.js";
-import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
-import { initReplica } from "../common/replica-schema.js";
-import { stream } from "../../../types/streams.js";
-import { ChangeProcessor } from "../../replicator/change-processor.js";
-import { ReplicationStatusPublisher } from "../../replicator/replication-status.js";
-import { WebSocket as WebSocket$1 } from "ws";
-//#region ../zero-cache/src/services/change-source/custom/change-source.ts
-/**
-* Initializes a Custom change source before streaming changes from the
-* corresponding logical replication stream.
-*/
-async function initializeCustomChangeSource(lc, upstreamURI, shard, replicaDbFile, context) {
-	await initReplica(lc, `replica-${shard.appID}-${shard.shardNum}`, replicaDbFile, (log, tx) => initialSync(log, shard, tx, upstreamURI, context));
-	const replica = new Database(lc, replicaDbFile);
-	const subscriptionState = getSubscriptionState(new StatementRunner(replica));
-	replica.close();
-	if (shard.publications.length) {
-		const requested = shard.publications.toSorted();
-		const replicated = subscriptionState.publications.sort();
-		if (!deepEqual(requested, replicated)) throw new Error(`Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`);
-	}
-	return {
-		subscriptionState,
-		changeSource: new CustomChangeSource(lc, upstreamURI, shard, subscriptionState)
-	};
-}
-var CustomChangeSource = class {
-	#lc;
-	#upstreamUri;
-	#shard;
-	#replicationConfig;
-	constructor(lc, upstreamUri, shard, replicationConfig) {
-		this.#lc = lc.withContext("component", "change-source");
-		this.#upstreamUri = upstreamUri;
-		this.#shard = shard;
-		this.#replicationConfig = replicationConfig;
-	}
-	initialSync() {
-		return this.#startStream();
-	}
-	startLagReporter() {
-		return null;
-	}
-	stop() {
-		return Promise.resolve();
-	}
-	startStream(clientWatermark, backfillRequests = []) {
-		if (backfillRequests?.length) throw new Error("backfill is yet not supported for custom change sources");
-		return Promise.resolve(this.#startStream(clientWatermark));
-	}
-	#startStream(clientWatermark) {
-		const { publications, replicaVersion } = this.#replicationConfig;
-		const { appID, shardNum } = this.#shard;
-		const url = new URL(this.#upstreamUri);
-		url.searchParams.set("appID", appID);
-		url.searchParams.set("shardNum", String(shardNum));
-		for (const pub of publications) url.searchParams.append("publications", pub);
-		if (clientWatermark) {
-			assert(replicaVersion.length, "replicaVersion is required when clientWatermark is set");
-			url.searchParams.set("lastWatermark", clientWatermark);
-			url.searchParams.set("replicaVersion", replicaVersion);
-		}
-		const ws = new WebSocket$1(url);
-		const { instream, outstream } = stream(this.#lc, ws, changeStreamMessageSchema, { coalesce: (curr) => curr });
-		return {
-			changes: instream,
-			acks: outstream
-		};
-	}
-};
-/**
-* Initial sync for a custom change source makes a request to the
-* change source endpoint with no `replicaVersion` or `lastWatermark`.
-* The initial transaction returned by the endpoint is treated as
-* the initial sync, and the commit watermark of that transaction
-* becomes the `replicaVersion` of the initialized replica.
-*
-* Note that this is equivalent to how the LSN of the Postgres WAL
-* at initial sync time is the `replicaVersion` (and starting
-* version for all initially-synced rows).
-*/
-async function initialSync(lc, shard, tx, upstreamURI, context) {
-	const { appID: id, publications } = shard;
-	const { changes } = new CustomChangeSource(lc, upstreamURI, shard, {
-		replicaVersion: "",
-		publications
-	}).initialSync();
-	createReplicationStateTables(tx);
-	const processor = new ChangeProcessor(new StatementRunner(tx), "initial-sync", (_, err) => {
-		throw err;
-	});
-	const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(tx);
-	try {
-		let num = 0;
-		for await (const change of changes) {
-			const [tag] = change;
-			switch (tag) {
-				case "begin": {
-					const { commitWatermark } = change[2];
-					lc.info?.(`initial sync of shard ${id} at replicaVersion ${commitWatermark}`);
-					statusPublisher.publish(lc, "Initializing", `Copying upstream tables at version ${commitWatermark}`, 5e3);
-					initReplicationState(tx, publications.toSorted(), commitWatermark, context, false);
-					processor.processMessage(lc, change);
-					break;
-				}
-				case "data":
-					processor.processMessage(lc, change);
-					if (++num % 1e3 === 0) lc.debug?.(`processed ${num} changes`);
-					break;
-				case "commit":
-					processor.processMessage(lc, change);
-					validateInitiallySyncedData(lc, tx, shard);
-					lc.info?.(`finished initial-sync of ${num} changes`);
-					return;
-				case "status": break;
-				case "control": {
-					const { tag, message } = change[1];
-					if (tag === "reset-required") throw new AutoResetSignal(message ?? "auto-reset signaled by change source");
-				}
-				case "rollback": throw new Error(`unexpected message during initial-sync: ${stringify(change)}`);
-				default: unreachable(change);
-			}
-		}
-		throw new Error(`change source ${upstreamURI} closed before initial-sync completed`);
-	} catch (e) {
-		await statusPublisher.publishAndThrowError(lc, "Initializing", e);
-	} finally {
-		statusPublisher.stop();
-	}
-}
-function getRequiredTables({ appID, shardNum }) {
-	return {
-		[`${appID}_${shardNum}.clients`]: {
-			clientGroupID: { type: "string" },
-			clientID: { type: "string" },
-			lastMutationID: { type: "number" },
-			userID: { type: "string" }
-		},
-		[`${appID}_${shardNum}.mutations`]: {
-			clientGroupID: { type: "string" },
-			clientID: { type: "string" },
-			mutationID: { type: "number" },
-			mutation: { type: "json" }
-		},
-		[`${appID}.permissions`]: {
-			permissions: { type: "json" },
-			hash: { type: "string" }
-		}
-	};
-}
-function validateInitiallySyncedData(lc, db, shard) {
-	const tables = computeZqlSpecs(lc, db, { includeBackfillingColumns: true });
-	const required = getRequiredTables(shard);
-	for (const [name, columns] of Object.entries(required)) {
-		const table = tables.get(name)?.zqlSpec;
-		if (!table) throw new Error(`Upstream is missing the "${name}" table. (Found ${[...tables.keys()]})Please ensure that each table has a unique index over one or more non-null columns.`);
-		for (const [col, { type }] of Object.entries(columns)) {
-			const found = table[col];
-			if (!found) throw new Error(`Upstream "${table}" table is missing the "${col}" column`);
-			if (found.type !== type) throw new Error(`Upstream "${table}.${col}" column is a ${found.type} type but must be a ${type} type.`);
-		}
-	}
-}
-//#endregion
-export { initializeCustomChangeSource };
-
-//# sourceMappingURL=change-source.js.map

package/out/zero-cache/src/services/change-source/custom/change-source.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"change-source.js","names":["#lc","#upstreamUri","#shard","#replicationConfig","#startStream"],"sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {WebSocket} from 'ws';\nimport {assert, unreachable} from '../../../../../shared/src/asserts.ts';\nimport {\n  stringify,\n  type JSONObject,\n} from '../../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport type {SchemaValue} from '../../../../../zero-schema/src/table-schema.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs} from '../../../db/lite-tables.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport type {ShardConfig, ShardID} from '../../../types/shards.ts';\nimport {stream} from '../../../types/streams.ts';\nimport {\n  AutoResetSignal,\n  type ReplicationConfig,\n} from '../../change-streamer/schema/tables.ts';\nimport {ChangeProcessor} from '../../replicator/change-processor.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {\n  createReplicationStateTables,\n  getSubscriptionState,\n  initReplicationState,\n  type SubscriptionState,\n} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource, ChangeStream} from '../change-source.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {changeStreamMessageSchema} from '../protocol/current/downstream.ts';\nimport {\n  type BackfillRequest,\n  type ChangeSourceUpstream,\n} from '../protocol/current/upstream.ts';\n\n/** Server context to store with the initial sync metadata for debugging. */\nexport type ServerContext = JSONObject;\n\n/**\n * Initializes a Custom change source before streaming changes from the\n * corresponding logical replication stream.\n */\nexport async function initializeCustomChangeSource(\n  lc: LogContext,\n  upstreamURI: string,\n  shard: ShardConfig,\n  replicaDbFile: string,\n  context: ServerContext,\n): Promise<{subscriptionState: SubscriptionState; changeSource: ChangeSource}> {\n  await initReplica(\n    lc,\n    `replica-${shard.appID}-${shard.shardNum}`,\n    replicaDbFile,\n    (log, tx) => initialSync(log, shard, tx, upstreamURI, context),\n  );\n\n  const replica = new Database(lc, replicaDbFile);\n  const subscriptionState = getSubscriptionState(new StatementRunner(replica));\n  replica.close();\n\n  if (shard.publications.length) {\n    // Verify that the publications match what has been synced.\n    const requested = shard.publications.toSorted();\n    const replicated = subscriptionState.publications.sort();\n    if (!deepEqual(requested, replicated)) {\n      throw new Error(\n        `Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`,\n      );\n    }\n  }\n\n  const changeSource = new CustomChangeSource(\n    lc,\n    upstreamURI,\n    shard,\n    subscriptionState,\n  );\n\n  return {subscriptionState, changeSource};\n}\n\nclass CustomChangeSource implements ChangeSource {\n  readonly #lc: LogContext;\n  readonly #upstreamUri: string;\n  readonly #shard: ShardID;\n  readonly #replicationConfig: ReplicationConfig;\n\n  constructor(\n    lc: LogContext,\n    upstreamUri: string,\n    shard: ShardID,\n    replicationConfig: ReplicationConfig,\n  ) {\n    this.#lc = lc.withContext('component', 'change-source');\n    this.#upstreamUri = upstreamUri;\n    this.#shard = shard;\n    this.#replicationConfig = replicationConfig;\n  }\n\n  initialSync(): ChangeStream {\n    return this.#startStream();\n  }\n\n  startLagReporter() {\n    return null; // Not supported for custom sources\n  }\n\n  stop(): Promise<void> {\n    return Promise.resolve();\n  }\n\n  startStream(\n    clientWatermark: string,\n    backfillRequests: BackfillRequest[] = [],\n  ): Promise<ChangeStream> {\n    if (backfillRequests?.length) {\n      throw new Error(\n        'backfill is yet not supported for custom change sources',\n      );\n    }\n    return Promise.resolve(this.#startStream(clientWatermark));\n  }\n\n  #startStream(clientWatermark?: string): ChangeStream {\n    const {publications, replicaVersion} = this.#replicationConfig;\n    const {appID, shardNum} = this.#shard;\n    const url = new URL(this.#upstreamUri);\n    url.searchParams.set('appID', appID);\n    url.searchParams.set('shardNum', String(shardNum));\n    for (const pub of publications) {\n      url.searchParams.append('publications', pub);\n    }\n    if (clientWatermark) {\n      assert(\n        replicaVersion.length,\n        'replicaVersion is required when clientWatermark is set',\n      );\n      url.searchParams.set('lastWatermark', clientWatermark);\n      url.searchParams.set('replicaVersion', replicaVersion);\n    }\n\n    const ws = new WebSocket(url);\n    const {instream, outstream} = stream(\n      this.#lc,\n      ws,\n      changeStreamMessageSchema,\n      // Upstream acks coalesce. If upstream exhibits back-pressure,\n      // only the last ACK is kept / buffered.\n      {coalesce: (curr: ChangeSourceUpstream) => curr},\n    );\n    return {changes: instream, acks: outstream};\n  }\n}\n\n/**\n * Initial sync for a custom change source makes a request to the\n * change source endpoint with no `replicaVersion` or `lastWatermark`.\n * The initial transaction returned by the endpoint is treated as\n * the initial sync, and the commit watermark of that transaction\n * becomes the `replicaVersion` of the initialized replica.\n *\n * Note that this is equivalent to how the LSN of the Postgres WAL\n * at initial sync time is the `replicaVersion` (and starting\n * version for all initially-synced rows).\n */\nexport async function initialSync(\n  lc: LogContext,\n  shard: ShardConfig,\n  tx: Database,\n  upstreamURI: string,\n  context: ServerContext,\n) {\n  const {appID: id, publications} = shard;\n  const changeSource = new CustomChangeSource(lc, upstreamURI, shard, {\n    replicaVersion: '', // ignored for initialSync()\n    publications,\n  });\n  const {changes} = changeSource.initialSync();\n\n  createReplicationStateTables(tx);\n  const processor = new ChangeProcessor(\n    new StatementRunner(tx),\n    'initial-sync',\n    (_, err) => {\n      throw err;\n    },\n  );\n\n  const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(tx);\n  try {\n    let num = 0;\n    for await (const change of changes) {\n      const [tag] = change;\n      switch (tag) {\n        case 'begin': {\n          const {commitWatermark} = change[2];\n          lc.info?.(\n            `initial sync of shard ${id} at replicaVersion ${commitWatermark}`,\n          );\n          statusPublisher.publish(\n            lc,\n            'Initializing',\n            `Copying upstream tables at version ${commitWatermark}`,\n            5000,\n          );\n          initReplicationState(\n            tx,\n            publications.toSorted(),\n            commitWatermark,\n            context,\n            false,\n          );\n          processor.processMessage(lc, change);\n          break;\n        }\n        case 'data':\n          processor.processMessage(lc, change);\n          if (++num % 1000 === 0) {\n            lc.debug?.(`processed ${num} changes`);\n          }\n          break;\n        case 'commit':\n          processor.processMessage(lc, change);\n          validateInitiallySyncedData(lc, tx, shard);\n          lc.info?.(`finished initial-sync of ${num} changes`);\n          return;\n\n        case 'status':\n          break; // Ignored\n        // @ts-expect-error: falls through if the tag is not 'reset-required\n        case 'control': {\n          const {tag, message} = change[1];\n          if (tag === 'reset-required') {\n            throw new AutoResetSignal(\n              message ?? 'auto-reset signaled by change source',\n            );\n          }\n        }\n        // falls through\n        case 'rollback':\n          throw new Error(\n            `unexpected message during initial-sync: ${stringify(change)}`,\n          );\n        default:\n          unreachable(change);\n      }\n    }\n    throw new Error(\n      `change source ${upstreamURI} closed before initial-sync completed`,\n    );\n  } catch (e) {\n    await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n  } finally {\n    statusPublisher.stop();\n  }\n}\n\n// Verify that the upstream tables expected by the sync logic\n// have been properly initialized.\nfunction getRequiredTables({\n  appID,\n  shardNum,\n}: ShardID): Record<string, Record<string, SchemaValue>> {\n  return {\n    [`${appID}_${shardNum}.clients`]: {\n      clientGroupID: {type: 'string'},\n      clientID: {type: 'string'},\n      lastMutationID: {type: 'number'},\n      userID: {type: 'string'},\n    },\n    [`${appID}_${shardNum}.mutations`]: {\n      clientGroupID: {type: 'string'},\n      clientID: {type: 'string'},\n      mutationID: {type: 'number'},\n      mutation: {type: 'json'},\n    },\n    [`${appID}.permissions`]: {\n      permissions: {type: 'json'},\n      hash: {type: 'string'},\n    },\n  };\n}\n\nfunction validateInitiallySyncedData(\n  lc: LogContext,\n  db: Database,\n  shard: ShardID,\n) {\n  const tables = computeZqlSpecs(lc, db, {includeBackfillingColumns: true});\n  const required = getRequiredTables(shard);\n  for (const [name, columns] of Object.entries(required)) {\n    const table = tables.get(name)?.zqlSpec;\n    if (!table) {\n      throw new Error(\n        `Upstream is missing the \"${name}\" table. (Found ${[\n          ...tables.keys(),\n        ]})` +\n          `Please ensure that each table has a unique index over one ` +\n          `or more non-null columns.`,\n      );\n    }\n    for (const [col, {type}] of Object.entries(columns)) {\n      const found = table[col];\n      if (!found) {\n        throw new Error(\n          `Upstream \"${table}\" table is missing the \"${col}\" column`,\n        );\n      }\n      if (found.type !== type) {\n        throw new Error(\n          `Upstream \"${table}.${col}\" column is a ${found.type} type but must be a ${type} type.`,\n        );\n      }\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAyCA,eAAsB,6BACpB,IACA,aACA,OACA,eACA,SAC6E;AAC7E,OAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,gBACC,KAAK,OAAO,YAAY,KAAK,OAAO,IAAI,aAAa,QAAQ,CAC/D;CAED,MAAM,UAAU,IAAI,SAAS,IAAI,cAAc;CAC/C,MAAM,oBAAoB,qBAAqB,IAAI,gBAAgB,QAAQ,CAAC;AAC5E,SAAQ,OAAO;AAEf,KAAI,MAAM,aAAa,QAAQ;EAE7B,MAAM,YAAY,MAAM,aAAa,UAAU;EAC/C,MAAM,aAAa,kBAAkB,aAAa,MAAM;AACxD,MAAI,CAAC,UAAU,WAAW,WAAW,CACnC,OAAM,IAAI,MACR,gDAAgD,UAAU,uCAAuC,WAAW,GAC7G;;AAWL,QAAO;EAAC;EAAmB,cAPN,IAAI,mBACvB,IACA,aACA,OACA,kBACD;EAEuC;;AAG1C,IAAM,qBAAN,MAAiD;CAC/C;CACA;CACA;CACA;CAEA,YACE,IACA,aACA,OACA,mBACA;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,gBAAgB;AACvD,QAAA,cAAoB;AACpB,QAAA,QAAc;AACd,QAAA,oBAA0B;;CAG5B,cAA4B;AAC1B,SAAO,MAAA,aAAmB;;CAG5B,mBAAmB;AACjB,SAAO;;CAGT,OAAsB;AACpB,SAAO,QAAQ,SAAS;;CAG1B,YACE,iBACA,mBAAsC,EAAE,EACjB;AACvB,MAAI,kBAAkB,OACpB,OAAM,IAAI,MACR,0DACD;AAEH,SAAO,QAAQ,QAAQ,MAAA,YAAkB,gBAAgB,CAAC;;CAG5D,aAAa,iBAAwC;EACnD,MAAM,EAAC,cAAc,mBAAkB,MAAA;EACvC,MAAM,EAAC,OAAO,aAAY,MAAA;EAC1B,MAAM,MAAM,IAAI,IAAI,MAAA,YAAkB;AACtC,MAAI,aAAa,IAAI,SAAS,MAAM;AACpC,MAAI,aAAa,IAAI,YAAY,OAAO,SAAS,CAAC;AAClD,OAAK,MAAM,OAAO,aAChB,KAAI,aAAa,OAAO,gBAAgB,IAAI;AAE9C,MAAI,iBAAiB;AACnB,UACE,eAAe,QACf,yDACD;AACD,OAAI,aAAa,IAAI,iBAAiB,gBAAgB;AACtD,OAAI,aAAa,IAAI,kBAAkB,eAAe;;EAGxD,MAAM,KAAK,IAAI,YAAU,IAAI;EAC7B,MAAM,EAAC,UAAU,cAAa,OAC5B,MAAA,IACA,IACA,2BAGA,EAAC,WAAW,SAA+B,MAAK,CACjD;AACD,SAAO;GAAC,SAAS;GAAU,MAAM;GAAU;;;;;;;;;;;;;;AAe/C,eAAsB,YACpB,IACA,OACA,IACA,aACA,SACA;CACA,MAAM,EAAC,OAAO,IAAI,iBAAgB;CAKlC,MAAM,EAAC,YAJc,IAAI,mBAAmB,IAAI,aAAa,OAAO;EAClE,gBAAgB;EAChB;EACD,CAAC,CAC6B,aAAa;AAE5C,8BAA6B,GAAG;CAChC,MAAM,YAAY,IAAI,gBACpB,IAAI,gBAAgB,GAAG,EACvB,iBACC,GAAG,QAAQ;AACV,QAAM;GAET;CAED,MAAM,kBAAkB,2BAA2B,sBAAsB,GAAG;AAC5E,KAAI;EACF,IAAI,MAAM;AACV,aAAW,MAAM,UAAU,SAAS;GAClC,MAAM,CAAC,OAAO;AACd,WAAQ,KAAR;IACE,KAAK,SAAS;KACZ,MAAM,EAAC,oBAAmB,OAAO;AACjC,QAAG,OACD,yBAAyB,GAAG,qBAAqB,kBAClD;AACD,qBAAgB,QACd,IACA,gBACA,sCAAsC,mBACtC,IACD;AACD,0BACE,IACA,aAAa,UAAU,EACvB,iBACA,SACA,MACD;AACD,eAAU,eAAe,IAAI,OAAO;AACpC;;IAEF,KAAK;AACH,eAAU,eAAe,IAAI,OAAO;AACpC,SAAI,EAAE,MAAM,QAAS,EACnB,IAAG,QAAQ,aAAa,IAAI,UAAU;AAExC;IACF,KAAK;AACH,eAAU,eAAe,IAAI,OAAO;AACpC,iCAA4B,IAAI,IAAI,MAAM;AAC1C,QAAG,OAAO,4BAA4B,IAAI,UAAU;AACpD;IAEF,KAAK,SACH;IAEF,KAAK,WAAW;KACd,MAAM,EAAC,KAAK,YAAW,OAAO;AAC9B,SAAI,QAAQ,iBACV,OAAM,IAAI,gBACR,WAAW,uCACZ;;IAIL,KAAK,WACH,OAAM,IAAI,MACR,2CAA2C,UAAU,OAAO,GAC7D;IACH,QACE,aAAY,OAAO;;;AAGzB,QAAM,IAAI,MACR,iBAAiB,YAAY,uCAC9B;UACM,GAAG;AACV,QAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,EAAE;WACzD;AACR,kBAAgB,MAAM;;;AAM1B,SAAS,kBAAkB,EACzB,OACA,YACuD;AACvD,QAAO;GACJ,GAAG,MAAM,GAAG,SAAS,YAAY;GAChC,eAAe,EAAC,MAAM,UAAS;GAC/B,UAAU,EAAC,MAAM,UAAS;GAC1B,gBAAgB,EAAC,MAAM,UAAS;GAChC,QAAQ,EAAC,MAAM,UAAS;GACzB;GACA,GAAG,MAAM,GAAG,SAAS,cAAc;GAClC,eAAe,EAAC,MAAM,UAAS;GAC/B,UAAU,EAAC,MAAM,UAAS;GAC1B,YAAY,EAAC,MAAM,UAAS;GAC5B,UAAU,EAAC,MAAM,QAAO;GACzB;GACA,GAAG,MAAM,gBAAgB;GACxB,aAAa,EAAC,MAAM,QAAO;GAC3B,MAAM,EAAC,MAAM,UAAS;GACvB;EACF;;AAGH,SAAS,4BACP,IACA,IACA,OACA;CACA,MAAM,SAAS,gBAAgB,IAAI,IAAI,EAAC,2BAA2B,MAAK,CAAC;CACzE,MAAM,WAAW,kBAAkB,MAAM;AACzC,MAAK,MAAM,CAAC,MAAM,YAAY,OAAO,QAAQ,SAAS,EAAE;EACtD,MAAM,QAAQ,OAAO,IAAI,KAAK,EAAE;AAChC,MAAI,CAAC,MACH,OAAM,IAAI,MACR,4BAA4B,KAAK,kBAAkB,CACjD,GAAG,OAAO,MAAM,CACjB,CAAC,sFAGH;AAEH,OAAK,MAAM,CAAC,KAAK,EAAC,WAAU,OAAO,QAAQ,QAAQ,EAAE;GACnD,MAAM,QAAQ,MAAM;AACpB,OAAI,CAAC,MACH,OAAM,IAAI,MACR,aAAa,MAAM,0BAA0B,IAAI,UAClD;AAEH,OAAI,MAAM,SAAS,KACjB,OAAM,IAAI,MACR,aAAa,MAAM,GAAG,IAAI,gBAAgB,MAAM,KAAK,sBAAsB,KAAK,QACjF"}

package/out/zero-cache/src/services/change-source/pg/backfill-metadata.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"backfill-metadata.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-metadata.ts"],"sourcesContent":["import * as v from '../../../../../shared/src/valita.ts';\n\n// PG-specific messages passed in the table metadata and backfill ID\n// messages.\n\nexport const columnMetadataSchema = v.object({\n  attNum: v.number(),\n});\n\nexport type ColumnMetadata = v.Infer<typeof columnMetadataSchema>;\n\nexport const tableMetadataSchema = v.object({\n  schemaOID: v.number(),\n  relationOID: v.number(),\n  rowKey: v.record(columnMetadataSchema),\n});\n\nexport type TableMetadata = v.Infer<typeof tableMetadataSchema>;\n"],"mappings":";;AAKA,IAAa,uBAAuB,eAAE,OAAO,EAC3C,QAAQ,eAAE,QAAQ,EACnB,CAAC;AAIF,IAAa,sBAAsB,eAAE,OAAO;CAC1C,WAAW,eAAE,QAAQ;CACrB,aAAa,eAAE,QAAQ;CACvB,QAAQ,eAAE,OAAO,qBAAqB;CACvC,CAAC"}

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js

@@ -1,197 +0,0 @@
-import { assert } from "../../../../../shared/src/asserts.js";
-import { parse } from "../../../../../shared/src/valita.js";
-import { equals } from "../../../../../shared/src/set-utils.js";
-import { pgClient } from "../../../types/pg.js";
-import { toStateVersionString } from "./lsn.js";
-import { READONLY } from "../../../db/mode-enum.js";
-import { getPublicationInfo } from "./schema/published.js";
-import { SchemaIncompatibilityError } from "../common/backfill-manager.js";
-import { BinaryCopyParser, hasBinaryDecoder, makeBinaryDecoder, textCastDecoder } from "../../../db/pg-copy-binary.js";
-import { TsvParser } from "../../../db/pg-copy.js";
-import { getTypeParsers } from "../../../db/pg-type-parser.js";
-import { TransactionPool, importSnapshot } from "../../../db/transaction-pool.js";
-import { columnMetadataSchema, tableMetadataSchema } from "./backfill-metadata.js";
-import { createReplicationSlot } from "./replication-slots.js";
-import { makeBinarySelectExprs, makeDownloadStatements } from "./initial-sync.js";
-import postgres from "postgres";
-import { PG_UNDEFINED_COLUMN, PG_UNDEFINED_TABLE } from "@drdgvhbh/postgres-error-codes";
-//#region ../zero-cache/src/services/change-source/pg/backfill-stream.ts
-var POSTGRES_COPY_CHUNK_SIZE = 64 * 1024;
-var SAMPLE_OR_LIMIT_RE = /\sTABLESAMPLE\s+BERNOULLI\b|\sLIMIT\s+\d/i;
-/**
-* Streams a series of `backfill` messages (ending with `backfill-complete`)
-* at a set watermark (i.e. LSN). The data is retrieved via a COPY stream
-* made at a transaction snapshot corresponding to specific LSN, obtained by
-* creating a short-lived replication slot.
-*/
-async function* streamBackfill(lc, upstreamURI, { slot, publications }, bf, opts = {}) {
-	lc = lc.withContext("component", "backfill").withContext("table", bf.table.name);
-	const { flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE, textCopy = false } = opts;
-	const db = pgClient(lc, upstreamURI, "backfill-stream", { ["max_lifetime"]: 7200 });
-	let tx;
-	let watermark;
-	try {
-		({tx, watermark} = await createSnapshotTransaction(lc, upstreamURI, db, slot));
-		const { tableSpec, backfill } = await validateSchema(tx, publications, bf, watermark);
-		const { relation, columns } = backfill;
-		const cols = [...relation.rowKey.columns, ...columns];
-		const stmts = makeDownloadStatements(tableSpec, cols);
-		if (textCopy) {
-			const types = await getTypeParsers(db, { returnJsonAsString: true });
-			yield* stream(lc, tx, backfill, stmts, `COPY (${stmts.select}) TO STDOUT`, new TsvParser(), cols.map((col) => {
-				const parser = types.getTypeParser(tableSpec.columns[col].typeOID);
-				return (text) => parser(text);
-			}), flushThresholdBytes);
-		} else {
-			const binaryStmts = makeDownloadStatements(tableSpec, cols, void 0, void 0, makeBinarySelectExprs(tableSpec, cols));
-			yield* stream(lc, tx, backfill, stmts, `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`, new BinaryCopyParser(), cols.map((col) => {
-				const spec = tableSpec.columns[col];
-				const decoder = hasBinaryDecoder(spec) ? makeBinaryDecoder(spec) : textCastDecoder;
-				return (buf) => decoder(buf);
-			}), flushThresholdBytes);
-		}
-	} catch (e) {
-		if (e instanceof postgres.PostgresError && (e.code === PG_UNDEFINED_TABLE || e.code === PG_UNDEFINED_COLUMN)) throw new SchemaIncompatibilityError(bf, String(e), { cause: e });
-		throw e;
-	} finally {
-		tx?.setDone();
-		db.end().catch((e) => lc.warn?.(`error closing backfill connection`, e));
-	}
-}
-async function* stream(lc, tx, backfill, { getTotalRows, getTotalBytes }, copyCommand, parser, decoders, flushThresholdBytes) {
-	assert(!SAMPLE_OR_LIMIT_RE.test(copyCommand), `backfill COPY must not sample or limit: ${copyCommand}`);
-	const start = performance.now();
-	const [rows, bytes] = await tx.processReadTask((sql) => Promise.all([sql.unsafe(getTotalRows), sql.unsafe(getTotalBytes)]));
-	const status = {
-		rows: 0,
-		totalRows: Number(rows[0].totalRows),
-		totalBytes: Number(bytes[0].totalBytes)
-	};
-	let elapsed = (performance.now() - start).toFixed(3);
-	lc.info?.(`Computed total rows and bytes for: ${copyCommand} (${elapsed} ms)`, { status });
-	const copyStream = await tx.processReadTask((sql) => sql.unsafe(copyCommand).readable());
-	let totalBytes = 0;
-	let totalMsgs = 0;
-	let rowValues = [];
-	let bufferedBytes = 0;
-	const logFlushed = () => {
-		lc.debug?.(`Flushed ${rowValues.length} rows, ${bufferedBytes} bytes (total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`);
-	};
-	let row = Array.from({ length: decoders.length });
-	let col = 0;
-	for await (const data of copyStream) {
-		const chunk = data;
-		for (const field of parser.parse(chunk)) {
-			row[col] = field === null ? null : decoders[col](field);
-			if (++col === decoders.length) {
-				rowValues.push(row);
-				status.rows++;
-				row = Array.from({ length: decoders.length });
-				col = 0;
-			}
-		}
-		bufferedBytes += chunk.byteLength;
-		totalBytes += chunk.byteLength;
-		if (bufferedBytes >= flushThresholdBytes) {
-			yield {
-				tag: "backfill",
-				...backfill,
-				rowValues,
-				status
-			};
-			totalMsgs++;
-			logFlushed();
-			rowValues = [];
-			bufferedBytes = 0;
-		}
-	}
-	if (rowValues.length > 0) {
-		yield {
-			tag: "backfill",
-			...backfill,
-			rowValues,
-			status
-		};
-		totalMsgs++;
-		logFlushed();
-	}
-	yield {
-		tag: "backfill-completed",
-		...backfill,
-		status
-	};
-	elapsed = (performance.now() - start).toFixed(3);
-	lc.info?.(`Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes (${elapsed} ms)`);
-}
-/**
-* Creates (and drops) a replication slot in order to obtain a snapshot
-* that corresponds with a specific LSN. Sets the snapshot on the
-* TransactionPool and returns the watermark corresponding to the LSN.
-*
-* (Note that PG's other LSN-related functions are not scoped to a
-*  transaction; this is the only way to get set a transaction at a specific
-*  LSN.)
-*/
-async function createSnapshotTransaction(lc, upstreamURI, db, slotNamePrefix) {
-	const replicationSession = pgClient(lc, upstreamURI, "backfill-replication-session", {
-		["fetch_types"]: false,
-		connection: { replication: "database" }
-	});
-	const slotName = `${slotNamePrefix}_bf_${Date.now()}`;
-	try {
-		const { snapshot_name: snapshot, consistent_point: lsn } = await createReplicationSlot(lc, replicationSession, { slotName });
-		const { init, imported } = importSnapshot(snapshot);
-		const tx = new TransactionPool(lc, {
-			mode: READONLY,
-			init
-		}).run(db);
-		await imported;
-		await replicationSession.unsafe(`DROP_REPLICATION_SLOT "${slotName}"`);
-		const watermark = toStateVersionString(lsn);
-		lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);
-		return {
-			tx,
-			watermark
-		};
-	} catch (e) {
-		await replicationSession.unsafe(`SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots
-         WHERE slot_name = '${slotName}'`);
-		lc.warn?.(`Failed to create backfill snapshot`, e);
-		throw e;
-	} finally {
-		await replicationSession.end();
-	}
-}
-function validateSchema(tx, publications, bf, watermark) {
-	return tx.processReadTask(async (sql) => {
-		const { tables } = await getPublicationInfo(sql, publications);
-		const spec = tables.find((spec) => spec.schema === bf.table.schema && spec.name === bf.table.name);
-		if (!spec) throw new SchemaIncompatibilityError(bf, `Table has been renamed or dropped`);
-		const tableMeta = parse(bf.table.metadata, tableMetadataSchema);
-		if (spec.schemaOID !== tableMeta.schemaOID) throw new SchemaIncompatibilityError(bf, `Schema no longer corresponds to the original schema`);
-		if (spec.oid !== tableMeta.relationOID) throw new SchemaIncompatibilityError(bf, `Table no longer corresponds to the original table`);
-		if (!equals(new Set(Object.keys(tableMeta.rowKey)), new Set(spec.replicaIdentityColumns))) throw new SchemaIncompatibilityError(bf, "Row key (e.g. PRIMARY KEY or INDEX) has changed");
-		const allCols = [...Object.entries(tableMeta.rowKey), ...Object.entries(bf.columns)];
-		for (const [col, val] of allCols) {
-			const colSpec = spec.columns[col];
-			if (!colSpec) throw new SchemaIncompatibilityError(bf, `Column ${col} has been renamed or dropped`);
-			if (parse(val, columnMetadataSchema).attNum !== colSpec.pos) throw new SchemaIncompatibilityError(bf, `Column ${col} no longer corresponds to the original column`);
-		}
-		return {
-			tableSpec: spec,
-			backfill: {
-				relation: {
-					schema: bf.table.schema,
-					name: bf.table.name,
-					rowKey: { columns: Object.keys(tableMeta.rowKey) }
-				},
-				columns: Object.keys(bf.columns).filter((col) => !(col in tableMeta.rowKey)),
-				watermark
-			}
-		};
-	});
-}
-//#endregion
-export { streamBackfill };
-
-//# sourceMappingURL=backfill-stream.js.map

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"backfill-stream.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"sourcesContent":["import {\n  PG_UNDEFINED_COLUMN,\n  PG_UNDEFINED_TABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {READONLY} from '../../../db/mode-enum.ts';\nimport {\n  BinaryCopyParser,\n  hasBinaryDecoder,\n  makeBinaryDecoder,\n  textCastDecoder,\n} from '../../../db/pg-copy-binary.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport type {PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {pgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {SchemaIncompatibilityError} from '../common/backfill-manager.ts';\nimport type {\n  BackfillCompleted,\n  BackfillRequest,\n  DownloadStatus,\n  JSONValue,\n  MessageBackfill,\n} from '../protocol/current.ts';\nimport {\n  columnMetadataSchema,\n  tableMetadataSchema,\n} from './backfill-metadata.ts';\nimport {\n  makeBinarySelectExprs,\n  makeDownloadStatements,\n  type DownloadStatements,\n} from './initial-sync.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {createReplicationSlot} from './replication-slots.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport type {Replica} from './schema/shard.ts';\n\ntype BackfillParams = Omit<BackfillCompleted, 'tag'>;\n\ntype StreamOptions = {\n  /**\n   * The number of bytes at which to flush a batch of rows in a\n   * backfill message. Defaults to Node's getDefaultHighWatermark().\n   */\n  flushThresholdBytes?: number | undefined;\n\n  /**\n   * Use text-format COPY instead of binary COPY.\n   * Binary is faster and handles all types (unknown types are cast to\n   * `::text` in the SELECT). This flag exists as an escape hatch to\n   * revert to the old code path if needed.\n   */\n  textCopy?: boolean | undefined;\n};\n\n// The size of chunks that Postgres sends on COPY stream.\n// This happens to match NodeJS's getDefaultHighWatermark()\n// (for Node v20+).\nconst POSTGRES_COPY_CHUNK_SIZE = 64 * 1024;\n\n// Matches the exact clauses emitted by makeDownloadStatements; quoted\n// identifiers like \"limit\" won't match because they lack the surrounding\n// whitespace.\nconst SAMPLE_OR_LIMIT_RE = /\\sTABLESAMPLE\\s+BERNOULLI\\b|\\sLIMIT\\s+\\d/i;\n\n/**\n * Streams a series of `backfill` messages (ending with `backfill-complete`)\n * at a set watermark (i.e. LSN). The data is retrieved via a COPY stream\n * made at a transaction snapshot corresponding to specific LSN, obtained by\n * creating a short-lived replication slot.\n */\nexport async function* streamBackfill(\n  lc: LogContext,\n  upstreamURI: string,\n  {slot, publications}: Pick<Replica, 'slot' | 'publications'>,\n  bf: BackfillRequest,\n  opts: StreamOptions = {},\n): AsyncGenerator<MessageBackfill | BackfillCompleted> {\n  lc = lc\n    .withContext('component', 'backfill')\n    .withContext('table', bf.table.name);\n\n  const {flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE, textCopy = false} =\n    opts;\n  const db = pgClient(lc, upstreamURI, 'backfill-stream', {\n    ['max_lifetime']: 120 * 60, // set a long (2h) limit for COPY streaming\n  });\n  let tx: TransactionPool | undefined;\n  let watermark: string;\n  try {\n    ({tx, watermark} = await createSnapshotTransaction(\n      lc,\n      upstreamURI,\n      db,\n      slot,\n    ));\n    const {tableSpec, backfill} = await validateSchema(\n      tx,\n      publications,\n      bf,\n      watermark,\n    );\n\n    // Note: validateSchema ensures that the rowKey and columns are disjoint\n    const {relation, columns} = backfill;\n    const cols = [...relation.rowKey.columns, ...columns];\n    const stmts = makeDownloadStatements(tableSpec, cols);\n\n    if (textCopy) {\n      const types = await getTypeParsers(db, {returnJsonAsString: true});\n      yield* stream(\n        lc,\n        tx,\n        backfill,\n        stmts,\n        `COPY (${stmts.select}) TO STDOUT`,\n        new TsvParser(),\n        cols.map(col => {\n          const parser = types.getTypeParser(tableSpec.columns[col].typeOID);\n          return (text: string) => parser(text) as JSONValue;\n        }),\n        flushThresholdBytes,\n      );\n    } else {\n      const binaryStmts = makeDownloadStatements(\n        tableSpec,\n        cols,\n        undefined,\n        undefined,\n        makeBinarySelectExprs(tableSpec, cols),\n      );\n\n      yield* stream(\n        lc,\n        tx,\n        backfill,\n        stmts,\n        `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`,\n        new BinaryCopyParser(),\n        cols.map(col => {\n          const spec = tableSpec.columns[col];\n          const decoder = hasBinaryDecoder(spec)\n            ? makeBinaryDecoder(spec)\n            : textCastDecoder;\n          return (buf: Buffer) => decoder(buf) as unknown as JSONValue;\n        }),\n        flushThresholdBytes,\n      );\n    }\n  } catch (e) {\n    // Although we make the best effort to validate the schema at the\n    // transaction snapshot, certain forms of `ALTER TABLE` are not\n    // MVCC safe and not \"frozen\" in the snapshot:\n    //\n    // https://www.postgresql.org/docs/current/mvcc-caveats.html\n    //\n    // Handle these errors as schema incompatibility errors rather than\n    // unknown runtime errors.\n    if (\n      e instanceof postgres.PostgresError &&\n      (e.code === PG_UNDEFINED_TABLE || e.code === PG_UNDEFINED_COLUMN)\n    ) {\n      throw new SchemaIncompatibilityError(bf, String(e), {cause: e});\n    }\n    throw e;\n  } finally {\n    tx?.setDone();\n    // Workaround postgres.js hanging at the end of some COPY commands:\n    // https://github.com/porsager/postgres/issues/499\n    void db.end().catch(e => lc.warn?.(`error closing backfill connection`, e));\n  }\n}\n\nasync function* stream<T>(\n  lc: LogContext,\n  tx: TransactionPool,\n  backfill: BackfillParams,\n  {\n    getTotalRows,\n    getTotalBytes,\n  }: Pick<DownloadStatements, 'getTotalRows' | 'getTotalBytes'>,\n  copyCommand: string,\n  parser: {parse(chunk: Buffer): Iterable<T | null>},\n  decoders: ((field: T) => JSONValue)[],\n  flushThresholdBytes: number,\n): AsyncGenerator<MessageBackfill | BackfillCompleted> {\n  // Backfill must read every row: TABLESAMPLE / LIMIT are reserved for shadow\n  // sync and must never appear in a backfill COPY.\n  assert(\n    !SAMPLE_OR_LIMIT_RE.test(copyCommand),\n    `backfill COPY must not sample or limit: ${copyCommand}`,\n  );\n  const start = performance.now();\n  const [rows, bytes] = await tx.processReadTask(sql =>\n    Promise.all([\n      sql.unsafe<{totalRows: bigint}[]>(getTotalRows),\n      sql.unsafe<{totalBytes: bigint}[]>(getTotalBytes),\n    ]),\n  );\n  const status: DownloadStatus = {\n    rows: 0,\n    totalRows: Number(rows[0].totalRows),\n    totalBytes: Number(bytes[0].totalBytes),\n  };\n\n  let elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(\n    `Computed total rows and bytes for: ${copyCommand} (${elapsed} ms)`,\n    {\n      status,\n    },\n  );\n  const copyStream = await tx.processReadTask(sql =>\n    sql.unsafe(copyCommand).readable(),\n  );\n\n  let totalBytes = 0;\n  let totalMsgs = 0;\n  let rowValues: JSONValue[][] = [];\n  let bufferedBytes = 0;\n\n  const logFlushed = () => {\n    lc.debug?.(\n      `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes ` +\n        `(total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`,\n    );\n  };\n\n  // Tracks the row being parsed.\n  let row: JSONValue[] = Array.from({length: decoders.length});\n  let col = 0;\n\n  for await (const data of copyStream) {\n    const chunk = data as Buffer;\n    for (const field of parser.parse(chunk)) {\n      row[col] = field === null ? null : decoders[col](field);\n\n      if (++col === decoders.length) {\n        rowValues.push(row);\n        status.rows++;\n        row = Array.from({length: decoders.length});\n        col = 0;\n      }\n    }\n    bufferedBytes += chunk.byteLength;\n    totalBytes += chunk.byteLength;\n\n    if (bufferedBytes >= flushThresholdBytes) {\n      yield {tag: 'backfill', ...backfill, rowValues, status};\n      totalMsgs++;\n      logFlushed();\n      rowValues = [];\n      bufferedBytes = 0;\n    }\n  }\n\n  // Flush the last batch of rows.\n  if (rowValues.length > 0) {\n    yield {tag: 'backfill', ...backfill, rowValues, status};\n    totalMsgs++;\n    logFlushed();\n  }\n\n  yield {tag: 'backfill-completed', ...backfill, status};\n  elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(\n    `Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes ` +\n      `(${elapsed} ms)`,\n  );\n}\n\n/**\n * Creates (and drops) a replication slot in order to obtain a snapshot\n * that corresponds with a specific LSN. Sets the snapshot on the\n * TransactionPool and returns the watermark corresponding to the LSN.\n *\n * (Note that PG's other LSN-related functions are not scoped to a\n *  transaction; this is the only way to get set a transaction at a specific\n *  LSN.)\n */\nasync function createSnapshotTransaction(\n  lc: LogContext,\n  upstreamURI: string,\n  db: PostgresDB,\n  slotNamePrefix: string,\n) {\n  const replicationSession = pgClient(\n    lc,\n    upstreamURI,\n    'backfill-replication-session',\n    {\n      ['fetch_types']: false, // Necessary for the streaming protocol\n      connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n    },\n  );\n  const slotName = `${slotNamePrefix}_bf_${Date.now()}`;\n  try {\n    const {snapshot_name: snapshot, consistent_point: lsn} =\n      await createReplicationSlot(lc, replicationSession, {slotName});\n\n    const {init, imported} = importSnapshot(snapshot);\n    const tx = new TransactionPool(lc, {mode: READONLY, init}).run(db);\n    await imported;\n    await replicationSession.unsafe(`DROP_REPLICATION_SLOT \"${slotName}\"`);\n\n    const watermark = toStateVersionString(lsn);\n    lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);\n    return {tx, watermark};\n  } catch (e) {\n    // In the event of a failure, clean up the replication slot if created.\n    await replicationSession.unsafe(\n      /*sql*/\n      `SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots\n         WHERE slot_name = '${slotName}'`,\n    );\n    lc.warn?.(`Failed to create backfill snapshot`, e);\n    throw e;\n  } finally {\n    await replicationSession.end();\n  }\n}\n\nfunction validateSchema(\n  tx: TransactionPool,\n  publications: string[],\n  bf: BackfillRequest,\n  watermark: string,\n): Promise<{\n  tableSpec: PublishedTableSpec;\n  backfill: BackfillParams;\n}> {\n  return tx.processReadTask(async sql => {\n    const {tables} = await getPublicationInfo(sql, publications);\n    const spec = tables.find(\n      spec => spec.schema === bf.table.schema && spec.name === bf.table.name,\n    );\n    if (!spec) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Table has been renamed or dropped`,\n      );\n    }\n    const tableMeta = v.parse(bf.table.metadata, tableMetadataSchema);\n    if (spec.schemaOID !== tableMeta.schemaOID) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Schema no longer corresponds to the original schema`,\n      );\n    }\n    if (spec.oid !== tableMeta.relationOID) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Table no longer corresponds to the original table`,\n      );\n    }\n    if (\n      !equals(\n        new Set(Object.keys(tableMeta.rowKey)),\n        new Set(spec.replicaIdentityColumns),\n      )\n    ) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        'Row key (e.g. PRIMARY KEY or INDEX) has changed',\n      );\n    }\n    const allCols = [\n      ...Object.entries(tableMeta.rowKey),\n      ...Object.entries(bf.columns),\n    ];\n    for (const [col, val] of allCols) {\n      const colSpec = spec.columns[col];\n      if (!colSpec) {\n        throw new SchemaIncompatibilityError(\n          bf,\n          `Column ${col} has been renamed or dropped`,\n        );\n      }\n      const colMeta = v.parse(val, columnMetadataSchema);\n      if (colMeta.attNum !== colSpec.pos) {\n        throw new SchemaIncompatibilityError(\n          bf,\n          `Column ${col} no longer corresponds to the original column`,\n        );\n      }\n    }\n    const backfill: BackfillParams = {\n      relation: {\n        schema: bf.table.schema,\n        name: bf.table.name,\n        rowKey: {columns: Object.keys(tableMeta.rowKey)},\n      },\n      columns: Object.keys(bf.columns).filter(\n        col => !(col in tableMeta.rowKey),\n      ),\n      watermark,\n    };\n    return {tableSpec: spec, backfill};\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAgEA,IAAM,2BAA2B,KAAK;AAKtC,IAAM,qBAAqB;;;;;;;AAQ3B,gBAAuB,eACrB,IACA,aACA,EAAC,MAAM,gBACP,IACA,OAAsB,EAAE,EAC6B;AACrD,MAAK,GACF,YAAY,aAAa,WAAW,CACpC,YAAY,SAAS,GAAG,MAAM,KAAK;CAEtC,MAAM,EAAC,sBAAsB,0BAA0B,WAAW,UAChE;CACF,MAAM,KAAK,SAAS,IAAI,aAAa,mBAAmB,GACrD,iBAAiB,MACnB,CAAC;CACF,IAAI;CACJ,IAAI;AACJ,KAAI;AACF,GAAC,CAAC,IAAI,aAAa,MAAM,0BACvB,IACA,aACA,IACA,KACD;EACD,MAAM,EAAC,WAAW,aAAY,MAAM,eAClC,IACA,cACA,IACA,UACD;EAGD,MAAM,EAAC,UAAU,YAAW;EAC5B,MAAM,OAAO,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,QAAQ;EACrD,MAAM,QAAQ,uBAAuB,WAAW,KAAK;AAErD,MAAI,UAAU;GACZ,MAAM,QAAQ,MAAM,eAAe,IAAI,EAAC,oBAAoB,MAAK,CAAC;AAClE,UAAO,OACL,IACA,IACA,UACA,OACA,SAAS,MAAM,OAAO,cACtB,IAAI,WAAW,EACf,KAAK,KAAI,QAAO;IACd,MAAM,SAAS,MAAM,cAAc,UAAU,QAAQ,KAAK,QAAQ;AAClE,YAAQ,SAAiB,OAAO,KAAK;KACrC,EACF,oBACD;SACI;GACL,MAAM,cAAc,uBAClB,WACA,MACA,KAAA,GACA,KAAA,GACA,sBAAsB,WAAW,KAAK,CACvC;AAED,UAAO,OACL,IACA,IACA,UACA,OACA,SAAS,YAAY,OAAO,mCAC5B,IAAI,kBAAkB,EACtB,KAAK,KAAI,QAAO;IACd,MAAM,OAAO,UAAU,QAAQ;IAC/B,MAAM,UAAU,iBAAiB,KAAK,GAClC,kBAAkB,KAAK,GACvB;AACJ,YAAQ,QAAgB,QAAQ,IAAI;KACpC,EACF,oBACD;;UAEI,GAAG;AASV,MACE,aAAa,SAAS,kBACrB,EAAE,SAAS,sBAAsB,EAAE,SAAS,qBAE7C,OAAM,IAAI,2BAA2B,IAAI,OAAO,EAAE,EAAE,EAAC,OAAO,GAAE,CAAC;AAEjE,QAAM;WACE;AACR,MAAI,SAAS;AAGR,KAAG,KAAK,CAAC,OAAM,MAAK,GAAG,OAAO,qCAAqC,EAAE,CAAC;;;AAI/E,gBAAgB,OACd,IACA,IACA,UACA,EACE,cACA,iBAEF,aACA,QACA,UACA,qBACqD;AAGrD,QACE,CAAC,mBAAmB,KAAK,YAAY,EACrC,2CAA2C,cAC5C;CACD,MAAM,QAAQ,YAAY,KAAK;CAC/B,MAAM,CAAC,MAAM,SAAS,MAAM,GAAG,iBAAgB,QAC7C,QAAQ,IAAI,CACV,IAAI,OAA8B,aAAa,EAC/C,IAAI,OAA+B,cAAc,CAClD,CAAC,CACH;CACD,MAAM,SAAyB;EAC7B,MAAM;EACN,WAAW,OAAO,KAAK,GAAG,UAAU;EACpC,YAAY,OAAO,MAAM,GAAG,WAAW;EACxC;CAED,IAAI,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACpD,IAAG,OACD,sCAAsC,YAAY,IAAI,QAAQ,OAC9D,EACE,QACD,CACF;CACD,MAAM,aAAa,MAAM,GAAG,iBAAgB,QAC1C,IAAI,OAAO,YAAY,CAAC,UAAU,CACnC;CAED,IAAI,aAAa;CACjB,IAAI,YAAY;CAChB,IAAI,YAA2B,EAAE;CACjC,IAAI,gBAAgB;CAEpB,MAAM,mBAAmB;AACvB,KAAG,QACD,WAAW,UAAU,OAAO,SAAS,cAAc,sBACjC,OAAO,KAAK,SAAS,UAAU,UAAU,WAAW,GACvE;;CAIH,IAAI,MAAmB,MAAM,KAAK,EAAC,QAAQ,SAAS,QAAO,CAAC;CAC5D,IAAI,MAAM;AAEV,YAAW,MAAM,QAAQ,YAAY;EACnC,MAAM,QAAQ;AACd,OAAK,MAAM,SAAS,OAAO,MAAM,MAAM,EAAE;AACvC,OAAI,OAAO,UAAU,OAAO,OAAO,SAAS,KAAK,MAAM;AAEvD,OAAI,EAAE,QAAQ,SAAS,QAAQ;AAC7B,cAAU,KAAK,IAAI;AACnB,WAAO;AACP,UAAM,MAAM,KAAK,EAAC,QAAQ,SAAS,QAAO,CAAC;AAC3C,UAAM;;;AAGV,mBAAiB,MAAM;AACvB,gBAAc,MAAM;AAEpB,MAAI,iBAAiB,qBAAqB;AACxC,SAAM;IAAC,KAAK;IAAY,GAAG;IAAU;IAAW;IAAO;AACvD;AACA,eAAY;AACZ,eAAY,EAAE;AACd,mBAAgB;;;AAKpB,KAAI,UAAU,SAAS,GAAG;AACxB,QAAM;GAAC,KAAK;GAAY,GAAG;GAAU;GAAW;GAAO;AACvD;AACA,cAAY;;AAGd,OAAM;EAAC,KAAK;EAAsB,GAAG;EAAU;EAAO;AACtD,YAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AAChD,IAAG,OACD,sBAAsB,OAAO,KAAK,SAAS,UAAU,SAAS,WAAW,UACnE,QAAQ,MACf;;;;;;;;;;;AAYH,eAAe,0BACb,IACA,aACA,IACA,gBACA;CACA,MAAM,qBAAqB,SACzB,IACA,aACA,gCACA;GACG,gBAAgB;EACjB,YAAY,EAAC,aAAa,YAAW;EACtC,CACF;CACD,MAAM,WAAW,GAAG,eAAe,MAAM,KAAK,KAAK;AACnD,KAAI;EACF,MAAM,EAAC,eAAe,UAAU,kBAAkB,QAChD,MAAM,sBAAsB,IAAI,oBAAoB,EAAC,UAAS,CAAC;EAEjE,MAAM,EAAC,MAAM,aAAY,eAAe,SAAS;EACjD,MAAM,KAAK,IAAI,gBAAgB,IAAI;GAAC,MAAM;GAAU;GAAK,CAAC,CAAC,IAAI,GAAG;AAClE,QAAM;AACN,QAAM,mBAAmB,OAAO,0BAA0B,SAAS,GAAG;EAEtE,MAAM,YAAY,qBAAqB,IAAI;AAC3C,KAAG,OAAO,sCAAsC,IAAI,IAAI,UAAU,GAAG;AACrE,SAAO;GAAC;GAAI;GAAU;UACf,GAAG;AAEV,QAAM,mBAAmB,OAEvB;8BACwB,SAAS,GAClC;AACD,KAAG,OAAO,sCAAsC,EAAE;AAClD,QAAM;WACE;AACR,QAAM,mBAAmB,KAAK;;;AAIlC,SAAS,eACP,IACA,cACA,IACA,WAIC;AACD,QAAO,GAAG,gBAAgB,OAAM,QAAO;EACrC,MAAM,EAAC,WAAU,MAAM,mBAAmB,KAAK,aAAa;EAC5D,MAAM,OAAO,OAAO,MAClB,SAAQ,KAAK,WAAW,GAAG,MAAM,UAAU,KAAK,SAAS,GAAG,MAAM,KACnE;AACD,MAAI,CAAC,KACH,OAAM,IAAI,2BACR,IACA,oCACD;EAEH,MAAM,YAAY,MAAQ,GAAG,MAAM,UAAU,oBAAoB;AACjE,MAAI,KAAK,cAAc,UAAU,UAC/B,OAAM,IAAI,2BACR,IACA,sDACD;AAEH,MAAI,KAAK,QAAQ,UAAU,YACzB,OAAM,IAAI,2BACR,IACA,oDACD;AAEH,MACE,CAAC,OACC,IAAI,IAAI,OAAO,KAAK,UAAU,OAAO,CAAC,EACtC,IAAI,IAAI,KAAK,uBAAuB,CACrC,CAED,OAAM,IAAI,2BACR,IACA,kDACD;EAEH,MAAM,UAAU,CACd,GAAG,OAAO,QAAQ,UAAU,OAAO,EACnC,GAAG,OAAO,QAAQ,GAAG,QAAQ,CAC9B;AACD,OAAK,MAAM,CAAC,KAAK,QAAQ,SAAS;GAChC,MAAM,UAAU,KAAK,QAAQ;AAC7B,OAAI,CAAC,QACH,OAAM,IAAI,2BACR,IACA,UAAU,IAAI,8BACf;AAGH,OADgB,MAAQ,KAAK,qBAAqB,CACtC,WAAW,QAAQ,IAC7B,OAAM,IAAI,2BACR,IACA,UAAU,IAAI,+CACf;;AAcL,SAAO;GAAC,WAAW;GAAM,UAXQ;IAC/B,UAAU;KACR,QAAQ,GAAG,MAAM;KACjB,MAAM,GAAG,MAAM;KACf,QAAQ,EAAC,SAAS,OAAO,KAAK,UAAU,OAAO,EAAC;KACjD;IACD,SAAS,OAAO,KAAK,GAAG,QAAQ,CAAC,QAC/B,QAAO,EAAE,OAAO,UAAU,QAC3B;IACD;IACD;GACiC;GAClC"}