@rocicorp/zero 0.25.7 → 0.26.0-canary.2

package/out/zero-cache/src/services/view-syncer/pipeline-driver.js.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline-driver.js","sources":["../../../../../../zero-cache/src/services/view-syncer/pipeline-driver.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport {deepEqual, type JSONValue} from '../../../../shared/src/json.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport type {AST} from '../../../../zero-protocol/src/ast.ts';\nimport type {ClientSchema} from '../../../../zero-protocol/src/client-schema.ts';\nimport type {Row} from '../../../../zero-protocol/src/data.ts';\nimport type {PrimaryKey} from '../../../../zero-protocol/src/primary-key.ts';\nimport {buildPipeline} from '../../../../zql/src/builder/builder.ts';\nimport {\n  Debug,\n  runtimeDebugFlags,\n} from '../../../../zql/src/builder/debug-delegate.ts';\nimport type {Change} from '../../../../zql/src/ivm/change.ts';\nimport type {Node} from '../../../../zql/src/ivm/data.ts';\nimport {type Input, type Storage} from '../../../../zql/src/ivm/operator.ts';\nimport type {SourceSchema} from '../../../../zql/src/ivm/schema.ts';\nimport type {\n  Source,\n  SourceChange,\n  SourceInput,\n} from '../../../../zql/src/ivm/source.ts';\nimport type {ConnectionCostModel} from '../../../../zql/src/planner/planner-connection.ts';\nimport {MeasurePushOperator} from '../../../../zql/src/query/measure-push-operator.ts';\nimport type {ClientGroupStorage} from '../../../../zqlite/src/database-storage.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {createSQLiteCostModel} from '../../../../zqlite/src/sqlite-cost-model.ts';\nimport {TableSource} from '../../../../zqlite/src/table-source.ts';\nimport {\n  reloadPermissionsIfChanged,\n  type LoadedPermissions,\n} from '../../auth/load-permissions.ts';\nimport type {LogConfig} from '../../config/zero-config.ts';\nimport {computeZqlSpecs, mustGetTableSpec} from '../../db/lite-tables.ts';\nimport type {LiteAndZqlSpec, LiteTableSpec} from '../../db/specs.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateHistogram,\n} from '../../observability/metrics.ts';\nimport type {InspectorDelegate} from '../../server/inspector-delegate.ts';\nimport {type RowKey} from '../../types/row-key.ts';\nimport type {SchemaVersions} from '../../types/schema-versions.ts';\nimport {upstreamSchema, type ShardID} from '../../types/shards.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {checkClientSchema} from './client-schema.ts';\nimport type {Snapshotter} from './snapshotter.ts';\nimport {ResetPipelinesSignal, type SnapshotDiff} from './snapshotter.ts';\n\nexport type RowAdd = {\n  readonly type: 'add';\n  readonly queryHash: string;\n  readonly table: string;\n  readonly rowKey: Row;\n  readonly row: Row;\n};\n\nexport type RowRemove = {\n  readonly type: 'remove';\n  readonly queryHash: string;\n  readonly table: string;\n  readonly rowKey: Row;\n  readonly row: undefined;\n};\n\nexport type RowEdit = {\n  readonly type: 'edit';\n  readonly queryHash: string;\n  readonly table: string;\n  readonly rowKey: Row;\n  readonly row: Row;\n};\n\nexport type RowChange = RowAdd | RowRemove | RowEdit;\n\ntype Pipeline = {\n  readonly input: Input;\n  readonly hydrationTimeMs: number;\n  readonly originalHash: string;\n  readonly transformedAst: AST; // Optional, only set after hydration\n  readonly transformationHash: string; // The hash of the transformed AST\n};\n\ntype AdvanceContext = {\n  readonly timer: Timer;\n  readonly totalHydrationTimeMs: number;\n  readonly numChanges: number;\n  pos: number;\n};\n\ntype HydrateContext = {\n  readonly timer: Timer;\n};\n\nexport type Timer = {\n  elapsedLap: () => number;\n  totalElapsed: () => number;\n};\n\n/**\n * No matter how fast hydration is, advancement is given at least this long to\n * complete before doing a pipeline reset.\n */\nconst MIN_ADVANCEMENT_TIME_LIMIT_MS = 50;\n\n/**\n * Manages the state of IVM pipelines for a given ViewSyncer (i.e. client group).\n */\nexport class PipelineDriver {\n  readonly #tables = new Map<string, TableSource>();\n  // We probs need the original query hash\n  // so we can decide not to re-transform a custom query\n  // that is already hydrated.\n  readonly #pipelines = new Map<string, Pipeline>();\n\n  readonly #lc: LogContext;\n  readonly #snapshotter: Snapshotter;\n  readonly #storage: ClientGroupStorage;\n  readonly #shardID: ShardID;\n  readonly #logConfig: LogConfig;\n  readonly #tableSpecs = new Map<string, LiteAndZqlSpec>();\n  readonly #costModels: WeakMap<Database, ConnectionCostModel> | undefined;\n  readonly #yieldThresholdMs: number;\n  #streamer: Streamer | null = null;\n  #hydrateContext: HydrateContext | null = null;\n  #advanceContext: AdvanceContext | null = null;\n  #replicaVersion: string | null = null;\n  #primaryKeys: Map<string, PrimaryKey> | null = null;\n  #permissions: LoadedPermissions | null = null;\n\n  readonly #advanceTime = getOrCreateHistogram('sync', 'ivm.advance-time', {\n    description:\n      'Time to advance all queries for a given client group for in response to a single change.',\n    unit: 's',\n  });\n\n  readonly #conflictRowsDeleted = getOrCreateCounter(\n    'sync',\n    'ivm.conflict-rows-deleted',\n    'Number of rows deleted because they conflicted with added row',\n  );\n\n  readonly #inspectorDelegate: InspectorDelegate;\n\n  constructor(\n    lc: LogContext,\n    logConfig: LogConfig,\n    snapshotter: Snapshotter,\n    shardID: ShardID,\n    storage: ClientGroupStorage,\n    clientGroupID: string,\n    inspectorDelegate: InspectorDelegate,\n    yieldThresholdMs: number,\n    enablePlanner?: boolean,\n  ) {\n    this.#lc = lc.withContext('clientGroupID', clientGroupID);\n    this.#snapshotter = snapshotter;\n    this.#storage = storage;\n    this.#shardID = shardID;\n    this.#logConfig = logConfig;\n    this.#inspectorDelegate = inspectorDelegate;\n    this.#costModels = enablePlanner ? new WeakMap() : undefined;\n    this.#yieldThresholdMs = yieldThresholdMs;\n  }\n\n  /**\n   * Initializes the PipelineDriver to the current head of the database.\n   * Queries can then be added (i.e. hydrated) with {@link addQuery()}.\n   *\n   * Must only be called once.\n   */\n  init(clientSchema: ClientSchema) {\n    assert(!this.#snapshotter.initialized(), 'Already initialized');\n    this.#snapshotter.init();\n    this.#initAndResetCommon(clientSchema);\n  }\n\n  /**\n   * @returns Whether the PipelineDriver has been initialized.\n   */\n  initialized(): boolean {\n    return this.#snapshotter.initialized();\n  }\n\n  /**\n   * Clears the current pipelines and TableSources, returning the PipelineDriver\n   * to its initial state. This should be called in response to a schema change,\n   * as TableSources need to be recomputed.\n   */\n  reset(clientSchema: ClientSchema) {\n    for (const {input} of this.#pipelines.values()) {\n      input.destroy();\n    }\n    this.#pipelines.clear();\n    this.#tables.clear();\n    this.#initAndResetCommon(clientSchema);\n  }\n\n  #initAndResetCommon(clientSchema: ClientSchema) {\n    const {db} = this.#snapshotter.current();\n    const fullTables = new Map<string, LiteTableSpec>();\n    computeZqlSpecs(this.#lc, db.db, this.#tableSpecs, fullTables);\n    checkClientSchema(\n      this.#shardID,\n      clientSchema,\n      this.#tableSpecs,\n      fullTables,\n    );\n    const primaryKeys = this.#primaryKeys ?? new Map<string, PrimaryKey>();\n    this.#primaryKeys = primaryKeys;\n    primaryKeys.clear();\n    for (const [table, spec] of this.#tableSpecs.entries()) {\n      if (table.startsWith(upstreamSchema(this.#shardID))) {\n        primaryKeys.set(table, spec.tableSpec.primaryKey);\n      }\n    }\n    buildPrimaryKeys(clientSchema, primaryKeys);\n    const {replicaVersion} = getSubscriptionState(db);\n    this.#replicaVersion = replicaVersion;\n  }\n\n  /** @returns The replica version. The PipelineDriver must have been initialized. */\n  get replicaVersion(): string {\n    return must(this.#replicaVersion, 'Not yet initialized');\n  }\n\n  /**\n   * Returns the current version of the database. This will reflect the\n   * latest version change when calling {@link advance()} once the\n   * iteration has begun.\n   */\n  currentVersion(): string {\n    assert(this.initialized(), 'Not yet initialized');\n    return this.#snapshotter.current().version;\n  }\n\n  /**\n   * Returns the current supported schema version range of the database.  This\n   * will reflect changes to supported schema version range when calling\n   * {@link advance()} once the iteration has begun.\n   */\n  currentSchemaVersions(): SchemaVersions {\n    assert(this.initialized(), 'Not yet initialized');\n    return this.#snapshotter.current().schemaVersions;\n  }\n\n  /**\n   * Returns the current upstream {app}.permissions, or `null` if none are defined.\n   */\n  currentPermissions(): LoadedPermissions | null {\n    assert(this.initialized(), 'Not yet initialized');\n    const res = reloadPermissionsIfChanged(\n      this.#lc,\n      this.#snapshotter.current().db,\n      this.#shardID.appID,\n      this.#permissions,\n    );\n    if (res.changed) {\n      this.#permissions = res.permissions;\n      this.#lc.debug?.(\n        'Reloaded permissions',\n        JSON.stringify(this.#permissions),\n      );\n    }\n    return this.#permissions;\n  }\n\n  advanceWithoutDiff(): string {\n    const {db, version} = this.#snapshotter.advanceWithoutDiff().curr;\n    for (const table of this.#tables.values()) {\n      table.setDB(db.db);\n    }\n    return version;\n  }\n\n  #ensureCostModelExistsIfEnabled(db: Database) {\n    let existing = this.#costModels?.get(db);\n    if (existing) {\n      return existing;\n    }\n    if (this.#costModels) {\n      const costModel = createSQLiteCostModel(db, this.#tableSpecs);\n      this.#costModels.set(db, costModel);\n      return costModel;\n    }\n    return undefined;\n  }\n\n  /**\n   * Clears storage used for the pipelines. Call this when the\n   * PipelineDriver will no longer be used.\n   */\n  destroy() {\n    this.#storage.destroy();\n    this.#snapshotter.destroy();\n  }\n\n  /** @return The Set of query hashes for all added queries. */\n  addedQueries(): [\n    transformationHashes: Set<string>,\n    byOriginalHash: Map<\n      string,\n      {\n        transformationHash: string;\n        transformedAst: AST;\n      }[]\n    >,\n  ] {\n    const byOriginalHash = new Map<\n      string,\n      {transformationHash: string; transformedAst: AST}[]\n    >();\n    for (const pipeline of this.#pipelines.values()) {\n      const {originalHash, transformedAst, transformationHash} = pipeline;\n\n      if (!byOriginalHash.has(originalHash)) {\n        byOriginalHash.set(originalHash, []);\n      }\n      byOriginalHash.get(originalHash)!.push({\n        transformationHash,\n        transformedAst,\n      });\n    }\n    return [new Set(this.#pipelines.keys()), byOriginalHash];\n  }\n\n  totalHydrationTimeMs(): number {\n    let total = 0;\n    for (const pipeline of this.#pipelines.values()) {\n      total += pipeline.hydrationTimeMs;\n    }\n    return total;\n  }\n\n  /**\n   * Adds a pipeline for the query. The method will hydrate the query using the\n   * driver's current snapshot of the database and return a stream of results.\n   * Henceforth, updates to the query will be returned when the driver is\n   * {@link advance}d. The query and its pipeline can be removed with\n   * {@link removeQuery()}.\n   *\n   * If a query with an identical hash has already been added, this method is a\n   * no-op and no RowChanges are generated.\n   *\n   * @param timer The caller-controlled {@link Timer} used to determine the\n   *        final hydration time. (The caller may pause and resume the timer\n   *        when yielding the thread for time-slicing).\n   * @return The rows from the initial hydration of the query.\n   */\n  *addQuery(\n    transformationHash: string,\n    queryID: string,\n    query: AST,\n    timer: Timer,\n  ): Iterable<RowChange | 'yield'> {\n    assert(this.initialized());\n    this.#inspectorDelegate.addQuery(transformationHash, queryID, query);\n    if (this.#pipelines.has(transformationHash)) {\n      this.#lc.info?.(`query ${transformationHash} already added`, query);\n      return;\n    }\n    const debugDelegate = runtimeDebugFlags.trackRowsVended\n      ? new Debug()\n      : undefined;\n\n    const costModel = this.#ensureCostModelExistsIfEnabled(\n      this.#snapshotter.current().db.db,\n    );\n\n    const input = buildPipeline(\n      query,\n      {\n        debug: debugDelegate,\n        enableNotExists: true, // Server-side can handle NOT EXISTS\n        getSource: name => this.#getSource(name),\n        createStorage: () => this.#createStorage(),\n        decorateSourceInput: (input: SourceInput, _queryID: string): Input =>\n          new MeasurePushOperator(\n            input,\n            transformationHash,\n            this.#inspectorDelegate,\n            'query-update-server',\n          ),\n        decorateInput: input => input,\n        addEdge() {},\n        decorateFilterInput: input => input,\n      },\n      queryID,\n      costModel,\n    );\n    const schema = input.getSchema();\n    input.setOutput({\n      push: change => {\n        const streamer = this.#streamer;\n        assert(streamer, 'must #startAccumulating() before pushing changes');\n        streamer.accumulate(transformationHash, schema, [change]);\n        return [];\n      },\n    });\n\n    assert(this.#advanceContext === null);\n    this.#hydrateContext = {\n      timer,\n    };\n    try {\n      yield* hydrateInternal(\n        input,\n        transformationHash,\n        must(this.#primaryKeys),\n      );\n    } finally {\n      this.#hydrateContext = null;\n    }\n\n    const hydrationTimeMs = timer.totalElapsed();\n    if (runtimeDebugFlags.trackRowCountsVended) {\n      if (hydrationTimeMs > this.#logConfig.slowHydrateThreshold) {\n        let totalRowsConsidered = 0;\n        const lc = this.#lc\n          .withContext('hash', transformationHash)\n          .withContext('hydrationTimeMs', hydrationTimeMs);\n        for (const tableName of this.#tables.keys()) {\n          const entries = Object.entries(\n            debugDelegate?.getVendedRowCounts()[tableName] ?? {},\n          );\n          totalRowsConsidered += entries.reduce(\n            (acc, entry) => acc + entry[1],\n            0,\n          );\n          lc.info?.(tableName + ' VENDED: ', entries);\n        }\n        lc.info?.(`Total rows considered: ${totalRowsConsidered}`);\n      }\n    }\n    debugDelegate?.reset();\n\n    // Note: This hydrationTime is a wall-clock overestimate, as it does\n    // not take time slicing into account. The view-syncer resets this\n    // to a more precise processing-time measurement with setHydrationTime().\n    this.#pipelines.set(transformationHash, {\n      input,\n      hydrationTimeMs,\n      originalHash: queryID,\n      transformedAst: query,\n      transformationHash,\n    });\n  }\n\n  /**\n   * Removes the pipeline for the query. This is a no-op if the query\n   * was not added.\n   */\n  removeQuery(hash: string) {\n    const pipeline = this.#pipelines.get(hash);\n    if (pipeline) {\n      this.#pipelines.delete(hash);\n      pipeline.input.destroy();\n    }\n  }\n\n  /**\n   * Returns the value of the row with the given primary key `pk`,\n   * or `undefined` if there is no such row. The pipeline must have been\n   * initialized.\n   */\n  getRow(table: string, pk: RowKey): Row | undefined {\n    assert(this.initialized(), 'Not yet initialized');\n    const source = must(this.#tables.get(table));\n    return source.getRow(pk as Row);\n  }\n\n  /**\n   * Advances to the new head of the database.\n   *\n   * @param timer The caller-controlled {@link Timer} that will be used to\n   *        measure the progress of the advancement and abort with a\n   *        {@link ResetPipelinesSignal} if it is estimated to take longer\n   *        than a hydration.\n   * @return The resulting row changes for all added queries. Note that the\n   *         `changes` must be iterated over in their entirety in order to\n   *         advance the database snapshot.\n   */\n  advance(timer: Timer): {\n    version: string;\n    numChanges: number;\n    changes: Iterable<RowChange | 'yield'>;\n  } {\n    assert(this.initialized());\n    const diff = this.#snapshotter.advance(this.#tableSpecs);\n    const {prev, curr, changes} = diff;\n    this.#lc.debug?.(\n      `advance ${prev.version} => ${curr.version}: ${changes} changes`,\n    );\n\n    return {\n      version: curr.version,\n      numChanges: changes,\n      changes: this.#advance(diff, timer, changes),\n    };\n  }\n\n  *#advance(\n    diff: SnapshotDiff,\n    timer: Timer,\n    numChanges: number,\n  ): Iterable<RowChange | 'yield'> {\n    assert(this.#hydrateContext === null);\n    this.#advanceContext = {\n      timer,\n      totalHydrationTimeMs: this.totalHydrationTimeMs(),\n      numChanges,\n      pos: 0,\n    };\n    try {\n      for (const {table, prevValues, nextValue} of diff) {\n        // Advance progress is checked each time a row is fetched\n        // from a TableSource during push processing, but some pushes\n        // don't read any rows.  Check progress here before processing\n        // the next change.\n        if (this.#shouldAdvanceYieldMaybeAbortAdvance()) {\n          yield 'yield';\n        }\n        const start = performance.now();\n        let type;\n        try {\n          const tableSource = this.#tables.get(table);\n          if (!tableSource) {\n            // no pipelines read from this table, so no need to process the change\n            continue;\n          }\n          const primaryKey = mustGetPrimaryKey(this.#primaryKeys, table);\n          let editOldRow: Row | undefined = undefined;\n          for (const prevValue of prevValues) {\n            if (\n              nextValue &&\n              deepEqual(\n                getRowKey(primaryKey, prevValue as Row) as JSONValue,\n                getRowKey(primaryKey, nextValue as Row) as JSONValue,\n              )\n            ) {\n              editOldRow = prevValue;\n            } else {\n              if (nextValue) {\n                this.#conflictRowsDeleted.add(1);\n              }\n              yield* this.#push(tableSource, {\n                type: 'remove',\n                row: prevValue,\n              });\n            }\n          }\n          if (nextValue) {\n            if (editOldRow) {\n              yield* this.#push(tableSource, {\n                type: 'edit',\n                row: nextValue,\n                oldRow: editOldRow,\n              });\n            } else {\n              yield* this.#push(tableSource, {\n                type: 'add',\n                row: nextValue,\n              });\n            }\n          }\n        } finally {\n          this.#advanceContext.pos++;\n        }\n\n        const elapsed = performance.now() - start;\n        this.#advanceTime.record(elapsed / 1000, {\n          table,\n          type,\n        });\n      }\n\n      // Set the new snapshot on all TableSources.\n      const {curr} = diff;\n      for (const table of this.#tables.values()) {\n        table.setDB(curr.db.db);\n      }\n      this.#ensureCostModelExistsIfEnabled(curr.db.db);\n      this.#lc.debug?.(`Advanced to ${curr.version}`);\n    } finally {\n      this.#advanceContext = null;\n    }\n  }\n\n  /** Implements `BuilderDelegate.getSource()` */\n  #getSource(tableName: string): Source {\n    let source = this.#tables.get(tableName);\n    if (source) {\n      return source;\n    }\n\n    const tableSpec = mustGetTableSpec(this.#tableSpecs, tableName);\n    const primaryKey = mustGetPrimaryKey(this.#primaryKeys, tableName);\n\n    const {db} = this.#snapshotter.current();\n    source = new TableSource(\n      this.#lc,\n      this.#logConfig,\n      db.db,\n      tableName,\n      tableSpec.zqlSpec,\n      primaryKey,\n      () => this.#shouldYield(),\n    );\n    this.#tables.set(tableName, source);\n    this.#lc.debug?.(`created TableSource for ${tableName}`);\n    return source;\n  }\n\n  #shouldYield(): boolean {\n    if (this.#hydrateContext) {\n      return this.#hydrateContext.timer.elapsedLap() > this.#yieldThresholdMs;\n    }\n    if (this.#advanceContext) {\n      return this.#shouldAdvanceYieldMaybeAbortAdvance();\n    }\n    throw new Error('shouldYield called outside of hydration or advancement');\n  }\n\n  /**\n   * Cancel the advancement processing, by throwing a ResetPipelinesSignal, if\n   * it has taken longer than half the total hydration time to make it through\n   * half of the advancement, or if processing time exceeds total hydration\n   * time.  This serves as both a circuit breaker for very large transactions,\n   * as well as a bound on the amount of time the previous connection locks\n   * the inactive WAL file (as the lock prevents WAL2 from switching to the\n   * free WAL when the current one is over the size limit, which can make\n   * the WAL grow continuously and compound slowness).\n   * This is checked:\n   * 1. before starting to process each change in an advancement is processed\n   * 2. whenever a row is fetched from a TableSource during push processing\n   */\n  #shouldAdvanceYieldMaybeAbortAdvance(): boolean {\n    const {\n      pos,\n      numChanges,\n      timer: advanceTimer,\n      totalHydrationTimeMs,\n    } = must(this.#advanceContext);\n    const elapsed = advanceTimer.totalElapsed();\n    if (\n      elapsed > MIN_ADVANCEMENT_TIME_LIMIT_MS &&\n      (elapsed > totalHydrationTimeMs ||\n        (elapsed > totalHydrationTimeMs / 2 && pos <= numChanges / 2))\n    ) {\n      throw new ResetPipelinesSignal(\n        `Advancement exceeded timeout at ${pos} of ${numChanges} changes ` +\n          `after ${elapsed} ms. Advancement time limited based on total ` +\n          `hydration time of ${totalHydrationTimeMs} ms.`,\n      );\n    }\n    return advanceTimer.elapsedLap() > this.#yieldThresholdMs;\n  }\n\n  /** Implements `BuilderDelegate.createStorage()` */\n  #createStorage(): Storage {\n    return this.#storage.createStorage();\n  }\n\n  *#push(\n    source: TableSource,\n    change: SourceChange,\n  ): Iterable<RowChange | 'yield'> {\n    this.#startAccumulating();\n    try {\n      for (const val of source.genPush(change)) {\n        if (val === 'yield') {\n          yield 'yield';\n        }\n        for (const changeOrYield of this.#stopAccumulating().stream()) {\n          yield changeOrYield;\n        }\n        this.#startAccumulating();\n      }\n    } finally {\n      if (this.#streamer !== null) {\n        this.#stopAccumulating();\n      }\n    }\n  }\n\n  #startAccumulating() {\n    assert(this.#streamer === null);\n    this.#streamer = new Streamer(must(this.#primaryKeys));\n  }\n\n  #stopAccumulating(): Streamer {\n    const streamer = this.#streamer;\n    assert(streamer);\n    this.#streamer = null;\n    return streamer;\n  }\n}\n\nclass Streamer {\n  readonly #primaryKeys: Map<string, PrimaryKey>;\n\n  constructor(primaryKeys: Map<string, PrimaryKey>) {\n    this.#primaryKeys = primaryKeys;\n  }\n\n  readonly #changes: [\n    hash: string,\n    schema: SourceSchema,\n    changes: Iterable<Change | 'yield'>,\n  ][] = [];\n\n  accumulate(\n    hash: string,\n    schema: SourceSchema,\n    changes: Iterable<Change | 'yield'>,\n  ): this {\n    this.#changes.push([hash, schema, changes]);\n    return this;\n  }\n\n  *stream(): Iterable<RowChange | 'yield'> {\n    for (const [hash, schema, changes] of this.#changes) {\n      yield* this.#streamChanges(hash, schema, changes);\n    }\n  }\n\n  *#streamChanges(\n    queryHash: string,\n    schema: SourceSchema,\n    changes: Iterable<Change | 'yield'>,\n  ): Iterable<RowChange | 'yield'> {\n    // We do not sync rows gathered by the permissions\n    // system to the client.\n    if (schema.system === 'permissions') {\n      return;\n    }\n\n    for (const change of changes) {\n      if (change === 'yield') {\n        yield change;\n        continue;\n      }\n      const {type} = change;\n\n      switch (type) {\n        case 'add':\n        case 'remove': {\n          yield* this.#streamNodes(queryHash, schema, type, () => [\n            change.node,\n          ]);\n          break;\n        }\n        case 'child': {\n          const {child} = change;\n          const childSchema = must(\n            schema.relationships[child.relationshipName],\n          );\n\n          yield* this.#streamChanges(queryHash, childSchema, [child.change]);\n          break;\n        }\n        case 'edit':\n          yield* this.#streamNodes(queryHash, schema, type, () => [\n            {row: change.node.row, relationships: {}},\n          ]);\n          break;\n        default:\n          unreachable(type);\n      }\n    }\n  }\n\n  *#streamNodes(\n    queryHash: string,\n    schema: SourceSchema,\n    op: 'add' | 'remove' | 'edit',\n    nodes: () => Iterable<Node | 'yield'>,\n  ): Iterable<RowChange | 'yield'> {\n    const {tableName: table, system} = schema;\n\n    const primaryKey = must(this.#primaryKeys.get(table));\n\n    // We do not sync rows gathered by the permissions\n    // system to the client.\n    if (system === 'permissions') {\n      return;\n    }\n\n    for (const node of nodes()) {\n      if (node === 'yield') {\n        yield node;\n        continue;\n      }\n      const {relationships, row} = node;\n      const rowKey = getRowKey(primaryKey, row);\n\n      yield {\n        type: op,\n        queryHash,\n        table,\n        rowKey,\n        row: op === 'remove' ? undefined : row,\n      } as RowChange;\n\n      for (const [relationship, children] of Object.entries(relationships)) {\n        const childSchema = must(schema.relationships[relationship]);\n        yield* this.#streamNodes(queryHash, childSchema, op, children);\n      }\n    }\n  }\n}\n\nfunction* toAdds(nodes: Iterable<Node | 'yield'>): Iterable<Change | 'yield'> {\n  for (const node of nodes) {\n    if (node === 'yield') {\n      yield node;\n      continue;\n    }\n    yield {type: 'add', node};\n  }\n}\n\nfunction getRowKey(cols: PrimaryKey, row: Row): RowKey {\n  return Object.fromEntries(cols.map(col => [col, must(row[col])]));\n}\n\n/**\n * Core hydration logic used by {@link PipelineDriver#addQuery}, extracted to a\n * function for reuse by bin-analyze so that bin-analyze's hydration logic\n * is as close as possible to zero-cache's real hydration logic.\n */\nexport function* hydrate(\n  input: Input,\n  hash: string,\n  clientSchema: ClientSchema,\n): Iterable<RowChange | 'yield'> {\n  const res = input.fetch({});\n  const streamer = new Streamer(buildPrimaryKeys(clientSchema)).accumulate(\n    hash,\n    input.getSchema(),\n    toAdds(res),\n  );\n  yield* streamer.stream();\n}\n\nexport function* hydrateInternal(\n  input: Input,\n  hash: string,\n  primaryKeys: Map<string, PrimaryKey>,\n): Iterable<RowChange | 'yield'> {\n  const res = input.fetch({});\n  const streamer = new Streamer(primaryKeys).accumulate(\n    hash,\n    input.getSchema(),\n    toAdds(res),\n  );\n  yield* streamer.stream();\n}\n\nfunction buildPrimaryKeys(\n  clientSchema: ClientSchema,\n  primaryKeys: Map<string, PrimaryKey> = new Map<string, PrimaryKey>(),\n) {\n  for (const [tableName, {primaryKey}] of Object.entries(clientSchema.tables)) {\n    primaryKeys.set(tableName, primaryKey as unknown as PrimaryKey);\n  }\n  return primaryKeys;\n}\n\nfunction mustGetPrimaryKey(\n  primaryKeys: Map<string, PrimaryKey> | null,\n  table: string,\n): PrimaryKey {\n  const pKeys = must(primaryKeys, 'primaryKey map must be non-null');\n\n  return must(\n    pKeys.get(table),\n    `table '${table}' is not one of: ${[...pKeys.keys()].sort()}. ` +\n      `Check the spelling and ensure that the table has a primary key.`,\n  );\n}\n"],"names":["input"],"mappings":";;;;;;;;;;;;;;;;;AAsGA,MAAM,gCAAgC;AAK/B,MAAM,eAAe;AAAA,EACjB,8BAAc,IAAA;AAAA;AAAA;AAAA;AAAA,EAId,iCAAiB,IAAA;AAAA,EAEjB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,kCAAkB,IAAA;AAAA,EAClB;AAAA,EACA;AAAA,EACT,YAA6B;AAAA,EAC7B,kBAAyC;AAAA,EACzC,kBAAyC;AAAA,EACzC,kBAAiC;AAAA,EACjC,eAA+C;AAAA,EAC/C,eAAyC;AAAA,EAEhC,eAAe,qBAAqB,QAAQ,oBAAoB;AAAA,IACvE,aACE;AAAA,IACF,MAAM;AAAA,EAAA,CACP;AAAA,EAEQ,uBAAuB;AAAA,IAC9B;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAAA,EAGO;AAAA,EAET,YACE,IACA,WACA,aACA,SACA,SACA,eACA,mBACA,kBACA,eACA;AACA,SAAK,MAAM,GAAG,YAAY,iBAAiB,aAAa;AACxD,SAAK,eAAe;AACpB,SAAK,WAAW;AAChB,SAAK,WAAW;AAChB,SAAK,aAAa;AAClB,SAAK,qBAAqB;AAC1B,SAAK,cAAc,gBAAgB,oBAAI,QAAA,IAAY;AACnD,SAAK,oBAAoB;AAAA,EAC3B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,KAAK,cAA4B;AAC/B,WAAO,CAAC,KAAK,aAAa,YAAA,GAAe,qBAAqB;AAC9D,SAAK,aAAa,KAAA;AAClB,SAAK,oBAAoB,YAAY;AAAA,EACvC;AAAA;AAAA;AAAA;AAAA,EAKA,cAAuB;AACrB,WAAO,KAAK,aAAa,YAAA;AAAA,EAC3B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,cAA4B;AAChC,eAAW,EAAC,MAAA,KAAU,KAAK,WAAW,UAAU;AAC9C,YAAM,QAAA;AAAA,IACR;AACA,SAAK,WAAW,MAAA;AAChB,SAAK,QAAQ,MAAA;AACb,SAAK,oBAAoB,YAAY;AAAA,EACvC;AAAA,EAEA,oBAAoB,cAA4B;AAC9C,UAAM,EAAC,GAAA,IAAM,KAAK,aAAa,QAAA;AAC/B,UAAM,iCAAiB,IAAA;AACvB,oBAAgB,KAAK,KAAK,GAAG,IAAI,KAAK,aAAa,UAAU;AAC7D;AAAA,MACE,KAAK;AAAA,MACL;AAAA,MACA,KAAK;AAAA,MACL;AAAA,IAAA;AAEF,UAAM,cAAc,KAAK,gBAAgB,oBAAI,IAAA;AAC7C,SAAK,eAAe;AACpB,gBAAY,MAAA;AACZ,eAAW,CAAC,OAAO,IAAI,KAAK,KAAK,YAAY,WAAW;AACtD,UAAI,MAAM,WAAW,eAAe,KAAK,QAAQ,CAAC,GAAG;AACnD,oBAAY,IAAI,OAAO,KAAK,UAAU,UAAU;AAAA,MAClD;AAAA,IACF;AACA,qBAAiB,cAAc,WAAW;AAC1C,UAAM,EAAC,eAAA,IAAkB,qBAAqB,EAAE;AAChD,SAAK,kBAAkB;AAAA,EACzB;AAAA;AAAA,EAGA,IAAI,iBAAyB;AAC3B,WAAO,KAAK,KAAK,iBAAiB,qBAAqB;AAAA,EACzD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,iBAAyB;AACvB,WAAO,KAAK,YAAA,GAAe,qBAAqB;AAChD,WAAO,KAAK,aAAa,QAAA,EAAU;AAAA,EACrC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,wBAAwC;AACtC,WAAO,KAAK,YAAA,GAAe,qBAAqB;AAChD,WAAO,KAAK,aAAa,QAAA,EAAU;AAAA,EACrC;AAAA;AAAA;AAAA;AAAA,EAKA,qBAA+C;AAC7C,WAAO,KAAK,YAAA,GAAe,qBAAqB;AAChD,UAAM,MAAM;AAAA,MACV,KAAK;AAAA,MACL,KAAK,aAAa,QAAA,EAAU;AAAA,MAC5B,KAAK,SAAS;AAAA,MACd,KAAK;AAAA,IAAA;AAEP,QAAI,IAAI,SAAS;AACf,WAAK,eAAe,IAAI;AACxB,WAAK,IAAI;AAAA,QACP;AAAA,QACA,KAAK,UAAU,KAAK,YAAY;AAAA,MAAA;AAAA,IAEpC;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,qBAA6B;AAC3B,UAAM,EAAC,IAAI,QAAA,IAAW,KAAK,aAAa,qBAAqB;AAC7D,eAAW,SAAS,KAAK,QAAQ,OAAA,GAAU;AACzC,YAAM,MAAM,GAAG,EAAE;AAAA,IACnB;AACA,WAAO;AAAA,EACT;AAAA,EAEA,gCAAgC,IAAc;AAC5C,QAAI,WAAW,KAAK,aAAa,IAAI,EAAE;AACvC,QAAI,UAAU;AACZ,aAAO;AAAA,IACT;AACA,QAAI,KAAK,aAAa;AACpB,YAAM,YAAY,sBAAsB,IAAI,KAAK,WAAW;AAC5D,WAAK,YAAY,IAAI,IAAI,SAAS;AAClC,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,UAAU;AACR,SAAK,SAAS,QAAA;AACd,SAAK,aAAa,QAAA;AAAA,EACpB;AAAA;AAAA,EAGA,eASE;AACA,UAAM,qCAAqB,IAAA;AAI3B,eAAW,YAAY,KAAK,WAAW,OAAA,GAAU;AAC/C,YAAM,EAAC,cAAc,gBAAgB,mBAAA,IAAsB;AAE3D,UAAI,CAAC,eAAe,IAAI,YAAY,GAAG;AACrC,uBAAe,IAAI,cAAc,EAAE;AAAA,MACrC;AACA,qBAAe,IAAI,YAAY,EAAG,KAAK;AAAA,QACrC;AAAA,QACA;AAAA,MAAA,CACD;AAAA,IACH;AACA,WAAO,CAAC,IAAI,IAAI,KAAK,WAAW,KAAA,CAAM,GAAG,cAAc;AAAA,EACzD;AAAA,EAEA,uBAA+B;AAC7B,QAAI,QAAQ;AACZ,eAAW,YAAY,KAAK,WAAW,OAAA,GAAU;AAC/C,eAAS,SAAS;AAAA,IACpB;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAiBA,CAAC,SACC,oBACA,SACA,OACA,OAC+B;AAC/B,WAAO,KAAK,aAAa;AACzB,SAAK,mBAAmB,SAAS,oBAAoB,SAAS,KAAK;AACnE,QAAI,KAAK,WAAW,IAAI,kBAAkB,GAAG;AAC3C,WAAK,IAAI,OAAO,SAAS,kBAAkB,kBAAkB,KAAK;AAClE;AAAA,IACF;AACA,UAAM,gBAAgB,kBAAkB,kBACpC,IAAI,UACJ;AAEJ,UAAM,YAAY,KAAK;AAAA,MACrB,KAAK,aAAa,QAAA,EAAU,GAAG;AAAA,IAAA;AAGjC,UAAM,QAAQ;AAAA,MACZ;AAAA,MACA;AAAA,QACE,OAAO;AAAA,QACP,iBAAiB;AAAA;AAAA,QACjB,WAAW,CAAA,SAAQ,KAAK,WAAW,IAAI;AAAA,QACvC,eAAe,MAAM,KAAK,eAAA;AAAA,QAC1B,qBAAqB,CAACA,QAAoB,aACxC,IAAI;AAAA,UACFA;AAAAA,UACA;AAAA,UACA,KAAK;AAAA,UACL;AAAA,QAAA;AAAA,QAEJ,eAAe,CAAAA,WAASA;AAAAA,QACxB,UAAU;AAAA,QAAC;AAAA,QACX,qBAAqB,CAAAA,WAASA;AAAAA,MAAA;AAAA,MAEhC;AAAA,MACA;AAAA,IAAA;AAEF,UAAM,SAAS,MAAM,UAAA;AACrB,UAAM,UAAU;AAAA,MACd,MAAM,CAAA,WAAU;AACd,cAAM,WAAW,KAAK;AACtB,eAAO,UAAU,kDAAkD;AACnE,iBAAS,WAAW,oBAAoB,QAAQ,CAAC,MAAM,CAAC;AACxD,eAAO,CAAA;AAAA,MACT;AAAA,IAAA,CACD;AAED,WAAO,KAAK,oBAAoB,IAAI;AACpC,SAAK,kBAAkB;AAAA,MACrB;AAAA,IAAA;AAEF,QAAI;AACF,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA,KAAK,KAAK,YAAY;AAAA,MAAA;AAAA,IAE1B,UAAA;AACE,WAAK,kBAAkB;AAAA,IACzB;AAEA,UAAM,kBAAkB,MAAM,aAAA;AAC9B,QAAI,kBAAkB,sBAAsB;AAC1C,UAAI,kBAAkB,KAAK,WAAW,sBAAsB;AAC1D,YAAI,sBAAsB;AAC1B,cAAM,KAAK,KAAK,IACb,YAAY,QAAQ,kBAAkB,EACtC,YAAY,mBAAmB,eAAe;AACjD,mBAAW,aAAa,KAAK,QAAQ,KAAA,GAAQ;AAC3C,gBAAM,UAAU,OAAO;AAAA,YACrB,eAAe,qBAAqB,SAAS,KAAK,CAAA;AAAA,UAAC;AAErD,iCAAuB,QAAQ;AAAA,YAC7B,CAAC,KAAK,UAAU,MAAM,MAAM,CAAC;AAAA,YAC7B;AAAA,UAAA;AAEF,aAAG,OAAO,YAAY,aAAa,OAAO;AAAA,QAC5C;AACA,WAAG,OAAO,0BAA0B,mBAAmB,EAAE;AAAA,MAC3D;AAAA,IACF;AACA,mBAAe,MAAA;AAKf,SAAK,WAAW,IAAI,oBAAoB;AAAA,MACtC;AAAA,MACA;AAAA,MACA,cAAc;AAAA,MACd,gBAAgB;AAAA,MAChB;AAAA,IAAA,CACD;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,YAAY,MAAc;AACxB,UAAM,WAAW,KAAK,WAAW,IAAI,IAAI;AACzC,QAAI,UAAU;AACZ,WAAK,WAAW,OAAO,IAAI;AAC3B,eAAS,MAAM,QAAA;AAAA,IACjB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,OAAO,OAAe,IAA6B;AACjD,WAAO,KAAK,YAAA,GAAe,qBAAqB;AAChD,UAAM,SAAS,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC;AAC3C,WAAO,OAAO,OAAO,EAAS;AAAA,EAChC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAaA,QAAQ,OAIN;AACA,WAAO,KAAK,aAAa;AACzB,UAAM,OAAO,KAAK,aAAa,QAAQ,KAAK,WAAW;AACvD,UAAM,EAAC,MAAM,MAAM,QAAA,IAAW;AAC9B,SAAK,IAAI;AAAA,MACP,WAAW,KAAK,OAAO,OAAO,KAAK,OAAO,KAAK,OAAO;AAAA,IAAA;AAGxD,WAAO;AAAA,MACL,SAAS,KAAK;AAAA,MACd,YAAY;AAAA,MACZ,SAAS,KAAK,SAAS,MAAM,OAAO,OAAO;AAAA,IAAA;AAAA,EAE/C;AAAA,EAEA,CAAC,SACC,MACA,OACA,YAC+B;AAC/B,WAAO,KAAK,oBAAoB,IAAI;AACpC,SAAK,kBAAkB;AAAA,MACrB;AAAA,MACA,sBAAsB,KAAK,qBAAA;AAAA,MAC3B;AAAA,MACA,KAAK;AAAA,IAAA;AAEP,QAAI;AACF,iBAAW,EAAC,OAAO,YAAY,UAAA,KAAc,MAAM;AAKjD,YAAI,KAAK,wCAAwC;AAC/C,gBAAM;AAAA,QACR;AACA,cAAM,QAAQ,YAAY,IAAA;AAC1B,YAAI;AACJ,YAAI;AACF,gBAAM,cAAc,KAAK,QAAQ,IAAI,KAAK;AAC1C,cAAI,CAAC,aAAa;AAEhB;AAAA,UACF;AACA,gBAAM,aAAa,kBAAkB,KAAK,cAAc,KAAK;AAC7D,cAAI,aAA8B;AAClC,qBAAW,aAAa,YAAY;AAClC,gBACE,aACA;AAAA,cACE,UAAU,YAAY,SAAgB;AAAA,cACtC,UAAU,YAAY,SAAgB;AAAA,YAAA,GAExC;AACA,2BAAa;AAAA,YACf,OAAO;AACL,kBAAI,WAAW;AACb,qBAAK,qBAAqB,IAAI,CAAC;AAAA,cACjC;AACA,qBAAO,KAAK,MAAM,aAAa;AAAA,gBAC7B,MAAM;AAAA,gBACN,KAAK;AAAA,cAAA,CACN;AAAA,YACH;AAAA,UACF;AACA,cAAI,WAAW;AACb,gBAAI,YAAY;AACd,qBAAO,KAAK,MAAM,aAAa;AAAA,gBAC7B,MAAM;AAAA,gBACN,KAAK;AAAA,gBACL,QAAQ;AAAA,cAAA,CACT;AAAA,YACH,OAAO;AACL,qBAAO,KAAK,MAAM,aAAa;AAAA,gBAC7B,MAAM;AAAA,gBACN,KAAK;AAAA,cAAA,CACN;AAAA,YACH;AAAA,UACF;AAAA,QACF,UAAA;AACE,eAAK,gBAAgB;AAAA,QACvB;AAEA,cAAM,UAAU,YAAY,IAAA,IAAQ;AACpC,aAAK,aAAa,OAAO,UAAU,KAAM;AAAA,UACvC;AAAA,UACA;AAAA,QAAA,CACD;AAAA,MACH;AAGA,YAAM,EAAC,SAAQ;AACf,iBAAW,SAAS,KAAK,QAAQ,OAAA,GAAU;AACzC,cAAM,MAAM,KAAK,GAAG,EAAE;AAAA,MACxB;AACA,WAAK,gCAAgC,KAAK,GAAG,EAAE;AAC/C,WAAK,IAAI,QAAQ,eAAe,KAAK,OAAO,EAAE;AAAA,IAChD,UAAA;AACE,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA;AAAA,EAGA,WAAW,WAA2B;AACpC,QAAI,SAAS,KAAK,QAAQ,IAAI,SAAS;AACvC,QAAI,QAAQ;AACV,aAAO;AAAA,IACT;AAEA,UAAM,YAAY,iBAAiB,KAAK,aAAa,SAAS;AAC9D,UAAM,aAAa,kBAAkB,KAAK,cAAc,SAAS;AAEjE,UAAM,EAAC,GAAA,IAAM,KAAK,aAAa,QAAA;AAC/B,aAAS,IAAI;AAAA,MACX,KAAK;AAAA,MACL,KAAK;AAAA,MACL,GAAG;AAAA,MACH;AAAA,MACA,UAAU;AAAA,MACV;AAAA,MACA,MAAM,KAAK,aAAA;AAAA,IAAa;AAE1B,SAAK,QAAQ,IAAI,WAAW,MAAM;AAClC,SAAK,IAAI,QAAQ,2BAA2B,SAAS,EAAE;AACvD,WAAO;AAAA,EACT;AAAA,EAEA,eAAwB;AACtB,QAAI,KAAK,iBAAiB;AACxB,aAAO,KAAK,gBAAgB,MAAM,WAAA,IAAe,KAAK;AAAA,IACxD;AACA,QAAI,KAAK,iBAAiB;AACxB,aAAO,KAAK,qCAAA;AAAA,IACd;AACA,UAAM,IAAI,MAAM,wDAAwD;AAAA,EAC1E;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAeA,uCAAgD;AAC9C,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA,OAAO;AAAA,MACP;AAAA,IAAA,IACE,KAAK,KAAK,eAAe;AAC7B,UAAM,UAAU,aAAa,aAAA;AAC7B,QACE,UAAU,kCACT,UAAU,wBACR,UAAU,uBAAuB,KAAK,OAAO,aAAa,IAC7D;AACA,YAAM,IAAI;AAAA,QACR,mCAAmC,GAAG,OAAO,UAAU,kBAC5C,OAAO,kEACK,oBAAoB;AAAA,MAAA;AAAA,IAE/C;AACA,WAAO,aAAa,eAAe,KAAK;AAAA,EAC1C;AAAA;AAAA,EAGA,iBAA0B;AACxB,WAAO,KAAK,SAAS,cAAA;AAAA,EACvB;AAAA,EAEA,CAAC,MACC,QACA,QAC+B;AAC/B,SAAK,mBAAA;AACL,QAAI;AACF,iBAAW,OAAO,OAAO,QAAQ,MAAM,GAAG;AACxC,YAAI,QAAQ,SAAS;AACnB,gBAAM;AAAA,QACR;AACA,mBAAW,iBAAiB,KAAK,kBAAA,EAAoB,UAAU;AAC7D,gBAAM;AAAA,QACR;AACA,aAAK,mBAAA;AAAA,MACP;AAAA,IACF,UAAA;AACE,UAAI,KAAK,cAAc,MAAM;AAC3B,aAAK,kBAAA;AAAA,MACP;AAAA,IACF;AAAA,EACF;AAAA,EAEA,qBAAqB;AACnB,WAAO,KAAK,cAAc,IAAI;AAC9B,SAAK,YAAY,IAAI,SAAS,KAAK,KAAK,YAAY,CAAC;AAAA,EACvD;AAAA,EAEA,oBAA8B;AAC5B,UAAM,WAAW,KAAK;AACtB,WAAO,QAAQ;AACf,SAAK,YAAY;AACjB,WAAO;AAAA,EACT;AACF;AAEA,MAAM,SAAS;AAAA,EACJ;AAAA,EAET,YAAY,aAAsC;AAChD,SAAK,eAAe;AAAA,EACtB;AAAA,EAES,WAIH,CAAA;AAAA,EAEN,WACE,MACA,QACA,SACM;AACN,SAAK,SAAS,KAAK,CAAC,MAAM,QAAQ,OAAO,CAAC;AAC1C,WAAO;AAAA,EACT;AAAA,EAEA,CAAC,SAAwC;AACvC,eAAW,CAAC,MAAM,QAAQ,OAAO,KAAK,KAAK,UAAU;AACnD,aAAO,KAAK,eAAe,MAAM,QAAQ,OAAO;AAAA,IAClD;AAAA,EACF;AAAA,EAEA,CAAC,eACC,WACA,QACA,SAC+B;AAG/B,QAAI,OAAO,WAAW,eAAe;AACnC;AAAA,IACF;AAEA,eAAW,UAAU,SAAS;AAC5B,UAAI,WAAW,SAAS;AACtB,cAAM;AACN;AAAA,MACF;AACA,YAAM,EAAC,SAAQ;AAEf,cAAQ,MAAA;AAAA,QACN,KAAK;AAAA,QACL,KAAK,UAAU;AACb,iBAAO,KAAK,aAAa,WAAW,QAAQ,MAAM,MAAM;AAAA,YACtD,OAAO;AAAA,UAAA,CACR;AACD;AAAA,QACF;AAAA,QACA,KAAK,SAAS;AACZ,gBAAM,EAAC,UAAS;AAChB,gBAAM,cAAc;AAAA,YAClB,OAAO,cAAc,MAAM,gBAAgB;AAAA,UAAA;AAG7C,iBAAO,KAAK,eAAe,WAAW,aAAa,CAAC,MAAM,MAAM,CAAC;AACjE;AAAA,QACF;AAAA,QACA,KAAK;AACH,iBAAO,KAAK,aAAa,WAAW,QAAQ,MAAM,MAAM;AAAA,YACtD,EAAC,KAAK,OAAO,KAAK,KAAK,eAAe,CAAA,EAAC;AAAA,UAAC,CACzC;AACD;AAAA,QACF;AACE,sBAAgB;AAAA,MAAA;AAAA,IAEtB;AAAA,EACF;AAAA,EAEA,CAAC,aACC,WACA,QACA,IACA,OAC+B;AAC/B,UAAM,EAAC,WAAW,OAAO,OAAA,IAAU;AAEnC,UAAM,aAAa,KAAK,KAAK,aAAa,IAAI,KAAK,CAAC;AAIpD,QAAI,WAAW,eAAe;AAC5B;AAAA,IACF;AAEA,eAAW,QAAQ,SAAS;AAC1B,UAAI,SAAS,SAAS;AACpB,cAAM;AACN;AAAA,MACF;AACA,YAAM,EAAC,eAAe,IAAA,IAAO;AAC7B,YAAM,SAAS,UAAU,YAAY,GAAG;AAExC,YAAM;AAAA,QACJ,MAAM;AAAA,QACN;AAAA,QACA;AAAA,QACA;AAAA,QACA,KAAK,OAAO,WAAW,SAAY;AAAA,MAAA;AAGrC,iBAAW,CAAC,cAAc,QAAQ,KAAK,OAAO,QAAQ,aAAa,GAAG;AACpE,cAAM,cAAc,KAAK,OAAO,cAAc,YAAY,CAAC;AAC3D,eAAO,KAAK,aAAa,WAAW,aAAa,IAAI,QAAQ;AAAA,MAC/D;AAAA,IACF;AAAA,EACF;AACF;AAEA,UAAU,OAAO,OAA6D;AAC5E,aAAW,QAAQ,OAAO;AACxB,QAAI,SAAS,SAAS;AACpB,YAAM;AACN;AAAA,IACF;AACA,UAAM,EAAC,MAAM,OAAO,KAAA;AAAA,EACtB;AACF;AAEA,SAAS,UAAU,MAAkB,KAAkB;AACrD,SAAO,OAAO,YAAY,KAAK,IAAI,CAAA,QAAO,CAAC,KAAK,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;AAClE;AAOO,UAAU,QACf,OACA,MACA,cAC+B;AAC/B,QAAM,MAAM,MAAM,MAAM,EAAE;AAC1B,QAAM,WAAW,IAAI,SAAS,iBAAiB,YAAY,CAAC,EAAE;AAAA,IAC5D;AAAA,IACA,MAAM,UAAA;AAAA,IACN,OAAO,GAAG;AAAA,EAAA;AAEZ,SAAO,SAAS,OAAA;AAClB;AAEO,UAAU,gBACf,OACA,MACA,aAC+B;AAC/B,QAAM,MAAM,MAAM,MAAM,EAAE;AAC1B,QAAM,WAAW,IAAI,SAAS,WAAW,EAAE;AAAA,IACzC;AAAA,IACA,MAAM,UAAA;AAAA,IACN,OAAO,GAAG;AAAA,EAAA;AAEZ,SAAO,SAAS,OAAA;AAClB;AAEA,SAAS,iBACP,cACA,cAAuC,oBAAI,OAC3C;AACA,aAAW,CAAC,WAAW,EAAC,WAAA,CAAW,KAAK,OAAO,QAAQ,aAAa,MAAM,GAAG;AAC3E,gBAAY,IAAI,WAAW,UAAmC;AAAA,EAChE;AACA,SAAO;AACT;AAEA,SAAS,kBACP,aACA,OACY;AACZ,QAAM,QAAQ,KAAK,aAAa,iCAAiC;AAEjE,SAAO;AAAA,IACL,MAAM,IAAI,KAAK;AAAA,IACf,UAAU,KAAK,oBAAoB,CAAC,GAAG,MAAM,KAAA,CAAM,EAAE,KAAA,CAAM;AAAA,EAAA;AAG/D;"}
+{"version":3,"file":"pipeline-driver.js","sources":["../../../../../../zero-cache/src/services/view-syncer/pipeline-driver.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport {deepEqual, type JSONValue} from '../../../../shared/src/json.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport type {AST} from '../../../../zero-protocol/src/ast.ts';\nimport type {ClientSchema} from '../../../../zero-protocol/src/client-schema.ts';\nimport type {Row} from '../../../../zero-protocol/src/data.ts';\nimport type {PrimaryKey} from '../../../../zero-protocol/src/primary-key.ts';\nimport {buildPipeline} from '../../../../zql/src/builder/builder.ts';\nimport {\n  Debug,\n  runtimeDebugFlags,\n} from '../../../../zql/src/builder/debug-delegate.ts';\nimport type {Change} from '../../../../zql/src/ivm/change.ts';\nimport type {Node} from '../../../../zql/src/ivm/data.ts';\nimport {type Input, type Storage} from '../../../../zql/src/ivm/operator.ts';\nimport type {SourceSchema} from '../../../../zql/src/ivm/schema.ts';\nimport type {\n  Source,\n  SourceChange,\n  SourceInput,\n} from '../../../../zql/src/ivm/source.ts';\nimport type {ConnectionCostModel} from '../../../../zql/src/planner/planner-connection.ts';\nimport {MeasurePushOperator} from '../../../../zql/src/query/measure-push-operator.ts';\nimport type {ClientGroupStorage} from '../../../../zqlite/src/database-storage.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {createSQLiteCostModel} from '../../../../zqlite/src/sqlite-cost-model.ts';\nimport {TableSource} from '../../../../zqlite/src/table-source.ts';\nimport {\n  reloadPermissionsIfChanged,\n  type LoadedPermissions,\n} from '../../auth/load-permissions.ts';\nimport type {LogConfig} from '../../config/zero-config.ts';\nimport {computeZqlSpecs, mustGetTableSpec} from '../../db/lite-tables.ts';\nimport type {LiteAndZqlSpec, LiteTableSpec} from '../../db/specs.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateHistogram,\n} from '../../observability/metrics.ts';\nimport type {InspectorDelegate} from '../../server/inspector-delegate.ts';\nimport {type RowKey} from '../../types/row-key.ts';\nimport {upstreamSchema, type ShardID} from '../../types/shards.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {checkClientSchema} from './client-schema.ts';\nimport type {Snapshotter} from './snapshotter.ts';\nimport {ResetPipelinesSignal, type SnapshotDiff} from './snapshotter.ts';\n\nexport type RowAdd = {\n  readonly type: 'add';\n  readonly queryHash: string;\n  readonly table: string;\n  readonly rowKey: Row;\n  readonly row: Row;\n};\n\nexport type RowRemove = {\n  readonly type: 'remove';\n  readonly queryHash: string;\n  readonly table: string;\n  readonly rowKey: Row;\n  readonly row: undefined;\n};\n\nexport type RowEdit = {\n  readonly type: 'edit';\n  readonly queryHash: string;\n  readonly table: string;\n  readonly rowKey: Row;\n  readonly row: Row;\n};\n\nexport type RowChange = RowAdd | RowRemove | RowEdit;\n\ntype Pipeline = {\n  readonly input: Input;\n  readonly hydrationTimeMs: number;\n  readonly originalHash: string;\n  readonly transformedAst: AST; // Optional, only set after hydration\n  readonly transformationHash: string; // The hash of the transformed AST\n};\n\ntype AdvanceContext = {\n  readonly timer: Timer;\n  readonly totalHydrationTimeMs: number;\n  readonly numChanges: number;\n  pos: number;\n};\n\ntype HydrateContext = {\n  readonly timer: Timer;\n};\n\nexport type Timer = {\n  elapsedLap: () => number;\n  totalElapsed: () => number;\n};\n\n/**\n * No matter how fast hydration is, advancement is given at least this long to\n * complete before doing a pipeline reset.\n */\nconst MIN_ADVANCEMENT_TIME_LIMIT_MS = 50;\n\n/**\n * Manages the state of IVM pipelines for a given ViewSyncer (i.e. client group).\n */\nexport class PipelineDriver {\n  readonly #tables = new Map<string, TableSource>();\n  // We probs need the original query hash\n  // so we can decide not to re-transform a custom query\n  // that is already hydrated.\n  readonly #pipelines = new Map<string, Pipeline>();\n\n  readonly #lc: LogContext;\n  readonly #snapshotter: Snapshotter;\n  readonly #storage: ClientGroupStorage;\n  readonly #shardID: ShardID;\n  readonly #logConfig: LogConfig;\n  readonly #tableSpecs = new Map<string, LiteAndZqlSpec>();\n  readonly #costModels: WeakMap<Database, ConnectionCostModel> | undefined;\n  readonly #yieldThresholdMs: () => number;\n  #streamer: Streamer | null = null;\n  #hydrateContext: HydrateContext | null = null;\n  #advanceContext: AdvanceContext | null = null;\n  #replicaVersion: string | null = null;\n  #primaryKeys: Map<string, PrimaryKey> | null = null;\n  #permissions: LoadedPermissions | null = null;\n\n  readonly #advanceTime = getOrCreateHistogram('sync', 'ivm.advance-time', {\n    description:\n      'Time to advance all queries for a given client group for in response to a single change.',\n    unit: 's',\n  });\n\n  readonly #conflictRowsDeleted = getOrCreateCounter(\n    'sync',\n    'ivm.conflict-rows-deleted',\n    'Number of rows deleted because they conflicted with added row',\n  );\n\n  readonly #inspectorDelegate: InspectorDelegate;\n\n  constructor(\n    lc: LogContext,\n    logConfig: LogConfig,\n    snapshotter: Snapshotter,\n    shardID: ShardID,\n    storage: ClientGroupStorage,\n    clientGroupID: string,\n    inspectorDelegate: InspectorDelegate,\n    yieldThresholdMs: () => number,\n    enablePlanner?: boolean,\n  ) {\n    this.#lc = lc.withContext('clientGroupID', clientGroupID);\n    this.#snapshotter = snapshotter;\n    this.#storage = storage;\n    this.#shardID = shardID;\n    this.#logConfig = logConfig;\n    this.#inspectorDelegate = inspectorDelegate;\n    this.#costModels = enablePlanner ? new WeakMap() : undefined;\n    this.#yieldThresholdMs = yieldThresholdMs;\n  }\n\n  /**\n   * Initializes the PipelineDriver to the current head of the database.\n   * Queries can then be added (i.e. hydrated) with {@link addQuery()}.\n   *\n   * Must only be called once.\n   */\n  init(clientSchema: ClientSchema) {\n    assert(!this.#snapshotter.initialized(), 'Already initialized');\n    this.#snapshotter.init();\n    this.#initAndResetCommon(clientSchema);\n  }\n\n  /**\n   * @returns Whether the PipelineDriver has been initialized.\n   */\n  initialized(): boolean {\n    return this.#snapshotter.initialized();\n  }\n\n  /**\n   * Clears the current pipelines and TableSources, returning the PipelineDriver\n   * to its initial state. This should be called in response to a schema change,\n   * as TableSources need to be recomputed.\n   */\n  reset(clientSchema: ClientSchema) {\n    for (const {input} of this.#pipelines.values()) {\n      input.destroy();\n    }\n    this.#pipelines.clear();\n    this.#tables.clear();\n    this.#initAndResetCommon(clientSchema);\n  }\n\n  #initAndResetCommon(clientSchema: ClientSchema) {\n    const {db} = this.#snapshotter.current();\n    const fullTables = new Map<string, LiteTableSpec>();\n    computeZqlSpecs(this.#lc, db.db, this.#tableSpecs, fullTables);\n    checkClientSchema(\n      this.#shardID,\n      clientSchema,\n      this.#tableSpecs,\n      fullTables,\n    );\n    const primaryKeys = this.#primaryKeys ?? new Map<string, PrimaryKey>();\n    this.#primaryKeys = primaryKeys;\n    primaryKeys.clear();\n    for (const [table, spec] of this.#tableSpecs.entries()) {\n      if (table.startsWith(upstreamSchema(this.#shardID))) {\n        primaryKeys.set(table, spec.tableSpec.primaryKey);\n      }\n    }\n    buildPrimaryKeys(clientSchema, primaryKeys);\n    const {replicaVersion} = getSubscriptionState(db);\n    this.#replicaVersion = replicaVersion;\n  }\n\n  /** @returns The replica version. The PipelineDriver must have been initialized. */\n  get replicaVersion(): string {\n    return must(this.#replicaVersion, 'Not yet initialized');\n  }\n\n  /**\n   * Returns the current version of the database. This will reflect the\n   * latest version change when calling {@link advance()} once the\n   * iteration has begun.\n   */\n  currentVersion(): string {\n    assert(this.initialized(), 'Not yet initialized');\n    return this.#snapshotter.current().version;\n  }\n\n  /**\n   * Returns the current upstream {app}.permissions, or `null` if none are defined.\n   */\n  currentPermissions(): LoadedPermissions | null {\n    assert(this.initialized(), 'Not yet initialized');\n    const res = reloadPermissionsIfChanged(\n      this.#lc,\n      this.#snapshotter.current().db,\n      this.#shardID.appID,\n      this.#permissions,\n    );\n    if (res.changed) {\n      this.#permissions = res.permissions;\n      this.#lc.debug?.(\n        'Reloaded permissions',\n        JSON.stringify(this.#permissions),\n      );\n    }\n    return this.#permissions;\n  }\n\n  advanceWithoutDiff(): string {\n    const {db, version} = this.#snapshotter.advanceWithoutDiff().curr;\n    for (const table of this.#tables.values()) {\n      table.setDB(db.db);\n    }\n    return version;\n  }\n\n  #ensureCostModelExistsIfEnabled(db: Database) {\n    let existing = this.#costModels?.get(db);\n    if (existing) {\n      return existing;\n    }\n    if (this.#costModels) {\n      const costModel = createSQLiteCostModel(db, this.#tableSpecs);\n      this.#costModels.set(db, costModel);\n      return costModel;\n    }\n    return undefined;\n  }\n\n  /**\n   * Clears storage used for the pipelines. Call this when the\n   * PipelineDriver will no longer be used.\n   */\n  destroy() {\n    this.#storage.destroy();\n    this.#snapshotter.destroy();\n  }\n\n  /** @return The Set of query hashes for all added queries. */\n  addedQueries(): [\n    transformationHashes: Set<string>,\n    byOriginalHash: Map<\n      string,\n      {\n        transformationHash: string;\n        transformedAst: AST;\n      }[]\n    >,\n  ] {\n    const byOriginalHash = new Map<\n      string,\n      {transformationHash: string; transformedAst: AST}[]\n    >();\n    for (const pipeline of this.#pipelines.values()) {\n      const {originalHash, transformedAst, transformationHash} = pipeline;\n\n      if (!byOriginalHash.has(originalHash)) {\n        byOriginalHash.set(originalHash, []);\n      }\n      byOriginalHash.get(originalHash)!.push({\n        transformationHash,\n        transformedAst,\n      });\n    }\n    return [new Set(this.#pipelines.keys()), byOriginalHash];\n  }\n\n  totalHydrationTimeMs(): number {\n    let total = 0;\n    for (const pipeline of this.#pipelines.values()) {\n      total += pipeline.hydrationTimeMs;\n    }\n    return total;\n  }\n\n  /**\n   * Adds a pipeline for the query. The method will hydrate the query using the\n   * driver's current snapshot of the database and return a stream of results.\n   * Henceforth, updates to the query will be returned when the driver is\n   * {@link advance}d. The query and its pipeline can be removed with\n   * {@link removeQuery()}.\n   *\n   * If a query with an identical hash has already been added, this method is a\n   * no-op and no RowChanges are generated.\n   *\n   * @param timer The caller-controlled {@link Timer} used to determine the\n   *        final hydration time. (The caller may pause and resume the timer\n   *        when yielding the thread for time-slicing).\n   * @return The rows from the initial hydration of the query.\n   */\n  *addQuery(\n    transformationHash: string,\n    queryID: string,\n    query: AST,\n    timer: Timer,\n  ): Iterable<RowChange | 'yield'> {\n    assert(this.initialized());\n    this.#inspectorDelegate.addQuery(transformationHash, queryID, query);\n    if (this.#pipelines.has(transformationHash)) {\n      this.#lc.info?.(`query ${transformationHash} already added`, query);\n      return;\n    }\n    const debugDelegate = runtimeDebugFlags.trackRowsVended\n      ? new Debug()\n      : undefined;\n\n    const costModel = this.#ensureCostModelExistsIfEnabled(\n      this.#snapshotter.current().db.db,\n    );\n\n    const input = buildPipeline(\n      query,\n      {\n        debug: debugDelegate,\n        enableNotExists: true, // Server-side can handle NOT EXISTS\n        getSource: name => this.#getSource(name),\n        createStorage: () => this.#createStorage(),\n        decorateSourceInput: (input: SourceInput, _queryID: string): Input =>\n          new MeasurePushOperator(\n            input,\n            transformationHash,\n            this.#inspectorDelegate,\n            'query-update-server',\n          ),\n        decorateInput: input => input,\n        addEdge() {},\n        decorateFilterInput: input => input,\n      },\n      queryID,\n      costModel,\n    );\n    const schema = input.getSchema();\n    input.setOutput({\n      push: change => {\n        const streamer = this.#streamer;\n        assert(streamer, 'must #startAccumulating() before pushing changes');\n        streamer.accumulate(transformationHash, schema, [change]);\n        return [];\n      },\n    });\n\n    assert(this.#advanceContext === null);\n    this.#hydrateContext = {\n      timer,\n    };\n    try {\n      yield* hydrateInternal(\n        input,\n        transformationHash,\n        must(this.#primaryKeys),\n      );\n    } finally {\n      this.#hydrateContext = null;\n    }\n\n    const hydrationTimeMs = timer.totalElapsed();\n    if (runtimeDebugFlags.trackRowCountsVended) {\n      if (hydrationTimeMs > this.#logConfig.slowHydrateThreshold) {\n        let totalRowsConsidered = 0;\n        const lc = this.#lc\n          .withContext('hash', transformationHash)\n          .withContext('hydrationTimeMs', hydrationTimeMs);\n        for (const tableName of this.#tables.keys()) {\n          const entries = Object.entries(\n            debugDelegate?.getVendedRowCounts()[tableName] ?? {},\n          );\n          totalRowsConsidered += entries.reduce(\n            (acc, entry) => acc + entry[1],\n            0,\n          );\n          lc.info?.(tableName + ' VENDED: ', entries);\n        }\n        lc.info?.(`Total rows considered: ${totalRowsConsidered}`);\n      }\n    }\n    debugDelegate?.reset();\n\n    // Note: This hydrationTime is a wall-clock overestimate, as it does\n    // not take time slicing into account. The view-syncer resets this\n    // to a more precise processing-time measurement with setHydrationTime().\n    this.#pipelines.set(transformationHash, {\n      input,\n      hydrationTimeMs,\n      originalHash: queryID,\n      transformedAst: query,\n      transformationHash,\n    });\n  }\n\n  /**\n   * Removes the pipeline for the query. This is a no-op if the query\n   * was not added.\n   */\n  removeQuery(hash: string) {\n    const pipeline = this.#pipelines.get(hash);\n    if (pipeline) {\n      this.#pipelines.delete(hash);\n      pipeline.input.destroy();\n    }\n  }\n\n  /**\n   * Returns the value of the row with the given primary key `pk`,\n   * or `undefined` if there is no such row. The pipeline must have been\n   * initialized.\n   */\n  getRow(table: string, pk: RowKey): Row | undefined {\n    assert(this.initialized(), 'Not yet initialized');\n    const source = must(this.#tables.get(table));\n    return source.getRow(pk as Row);\n  }\n\n  /**\n   * Advances to the new head of the database.\n   *\n   * @param timer The caller-controlled {@link Timer} that will be used to\n   *        measure the progress of the advancement and abort with a\n   *        {@link ResetPipelinesSignal} if it is estimated to take longer\n   *        than a hydration.\n   * @return The resulting row changes for all added queries. Note that the\n   *         `changes` must be iterated over in their entirety in order to\n   *         advance the database snapshot.\n   */\n  advance(timer: Timer): {\n    version: string;\n    numChanges: number;\n    changes: Iterable<RowChange | 'yield'>;\n  } {\n    assert(this.initialized());\n    const diff = this.#snapshotter.advance(this.#tableSpecs);\n    const {prev, curr, changes} = diff;\n    this.#lc.debug?.(\n      `advance ${prev.version} => ${curr.version}: ${changes} changes`,\n    );\n\n    return {\n      version: curr.version,\n      numChanges: changes,\n      changes: this.#advance(diff, timer, changes),\n    };\n  }\n\n  *#advance(\n    diff: SnapshotDiff,\n    timer: Timer,\n    numChanges: number,\n  ): Iterable<RowChange | 'yield'> {\n    assert(this.#hydrateContext === null);\n    this.#advanceContext = {\n      timer,\n      totalHydrationTimeMs: this.totalHydrationTimeMs(),\n      numChanges,\n      pos: 0,\n    };\n    try {\n      for (const {table, prevValues, nextValue} of diff) {\n        // Advance progress is checked each time a row is fetched\n        // from a TableSource during push processing, but some pushes\n        // don't read any rows.  Check progress here before processing\n        // the next change.\n        if (this.#shouldAdvanceYieldMaybeAbortAdvance()) {\n          yield 'yield';\n        }\n        const start = performance.now();\n        let type;\n        try {\n          const tableSource = this.#tables.get(table);\n          if (!tableSource) {\n            // no pipelines read from this table, so no need to process the change\n            continue;\n          }\n          const primaryKey = mustGetPrimaryKey(this.#primaryKeys, table);\n          let editOldRow: Row | undefined = undefined;\n          for (const prevValue of prevValues) {\n            if (\n              nextValue &&\n              deepEqual(\n                getRowKey(primaryKey, prevValue as Row) as JSONValue,\n                getRowKey(primaryKey, nextValue as Row) as JSONValue,\n              )\n            ) {\n              editOldRow = prevValue;\n            } else {\n              if (nextValue) {\n                this.#conflictRowsDeleted.add(1);\n              }\n              yield* this.#push(tableSource, {\n                type: 'remove',\n                row: prevValue,\n              });\n            }\n          }\n          if (nextValue) {\n            if (editOldRow) {\n              yield* this.#push(tableSource, {\n                type: 'edit',\n                row: nextValue,\n                oldRow: editOldRow,\n              });\n            } else {\n              yield* this.#push(tableSource, {\n                type: 'add',\n                row: nextValue,\n              });\n            }\n          }\n        } finally {\n          this.#advanceContext.pos++;\n        }\n\n        const elapsed = performance.now() - start;\n        this.#advanceTime.record(elapsed / 1000, {\n          table,\n          type,\n        });\n      }\n\n      // Set the new snapshot on all TableSources.\n      const {curr} = diff;\n      for (const table of this.#tables.values()) {\n        table.setDB(curr.db.db);\n      }\n      this.#ensureCostModelExistsIfEnabled(curr.db.db);\n      this.#lc.debug?.(`Advanced to ${curr.version}`);\n    } finally {\n      this.#advanceContext = null;\n    }\n  }\n\n  /** Implements `BuilderDelegate.getSource()` */\n  #getSource(tableName: string): Source {\n    let source = this.#tables.get(tableName);\n    if (source) {\n      return source;\n    }\n\n    const tableSpec = mustGetTableSpec(this.#tableSpecs, tableName);\n    const primaryKey = mustGetPrimaryKey(this.#primaryKeys, tableName);\n\n    const {db} = this.#snapshotter.current();\n    source = new TableSource(\n      this.#lc,\n      this.#logConfig,\n      db.db,\n      tableName,\n      tableSpec.zqlSpec,\n      primaryKey,\n      () => this.#shouldYield(),\n    );\n    this.#tables.set(tableName, source);\n    this.#lc.debug?.(`created TableSource for ${tableName}`);\n    return source;\n  }\n\n  #shouldYield(): boolean {\n    if (this.#hydrateContext) {\n      return this.#hydrateContext.timer.elapsedLap() > this.#yieldThresholdMs();\n    }\n    if (this.#advanceContext) {\n      return this.#shouldAdvanceYieldMaybeAbortAdvance();\n    }\n    throw new Error('shouldYield called outside of hydration or advancement');\n  }\n\n  /**\n   * Cancel the advancement processing, by throwing a ResetPipelinesSignal, if\n   * it has taken longer than half the total hydration time to make it through\n   * half of the advancement, or if processing time exceeds total hydration\n   * time.  This serves as both a circuit breaker for very large transactions,\n   * as well as a bound on the amount of time the previous connection locks\n   * the inactive WAL file (as the lock prevents WAL2 from switching to the\n   * free WAL when the current one is over the size limit, which can make\n   * the WAL grow continuously and compound slowness).\n   * This is checked:\n   * 1. before starting to process each change in an advancement is processed\n   * 2. whenever a row is fetched from a TableSource during push processing\n   */\n  #shouldAdvanceYieldMaybeAbortAdvance(): boolean {\n    const {\n      pos,\n      numChanges,\n      timer: advanceTimer,\n      totalHydrationTimeMs,\n    } = must(this.#advanceContext);\n    const elapsed = advanceTimer.totalElapsed();\n    if (\n      elapsed > MIN_ADVANCEMENT_TIME_LIMIT_MS &&\n      (elapsed > totalHydrationTimeMs ||\n        (elapsed > totalHydrationTimeMs / 2 && pos <= numChanges / 2))\n    ) {\n      throw new ResetPipelinesSignal(\n        `Advancement exceeded timeout at ${pos} of ${numChanges} changes ` +\n          `after ${elapsed} ms. Advancement time limited based on total ` +\n          `hydration time of ${totalHydrationTimeMs} ms.`,\n      );\n    }\n    return advanceTimer.elapsedLap() > this.#yieldThresholdMs();\n  }\n\n  /** Implements `BuilderDelegate.createStorage()` */\n  #createStorage(): Storage {\n    return this.#storage.createStorage();\n  }\n\n  *#push(\n    source: TableSource,\n    change: SourceChange,\n  ): Iterable<RowChange | 'yield'> {\n    this.#startAccumulating();\n    try {\n      for (const val of source.genPush(change)) {\n        if (val === 'yield') {\n          yield 'yield';\n        }\n        for (const changeOrYield of this.#stopAccumulating().stream()) {\n          yield changeOrYield;\n        }\n        this.#startAccumulating();\n      }\n    } finally {\n      if (this.#streamer !== null) {\n        this.#stopAccumulating();\n      }\n    }\n  }\n\n  #startAccumulating() {\n    assert(this.#streamer === null);\n    this.#streamer = new Streamer(must(this.#primaryKeys));\n  }\n\n  #stopAccumulating(): Streamer {\n    const streamer = this.#streamer;\n    assert(streamer);\n    this.#streamer = null;\n    return streamer;\n  }\n}\n\nclass Streamer {\n  readonly #primaryKeys: Map<string, PrimaryKey>;\n\n  constructor(primaryKeys: Map<string, PrimaryKey>) {\n    this.#primaryKeys = primaryKeys;\n  }\n\n  readonly #changes: [\n    hash: string,\n    schema: SourceSchema,\n    changes: Iterable<Change | 'yield'>,\n  ][] = [];\n\n  accumulate(\n    hash: string,\n    schema: SourceSchema,\n    changes: Iterable<Change | 'yield'>,\n  ): this {\n    this.#changes.push([hash, schema, changes]);\n    return this;\n  }\n\n  *stream(): Iterable<RowChange | 'yield'> {\n    for (const [hash, schema, changes] of this.#changes) {\n      yield* this.#streamChanges(hash, schema, changes);\n    }\n  }\n\n  *#streamChanges(\n    queryHash: string,\n    schema: SourceSchema,\n    changes: Iterable<Change | 'yield'>,\n  ): Iterable<RowChange | 'yield'> {\n    // We do not sync rows gathered by the permissions\n    // system to the client.\n    if (schema.system === 'permissions') {\n      return;\n    }\n\n    for (const change of changes) {\n      if (change === 'yield') {\n        yield change;\n        continue;\n      }\n      const {type} = change;\n\n      switch (type) {\n        case 'add':\n        case 'remove': {\n          yield* this.#streamNodes(queryHash, schema, type, () => [\n            change.node,\n          ]);\n          break;\n        }\n        case 'child': {\n          const {child} = change;\n          const childSchema = must(\n            schema.relationships[child.relationshipName],\n          );\n\n          yield* this.#streamChanges(queryHash, childSchema, [child.change]);\n          break;\n        }\n        case 'edit':\n          yield* this.#streamNodes(queryHash, schema, type, () => [\n            {row: change.node.row, relationships: {}},\n          ]);\n          break;\n        default:\n          unreachable(type);\n      }\n    }\n  }\n\n  *#streamNodes(\n    queryHash: string,\n    schema: SourceSchema,\n    op: 'add' | 'remove' | 'edit',\n    nodes: () => Iterable<Node | 'yield'>,\n  ): Iterable<RowChange | 'yield'> {\n    const {tableName: table, system} = schema;\n\n    const primaryKey = must(this.#primaryKeys.get(table));\n\n    // We do not sync rows gathered by the permissions\n    // system to the client.\n    if (system === 'permissions') {\n      return;\n    }\n\n    for (const node of nodes()) {\n      if (node === 'yield') {\n        yield node;\n        continue;\n      }\n      const {relationships, row} = node;\n      const rowKey = getRowKey(primaryKey, row);\n\n      yield {\n        type: op,\n        queryHash,\n        table,\n        rowKey,\n        row: op === 'remove' ? undefined : row,\n      } as RowChange;\n\n      for (const [relationship, children] of Object.entries(relationships)) {\n        const childSchema = must(schema.relationships[relationship]);\n        yield* this.#streamNodes(queryHash, childSchema, op, children);\n      }\n    }\n  }\n}\n\nfunction* toAdds(nodes: Iterable<Node | 'yield'>): Iterable<Change | 'yield'> {\n  for (const node of nodes) {\n    if (node === 'yield') {\n      yield node;\n      continue;\n    }\n    yield {type: 'add', node};\n  }\n}\n\nfunction getRowKey(cols: PrimaryKey, row: Row): RowKey {\n  return Object.fromEntries(cols.map(col => [col, must(row[col])]));\n}\n\n/**\n * Core hydration logic used by {@link PipelineDriver#addQuery}, extracted to a\n * function for reuse by bin-analyze so that bin-analyze's hydration logic\n * is as close as possible to zero-cache's real hydration logic.\n */\nexport function* hydrate(\n  input: Input,\n  hash: string,\n  clientSchema: ClientSchema,\n): Iterable<RowChange | 'yield'> {\n  const res = input.fetch({});\n  const streamer = new Streamer(buildPrimaryKeys(clientSchema)).accumulate(\n    hash,\n    input.getSchema(),\n    toAdds(res),\n  );\n  yield* streamer.stream();\n}\n\nexport function* hydrateInternal(\n  input: Input,\n  hash: string,\n  primaryKeys: Map<string, PrimaryKey>,\n): Iterable<RowChange | 'yield'> {\n  const res = input.fetch({});\n  const streamer = new Streamer(primaryKeys).accumulate(\n    hash,\n    input.getSchema(),\n    toAdds(res),\n  );\n  yield* streamer.stream();\n}\n\nfunction buildPrimaryKeys(\n  clientSchema: ClientSchema,\n  primaryKeys: Map<string, PrimaryKey> = new Map<string, PrimaryKey>(),\n) {\n  for (const [tableName, {primaryKey}] of Object.entries(clientSchema.tables)) {\n    primaryKeys.set(tableName, primaryKey as unknown as PrimaryKey);\n  }\n  return primaryKeys;\n}\n\nfunction mustGetPrimaryKey(\n  primaryKeys: Map<string, PrimaryKey> | null,\n  table: string,\n): PrimaryKey {\n  const pKeys = must(primaryKeys, 'primaryKey map must be non-null');\n\n  return must(\n    pKeys.get(table),\n    `table '${table}' is not one of: ${[...pKeys.keys()].sort()}. ` +\n      `Check the spelling and ensure that the table has a primary key.`,\n  );\n}\n"],"names":["input"],"mappings":";;;;;;;;;;;;;;;;;AAqGA,MAAM,gCAAgC;AAK/B,MAAM,eAAe;AAAA,EACjB,8BAAc,IAAA;AAAA;AAAA;AAAA;AAAA,EAId,iCAAiB,IAAA;AAAA,EAEjB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,kCAAkB,IAAA;AAAA,EAClB;AAAA,EACA;AAAA,EACT,YAA6B;AAAA,EAC7B,kBAAyC;AAAA,EACzC,kBAAyC;AAAA,EACzC,kBAAiC;AAAA,EACjC,eAA+C;AAAA,EAC/C,eAAyC;AAAA,EAEhC,eAAe,qBAAqB,QAAQ,oBAAoB;AAAA,IACvE,aACE;AAAA,IACF,MAAM;AAAA,EAAA,CACP;AAAA,EAEQ,uBAAuB;AAAA,IAC9B;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAAA,EAGO;AAAA,EAET,YACE,IACA,WACA,aACA,SACA,SACA,eACA,mBACA,kBACA,eACA;AACA,SAAK,MAAM,GAAG,YAAY,iBAAiB,aAAa;AACxD,SAAK,eAAe;AACpB,SAAK,WAAW;AAChB,SAAK,WAAW;AAChB,SAAK,aAAa;AAClB,SAAK,qBAAqB;AAC1B,SAAK,cAAc,gBAAgB,oBAAI,QAAA,IAAY;AACnD,SAAK,oBAAoB;AAAA,EAC3B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,KAAK,cAA4B;AAC/B,WAAO,CAAC,KAAK,aAAa,YAAA,GAAe,qBAAqB;AAC9D,SAAK,aAAa,KAAA;AAClB,SAAK,oBAAoB,YAAY;AAAA,EACvC;AAAA;AAAA;AAAA;AAAA,EAKA,cAAuB;AACrB,WAAO,KAAK,aAAa,YAAA;AAAA,EAC3B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,cAA4B;AAChC,eAAW,EAAC,MAAA,KAAU,KAAK,WAAW,UAAU;AAC9C,YAAM,QAAA;AAAA,IACR;AACA,SAAK,WAAW,MAAA;AAChB,SAAK,QAAQ,MAAA;AACb,SAAK,oBAAoB,YAAY;AAAA,EACvC;AAAA,EAEA,oBAAoB,cAA4B;AAC9C,UAAM,EAAC,GAAA,IAAM,KAAK,aAAa,QAAA;AAC/B,UAAM,iCAAiB,IAAA;AACvB,oBAAgB,KAAK,KAAK,GAAG,IAAI,KAAK,aAAa,UAAU;AAC7D;AAAA,MACE,KAAK;AAAA,MACL;AAAA,MACA,KAAK;AAAA,MACL;AAAA,IAAA;AAEF,UAAM,cAAc,KAAK,gBAAgB,oBAAI,IAAA;AAC7C,SAAK,eAAe;AACpB,gBAAY,MAAA;AACZ,eAAW,CAAC,OAAO,IAAI,KAAK,KAAK,YAAY,WAAW;AACtD,UAAI,MAAM,WAAW,eAAe,KAAK,QAAQ,CAAC,GAAG;AACnD,oBAAY,IAAI,OAAO,KAAK,UAAU,UAAU;AAAA,MAClD;AAAA,IACF;AACA,qBAAiB,cAAc,WAAW;AAC1C,UAAM,EAAC,eAAA,IAAkB,qBAAqB,EAAE;AAChD,SAAK,kBAAkB;AAAA,EACzB;AAAA;AAAA,EAGA,IAAI,iBAAyB;AAC3B,WAAO,KAAK,KAAK,iBAAiB,qBAAqB;AAAA,EACzD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,iBAAyB;AACvB,WAAO,KAAK,YAAA,GAAe,qBAAqB;AAChD,WAAO,KAAK,aAAa,QAAA,EAAU;AAAA,EACrC;AAAA;AAAA;AAAA;AAAA,EAKA,qBAA+C;AAC7C,WAAO,KAAK,YAAA,GAAe,qBAAqB;AAChD,UAAM,MAAM;AAAA,MACV,KAAK;AAAA,MACL,KAAK,aAAa,QAAA,EAAU;AAAA,MAC5B,KAAK,SAAS;AAAA,MACd,KAAK;AAAA,IAAA;AAEP,QAAI,IAAI,SAAS;AACf,WAAK,eAAe,IAAI;AACxB,WAAK,IAAI;AAAA,QACP;AAAA,QACA,KAAK,UAAU,KAAK,YAAY;AAAA,MAAA;AAAA,IAEpC;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,qBAA6B;AAC3B,UAAM,EAAC,IAAI,QAAA,IAAW,KAAK,aAAa,qBAAqB;AAC7D,eAAW,SAAS,KAAK,QAAQ,OAAA,GAAU;AACzC,YAAM,MAAM,GAAG,EAAE;AAAA,IACnB;AACA,WAAO;AAAA,EACT;AAAA,EAEA,gCAAgC,IAAc;AAC5C,QAAI,WAAW,KAAK,aAAa,IAAI,EAAE;AACvC,QAAI,UAAU;AACZ,aAAO;AAAA,IACT;AACA,QAAI,KAAK,aAAa;AACpB,YAAM,YAAY,sBAAsB,IAAI,KAAK,WAAW;AAC5D,WAAK,YAAY,IAAI,IAAI,SAAS;AAClC,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,UAAU;AACR,SAAK,SAAS,QAAA;AACd,SAAK,aAAa,QAAA;AAAA,EACpB;AAAA;AAAA,EAGA,eASE;AACA,UAAM,qCAAqB,IAAA;AAI3B,eAAW,YAAY,KAAK,WAAW,OAAA,GAAU;AAC/C,YAAM,EAAC,cAAc,gBAAgB,mBAAA,IAAsB;AAE3D,UAAI,CAAC,eAAe,IAAI,YAAY,GAAG;AACrC,uBAAe,IAAI,cAAc,EAAE;AAAA,MACrC;AACA,qBAAe,IAAI,YAAY,EAAG,KAAK;AAAA,QACrC;AAAA,QACA;AAAA,MAAA,CACD;AAAA,IACH;AACA,WAAO,CAAC,IAAI,IAAI,KAAK,WAAW,KAAA,CAAM,GAAG,cAAc;AAAA,EACzD;AAAA,EAEA,uBAA+B;AAC7B,QAAI,QAAQ;AACZ,eAAW,YAAY,KAAK,WAAW,OAAA,GAAU;AAC/C,eAAS,SAAS;AAAA,IACpB;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAiBA,CAAC,SACC,oBACA,SACA,OACA,OAC+B;AAC/B,WAAO,KAAK,aAAa;AACzB,SAAK,mBAAmB,SAAS,oBAAoB,SAAS,KAAK;AACnE,QAAI,KAAK,WAAW,IAAI,kBAAkB,GAAG;AAC3C,WAAK,IAAI,OAAO,SAAS,kBAAkB,kBAAkB,KAAK;AAClE;AAAA,IACF;AACA,UAAM,gBAAgB,kBAAkB,kBACpC,IAAI,UACJ;AAEJ,UAAM,YAAY,KAAK;AAAA,MACrB,KAAK,aAAa,QAAA,EAAU,GAAG;AAAA,IAAA;AAGjC,UAAM,QAAQ;AAAA,MACZ;AAAA,MACA;AAAA,QACE,OAAO;AAAA,QACP,iBAAiB;AAAA;AAAA,QACjB,WAAW,CAAA,SAAQ,KAAK,WAAW,IAAI;AAAA,QACvC,eAAe,MAAM,KAAK,eAAA;AAAA,QAC1B,qBAAqB,CAACA,QAAoB,aACxC,IAAI;AAAA,UACFA;AAAAA,UACA;AAAA,UACA,KAAK;AAAA,UACL;AAAA,QAAA;AAAA,QAEJ,eAAe,CAAAA,WAASA;AAAAA,QACxB,UAAU;AAAA,QAAC;AAAA,QACX,qBAAqB,CAAAA,WAASA;AAAAA,MAAA;AAAA,MAEhC;AAAA,MACA;AAAA,IAAA;AAEF,UAAM,SAAS,MAAM,UAAA;AACrB,UAAM,UAAU;AAAA,MACd,MAAM,CAAA,WAAU;AACd,cAAM,WAAW,KAAK;AACtB,eAAO,UAAU,kDAAkD;AACnE,iBAAS,WAAW,oBAAoB,QAAQ,CAAC,MAAM,CAAC;AACxD,eAAO,CAAA;AAAA,MACT;AAAA,IAAA,CACD;AAED,WAAO,KAAK,oBAAoB,IAAI;AACpC,SAAK,kBAAkB;AAAA,MACrB;AAAA,IAAA;AAEF,QAAI;AACF,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA,KAAK,KAAK,YAAY;AAAA,MAAA;AAAA,IAE1B,UAAA;AACE,WAAK,kBAAkB;AAAA,IACzB;AAEA,UAAM,kBAAkB,MAAM,aAAA;AAC9B,QAAI,kBAAkB,sBAAsB;AAC1C,UAAI,kBAAkB,KAAK,WAAW,sBAAsB;AAC1D,YAAI,sBAAsB;AAC1B,cAAM,KAAK,KAAK,IACb,YAAY,QAAQ,kBAAkB,EACtC,YAAY,mBAAmB,eAAe;AACjD,mBAAW,aAAa,KAAK,QAAQ,KAAA,GAAQ;AAC3C,gBAAM,UAAU,OAAO;AAAA,YACrB,eAAe,qBAAqB,SAAS,KAAK,CAAA;AAAA,UAAC;AAErD,iCAAuB,QAAQ;AAAA,YAC7B,CAAC,KAAK,UAAU,MAAM,MAAM,CAAC;AAAA,YAC7B;AAAA,UAAA;AAEF,aAAG,OAAO,YAAY,aAAa,OAAO;AAAA,QAC5C;AACA,WAAG,OAAO,0BAA0B,mBAAmB,EAAE;AAAA,MAC3D;AAAA,IACF;AACA,mBAAe,MAAA;AAKf,SAAK,WAAW,IAAI,oBAAoB;AAAA,MACtC;AAAA,MACA;AAAA,MACA,cAAc;AAAA,MACd,gBAAgB;AAAA,MAChB;AAAA,IAAA,CACD;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,YAAY,MAAc;AACxB,UAAM,WAAW,KAAK,WAAW,IAAI,IAAI;AACzC,QAAI,UAAU;AACZ,WAAK,WAAW,OAAO,IAAI;AAC3B,eAAS,MAAM,QAAA;AAAA,IACjB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,OAAO,OAAe,IAA6B;AACjD,WAAO,KAAK,YAAA,GAAe,qBAAqB;AAChD,UAAM,SAAS,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC;AAC3C,WAAO,OAAO,OAAO,EAAS;AAAA,EAChC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAaA,QAAQ,OAIN;AACA,WAAO,KAAK,aAAa;AACzB,UAAM,OAAO,KAAK,aAAa,QAAQ,KAAK,WAAW;AACvD,UAAM,EAAC,MAAM,MAAM,QAAA,IAAW;AAC9B,SAAK,IAAI;AAAA,MACP,WAAW,KAAK,OAAO,OAAO,KAAK,OAAO,KAAK,OAAO;AAAA,IAAA;AAGxD,WAAO;AAAA,MACL,SAAS,KAAK;AAAA,MACd,YAAY;AAAA,MACZ,SAAS,KAAK,SAAS,MAAM,OAAO,OAAO;AAAA,IAAA;AAAA,EAE/C;AAAA,EAEA,CAAC,SACC,MACA,OACA,YAC+B;AAC/B,WAAO,KAAK,oBAAoB,IAAI;AACpC,SAAK,kBAAkB;AAAA,MACrB;AAAA,MACA,sBAAsB,KAAK,qBAAA;AAAA,MAC3B;AAAA,MACA,KAAK;AAAA,IAAA;AAEP,QAAI;AACF,iBAAW,EAAC,OAAO,YAAY,UAAA,KAAc,MAAM;AAKjD,YAAI,KAAK,wCAAwC;AAC/C,gBAAM;AAAA,QACR;AACA,cAAM,QAAQ,YAAY,IAAA;AAC1B,YAAI;AACJ,YAAI;AACF,gBAAM,cAAc,KAAK,QAAQ,IAAI,KAAK;AAC1C,cAAI,CAAC,aAAa;AAEhB;AAAA,UACF;AACA,gBAAM,aAAa,kBAAkB,KAAK,cAAc,KAAK;AAC7D,cAAI,aAA8B;AAClC,qBAAW,aAAa,YAAY;AAClC,gBACE,aACA;AAAA,cACE,UAAU,YAAY,SAAgB;AAAA,cACtC,UAAU,YAAY,SAAgB;AAAA,YAAA,GAExC;AACA,2BAAa;AAAA,YACf,OAAO;AACL,kBAAI,WAAW;AACb,qBAAK,qBAAqB,IAAI,CAAC;AAAA,cACjC;AACA,qBAAO,KAAK,MAAM,aAAa;AAAA,gBAC7B,MAAM;AAAA,gBACN,KAAK;AAAA,cAAA,CACN;AAAA,YACH;AAAA,UACF;AACA,cAAI,WAAW;AACb,gBAAI,YAAY;AACd,qBAAO,KAAK,MAAM,aAAa;AAAA,gBAC7B,MAAM;AAAA,gBACN,KAAK;AAAA,gBACL,QAAQ;AAAA,cAAA,CACT;AAAA,YACH,OAAO;AACL,qBAAO,KAAK,MAAM,aAAa;AAAA,gBAC7B,MAAM;AAAA,gBACN,KAAK;AAAA,cAAA,CACN;AAAA,YACH;AAAA,UACF;AAAA,QACF,UAAA;AACE,eAAK,gBAAgB;AAAA,QACvB;AAEA,cAAM,UAAU,YAAY,IAAA,IAAQ;AACpC,aAAK,aAAa,OAAO,UAAU,KAAM;AAAA,UACvC;AAAA,UACA;AAAA,QAAA,CACD;AAAA,MACH;AAGA,YAAM,EAAC,SAAQ;AACf,iBAAW,SAAS,KAAK,QAAQ,OAAA,GAAU;AACzC,cAAM,MAAM,KAAK,GAAG,EAAE;AAAA,MACxB;AACA,WAAK,gCAAgC,KAAK,GAAG,EAAE;AAC/C,WAAK,IAAI,QAAQ,eAAe,KAAK,OAAO,EAAE;AAAA,IAChD,UAAA;AACE,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA;AAAA,EAGA,WAAW,WAA2B;AACpC,QAAI,SAAS,KAAK,QAAQ,IAAI,SAAS;AACvC,QAAI,QAAQ;AACV,aAAO;AAAA,IACT;AAEA,UAAM,YAAY,iBAAiB,KAAK,aAAa,SAAS;AAC9D,UAAM,aAAa,kBAAkB,KAAK,cAAc,SAAS;AAEjE,UAAM,EAAC,GAAA,IAAM,KAAK,aAAa,QAAA;AAC/B,aAAS,IAAI;AAAA,MACX,KAAK;AAAA,MACL,KAAK;AAAA,MACL,GAAG;AAAA,MACH;AAAA,MACA,UAAU;AAAA,MACV;AAAA,MACA,MAAM,KAAK,aAAA;AAAA,IAAa;AAE1B,SAAK,QAAQ,IAAI,WAAW,MAAM;AAClC,SAAK,IAAI,QAAQ,2BAA2B,SAAS,EAAE;AACvD,WAAO;AAAA,EACT;AAAA,EAEA,eAAwB;AACtB,QAAI,KAAK,iBAAiB;AACxB,aAAO,KAAK,gBAAgB,MAAM,WAAA,IAAe,KAAK,kBAAA;AAAA,IACxD;AACA,QAAI,KAAK,iBAAiB;AACxB,aAAO,KAAK,qCAAA;AAAA,IACd;AACA,UAAM,IAAI,MAAM,wDAAwD;AAAA,EAC1E;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAeA,uCAAgD;AAC9C,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA,OAAO;AAAA,MACP;AAAA,IAAA,IACE,KAAK,KAAK,eAAe;AAC7B,UAAM,UAAU,aAAa,aAAA;AAC7B,QACE,UAAU,kCACT,UAAU,wBACR,UAAU,uBAAuB,KAAK,OAAO,aAAa,IAC7D;AACA,YAAM,IAAI;AAAA,QACR,mCAAmC,GAAG,OAAO,UAAU,kBAC5C,OAAO,kEACK,oBAAoB;AAAA,MAAA;AAAA,IAE/C;AACA,WAAO,aAAa,eAAe,KAAK,kBAAA;AAAA,EAC1C;AAAA;AAAA,EAGA,iBAA0B;AACxB,WAAO,KAAK,SAAS,cAAA;AAAA,EACvB;AAAA,EAEA,CAAC,MACC,QACA,QAC+B;AAC/B,SAAK,mBAAA;AACL,QAAI;AACF,iBAAW,OAAO,OAAO,QAAQ,MAAM,GAAG;AACxC,YAAI,QAAQ,SAAS;AACnB,gBAAM;AAAA,QACR;AACA,mBAAW,iBAAiB,KAAK,kBAAA,EAAoB,UAAU;AAC7D,gBAAM;AAAA,QACR;AACA,aAAK,mBAAA;AAAA,MACP;AAAA,IACF,UAAA;AACE,UAAI,KAAK,cAAc,MAAM;AAC3B,aAAK,kBAAA;AAAA,MACP;AAAA,IACF;AAAA,EACF;AAAA,EAEA,qBAAqB;AACnB,WAAO,KAAK,cAAc,IAAI;AAC9B,SAAK,YAAY,IAAI,SAAS,KAAK,KAAK,YAAY,CAAC;AAAA,EACvD;AAAA,EAEA,oBAA8B;AAC5B,UAAM,WAAW,KAAK;AACtB,WAAO,QAAQ;AACf,SAAK,YAAY;AACjB,WAAO;AAAA,EACT;AACF;AAEA,MAAM,SAAS;AAAA,EACJ;AAAA,EAET,YAAY,aAAsC;AAChD,SAAK,eAAe;AAAA,EACtB;AAAA,EAES,WAIH,CAAA;AAAA,EAEN,WACE,MACA,QACA,SACM;AACN,SAAK,SAAS,KAAK,CAAC,MAAM,QAAQ,OAAO,CAAC;AAC1C,WAAO;AAAA,EACT;AAAA,EAEA,CAAC,SAAwC;AACvC,eAAW,CAAC,MAAM,QAAQ,OAAO,KAAK,KAAK,UAAU;AACnD,aAAO,KAAK,eAAe,MAAM,QAAQ,OAAO;AAAA,IAClD;AAAA,EACF;AAAA,EAEA,CAAC,eACC,WACA,QACA,SAC+B;AAG/B,QAAI,OAAO,WAAW,eAAe;AACnC;AAAA,IACF;AAEA,eAAW,UAAU,SAAS;AAC5B,UAAI,WAAW,SAAS;AACtB,cAAM;AACN;AAAA,MACF;AACA,YAAM,EAAC,SAAQ;AAEf,cAAQ,MAAA;AAAA,QACN,KAAK;AAAA,QACL,KAAK,UAAU;AACb,iBAAO,KAAK,aAAa,WAAW,QAAQ,MAAM,MAAM;AAAA,YACtD,OAAO;AAAA,UAAA,CACR;AACD;AAAA,QACF;AAAA,QACA,KAAK,SAAS;AACZ,gBAAM,EAAC,UAAS;AAChB,gBAAM,cAAc;AAAA,YAClB,OAAO,cAAc,MAAM,gBAAgB;AAAA,UAAA;AAG7C,iBAAO,KAAK,eAAe,WAAW,aAAa,CAAC,MAAM,MAAM,CAAC;AACjE;AAAA,QACF;AAAA,QACA,KAAK;AACH,iBAAO,KAAK,aAAa,WAAW,QAAQ,MAAM,MAAM;AAAA,YACtD,EAAC,KAAK,OAAO,KAAK,KAAK,eAAe,CAAA,EAAC;AAAA,UAAC,CACzC;AACD;AAAA,QACF;AACE,sBAAgB;AAAA,MAAA;AAAA,IAEtB;AAAA,EACF;AAAA,EAEA,CAAC,aACC,WACA,QACA,IACA,OAC+B;AAC/B,UAAM,EAAC,WAAW,OAAO,OAAA,IAAU;AAEnC,UAAM,aAAa,KAAK,KAAK,aAAa,IAAI,KAAK,CAAC;AAIpD,QAAI,WAAW,eAAe;AAC5B;AAAA,IACF;AAEA,eAAW,QAAQ,SAAS;AAC1B,UAAI,SAAS,SAAS;AACpB,cAAM;AACN;AAAA,MACF;AACA,YAAM,EAAC,eAAe,IAAA,IAAO;AAC7B,YAAM,SAAS,UAAU,YAAY,GAAG;AAExC,YAAM;AAAA,QACJ,MAAM;AAAA,QACN;AAAA,QACA;AAAA,QACA;AAAA,QACA,KAAK,OAAO,WAAW,SAAY;AAAA,MAAA;AAGrC,iBAAW,CAAC,cAAc,QAAQ,KAAK,OAAO,QAAQ,aAAa,GAAG;AACpE,cAAM,cAAc,KAAK,OAAO,cAAc,YAAY,CAAC;AAC3D,eAAO,KAAK,aAAa,WAAW,aAAa,IAAI,QAAQ;AAAA,MAC/D;AAAA,IACF;AAAA,EACF;AACF;AAEA,UAAU,OAAO,OAA6D;AAC5E,aAAW,QAAQ,OAAO;AACxB,QAAI,SAAS,SAAS;AACpB,YAAM;AACN;AAAA,IACF;AACA,UAAM,EAAC,MAAM,OAAO,KAAA;AAAA,EACtB;AACF;AAEA,SAAS,UAAU,MAAkB,KAAkB;AACrD,SAAO,OAAO,YAAY,KAAK,IAAI,CAAA,QAAO,CAAC,KAAK,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;AAClE;AAOO,UAAU,QACf,OACA,MACA,cAC+B;AAC/B,QAAM,MAAM,MAAM,MAAM,EAAE;AAC1B,QAAM,WAAW,IAAI,SAAS,iBAAiB,YAAY,CAAC,EAAE;AAAA,IAC5D;AAAA,IACA,MAAM,UAAA;AAAA,IACN,OAAO,GAAG;AAAA,EAAA;AAEZ,SAAO,SAAS,OAAA;AAClB;AAEO,UAAU,gBACf,OACA,MACA,aAC+B;AAC/B,QAAM,MAAM,MAAM,MAAM,EAAE;AAC1B,QAAM,WAAW,IAAI,SAAS,WAAW,EAAE;AAAA,IACzC;AAAA,IACA,MAAM,UAAA;AAAA,IACN,OAAO,GAAG;AAAA,EAAA;AAEZ,SAAO,SAAS,OAAA;AAClB;AAEA,SAAS,iBACP,cACA,cAAuC,oBAAI,OAC3C;AACA,aAAW,CAAC,WAAW,EAAC,WAAA,CAAW,KAAK,OAAO,QAAQ,aAAa,MAAM,GAAG;AAC3E,gBAAY,IAAI,WAAW,UAAmC;AAAA,EAChE;AACA,SAAO;AACT;AAEA,SAAS,kBACP,aACA,OACY;AACZ,QAAM,QAAQ,KAAK,aAAa,iCAAiC;AAEjE,SAAO;AAAA,IACL,MAAM,IAAI,KAAK;AAAA,IACf,UAAU,KAAK,oBAAoB,CAAC,GAAG,MAAM,KAAA,CAAM,EAAE,KAAA,CAAM;AAAA,EAAA;AAG/D;"}

package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts

@@ -5,7 +5,6 @@ import type { PrimaryKey } from '../../../../zero-types/src/schema.ts';
 import type { LiteAndZqlSpec, LiteTableSpecWithKeys } from '../../db/specs.ts';
 import { StatementRunner } from '../../db/statements.ts';
 import { type RowKey, type RowValue } from '../../types/row-key.ts';
-import { type SchemaVersions } from '../../types/schema-versions.ts';
 import type { AppID } from '../../types/shards.ts';
 /**
  * A `Snapshotter` manages the progression of database snapshots for a
@@ -181,7 +180,6 @@ declare class Snapshot {
     static create(lc: LogContext, dbFile: string, appID: string, pageCacheSizeKib: number | undefined): Snapshot;
     readonly db: StatementRunner;
     readonly version: string;
-    readonly schemaVersions: SchemaVersions;
     constructor(db: StatementRunner, appID: string);
     numChangesSince(prevVersion: string): any;
     changesSince(prevVersion: string): {

package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAY,KAAK,SAAS,EAAC,MAAM,uCAAuC,CAAC;AAGhF,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,uCAAuC,CAAC;AAC/D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,sCAAsC,CAAC;AAGrE,OAAO,KAAK,EAAC,cAAc,EAAE,qBAAqB,EAAC,MAAM,mBAAmB,CAAC;AAC7E,OAAO,EAAC,eAAe,EAAC,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAEL,KAAK,MAAM,EACX,KAAK,QAAQ,EACd,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAC,KAAK,cAAc,EAAC,MAAM,gCAAgC,CAAC;AACnE,OAAO,KAAK,EAAC,KAAK,EAAC,MAAM,uBAAuB,CAAC;AAajD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0DG;AACH,qBAAa,WAAW;;gBASpB,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAC,KAAK,EAAC,EAAE,KAAK,EACd,gBAAgB,CAAC,EAAE,MAAM,GAAG,SAAS;IAQvC;;;;OAIG;IACH,IAAI,IAAI,IAAI;IAYZ,WAAW,IAAI,OAAO;IAItB,+EAA+E;IAC/E,OAAO,IAAI,QAAQ;IAKnB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoCG;IACH,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,GAAG,YAAY;IAK1D,kBAAkB;;;;IAelB;;;OAGG;IACH,OAAO;CAKR;AAED,MAAM,MAAM,MAAM,GAAG;IACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,WAAW,YAAa,SAAQ,QAAQ,CAAC,MAAM,CAAC;IACpD,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;QAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;KAC1B,CAAC;IACF,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;QAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;KAC1B,CAAC;IAEF;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED;;;;GAIG;AACH,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,IAAI,0BAA0B;gBAE3B,GAAG,EAAE,MAAM;CAGxB;AAQD,cAAM,QAAQ;;IACZ,MAAM,CAAC,MAAM,CACX,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,gBAAgB,EAAE,MAAM,GAAG,SAAS;IAsBtC,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;IAE7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;gBAE5B,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE,MAAM;IAa9C,eAAe,CAAC,WAAW,EAAE,MAAM;IAQnC,YAAY,CAAC,WAAW,EAAE,MAAM;;;;IAUhC,MAAM,CAAC,KAAK,EAAE,qBAAqB,EAAE,MAAM,EAAE,SAAS;IAkBtD,OAAO,CAAC,KAAK,EAAE,qBAAqB,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,QAAQ;IAmBvE,WAAW,IAAI,QAAQ;CAIxB;AAmKD,qBAAa,gBAAiB,SAAQ,KAAK;gBAC7B,GAAG,EAAE,MAAM;CAGxB"}
+{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAY,KAAK,SAAS,EAAC,MAAM,uCAAuC,CAAC;AAGhF,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,uCAAuC,CAAC;AAC/D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,sCAAsC,CAAC;AAGrE,OAAO,KAAK,EAAC,cAAc,EAAE,qBAAqB,EAAC,MAAM,mBAAmB,CAAC;AAC7E,OAAO,EAAC,eAAe,EAAC,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAEL,KAAK,MAAM,EACX,KAAK,QAAQ,EACd,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAC,KAAK,EAAC,MAAM,uBAAuB,CAAC;AAajD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0DG;AACH,qBAAa,WAAW;;gBASpB,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAC,KAAK,EAAC,EAAE,KAAK,EACd,gBAAgB,CAAC,EAAE,MAAM,GAAG,SAAS;IAQvC;;;;OAIG;IACH,IAAI,IAAI,IAAI;IAYZ,WAAW,IAAI,OAAO;IAItB,+EAA+E;IAC/E,OAAO,IAAI,QAAQ;IAKnB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoCG;IACH,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,GAAG,YAAY;IAK1D,kBAAkB;;;;IAelB;;;OAGG;IACH,OAAO;CAKR;AAED,MAAM,MAAM,MAAM,GAAG;IACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,WAAW,YAAa,SAAQ,QAAQ,CAAC,MAAM,CAAC;IACpD,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;QAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;KAC1B,CAAC;IACF,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;QAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;KAC1B,CAAC;IAEF;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED;;;;GAIG;AACH,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,IAAI,0BAA0B;gBAE3B,GAAG,EAAE,MAAM;CAGxB;AAED,cAAM,QAAQ;;IACZ,MAAM,CAAC,MAAM,CACX,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,gBAAgB,EAAE,MAAM,GAAG,SAAS;IAsBtC,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;IAE7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;gBAEb,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE,MAAM;IAY9C,eAAe,CAAC,WAAW,EAAE,MAAM;IAQnC,YAAY,CAAC,WAAW,EAAE,MAAM;;;;IAUhC,MAAM,CAAC,KAAK,EAAE,qBAAqB,EAAE,MAAM,EAAE,SAAS;IAkBtD,OAAO,CAAC,KAAK,EAAE,qBAAqB,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,QAAQ;IAmBvE,WAAW,IAAI,QAAQ;CAIxB;AAmKD,qBAAa,gBAAiB,SAAQ,KAAK;gBAC7B,GAAG,EAAE,MAAM;CAGxB"}

package/out/zero-cache/src/services/view-syncer/snapshotter.js

@@ -6,7 +6,6 @@ import { Database } from "../../../../zqlite/src/db.js";
 import { fromSQLiteTypes } from "../../../../zqlite/src/table-source.js";
 import { StatementRunner } from "../../db/statements.js";
 import { normalizedKeyOrder } from "../../types/row-key.js";
-import "../../../../zero-protocol/src/error.js";
 import { id } from "../../types/sql.js";
 import { changeLogEntrySchema, RESET_OP, TRUNCATE_OP, SET_OP } from "../replicator/schema/change-log.js";
 import { getReplicationState } from "../replicator/schema/replication-state.js";
@@ -117,11 +116,6 @@ class ResetPipelinesSignal extends Error {
     super(msg);
   }
 }
-function getSchemaVersions(db, appID) {
-  return db.get(
-    `SELECT minSupportedVersion, maxSupportedVersion FROM "${appID}.schemaVersions"`
-  );
-}
 class Snapshot {
   static create(lc, dbFile, appID, pageCacheSizeKib) {
     const conn = new Database(lc, dbFile);
@@ -140,25 +134,23 @@ class Snapshot {
   db;
   #appID;
   version;
-  schemaVersions;
   constructor(db, appID) {
     db.beginConcurrent();
     const { stateVersion } = getReplicationState(db);
     this.db = db;
     this.#appID = appID;
     this.version = stateVersion;
-    this.schemaVersions = getSchemaVersions(db, appID);
   }
   numChangesSince(prevVersion) {
     const { count } = this.db.get(
-      'SELECT COUNT(*) AS count FROM "_zero.changeLog" WHERE stateVersion > ?',
+      'SELECT COUNT(*) AS count FROM "_zero.changeLog2" WHERE stateVersion > ?',
       prevVersion
     );
     return count;
   }
   changesSince(prevVersion) {
     const cached = this.db.statementCache.get(
-      'SELECT * FROM "_zero.changeLog" WHERE stateVersion > ?'
+      'SELECT * FROM "_zero.changeLog2" WHERE stateVersion > ? ORDER BY stateVersion ASC, pos ASC'
     );
     return {
       changes: cached.statement.iterate(prevVersion),

package/out/zero-cache/src/services/view-syncer/snapshotter.js.map

@@ -1 +1 @@
-{"version":3,"file":"snapshotter.js","sources":["../../../../../../zero-cache/src/services/view-syncer/snapshotter.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {stringify, type JSONValue} from '../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {Row} from '../../../../zero-protocol/src/data.ts';\nimport type {PrimaryKey} from '../../../../zero-types/src/schema.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {fromSQLiteTypes} from '../../../../zqlite/src/table-source.ts';\nimport type {LiteAndZqlSpec, LiteTableSpecWithKeys} from '../../db/specs.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {\n  normalizedKeyOrder,\n  type RowKey,\n  type RowValue,\n} from '../../types/row-key.ts';\nimport {type SchemaVersions} from '../../types/schema-versions.ts';\nimport type {AppID} from '../../types/shards.ts';\nimport {id} from '../../types/sql.ts';\nimport {\n  RESET_OP,\n  changeLogEntrySchema as schema,\n  SET_OP,\n  TRUNCATE_OP,\n} from '../replicator/schema/change-log.ts';\nimport {\n  getReplicationState,\n  ZERO_VERSION_COLUMN_NAME as ROW_VERSION,\n} from '../replicator/schema/replication-state.ts';\n\n/**\n * A `Snapshotter` manages the progression of database snapshots for a\n * ViewSyncer.\n *\n * The Replicator and ViewSyncers operate on the same SQLite file, with the\n * Replicator being the sole writer to the database. The IVM logic in\n * ViewSyncers, however, rely on incrementally applying changes to the DB to\n * update the state of its pipelines.\n *\n * To avoid coupling the progress of the Replicator and all IVM pipelines on\n * each other, ViewSyncers operate on ephemeral forks of the database by holding\n * [concurrent](https://sqlite.org/src/doc/begin-concurrent/doc/begin_concurrent.md)\n * snapshots of the database and simulating (but ultimately rolling back)\n * mutations on these snapshots.\n *\n * Example:\n * 1. ViewSyncer takes `snapshot_a` at version `t1` of the database and\n *    hydrates its pipeline(s).\n * 2. Replicator applies a new transaction to the database and notifies\n *    subscribers.\n * 3. ViewSyncer takes `snapshot_b` at `t2`, and queries the `ChangeLog` at\n *    that snapshot for changes since `t1`.\n * 4. ViewSyncer applies those changes to `snapshot_a` for IVM, but does not\n *    commit them. (Recall that the Replicator is the sole writer to the db, so\n *    the ViewSyncer never commits any writes.)\n * 5. Replicator applies the next transaction and advances the database to `t3`.\n * 6. ViewSyncer rolls back `snapshot_a` and opens `snapshot_c` at `t3`, using\n *    `snapshot_b` to simulate changes from `t2` to `t3`.\n *\n * ```\n * Replicator:  t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n * ```\n *\n * Note that the Replicator (and ViewSyncers) do not wait on the progress of\n * other ViewSyncers. If a ViewSyncer is busy hydrating at `t1`, the Replicator\n * and other ViewSyncers can progress through `t2`, `t3`, etc. independently,\n * as the busy ViewSyncer simply takes its own snapshot when it is ready.\n *\n * ```\n * Replicator:   t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer1:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n * ViewSyncer2:       [.......... snapshot_a ..........] ----> [snapshot_b]\n * ```\n *\n * To minimize Database connections (and statement preparation, etc.), the\n * Snapshotter reuses the connection from the previous (rolled back)\n * snapshot when opening the new one.\n *\n * ```\n * Replicator:  t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n *                     (conn_1)           (conn_2)           (conn_1)\n * ```\n *\n * In this manner, each ViewSyncer uses two connections that continually\n * \"leapfrog\" each other to replay the timeline of changes in isolation from\n * the Replicator and other ViewSyncers.\n */\nexport class Snapshotter {\n  readonly #lc: LogContext;\n  readonly #dbFile: string;\n  readonly #appID: string;\n  readonly #pageCacheSizeKib: number | undefined;\n  #curr: Snapshot | undefined;\n  #prev: Snapshot | undefined;\n\n  constructor(\n    lc: LogContext,\n    dbFile: string,\n    {appID}: AppID,\n    pageCacheSizeKib?: number | undefined,\n  ) {\n    this.#lc = lc;\n    this.#dbFile = dbFile;\n    this.#appID = appID;\n    this.#pageCacheSizeKib = pageCacheSizeKib;\n  }\n\n  /**\n   * Initializes the snapshot to the current head of the database. This must be\n   * only be called once. The state of whether a Snapshotter has been initialized\n   * can be determined by calling {@link initialized()}.\n   */\n  init(): this {\n    assert(this.#curr === undefined, 'Already initialized');\n    this.#curr = Snapshot.create(\n      this.#lc,\n      this.#dbFile,\n      this.#appID,\n      this.#pageCacheSizeKib,\n    );\n    this.#lc.debug?.(`Initial snapshot at version ${this.#curr.version}`);\n    return this;\n  }\n\n  initialized(): boolean {\n    return this.#curr !== undefined;\n  }\n\n  /** Returns the current snapshot. Asserts if {@link initialized()} is false. */\n  current(): Snapshot {\n    assert(this.#curr !== undefined, 'Snapshotter has not been initialized');\n    return this.#curr;\n  }\n\n  /**\n   * Advances to the head of the Database, returning a diff between the\n   * previously current Snapshot and a new Snapshot at head. This is called\n   * in response to a notification from a Replicator subscription. Subsequent\n   * calls to {@link current()} return the new Snapshot. Note that the Snapshotter\n   * must be initialized before advancing.\n   *\n   * The returned {@link SnapshotDiff} contains snapshots at the endpoints\n   * of the database timeline. Iterating over the diff generates a sequence\n   * of {@link Change}s between the two snapshots.\n   *\n   * Note that this sequence is not chronological; rather, the sequence is\n   * ordered by `<table, row-key>`, such that a row can appear at most once\n   * in the common case, or twice if its table is `TRUNCATE`'d and a new value\n   * is subsequently `INSERT`'ed. This results in dropping most intermediate\n   * changes to a row and bounds the amount of work needed to catch up;\n   * however, as a consequence, a consistent database state is only guaranteed\n   * when the sequence has been fully consumed.\n   *\n   * Note that Change generation relies on the state of the underlying\n   * database connections, and because the connection for the previous snapshot\n   * is reused to produce the next snapshot, the diff object is only valid\n   * until the next call to `advance()`.\n   *\n   * It is okay for the caller to apply `Change`s to the `prev` snapshot\n   * during the iteration (e.g. this is necessary for IVM); the remainder\n   * of the iteration is not affected because a given row can appear at most\n   * once in the sequence (with the exception being TRUNCATE, after which the\n   * deleted rows can be re-inserted, but this will also behave correctly if\n   * the changes are applied).\n   *\n   * Once the changes have been applied, however, a _subsequent_ iteration\n   * will not produce the correct results. In order to perform multiple\n   * change-applying iterations, the caller must (1) create a save point\n   * on `prev` before each iteration, and (2) rollback to the save point after\n   * the iteration.\n   */\n  advance(tables: Map<string, LiteAndZqlSpec>): SnapshotDiff {\n    const {prev, curr} = this.advanceWithoutDiff();\n    return new Diff(this.#appID, tables, prev, curr);\n  }\n\n  advanceWithoutDiff() {\n    assert(this.#curr !== undefined, 'Snapshotter has not been initialized');\n    const next = this.#prev\n      ? this.#prev.resetToHead()\n      : Snapshot.create(\n          this.#lc,\n          this.#curr.db.db.name,\n          this.#appID,\n          this.#pageCacheSizeKib,\n        );\n    this.#prev = this.#curr;\n    this.#curr = next;\n    return {prev: this.#prev, curr: this.#curr};\n  }\n\n  /**\n   * Call this to close the database connections when the Snapshotter is\n   * no longer needed.\n   */\n  destroy() {\n    this.#curr?.db.db.close();\n    this.#prev?.db.db.close();\n    this.#lc.debug?.('closed database connections');\n  }\n}\n\nexport type Change = {\n  readonly table: string;\n  /**\n   * If this change represents a remove the row to remove,\n   * if nextValue is not null then all rows that have a unique constraint\n   * violation with nextValue.\n   * In both cases these rows should be removed.\n   */\n  readonly prevValues: Readonly<Row>[];\n  readonly nextValue: Readonly<Row> | null;\n  readonly rowKey: RowKey;\n};\n\n/**\n * Represents the difference between two database Snapshots.\n * Iterating over the object will produce a sequence of {@link Change}s\n * between the two snapshots.\n *\n * See {@link Snapshotter.advance()} for semantics and usage.\n */\nexport interface SnapshotDiff extends Iterable<Change> {\n  readonly prev: {\n    readonly db: StatementRunner;\n    readonly version: string;\n  };\n  readonly curr: {\n    readonly db: StatementRunner;\n    readonly version: string;\n  };\n\n  /**\n   * The number of ChangeLog entries between the snapshots. Note that this\n   * may not necessarily equal the number of `Change` objects that the iteration\n   * will produce, as `TRUNCATE` entries are counted as a single log entry which\n   * may be expanded into many changes (i.e. row deletes).\n   *\n   * TODO: Determine if it is worth changing the definition to count the\n   *       truncated rows. This would make diff computation more expensive\n   *       (requiring the count to be aggregated by operation type), which\n   *       may not be worth it for a presumable rare operation.\n   */\n  readonly changes: number;\n}\n\n/**\n * Thrown during an iteration of a {@link SnapshotDiff} when a schema\n * change or truncate is encountered, which result in aborting the\n * advancement and resetting / rehydrating the pipelines.\n */\nexport class ResetPipelinesSignal extends Error {\n  readonly name = 'ResetPipelinesSignal';\n\n  constructor(msg: string) {\n    super(msg);\n  }\n}\n\nfunction getSchemaVersions(db: StatementRunner, appID: string): SchemaVersions {\n  return db.get(\n    `SELECT minSupportedVersion, maxSupportedVersion FROM \"${appID}.schemaVersions\"`,\n  );\n}\n\nclass Snapshot {\n  static create(\n    lc: LogContext,\n    dbFile: string,\n    appID: string,\n    pageCacheSizeKib: number | undefined,\n  ) {\n    const conn = new Database(lc, dbFile);\n    conn.pragma('synchronous = OFF'); // Applied changes are ephemeral; COMMIT is never called.\n    if (pageCacheSizeKib !== undefined) {\n      conn.pragma(`cache_size = -${pageCacheSizeKib}`); // Negative = size in KiB\n    }\n    const [{journal_mode: mode}] = conn.pragma('journal_mode') as [\n      {journal_mode: string},\n    ];\n    // The Snapshotter operates on the replica file with BEGIN CONCURRENT,\n    // which must be used in concert with the replicator using BEGIN CONCURRENT\n    // on a db in the wal2 journal_mode.\n    assert(\n      mode === 'wal2',\n      `replica db must be in wal2 mode (current: ${mode})`,\n    );\n\n    const db = new StatementRunner(conn);\n    return new Snapshot(db, appID);\n  }\n\n  readonly db: StatementRunner;\n  readonly #appID: string;\n  readonly version: string;\n  readonly schemaVersions: SchemaVersions;\n\n  constructor(db: StatementRunner, appID: string) {\n    db.beginConcurrent();\n    // Note: The subsequent read is necessary to acquire the read lock\n    // (which results in the logical creation of the snapshot). Calling\n    // `BEGIN CONCURRENT` alone does not result in acquiring the read lock.\n    const {stateVersion} = getReplicationState(db);\n\n    this.db = db;\n    this.#appID = appID;\n    this.version = stateVersion;\n    this.schemaVersions = getSchemaVersions(db, appID);\n  }\n\n  numChangesSince(prevVersion: string) {\n    const {count} = this.db.get(\n      'SELECT COUNT(*) AS count FROM \"_zero.changeLog\" WHERE stateVersion > ?',\n      prevVersion,\n    );\n    return count;\n  }\n\n  changesSince(prevVersion: string) {\n    const cached = this.db.statementCache.get(\n      'SELECT * FROM \"_zero.changeLog\" WHERE stateVersion > ?',\n    );\n    return {\n      changes: cached.statement.iterate(prevVersion),\n      cleanup: () => this.db.statementCache.return(cached),\n    };\n  }\n\n  getRow(table: LiteTableSpecWithKeys, rowKey: JSONValue) {\n    const key = normalizedKeyOrder(rowKey as RowKey);\n    const conds = Object.keys(key).map(c => `${id(c)}=?`);\n    const cols = Object.keys(table.columns);\n    const cached = this.db.statementCache.get(\n      `SELECT ${cols.map(c => id(c)).join(',')} FROM ${id(\n        table.name,\n      )} WHERE ${conds.join(' AND ')}`,\n    );\n    cached.statement.safeIntegers(true);\n    try {\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      return cached.statement.get<any>(Object.values(key));\n    } finally {\n      this.db.statementCache.return(cached);\n    }\n  }\n\n  getRows(table: LiteTableSpecWithKeys, keys: PrimaryKey[], row: RowValue) {\n    const conds = keys.map(key => key.map(c => `${id(c)}=?`));\n    const cols = Object.keys(table.columns);\n    const cached = this.db.statementCache.get(\n      `SELECT ${cols.map(c => id(c)).join(',')} FROM ${id(\n        table.name,\n      )} WHERE ${conds.map(cond => cond.join(' AND ')).join(' OR ')}`,\n    );\n    cached.statement.safeIntegers(true);\n    try {\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      return cached.statement.all<any>(\n        keys.flatMap(key => key.map(column => row[column])),\n      );\n    } finally {\n      this.db.statementCache.return(cached);\n    }\n  }\n\n  resetToHead(): Snapshot {\n    this.db.rollback();\n    return new Snapshot(this.db, this.#appID);\n  }\n}\n\nclass Diff implements SnapshotDiff {\n  readonly #permissionsTable: string;\n  readonly tables: Map<string, LiteAndZqlSpec>;\n  readonly prev: Snapshot;\n  readonly curr: Snapshot;\n  readonly changes: number;\n\n  constructor(\n    appID: string,\n    tables: Map<string, LiteAndZqlSpec>,\n    prev: Snapshot,\n    curr: Snapshot,\n  ) {\n    this.#permissionsTable = `${appID}.permissions`;\n    this.tables = tables;\n    this.prev = prev;\n    this.curr = curr;\n    this.changes = curr.numChangesSince(prev.version);\n  }\n\n  [Symbol.iterator](): Iterator<Change> {\n    const {changes, cleanup: done} = this.curr.changesSince(this.prev.version);\n\n    const cleanup = () => {\n      try {\n        // Allow open iterators to clean up their state.\n        changes.return?.(undefined);\n      } finally {\n        done();\n      }\n    };\n\n    return {\n      next: () => {\n        try {\n          for (;;) {\n            const {value, done} = changes.next();\n            if (done) {\n              cleanup();\n              return {value, done: true};\n            }\n\n            const {table, rowKey, op, stateVersion} = v.parse(value, schema);\n            if (op === RESET_OP) {\n              // The current map of `TableSpec`s may not have the correct or complete information.\n              throw new ResetPipelinesSignal(\n                `schema for table ${table} has changed`,\n              );\n            }\n            if (op === TRUNCATE_OP) {\n              // Truncates are also processed by rehydrating pipelines at current.\n              throw new ResetPipelinesSignal(\n                `table ${table} has been truncated`,\n              );\n            }\n            const {tableSpec, zqlSpec} = must(this.tables.get(table));\n\n            assert(rowKey !== null);\n            const nextValue =\n              op === SET_OP ? this.curr.getRow(tableSpec, rowKey) : null;\n            let prevValues;\n            if (nextValue) {\n              prevValues = this.prev.getRows(\n                tableSpec,\n                tableSpec.uniqueKeys,\n                nextValue,\n              );\n            } else {\n              const prevValue = this.prev.getRow(tableSpec, rowKey);\n              prevValues = prevValue ? [prevValue] : [];\n            }\n            if (nextValue === undefined) {\n              throw new Error(\n                `Missing value for ${table} ${stringify(rowKey)}`,\n              );\n            }\n            // Sanity check detects if the diff is being accessed after the Snapshots have advanced.\n            this.checkThatDiffIsValid(stateVersion, op, prevValues, nextValue);\n\n            if (prevValues.length === 0 && nextValue === null) {\n              // Filter out no-op changes (e.g. a delete of a row that does not exist in prev).\n              // TODO: Consider doing this for deep-equal values.\n              continue;\n            }\n\n            if (\n              table === this.#permissionsTable &&\n              prevValues.find(\n                prevValue => prevValue.permissions !== nextValue.permissions,\n              )\n            ) {\n              throw new ResetPipelinesSignal(\n                `Permissions have changed ${\n                  prevValues.find(\n                    prevValue =>\n                      prevValue.permissions !== nextValue.permissions,\n                  ).hash\n                } => ${nextValue.hash}`,\n              );\n            }\n\n            // Modify the values in place when converting to ZQL rows\n            // This is safe since we're the first node in the iterator chain.\n            // TODO Can we get rid of these RowValue casts?\n            return {\n              value: {\n                table,\n                prevValues: prevValues.map(prevValue =>\n                  fromSQLiteTypes(zqlSpec, prevValue, table),\n                ),\n                nextValue: nextValue\n                  ? fromSQLiteTypes(zqlSpec, nextValue, table)\n                  : null,\n                rowKey,\n              } satisfies Change,\n            };\n          }\n        } catch (e) {\n          // This control flow path is not covered by the return() method (i.e. `break`).\n          cleanup();\n          throw e;\n        }\n      },\n\n      return: (value: unknown) => {\n        cleanup();\n        return {value, done: true};\n      },\n    };\n  }\n\n  checkThatDiffIsValid(\n    stateVersion: string,\n    op: string,\n    prevValues: RowValue[],\n    nextValue: RowValue,\n  ) {\n    // Sanity checks to detect that the diff is not being accessed after\n    // the Snapshots have advanced.\n    if (stateVersion > this.curr.version) {\n      throw new InvalidDiffError(\n        `Diff is no longer valid. curr db has advanced past ${this.curr.version}`,\n      );\n    }\n    if (\n      prevValues.findIndex(\n        prevValue => (prevValue[ROW_VERSION] ?? '~') > this.prev.version,\n      ) !== -1\n    ) {\n      throw new InvalidDiffError(\n        `Diff is no longer valid. prev db has advanced past ${this.prev.version}.`,\n      );\n    }\n    if (op === SET_OP && nextValue[ROW_VERSION] !== stateVersion) {\n      throw new InvalidDiffError(\n        'Diff is no longer valid. curr db has advanced.',\n      );\n    }\n  }\n}\n\nexport class InvalidDiffError extends Error {\n  constructor(msg: string) {\n    super(msg);\n  }\n}\n"],"names":["done","v.parse","schema","ROW_VERSION"],"mappings":";;;;;;;;;;;;;AAyFO,MAAM,YAAY;AAAA,EACd;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACT;AAAA,EACA;AAAA,EAEA,YACE,IACA,QACA,EAAC,MAAA,GACD,kBACA;AACA,SAAK,MAAM;AACX,SAAK,UAAU;AACf,SAAK,SAAS;AACd,SAAK,oBAAoB;AAAA,EAC3B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,OAAa;AACX,WAAO,KAAK,UAAU,QAAW,qBAAqB;AACtD,SAAK,QAAQ,SAAS;AAAA,MACpB,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,IAAA;AAEP,SAAK,IAAI,QAAQ,+BAA+B,KAAK,MAAM,OAAO,EAAE;AACpE,WAAO;AAAA,EACT;AAAA,EAEA,cAAuB;AACrB,WAAO,KAAK,UAAU;AAAA,EACxB;AAAA;AAAA,EAGA,UAAoB;AAClB,WAAO,KAAK,UAAU,QAAW,sCAAsC;AACvE,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAuCA,QAAQ,QAAmD;AACzD,UAAM,EAAC,MAAM,SAAQ,KAAK,mBAAA;AAC1B,WAAO,IAAI,KAAK,KAAK,QAAQ,QAAQ,MAAM,IAAI;AAAA,EACjD;AAAA,EAEA,qBAAqB;AACnB,WAAO,KAAK,UAAU,QAAW,sCAAsC;AACvE,UAAM,OAAO,KAAK,QACd,KAAK,MAAM,YAAA,IACX,SAAS;AAAA,MACP,KAAK;AAAA,MACL,KAAK,MAAM,GAAG,GAAG;AAAA,MACjB,KAAK;AAAA,MACL,KAAK;AAAA,IAAA;AAEX,SAAK,QAAQ,KAAK;AAClB,SAAK,QAAQ;AACb,WAAO,EAAC,MAAM,KAAK,OAAO,MAAM,KAAK,MAAA;AAAA,EACvC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,UAAU;AACR,SAAK,OAAO,GAAG,GAAG,MAAA;AAClB,SAAK,OAAO,GAAG,GAAG,MAAA;AAClB,SAAK,IAAI,QAAQ,6BAA6B;AAAA,EAChD;AACF;AAmDO,MAAM,6BAA6B,MAAM;AAAA,EACrC,OAAO;AAAA,EAEhB,YAAY,KAAa;AACvB,UAAM,GAAG;AAAA,EACX;AACF;AAEA,SAAS,kBAAkB,IAAqB,OAA+B;AAC7E,SAAO,GAAG;AAAA,IACR,yDAAyD,KAAK;AAAA,EAAA;AAElE;AAEA,MAAM,SAAS;AAAA,EACb,OAAO,OACL,IACA,QACA,OACA,kBACA;AACA,UAAM,OAAO,IAAI,SAAS,IAAI,MAAM;AACpC,SAAK,OAAO,mBAAmB;AAC/B,QAAI,qBAAqB,QAAW;AAClC,WAAK,OAAO,iBAAiB,gBAAgB,EAAE;AAAA,IACjD;AACA,UAAM,CAAC,EAAC,cAAc,KAAA,CAAK,IAAI,KAAK,OAAO,cAAc;AAMzD;AAAA,MACE,SAAS;AAAA,MACT,6CAA6C,IAAI;AAAA,IAAA;AAGnD,UAAM,KAAK,IAAI,gBAAgB,IAAI;AACnC,WAAO,IAAI,SAAS,IAAI,KAAK;AAAA,EAC/B;AAAA,EAES;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAET,YAAY,IAAqB,OAAe;AAC9C,OAAG,gBAAA;AAIH,UAAM,EAAC,aAAA,IAAgB,oBAAoB,EAAE;AAE7C,SAAK,KAAK;AACV,SAAK,SAAS;AACd,SAAK,UAAU;AACf,SAAK,iBAAiB,kBAAkB,IAAI,KAAK;AAAA,EACnD;AAAA,EAEA,gBAAgB,aAAqB;AACnC,UAAM,EAAC,MAAA,IAAS,KAAK,GAAG;AAAA,MACtB;AAAA,MACA;AAAA,IAAA;AAEF,WAAO;AAAA,EACT;AAAA,EAEA,aAAa,aAAqB;AAChC,UAAM,SAAS,KAAK,GAAG,eAAe;AAAA,MACpC;AAAA,IAAA;AAEF,WAAO;AAAA,MACL,SAAS,OAAO,UAAU,QAAQ,WAAW;AAAA,MAC7C,SAAS,MAAM,KAAK,GAAG,eAAe,OAAO,MAAM;AAAA,IAAA;AAAA,EAEvD;AAAA,EAEA,OAAO,OAA8B,QAAmB;AACtD,UAAM,MAAM,mBAAmB,MAAgB;AAC/C,UAAM,QAAQ,OAAO,KAAK,GAAG,EAAE,IAAI,CAAA,MAAK,GAAG,GAAG,CAAC,CAAC,IAAI;AACpD,UAAM,OAAO,OAAO,KAAK,MAAM,OAAO;AACtC,UAAM,SAAS,KAAK,GAAG,eAAe;AAAA,MACpC,UAAU,KAAK,IAAI,CAAA,MAAK,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,SAAS;AAAA,QAC/C,MAAM;AAAA,MAAA,CACP,UAAU,MAAM,KAAK,OAAO,CAAC;AAAA,IAAA;AAEhC,WAAO,UAAU,aAAa,IAAI;AAClC,QAAI;AAEF,aAAO,OAAO,UAAU,IAAS,OAAO,OAAO,GAAG,CAAC;AAAA,IACrD,UAAA;AACE,WAAK,GAAG,eAAe,OAAO,MAAM;AAAA,IACtC;AAAA,EACF;AAAA,EAEA,QAAQ,OAA8B,MAAoB,KAAe;AACvE,UAAM,QAAQ,KAAK,IAAI,CAAA,QAAO,IAAI,IAAI,CAAA,MAAK,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC;AACxD,UAAM,OAAO,OAAO,KAAK,MAAM,OAAO;AACtC,UAAM,SAAS,KAAK,GAAG,eAAe;AAAA,MACpC,UAAU,KAAK,IAAI,CAAA,MAAK,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,SAAS;AAAA,QAC/C,MAAM;AAAA,MAAA,CACP,UAAU,MAAM,IAAI,CAAA,SAAQ,KAAK,KAAK,OAAO,CAAC,EAAE,KAAK,MAAM,CAAC;AAAA,IAAA;AAE/D,WAAO,UAAU,aAAa,IAAI;AAClC,QAAI;AAEF,aAAO,OAAO,UAAU;AAAA,QACtB,KAAK,QAAQ,CAAA,QAAO,IAAI,IAAI,CAAA,WAAU,IAAI,MAAM,CAAC,CAAC;AAAA,MAAA;AAAA,IAEtD,UAAA;AACE,WAAK,GAAG,eAAe,OAAO,MAAM;AAAA,IACtC;AAAA,EACF;AAAA,EAEA,cAAwB;AACtB,SAAK,GAAG,SAAA;AACR,WAAO,IAAI,SAAS,KAAK,IAAI,KAAK,MAAM;AAAA,EAC1C;AACF;AAEA,MAAM,KAA6B;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAET,YACE,OACA,QACA,MACA,MACA;AACA,SAAK,oBAAoB,GAAG,KAAK;AACjC,SAAK,SAAS;AACd,SAAK,OAAO;AACZ,SAAK,OAAO;AACZ,SAAK,UAAU,KAAK,gBAAgB,KAAK,OAAO;AAAA,EAClD;AAAA,EAEA,CAAC,OAAO,QAAQ,IAAsB;AACpC,UAAM,EAAC,SAAS,SAAS,SAAQ,KAAK,KAAK,aAAa,KAAK,KAAK,OAAO;AAEzE,UAAM,UAAU,MAAM;AACpB,UAAI;AAEF,gBAAQ,SAAS,MAAS;AAAA,MAC5B,UAAA;AACE,aAAA;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,MACL,MAAM,MAAM;AACV,YAAI;AACF,qBAAS;AACP,kBAAM,EAAC,OAAO,MAAAA,MAAAA,IAAQ,QAAQ,KAAA;AAC9B,gBAAIA,OAAM;AACR,sBAAA;AACA,qBAAO,EAAC,OAAO,MAAM,KAAA;AAAA,YACvB;AAEA,kBAAM,EAAC,OAAO,QAAQ,IAAI,iBAAgBC,MAAQ,OAAOC,oBAAM;AAC/D,gBAAI,OAAO,UAAU;AAEnB,oBAAM,IAAI;AAAA,gBACR,oBAAoB,KAAK;AAAA,cAAA;AAAA,YAE7B;AACA,gBAAI,OAAO,aAAa;AAEtB,oBAAM,IAAI;AAAA,gBACR,SAAS,KAAK;AAAA,cAAA;AAAA,YAElB;AACA,kBAAM,EAAC,WAAW,YAAW,KAAK,KAAK,OAAO,IAAI,KAAK,CAAC;AAExD,mBAAO,WAAW,IAAI;AACtB,kBAAM,YACJ,OAAO,SAAS,KAAK,KAAK,OAAO,WAAW,MAAM,IAAI;AACxD,gBAAI;AACJ,gBAAI,WAAW;AACb,2BAAa,KAAK,KAAK;AAAA,gBACrB;AAAA,gBACA,UAAU;AAAA,gBACV;AAAA,cAAA;AAAA,YAEJ,OAAO;AACL,oBAAM,YAAY,KAAK,KAAK,OAAO,WAAW,MAAM;AACpD,2BAAa,YAAY,CAAC,SAAS,IAAI,CAAA;AAAA,YACzC;AACA,gBAAI,cAAc,QAAW;AAC3B,oBAAM,IAAI;AAAA,gBACR,qBAAqB,KAAK,IAAI,UAAU,MAAM,CAAC;AAAA,cAAA;AAAA,YAEnD;AAEA,iBAAK,qBAAqB,cAAc,IAAI,YAAY,SAAS;AAEjE,gBAAI,WAAW,WAAW,KAAK,cAAc,MAAM;AAGjD;AAAA,YACF;AAEA,gBACE,UAAU,KAAK,qBACf,WAAW;AAAA,cACT,CAAA,cAAa,UAAU,gBAAgB,UAAU;AAAA,YAAA,GAEnD;AACA,oBAAM,IAAI;AAAA,gBACR,4BACE,WAAW;AAAA,kBACT,CAAA,cACE,UAAU,gBAAgB,UAAU;AAAA,gBAAA,EACtC,IACJ,OAAO,UAAU,IAAI;AAAA,cAAA;AAAA,YAEzB;AAKA,mBAAO;AAAA,cACL,OAAO;AAAA,gBACL;AAAA,gBACA,YAAY,WAAW;AAAA,kBAAI,CAAA,cACzB,gBAAgB,SAAS,WAAW,KAAK;AAAA,gBAAA;AAAA,gBAE3C,WAAW,YACP,gBAAgB,SAAS,WAAW,KAAK,IACzC;AAAA,gBACJ;AAAA,cAAA;AAAA,YACF;AAAA,UAEJ;AAAA,QACF,SAAS,GAAG;AAEV,kBAAA;AACA,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,MAEA,QAAQ,CAAC,UAAmB;AAC1B,gBAAA;AACA,eAAO,EAAC,OAAO,MAAM,KAAA;AAAA,MACvB;AAAA,IAAA;AAAA,EAEJ;AAAA,EAEA,qBACE,cACA,IACA,YACA,WACA;AAGA,QAAI,eAAe,KAAK,KAAK,SAAS;AACpC,YAAM,IAAI;AAAA,QACR,sDAAsD,KAAK,KAAK,OAAO;AAAA,MAAA;AAAA,IAE3E;AACA,QACE,WAAW;AAAA,MACT,gBAAc,UAAUC,wBAAW,KAAK,OAAO,KAAK,KAAK;AAAA,IAAA,MACrD,IACN;AACA,YAAM,IAAI;AAAA,QACR,sDAAsD,KAAK,KAAK,OAAO;AAAA,MAAA;AAAA,IAE3E;AACA,QAAI,OAAO,UAAU,UAAUA,wBAAW,MAAM,cAAc;AAC5D,YAAM,IAAI;AAAA,QACR;AAAA,MAAA;AAAA,IAEJ;AAAA,EACF;AACF;AAEO,MAAM,yBAAyB,MAAM;AAAA,EAC1C,YAAY,KAAa;AACvB,UAAM,GAAG;AAAA,EACX;AACF;"}
+{"version":3,"file":"snapshotter.js","sources":["../../../../../../zero-cache/src/services/view-syncer/snapshotter.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {stringify, type JSONValue} from '../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {Row} from '../../../../zero-protocol/src/data.ts';\nimport type {PrimaryKey} from '../../../../zero-types/src/schema.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {fromSQLiteTypes} from '../../../../zqlite/src/table-source.ts';\nimport type {LiteAndZqlSpec, LiteTableSpecWithKeys} from '../../db/specs.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {\n  normalizedKeyOrder,\n  type RowKey,\n  type RowValue,\n} from '../../types/row-key.ts';\nimport type {AppID} from '../../types/shards.ts';\nimport {id} from '../../types/sql.ts';\nimport {\n  RESET_OP,\n  changeLogEntrySchema as schema,\n  SET_OP,\n  TRUNCATE_OP,\n} from '../replicator/schema/change-log.ts';\nimport {\n  getReplicationState,\n  ZERO_VERSION_COLUMN_NAME as ROW_VERSION,\n} from '../replicator/schema/replication-state.ts';\n\n/**\n * A `Snapshotter` manages the progression of database snapshots for a\n * ViewSyncer.\n *\n * The Replicator and ViewSyncers operate on the same SQLite file, with the\n * Replicator being the sole writer to the database. The IVM logic in\n * ViewSyncers, however, rely on incrementally applying changes to the DB to\n * update the state of its pipelines.\n *\n * To avoid coupling the progress of the Replicator and all IVM pipelines on\n * each other, ViewSyncers operate on ephemeral forks of the database by holding\n * [concurrent](https://sqlite.org/src/doc/begin-concurrent/doc/begin_concurrent.md)\n * snapshots of the database and simulating (but ultimately rolling back)\n * mutations on these snapshots.\n *\n * Example:\n * 1. ViewSyncer takes `snapshot_a` at version `t1` of the database and\n *    hydrates its pipeline(s).\n * 2. Replicator applies a new transaction to the database and notifies\n *    subscribers.\n * 3. ViewSyncer takes `snapshot_b` at `t2`, and queries the `ChangeLog` at\n *    that snapshot for changes since `t1`.\n * 4. ViewSyncer applies those changes to `snapshot_a` for IVM, but does not\n *    commit them. (Recall that the Replicator is the sole writer to the db, so\n *    the ViewSyncer never commits any writes.)\n * 5. Replicator applies the next transaction and advances the database to `t3`.\n * 6. ViewSyncer rolls back `snapshot_a` and opens `snapshot_c` at `t3`, using\n *    `snapshot_b` to simulate changes from `t2` to `t3`.\n *\n * ```\n * Replicator:  t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n * ```\n *\n * Note that the Replicator (and ViewSyncers) do not wait on the progress of\n * other ViewSyncers. If a ViewSyncer is busy hydrating at `t1`, the Replicator\n * and other ViewSyncers can progress through `t2`, `t3`, etc. independently,\n * as the busy ViewSyncer simply takes its own snapshot when it is ready.\n *\n * ```\n * Replicator:   t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer1:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n * ViewSyncer2:       [.......... snapshot_a ..........] ----> [snapshot_b]\n * ```\n *\n * To minimize Database connections (and statement preparation, etc.), the\n * Snapshotter reuses the connection from the previous (rolled back)\n * snapshot when opening the new one.\n *\n * ```\n * Replicator:  t1 --------------> t2 --------------> t3 --------------->\n * ViewSyncer:       [snapshot_a] ----> [snapshot_b] ----> [snapshot_c]\n *                     (conn_1)           (conn_2)           (conn_1)\n * ```\n *\n * In this manner, each ViewSyncer uses two connections that continually\n * \"leapfrog\" each other to replay the timeline of changes in isolation from\n * the Replicator and other ViewSyncers.\n */\nexport class Snapshotter {\n  readonly #lc: LogContext;\n  readonly #dbFile: string;\n  readonly #appID: string;\n  readonly #pageCacheSizeKib: number | undefined;\n  #curr: Snapshot | undefined;\n  #prev: Snapshot | undefined;\n\n  constructor(\n    lc: LogContext,\n    dbFile: string,\n    {appID}: AppID,\n    pageCacheSizeKib?: number | undefined,\n  ) {\n    this.#lc = lc;\n    this.#dbFile = dbFile;\n    this.#appID = appID;\n    this.#pageCacheSizeKib = pageCacheSizeKib;\n  }\n\n  /**\n   * Initializes the snapshot to the current head of the database. This must be\n   * only be called once. The state of whether a Snapshotter has been initialized\n   * can be determined by calling {@link initialized()}.\n   */\n  init(): this {\n    assert(this.#curr === undefined, 'Already initialized');\n    this.#curr = Snapshot.create(\n      this.#lc,\n      this.#dbFile,\n      this.#appID,\n      this.#pageCacheSizeKib,\n    );\n    this.#lc.debug?.(`Initial snapshot at version ${this.#curr.version}`);\n    return this;\n  }\n\n  initialized(): boolean {\n    return this.#curr !== undefined;\n  }\n\n  /** Returns the current snapshot. Asserts if {@link initialized()} is false. */\n  current(): Snapshot {\n    assert(this.#curr !== undefined, 'Snapshotter has not been initialized');\n    return this.#curr;\n  }\n\n  /**\n   * Advances to the head of the Database, returning a diff between the\n   * previously current Snapshot and a new Snapshot at head. This is called\n   * in response to a notification from a Replicator subscription. Subsequent\n   * calls to {@link current()} return the new Snapshot. Note that the Snapshotter\n   * must be initialized before advancing.\n   *\n   * The returned {@link SnapshotDiff} contains snapshots at the endpoints\n   * of the database timeline. Iterating over the diff generates a sequence\n   * of {@link Change}s between the two snapshots.\n   *\n   * Note that this sequence is not chronological; rather, the sequence is\n   * ordered by `<table, row-key>`, such that a row can appear at most once\n   * in the common case, or twice if its table is `TRUNCATE`'d and a new value\n   * is subsequently `INSERT`'ed. This results in dropping most intermediate\n   * changes to a row and bounds the amount of work needed to catch up;\n   * however, as a consequence, a consistent database state is only guaranteed\n   * when the sequence has been fully consumed.\n   *\n   * Note that Change generation relies on the state of the underlying\n   * database connections, and because the connection for the previous snapshot\n   * is reused to produce the next snapshot, the diff object is only valid\n   * until the next call to `advance()`.\n   *\n   * It is okay for the caller to apply `Change`s to the `prev` snapshot\n   * during the iteration (e.g. this is necessary for IVM); the remainder\n   * of the iteration is not affected because a given row can appear at most\n   * once in the sequence (with the exception being TRUNCATE, after which the\n   * deleted rows can be re-inserted, but this will also behave correctly if\n   * the changes are applied).\n   *\n   * Once the changes have been applied, however, a _subsequent_ iteration\n   * will not produce the correct results. In order to perform multiple\n   * change-applying iterations, the caller must (1) create a save point\n   * on `prev` before each iteration, and (2) rollback to the save point after\n   * the iteration.\n   */\n  advance(tables: Map<string, LiteAndZqlSpec>): SnapshotDiff {\n    const {prev, curr} = this.advanceWithoutDiff();\n    return new Diff(this.#appID, tables, prev, curr);\n  }\n\n  advanceWithoutDiff() {\n    assert(this.#curr !== undefined, 'Snapshotter has not been initialized');\n    const next = this.#prev\n      ? this.#prev.resetToHead()\n      : Snapshot.create(\n          this.#lc,\n          this.#curr.db.db.name,\n          this.#appID,\n          this.#pageCacheSizeKib,\n        );\n    this.#prev = this.#curr;\n    this.#curr = next;\n    return {prev: this.#prev, curr: this.#curr};\n  }\n\n  /**\n   * Call this to close the database connections when the Snapshotter is\n   * no longer needed.\n   */\n  destroy() {\n    this.#curr?.db.db.close();\n    this.#prev?.db.db.close();\n    this.#lc.debug?.('closed database connections');\n  }\n}\n\nexport type Change = {\n  readonly table: string;\n  /**\n   * If this change represents a remove the row to remove,\n   * if nextValue is not null then all rows that have a unique constraint\n   * violation with nextValue.\n   * In both cases these rows should be removed.\n   */\n  readonly prevValues: Readonly<Row>[];\n  readonly nextValue: Readonly<Row> | null;\n  readonly rowKey: RowKey;\n};\n\n/**\n * Represents the difference between two database Snapshots.\n * Iterating over the object will produce a sequence of {@link Change}s\n * between the two snapshots.\n *\n * See {@link Snapshotter.advance()} for semantics and usage.\n */\nexport interface SnapshotDiff extends Iterable<Change> {\n  readonly prev: {\n    readonly db: StatementRunner;\n    readonly version: string;\n  };\n  readonly curr: {\n    readonly db: StatementRunner;\n    readonly version: string;\n  };\n\n  /**\n   * The number of ChangeLog entries between the snapshots. Note that this\n   * may not necessarily equal the number of `Change` objects that the iteration\n   * will produce, as `TRUNCATE` entries are counted as a single log entry which\n   * may be expanded into many changes (i.e. row deletes).\n   *\n   * TODO: Determine if it is worth changing the definition to count the\n   *       truncated rows. This would make diff computation more expensive\n   *       (requiring the count to be aggregated by operation type), which\n   *       may not be worth it for a presumable rare operation.\n   */\n  readonly changes: number;\n}\n\n/**\n * Thrown during an iteration of a {@link SnapshotDiff} when a schema\n * change or truncate is encountered, which result in aborting the\n * advancement and resetting / rehydrating the pipelines.\n */\nexport class ResetPipelinesSignal extends Error {\n  readonly name = 'ResetPipelinesSignal';\n\n  constructor(msg: string) {\n    super(msg);\n  }\n}\n\nclass Snapshot {\n  static create(\n    lc: LogContext,\n    dbFile: string,\n    appID: string,\n    pageCacheSizeKib: number | undefined,\n  ) {\n    const conn = new Database(lc, dbFile);\n    conn.pragma('synchronous = OFF'); // Applied changes are ephemeral; COMMIT is never called.\n    if (pageCacheSizeKib !== undefined) {\n      conn.pragma(`cache_size = -${pageCacheSizeKib}`); // Negative = size in KiB\n    }\n    const [{journal_mode: mode}] = conn.pragma('journal_mode') as [\n      {journal_mode: string},\n    ];\n    // The Snapshotter operates on the replica file with BEGIN CONCURRENT,\n    // which must be used in concert with the replicator using BEGIN CONCURRENT\n    // on a db in the wal2 journal_mode.\n    assert(\n      mode === 'wal2',\n      `replica db must be in wal2 mode (current: ${mode})`,\n    );\n\n    const db = new StatementRunner(conn);\n    return new Snapshot(db, appID);\n  }\n\n  readonly db: StatementRunner;\n  readonly #appID: string;\n  readonly version: string;\n\n  constructor(db: StatementRunner, appID: string) {\n    db.beginConcurrent();\n    // Note: The subsequent read is necessary to acquire the read lock\n    // (which results in the logical creation of the snapshot). Calling\n    // `BEGIN CONCURRENT` alone does not result in acquiring the read lock.\n    const {stateVersion} = getReplicationState(db);\n\n    this.db = db;\n    this.#appID = appID;\n    this.version = stateVersion;\n  }\n\n  numChangesSince(prevVersion: string) {\n    const {count} = this.db.get(\n      'SELECT COUNT(*) AS count FROM \"_zero.changeLog2\" WHERE stateVersion > ?',\n      prevVersion,\n    );\n    return count;\n  }\n\n  changesSince(prevVersion: string) {\n    const cached = this.db.statementCache.get(\n      'SELECT * FROM \"_zero.changeLog2\" WHERE stateVersion > ? ORDER BY stateVersion ASC, pos ASC',\n    );\n    return {\n      changes: cached.statement.iterate(prevVersion),\n      cleanup: () => this.db.statementCache.return(cached),\n    };\n  }\n\n  getRow(table: LiteTableSpecWithKeys, rowKey: JSONValue) {\n    const key = normalizedKeyOrder(rowKey as RowKey);\n    const conds = Object.keys(key).map(c => `${id(c)}=?`);\n    const cols = Object.keys(table.columns);\n    const cached = this.db.statementCache.get(\n      `SELECT ${cols.map(c => id(c)).join(',')} FROM ${id(\n        table.name,\n      )} WHERE ${conds.join(' AND ')}`,\n    );\n    cached.statement.safeIntegers(true);\n    try {\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      return cached.statement.get<any>(Object.values(key));\n    } finally {\n      this.db.statementCache.return(cached);\n    }\n  }\n\n  getRows(table: LiteTableSpecWithKeys, keys: PrimaryKey[], row: RowValue) {\n    const conds = keys.map(key => key.map(c => `${id(c)}=?`));\n    const cols = Object.keys(table.columns);\n    const cached = this.db.statementCache.get(\n      `SELECT ${cols.map(c => id(c)).join(',')} FROM ${id(\n        table.name,\n      )} WHERE ${conds.map(cond => cond.join(' AND ')).join(' OR ')}`,\n    );\n    cached.statement.safeIntegers(true);\n    try {\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n      return cached.statement.all<any>(\n        keys.flatMap(key => key.map(column => row[column])),\n      );\n    } finally {\n      this.db.statementCache.return(cached);\n    }\n  }\n\n  resetToHead(): Snapshot {\n    this.db.rollback();\n    return new Snapshot(this.db, this.#appID);\n  }\n}\n\nclass Diff implements SnapshotDiff {\n  readonly #permissionsTable: string;\n  readonly tables: Map<string, LiteAndZqlSpec>;\n  readonly prev: Snapshot;\n  readonly curr: Snapshot;\n  readonly changes: number;\n\n  constructor(\n    appID: string,\n    tables: Map<string, LiteAndZqlSpec>,\n    prev: Snapshot,\n    curr: Snapshot,\n  ) {\n    this.#permissionsTable = `${appID}.permissions`;\n    this.tables = tables;\n    this.prev = prev;\n    this.curr = curr;\n    this.changes = curr.numChangesSince(prev.version);\n  }\n\n  [Symbol.iterator](): Iterator<Change> {\n    const {changes, cleanup: done} = this.curr.changesSince(this.prev.version);\n\n    const cleanup = () => {\n      try {\n        // Allow open iterators to clean up their state.\n        changes.return?.(undefined);\n      } finally {\n        done();\n      }\n    };\n\n    return {\n      next: () => {\n        try {\n          for (;;) {\n            const {value, done} = changes.next();\n            if (done) {\n              cleanup();\n              return {value, done: true};\n            }\n\n            const {table, rowKey, op, stateVersion} = v.parse(value, schema);\n            if (op === RESET_OP) {\n              // The current map of `TableSpec`s may not have the correct or complete information.\n              throw new ResetPipelinesSignal(\n                `schema for table ${table} has changed`,\n              );\n            }\n            if (op === TRUNCATE_OP) {\n              // Truncates are also processed by rehydrating pipelines at current.\n              throw new ResetPipelinesSignal(\n                `table ${table} has been truncated`,\n              );\n            }\n            const {tableSpec, zqlSpec} = must(this.tables.get(table));\n\n            assert(rowKey !== null);\n            const nextValue =\n              op === SET_OP ? this.curr.getRow(tableSpec, rowKey) : null;\n            let prevValues;\n            if (nextValue) {\n              prevValues = this.prev.getRows(\n                tableSpec,\n                tableSpec.uniqueKeys,\n                nextValue,\n              );\n            } else {\n              const prevValue = this.prev.getRow(tableSpec, rowKey);\n              prevValues = prevValue ? [prevValue] : [];\n            }\n            if (nextValue === undefined) {\n              throw new Error(\n                `Missing value for ${table} ${stringify(rowKey)}`,\n              );\n            }\n            // Sanity check detects if the diff is being accessed after the Snapshots have advanced.\n            this.checkThatDiffIsValid(stateVersion, op, prevValues, nextValue);\n\n            if (prevValues.length === 0 && nextValue === null) {\n              // Filter out no-op changes (e.g. a delete of a row that does not exist in prev).\n              // TODO: Consider doing this for deep-equal values.\n              continue;\n            }\n\n            if (\n              table === this.#permissionsTable &&\n              prevValues.find(\n                prevValue => prevValue.permissions !== nextValue.permissions,\n              )\n            ) {\n              throw new ResetPipelinesSignal(\n                `Permissions have changed ${\n                  prevValues.find(\n                    prevValue =>\n                      prevValue.permissions !== nextValue.permissions,\n                  ).hash\n                } => ${nextValue.hash}`,\n              );\n            }\n\n            // Modify the values in place when converting to ZQL rows\n            // This is safe since we're the first node in the iterator chain.\n            // TODO Can we get rid of these RowValue casts?\n            return {\n              value: {\n                table,\n                prevValues: prevValues.map(prevValue =>\n                  fromSQLiteTypes(zqlSpec, prevValue, table),\n                ),\n                nextValue: nextValue\n                  ? fromSQLiteTypes(zqlSpec, nextValue, table)\n                  : null,\n                rowKey,\n              } satisfies Change,\n            };\n          }\n        } catch (e) {\n          // This control flow path is not covered by the return() method (i.e. `break`).\n          cleanup();\n          throw e;\n        }\n      },\n\n      return: (value: unknown) => {\n        cleanup();\n        return {value, done: true};\n      },\n    };\n  }\n\n  checkThatDiffIsValid(\n    stateVersion: string,\n    op: string,\n    prevValues: RowValue[],\n    nextValue: RowValue,\n  ) {\n    // Sanity checks to detect that the diff is not being accessed after\n    // the Snapshots have advanced.\n    if (stateVersion > this.curr.version) {\n      throw new InvalidDiffError(\n        `Diff is no longer valid. curr db has advanced past ${this.curr.version}`,\n      );\n    }\n    if (\n      prevValues.findIndex(\n        prevValue => (prevValue[ROW_VERSION] ?? '~') > this.prev.version,\n      ) !== -1\n    ) {\n      throw new InvalidDiffError(\n        `Diff is no longer valid. prev db has advanced past ${this.prev.version}.`,\n      );\n    }\n    if (op === SET_OP && nextValue[ROW_VERSION] !== stateVersion) {\n      throw new InvalidDiffError(\n        'Diff is no longer valid. curr db has advanced.',\n      );\n    }\n  }\n}\n\nexport class InvalidDiffError extends Error {\n  constructor(msg: string) {\n    super(msg);\n  }\n}\n"],"names":["done","v.parse","schema","ROW_VERSION"],"mappings":";;;;;;;;;;;;AAwFO,MAAM,YAAY;AAAA,EACd;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACT;AAAA,EACA;AAAA,EAEA,YACE,IACA,QACA,EAAC,MAAA,GACD,kBACA;AACA,SAAK,MAAM;AACX,SAAK,UAAU;AACf,SAAK,SAAS;AACd,SAAK,oBAAoB;AAAA,EAC3B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,OAAa;AACX,WAAO,KAAK,UAAU,QAAW,qBAAqB;AACtD,SAAK,QAAQ,SAAS;AAAA,MACpB,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,IAAA;AAEP,SAAK,IAAI,QAAQ,+BAA+B,KAAK,MAAM,OAAO,EAAE;AACpE,WAAO;AAAA,EACT;AAAA,EAEA,cAAuB;AACrB,WAAO,KAAK,UAAU;AAAA,EACxB;AAAA;AAAA,EAGA,UAAoB;AAClB,WAAO,KAAK,UAAU,QAAW,sCAAsC;AACvE,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAuCA,QAAQ,QAAmD;AACzD,UAAM,EAAC,MAAM,SAAQ,KAAK,mBAAA;AAC1B,WAAO,IAAI,KAAK,KAAK,QAAQ,QAAQ,MAAM,IAAI;AAAA,EACjD;AAAA,EAEA,qBAAqB;AACnB,WAAO,KAAK,UAAU,QAAW,sCAAsC;AACvE,UAAM,OAAO,KAAK,QACd,KAAK,MAAM,YAAA,IACX,SAAS;AAAA,MACP,KAAK;AAAA,MACL,KAAK,MAAM,GAAG,GAAG;AAAA,MACjB,KAAK;AAAA,MACL,KAAK;AAAA,IAAA;AAEX,SAAK,QAAQ,KAAK;AAClB,SAAK,QAAQ;AACb,WAAO,EAAC,MAAM,KAAK,OAAO,MAAM,KAAK,MAAA;AAAA,EACvC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,UAAU;AACR,SAAK,OAAO,GAAG,GAAG,MAAA;AAClB,SAAK,OAAO,GAAG,GAAG,MAAA;AAClB,SAAK,IAAI,QAAQ,6BAA6B;AAAA,EAChD;AACF;AAmDO,MAAM,6BAA6B,MAAM;AAAA,EACrC,OAAO;AAAA,EAEhB,YAAY,KAAa;AACvB,UAAM,GAAG;AAAA,EACX;AACF;AAEA,MAAM,SAAS;AAAA,EACb,OAAO,OACL,IACA,QACA,OACA,kBACA;AACA,UAAM,OAAO,IAAI,SAAS,IAAI,MAAM;AACpC,SAAK,OAAO,mBAAmB;AAC/B,QAAI,qBAAqB,QAAW;AAClC,WAAK,OAAO,iBAAiB,gBAAgB,EAAE;AAAA,IACjD;AACA,UAAM,CAAC,EAAC,cAAc,KAAA,CAAK,IAAI,KAAK,OAAO,cAAc;AAMzD;AAAA,MACE,SAAS;AAAA,MACT,6CAA6C,IAAI;AAAA,IAAA;AAGnD,UAAM,KAAK,IAAI,gBAAgB,IAAI;AACnC,WAAO,IAAI,SAAS,IAAI,KAAK;AAAA,EAC/B;AAAA,EAES;AAAA,EACA;AAAA,EACA;AAAA,EAET,YAAY,IAAqB,OAAe;AAC9C,OAAG,gBAAA;AAIH,UAAM,EAAC,aAAA,IAAgB,oBAAoB,EAAE;AAE7C,SAAK,KAAK;AACV,SAAK,SAAS;AACd,SAAK,UAAU;AAAA,EACjB;AAAA,EAEA,gBAAgB,aAAqB;AACnC,UAAM,EAAC,MAAA,IAAS,KAAK,GAAG;AAAA,MACtB;AAAA,MACA;AAAA,IAAA;AAEF,WAAO;AAAA,EACT;AAAA,EAEA,aAAa,aAAqB;AAChC,UAAM,SAAS,KAAK,GAAG,eAAe;AAAA,MACpC;AAAA,IAAA;AAEF,WAAO;AAAA,MACL,SAAS,OAAO,UAAU,QAAQ,WAAW;AAAA,MAC7C,SAAS,MAAM,KAAK,GAAG,eAAe,OAAO,MAAM;AAAA,IAAA;AAAA,EAEvD;AAAA,EAEA,OAAO,OAA8B,QAAmB;AACtD,UAAM,MAAM,mBAAmB,MAAgB;AAC/C,UAAM,QAAQ,OAAO,KAAK,GAAG,EAAE,IAAI,CAAA,MAAK,GAAG,GAAG,CAAC,CAAC,IAAI;AACpD,UAAM,OAAO,OAAO,KAAK,MAAM,OAAO;AACtC,UAAM,SAAS,KAAK,GAAG,eAAe;AAAA,MACpC,UAAU,KAAK,IAAI,CAAA,MAAK,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,SAAS;AAAA,QAC/C,MAAM;AAAA,MAAA,CACP,UAAU,MAAM,KAAK,OAAO,CAAC;AAAA,IAAA;AAEhC,WAAO,UAAU,aAAa,IAAI;AAClC,QAAI;AAEF,aAAO,OAAO,UAAU,IAAS,OAAO,OAAO,GAAG,CAAC;AAAA,IACrD,UAAA;AACE,WAAK,GAAG,eAAe,OAAO,MAAM;AAAA,IACtC;AAAA,EACF;AAAA,EAEA,QAAQ,OAA8B,MAAoB,KAAe;AACvE,UAAM,QAAQ,KAAK,IAAI,CAAA,QAAO,IAAI,IAAI,CAAA,MAAK,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC;AACxD,UAAM,OAAO,OAAO,KAAK,MAAM,OAAO;AACtC,UAAM,SAAS,KAAK,GAAG,eAAe;AAAA,MACpC,UAAU,KAAK,IAAI,CAAA,MAAK,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,SAAS;AAAA,QAC/C,MAAM;AAAA,MAAA,CACP,UAAU,MAAM,IAAI,CAAA,SAAQ,KAAK,KAAK,OAAO,CAAC,EAAE,KAAK,MAAM,CAAC;AAAA,IAAA;AAE/D,WAAO,UAAU,aAAa,IAAI;AAClC,QAAI;AAEF,aAAO,OAAO,UAAU;AAAA,QACtB,KAAK,QAAQ,CAAA,QAAO,IAAI,IAAI,CAAA,WAAU,IAAI,MAAM,CAAC,CAAC;AAAA,MAAA;AAAA,IAEtD,UAAA;AACE,WAAK,GAAG,eAAe,OAAO,MAAM;AAAA,IACtC;AAAA,EACF;AAAA,EAEA,cAAwB;AACtB,SAAK,GAAG,SAAA;AACR,WAAO,IAAI,SAAS,KAAK,IAAI,KAAK,MAAM;AAAA,EAC1C;AACF;AAEA,MAAM,KAA6B;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAET,YACE,OACA,QACA,MACA,MACA;AACA,SAAK,oBAAoB,GAAG,KAAK;AACjC,SAAK,SAAS;AACd,SAAK,OAAO;AACZ,SAAK,OAAO;AACZ,SAAK,UAAU,KAAK,gBAAgB,KAAK,OAAO;AAAA,EAClD;AAAA,EAEA,CAAC,OAAO,QAAQ,IAAsB;AACpC,UAAM,EAAC,SAAS,SAAS,SAAQ,KAAK,KAAK,aAAa,KAAK,KAAK,OAAO;AAEzE,UAAM,UAAU,MAAM;AACpB,UAAI;AAEF,gBAAQ,SAAS,MAAS;AAAA,MAC5B,UAAA;AACE,aAAA;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,MACL,MAAM,MAAM;AACV,YAAI;AACF,qBAAS;AACP,kBAAM,EAAC,OAAO,MAAAA,MAAAA,IAAQ,QAAQ,KAAA;AAC9B,gBAAIA,OAAM;AACR,sBAAA;AACA,qBAAO,EAAC,OAAO,MAAM,KAAA;AAAA,YACvB;AAEA,kBAAM,EAAC,OAAO,QAAQ,IAAI,iBAAgBC,MAAQ,OAAOC,oBAAM;AAC/D,gBAAI,OAAO,UAAU;AAEnB,oBAAM,IAAI;AAAA,gBACR,oBAAoB,KAAK;AAAA,cAAA;AAAA,YAE7B;AACA,gBAAI,OAAO,aAAa;AAEtB,oBAAM,IAAI;AAAA,gBACR,SAAS,KAAK;AAAA,cAAA;AAAA,YAElB;AACA,kBAAM,EAAC,WAAW,YAAW,KAAK,KAAK,OAAO,IAAI,KAAK,CAAC;AAExD,mBAAO,WAAW,IAAI;AACtB,kBAAM,YACJ,OAAO,SAAS,KAAK,KAAK,OAAO,WAAW,MAAM,IAAI;AACxD,gBAAI;AACJ,gBAAI,WAAW;AACb,2BAAa,KAAK,KAAK;AAAA,gBACrB;AAAA,gBACA,UAAU;AAAA,gBACV;AAAA,cAAA;AAAA,YAEJ,OAAO;AACL,oBAAM,YAAY,KAAK,KAAK,OAAO,WAAW,MAAM;AACpD,2BAAa,YAAY,CAAC,SAAS,IAAI,CAAA;AAAA,YACzC;AACA,gBAAI,cAAc,QAAW;AAC3B,oBAAM,IAAI;AAAA,gBACR,qBAAqB,KAAK,IAAI,UAAU,MAAM,CAAC;AAAA,cAAA;AAAA,YAEnD;AAEA,iBAAK,qBAAqB,cAAc,IAAI,YAAY,SAAS;AAEjE,gBAAI,WAAW,WAAW,KAAK,cAAc,MAAM;AAGjD;AAAA,YACF;AAEA,gBACE,UAAU,KAAK,qBACf,WAAW;AAAA,cACT,CAAA,cAAa,UAAU,gBAAgB,UAAU;AAAA,YAAA,GAEnD;AACA,oBAAM,IAAI;AAAA,gBACR,4BACE,WAAW;AAAA,kBACT,CAAA,cACE,UAAU,gBAAgB,UAAU;AAAA,gBAAA,EACtC,IACJ,OAAO,UAAU,IAAI;AAAA,cAAA;AAAA,YAEzB;AAKA,mBAAO;AAAA,cACL,OAAO;AAAA,gBACL;AAAA,gBACA,YAAY,WAAW;AAAA,kBAAI,CAAA,cACzB,gBAAgB,SAAS,WAAW,KAAK;AAAA,gBAAA;AAAA,gBAE3C,WAAW,YACP,gBAAgB,SAAS,WAAW,KAAK,IACzC;AAAA,gBACJ;AAAA,cAAA;AAAA,YACF;AAAA,UAEJ;AAAA,QACF,SAAS,GAAG;AAEV,kBAAA;AACA,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,MAEA,QAAQ,CAAC,UAAmB;AAC1B,gBAAA;AACA,eAAO,EAAC,OAAO,MAAM,KAAA;AAAA,MACvB;AAAA,IAAA;AAAA,EAEJ;AAAA,EAEA,qBACE,cACA,IACA,YACA,WACA;AAGA,QAAI,eAAe,KAAK,KAAK,SAAS;AACpC,YAAM,IAAI;AAAA,QACR,sDAAsD,KAAK,KAAK,OAAO;AAAA,MAAA;AAAA,IAE3E;AACA,QACE,WAAW;AAAA,MACT,gBAAc,UAAUC,wBAAW,KAAK,OAAO,KAAK,KAAK;AAAA,IAAA,MACrD,IACN;AACA,YAAM,IAAI;AAAA,QACR,sDAAsD,KAAK,KAAK,OAAO;AAAA,MAAA;AAAA,IAE3E;AACA,QAAI,OAAO,UAAU,UAAUA,wBAAW,MAAM,cAAc;AAC5D,YAAM,IAAI;AAAA,QACR;AAAA,MAAA;AAAA,IAEJ;AAAA,EACF;AACF;AAEO,MAAM,yBAAyB,MAAM;AAAA,EAC1C,YAAY,KAAa;AACvB,UAAM,GAAG;AAAA,EACX;AACF;"}

package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts

@@ -27,7 +27,6 @@ export type SyncContext = {
     readonly profileID: string | null;
     readonly baseCookie: string | null;
     readonly protocolVersion: number;
-    readonly schemaVersion: number | null;
     readonly tokenData: TokenData | undefined;
     readonly httpCookie: string | undefined;
 };
@@ -55,7 +54,7 @@ export declare class ViewSyncerService implements ViewSyncer, ActivityBasedServi
     #private;
     readonly id: string;
     userQueryURL?: string | undefined;
-    constructor(config: NormalizedZeroConfig, lc: LogContext, shard: ShardID, taskID: string, clientGroupID: string, cvrDb: PostgresDB, upstreamDb: PostgresDB | undefined, pipelineDriver: PipelineDriver, versionChanges: Subscription<ReplicaState>, drainCoordinator: DrainCoordinator, slowHydrateThreshold: number, inspectorDelegate: InspectorDelegate, customQueryTransformer: CustomQueryTransformer | undefined, keepaliveMs?: number, setTimeoutFn?: SetTimeout);
+    constructor(config: NormalizedZeroConfig, lc: LogContext, shard: ShardID, taskID: string, clientGroupID: string, cvrDb: PostgresDB, upstreamDb: PostgresDB | undefined, pipelineDriver: PipelineDriver, versionChanges: Subscription<ReplicaState>, drainCoordinator: DrainCoordinator, slowHydrateThreshold: number, inspectorDelegate: InspectorDelegate, customQueryTransformer: CustomQueryTransformer | undefined, runPriorityOp: <T>(op: () => Promise<T>) => Promise<T>, keepaliveMs?: number, setTimeoutFn?: SetTimeout);
     readyState(): Promise<'initialized' | 'draining'>;
     run(): Promise<void>;
     /**

package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"view-syncer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/view-syncer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,MAAM,CAAC;AAcrC,OAAO,KAAK,EAAC,2BAA2B,EAAC,MAAM,yDAAyD,CAAC;AACzG,OAAO,KAAK,EAEV,qBAAqB,EACtB,MAAM,0CAA0C,CAAC;AAElD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,iDAAiD,CAAC;AAC1F,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,uCAAuC,CAAC;AAOtE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,6CAA6C,CAAC;AAMrD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAEpE,OAAO,KAAK,EAAC,sBAAsB,EAAC,MAAM,yCAAyC,CAAC;AAOpF,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,oCAAoC,CAAC;AAK1E,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAElD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AACzD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAE9D,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,eAAe,CAAC;AAiBxD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AAE7D,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AAuBzD,MAAM,MAAM,SAAS,GAAG;IACtB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,kBAAkB;IAClB,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,SAAS,EAAE,SAAS,GAAG,SAAS,CAAC;IAC1C,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;CACzC,CAAC;AAMF,MAAM,WAAW,UAAU;IACzB,cAAc,CACZ,GAAG,EAAE,WAAW,EAChB,GAAG,EAAE,qBAAqB,GACzB,MAAM,CAAC,UAAU,CAAC,CAAC;IAEtB,oBAAoB,CAClB,GAAG,EAAE,WAAW,EAChB,GAAG,EAAE,2BAA2B,GAC/B,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB,aAAa,CAAC,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1E,OAAO,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACrE;AAQD,KAAK,UAAU,GAAG,CAChB,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,EAChC,KAAK,CAAC,EAAE,MAAM,KACX,UAAU,CAAC,OAAO,UAAU,CAAC,CAAC;AAEnC;;;;GAIG;AACH,eAAO,MAAM,kBAAkB,QAAS,CAAC;AAEzC;;;;;GAKG;AACH,eAAO,MAAM,oBAAoB,KAAK,CAAC;AAEvC,qBAAa,iBAAkB,YAAW,UAAU,EAAE,oBAAoB;;IACxE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAUpB,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;gBAoHhC,MAAM,EAAE,oBAAoB,EAC5B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,MAAM,EACrB,KAAK,EAAE,UAAU,EACjB,UAAU,EAAE,UAAU,GAAG,SAAS,EAClC,cAAc,EAAE,cAAc,EAC9B,cAAc,EAAE,YAAY,CAAC,YAAY,CAAC,EAC1C,gBAAgB,EAAE,gBAAgB,EAClC,oBAAoB,EAAE,MAAM,EAC5B,iBAAiB,EAAE,iBAAiB,EACpC,sBAAsB,EAAE,sBAAsB,GAAG,SAAS,EAC1D,WAAW,SAAuB,EAClC,YAAY,GAAE,UAAwC;IA2FxD,UAAU,IAAI,OAAO,CAAC,aAAa,GAAG,UAAU,CAAC;IAO3C,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAiH1B;;;;;;;;OAQG;IACH,SAAS,IAAI,OAAO;IAwEpB,cAAc,CACZ,GAAG,EAAE,WAAW,EAChB,qBAAqB,EAAE,qBAAqB,GAC3C,MAAM,CAAC,UAAU,CAAC;IAuHf,oBAAoB,CACxB,GAAG,EAAE,WAAW,EAChB,GAAG,EAAE,2BAA2B,GAC/B,OAAO,CAAC,IAAI,CAAC;IAIV,aAAa,CACjB,GAAG,EAAE,WAAW,EAChB,GAAG,EAAE,oBAAoB,GACxB,OAAO,CAAC,IAAI,CAAC;IAyuChB,OAAO,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IA2BnE,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAqBrB;;;OAGG;IACH,eAAe;CAGhB;AAuED,wBAAgB,SAAS,CACvB,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,SAAS,GAAG,SAAS,EACpC,QAAQ,EAAE,SAAS,GAAG,SAAS,yBAkDhC;AAyCD,qBAAa,cAAc;;IAInB,KAAK;IAOX,oBAAoB;IAMd,YAAY,CAAC,cAAc,CAAC,EAAE,MAAM;IAW1C,UAAU;IAWV,sCAAsC;IACtC,IAAI,IAAI,MAAM;IAKd;;;OAGG;IACH,YAAY,IAAI,MAAM;CAKvB"}
+{"version":3,"file":"view-syncer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/view-syncer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,MAAM,CAAC;AAcrC,OAAO,KAAK,EAAC,2BAA2B,EAAC,MAAM,yDAAyD,CAAC;AACzG,OAAO,KAAK,EAEV,qBAAqB,EACtB,MAAM,0CAA0C,CAAC;AAElD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,iDAAiD,CAAC;AAC1F,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,uCAAuC,CAAC;AAOtE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,6CAA6C,CAAC;AAMrD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAEpE,OAAO,KAAK,EAAC,sBAAsB,EAAC,MAAM,yCAAyC,CAAC;AAOpF,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,oCAAoC,CAAC;AAK1E,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAElD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AACzD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAE9D,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,eAAe,CAAC;AAiBxD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AAE7D,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AAuBzD,MAAM,MAAM,SAAS,GAAG;IACtB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,kBAAkB;IAClB,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,SAAS,GAAG,SAAS,CAAC;IAC1C,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;CACzC,CAAC;AAMF,MAAM,WAAW,UAAU;IACzB,cAAc,CACZ,GAAG,EAAE,WAAW,EAChB,GAAG,EAAE,qBAAqB,GACzB,MAAM,CAAC,UAAU,CAAC,CAAC;IAEtB,oBAAoB,CAClB,GAAG,EAAE,WAAW,EAChB,GAAG,EAAE,2BAA2B,GAC/B,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB,aAAa,CAAC,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1E,OAAO,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACrE;AAQD,KAAK,UAAU,GAAG,CAChB,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,EAChC,KAAK,CAAC,EAAE,MAAM,KACX,UAAU,CAAC,OAAO,UAAU,CAAC,CAAC;AAEnC;;;;GAIG;AACH,eAAO,MAAM,kBAAkB,QAAS,CAAC;AAEzC;;;;;GAKG;AACH,eAAO,MAAM,oBAAoB,KAAK,CAAC;AAEvC,qBAAa,iBAAkB,YAAW,UAAU,EAAE,oBAAoB;;IACxE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAUpB,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;gBAqHhC,MAAM,EAAE,oBAAoB,EAC5B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,MAAM,EACrB,KAAK,EAAE,UAAU,EACjB,UAAU,EAAE,UAAU,GAAG,SAAS,EAClC,cAAc,EAAE,cAAc,EAC9B,cAAc,EAAE,YAAY,CAAC,YAAY,CAAC,EAC1C,gBAAgB,EAAE,gBAAgB,EAClC,oBAAoB,EAAE,MAAM,EAC5B,iBAAiB,EAAE,iBAAiB,EACpC,sBAAsB,EAAE,sBAAsB,GAAG,SAAS,EAC1D,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,EACtD,WAAW,SAAuB,EAClC,YAAY,GAAE,UAAwC;IA6FxD,UAAU,IAAI,OAAO,CAAC,aAAa,GAAG,UAAU,CAAC;IAO3C,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAiH1B;;;;;;;;OAQG;IACH,SAAS,IAAI,OAAO;IAwEpB,cAAc,CACZ,GAAG,EAAE,WAAW,EAChB,qBAAqB,EAAE,qBAAqB,GAC3C,MAAM,CAAC,UAAU,CAAC;IAqHf,oBAAoB,CACxB,GAAG,EAAE,WAAW,EAChB,GAAG,EAAE,2BAA2B,GAC/B,OAAO,CAAC,IAAI,CAAC;IAIV,aAAa,CACjB,GAAG,EAAE,WAAW,EAChB,GAAG,EAAE,oBAAoB,GACxB,OAAO,CAAC,IAAI,CAAC;IA+tChB,OAAO,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IA2BnE,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAqBrB;;;OAGG;IACH,eAAe;CAGhB;AAuED,wBAAgB,SAAS,CACvB,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,SAAS,GAAG,SAAS,EACpC,QAAQ,EAAE,SAAS,GAAG,SAAS,yBAkDhC;AAyCD,qBAAa,cAAc;;IAInB,KAAK;IAOX,oBAAoB;IAMd,YAAY,CAAC,cAAc,CAAC,EAAE,MAAM;IAW1C,UAAU;IAWV,sCAAsC;IACtC,IAAI,IAAI,MAAM;IAKd;;;OAGG;IACH,YAAY,IAAI,MAAM;CAKvB"}

package/out/zero-cache/src/services/view-syncer/view-syncer.js

@@ -30,6 +30,7 @@ import "@rocicorp/zero-sqlite3";
 import "../../../../zqlite/src/internal/sql.js";
 import "@databases/escape-identifier";
 import "../../../../shared/src/btree-set.js";
+import "../../../../shared/src/iterables.js";
 import "../../../../shared/src/valita.js";
 import "../../../../zero-schema/src/compiled-permissions.js";
 import "../../../../zero-protocol/src/primary-key.js";
@@ -154,7 +155,8 @@ class ViewSyncerService {
   );
   #inspectorDelegate;
   #config;
-  constructor(config, lc, shard, taskID, clientGroupID, cvrDb, upstreamDb, pipelineDriver, versionChanges, drainCoordinator, slowHydrateThreshold, inspectorDelegate, customQueryTransformer, keepaliveMs = DEFAULT_KEEPALIVE_MS, setTimeoutFn = setTimeout.bind(globalThis)) {
+  #runPriorityOp;
+  constructor(config, lc, shard, taskID, clientGroupID, cvrDb, upstreamDb, pipelineDriver, versionChanges, drainCoordinator, slowHydrateThreshold, inspectorDelegate, customQueryTransformer, runPriorityOp, keepaliveMs = DEFAULT_KEEPALIVE_MS, setTimeoutFn = setTimeout.bind(globalThis)) {
     const queryConfig = config.query?.url ? config.query : config.getQueries;
     this.#config = config;
     this.id = clientGroupID;
@@ -180,6 +182,7 @@ class ViewSyncerService {
       () => this.#stateChanges.cancel()
     );
     this.#setTimeout = setTimeoutFn;
+    this.#runPriorityOp = runPriorityOp;
     this.keepalive();
   }
   #getHeaderOptions(forwardCookie) {
@@ -211,7 +214,9 @@ class ViewSyncerService {
       }
       if (!this.#cvr) {
         this.#lc.debug?.("loading CVR");
-        this.#cvr = await this.#cvrStore.load(lc, this.#lastConnectTime);
+        this.#cvr = await this.#runPriorityOp(
+          () => this.#cvrStore.load(lc, this.#lastConnectTime)
+        );
         this.#ttlClock = this.#cvr.ttlClock;
         this.#ttlClockBase = Date.now();
       } else {
@@ -378,7 +383,6 @@ class ViewSyncerService {
         profileID,
         wsID,
         baseCookie,
-        schemaVersion,
         tokenData,
         httpCookie,
         protocolVersion
@@ -427,7 +431,6 @@ class ViewSyncerService {
         wsID,
         this.#shard,
         baseCookie,
-        schemaVersion,
         downstream
       );
       this.#clients.get(clientID)?.close(`replaced by wsID: ${wsID}`);
@@ -485,19 +488,21 @@ class ViewSyncerService {
     this.#ttlClockBase = now;
     return ttlClock;
   }
-  async #flushUpdater(lc, updater) {
-    const now = Date.now();
-    const ttlClock = this.#getTTLClock(now);
-    const { cvr, flushed } = await updater.flush(
-      lc,
-      this.#lastConnectTime,
-      now,
-      ttlClock
-    );
-    if (flushed) {
-      this.#startTTLClockInterval(lc);
-    }
-    return cvr;
+  #flushUpdater(lc, updater) {
+    return this.#runPriorityOp(async () => {
+      const now = Date.now();
+      const ttlClock = this.#getTTLClock(now);
+      const { cvr, flushed } = await updater.flush(
+        lc,
+        this.#lastConnectTime,
+        now,
+        ttlClock
+      );
+      if (flushed) {
+        this.#startTTLClockInterval(lc);
+      }
+      return cvr;
+    });
   }
   #startTTLClockInterval(lc) {
     this.#stopTTLClockInterval();
@@ -754,19 +759,21 @@ class ViewSyncerService {
         "Custom/named queries were requested but no `ZERO_QUERY_URL` is configured for Zero Cache."
       );
     }
-    if (this.#customQueryTransformer && customQueries.size > 0) {
-      const transformedCustomQueries = await this.#customQueryTransformer.transform(
-        this.#getHeaderOptions(this.#queryConfig.forwardCookies),
-        customQueries.values(),
-        this.userQueryURL
-      );
-      this.#processTransformedCustomQueries(
-        lc,
-        transformedCustomQueries,
-        (q) => transformedQueries.push(q),
-        customQueries
-      );
-    }
+    await this.#runPriorityOp(async () => {
+      if (this.#customQueryTransformer && customQueries.size > 0) {
+        const transformedCustomQueries = await this.#customQueryTransformer.transform(
+          this.#getHeaderOptions(this.#queryConfig.forwardCookies),
+          customQueries.values(),
+          this.userQueryURL
+        );
+        this.#processTransformedCustomQueries(
+          lc,
+          transformedCustomQueries,
+          (q) => transformedQueries.push(q),
+          customQueries
+        );
+      }
+    });
     for (const q of otherQueries) {
       const transformed = transformAndHashQuery(
         lc,
@@ -1104,11 +1111,7 @@ class ViewSyncerService {
         removeQueries
       );
       const clients = this.#getClients();
-      const pokers = startPoke(
-        clients,
-        newVersion,
-        this.#pipelines.currentSchemaVersions()
-      );
+      const pokers = startPoke(clients, newVersion);
       for (const patch of queryPatches) {
         await pokers.addPatch(patch);
       }
@@ -1213,11 +1216,7 @@ class ViewSyncerService {
     return startAsyncSpan(tracer, "vs.#catchupClients", async (span) => {
       current ??= cvr.version;
       const clients = this.#getClients();
-      const pokers = usePokers ?? startPoke(
-        clients,
-        cvr.version,
-        this.#pipelines.currentSchemaVersions()
-      );
+      const pokers = usePokers ?? startPoke(clients, cvr.version);
       span.setAttribute("numClients", clients.length);
       const catchupFrom = clients.map((c) => c.version()).reduce((a, b) => cmpVersions(a, b) < 0 ? a : b, cvr.version);
       const rowPatches = this.#cvrStore.catchupRowPatches(
@@ -1365,8 +1364,7 @@ class ViewSyncerService {
       );
       const pokers = startPoke(
         this.#getClients(cvr.version),
-        updater.updatedVersion(),
-        this.#pipelines.currentSchemaVersions()
+        updater.updatedVersion()
       );
       lc.debug?.(`applying ${numChanges} to advance to ${version2}`);
       const hashToIDs = createHashToIDs(cvr);