@sixb/ducklake 0.1.7 → 0.1.9

package/src/internal/ducklake-snapshot-reader.ts

@@ -1,29 +1,45 @@
-import type { DatasetDefinition } from "@sixb/core"
+import type { DatasetDefinition, DatasetSchema } from "@sixb/core"
 import type {
   DatasetCatalogState,
   DatasetLatestVersionSummary,
+  DatasetProducer,
   DatasetVersion,
+  DatasetVersionLineage,
   DatasetVersionMode,
   DatasetVersionRef,
+  LatestVersionsSince,
+  ListLatestVersionsSinceInput,
 } from "@sixb/core/lake-storage"
 import { LakeStorageError } from "@sixb/core/lake-storage"
 import type { DuckLakeStorageOptions } from "../types"
-import { getBigIntLike, getBoolean, getDate, getOptionalString, getString } from "./duckdb-row"
+import { getBigIntLike, getDate, getOptionalString, getString } from "./duckdb-row"
 import type { DuckDbQueryRuntime } from "./duckdb-runtime"
 import type { DuckLakeConnectionManager } from "./ducklake-connection-manager"
 import type { DuckLakeDatasetCatalog } from "./ducklake-dataset-catalog"
-import { type DatasetTableRef, resolveDatasetTableRef } from "./ducklake-dataset-table-ref"
-import { encodeDatasetTableName } from "./names"
-import { duckLakeMetadataTableName, quoteSqlString } from "./sql"
 import {
+  type DatasetTableRef,
+  datasetTableIds,
+  resolveDatasetTableRef,
+  resolveDatasetTableRefAtCurrentSnapshot,
+} from "./ducklake-dataset-table-ref"
+import type { LatestVersionAtSnapshot } from "./ducklake-version-transaction"
+import { decodeLineage } from "./lineage-encoding"
+import {
+  decodeDatasetTableName,
+  encodeDatasetSequenceTableName,
+  encodeDatasetTableName,
+} from "./names"
+import { buildDuckLakeMetadataQuery, duckLakeMetadataTableName, quoteSqlString } from "./sql"
+import {
+  changesTouchTables,
   parseCommitMetadata,
-  parseInlineDataChange,
+  parseTableDataChange,
   parseVersionId,
   type SixbCommitMetadata,
   toVersionId,
 } from "./versions"
 
-interface DatasetSnapshotRow {
+export interface DatasetSnapshotRow {
   readonly snapshotId: string
   readonly tableId: bigint
   readonly createdAt: Date
@@ -36,30 +52,44 @@ interface DatasetSnapshotCandidateRow {
   readonly snapshotId: string
   readonly createdAt: Date
   readonly changesMade: string
-  readonly hasFileChange: boolean
-  readonly hasFileDeleteChange: boolean
   readonly metadata?: SixbCommitMetadata
 }
 
-interface SnapshotCandidateQueryInput {
+interface FeedTable {
+  readonly datasetId: string
   readonly tableId: bigint
+  /** A dataset cannot have a version before its table exists. */
+  readonly startSnapshotId: bigint
+}
+
+interface SnapshotCandidateQueryInput {
+  readonly table: DatasetTableRef
   readonly exactSnapshotId?: string
   readonly beforeSnapshotId?: string
-  readonly limit?: number
+  readonly limit: number
 }
 
 interface VisibleSnapshotRowsInput {
-  readonly datasetId: string
-  readonly tableId: bigint
+  readonly table: DatasetTableRef
   readonly exactSnapshotId?: string
   readonly beforeSnapshotId?: string
   readonly visibleRowLimit?: number
 }
 
+/** A version reference, with the DuckLake table and snapshot it was resolved through. */
+export interface DuckLakeTableVersionRef extends DatasetVersionRef {
+  readonly table: DatasetTableRef
+  readonly snapshot: DatasetSnapshotRow
+}
+
+/** What a commit needs of the version it builds on. */
 export interface DuckLakeVersionSummary {
   readonly datasetId: string
   readonly versionId: string
+  readonly schema: DatasetSchema
+  readonly producer?: DatasetProducer
   readonly inputs?: readonly DatasetVersionRef[]
+  readonly lineage?: DatasetVersionLineage
   readonly rowCount?: number
   readonly validatedPrimaryKeyColumns?: readonly string[]
 }
@@ -69,16 +99,16 @@ interface CatalogScanSnapshot {
   readonly snapshotId: string
   readonly createdAt: Date
   readonly changesMade: string
-  readonly metadata?: SixbCommitMetadata
-}
-
-/** Per (snapshot, table) file-change flags merged into the catalog scan. */
-interface FileChangeFlags {
-  readonly hasFileChange: boolean
-  readonly hasFileDeleteChange: boolean
 }
 
 const SNAPSHOT_ROW_BATCH_SIZE = 128
+// One feed read covers at most this many commits; the next read continues from its cursor.
+const FEED_PAGE_SIZE = 512
+
+// A read of the latest version, or of the one before a version, usually finds it among the
+// dataset's most recent snapshots. It first reads only a few of them: each carries the commit
+// metadata of its version, whose lineage may be large.
+const SHORT_READ_BATCH_SIZE = 8
 
 // The bulk catalog scan shares one descending walk of recent snapshots across
 // all requested datasets, so its cost is bounded by this window rather than by
@@ -96,8 +126,13 @@ const CATALOG_SNAPSHOT_SCAN_LIMIT = 512
  *
  * This class intentionally reads DuckLake metadata directly. The provider does
  * not keep a Sixb side table for versions; a Sixb version is a DuckLake
- * snapshot that either changed the dataset table or was explicitly tagged with
- * Sixb dataset metadata.
+ * snapshot whose `changes_made` names the dataset table: one that changed its
+ * rows, or that altered the table with Sixb commit metadata naming the dataset,
+ * as a version that changes no rows does (see the write coordinator's
+ * `claimTable`).
+ *
+ * DuckLake names a table in `changes_made` for every change to its rows, inlined
+ * or written to files, so version reads never scan the catalog's file tables.
  */
 export class DuckLakeSnapshotReader {
   constructor(
@@ -172,11 +207,11 @@ export class DuckLakeSnapshotReader {
     datasetIds: readonly string[]
   ): Promise<readonly DatasetCatalogState[]> {
     const uniqueIds = [...new Set(datasetIds)]
-    const tableIdsByDatasetId = await this.resolveDatasetTableIds(runtime, uniqueIds)
-    const latestRowByDatasetId = await this.resolveLatestSnapshotRows(runtime, tableIdsByDatasetId)
+    const tablesByDatasetId = await this.resolveDatasetTables(runtime, uniqueIds)
+    const latestRowByDatasetId = await this.resolveLatestSnapshotRows(runtime, tablesByDatasetId)
 
     return uniqueIds.map((datasetId) => {
-      if (!tableIdsByDatasetId.has(datasetId)) {
+      if (!tablesByDatasetId.has(datasetId)) {
         return { datasetId, materialized: false, latestVersion: null }
       }
 
@@ -189,6 +224,205 @@ export class DuckLakeSnapshotReader {
     })
   }
 
+  /**
+   * Change feed over DuckLake's snapshot table, which is an append-only commit log: snapshot ids
+   * only increase, and a commit becomes visible only after every lower id has (concurrent commits
+   * that pick the same id conflict, and the loser retries with the next one). The cursor is the
+   * last snapshot id read. Every query is bounded by a snapshot-id range, so an idle read is one
+   * indexed query that returns the cursor's own snapshot, whatever the history length.
+   */
+  async listLatestVersionsSince(
+    input: ListLatestVersionsSinceInput
+  ): Promise<LatestVersionsSince | null> {
+    this.connections.assertOpen()
+
+    return this.connections.withAttachedRuntime(async (runtime) => {
+      if (input.cursor === null) {
+        return { cursor: await this.queryHeadSnapshotId(runtime), versions: [] }
+      }
+      if (!/^\d+$/.test(input.cursor)) {
+        return null
+      }
+
+      const snapshots = await this.querySnapshotsFrom(runtime, input.cursor)
+      // The cursor's own snapshot proves the cursor is still a position in this lake's history.
+      if (snapshots[0]?.snapshotId !== input.cursor) {
+        return null
+      }
+      const newSnapshots = snapshots.slice(1)
+      if (newSnapshots.length === 0) {
+        return { cursor: input.cursor, versions: [] }
+      }
+
+      const tables = await this.resolveFeedTables(runtime, input.datasetIds)
+      const latest = await this.collectFeedPage(runtime, tables, newSnapshots)
+      return {
+        cursor: newSnapshots[newSnapshots.length - 1]?.snapshotId ?? input.cursor,
+        versions: [...latest].map(([datasetId, row]) => this.snapshotRowToSummary(datasetId, row)),
+      }
+    })
+  }
+
+  private async queryHeadSnapshotId(runtime: DuckDbQueryRuntime): Promise<string> {
+    const [row] = await runtime.query(
+      buildDuckLakeMetadataQuery(
+        this.options,
+        (table) => `SELECT max(snapshot_id) AS snapshot_id FROM ${table("ducklake_snapshot")}`
+      )
+    )
+    if (row?.snapshot_id === null || row?.snapshot_id === undefined) {
+      throw new LakeStorageError("[SixbDuckLake] DuckLake catalog has no snapshots.")
+    }
+    return String(getBigIntLike(row, "snapshot_id"))
+  }
+
+  /** The cursor's snapshot and up to one feed page of snapshots after it, oldest first. */
+  private async querySnapshotsFrom(
+    runtime: DuckDbQueryRuntime,
+    cursor: string
+  ): Promise<readonly { readonly snapshotId: string; readonly createdAt: Date }[]> {
+    const rows = await runtime.query(
+      buildDuckLakeMetadataQuery(
+        this.options,
+        (table) => `
+          SELECT snapshot_id, snapshot_time
+          FROM ${table("ducklake_snapshot")}
+          WHERE snapshot_id >= ${cursor}
+          ORDER BY snapshot_id
+          LIMIT ${FEED_PAGE_SIZE + 1}
+        `
+      )
+    )
+    return rows.map((row) => ({
+      snapshotId: String(getBigIntLike(row, "snapshot_id")),
+      createdAt: getDate(row, "snapshot_time"),
+    }))
+  }
+
+  /** Current dataset tables, with the first snapshot each table id existed in. */
+  private async resolveFeedTables(
+    runtime: DuckDbQueryRuntime,
+    datasetIds: readonly string[] | undefined
+  ): Promise<readonly FeedTable[]> {
+    const requested = datasetIds === undefined ? null : new Set(datasetIds)
+    const rows = await runtime.query(`
+      SELECT table_id, table_name, begin_snapshot, end_snapshot
+      FROM ${duckLakeMetadataTableName(this.options, "ducklake_table")}
+    `)
+
+    const startByTableId = new Map<bigint, bigint>()
+    const current: { datasetId: string; tableId: bigint }[] = []
+    for (const row of rows) {
+      const tableId = getBigIntLike(row, "table_id")
+      const begin = getBigIntLike(row, "begin_snapshot")
+      const start = startByTableId.get(tableId)
+      if (start === undefined || begin < start) {
+        startByTableId.set(tableId, begin)
+      }
+      if (row.end_snapshot !== null && row.end_snapshot !== undefined) {
+        continue
+      }
+      const datasetId = decodeDatasetTableName(getString(row, "table_name"))
+      if (datasetId !== null && (requested === null || requested.has(datasetId))) {
+        current.push({ datasetId, tableId })
+      }
+    }
+
+    return current.map((table) => ({
+      ...table,
+      startSnapshotId: startByTableId.get(table.tableId) ?? 0n,
+    }))
+  }
+
+  /** Folds a page of snapshots, oldest first, into each dataset's newest data version. */
+  private async collectFeedPage(
+    runtime: DuckDbQueryRuntime,
+    tables: readonly FeedTable[],
+    snapshots: readonly { readonly snapshotId: string; readonly createdAt: Date }[]
+  ): Promise<Map<string, DatasetSnapshotRow>> {
+    const latest = new Map<string, DatasetSnapshotRow>()
+    const first = snapshots[0]?.snapshotId
+    const last = snapshots[snapshots.length - 1]?.snapshotId
+    if (first === undefined || last === undefined || tables.length === 0) {
+      return latest
+    }
+
+    const changeRows = await runtime.query(
+      buildDuckLakeMetadataQuery(
+        this.options,
+        (table) => `
+          SELECT snapshot_id, changes_made, commit_extra_info
+          FROM ${table("ducklake_snapshot_changes")}
+          WHERE snapshot_id >= ${first} AND snapshot_id <= ${last}
+        `
+      )
+    )
+    const changesBySnapshotId = new Map(
+      changeRows.map((row) => [String(getBigIntLike(row, "snapshot_id")), row])
+    )
+    const tablesByTableId = new Map(tables.map((table) => [table.tableId.toString(), table]))
+
+    for (const snapshot of snapshots) {
+      const changes = changesBySnapshotId.get(snapshot.snapshotId)
+      if (changes === undefined) {
+        continue
+      }
+      const changesMade = getString(changes, "changes_made")
+      const metadata = parseCommitMetadata(changes.commit_extra_info)
+
+      // Only the tables this snapshot names can have a version in it.
+      const touched = new Set<FeedTable>()
+      for (const change of changesMade.split(",")) {
+        const table = tablesByTableId.get(change.split(":")[1] ?? "")
+        if (table) touched.add(table)
+      }
+
+      for (const table of touched) {
+        if (BigInt(snapshot.snapshotId) < table.startSnapshotId) {
+          continue
+        }
+        const row = this.candidateToSnapshotRow(table.datasetId, table.tableId, {
+          snapshotId: snapshot.snapshotId,
+          createdAt: snapshot.createdAt,
+          changesMade,
+          ...(metadata !== undefined ? { metadata } : {}),
+        })
+        if (row && row.mode !== "schema") {
+          latest.set(table.datasetId, row)
+        }
+      }
+    }
+
+    return latest
+  }
+
+  /**
+   * The snapshot of every dataset's latest version, however old, from one shared walk down the
+   * whole history: maintenance must never expire one.
+   */
+  async latestVersionSnapshotIds(runtime: DuckDbQueryRuntime): Promise<readonly string[]> {
+    const tables = await runtime.query(`
+      SELECT table_name
+      FROM ${duckLakeMetadataTableName(this.options, "ducklake_table")}
+      WHERE end_snapshot IS NULL
+    `)
+    const datasetIds = tables.flatMap((table) => {
+      const datasetId = decodeDatasetTableName(getString(table, "table_name"))
+      return datasetId === null ? [] : [datasetId]
+    })
+
+    if (datasetIds.length === 0) {
+      return []
+    }
+
+    const latest = await this.resolveLatestSnapshotRows(
+      runtime,
+      await this.resolveDatasetTables(runtime, datasetIds),
+      Number.POSITIVE_INFINITY
+    )
+    return [...latest.values()].map((row) => row.snapshotId)
+  }
+
   private snapshotRowToSummary(
     datasetId: string,
     row: DatasetSnapshotRow
@@ -202,19 +436,21 @@ export class DuckLakeSnapshotReader {
     }
   }
 
-  private async resolveDatasetTableIds(
+  private async resolveDatasetTables(
     runtime: DuckDbQueryRuntime,
     datasetIds: readonly string[]
-  ): Promise<Map<string, bigint>> {
+  ): Promise<Map<string, DatasetTableRef>> {
     // Encoded table names are a collision-free bijection with dataset ids, so a
-    // single `ducklake_table` read maps every requested id to its current table.
+    // single `ducklake_table` read maps every requested id to its current tables.
     const datasetIdByTableName = new Map<string, string>()
+    const datasetIdBySequenceTableName = new Map<string, string>()
     for (const datasetId of datasetIds) {
       datasetIdByTableName.set(encodeDatasetTableName(datasetId), datasetId)
+      datasetIdBySequenceTableName.set(encodeDatasetSequenceTableName(datasetId), datasetId)
     }
 
     const ducklakeTable = duckLakeMetadataTableName(this.options, "ducklake_table")
-    const tableNameList = [...datasetIdByTableName.keys()]
+    const tableNameList = [...datasetIdByTableName.keys(), ...datasetIdBySequenceTableName.keys()]
       .map((name) => quoteSqlString(name))
       .join(", ")
     const rows = await runtime.query(`
@@ -224,15 +460,31 @@ export class DuckLakeSnapshotReader {
         AND end_snapshot IS NULL
     `)
 
-    const tableIdsByDatasetId = new Map<string, bigint>()
+    const tablesByDatasetId = new Map<string, DatasetTableRef>()
     for (const row of rows) {
-      const datasetId = datasetIdByTableName.get(getString(row, "table_name"))
+      const tableName = getString(row, "table_name")
+      const datasetId = datasetIdByTableName.get(tableName)
       if (datasetId !== undefined) {
-        tableIdsByDatasetId.set(datasetId, getBigIntLike(row, "table_id"))
+        tablesByDatasetId.set(datasetId, {
+          datasetId,
+          tableName,
+          tableId: getBigIntLike(row, "table_id"),
+        })
       }
     }
 
-    return tableIdsByDatasetId
+    for (const row of rows) {
+      const datasetId = datasetIdBySequenceTableName.get(getString(row, "table_name"))
+      const table = datasetId === undefined ? undefined : tablesByDatasetId.get(datasetId)
+      if (datasetId !== undefined && table !== undefined) {
+        tablesByDatasetId.set(datasetId, {
+          ...table,
+          sequenceTableId: getBigIntLike(row, "table_id"),
+        })
+      }
+    }
+
+    return tablesByDatasetId
   }
 
   /**
@@ -244,19 +496,19 @@ export class DuckLakeSnapshotReader {
    */
   private async resolveLatestSnapshotRows(
     runtime: DuckDbQueryRuntime,
-    tableIdsByDatasetId: ReadonlyMap<string, bigint>
+    tablesByDatasetId: ReadonlyMap<string, DatasetTableRef>,
+    scanLimit = CATALOG_SNAPSHOT_SCAN_LIMIT
   ): Promise<Map<string, DatasetSnapshotRow>> {
     const result = new Map<string, DatasetSnapshotRow>()
-    if (tableIdsByDatasetId.size === 0) {
+    if (tablesByDatasetId.size === 0) {
       return result
     }
 
-    const unresolved = new Map<string, bigint>(tableIdsByDatasetId)
-    const tableIds = [...new Set(tableIdsByDatasetId.values())]
+    const unresolved = new Map<string, DatasetTableRef>(tablesByDatasetId)
 
     let beforeSnapshotId: string | undefined
     let scanned = 0
-    while (unresolved.size > 0 && scanned < CATALOG_SNAPSHOT_SCAN_LIMIT) {
+    while (unresolved.size > 0 && scanned < scanLimit) {
       const candidates = await this.queryCatalogSnapshotBatch(
         runtime,
         beforeSnapshotId,
@@ -266,23 +518,27 @@ export class DuckLakeSnapshotReader {
         break
       }
 
-      const fileFlags = await this.queryFileChangeFlags(
+      const tables = [...unresolved.values()]
+      const metadata = await this.readCommitMetadata(
         runtime,
-        candidates.map((candidate) => candidate.snapshotId),
-        tableIds
+        candidates
+          .filter((candidate) =>
+            tables.some((table) =>
+              changesTouchTables(candidate.changesMade, datasetTableIds(table))
+            )
+          )
+          .map((candidate) => candidate.snapshotId)
       )
 
       for (const candidate of candidates) {
         scanned += 1
-        for (const [datasetId, tableId] of [...unresolved]) {
-          const flags = fileFlags.get(fileChangeKey(candidate.snapshotId, tableId))
-          const row = this.candidateToSnapshotRow(datasetId, tableId, {
+        const candidateMetadata = metadata.get(candidate.snapshotId)
+        for (const [datasetId, table] of [...unresolved]) {
+          const row = this.candidateToSnapshotRow(datasetId, table.tableId, {
             snapshotId: candidate.snapshotId,
             createdAt: candidate.createdAt,
             changesMade: candidate.changesMade,
-            hasFileChange: flags?.hasFileChange ?? false,
-            hasFileDeleteChange: flags?.hasFileDeleteChange ?? false,
-            ...(candidate.metadata !== undefined ? { metadata: candidate.metadata } : {}),
+            ...(candidateMetadata !== undefined ? { metadata: candidateMetadata } : {}),
           })
           if (row) {
             result.set(datasetId, row)
@@ -325,8 +581,7 @@ export class DuckLakeSnapshotReader {
       SELECT
         snapshot.snapshot_id,
         snapshot.snapshot_time,
-        changes.changes_made,
-        changes.commit_extra_info
+        changes.changes_made
       FROM ${ducklakeSnapshot} snapshot
       JOIN ${ducklakeSnapshotChanges} changes ON changes.snapshot_id = snapshot.snapshot_id
       ${whereSql}
@@ -334,70 +589,11 @@ export class DuckLakeSnapshotReader {
       LIMIT ${Math.max(0, Math.trunc(limit))}
     `)
 
-    return rows.map((row) => {
-      const metadata = parseCommitMetadata(getOptionalString(row, "commit_extra_info"))
-      return {
-        snapshotId: String(getBigIntLike(row, "snapshot_id")),
-        createdAt: getDate(row, "snapshot_time"),
-        changesMade: getString(row, "changes_made"),
-        ...(metadata !== undefined ? { metadata } : {}),
-      }
-    })
-  }
-
-  private async queryFileChangeFlags(
-    runtime: DuckDbQueryRuntime,
-    snapshotIds: readonly string[],
-    tableIds: readonly bigint[]
-  ): Promise<Map<string, FileChangeFlags>> {
-    const flags = new Map<string, FileChangeFlags>()
-    if (snapshotIds.length === 0 || tableIds.length === 0) {
-      return flags
-    }
-
-    for (const snapshotId of snapshotIds) {
-      assertDuckLakeSnapshotId(snapshotId)
-    }
-
-    const snapshotIdList = snapshotIds.join(", ")
-    const tableIdList = tableIds.map((tableId) => tableId.toString()).join(", ")
-    const ducklakeDataFile = duckLakeMetadataTableName(this.options, "ducklake_data_file")
-    const ducklakeDeleteFile = duckLakeMetadataTableName(this.options, "ducklake_delete_file")
-
-    // Large tables keep their changes in data/delete files instead of inline
-    // `changes_made`, so merge file-level changes for the scanned snapshots.
-    const rows = await runtime.query(`
-      WITH file_changes AS (
-        SELECT begin_snapshot AS snapshot_id, table_id, false AS is_delete
-        FROM ${ducklakeDataFile}
-        WHERE table_id IN (${tableIdList}) AND begin_snapshot IN (${snapshotIdList})
-        UNION ALL
-        SELECT end_snapshot, table_id, true
-        FROM ${ducklakeDataFile}
-        WHERE table_id IN (${tableIdList}) AND end_snapshot IN (${snapshotIdList})
-        UNION ALL
-        SELECT begin_snapshot, table_id, true
-        FROM ${ducklakeDeleteFile}
-        WHERE table_id IN (${tableIdList}) AND begin_snapshot IN (${snapshotIdList})
-        UNION ALL
-        SELECT end_snapshot, table_id, true
-        FROM ${ducklakeDeleteFile}
-        WHERE table_id IN (${tableIdList}) AND end_snapshot IN (${snapshotIdList})
-      )
-      SELECT snapshot_id, table_id, bool_or(is_delete) AS has_delete
-      FROM file_changes
-      GROUP BY snapshot_id, table_id
-    `)
-
-    for (const row of rows) {
-      const key = fileChangeKey(
-        String(getBigIntLike(row, "snapshot_id")),
-        getBigIntLike(row, "table_id")
-      )
-      flags.set(key, { hasFileChange: true, hasFileDeleteChange: getBoolean(row, "has_delete") })
-    }
-
-    return flags
+    return rows.map((row) => ({
+      snapshotId: String(getBigIntLike(row, "snapshot_id")),
+      createdAt: getDate(row, "snapshot_time"),
+      changesMade: getString(row, "changes_made"),
+    }))
   }
 
   private assertNoMetadataConflict(
@@ -423,9 +619,14 @@ export class DuckLakeSnapshotReader {
   async getLatestVersionRefForDefinition(
     runtime: DuckDbQueryRuntime,
     dataset: DatasetDefinition
-  ): Promise<DatasetVersionRef | null> {
-    const summary = await this.getLatestVersionSummaryForDefinition(runtime, dataset)
-    return summary ? { datasetId: summary.datasetId, versionId: summary.versionId } : null
+  ): Promise<DuckLakeTableVersionRef | null> {
+    const tableRef = await resolveDatasetTableRef(this.options, runtime, dataset.id)
+    if (!tableRef) {
+      return null
+    }
+
+    const row = await this.getLatestSnapshotRowForTableRef(runtime, tableRef)
+    return row ? tableVersionRef(tableRef, row) : null
   }
 
   async getLatestVersionSummaryForDefinition(
@@ -433,19 +634,49 @@ export class DuckLakeSnapshotReader {
     dataset: DatasetDefinition
   ): Promise<DuckLakeVersionSummary | null> {
     const tableRef = await resolveDatasetTableRef(this.options, runtime, dataset.id)
-    if (!tableRef) {
-      return null
+    return tableRef ? this.getLatestVersionSummaryForTableRef(runtime, tableRef) : null
+  }
+
+  /**
+   * The latest version a commit builds on, with the lake's snapshot it was read at. Its first
+   * read fixes the snapshot of a transaction it runs in (see `beginTransactionOnLatestVersion`).
+   */
+  async getLatestVersionSummaryAtCurrentSnapshot(
+    runtime: DuckDbQueryRuntime,
+    dataset: DatasetDefinition
+  ): Promise<LatestVersionAtSnapshot<DuckLakeVersionSummary>> {
+    const { snapshotId, table } = await resolveDatasetTableRefAtCurrentSnapshot(
+      this.options,
+      runtime,
+      dataset.id
+    )
+
+    return {
+      snapshotId,
+      latestVersion: table ? await this.getLatestVersionSummaryForTableRef(runtime, table) : null,
     }
+  }
 
+  private async getLatestVersionSummaryForTableRef(
+    runtime: DuckDbQueryRuntime,
+    tableRef: DatasetTableRef
+  ): Promise<DuckLakeVersionSummary | null> {
     const row = await this.getLatestSnapshotRowForTableRef(runtime, tableRef)
     if (!row) {
       return null
     }
 
+    const [schema] = await this.datasets.getDatasetSchemasAtSnapshots(runtime, [
+      { table: tableRef, snapshotId: row.snapshotId },
+    ])
+
     return {
       datasetId: tableRef.datasetId,
       versionId: toVersionId(row.snapshotId),
+      schema,
+      producer: row.metadata?.producer,
       inputs: row.metadata?.inputs,
+      ...storedLineage(tableRef.datasetId, row),
       ...(row.metadata?.rowCount !== undefined ? { rowCount: row.metadata.rowCount } : {}),
       ...(row.metadata?.validatedPrimaryKeyColumns !== undefined
         ? { validatedPrimaryKeyColumns: row.metadata.validatedPrimaryKeyColumns }
@@ -472,14 +703,14 @@ export class DuckLakeSnapshotReader {
     const match = await this.getSnapshotRowForTableRef(runtime, tableRef, snapshotId, {
       includeParent: true,
     })
-    return match ? this.snapshotToVersion(runtime, tableRef, match) : null
+    return match ? (await this.snapshotsToVersions(runtime, tableRef, [match]))[0] : null
   }
 
   async getVersionRefForSnapshot(
     runtime: DuckDbQueryRuntime,
     dataset: DatasetDefinition,
     snapshotId: string
-  ): Promise<DatasetVersionRef | null> {
+  ): Promise<DuckLakeTableVersionRef | null> {
     assertDuckLakeSnapshotId(snapshotId)
 
     const tableRef = await resolveDatasetTableRef(this.options, runtime, dataset.id)
@@ -490,7 +721,23 @@ export class DuckLakeSnapshotReader {
     const row = await this.getSnapshotRowForTableRef(runtime, tableRef, snapshotId, {
       includeParent: false,
     })
-    return row ? { datasetId: tableRef.datasetId, versionId: toVersionId(row.snapshotId) } : null
+    return row ? tableVersionRef(tableRef, row) : null
+  }
+
+  /**
+   * The versions resolved refs point to, with their schemas from one catalog read. Refs resolve
+   * without their parent, so these versions carry no `parentVersionId`.
+   */
+  async getVersionsForRefs(
+    runtime: DuckDbQueryRuntime,
+    refs: readonly DuckLakeTableVersionRef[]
+  ): Promise<DatasetVersion[]> {
+    const schemas = await this.datasets.getDatasetSchemasAtSnapshots(
+      runtime,
+      refs.map((ref) => ({ table: ref.table, snapshotId: ref.snapshot.snapshotId }))
+    )
+
+    return refs.map((ref, index) => snapshotToVersion(ref.table, ref.snapshot, schemas[index]))
   }
 
   async listVersionsForDefinition(
@@ -508,13 +755,7 @@ export class DuckLakeSnapshotReader {
     limit?: number
   ): Promise<readonly DatasetVersion[]> {
     const limitedRows = await this.getSnapshotRowsForTableRef(runtime, tableRef, limit)
-    const versions: DatasetVersion[] = []
-
-    for (const row of limitedRows) {
-      versions.push(await this.snapshotToVersion(runtime, tableRef, row))
-    }
-
-    return versions
+    return this.snapshotsToVersions(runtime, tableRef, limitedRows)
   }
 
   async getLatestVersionForTableRef(
@@ -530,8 +771,7 @@ export class DuckLakeSnapshotReader {
     tableRef: DatasetTableRef
   ): Promise<DatasetSnapshotRow | null> {
     const [row] = await this.collectVisibleSnapshotRows(runtime, {
-      datasetId: tableRef.datasetId,
-      tableId: tableRef.tableId,
+      table: tableRef,
       visibleRowLimit: 1,
     })
     return row ?? null
@@ -546,8 +786,7 @@ export class DuckLakeSnapshotReader {
     assertDuckLakeSnapshotId(snapshotId)
 
     const [row] = await this.collectVisibleSnapshotRows(runtime, {
-      datasetId: tableRef.datasetId,
-      tableId: tableRef.tableId,
+      table: tableRef,
       exactSnapshotId: snapshotId,
       visibleRowLimit: 1,
     })
@@ -563,8 +802,7 @@ export class DuckLakeSnapshotReader {
     }
 
     const [parent] = await this.collectVisibleSnapshotRows(runtime, {
-      datasetId: tableRef.datasetId,
-      tableId: tableRef.tableId,
+      table: tableRef,
       beforeSnapshotId: snapshotId,
       visibleRowLimit: 1,
     })
@@ -583,8 +821,7 @@ export class DuckLakeSnapshotReader {
     }
 
     const rows = await this.collectVisibleSnapshotRows(runtime, {
-      datasetId: tableRef.datasetId,
-      tableId: tableRef.tableId,
+      table: tableRef,
       visibleRowLimit: visibleLimit === undefined ? undefined : visibleLimit + 1,
     })
     const rowsWithParents = this.withParentSnapshotIds(rows)
@@ -601,20 +838,29 @@ export class DuckLakeSnapshotReader {
 
     const snapshots: DatasetSnapshotRow[] = []
     let beforeSnapshotId = input.beforeSnapshotId
+    let shortRead =
+      input.exactSnapshotId !== undefined ||
+      (input.visibleRowLimit !== undefined && input.visibleRowLimit <= 2)
 
     while (input.visibleRowLimit === undefined || snapshots.length < input.visibleRowLimit) {
+      const batchSize = shortRead ? SHORT_READ_BATCH_SIZE : SNAPSHOT_ROW_BATCH_SIZE
       const candidates = await this.querySnapshotCandidates(runtime, {
-        tableId: input.tableId,
+        table: input.table,
         exactSnapshotId: input.exactSnapshotId,
         beforeSnapshotId,
-        limit: input.exactSnapshotId === undefined ? SNAPSHOT_ROW_BATCH_SIZE : 1,
+        limit: input.exactSnapshotId === undefined ? batchSize : 1,
       })
+      shortRead = false
       if (candidates.length === 0) {
         break
       }
 
       for (const candidate of candidates) {
-        const snapshot = this.candidateToSnapshotRow(input.datasetId, input.tableId, candidate)
+        const snapshot = this.candidateToSnapshotRow(
+          input.table.datasetId,
+          input.table.tableId,
+          candidate
+        )
         if (snapshot) {
           snapshots.push(snapshot)
         }
@@ -624,7 +870,7 @@ export class DuckLakeSnapshotReader {
         }
       }
 
-      if (input.exactSnapshotId !== undefined || candidates.length < SNAPSHOT_ROW_BATCH_SIZE) {
+      if (input.exactSnapshotId !== undefined || candidates.length < batchSize) {
         break
       }
 
@@ -634,121 +880,133 @@ export class DuckLakeSnapshotReader {
     return snapshots
   }
 
+  /**
+   * Hydrates, newest first, the snapshots whose changes name one of the dataset's tables: every
+   * snapshot that may hold one of its versions. The text match is a superset;
+   * candidateToSnapshotRow decides visibility.
+   *
+   * The query walks the catalog's snapshot ids downwards, from `beforeSnapshotId` or the head,
+   * and stops at the limit or at the table's creation. A lookup therefore reads the snapshots
+   * between where it starts and the versions it returns: a few for an active dataset, at most the
+   * table's lifetime for one written long ago. It never reads the file tables, whose replaced
+   * files stay until cleanup.
+   */
   private async querySnapshotCandidates(
     runtime: DuckDbQueryRuntime,
     input: SnapshotCandidateQueryInput
   ): Promise<readonly DatasetSnapshotCandidateRow[]> {
-    if (input.limit !== undefined && input.limit <= 0) {
+    const limit = Math.max(0, Math.trunc(input.limit))
+    if (limit === 0) {
       return []
     }
-
-    // A dataset cannot have a version before its table exists. Use the earliest table record:
-    // a rename may create a newer record with the same table id and must not hide older versions.
-    const ducklakeTable = duckLakeMetadataTableName(this.options, "ducklake_table")
-    const where = [
-      `snapshot.snapshot_id >= (SELECT min(begin_snapshot) FROM ${ducklakeTable} WHERE table_id = ${input.tableId})`,
-    ]
     if (input.exactSnapshotId !== undefined) {
       assertDuckLakeSnapshotId(input.exactSnapshotId)
-      where.push(`snapshot.snapshot_id = ${input.exactSnapshotId}`)
     }
     if (input.beforeSnapshotId !== undefined) {
       assertDuckLakeSnapshotId(input.beforeSnapshotId)
-      where.push(`snapshot.snapshot_id < ${input.beforeSnapshotId}`)
     }
 
-    const whereSql = `WHERE ${where.join(" AND ")}`
-    const limitSql =
-      input.limit === undefined ? "" : `LIMIT ${Math.max(0, Math.trunc(input.limit))}`
-    const ducklakeSnapshot = duckLakeMetadataTableName(this.options, "ducklake_snapshot")
-    const ducklakeSnapshotChanges = duckLakeMetadataTableName(
-      this.options,
-      "ducklake_snapshot_changes"
-    )
-    const ducklakeDataFile = duckLakeMetadataTableName(this.options, "ducklake_data_file")
-    const ducklakeDeleteFile = duckLakeMetadataTableName(this.options, "ducklake_delete_file")
+    const rows = await runtime.query(
+      buildDuckLakeMetadataQuery(this.options, (table) => {
+        // A dataset cannot have a version before its table exists. Use the earliest table record:
+        // a rename may create a newer record with the same table id and must not hide older
+        // versions.
+        const bounds = [
+          `changes.snapshot_id >= (
+            SELECT min(begin_snapshot)
+            FROM ${table("ducklake_table")}
+            WHERE table_id = ${input.table.tableId}
+          )`,
+        ]
+        if (input.exactSnapshotId !== undefined) {
+          bounds.push(`changes.snapshot_id = ${input.exactSnapshotId}`)
+        }
+        if (input.beforeSnapshotId !== undefined) {
+          bounds.push(`changes.snapshot_id < ${input.beforeSnapshotId}`)
+        }
 
-    const rows = await runtime.query(`
-      WITH candidate_snapshots AS (
-        -- Recent DuckLake snapshots to inspect; Sixb visibility is filtered later.
-        SELECT
-          snapshot.snapshot_id,
-          snapshot.snapshot_time,
-          changes.changes_made,
-          changes.commit_extra_info
-        FROM ${ducklakeSnapshot} snapshot
-        JOIN ${ducklakeSnapshotChanges} changes
-          ON changes.snapshot_id = snapshot.snapshot_id
-        ${whereSql}
-        ORDER BY snapshot.snapshot_id DESC
-        ${limitSql}
-      ),
-      file_changes AS (
-        -- File metadata tells whether this table changed in each candidate snapshot.
-        SELECT begin_snapshot AS snapshot_id, false AS is_delete_change
-        FROM ${ducklakeDataFile}
-        WHERE table_id = ${input.tableId}
-          AND begin_snapshot IN (SELECT snapshot_id FROM candidate_snapshots)
-        UNION ALL
-        SELECT end_snapshot AS snapshot_id, true AS is_delete_change
-        FROM ${ducklakeDataFile}
-        WHERE table_id = ${input.tableId}
-          AND end_snapshot IN (SELECT snapshot_id FROM candidate_snapshots)
-        UNION ALL
-        SELECT begin_snapshot AS snapshot_id, true AS is_delete_change
-        FROM ${ducklakeDeleteFile}
-        WHERE table_id = ${input.tableId}
-          AND begin_snapshot IN (SELECT snapshot_id FROM candidate_snapshots)
-        UNION ALL
-        SELECT end_snapshot AS snapshot_id, true AS is_delete_change
-        FROM ${ducklakeDeleteFile}
-        WHERE table_id = ${input.tableId}
-          AND end_snapshot IN (SELECT snapshot_id FROM candidate_snapshots)
-      ),
-      file_changes_by_snapshot AS (
-        -- Collapse file-level changes into one row per snapshot.
-        SELECT
-          snapshot_id,
-          count(*) > 0 AS has_file_change,
-          count(*) FILTER (WHERE is_delete_change) > 0 AS has_file_delete_change
-        FROM file_changes
-        GROUP BY snapshot_id
-      )
-      SELECT
-        -- Keep metadata-only candidates; Sixb filters them with commit_extra_info.
-        candidate.snapshot_id,
-        candidate.snapshot_time,
-        candidate.changes_made,
-        candidate.commit_extra_info,
-        coalesce(file_changes.has_file_change, false) AS has_file_change,
-        coalesce(file_changes.has_file_delete_change, false) AS has_file_delete_change
-      FROM candidate_snapshots candidate
-      LEFT JOIN file_changes_by_snapshot file_changes
-        ON file_changes.snapshot_id = candidate.snapshot_id
-      ORDER BY candidate.snapshot_id DESC
-    `)
+        const namesDatasetTable = datasetTableIds(input.table)
+          .map(
+            (tableId) =>
+              `strpos(',' || changes.changes_made || ',', ${quoteSqlString(`:${tableId},`)}) > 0`
+          )
+          .join(" OR ")
+
+        // Bounds and order follow the snapshot-id primary key, so PostgreSQL reads it backwards
+        // and stops at the limit; the text match only filters the rows it passes.
+        return `
+          SELECT
+            changes.snapshot_id,
+            snapshot.snapshot_time,
+            changes.changes_made,
+            changes.commit_extra_info
+          FROM ${table("ducklake_snapshot_changes")} changes
+          JOIN ${table("ducklake_snapshot")} snapshot
+            ON snapshot.snapshot_id = changes.snapshot_id
+          WHERE ${bounds.join(" AND ")}
+            AND (${namesDatasetTable})
+          ORDER BY changes.snapshot_id DESC
+          LIMIT ${limit}
+        `
+      })
+    )
 
     return rows.map((row) => {
       const metadata = parseCommitMetadata(row.commit_extra_info)
+
       return {
         snapshotId: String(getBigIntLike(row, "snapshot_id")),
         createdAt: getDate(row, "snapshot_time"),
         changesMade: getString(row, "changes_made"),
-        hasFileChange: getBoolean(row, "has_file_change"),
-        hasFileDeleteChange: getBoolean(row, "has_file_delete_change"),
         ...(metadata !== undefined ? { metadata } : {}),
       }
     })
   }
 
+  /**
+   * Sixb commit metadata of the snapshots that changed a scanned dataset. Each one carries the
+   * lineage of its version, which can be large: scans read it only for the snapshots they keep,
+   * never for the other datasets' snapshots they page past.
+   */
+  private async readCommitMetadata(
+    runtime: DuckDbQueryRuntime,
+    snapshotIds: readonly string[]
+  ): Promise<Map<string, SixbCommitMetadata>> {
+    const metadata = new Map<string, SixbCommitMetadata>()
+    if (snapshotIds.length === 0) {
+      return metadata
+    }
+
+    for (const snapshotId of snapshotIds) {
+      assertDuckLakeSnapshotId(snapshotId)
+    }
+
+    const ducklakeSnapshotChanges = duckLakeMetadataTableName(
+      this.options,
+      "ducklake_snapshot_changes"
+    )
+    const rows = await runtime.query(`
+      SELECT snapshot_id, commit_extra_info
+      FROM ${ducklakeSnapshotChanges}
+      WHERE snapshot_id IN (${snapshotIds.join(", ")})
+    `)
+
+    for (const row of rows) {
+      const parsed = parseCommitMetadata(getOptionalString(row, "commit_extra_info"))
+      if (parsed !== undefined) {
+        metadata.set(String(getBigIntLike(row, "snapshot_id")), parsed)
+      }
+    }
+
+    return metadata
+  }
+
   private candidateToSnapshotRow(
     datasetId: string,
     tableId: bigint,
     candidate: DatasetSnapshotCandidateRow
   ): DatasetSnapshotRow | null {
-    const inlineChange = parseInlineDataChange(candidate.changesMade, tableId)
-    const hasDataChange = inlineChange.hasDataChange || candidate.hasFileChange
-    const hasDeleteChange = inlineChange.hasDeleteChange || candidate.hasFileDeleteChange
+    const { hasDataChange, hasDeleteChange } = parseTableDataChange(candidate.changesMade, tableId)
 
     // Metadata-only snapshots are common for table comments, schema changes,
     // and other catalog operations. Treat them as dataset versions only when
@@ -782,36 +1040,17 @@ export class DuckLakeSnapshotReader {
     }
   }
 
-  private async snapshotToVersion(
+  /** One catalog read for the schemas of every listed version, not one per version. */
+  private async snapshotsToVersions(
     runtime: DuckDbQueryRuntime,
     tableRef: DatasetTableRef,
-    row: DatasetSnapshotRow
-  ): Promise<DatasetVersion> {
-    const schemaAtSnapshot = await this.datasets.getDatasetSchemaAtSnapshot(
+    rows: readonly DatasetSnapshotRow[]
+  ): Promise<DatasetVersion[]> {
+    const schemas = await this.datasets.getDatasetSchemasAtSnapshots(
       runtime,
-      tableRef.datasetId,
-      tableRef.tableName,
-      tableRef.tableId,
-      row.snapshotId
+      rows.map((row) => ({ table: tableRef, snapshotId: row.snapshotId }))
     )
-
-    // DuckLake gives us version id, timestamp, and time travel. Sixb metadata
-    // fills in caller intent such as append vs snapshot and producer lineage.
-    // Version listing is a metadata path, so it only carries row counts already
-    // stored in Sixb commit metadata and never counts historical table contents.
-
-    return {
-      datasetId: tableRef.datasetId,
-      versionId: toVersionId(row.snapshotId),
-      parentVersionId:
-        row.parentSnapshotId === undefined ? undefined : toVersionId(row.parentSnapshotId),
-      mode: row.mode,
-      createdAt: row.createdAt,
-      schema: schemaAtSnapshot,
-      producer: row.metadata?.producer,
-      inputs: row.metadata?.inputs,
-      ...(row.metadata?.rowCount !== undefined ? { rowCount: row.metadata.rowCount } : {}),
-    }
+    return rows.map((row, index) => snapshotToVersion(tableRef, row, schemas[index]))
   }
 
   private withParentSnapshotIds(
@@ -829,12 +1068,68 @@ export class DuckLakeSnapshotReader {
   }
 }
 
+function snapshotToVersion(
+  tableRef: DatasetTableRef,
+  row: DatasetSnapshotRow,
+  schema: DatasetVersion["schema"]
+): DatasetVersion {
+  // DuckLake gives us version id, timestamp, and time travel. Sixb metadata
+  // fills in caller intent such as append vs snapshot and producer lineage.
+  // Version listing is a metadata path, so it only carries row counts already
+  // stored in Sixb commit metadata and never counts historical table contents.
+
+  return {
+    datasetId: tableRef.datasetId,
+    versionId: toVersionId(row.snapshotId),
+    parentVersionId:
+      row.parentSnapshotId === undefined ? undefined : toVersionId(row.parentSnapshotId),
+    mode: row.mode,
+    createdAt: row.createdAt,
+    schema,
+    producer: row.metadata?.producer,
+    inputs: row.metadata?.inputs,
+    ...storedLineage(tableRef.datasetId, row),
+    ...(row.metadata?.rowCount !== undefined ? { rowCount: row.metadata.rowCount } : {}),
+  }
+}
+
+/**
+ * A data snapshot without Sixb metadata was written outside Sixb, by any DuckLake client. It may
+ * keep rows of its parent whatever change it made, and nothing says where its own rows come from:
+ * its lineage is unknown.
+ */
+function storedLineage(
+  datasetId: string,
+  row: DatasetSnapshotRow
+): Pick<DatasetVersion, "lineage"> {
+  const metadata = row.metadata
+  if (metadata === undefined) {
+    return {
+      lineage: {
+        unknown: `version '${toVersionId(row.snapshotId)}' of '${datasetId}' was written outside Sixb`,
+      },
+    }
+  }
+
+  return metadata.lineage === undefined
+    ? {}
+    : { lineage: decodeLineage(metadata.datasetId, metadata.lineage) }
+}
+
+function tableVersionRef(
+  table: DatasetTableRef,
+  snapshot: DatasetSnapshotRow
+): DuckLakeTableVersionRef {
+  return {
+    datasetId: table.datasetId,
+    versionId: toVersionId(snapshot.snapshotId),
+    table,
+    snapshot,
+  }
+}
+
 function assertDuckLakeSnapshotId(snapshotId: string): void {
   if (!/^\d+$/.test(snapshotId)) {
     throw new LakeStorageError(`[SixbDuckLake] Invalid DuckLake snapshot id '${snapshotId}'.`)
   }
 }
-
-function fileChangeKey(snapshotId: string, tableId: bigint): string {
-  return `${snapshotId}|${tableId}`
-}