@sixb/ducklake 0.1.8 → 0.1.9

package/src/internal/ducklake-snapshot-reader.ts

@@ -1,8 +1,10 @@
-import type { DatasetDefinition } from "@sixb/core"
+import type { DatasetDefinition, DatasetSchema } from "@sixb/core"
 import type {
   DatasetCatalogState,
   DatasetLatestVersionSummary,
+  DatasetProducer,
   DatasetVersion,
+  DatasetVersionLineage,
   DatasetVersionMode,
   DatasetVersionRef,
   LatestVersionsSince,
@@ -10,22 +12,34 @@ import type {
 } from "@sixb/core/lake-storage"
 import { LakeStorageError } from "@sixb/core/lake-storage"
 import type { DuckLakeStorageOptions } from "../types"
-import { getBigIntLike, getBoolean, getDate, getOptionalString, getString } from "./duckdb-row"
+import { getBigIntLike, getDate, getOptionalString, getString } from "./duckdb-row"
 import type { DuckDbQueryRuntime } from "./duckdb-runtime"
 import type { DuckLakeConnectionManager } from "./ducklake-connection-manager"
 import type { DuckLakeDatasetCatalog } from "./ducklake-dataset-catalog"
-import { type DatasetTableRef, resolveDatasetTableRef } from "./ducklake-dataset-table-ref"
-import { decodeDatasetTableName, encodeDatasetTableName } from "./names"
+import {
+  type DatasetTableRef,
+  datasetTableIds,
+  resolveDatasetTableRef,
+  resolveDatasetTableRefAtCurrentSnapshot,
+} from "./ducklake-dataset-table-ref"
+import type { LatestVersionAtSnapshot } from "./ducklake-version-transaction"
+import { decodeLineage } from "./lineage-encoding"
+import {
+  decodeDatasetTableName,
+  encodeDatasetSequenceTableName,
+  encodeDatasetTableName,
+} from "./names"
 import { buildDuckLakeMetadataQuery, duckLakeMetadataTableName, quoteSqlString } from "./sql"
 import {
+  changesTouchTables,
   parseCommitMetadata,
-  parseInlineDataChange,
+  parseTableDataChange,
   parseVersionId,
   type SixbCommitMetadata,
   toVersionId,
 } from "./versions"
 
-interface DatasetSnapshotRow {
+export interface DatasetSnapshotRow {
   readonly snapshotId: string
   readonly tableId: bigint
   readonly createdAt: Date
@@ -38,8 +52,6 @@ interface DatasetSnapshotCandidateRow {
   readonly snapshotId: string
   readonly createdAt: Date
   readonly changesMade: string
-  readonly hasFileChange: boolean
-  readonly hasFileDeleteChange: boolean
   readonly metadata?: SixbCommitMetadata
 }
 
@@ -50,31 +62,34 @@ interface FeedTable {
   readonly startSnapshotId: bigint
 }
 
-interface MissingSnapshotCandidate {
-  readonly snapshotId: string
-  readonly missing: true
-}
-
 interface SnapshotCandidateQueryInput {
-  readonly datasetId: string
-  readonly tableId: bigint
+  readonly table: DatasetTableRef
   readonly exactSnapshotId?: string
   readonly beforeSnapshotId?: string
   readonly limit: number
 }
 
 interface VisibleSnapshotRowsInput {
-  readonly datasetId: string
-  readonly tableId: bigint
+  readonly table: DatasetTableRef
   readonly exactSnapshotId?: string
   readonly beforeSnapshotId?: string
   readonly visibleRowLimit?: number
 }
 
+/** A version reference, with the DuckLake table and snapshot it was resolved through. */
+export interface DuckLakeTableVersionRef extends DatasetVersionRef {
+  readonly table: DatasetTableRef
+  readonly snapshot: DatasetSnapshotRow
+}
+
+/** What a commit needs of the version it builds on. */
 export interface DuckLakeVersionSummary {
   readonly datasetId: string
   readonly versionId: string
+  readonly schema: DatasetSchema
+  readonly producer?: DatasetProducer
   readonly inputs?: readonly DatasetVersionRef[]
+  readonly lineage?: DatasetVersionLineage
   readonly rowCount?: number
   readonly validatedPrimaryKeyColumns?: readonly string[]
 }
@@ -84,19 +99,17 @@ interface CatalogScanSnapshot {
   readonly snapshotId: string
   readonly createdAt: Date
   readonly changesMade: string
-  readonly metadata?: SixbCommitMetadata
-}
-
-/** Per (snapshot, table) file-change flags merged into the catalog scan. */
-interface FileChangeFlags {
-  readonly hasFileChange: boolean
-  readonly hasFileDeleteChange: boolean
 }
 
 const SNAPSHOT_ROW_BATCH_SIZE = 128
 // One feed read covers at most this many commits; the next read continues from its cursor.
 const FEED_PAGE_SIZE = 512
 
+// A read of the latest version, or of the one before a version, usually finds it among the
+// dataset's most recent snapshots. It first reads only a few of them: each carries the commit
+// metadata of its version, whose lineage may be large.
+const SHORT_READ_BATCH_SIZE = 8
+
 // The bulk catalog scan shares one descending walk of recent snapshots across
 // all requested datasets, so its cost is bounded by this window rather than by
 // the dataset count. A dataset whose latest version is older than the window
@@ -113,8 +126,13 @@ const CATALOG_SNAPSHOT_SCAN_LIMIT = 512
  *
  * This class intentionally reads DuckLake metadata directly. The provider does
  * not keep a Sixb side table for versions; a Sixb version is a DuckLake
- * snapshot that either changed the dataset table or was explicitly tagged with
- * Sixb dataset metadata.
+ * snapshot whose `changes_made` names the dataset table: one that changed its
+ * rows, or that altered the table with Sixb commit metadata naming the dataset,
+ * as a version that changes no rows does (see the write coordinator's
+ * `claimTable`).
+ *
+ * DuckLake names a table in `changes_made` for every change to its rows, inlined
+ * or written to files, so version reads never scan the catalog's file tables.
  */
 export class DuckLakeSnapshotReader {
   constructor(
@@ -189,11 +207,11 @@ export class DuckLakeSnapshotReader {
     datasetIds: readonly string[]
   ): Promise<readonly DatasetCatalogState[]> {
     const uniqueIds = [...new Set(datasetIds)]
-    const tableIdsByDatasetId = await this.resolveDatasetTableIds(runtime, uniqueIds)
-    const latestRowByDatasetId = await this.resolveLatestSnapshotRows(runtime, tableIdsByDatasetId)
+    const tablesByDatasetId = await this.resolveDatasetTables(runtime, uniqueIds)
+    const latestRowByDatasetId = await this.resolveLatestSnapshotRows(runtime, tablesByDatasetId)
 
     return uniqueIds.map((datasetId) => {
-      if (!tableIdsByDatasetId.has(datasetId)) {
+      if (!tablesByDatasetId.has(datasetId)) {
         return { datasetId, materialized: false, latestVersion: null }
       }
 
@@ -339,16 +357,10 @@ export class DuckLakeSnapshotReader {
         `
       )
     )
-    const fileFlags = await this.queryFileChangeFlags(
-      runtime,
-      snapshots.map((snapshot) => snapshot.snapshotId),
-      tables.map((table) => table.tableId)
-    )
     const changesBySnapshotId = new Map(
       changeRows.map((row) => [String(getBigIntLike(row, "snapshot_id")), row])
     )
     const tablesByTableId = new Map(tables.map((table) => [table.tableId.toString(), table]))
-    const tablesByDatasetId = new Map(tables.map((table) => [table.datasetId, table]))
 
     for (const snapshot of snapshots) {
       const changes = changesBySnapshotId.get(snapshot.snapshotId)
@@ -358,29 +370,21 @@ export class DuckLakeSnapshotReader {
       const changesMade = getString(changes, "changes_made")
       const metadata = parseCommitMetadata(changes.commit_extra_info)
 
-      // Only tables this snapshot names, touches through files, or tags can change visibility.
+      // Only the tables this snapshot names can have a version in it.
       const touched = new Set<FeedTable>()
       for (const change of changesMade.split(",")) {
         const table = tablesByTableId.get(change.split(":")[1] ?? "")
         if (table) touched.add(table)
       }
-      for (const table of tables) {
-        if (fileFlags.has(fileChangeKey(snapshot.snapshotId, table.tableId))) touched.add(table)
-      }
-      const tagged = metadata === undefined ? undefined : tablesByDatasetId.get(metadata.datasetId)
-      if (tagged) touched.add(tagged)
 
       for (const table of touched) {
         if (BigInt(snapshot.snapshotId) < table.startSnapshotId) {
           continue
         }
-        const flags = fileFlags.get(fileChangeKey(snapshot.snapshotId, table.tableId))
         const row = this.candidateToSnapshotRow(table.datasetId, table.tableId, {
           snapshotId: snapshot.snapshotId,
           createdAt: snapshot.createdAt,
           changesMade,
-          hasFileChange: flags?.hasFileChange ?? false,
-          hasFileDeleteChange: flags?.hasFileDeleteChange ?? false,
           ...(metadata !== undefined ? { metadata } : {}),
         })
         if (row && row.mode !== "schema") {
@@ -392,6 +396,33 @@ export class DuckLakeSnapshotReader {
     return latest
   }
 
+  /**
+   * The snapshot of every dataset's latest version, however old, from one shared walk down the
+   * whole history: maintenance must never expire one.
+   */
+  async latestVersionSnapshotIds(runtime: DuckDbQueryRuntime): Promise<readonly string[]> {
+    const tables = await runtime.query(`
+      SELECT table_name
+      FROM ${duckLakeMetadataTableName(this.options, "ducklake_table")}
+      WHERE end_snapshot IS NULL
+    `)
+    const datasetIds = tables.flatMap((table) => {
+      const datasetId = decodeDatasetTableName(getString(table, "table_name"))
+      return datasetId === null ? [] : [datasetId]
+    })
+
+    if (datasetIds.length === 0) {
+      return []
+    }
+
+    const latest = await this.resolveLatestSnapshotRows(
+      runtime,
+      await this.resolveDatasetTables(runtime, datasetIds),
+      Number.POSITIVE_INFINITY
+    )
+    return [...latest.values()].map((row) => row.snapshotId)
+  }
+
   private snapshotRowToSummary(
     datasetId: string,
     row: DatasetSnapshotRow
@@ -405,19 +436,21 @@ export class DuckLakeSnapshotReader {
     }
   }
 
-  private async resolveDatasetTableIds(
+  private async resolveDatasetTables(
     runtime: DuckDbQueryRuntime,
     datasetIds: readonly string[]
-  ): Promise<Map<string, bigint>> {
+  ): Promise<Map<string, DatasetTableRef>> {
     // Encoded table names are a collision-free bijection with dataset ids, so a
-    // single `ducklake_table` read maps every requested id to its current table.
+    // single `ducklake_table` read maps every requested id to its current tables.
     const datasetIdByTableName = new Map<string, string>()
+    const datasetIdBySequenceTableName = new Map<string, string>()
     for (const datasetId of datasetIds) {
       datasetIdByTableName.set(encodeDatasetTableName(datasetId), datasetId)
+      datasetIdBySequenceTableName.set(encodeDatasetSequenceTableName(datasetId), datasetId)
     }
 
     const ducklakeTable = duckLakeMetadataTableName(this.options, "ducklake_table")
-    const tableNameList = [...datasetIdByTableName.keys()]
+    const tableNameList = [...datasetIdByTableName.keys(), ...datasetIdBySequenceTableName.keys()]
       .map((name) => quoteSqlString(name))
       .join(", ")
     const rows = await runtime.query(`
@@ -427,15 +460,31 @@ export class DuckLakeSnapshotReader {
         AND end_snapshot IS NULL
     `)
 
-    const tableIdsByDatasetId = new Map<string, bigint>()
+    const tablesByDatasetId = new Map<string, DatasetTableRef>()
     for (const row of rows) {
-      const datasetId = datasetIdByTableName.get(getString(row, "table_name"))
+      const tableName = getString(row, "table_name")
+      const datasetId = datasetIdByTableName.get(tableName)
       if (datasetId !== undefined) {
-        tableIdsByDatasetId.set(datasetId, getBigIntLike(row, "table_id"))
+        tablesByDatasetId.set(datasetId, {
+          datasetId,
+          tableName,
+          tableId: getBigIntLike(row, "table_id"),
+        })
+      }
+    }
+
+    for (const row of rows) {
+      const datasetId = datasetIdBySequenceTableName.get(getString(row, "table_name"))
+      const table = datasetId === undefined ? undefined : tablesByDatasetId.get(datasetId)
+      if (datasetId !== undefined && table !== undefined) {
+        tablesByDatasetId.set(datasetId, {
+          ...table,
+          sequenceTableId: getBigIntLike(row, "table_id"),
+        })
       }
     }
 
-    return tableIdsByDatasetId
+    return tablesByDatasetId
   }
 
   /**
@@ -447,19 +496,19 @@ export class DuckLakeSnapshotReader {
    */
   private async resolveLatestSnapshotRows(
     runtime: DuckDbQueryRuntime,
-    tableIdsByDatasetId: ReadonlyMap<string, bigint>
+    tablesByDatasetId: ReadonlyMap<string, DatasetTableRef>,
+    scanLimit = CATALOG_SNAPSHOT_SCAN_LIMIT
   ): Promise<Map<string, DatasetSnapshotRow>> {
     const result = new Map<string, DatasetSnapshotRow>()
-    if (tableIdsByDatasetId.size === 0) {
+    if (tablesByDatasetId.size === 0) {
       return result
     }
 
-    const unresolved = new Map<string, bigint>(tableIdsByDatasetId)
-    const tableIds = [...new Set(tableIdsByDatasetId.values())]
+    const unresolved = new Map<string, DatasetTableRef>(tablesByDatasetId)
 
     let beforeSnapshotId: string | undefined
     let scanned = 0
-    while (unresolved.size > 0 && scanned < CATALOG_SNAPSHOT_SCAN_LIMIT) {
+    while (unresolved.size > 0 && scanned < scanLimit) {
       const candidates = await this.queryCatalogSnapshotBatch(
         runtime,
         beforeSnapshotId,
@@ -469,23 +518,27 @@ export class DuckLakeSnapshotReader {
         break
       }
 
-      const fileFlags = await this.queryFileChangeFlags(
+      const tables = [...unresolved.values()]
+      const metadata = await this.readCommitMetadata(
         runtime,
-        candidates.map((candidate) => candidate.snapshotId),
-        tableIds
+        candidates
+          .filter((candidate) =>
+            tables.some((table) =>
+              changesTouchTables(candidate.changesMade, datasetTableIds(table))
+            )
+          )
+          .map((candidate) => candidate.snapshotId)
       )
 
       for (const candidate of candidates) {
         scanned += 1
-        for (const [datasetId, tableId] of [...unresolved]) {
-          const flags = fileFlags.get(fileChangeKey(candidate.snapshotId, tableId))
-          const row = this.candidateToSnapshotRow(datasetId, tableId, {
+        const candidateMetadata = metadata.get(candidate.snapshotId)
+        for (const [datasetId, table] of [...unresolved]) {
+          const row = this.candidateToSnapshotRow(datasetId, table.tableId, {
             snapshotId: candidate.snapshotId,
             createdAt: candidate.createdAt,
             changesMade: candidate.changesMade,
-            hasFileChange: flags?.hasFileChange ?? false,
-            hasFileDeleteChange: flags?.hasFileDeleteChange ?? false,
-            ...(candidate.metadata !== undefined ? { metadata: candidate.metadata } : {}),
+            ...(candidateMetadata !== undefined ? { metadata: candidateMetadata } : {}),
           })
           if (row) {
             result.set(datasetId, row)
@@ -528,8 +581,7 @@ export class DuckLakeSnapshotReader {
       SELECT
         snapshot.snapshot_id,
         snapshot.snapshot_time,
-        changes.changes_made,
-        changes.commit_extra_info
+        changes.changes_made
       FROM ${ducklakeSnapshot} snapshot
       JOIN ${ducklakeSnapshotChanges} changes ON changes.snapshot_id = snapshot.snapshot_id
       ${whereSql}
@@ -537,70 +589,11 @@ export class DuckLakeSnapshotReader {
       LIMIT ${Math.max(0, Math.trunc(limit))}
     `)
 
-    return rows.map((row) => {
-      const metadata = parseCommitMetadata(getOptionalString(row, "commit_extra_info"))
-      return {
-        snapshotId: String(getBigIntLike(row, "snapshot_id")),
-        createdAt: getDate(row, "snapshot_time"),
-        changesMade: getString(row, "changes_made"),
-        ...(metadata !== undefined ? { metadata } : {}),
-      }
-    })
-  }
-
-  private async queryFileChangeFlags(
-    runtime: DuckDbQueryRuntime,
-    snapshotIds: readonly string[],
-    tableIds: readonly bigint[]
-  ): Promise<Map<string, FileChangeFlags>> {
-    const flags = new Map<string, FileChangeFlags>()
-    if (snapshotIds.length === 0 || tableIds.length === 0) {
-      return flags
-    }
-
-    for (const snapshotId of snapshotIds) {
-      assertDuckLakeSnapshotId(snapshotId)
-    }
-
-    const snapshotIdList = snapshotIds.join(", ")
-    const tableIdList = tableIds.map((tableId) => tableId.toString()).join(", ")
-    const ducklakeDataFile = duckLakeMetadataTableName(this.options, "ducklake_data_file")
-    const ducklakeDeleteFile = duckLakeMetadataTableName(this.options, "ducklake_delete_file")
-
-    // Large tables keep their changes in data/delete files instead of inline
-    // `changes_made`, so merge file-level changes for the scanned snapshots.
-    const rows = await runtime.query(`
-      WITH file_changes AS (
-        SELECT begin_snapshot AS snapshot_id, table_id, false AS is_delete
-        FROM ${ducklakeDataFile}
-        WHERE table_id IN (${tableIdList}) AND begin_snapshot IN (${snapshotIdList})
-        UNION ALL
-        SELECT end_snapshot, table_id, true
-        FROM ${ducklakeDataFile}
-        WHERE table_id IN (${tableIdList}) AND end_snapshot IN (${snapshotIdList})
-        UNION ALL
-        SELECT begin_snapshot, table_id, true
-        FROM ${ducklakeDeleteFile}
-        WHERE table_id IN (${tableIdList}) AND begin_snapshot IN (${snapshotIdList})
-        UNION ALL
-        SELECT end_snapshot, table_id, true
-        FROM ${ducklakeDeleteFile}
-        WHERE table_id IN (${tableIdList}) AND end_snapshot IN (${snapshotIdList})
-      )
-      SELECT snapshot_id, table_id, bool_or(is_delete) AS has_delete
-      FROM file_changes
-      GROUP BY snapshot_id, table_id
-    `)
-
-    for (const row of rows) {
-      const key = fileChangeKey(
-        String(getBigIntLike(row, "snapshot_id")),
-        getBigIntLike(row, "table_id")
-      )
-      flags.set(key, { hasFileChange: true, hasFileDeleteChange: getBoolean(row, "has_delete") })
-    }
-
-    return flags
+    return rows.map((row) => ({
+      snapshotId: String(getBigIntLike(row, "snapshot_id")),
+      createdAt: getDate(row, "snapshot_time"),
+      changesMade: getString(row, "changes_made"),
+    }))
   }
 
   private assertNoMetadataConflict(
@@ -626,9 +619,14 @@ export class DuckLakeSnapshotReader {
   async getLatestVersionRefForDefinition(
     runtime: DuckDbQueryRuntime,
     dataset: DatasetDefinition
-  ): Promise<DatasetVersionRef | null> {
-    const summary = await this.getLatestVersionSummaryForDefinition(runtime, dataset)
-    return summary ? { datasetId: summary.datasetId, versionId: summary.versionId } : null
+  ): Promise<DuckLakeTableVersionRef | null> {
+    const tableRef = await resolveDatasetTableRef(this.options, runtime, dataset.id)
+    if (!tableRef) {
+      return null
+    }
+
+    const row = await this.getLatestSnapshotRowForTableRef(runtime, tableRef)
+    return row ? tableVersionRef(tableRef, row) : null
   }
 
   async getLatestVersionSummaryForDefinition(
@@ -636,19 +634,49 @@ export class DuckLakeSnapshotReader {
     dataset: DatasetDefinition
   ): Promise<DuckLakeVersionSummary | null> {
     const tableRef = await resolveDatasetTableRef(this.options, runtime, dataset.id)
-    if (!tableRef) {
-      return null
+    return tableRef ? this.getLatestVersionSummaryForTableRef(runtime, tableRef) : null
+  }
+
+  /**
+   * The latest version a commit builds on, with the lake's snapshot it was read at. Its first
+   * read fixes the snapshot of a transaction it runs in (see `beginTransactionOnLatestVersion`).
+   */
+  async getLatestVersionSummaryAtCurrentSnapshot(
+    runtime: DuckDbQueryRuntime,
+    dataset: DatasetDefinition
+  ): Promise<LatestVersionAtSnapshot<DuckLakeVersionSummary>> {
+    const { snapshotId, table } = await resolveDatasetTableRefAtCurrentSnapshot(
+      this.options,
+      runtime,
+      dataset.id
+    )
+
+    return {
+      snapshotId,
+      latestVersion: table ? await this.getLatestVersionSummaryForTableRef(runtime, table) : null,
     }
+  }
 
+  private async getLatestVersionSummaryForTableRef(
+    runtime: DuckDbQueryRuntime,
+    tableRef: DatasetTableRef
+  ): Promise<DuckLakeVersionSummary | null> {
     const row = await this.getLatestSnapshotRowForTableRef(runtime, tableRef)
     if (!row) {
       return null
     }
 
+    const [schema] = await this.datasets.getDatasetSchemasAtSnapshots(runtime, [
+      { table: tableRef, snapshotId: row.snapshotId },
+    ])
+
     return {
       datasetId: tableRef.datasetId,
       versionId: toVersionId(row.snapshotId),
+      schema,
+      producer: row.metadata?.producer,
       inputs: row.metadata?.inputs,
+      ...storedLineage(tableRef.datasetId, row),
       ...(row.metadata?.rowCount !== undefined ? { rowCount: row.metadata.rowCount } : {}),
       ...(row.metadata?.validatedPrimaryKeyColumns !== undefined
         ? { validatedPrimaryKeyColumns: row.metadata.validatedPrimaryKeyColumns }
@@ -675,14 +703,14 @@ export class DuckLakeSnapshotReader {
     const match = await this.getSnapshotRowForTableRef(runtime, tableRef, snapshotId, {
       includeParent: true,
     })
-    return match ? this.snapshotToVersion(runtime, tableRef, match) : null
+    return match ? (await this.snapshotsToVersions(runtime, tableRef, [match]))[0] : null
   }
 
   async getVersionRefForSnapshot(
     runtime: DuckDbQueryRuntime,
     dataset: DatasetDefinition,
     snapshotId: string
-  ): Promise<DatasetVersionRef | null> {
+  ): Promise<DuckLakeTableVersionRef | null> {
     assertDuckLakeSnapshotId(snapshotId)
 
     const tableRef = await resolveDatasetTableRef(this.options, runtime, dataset.id)
@@ -693,7 +721,23 @@ export class DuckLakeSnapshotReader {
     const row = await this.getSnapshotRowForTableRef(runtime, tableRef, snapshotId, {
       includeParent: false,
     })
-    return row ? { datasetId: tableRef.datasetId, versionId: toVersionId(row.snapshotId) } : null
+    return row ? tableVersionRef(tableRef, row) : null
+  }
+
+  /**
+   * The versions resolved refs point to, with their schemas from one catalog read. Refs resolve
+   * without their parent, so these versions carry no `parentVersionId`.
+   */
+  async getVersionsForRefs(
+    runtime: DuckDbQueryRuntime,
+    refs: readonly DuckLakeTableVersionRef[]
+  ): Promise<DatasetVersion[]> {
+    const schemas = await this.datasets.getDatasetSchemasAtSnapshots(
+      runtime,
+      refs.map((ref) => ({ table: ref.table, snapshotId: ref.snapshot.snapshotId }))
+    )
+
+    return refs.map((ref, index) => snapshotToVersion(ref.table, ref.snapshot, schemas[index]))
   }
 
   async listVersionsForDefinition(
@@ -711,13 +755,7 @@ export class DuckLakeSnapshotReader {
     limit?: number
   ): Promise<readonly DatasetVersion[]> {
     const limitedRows = await this.getSnapshotRowsForTableRef(runtime, tableRef, limit)
-    const versions: DatasetVersion[] = []
-
-    for (const row of limitedRows) {
-      versions.push(await this.snapshotToVersion(runtime, tableRef, row))
-    }
-
-    return versions
+    return this.snapshotsToVersions(runtime, tableRef, limitedRows)
   }
 
   async getLatestVersionForTableRef(
@@ -733,8 +771,7 @@ export class DuckLakeSnapshotReader {
     tableRef: DatasetTableRef
   ): Promise<DatasetSnapshotRow | null> {
     const [row] = await this.collectVisibleSnapshotRows(runtime, {
-      datasetId: tableRef.datasetId,
-      tableId: tableRef.tableId,
+      table: tableRef,
       visibleRowLimit: 1,
     })
     return row ?? null
@@ -749,8 +786,7 @@ export class DuckLakeSnapshotReader {
     assertDuckLakeSnapshotId(snapshotId)
 
     const [row] = await this.collectVisibleSnapshotRows(runtime, {
-      datasetId: tableRef.datasetId,
-      tableId: tableRef.tableId,
+      table: tableRef,
       exactSnapshotId: snapshotId,
       visibleRowLimit: 1,
     })
@@ -766,8 +802,7 @@ export class DuckLakeSnapshotReader {
     }
 
     const [parent] = await this.collectVisibleSnapshotRows(runtime, {
-      datasetId: tableRef.datasetId,
-      tableId: tableRef.tableId,
+      table: tableRef,
       beforeSnapshotId: snapshotId,
       visibleRowLimit: 1,
     })
@@ -786,8 +821,7 @@ export class DuckLakeSnapshotReader {
     }
 
     const rows = await this.collectVisibleSnapshotRows(runtime, {
-      datasetId: tableRef.datasetId,
-      tableId: tableRef.tableId,
+      table: tableRef,
       visibleRowLimit: visibleLimit === undefined ? undefined : visibleLimit + 1,
     })
     const rowsWithParents = this.withParentSnapshotIds(rows)
@@ -804,24 +838,29 @@ export class DuckLakeSnapshotReader {
 
     const snapshots: DatasetSnapshotRow[] = []
     let beforeSnapshotId = input.beforeSnapshotId
+    let shortRead =
+      input.exactSnapshotId !== undefined ||
+      (input.visibleRowLimit !== undefined && input.visibleRowLimit <= 2)
 
     while (input.visibleRowLimit === undefined || snapshots.length < input.visibleRowLimit) {
+      const batchSize = shortRead ? SHORT_READ_BATCH_SIZE : SNAPSHOT_ROW_BATCH_SIZE
       const candidates = await this.querySnapshotCandidates(runtime, {
-        datasetId: input.datasetId,
-        tableId: input.tableId,
+        table: input.table,
         exactSnapshotId: input.exactSnapshotId,
         beforeSnapshotId,
-        limit: input.exactSnapshotId === undefined ? SNAPSHOT_ROW_BATCH_SIZE : 1,
+        limit: input.exactSnapshotId === undefined ? batchSize : 1,
       })
+      shortRead = false
       if (candidates.length === 0) {
         break
       }
 
       for (const candidate of candidates) {
-        if ("missing" in candidate) {
-          continue
-        }
-        const snapshot = this.candidateToSnapshotRow(input.datasetId, input.tableId, candidate)
+        const snapshot = this.candidateToSnapshotRow(
+          input.table.datasetId,
+          input.table.tableId,
+          candidate
+        )
         if (snapshot) {
           snapshots.push(snapshot)
         }
@@ -831,7 +870,7 @@ export class DuckLakeSnapshotReader {
         }
       }
 
-      if (input.exactSnapshotId !== undefined || candidates.length < SNAPSHOT_ROW_BATCH_SIZE) {
+      if (input.exactSnapshotId !== undefined || candidates.length < batchSize) {
         break
       }
 
@@ -842,20 +881,20 @@ export class DuckLakeSnapshotReader {
   }
 
   /**
-   * Hydrates, newest first, the snapshots that may hold a version of one dataset table: those
-   * that added or removed its data or delete files, that DuckLake recorded as changing it inline,
-   * or whose Sixb commit metadata names the dataset. The text matches are a superset;
-   * candidateToSnapshotRow decides visibility. Only this table's snapshots are read, so the cost
-   * follows the dataset's own history rather than every commit in the catalog.
+   * Hydrates, newest first, the snapshots whose changes name one of the dataset's tables: every
+   * snapshot that may hold one of its versions. The text match is a superset;
+   * candidateToSnapshotRow decides visibility.
    *
-   * Returns one row per candidate id, including ids whose snapshot no longer exists, so the caller
-   * can tell a short batch from the end of the history.
+   * The query walks the catalog's snapshot ids downwards, from `beforeSnapshotId` or the head,
+   * and stops at the limit or at the table's creation. A lookup therefore reads the snapshots
+   * between where it starts and the versions it returns: a few for an active dataset, at most the
+   * table's lifetime for one written long ago. It never reads the file tables, whose replaced
+   * files stay until cleanup.
    */
   private async querySnapshotCandidates(
     runtime: DuckDbQueryRuntime,
     input: SnapshotCandidateQueryInput
-  ): Promise<readonly (DatasetSnapshotCandidateRow | MissingSnapshotCandidate)[]> {
-    const tableId = input.tableId.toString()
+  ): Promise<readonly DatasetSnapshotCandidateRow[]> {
     const limit = Math.max(0, Math.trunc(input.limit))
     if (limit === 0) {
       return []
@@ -872,130 +911,102 @@ export class DuckLakeSnapshotReader {
         // A dataset cannot have a version before its table exists. Use the earliest table record:
         // a rename may create a newer record with the same table id and must not hide older
         // versions.
-        const tableStart = `(SELECT min(begin_snapshot) FROM ${table("ducklake_table")} WHERE table_id = ${tableId})`
-        const candidateIds =
-          input.exactSnapshotId !== undefined
-            ? `SELECT CAST(${input.exactSnapshotId} AS BIGINT) AS snapshot_id
-               WHERE ${input.exactSnapshotId} >= ${tableStart}`
-            : `SELECT DISTINCT snapshot_id
-               FROM (
-                 SELECT begin_snapshot AS snapshot_id
-                 FROM ${table("ducklake_data_file")} WHERE table_id = ${tableId}
-                 UNION ALL
-                 SELECT end_snapshot
-                 FROM ${table("ducklake_data_file")}
-                 WHERE table_id = ${tableId} AND end_snapshot IS NOT NULL
-                 UNION ALL
-                 SELECT begin_snapshot
-                 FROM ${table("ducklake_delete_file")} WHERE table_id = ${tableId}
-                 UNION ALL
-                 SELECT end_snapshot
-                 FROM ${table("ducklake_delete_file")}
-                 WHERE table_id = ${tableId} AND end_snapshot IS NOT NULL
-                 UNION ALL
-                 SELECT snapshot_id
-                 FROM ${table("ducklake_snapshot_changes")}
-                 WHERE strpos(',' || changes_made || ',', ${quoteSqlString(`:${tableId},`)}) > 0
-                   OR strpos(
-                     commit_extra_info,
-                     ${quoteSqlString(`"datasetId":${JSON.stringify(input.datasetId)}`)}
-                   ) > 0
-               ) touched
-               WHERE snapshot_id >= ${tableStart}
-                 ${input.beforeSnapshotId === undefined ? "" : `AND snapshot_id < ${input.beforeSnapshotId}`}
-               ORDER BY snapshot_id DESC
-               LIMIT ${limit}`
+        const bounds = [
+          `changes.snapshot_id >= (
+            SELECT min(begin_snapshot)
+            FROM ${table("ducklake_table")}
+            WHERE table_id = ${input.table.tableId}
+          )`,
+        ]
+        if (input.exactSnapshotId !== undefined) {
+          bounds.push(`changes.snapshot_id = ${input.exactSnapshotId}`)
+        }
+        if (input.beforeSnapshotId !== undefined) {
+          bounds.push(`changes.snapshot_id < ${input.beforeSnapshotId}`)
+        }
 
-        return `
-          WITH candidate_ids AS (
-            ${candidateIds}
-          ),
-          candidate_snapshots AS (
-            SELECT
-              snapshot.snapshot_id,
-              snapshot.snapshot_time,
-              changes.changes_made,
-              changes.commit_extra_info
-            FROM ${table("ducklake_snapshot")} snapshot
-            JOIN ${table("ducklake_snapshot_changes")} changes
-              ON changes.snapshot_id = snapshot.snapshot_id
-            WHERE snapshot.snapshot_id IN (SELECT snapshot_id FROM candidate_ids)
-          ),
-          file_changes AS (
-            -- File metadata tells whether this table changed in each candidate snapshot.
-            SELECT begin_snapshot AS snapshot_id, false AS is_delete_change
-            FROM ${table("ducklake_data_file")}
-            WHERE table_id = ${tableId}
-              AND begin_snapshot IN (SELECT snapshot_id FROM candidate_ids)
-            UNION ALL
-            SELECT end_snapshot AS snapshot_id, true AS is_delete_change
-            FROM ${table("ducklake_data_file")}
-            WHERE table_id = ${tableId}
-              AND end_snapshot IN (SELECT snapshot_id FROM candidate_ids)
-            UNION ALL
-            SELECT begin_snapshot AS snapshot_id, true AS is_delete_change
-            FROM ${table("ducklake_delete_file")}
-            WHERE table_id = ${tableId}
-              AND begin_snapshot IN (SELECT snapshot_id FROM candidate_ids)
-            UNION ALL
-            SELECT end_snapshot AS snapshot_id, true AS is_delete_change
-            FROM ${table("ducklake_delete_file")}
-            WHERE table_id = ${tableId}
-              AND end_snapshot IN (SELECT snapshot_id FROM candidate_ids)
-          ),
-          file_changes_by_snapshot AS (
-            -- Collapse file-level changes into one row per snapshot.
-            SELECT
-              snapshot_id,
-              count(*) > 0 AS has_file_change,
-              count(*) FILTER (WHERE is_delete_change) > 0 AS has_file_delete_change
-            FROM file_changes
-            GROUP BY snapshot_id
+        const namesDatasetTable = datasetTableIds(input.table)
+          .map(
+            (tableId) =>
+              `strpos(',' || changes.changes_made || ',', ${quoteSqlString(`:${tableId},`)}) > 0`
           )
+          .join(" OR ")
+
+        // Bounds and order follow the snapshot-id primary key, so PostgreSQL reads it backwards
+        // and stops at the limit; the text match only filters the rows it passes.
+        return `
           SELECT
-            -- Keep metadata-only candidates; Sixb filters them with commit_extra_info.
-            candidate_ids.snapshot_id,
-            candidate.snapshot_time,
-            candidate.changes_made,
-            candidate.commit_extra_info,
-            coalesce(file_changes.has_file_change, false) AS has_file_change,
-            coalesce(file_changes.has_file_delete_change, false) AS has_file_delete_change
-          FROM candidate_ids
-          LEFT JOIN candidate_snapshots candidate
-            ON candidate.snapshot_id = candidate_ids.snapshot_id
-          LEFT JOIN file_changes_by_snapshot file_changes
-            ON file_changes.snapshot_id = candidate_ids.snapshot_id
-          ORDER BY candidate_ids.snapshot_id DESC
+            changes.snapshot_id,
+            snapshot.snapshot_time,
+            changes.changes_made,
+            changes.commit_extra_info
+          FROM ${table("ducklake_snapshot_changes")} changes
+          JOIN ${table("ducklake_snapshot")} snapshot
+            ON snapshot.snapshot_id = changes.snapshot_id
+          WHERE ${bounds.join(" AND ")}
+            AND (${namesDatasetTable})
+          ORDER BY changes.snapshot_id DESC
+          LIMIT ${limit}
         `
       })
     )
 
     return rows.map((row) => {
-      const snapshotId = String(getBigIntLike(row, "snapshot_id"))
-      // Expired snapshots keep their file records; they are no longer versions.
-      if (row.snapshot_time === null || row.snapshot_time === undefined) {
-        return { snapshotId, missing: true } as const
-      }
       const metadata = parseCommitMetadata(row.commit_extra_info)
+
       return {
-        snapshotId,
+        snapshotId: String(getBigIntLike(row, "snapshot_id")),
         createdAt: getDate(row, "snapshot_time"),
         changesMade: getString(row, "changes_made"),
-        hasFileChange: getBoolean(row, "has_file_change"),
-        hasFileDeleteChange: getBoolean(row, "has_file_delete_change"),
         ...(metadata !== undefined ? { metadata } : {}),
       }
     })
   }
 
+  /**
+   * Sixb commit metadata of the snapshots that changed a scanned dataset. Each one carries the
+   * lineage of its version, which can be large: scans read it only for the snapshots they keep,
+   * never for the other datasets' snapshots they page past.
+   */
+  private async readCommitMetadata(
+    runtime: DuckDbQueryRuntime,
+    snapshotIds: readonly string[]
+  ): Promise<Map<string, SixbCommitMetadata>> {
+    const metadata = new Map<string, SixbCommitMetadata>()
+    if (snapshotIds.length === 0) {
+      return metadata
+    }
+
+    for (const snapshotId of snapshotIds) {
+      assertDuckLakeSnapshotId(snapshotId)
+    }
+
+    const ducklakeSnapshotChanges = duckLakeMetadataTableName(
+      this.options,
+      "ducklake_snapshot_changes"
+    )
+    const rows = await runtime.query(`
+      SELECT snapshot_id, commit_extra_info
+      FROM ${ducklakeSnapshotChanges}
+      WHERE snapshot_id IN (${snapshotIds.join(", ")})
+    `)
+
+    for (const row of rows) {
+      const parsed = parseCommitMetadata(getOptionalString(row, "commit_extra_info"))
+      if (parsed !== undefined) {
+        metadata.set(String(getBigIntLike(row, "snapshot_id")), parsed)
+      }
+    }
+
+    return metadata
+  }
+
   private candidateToSnapshotRow(
     datasetId: string,
     tableId: bigint,
     candidate: DatasetSnapshotCandidateRow
   ): DatasetSnapshotRow | null {
-    const inlineChange = parseInlineDataChange(candidate.changesMade, tableId)
-    const hasDataChange = inlineChange.hasDataChange || candidate.hasFileChange
-    const hasDeleteChange = inlineChange.hasDeleteChange || candidate.hasFileDeleteChange
+    const { hasDataChange, hasDeleteChange } = parseTableDataChange(candidate.changesMade, tableId)
 
     // Metadata-only snapshots are common for table comments, schema changes,
     // and other catalog operations. Treat them as dataset versions only when
@@ -1029,36 +1040,17 @@ export class DuckLakeSnapshotReader {
     }
   }
 
-  private async snapshotToVersion(
+  /** One catalog read for the schemas of every listed version, not one per version. */
+  private async snapshotsToVersions(
     runtime: DuckDbQueryRuntime,
     tableRef: DatasetTableRef,
-    row: DatasetSnapshotRow
-  ): Promise<DatasetVersion> {
-    const schemaAtSnapshot = await this.datasets.getDatasetSchemaAtSnapshot(
+    rows: readonly DatasetSnapshotRow[]
+  ): Promise<DatasetVersion[]> {
+    const schemas = await this.datasets.getDatasetSchemasAtSnapshots(
       runtime,
-      tableRef.datasetId,
-      tableRef.tableName,
-      tableRef.tableId,
-      row.snapshotId
+      rows.map((row) => ({ table: tableRef, snapshotId: row.snapshotId }))
     )
-
-    // DuckLake gives us version id, timestamp, and time travel. Sixb metadata
-    // fills in caller intent such as append vs snapshot and producer lineage.
-    // Version listing is a metadata path, so it only carries row counts already
-    // stored in Sixb commit metadata and never counts historical table contents.
-
-    return {
-      datasetId: tableRef.datasetId,
-      versionId: toVersionId(row.snapshotId),
-      parentVersionId:
-        row.parentSnapshotId === undefined ? undefined : toVersionId(row.parentSnapshotId),
-      mode: row.mode,
-      createdAt: row.createdAt,
-      schema: schemaAtSnapshot,
-      producer: row.metadata?.producer,
-      inputs: row.metadata?.inputs,
-      ...(row.metadata?.rowCount !== undefined ? { rowCount: row.metadata.rowCount } : {}),
-    }
+    return rows.map((row, index) => snapshotToVersion(tableRef, row, schemas[index]))
   }
 
   private withParentSnapshotIds(
@@ -1076,12 +1068,68 @@ export class DuckLakeSnapshotReader {
   }
 }
 
+function snapshotToVersion(
+  tableRef: DatasetTableRef,
+  row: DatasetSnapshotRow,
+  schema: DatasetVersion["schema"]
+): DatasetVersion {
+  // DuckLake gives us version id, timestamp, and time travel. Sixb metadata
+  // fills in caller intent such as append vs snapshot and producer lineage.
+  // Version listing is a metadata path, so it only carries row counts already
+  // stored in Sixb commit metadata and never counts historical table contents.
+
+  return {
+    datasetId: tableRef.datasetId,
+    versionId: toVersionId(row.snapshotId),
+    parentVersionId:
+      row.parentSnapshotId === undefined ? undefined : toVersionId(row.parentSnapshotId),
+    mode: row.mode,
+    createdAt: row.createdAt,
+    schema,
+    producer: row.metadata?.producer,
+    inputs: row.metadata?.inputs,
+    ...storedLineage(tableRef.datasetId, row),
+    ...(row.metadata?.rowCount !== undefined ? { rowCount: row.metadata.rowCount } : {}),
+  }
+}
+
+/**
+ * A data snapshot without Sixb metadata was written outside Sixb, by any DuckLake client. It may
+ * keep rows of its parent whatever change it made, and nothing says where its own rows come from:
+ * its lineage is unknown.
+ */
+function storedLineage(
+  datasetId: string,
+  row: DatasetSnapshotRow
+): Pick<DatasetVersion, "lineage"> {
+  const metadata = row.metadata
+  if (metadata === undefined) {
+    return {
+      lineage: {
+        unknown: `version '${toVersionId(row.snapshotId)}' of '${datasetId}' was written outside Sixb`,
+      },
+    }
+  }
+
+  return metadata.lineage === undefined
+    ? {}
+    : { lineage: decodeLineage(metadata.datasetId, metadata.lineage) }
+}
+
+function tableVersionRef(
+  table: DatasetTableRef,
+  snapshot: DatasetSnapshotRow
+): DuckLakeTableVersionRef {
+  return {
+    datasetId: table.datasetId,
+    versionId: toVersionId(snapshot.snapshotId),
+    table,
+    snapshot,
+  }
+}
+
 function assertDuckLakeSnapshotId(snapshotId: string): void {
   if (!/^\d+$/.test(snapshotId)) {
     throw new LakeStorageError(`[SixbDuckLake] Invalid DuckLake snapshot id '${snapshotId}'.`)
   }
 }
-
-function fileChangeKey(snapshotId: string, tableId: bigint): string {
-  return `${snapshotId}|${tableId}`
-}