@sixb/ducklake 0.1.6 → 0.1.8

package/src/internal/ducklake-row-reader.ts

@@ -2,6 +2,7 @@ import type { DatasetColumnDefinition, DatasetRow } from "@sixb/core"
 import type { DatasetVersion, ReadDatasetRowsInput } from "@sixb/core/lake-storage"
 import { LakeStorageError } from "@sixb/core/lake-storage"
 import type { DuckLakeStorageOptions } from "../types"
+import type { DuckDbReader } from "./duckdb-reader"
 import { getBigIntLike } from "./duckdb-row"
 import type { DuckDbQueryRuntime } from "./duckdb-runtime"
 import type { DuckLakeConnectionManager } from "./ducklake-connection-manager"
@@ -11,10 +12,7 @@ import { normalizeReadValue } from "./schema"
 import { qualifiedTableName, quoteIdentifier } from "./sql"
 import { parseVersionId } from "./versions"
 
-// Large reads release the provider's single DuckDB queue slot between bounded pages. Besides
-// bounding converted-row memory, this lets a JavaScript pipeline feed one DuckLake dataset into
-// another: the destination appender can use the runtime after each page instead of waiting forever
-// behind its own still-open source stream.
+// Bounded fallback for catalogs whose metadata transactions cannot stay open during a read.
 const READ_PAGE_ROWS = 5_000
 const PHYSICAL_ROW_ID_ALIAS = "__sixb_physical_row_id"
 
@@ -33,78 +31,88 @@ export class DuckLakeRowReader {
   ) {}
 
   async *readRows(input: ReadDatasetRowsInput): AsyncIterable<DatasetRow> {
-    this.connections.assertOpen()
-
-    // Keep the attachment lease across pages so local catalogs do not pay DETACH/ATTACH for every
-    // page. Individual query() calls still release the DuckDB queue before rows are yielded.
-    const lease = await this.connections.acquireAttachedRuntime()
+    const read = await this.connections.acquireRead(input.signal)
+    let reader: DuckDbReader | null = null
     try {
-      const runtime = lease.runtime
-      // Resolve latest exactly once so a paged read stays pinned even if another writer commits
-      // between pages. DuckLake snapshots are immutable; every page names this snapshot.
-      const tableRef = await resolveDatasetTableRef(this.options, runtime, input.datasetId)
-      if (!tableRef) {
-        throw new LakeStorageError(`[SixbDuckLake] Unknown dataset '${input.datasetId}'.`)
+      const prepared = await this.connections.withAttachedRuntime(async (runtime) => {
+        read.signal.throwIfAborted()
+        const tableRef = await resolveDatasetTableRef(this.options, runtime, input.datasetId)
+        if (!tableRef) {
+          throw new LakeStorageError(`[SixbDuckLake] Unknown dataset '${input.datasetId}'.`)
+        }
+        // Resolve latest once, before yielding any rows. Every query names the same snapshot.
+        const version = await this.resolveVersion(runtime, tableRef, input.versionId)
+        const columns = this.resolveReadColumns(tableRef.datasetId, version.schema, input.columns)
+        const names = new Set(version.schema.columns.map((column) => column.name.toLowerCase()))
+        const physicalOrder = !names.has("rowid") && !names.has(PHYSICAL_ROW_ID_ALIAS)
+        const relation = `${qualifiedTableName(this.options, tableRef.tableName)} AT (VERSION => ${parseVersionId(version.versionId)})`
+        const columnsSql = columns.map((column) => quoteIdentifier(column.name)).join(", ")
+        const offset = Math.max(0, Math.trunc(input.offset ?? 0))
+        const limit = input.limit === undefined ? undefined : Math.max(0, Math.trunc(input.limit))
+        const sql = `SELECT ${columnsSql} FROM ${relation}${physicalOrder ? " ORDER BY rowid" : ""}${limit === undefined ? "" : ` LIMIT ${limit}`}${offset > 0 ? ` OFFSET ${offset}` : ""}`
+        const cursor =
+          limit === undefined || limit > READ_PAGE_ROWS
+            ? await this.connections.openReader(runtime, sql, read.signal, read.release)
+            : null
+        const query = { columnsSql, relation, physicalOrder }
+        const firstPage =
+          cursor || limit === 0
+            ? []
+            : await runtime.query(
+                pageSql(query, Math.min(limit ?? READ_PAGE_ROWS, READ_PAGE_ROWS), offset, null)
+              )
+        return { columns, ...query, offset, limit, cursor, firstPage }
+      })
+      reader = prepared.cursor
+      // A paused paged reader owns no native query. Cancellation can release its read lease without
+      // waiting for the consumer to call next()/return(); native readers release after cleanup.
+      if (!reader) read.signal.addEventListener("abort", read.release, { once: true })
+      read.signal.throwIfAborted()
+      if (reader) {
+        yield* this.normalizeRows(reader.rows(), prepared.columns, read.signal)
+        return
       }
-      const version = await this.resolveVersion(runtime, tableRef, input.versionId)
-      const selectedColumns = this.resolveReadColumns(
-        tableRef.datasetId,
-        version.schema,
-        input.columns
-      )
-      const snapshotId = parseVersionId(version.versionId)
-      const columnsSql = selectedColumns.map((column) => quoteIdentifier(column.name)).join(", ")
-      const tableSql = qualifiedTableName(this.options, tableRef.tableName)
-      // A user column named `rowid` shadows DuckDB's virtual row id. In that edge case the
-      // connection-level invariant still preserves insertion order; otherwise make it explicit.
-      const schemaNames = new Set(version.schema.columns.map((column) => column.name.toLowerCase()))
-      const canUsePhysicalRowId =
-        !schemaNames.has("rowid") && !schemaNames.has(PHYSICAL_ROW_ID_ALIAS.toLowerCase())
-      const orderSql = canUsePhysicalRowId ? " ORDER BY rowid" : ""
-      const baseSql = `SELECT ${columnsSql} FROM ${tableSql} AT (VERSION => ${snapshotId})`
-      const requestedOffset = Math.max(0, Math.trunc(input.offset ?? 0))
-      let remaining = input.limit === undefined ? undefined : Math.max(0, Math.trunc(input.limit))
-      if (remaining === 0) return
 
-      // Materialize one bounded page and release its queue operation before yielding it. A
-      // consumer may enqueue another DuckLake operation while processing the yielded rows.
-      // Holding a native stream open here would make that operation wait behind itself.
-      let offset = requestedOffset
-      let physicalCursor: bigint | null = null
-      while (true) {
-        const pageRows =
-          remaining === undefined ? READ_PAGE_ROWS : Math.min(READ_PAGE_ROWS, remaining)
-        const pageSql = canUsePhysicalRowId
-          ? `SELECT rowid AS ${quoteIdentifier(PHYSICAL_ROW_ID_ALIAS)}, ${columnsSql}
-              FROM ${tableSql} AT (VERSION => ${snapshotId})
-              ${physicalCursor === null ? "" : `WHERE rowid > ${physicalCursor}`}
-              ORDER BY rowid LIMIT ${pageRows}
-              ${physicalCursor === null && offset > 0 ? `OFFSET ${offset}` : ""}`
-          : `${baseSql}${orderSql} LIMIT ${pageRows} OFFSET ${offset}`
-        const rows = await runtime.query(pageSql)
-        if (rows.length === 0) return
-
-        yield* this.normalizeRows(rows, selectedColumns)
+      // Each bounded query releases both the runtime queue and attachment lease before yielding.
+      // SQLite metadata locks and PostgreSQL pool slots are released before consumer operations.
+      let offset = prepared.offset
+      let remaining = prepared.limit
+      let cursor: bigint | null = null
+      let page: readonly Record<string, unknown>[] | undefined = prepared.firstPage
+      while (remaining !== 0) {
+        read.signal.throwIfAborted()
+        const pageSize = Math.min(remaining ?? READ_PAGE_ROWS, READ_PAGE_ROWS)
+        const rows: readonly Record<string, unknown>[] =
+          page ??
+          (await this.connections.withAttachedRuntime((runtime) =>
+            runtime.query(pageSql(prepared, pageSize, offset, cursor))
+          ))
+        page = undefined
+        read.signal.throwIfAborted()
+        yield* this.normalizeRows(rows, prepared.columns, read.signal)
+        if (remaining !== undefined) remaining -= rows.length
+        if (rows.length < pageSize) return
+        if (prepared.physicalOrder)
+          cursor = getBigIntLike(rows[rows.length - 1]!, PHYSICAL_ROW_ID_ALIAS)
         offset += rows.length
-        if (remaining !== undefined) {
-          remaining -= rows.length
-          if (remaining === 0) return
-        }
-        if (canUsePhysicalRowId) {
-          physicalCursor = getBigIntLike(rows[rows.length - 1]!, PHYSICAL_ROW_ID_ALIAS)
-        }
-        if (rows.length < pageRows) return
       }
     } finally {
-      await lease.release()
+      read.signal.removeEventListener("abort", read.release)
+      try {
+        await reader?.close()
+      } finally {
+        read.release()
+      }
     }
   }
 
   private async *normalizeRows(
     rows: Iterable<Record<string, unknown>> | AsyncIterable<Record<string, unknown>>,
-    selectedColumns: readonly DatasetColumnDefinition[]
+    selectedColumns: readonly DatasetColumnDefinition[],
+    signal: AbortSignal
   ): AsyncIterable<DatasetRow> {
     for await (const row of rows) {
+      signal.throwIfAborted()
       const output: Record<string, unknown> = {}
       for (const column of selectedColumns) {
         output[column.name] = normalizeReadValue(row[column.name], column)
@@ -165,3 +173,16 @@ export class DuckLakeRowReader {
     )
   }
 }
+
+function pageSql(
+  read: { readonly columnsSql: string; readonly relation: string; readonly physicalOrder: boolean },
+  limit: number,
+  offset: number,
+  cursor: bigint | null
+): string {
+  return read.physicalOrder
+    ? `SELECT rowid AS ${quoteIdentifier(PHYSICAL_ROW_ID_ALIAS)}, ${read.columnsSql}
+        FROM ${read.relation} ${cursor === null ? "" : `WHERE rowid > ${cursor}`}
+        ORDER BY rowid LIMIT ${limit} ${cursor === null && offset > 0 ? `OFFSET ${offset}` : ""}`
+    : `SELECT ${read.columnsSql} FROM ${read.relation} LIMIT ${limit} OFFSET ${offset}`
+}

package/src/internal/ducklake-snapshot-reader.ts

@@ -5,6 +5,8 @@ import type {
   DatasetVersion,
   DatasetVersionMode,
   DatasetVersionRef,
+  LatestVersionsSince,
+  ListLatestVersionsSinceInput,
 } from "@sixb/core/lake-storage"
 import { LakeStorageError } from "@sixb/core/lake-storage"
 import type { DuckLakeStorageOptions } from "../types"
@@ -13,8 +15,8 @@ import type { DuckDbQueryRuntime } from "./duckdb-runtime"
 import type { DuckLakeConnectionManager } from "./ducklake-connection-manager"
 import type { DuckLakeDatasetCatalog } from "./ducklake-dataset-catalog"
 import { type DatasetTableRef, resolveDatasetTableRef } from "./ducklake-dataset-table-ref"
-import { encodeDatasetTableName } from "./names"
-import { duckLakeMetadataTableName, quoteSqlString } from "./sql"
+import { decodeDatasetTableName, encodeDatasetTableName } from "./names"
+import { buildDuckLakeMetadataQuery, duckLakeMetadataTableName, quoteSqlString } from "./sql"
 import {
   parseCommitMetadata,
   parseInlineDataChange,
@@ -41,11 +43,24 @@ interface DatasetSnapshotCandidateRow {
   readonly metadata?: SixbCommitMetadata
 }
 
+interface FeedTable {
+  readonly datasetId: string
+  readonly tableId: bigint
+  /** A dataset cannot have a version before its table exists. */
+  readonly startSnapshotId: bigint
+}
+
+interface MissingSnapshotCandidate {
+  readonly snapshotId: string
+  readonly missing: true
+}
+
 interface SnapshotCandidateQueryInput {
+  readonly datasetId: string
   readonly tableId: bigint
   readonly exactSnapshotId?: string
   readonly beforeSnapshotId?: string
-  readonly limit?: number
+  readonly limit: number
 }
 
 interface VisibleSnapshotRowsInput {
@@ -79,6 +94,8 @@ interface FileChangeFlags {
 }
 
 const SNAPSHOT_ROW_BATCH_SIZE = 128
+// One feed read covers at most this many commits; the next read continues from its cursor.
+const FEED_PAGE_SIZE = 512
 
 // The bulk catalog scan shares one descending walk of recent snapshots across
 // all requested datasets, so its cost is bounded by this window rather than by
@@ -189,6 +206,192 @@ export class DuckLakeSnapshotReader {
     })
   }
 
+  /**
+   * Change feed over DuckLake's snapshot table, which is an append-only commit log: snapshot ids
+   * only increase, and a commit becomes visible only after every lower id has (concurrent commits
+   * that pick the same id conflict, and the loser retries with the next one). The cursor is the
+   * last snapshot id read. Every query is bounded by a snapshot-id range, so an idle read is one
+   * indexed query that returns the cursor's own snapshot, whatever the history length.
+   */
+  async listLatestVersionsSince(
+    input: ListLatestVersionsSinceInput
+  ): Promise<LatestVersionsSince | null> {
+    this.connections.assertOpen()
+
+    return this.connections.withAttachedRuntime(async (runtime) => {
+      if (input.cursor === null) {
+        return { cursor: await this.queryHeadSnapshotId(runtime), versions: [] }
+      }
+      if (!/^\d+$/.test(input.cursor)) {
+        return null
+      }
+
+      const snapshots = await this.querySnapshotsFrom(runtime, input.cursor)
+      // The cursor's own snapshot proves the cursor is still a position in this lake's history.
+      if (snapshots[0]?.snapshotId !== input.cursor) {
+        return null
+      }
+      const newSnapshots = snapshots.slice(1)
+      if (newSnapshots.length === 0) {
+        return { cursor: input.cursor, versions: [] }
+      }
+
+      const tables = await this.resolveFeedTables(runtime, input.datasetIds)
+      const latest = await this.collectFeedPage(runtime, tables, newSnapshots)
+      return {
+        cursor: newSnapshots[newSnapshots.length - 1]?.snapshotId ?? input.cursor,
+        versions: [...latest].map(([datasetId, row]) => this.snapshotRowToSummary(datasetId, row)),
+      }
+    })
+  }
+
+  private async queryHeadSnapshotId(runtime: DuckDbQueryRuntime): Promise<string> {
+    const [row] = await runtime.query(
+      buildDuckLakeMetadataQuery(
+        this.options,
+        (table) => `SELECT max(snapshot_id) AS snapshot_id FROM ${table("ducklake_snapshot")}`
+      )
+    )
+    if (row?.snapshot_id === null || row?.snapshot_id === undefined) {
+      throw new LakeStorageError("[SixbDuckLake] DuckLake catalog has no snapshots.")
+    }
+    return String(getBigIntLike(row, "snapshot_id"))
+  }
+
+  /** The cursor's snapshot and up to one feed page of snapshots after it, oldest first. */
+  private async querySnapshotsFrom(
+    runtime: DuckDbQueryRuntime,
+    cursor: string
+  ): Promise<readonly { readonly snapshotId: string; readonly createdAt: Date }[]> {
+    const rows = await runtime.query(
+      buildDuckLakeMetadataQuery(
+        this.options,
+        (table) => `
+          SELECT snapshot_id, snapshot_time
+          FROM ${table("ducklake_snapshot")}
+          WHERE snapshot_id >= ${cursor}
+          ORDER BY snapshot_id
+          LIMIT ${FEED_PAGE_SIZE + 1}
+        `
+      )
+    )
+    return rows.map((row) => ({
+      snapshotId: String(getBigIntLike(row, "snapshot_id")),
+      createdAt: getDate(row, "snapshot_time"),
+    }))
+  }
+
+  /** Current dataset tables, with the first snapshot each table id existed in. */
+  private async resolveFeedTables(
+    runtime: DuckDbQueryRuntime,
+    datasetIds: readonly string[] | undefined
+  ): Promise<readonly FeedTable[]> {
+    const requested = datasetIds === undefined ? null : new Set(datasetIds)
+    const rows = await runtime.query(`
+      SELECT table_id, table_name, begin_snapshot, end_snapshot
+      FROM ${duckLakeMetadataTableName(this.options, "ducklake_table")}
+    `)
+
+    const startByTableId = new Map<bigint, bigint>()
+    const current: { datasetId: string; tableId: bigint }[] = []
+    for (const row of rows) {
+      const tableId = getBigIntLike(row, "table_id")
+      const begin = getBigIntLike(row, "begin_snapshot")
+      const start = startByTableId.get(tableId)
+      if (start === undefined || begin < start) {
+        startByTableId.set(tableId, begin)
+      }
+      if (row.end_snapshot !== null && row.end_snapshot !== undefined) {
+        continue
+      }
+      const datasetId = decodeDatasetTableName(getString(row, "table_name"))
+      if (datasetId !== null && (requested === null || requested.has(datasetId))) {
+        current.push({ datasetId, tableId })
+      }
+    }
+
+    return current.map((table) => ({
+      ...table,
+      startSnapshotId: startByTableId.get(table.tableId) ?? 0n,
+    }))
+  }
+
+  /** Folds a page of snapshots, oldest first, into each dataset's newest data version. */
+  private async collectFeedPage(
+    runtime: DuckDbQueryRuntime,
+    tables: readonly FeedTable[],
+    snapshots: readonly { readonly snapshotId: string; readonly createdAt: Date }[]
+  ): Promise<Map<string, DatasetSnapshotRow>> {
+    const latest = new Map<string, DatasetSnapshotRow>()
+    const first = snapshots[0]?.snapshotId
+    const last = snapshots[snapshots.length - 1]?.snapshotId
+    if (first === undefined || last === undefined || tables.length === 0) {
+      return latest
+    }
+
+    const changeRows = await runtime.query(
+      buildDuckLakeMetadataQuery(
+        this.options,
+        (table) => `
+          SELECT snapshot_id, changes_made, commit_extra_info
+          FROM ${table("ducklake_snapshot_changes")}
+          WHERE snapshot_id >= ${first} AND snapshot_id <= ${last}
+        `
+      )
+    )
+    const fileFlags = await this.queryFileChangeFlags(
+      runtime,
+      snapshots.map((snapshot) => snapshot.snapshotId),
+      tables.map((table) => table.tableId)
+    )
+    const changesBySnapshotId = new Map(
+      changeRows.map((row) => [String(getBigIntLike(row, "snapshot_id")), row])
+    )
+    const tablesByTableId = new Map(tables.map((table) => [table.tableId.toString(), table]))
+    const tablesByDatasetId = new Map(tables.map((table) => [table.datasetId, table]))
+
+    for (const snapshot of snapshots) {
+      const changes = changesBySnapshotId.get(snapshot.snapshotId)
+      if (changes === undefined) {
+        continue
+      }
+      const changesMade = getString(changes, "changes_made")
+      const metadata = parseCommitMetadata(changes.commit_extra_info)
+
+      // Only tables this snapshot names, touches through files, or tags can change visibility.
+      const touched = new Set<FeedTable>()
+      for (const change of changesMade.split(",")) {
+        const table = tablesByTableId.get(change.split(":")[1] ?? "")
+        if (table) touched.add(table)
+      }
+      for (const table of tables) {
+        if (fileFlags.has(fileChangeKey(snapshot.snapshotId, table.tableId))) touched.add(table)
+      }
+      const tagged = metadata === undefined ? undefined : tablesByDatasetId.get(metadata.datasetId)
+      if (tagged) touched.add(tagged)
+
+      for (const table of touched) {
+        if (BigInt(snapshot.snapshotId) < table.startSnapshotId) {
+          continue
+        }
+        const flags = fileFlags.get(fileChangeKey(snapshot.snapshotId, table.tableId))
+        const row = this.candidateToSnapshotRow(table.datasetId, table.tableId, {
+          snapshotId: snapshot.snapshotId,
+          createdAt: snapshot.createdAt,
+          changesMade,
+          hasFileChange: flags?.hasFileChange ?? false,
+          hasFileDeleteChange: flags?.hasFileDeleteChange ?? false,
+          ...(metadata !== undefined ? { metadata } : {}),
+        })
+        if (row && row.mode !== "schema") {
+          latest.set(table.datasetId, row)
+        }
+      }
+    }
+
+    return latest
+  }
+
   private snapshotRowToSummary(
     datasetId: string,
     row: DatasetSnapshotRow
@@ -604,6 +807,7 @@ export class DuckLakeSnapshotReader {
 
     while (input.visibleRowLimit === undefined || snapshots.length < input.visibleRowLimit) {
       const candidates = await this.querySnapshotCandidates(runtime, {
+        datasetId: input.datasetId,
         tableId: input.tableId,
         exactSnapshotId: input.exactSnapshotId,
         beforeSnapshotId,
@@ -614,6 +818,9 @@ export class DuckLakeSnapshotReader {
       }
 
       for (const candidate of candidates) {
+        if ("missing" in candidate) {
+          continue
+        }
         const snapshot = this.candidateToSnapshotRow(input.datasetId, input.tableId, candidate)
         if (snapshot) {
           snapshots.push(snapshot)
@@ -634,99 +841,144 @@ export class DuckLakeSnapshotReader {
     return snapshots
   }
 
+  /**
+   * Hydrates, newest first, the snapshots that may hold a version of one dataset table: those
+   * that added or removed its data or delete files, that DuckLake recorded as changing it inline,
+   * or whose Sixb commit metadata names the dataset. The text matches are a superset;
+   * candidateToSnapshotRow decides visibility. Only this table's snapshots are read, so the cost
+   * follows the dataset's own history rather than every commit in the catalog.
+   *
+   * Returns one row per candidate id, including ids whose snapshot no longer exists, so the caller
+   * can tell a short batch from the end of the history.
+   */
   private async querySnapshotCandidates(
     runtime: DuckDbQueryRuntime,
     input: SnapshotCandidateQueryInput
-  ): Promise<readonly DatasetSnapshotCandidateRow[]> {
-    if (input.limit !== undefined && input.limit <= 0) {
+  ): Promise<readonly (DatasetSnapshotCandidateRow | MissingSnapshotCandidate)[]> {
+    const tableId = input.tableId.toString()
+    const limit = Math.max(0, Math.trunc(input.limit))
+    if (limit === 0) {
       return []
     }
-
-    const where: string[] = []
     if (input.exactSnapshotId !== undefined) {
       assertDuckLakeSnapshotId(input.exactSnapshotId)
-      where.push(`snapshot.snapshot_id = ${input.exactSnapshotId}`)
     }
     if (input.beforeSnapshotId !== undefined) {
       assertDuckLakeSnapshotId(input.beforeSnapshotId)
-      where.push(`snapshot.snapshot_id < ${input.beforeSnapshotId}`)
     }
 
-    const whereSql = where.length === 0 ? "" : `WHERE ${where.join(" AND ")}`
-    const limitSql =
-      input.limit === undefined ? "" : `LIMIT ${Math.max(0, Math.trunc(input.limit))}`
-    const ducklakeSnapshot = duckLakeMetadataTableName(this.options, "ducklake_snapshot")
-    const ducklakeSnapshotChanges = duckLakeMetadataTableName(
-      this.options,
-      "ducklake_snapshot_changes"
+    const rows = await runtime.query(
+      buildDuckLakeMetadataQuery(this.options, (table) => {
+        // A dataset cannot have a version before its table exists. Use the earliest table record:
+        // a rename may create a newer record with the same table id and must not hide older
+        // versions.
+        const tableStart = `(SELECT min(begin_snapshot) FROM ${table("ducklake_table")} WHERE table_id = ${tableId})`
+        const candidateIds =
+          input.exactSnapshotId !== undefined
+            ? `SELECT CAST(${input.exactSnapshotId} AS BIGINT) AS snapshot_id
+               WHERE ${input.exactSnapshotId} >= ${tableStart}`
+            : `SELECT DISTINCT snapshot_id
+               FROM (
+                 SELECT begin_snapshot AS snapshot_id
+                 FROM ${table("ducklake_data_file")} WHERE table_id = ${tableId}
+                 UNION ALL
+                 SELECT end_snapshot
+                 FROM ${table("ducklake_data_file")}
+                 WHERE table_id = ${tableId} AND end_snapshot IS NOT NULL
+                 UNION ALL
+                 SELECT begin_snapshot
+                 FROM ${table("ducklake_delete_file")} WHERE table_id = ${tableId}
+                 UNION ALL
+                 SELECT end_snapshot
+                 FROM ${table("ducklake_delete_file")}
+                 WHERE table_id = ${tableId} AND end_snapshot IS NOT NULL
+                 UNION ALL
+                 SELECT snapshot_id
+                 FROM ${table("ducklake_snapshot_changes")}
+                 WHERE strpos(',' || changes_made || ',', ${quoteSqlString(`:${tableId},`)}) > 0
+                   OR strpos(
+                     commit_extra_info,
+                     ${quoteSqlString(`"datasetId":${JSON.stringify(input.datasetId)}`)}
+                   ) > 0
+               ) touched
+               WHERE snapshot_id >= ${tableStart}
+                 ${input.beforeSnapshotId === undefined ? "" : `AND snapshot_id < ${input.beforeSnapshotId}`}
+               ORDER BY snapshot_id DESC
+               LIMIT ${limit}`
+
+        return `
+          WITH candidate_ids AS (
+            ${candidateIds}
+          ),
+          candidate_snapshots AS (
+            SELECT
+              snapshot.snapshot_id,
+              snapshot.snapshot_time,
+              changes.changes_made,
+              changes.commit_extra_info
+            FROM ${table("ducklake_snapshot")} snapshot
+            JOIN ${table("ducklake_snapshot_changes")} changes
+              ON changes.snapshot_id = snapshot.snapshot_id
+            WHERE snapshot.snapshot_id IN (SELECT snapshot_id FROM candidate_ids)
+          ),
+          file_changes AS (
+            -- File metadata tells whether this table changed in each candidate snapshot.
+            SELECT begin_snapshot AS snapshot_id, false AS is_delete_change
+            FROM ${table("ducklake_data_file")}
+            WHERE table_id = ${tableId}
+              AND begin_snapshot IN (SELECT snapshot_id FROM candidate_ids)
+            UNION ALL
+            SELECT end_snapshot AS snapshot_id, true AS is_delete_change
+            FROM ${table("ducklake_data_file")}
+            WHERE table_id = ${tableId}
+              AND end_snapshot IN (SELECT snapshot_id FROM candidate_ids)
+            UNION ALL
+            SELECT begin_snapshot AS snapshot_id, true AS is_delete_change
+            FROM ${table("ducklake_delete_file")}
+            WHERE table_id = ${tableId}
+              AND begin_snapshot IN (SELECT snapshot_id FROM candidate_ids)
+            UNION ALL
+            SELECT end_snapshot AS snapshot_id, true AS is_delete_change
+            FROM ${table("ducklake_delete_file")}
+            WHERE table_id = ${tableId}
+              AND end_snapshot IN (SELECT snapshot_id FROM candidate_ids)
+          ),
+          file_changes_by_snapshot AS (
+            -- Collapse file-level changes into one row per snapshot.
+            SELECT
+              snapshot_id,
+              count(*) > 0 AS has_file_change,
+              count(*) FILTER (WHERE is_delete_change) > 0 AS has_file_delete_change
+            FROM file_changes
+            GROUP BY snapshot_id
+          )
+          SELECT
+            -- Keep metadata-only candidates; Sixb filters them with commit_extra_info.
+            candidate_ids.snapshot_id,
+            candidate.snapshot_time,
+            candidate.changes_made,
+            candidate.commit_extra_info,
+            coalesce(file_changes.has_file_change, false) AS has_file_change,
+            coalesce(file_changes.has_file_delete_change, false) AS has_file_delete_change
+          FROM candidate_ids
+          LEFT JOIN candidate_snapshots candidate
+            ON candidate.snapshot_id = candidate_ids.snapshot_id
+          LEFT JOIN file_changes_by_snapshot file_changes
+            ON file_changes.snapshot_id = candidate_ids.snapshot_id
+          ORDER BY candidate_ids.snapshot_id DESC
+        `
+      })
     )
-    const ducklakeDataFile = duckLakeMetadataTableName(this.options, "ducklake_data_file")
-    const ducklakeDeleteFile = duckLakeMetadataTableName(this.options, "ducklake_delete_file")
-
-    const rows = await runtime.query(`
-      WITH candidate_snapshots AS (
-        -- Recent DuckLake snapshots to inspect; Sixb visibility is filtered later.
-        SELECT
-          snapshot.snapshot_id,
-          snapshot.snapshot_time,
-          changes.changes_made,
-          changes.commit_extra_info
-        FROM ${ducklakeSnapshot} snapshot
-        JOIN ${ducklakeSnapshotChanges} changes
-          ON changes.snapshot_id = snapshot.snapshot_id
-        ${whereSql}
-        ORDER BY snapshot.snapshot_id DESC
-        ${limitSql}
-      ),
-      file_changes AS (
-        -- File metadata tells whether this table changed in each candidate snapshot.
-        SELECT begin_snapshot AS snapshot_id, false AS is_delete_change
-        FROM ${ducklakeDataFile}
-        WHERE table_id = ${input.tableId}
-          AND begin_snapshot IN (SELECT snapshot_id FROM candidate_snapshots)
-        UNION ALL
-        SELECT end_snapshot AS snapshot_id, true AS is_delete_change
-        FROM ${ducklakeDataFile}
-        WHERE table_id = ${input.tableId}
-          AND end_snapshot IN (SELECT snapshot_id FROM candidate_snapshots)
-        UNION ALL
-        SELECT begin_snapshot AS snapshot_id, true AS is_delete_change
-        FROM ${ducklakeDeleteFile}
-        WHERE table_id = ${input.tableId}
-          AND begin_snapshot IN (SELECT snapshot_id FROM candidate_snapshots)
-        UNION ALL
-        SELECT end_snapshot AS snapshot_id, true AS is_delete_change
-        FROM ${ducklakeDeleteFile}
-        WHERE table_id = ${input.tableId}
-          AND end_snapshot IN (SELECT snapshot_id FROM candidate_snapshots)
-      ),
-      file_changes_by_snapshot AS (
-        -- Collapse file-level changes into one row per snapshot.
-        SELECT
-          snapshot_id,
-          count(*) > 0 AS has_file_change,
-          count(*) FILTER (WHERE is_delete_change) > 0 AS has_file_delete_change
-        FROM file_changes
-        GROUP BY snapshot_id
-      )
-      SELECT
-        -- Keep metadata-only candidates; Sixb filters them with commit_extra_info.
-        candidate.snapshot_id,
-        candidate.snapshot_time,
-        candidate.changes_made,
-        candidate.commit_extra_info,
-        coalesce(file_changes.has_file_change, false) AS has_file_change,
-        coalesce(file_changes.has_file_delete_change, false) AS has_file_delete_change
-      FROM candidate_snapshots candidate
-      LEFT JOIN file_changes_by_snapshot file_changes
-        ON file_changes.snapshot_id = candidate.snapshot_id
-      ORDER BY candidate.snapshot_id DESC
-    `)
 
     return rows.map((row) => {
+      const snapshotId = String(getBigIntLike(row, "snapshot_id"))
+      // Expired snapshots keep their file records; they are no longer versions.
+      if (row.snapshot_time === null || row.snapshot_time === undefined) {
+        return { snapshotId, missing: true } as const
+      }
       const metadata = parseCommitMetadata(row.commit_extra_info)
       return {
-        snapshotId: String(getBigIntLike(row, "snapshot_id")),
+        snapshotId,
         createdAt: getDate(row, "snapshot_time"),
         changesMade: getString(row, "changes_made"),
         hasFileChange: getBoolean(row, "has_file_change"),