@sixb/ducklake 0.1.7 → 0.1.8

package/README.md

@@ -534,7 +534,7 @@ Local DuckLake e2e tests:
 bun --filter @sixb/ducklake test:e2e:local
 ```
 
-Docker-backed PostgreSQL and MinIO/S3-compatible e2e tests:
+Docker-backed PostgreSQL and SeaweedFS (S3-compatible) e2e tests:
 
 ```bash
 bun --filter @sixb/ducklake test:e2e:remote

package/dist/ducklake-storage.d.ts

@@ -1,5 +1,5 @@
 import type { DatasetDefinition, DatasetRow, LakeStorageWithSql } from "@sixb/core";
-import type { BeginDatasetMergeInput, BeginDatasetWriteInput, DatasetCatalogState, DatasetChanges, DatasetVersion, LakeMergeSession, LakeWriteSession, ReadDatasetChangesInput, ReadDatasetRowsInput } from "@sixb/core/lake-storage";
+import type { BeginDatasetMergeInput, BeginDatasetWriteInput, DatasetCatalogState, DatasetChanges, DatasetVersion, LakeMergeSession, LakeWriteSession, LatestVersionsSince, ListLatestVersionsSinceInput, ReadDatasetChangesInput, ReadDatasetRowsInput } from "@sixb/core/lake-storage";
 import { DuckLakeSqlExecutor } from "./internal/ducklake-sql-executor";
 import type { DuckLakeMaintenanceOptions, DuckLakeMaintenanceReport, DuckLakeStorageOptions } from "./types";
 /**
@@ -33,6 +33,7 @@ export declare class DuckLakeStorage implements LakeStorageWithSql<"duckdb"> {
     beginMerge(input: BeginDatasetMergeInput): Promise<LakeMergeSession>;
     getLatestVersion(datasetId: string): Promise<DatasetVersion | null>;
     getVersion(datasetId: string, versionId: string): Promise<DatasetVersion | null>;
+    listLatestVersionsSince(input: ListLatestVersionsSinceInput): Promise<LatestVersionsSince | null>;
     readRows(input: ReadDatasetRowsInput): AsyncIterable<DatasetRow>;
     readChanges(input: ReadDatasetChangesInput): Promise<DatasetChanges | null>;
     runMaintenance(options?: DuckLakeMaintenanceOptions): Promise<DuckLakeMaintenanceReport>;

package/dist/ducklake-storage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ducklake-storage.d.ts","sourceRoot":"","sources":["../src/ducklake-storage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAA;AACnF,OAAO,KAAK,EACV,sBAAsB,EACtB,sBAAsB,EACtB,mBAAmB,EACnB,cAAc,EACd,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,uBAAuB,EACvB,oBAAoB,EACrB,MAAM,yBAAyB,CAAA;AAOhC,OAAO,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAA;AAEtE,OAAO,KAAK,EACV,0BAA0B,EAC1B,yBAAyB,EACzB,sBAAsB,EACvB,MAAM,SAAS,CAAA;AAEhB;;;;;;GAMG;AACH,qBAAa,eAAgB,YAAW,kBAAkB,CAAC,QAAQ,CAAC;IAClE,QAAQ,CAAC,QAAQ;;;MAA8C;IAC/D,QAAQ,CAAC,GAAG,EAAE,mBAAmB,CAAA;IAEjC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA2B;IACvD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAwB;IACjD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;IACjD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAmB;IACxC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAsB;IAC9C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAwB;IACvD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0B;gBAErC,OAAO,EAAE,sBAAsB;IAwCrC,aAAa,CAAC,UAAU,EAAE,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAIxE,kCAAkC,CACtC,WAAW,EAAE,SAAS,iBAAiB,EAAE,GACxC,OAAO,CAAC,IAAI,CAAC;IAIV,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAIhE,YAAY,IAAI,OAAO,CAAC,SAAS,iBAAiB,EAAE,CAAC;IAIrD,uBAAuB,CAC3B,UAAU,EAAE,SAAS,MAAM,EAAE,GAC5B,OAAO,CAAC,SAAS,mBAAmB,EAAE,CAAC;IAIpC,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC;IAInF,UAAU,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAIpE,UAAU,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAIpE,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAInE,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAItF,QAAQ,CAAC,KAAK,EAAE,oBAAoB,GAAG,aAAa,CAAC,UAAU,CAAC;IAIhE,WAAW,CAAC,KAAK,EAAE,uBAAuB,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAIrE,cAAc,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,yBAAyB,CAAC;IAI9F;;;OAGG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B"}
+{"version":3,"file":"ducklake-storage.d.ts","sourceRoot":"","sources":["../src/ducklake-storage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAA;AACnF,OAAO,KAAK,EACV,sBAAsB,EACtB,sBAAsB,EACtB,mBAAmB,EACnB,cAAc,EACd,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,mBAAmB,EACnB,4BAA4B,EAC5B,uBAAuB,EACvB,oBAAoB,EACrB,MAAM,yBAAyB,CAAA;AAOhC,OAAO,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAA;AAEtE,OAAO,KAAK,EACV,0BAA0B,EAC1B,yBAAyB,EACzB,sBAAsB,EACvB,MAAM,SAAS,CAAA;AAEhB;;;;;;GAMG;AACH,qBAAa,eAAgB,YAAW,kBAAkB,CAAC,QAAQ,CAAC;IAClE,QAAQ,CAAC,QAAQ;;;MAA8C;IAC/D,QAAQ,CAAC,GAAG,EAAE,mBAAmB,CAAA;IAEjC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA2B;IACvD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAwB;IACjD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;IACjD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAmB;IACxC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAsB;IAC9C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAwB;IACvD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0B;gBAErC,OAAO,EAAE,sBAAsB;IAwCrC,aAAa,CAAC,UAAU,EAAE,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAIxE,kCAAkC,CACtC,WAAW,EAAE,SAAS,iBAAiB,EAAE,GACxC,OAAO,CAAC,IAAI,CAAC;IAIV,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAIhE,YAAY,IAAI,OAAO,CAAC,SAAS,iBAAiB,EAAE,CAAC;IAIrD,uBAAuB,CAC3B,UAAU,EAAE,SAAS,MAAM,EAAE,GAC5B,OAAO,CAAC,SAAS,mBAAmB,EAAE,CAAC;IAIpC,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC;IAInF,UAAU,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAIpE,UAAU,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAIpE,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAInE,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAIhF,uBAAuB,CAC3B,KAAK,EAAE,4BAA4B,GAClC,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC;IAItC,QAAQ,CAAC,KAAK,EAAE,oBAAoB,GAAG,aAAa,CAAC,UAAU,CAAC;IAIhE,WAAW,CAAC,KAAK,EAAE,uBAAuB,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAIrE,cAAc,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,yBAAyB,CAAC;IAI9F;;;OAGG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B"}

package/dist/index.js

@@ -158,8 +158,20 @@ function buildSetPostgresPoolSql(options) {
   return postgresPoolParameters(options.postgresPool).map(({ name, value }) => `SET pg_pool_${name} = ${value}`);
 }
 function duckLakeMetadataTableName(options, tableName) {
-  const schemaName = options.catalog.type === "postgres" ? options.catalog.metadataSchema ?? "main" : "main";
-  return `${quoteIdentifier(duckLakeMetadataCatalog(options))}.${quoteIdentifier(schemaName)}.${quoteIdentifier(tableName)}`;
+  return `${quoteIdentifier(duckLakeMetadataCatalog(options))}.${quoteIdentifier(duckLakeMetadataSchema(options))}.${quoteIdentifier(tableName)}`;
+}
+function buildDuckLakeMetadataQuery(options, build) {
+  if (options.catalog.type !== "postgres") {
+    return build((tableName) => duckLakeMetadataTableName(options, tableName));
+  }
+  const schema = options.catalog.metadataSchema;
+  const sql = build((tableName) => schema === undefined ? quoteIdentifier(tableName) : `${quoteIdentifier(schema)}.${quoteIdentifier(tableName)}`);
+  return `SELECT * FROM postgres_query(${quoteSqlString(duckLakeMetadataCatalog(options))}, ${quoteSqlString(sql)})`;
+}
+function duckLakeMetadataSchema(options) {
+  if (options.catalog.type !== "postgres")
+    return "main";
+  return options.catalog.metadataSchema ?? "public";
 }
 function quoteIdentifier(identifier) {
   if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(identifier)) {
@@ -2311,6 +2323,7 @@ function pageSql(read, limit, offset, cursor) {
 // src/internal/ducklake-snapshot-reader.ts
 import { LakeStorageError as LakeStorageError17 } from "@sixb/core/lake-storage";
 var SNAPSHOT_ROW_BATCH_SIZE = 128;
+var FEED_PAGE_SIZE = 512;
 var CATALOG_SNAPSHOT_SCAN_LIMIT = 512;
 
 class DuckLakeSnapshotReader {
@@ -2376,6 +2389,135 @@ class DuckLakeSnapshotReader {
       };
     });
   }
+  async listLatestVersionsSince(input) {
+    this.connections.assertOpen();
+    return this.connections.withAttachedRuntime(async (runtime) => {
+      if (input.cursor === null) {
+        return { cursor: await this.queryHeadSnapshotId(runtime), versions: [] };
+      }
+      if (!/^\d+$/.test(input.cursor)) {
+        return null;
+      }
+      const snapshots = await this.querySnapshotsFrom(runtime, input.cursor);
+      if (snapshots[0]?.snapshotId !== input.cursor) {
+        return null;
+      }
+      const newSnapshots = snapshots.slice(1);
+      if (newSnapshots.length === 0) {
+        return { cursor: input.cursor, versions: [] };
+      }
+      const tables = await this.resolveFeedTables(runtime, input.datasetIds);
+      const latest = await this.collectFeedPage(runtime, tables, newSnapshots);
+      return {
+        cursor: newSnapshots[newSnapshots.length - 1]?.snapshotId ?? input.cursor,
+        versions: [...latest].map(([datasetId, row]) => this.snapshotRowToSummary(datasetId, row))
+      };
+    });
+  }
+  async queryHeadSnapshotId(runtime) {
+    const [row] = await runtime.query(buildDuckLakeMetadataQuery(this.options, (table) => `SELECT max(snapshot_id) AS snapshot_id FROM ${table("ducklake_snapshot")}`));
+    if (row?.snapshot_id === null || row?.snapshot_id === undefined) {
+      throw new LakeStorageError17("[SixbDuckLake] DuckLake catalog has no snapshots.");
+    }
+    return String(getBigIntLike(row, "snapshot_id"));
+  }
+  async querySnapshotsFrom(runtime, cursor) {
+    const rows = await runtime.query(buildDuckLakeMetadataQuery(this.options, (table) => `
+          SELECT snapshot_id, snapshot_time
+          FROM ${table("ducklake_snapshot")}
+          WHERE snapshot_id >= ${cursor}
+          ORDER BY snapshot_id
+          LIMIT ${FEED_PAGE_SIZE + 1}
+        `));
+    return rows.map((row) => ({
+      snapshotId: String(getBigIntLike(row, "snapshot_id")),
+      createdAt: getDate(row, "snapshot_time")
+    }));
+  }
+  async resolveFeedTables(runtime, datasetIds) {
+    const requested = datasetIds === undefined ? null : new Set(datasetIds);
+    const rows = await runtime.query(`
+      SELECT table_id, table_name, begin_snapshot, end_snapshot
+      FROM ${duckLakeMetadataTableName(this.options, "ducklake_table")}
+    `);
+    const startByTableId = new Map;
+    const current = [];
+    for (const row of rows) {
+      const tableId = getBigIntLike(row, "table_id");
+      const begin = getBigIntLike(row, "begin_snapshot");
+      const start = startByTableId.get(tableId);
+      if (start === undefined || begin < start) {
+        startByTableId.set(tableId, begin);
+      }
+      if (row.end_snapshot !== null && row.end_snapshot !== undefined) {
+        continue;
+      }
+      const datasetId = decodeDatasetTableName(getString(row, "table_name"));
+      if (datasetId !== null && (requested === null || requested.has(datasetId))) {
+        current.push({ datasetId, tableId });
+      }
+    }
+    return current.map((table) => ({
+      ...table,
+      startSnapshotId: startByTableId.get(table.tableId) ?? 0n
+    }));
+  }
+  async collectFeedPage(runtime, tables, snapshots) {
+    const latest = new Map;
+    const first = snapshots[0]?.snapshotId;
+    const last = snapshots[snapshots.length - 1]?.snapshotId;
+    if (first === undefined || last === undefined || tables.length === 0) {
+      return latest;
+    }
+    const changeRows = await runtime.query(buildDuckLakeMetadataQuery(this.options, (table) => `
+          SELECT snapshot_id, changes_made, commit_extra_info
+          FROM ${table("ducklake_snapshot_changes")}
+          WHERE snapshot_id >= ${first} AND snapshot_id <= ${last}
+        `));
+    const fileFlags = await this.queryFileChangeFlags(runtime, snapshots.map((snapshot) => snapshot.snapshotId), tables.map((table) => table.tableId));
+    const changesBySnapshotId = new Map(changeRows.map((row) => [String(getBigIntLike(row, "snapshot_id")), row]));
+    const tablesByTableId = new Map(tables.map((table) => [table.tableId.toString(), table]));
+    const tablesByDatasetId = new Map(tables.map((table) => [table.datasetId, table]));
+    for (const snapshot of snapshots) {
+      const changes = changesBySnapshotId.get(snapshot.snapshotId);
+      if (changes === undefined) {
+        continue;
+      }
+      const changesMade = getString(changes, "changes_made");
+      const metadata = parseCommitMetadata(changes.commit_extra_info);
+      const touched = new Set;
+      for (const change of changesMade.split(",")) {
+        const table = tablesByTableId.get(change.split(":")[1] ?? "");
+        if (table)
+          touched.add(table);
+      }
+      for (const table of tables) {
+        if (fileFlags.has(fileChangeKey(snapshot.snapshotId, table.tableId)))
+          touched.add(table);
+      }
+      const tagged = metadata === undefined ? undefined : tablesByDatasetId.get(metadata.datasetId);
+      if (tagged)
+        touched.add(tagged);
+      for (const table of touched) {
+        if (BigInt(snapshot.snapshotId) < table.startSnapshotId) {
+          continue;
+        }
+        const flags = fileFlags.get(fileChangeKey(snapshot.snapshotId, table.tableId));
+        const row = this.candidateToSnapshotRow(table.datasetId, table.tableId, {
+          snapshotId: snapshot.snapshotId,
+          createdAt: snapshot.createdAt,
+          changesMade,
+          hasFileChange: flags?.hasFileChange ?? false,
+          hasFileDeleteChange: flags?.hasFileDeleteChange ?? false,
+          ...metadata !== undefined ? { metadata } : {}
+        });
+        if (row && row.mode !== "schema") {
+          latest.set(table.datasetId, row);
+        }
+      }
+    }
+    return latest;
+  }
   snapshotRowToSummary(datasetId, row) {
     return {
       datasetId,
@@ -2639,6 +2781,7 @@ class DuckLakeSnapshotReader {
     let beforeSnapshotId = input.beforeSnapshotId;
     while (input.visibleRowLimit === undefined || snapshots.length < input.visibleRowLimit) {
       const candidates = await this.querySnapshotCandidates(runtime, {
+        datasetId: input.datasetId,
         tableId: input.tableId,
         exactSnapshotId: input.exactSnapshotId,
         beforeSnapshotId,
@@ -2648,6 +2791,9 @@ class DuckLakeSnapshotReader {
         break;
       }
       for (const candidate of candidates) {
+        if ("missing" in candidate) {
+          continue;
+        }
         const snapshot = this.candidateToSnapshotRow(input.datasetId, input.tableId, candidate);
         if (snapshot) {
           snapshots.push(snapshot);
@@ -2664,90 +2810,118 @@ class DuckLakeSnapshotReader {
     return snapshots;
   }
   async querySnapshotCandidates(runtime, input) {
-    if (input.limit !== undefined && input.limit <= 0) {
+    const tableId = input.tableId.toString();
+    const limit = Math.max(0, Math.trunc(input.limit));
+    if (limit === 0) {
       return [];
     }
-    const ducklakeTable = duckLakeMetadataTableName(this.options, "ducklake_table");
-    const where = [
-      `snapshot.snapshot_id >= (SELECT min(begin_snapshot) FROM ${ducklakeTable} WHERE table_id = ${input.tableId})`
-    ];
     if (input.exactSnapshotId !== undefined) {
       assertDuckLakeSnapshotId2(input.exactSnapshotId);
-      where.push(`snapshot.snapshot_id = ${input.exactSnapshotId}`);
     }
     if (input.beforeSnapshotId !== undefined) {
       assertDuckLakeSnapshotId2(input.beforeSnapshotId);
-      where.push(`snapshot.snapshot_id < ${input.beforeSnapshotId}`);
     }
-    const whereSql = `WHERE ${where.join(" AND ")}`;
-    const limitSql = input.limit === undefined ? "" : `LIMIT ${Math.max(0, Math.trunc(input.limit))}`;
-    const ducklakeSnapshot = duckLakeMetadataTableName(this.options, "ducklake_snapshot");
-    const ducklakeSnapshotChanges = duckLakeMetadataTableName(this.options, "ducklake_snapshot_changes");
-    const ducklakeDataFile = duckLakeMetadataTableName(this.options, "ducklake_data_file");
-    const ducklakeDeleteFile = duckLakeMetadataTableName(this.options, "ducklake_delete_file");
-    const rows = await runtime.query(`
-      WITH candidate_snapshots AS (
-        -- Recent DuckLake snapshots to inspect; Sixb visibility is filtered later.
-        SELECT
-          snapshot.snapshot_id,
-          snapshot.snapshot_time,
-          changes.changes_made,
-          changes.commit_extra_info
-        FROM ${ducklakeSnapshot} snapshot
-        JOIN ${ducklakeSnapshotChanges} changes
-          ON changes.snapshot_id = snapshot.snapshot_id
-        ${whereSql}
-        ORDER BY snapshot.snapshot_id DESC
-        ${limitSql}
-      ),
-      file_changes AS (
-        -- File metadata tells whether this table changed in each candidate snapshot.
-        SELECT begin_snapshot AS snapshot_id, false AS is_delete_change
-        FROM ${ducklakeDataFile}
-        WHERE table_id = ${input.tableId}
-          AND begin_snapshot IN (SELECT snapshot_id FROM candidate_snapshots)
-        UNION ALL
-        SELECT end_snapshot AS snapshot_id, true AS is_delete_change
-        FROM ${ducklakeDataFile}
-        WHERE table_id = ${input.tableId}
-          AND end_snapshot IN (SELECT snapshot_id FROM candidate_snapshots)
-        UNION ALL
-        SELECT begin_snapshot AS snapshot_id, true AS is_delete_change
-        FROM ${ducklakeDeleteFile}
-        WHERE table_id = ${input.tableId}
-          AND begin_snapshot IN (SELECT snapshot_id FROM candidate_snapshots)
-        UNION ALL
-        SELECT end_snapshot AS snapshot_id, true AS is_delete_change
-        FROM ${ducklakeDeleteFile}
-        WHERE table_id = ${input.tableId}
-          AND end_snapshot IN (SELECT snapshot_id FROM candidate_snapshots)
-      ),
-      file_changes_by_snapshot AS (
-        -- Collapse file-level changes into one row per snapshot.
-        SELECT
-          snapshot_id,
-          count(*) > 0 AS has_file_change,
-          count(*) FILTER (WHERE is_delete_change) > 0 AS has_file_delete_change
-        FROM file_changes
-        GROUP BY snapshot_id
-      )
-      SELECT
-        -- Keep metadata-only candidates; Sixb filters them with commit_extra_info.
-        candidate.snapshot_id,
-        candidate.snapshot_time,
-        candidate.changes_made,
-        candidate.commit_extra_info,
-        coalesce(file_changes.has_file_change, false) AS has_file_change,
-        coalesce(file_changes.has_file_delete_change, false) AS has_file_delete_change
-      FROM candidate_snapshots candidate
-      LEFT JOIN file_changes_by_snapshot file_changes
-        ON file_changes.snapshot_id = candidate.snapshot_id
-      ORDER BY candidate.snapshot_id DESC
-    `);
+    const rows = await runtime.query(buildDuckLakeMetadataQuery(this.options, (table) => {
+      const tableStart = `(SELECT min(begin_snapshot) FROM ${table("ducklake_table")} WHERE table_id = ${tableId})`;
+      const candidateIds = input.exactSnapshotId !== undefined ? `SELECT CAST(${input.exactSnapshotId} AS BIGINT) AS snapshot_id
+               WHERE ${input.exactSnapshotId} >= ${tableStart}` : `SELECT DISTINCT snapshot_id
+               FROM (
+                 SELECT begin_snapshot AS snapshot_id
+                 FROM ${table("ducklake_data_file")} WHERE table_id = ${tableId}
+                 UNION ALL
+                 SELECT end_snapshot
+                 FROM ${table("ducklake_data_file")}
+                 WHERE table_id = ${tableId} AND end_snapshot IS NOT NULL
+                 UNION ALL
+                 SELECT begin_snapshot
+                 FROM ${table("ducklake_delete_file")} WHERE table_id = ${tableId}
+                 UNION ALL
+                 SELECT end_snapshot
+                 FROM ${table("ducklake_delete_file")}
+                 WHERE table_id = ${tableId} AND end_snapshot IS NOT NULL
+                 UNION ALL
+                 SELECT snapshot_id
+                 FROM ${table("ducklake_snapshot_changes")}
+                 WHERE strpos(',' || changes_made || ',', ${quoteSqlString(`:${tableId},`)}) > 0
+                   OR strpos(
+                     commit_extra_info,
+                     ${quoteSqlString(`"datasetId":${JSON.stringify(input.datasetId)}`)}
+                   ) > 0
+               ) touched
+               WHERE snapshot_id >= ${tableStart}
+                 ${input.beforeSnapshotId === undefined ? "" : `AND snapshot_id < ${input.beforeSnapshotId}`}
+               ORDER BY snapshot_id DESC
+               LIMIT ${limit}`;
+      return `
+          WITH candidate_ids AS (
+            ${candidateIds}
+          ),
+          candidate_snapshots AS (
+            SELECT
+              snapshot.snapshot_id,
+              snapshot.snapshot_time,
+              changes.changes_made,
+              changes.commit_extra_info
+            FROM ${table("ducklake_snapshot")} snapshot
+            JOIN ${table("ducklake_snapshot_changes")} changes
+              ON changes.snapshot_id = snapshot.snapshot_id
+            WHERE snapshot.snapshot_id IN (SELECT snapshot_id FROM candidate_ids)
+          ),
+          file_changes AS (
+            -- File metadata tells whether this table changed in each candidate snapshot.
+            SELECT begin_snapshot AS snapshot_id, false AS is_delete_change
+            FROM ${table("ducklake_data_file")}
+            WHERE table_id = ${tableId}
+              AND begin_snapshot IN (SELECT snapshot_id FROM candidate_ids)
+            UNION ALL
+            SELECT end_snapshot AS snapshot_id, true AS is_delete_change
+            FROM ${table("ducklake_data_file")}
+            WHERE table_id = ${tableId}
+              AND end_snapshot IN (SELECT snapshot_id FROM candidate_ids)
+            UNION ALL
+            SELECT begin_snapshot AS snapshot_id, true AS is_delete_change
+            FROM ${table("ducklake_delete_file")}
+            WHERE table_id = ${tableId}
+              AND begin_snapshot IN (SELECT snapshot_id FROM candidate_ids)
+            UNION ALL
+            SELECT end_snapshot AS snapshot_id, true AS is_delete_change
+            FROM ${table("ducklake_delete_file")}
+            WHERE table_id = ${tableId}
+              AND end_snapshot IN (SELECT snapshot_id FROM candidate_ids)
+          ),
+          file_changes_by_snapshot AS (
+            -- Collapse file-level changes into one row per snapshot.
+            SELECT
+              snapshot_id,
+              count(*) > 0 AS has_file_change,
+              count(*) FILTER (WHERE is_delete_change) > 0 AS has_file_delete_change
+            FROM file_changes
+            GROUP BY snapshot_id
+          )
+          SELECT
+            -- Keep metadata-only candidates; Sixb filters them with commit_extra_info.
+            candidate_ids.snapshot_id,
+            candidate.snapshot_time,
+            candidate.changes_made,
+            candidate.commit_extra_info,
+            coalesce(file_changes.has_file_change, false) AS has_file_change,
+            coalesce(file_changes.has_file_delete_change, false) AS has_file_delete_change
+          FROM candidate_ids
+          LEFT JOIN candidate_snapshots candidate
+            ON candidate.snapshot_id = candidate_ids.snapshot_id
+          LEFT JOIN file_changes_by_snapshot file_changes
+            ON file_changes.snapshot_id = candidate_ids.snapshot_id
+          ORDER BY candidate_ids.snapshot_id DESC
+        `;
+    }));
     return rows.map((row) => {
+      const snapshotId = String(getBigIntLike(row, "snapshot_id"));
+      if (row.snapshot_time === null || row.snapshot_time === undefined) {
+        return { snapshotId, missing: true };
+      }
       const metadata = parseCommitMetadata(row.commit_extra_info);
       return {
-        snapshotId: String(getBigIntLike(row, "snapshot_id")),
+        snapshotId,
         createdAt: getDate(row, "snapshot_time"),
         changesMade: getString(row, "changes_made"),
         hasFileChange: getBoolean(row, "has_file_change"),
@@ -4237,6 +4411,9 @@ class DuckLakeStorage {
   async getVersion(datasetId, versionId) {
     return this.snapshotReader.getVersion(datasetId, versionId);
   }
+  async listLatestVersionsSince(input) {
+    return this.snapshotReader.listLatestVersionsSince(input);
+  }
   readRows(input) {
     return this.rows.readRows(input);
   }
@@ -4254,4 +4431,4 @@ export {
   DuckLakeStorage
 };
 
-//# debugId=35ABBEC5A94F36A964756E2164756E21
+//# debugId=99E61447F4C74C9E64756E2164756E21