@sixb/ducklake 0.1.7 → 0.1.8

package/dist/internal/ducklake-snapshot-reader.d.ts

@@ -1,5 +1,5 @@
 import type { DatasetDefinition } from "@sixb/core";
-import type { DatasetCatalogState, DatasetVersion, DatasetVersionRef } from "@sixb/core/lake-storage";
+import type { DatasetCatalogState, DatasetVersion, DatasetVersionRef, LatestVersionsSince, ListLatestVersionsSinceInput } from "@sixb/core/lake-storage";
 import type { DuckLakeStorageOptions } from "../types";
 import type { DuckDbQueryRuntime } from "./duckdb-runtime";
 import type { DuckLakeConnectionManager } from "./ducklake-connection-manager";
@@ -41,6 +41,21 @@ export declare class DuckLakeSnapshotReader {
      */
     listDatasetCatalogState(datasetIds: readonly string[]): Promise<readonly DatasetCatalogState[]>;
     private collectDatasetCatalogState;
+    /**
+     * Change feed over DuckLake's snapshot table, which is an append-only commit log: snapshot ids
+     * only increase, and a commit becomes visible only after every lower id has (concurrent commits
+     * that pick the same id conflict, and the loser retries with the next one). The cursor is the
+     * last snapshot id read. Every query is bounded by a snapshot-id range, so an idle read is one
+     * indexed query that returns the cursor's own snapshot, whatever the history length.
+     */
+    listLatestVersionsSince(input: ListLatestVersionsSinceInput): Promise<LatestVersionsSince | null>;
+    private queryHeadSnapshotId;
+    /** The cursor's snapshot and up to one feed page of snapshots after it, oldest first. */
+    private querySnapshotsFrom;
+    /** Current dataset tables, with the first snapshot each table id existed in. */
+    private resolveFeedTables;
+    /** Folds a page of snapshots, oldest first, into each dataset's newest data version. */
+    private collectFeedPage;
     private snapshotRowToSummary;
     private resolveDatasetTableIds;
     /**
@@ -67,6 +82,16 @@ export declare class DuckLakeSnapshotReader {
     private getSnapshotRowForTableRef;
     private getSnapshotRowsForTableRef;
     private collectVisibleSnapshotRows;
+    /**
+     * Hydrates, newest first, the snapshots that may hold a version of one dataset table: those
+     * that added or removed its data or delete files, that DuckLake recorded as changing it inline,
+     * or whose Sixb commit metadata names the dataset. The text matches are a superset;
+     * candidateToSnapshotRow decides visibility. Only this table's snapshots are read, so the cost
+     * follows the dataset's own history rather than every commit in the catalog.
+     *
+     * Returns one row per candidate id, including ids whose snapshot no longer exists, so the caller
+     * can tell a short batch from the end of the history.
+     */
     private querySnapshotCandidates;
     private candidateToSnapshotRow;
     private snapshotToVersion;

package/dist/internal/ducklake-snapshot-reader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ducklake-snapshot-reader.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-snapshot-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,KAAK,EACV,mBAAmB,EAEnB,cAAc,EAEd,iBAAiB,EAClB,MAAM,yBAAyB,CAAA;AAEhC,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAEtD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAC1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAA;AAC9E,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACxE,OAAO,EAAE,KAAK,eAAe,EAA0B,MAAM,8BAA8B,CAAA;AA4C3F,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,iBAAiB,EAAE,CAAA;IAC9C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,0BAA0B,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACxD;AAyBD;;;;;;;;;;;GAWG;AACH,qBAAa,sBAAsB;IAE/B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAFR,OAAO,EAAE,sBAAsB,EAC/B,WAAW,EAAE,yBAAyB,EACtC,QAAQ,EAAE,sBAAsB;IAG7C,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC;IAanF,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAanE,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IActF;;;;;;OAMG;IACG,uBAAuB,CAC3B,UAAU,EAAE,SAAS,MAAM,EAAE,GAC5B,OAAO,CAAC,SAAS,mBAAmB,EAAE,CAAC;YAY5B,0BAA0B;IAsBxC,OAAO,CAAC,oBAAoB;YAad,sBAAsB;IAiCpC;;;;;;OAMG;YACW,yBAAyB;YA8DzB,yBAAyB;YAyCzB,oBAAoB;IAuDlC,OAAO,CAAC,wBAAwB;IAY1B,6BAA6B,CACjC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAK3B,gCAAgC,CACpC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAK9B,oCAAoC,CACxC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC;IAsBnC,qBAAqB,CACzB,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,EAC1B,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAO3B,qBAAqB,CACzB,OAAO,EAAE,kBAAkB,EAC3B,QAAQ,EAAE,eAAe,EACzB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAO3B,wBAAwB,CAC5B,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,EAC1B,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAc9B,yBAAyB,CAC7B,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,EAC1B,KAAK,CAAC,EAAE,MAAM,GACb,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC;YAKvB,uBAAuB;IAe/B,2BAA2B,CAC/B,OAAO,EAAE,kBAAkB,EAC3B,QAAQ,EAAE,eAAe,GACxB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;YAKnB,+BAA+B;YAY/B,yBAAyB;YAmCzB,0BAA0B;YAmB1B,0BAA0B;YA2C1B,uBAAuB;IA2GrC,OAAO,CAAC,sBAAsB;YAyChB,iBAAiB;IAgC/B,OAAO,CAAC,qBAAqB;CAa9B"}
+{"version":3,"file":"ducklake-snapshot-reader.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-snapshot-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,KAAK,EACV,mBAAmB,EAEnB,cAAc,EAEd,iBAAiB,EACjB,mBAAmB,EACnB,4BAA4B,EAC7B,MAAM,yBAAyB,CAAA;AAEhC,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAEtD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAC1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAA;AAC9E,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACxE,OAAO,EAAE,KAAK,eAAe,EAA0B,MAAM,8BAA8B,CAAA;AAyD3F,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,iBAAiB,EAAE,CAAA;IAC9C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,0BAA0B,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACxD;AA2BD;;;;;;;;;;;GAWG;AACH,qBAAa,sBAAsB;IAE/B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAFR,OAAO,EAAE,sBAAsB,EAC/B,WAAW,EAAE,yBAAyB,EACtC,QAAQ,EAAE,sBAAsB;IAG7C,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC;IAanF,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAanE,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IActF;;;;;;OAMG;IACG,uBAAuB,CAC3B,UAAU,EAAE,SAAS,MAAM,EAAE,GAC5B,OAAO,CAAC,SAAS,mBAAmB,EAAE,CAAC;YAY5B,0BAA0B;IAsBxC;;;;;;OAMG;IACG,uBAAuB,CAC3B,KAAK,EAAE,4BAA4B,GAClC,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC;YA8BxB,mBAAmB;IAajC,yFAAyF;YAC3E,kBAAkB;IAsBhC,gFAAgF;YAClE,iBAAiB;IAkC/B,wFAAwF;YAC1E,eAAe;IA2E7B,OAAO,CAAC,oBAAoB;YAad,sBAAsB;IAiCpC;;;;;;OAMG;YACW,yBAAyB;YA8DzB,yBAAyB;YAyCzB,oBAAoB;IAuDlC,OAAO,CAAC,wBAAwB;IAY1B,6BAA6B,CACjC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAK3B,gCAAgC,CACpC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAK9B,oCAAoC,CACxC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC;IAsBnC,qBAAqB,CACzB,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,EAC1B,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAO3B,qBAAqB,CACzB,OAAO,EAAE,kBAAkB,EAC3B,QAAQ,EAAE,eAAe,EACzB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAO3B,wBAAwB,CAC5B,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,EAC1B,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAc9B,yBAAyB,CAC7B,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,EAC1B,KAAK,CAAC,EAAE,MAAM,GACb,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC;YAKvB,uBAAuB;IAe/B,2BAA2B,CAC/B,OAAO,EAAE,kBAAkB,EAC3B,QAAQ,EAAE,eAAe,GACxB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;YAKnB,+BAA+B;YAY/B,yBAAyB;YAmCzB,0BAA0B;YAmB1B,0BAA0B;IA+CxC;;;;;;;;;OASG;YACW,uBAAuB;IAyIrC,OAAO,CAAC,sBAAsB;YAyChB,iBAAiB;IAgC/B,OAAO,CAAC,qBAAqB;CAa9B"}

package/dist/internal/sql.d.ts

@@ -19,10 +19,19 @@ export declare function buildSetPostgresPoolSql(options: Pick<DuckLakeStorageOpt
  * Render a fully-qualified DuckLake metadata table path.
  *
  * PostgreSQL catalogs expose DuckLake metadata tables under the configured
- * metadata schema inside the attached metadata catalog. Local catalogs use
- * DuckDB's default `main` schema.
+ * metadata schema (PostgreSQL's `public` by default) inside the attached
+ * metadata catalog. Local catalogs use DuckDB's default `main` schema.
  */
 export declare function duckLakeMetadataTableName(options: Pick<DuckLakeStorageOptions, "alias" | "catalog">, tableName: string): string;
+/**
+ * Render a metadata query that runs where the catalog lives.
+ *
+ * DuckDB's PostgreSQL scanner pushes comparisons and constant IN lists down to PostgreSQL, but not
+ * string matching, so a query that filters on text would copy the whole column into DuckDB. On a
+ * PostgreSQL catalog the query therefore runs server-side through `postgres_query`; elsewhere the
+ * catalog is local and DuckDB runs it. `build` must produce SQL that both dialects accept.
+ */
+export declare function buildDuckLakeMetadataQuery(options: Pick<DuckLakeStorageOptions, "alias" | "catalog">, build: (table: (tableName: string) => string) => string): string;
 /**
  * Quote a SQL identifier after validating it is already identifier-shaped.
  *

package/dist/internal/sql.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sql.d.ts","sourceRoot":"","sources":["../../src/internal/sql.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAEV,mBAAmB,EACnB,sBAAsB,EACtB,sBAAsB,EAEtB,8BAA8B,EAG/B,MAAM,UAAU,CAAA;AAIjB;;GAEG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,sBAAsB,EAAE,OAAO,CAAC,GAAG,MAAM,CAEpF;AAUD;;;;;GAKG;AACH,wBAAgB,qCAAqC,CACnD,OAAO,EAAE,IAAI,CAAC,sBAAsB,EAAE,OAAO,GAAG,cAAc,CAAC,GAC9D,MAAM,CAOR;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,IAAI,CAAC,sBAAsB,EAAE,cAAc,CAAC,GACpD,SAAS,MAAM,EAAE,CAInB;AAED;;;;;;GAMG;AACH,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,IAAI,CAAC,sBAAsB,EAAE,OAAO,GAAG,SAAS,CAAC,EAC1D,SAAS,EAAE,MAAM,GAChB,MAAM,CAOR;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAM1D;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEpD;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAEpE;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,IAAI,CAAC,sBAAsB,EAAE,OAAO,CAAC,EAC9C,SAAS,EAAE,MAAM,GAChB,MAAM,CAER;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,sBAAsB,GAAG,MAAM,CAWlE;AAED;;GAEG;AACH,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,8BAA8B,GAAG,MAAM,CAuBxF;AAkBD;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,sBAAsB,GAAG,SAAS,MAAM,EAAE,CAcrF;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,mBAAmB,GAAG,MAAM,CAIxE;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,sBAAsB,GAAG,MAAM,CA+BtE"}
+{"version":3,"file":"sql.d.ts","sourceRoot":"","sources":["../../src/internal/sql.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAEV,mBAAmB,EACnB,sBAAsB,EACtB,sBAAsB,EAEtB,8BAA8B,EAG/B,MAAM,UAAU,CAAA;AAIjB;;GAEG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,sBAAsB,EAAE,OAAO,CAAC,GAAG,MAAM,CAEpF;AAUD;;;;;GAKG;AACH,wBAAgB,qCAAqC,CACnD,OAAO,EAAE,IAAI,CAAC,sBAAsB,EAAE,OAAO,GAAG,cAAc,CAAC,GAC9D,MAAM,CAOR;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,IAAI,CAAC,sBAAsB,EAAE,cAAc,CAAC,GACpD,SAAS,MAAM,EAAE,CAInB;AAED;;;;;;GAMG;AACH,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,IAAI,CAAC,sBAAsB,EAAE,OAAO,GAAG,SAAS,CAAC,EAC1D,SAAS,EAAE,MAAM,GAChB,MAAM,CAIR;AAED;;;;;;;GAOG;AACH,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,IAAI,CAAC,sBAAsB,EAAE,OAAO,GAAG,SAAS,CAAC,EAC1D,KAAK,EAAE,CAAC,KAAK,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,MAAM,KAAK,MAAM,GACtD,MAAM,CAaR;AAQD;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAM1D;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEpD;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAEpE;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,IAAI,CAAC,sBAAsB,EAAE,OAAO,CAAC,EAC9C,SAAS,EAAE,MAAM,GAChB,MAAM,CAER;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,sBAAsB,GAAG,MAAM,CAWlE;AAED;;GAEG;AACH,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,8BAA8B,GAAG,MAAM,CAuBxF;AAkBD;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,sBAAsB,GAAG,SAAS,MAAM,EAAE,CAcrF;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,mBAAmB,GAAG,MAAM,CAIxE;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,sBAAsB,GAAG,MAAM,CA+BtE"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sixb/ducklake",
-  "version": "0.1.7",
+  "version": "0.1.8",
   "description": "DuckDB/DuckLake lake storage provider for Sixb",
   "keywords": [
     "sixb",
@@ -36,7 +36,7 @@
     "typecheck": "tsc --noEmit",
     "test": "bun test",
     "test:e2e": "bun run test:e2e:local && bun run test:e2e:remote",
-    "test:e2e:local": "bun test ./tests/ducklake-reader.e2e.ts ./tests/ducklake-changes.e2e.ts ./tests/ducklake-contract.e2e.ts ./tests/ducklake-file-refs.e2e.ts ./tests/ducklake-catalog-state.e2e.ts ./tests/ducklake-catalogs.e2e.ts ./tests/ducklake-concurrency.e2e.ts ./tests/ducklake-datasets.e2e.ts ./tests/ducklake-driver.e2e.ts ./tests/ducklake-maintenance.e2e.ts ./tests/ducklake-merges.e2e.ts ./tests/ducklake-sql-transforms.e2e.ts ./tests/ducklake-versions.e2e.ts ./tests/ducklake-writes.e2e.ts ./tests/ducklake-sqlite-writes.e2e.ts",
+    "test:e2e:local": "bun test ./tests/ducklake-reader.e2e.ts ./tests/ducklake-changes.e2e.ts ./tests/ducklake-contract.e2e.ts ./tests/ducklake-file-refs.e2e.ts ./tests/ducklake-catalog-state.e2e.ts ./tests/ducklake-catalogs.e2e.ts ./tests/ducklake-concurrency.e2e.ts ./tests/ducklake-datasets.e2e.ts ./tests/ducklake-driver.e2e.ts ./tests/ducklake-maintenance.e2e.ts ./tests/ducklake-merges.e2e.ts ./tests/ducklake-sql-transforms.e2e.ts ./tests/ducklake-versions.e2e.ts ./tests/ducklake-long-history.e2e.ts ./tests/ducklake-writes.e2e.ts ./tests/ducklake-sqlite-writes.e2e.ts",
     "test:e2e:remote": "bun test --preload ./tests/setup.ts ./tests/ducklake-remote.e2e.ts",
     "prepack": "bun run build"
   },
@@ -44,10 +44,10 @@
     "@duckdb/node-api": "^1.5.2-r.1"
   },
   "peerDependencies": {
-    "@sixb/core": "^0.1.11"
+    "@sixb/core": "^0.1.17"
   },
   "devDependencies": {
-    "@sixb/core": "0.1.11",
+    "@sixb/core": "0.1.17",
     "@types/bun": "latest",
     "typescript": "^5.7.0"
   },

package/src/ducklake-storage.ts

@@ -7,6 +7,8 @@ import type {
   DatasetVersion,
   LakeMergeSession,
   LakeWriteSession,
+  LatestVersionsSince,
+  ListLatestVersionsSinceInput,
   ReadDatasetChangesInput,
   ReadDatasetRowsInput,
 } from "@sixb/core/lake-storage"
@@ -127,6 +129,12 @@ export class DuckLakeStorage implements LakeStorageWithSql<"duckdb"> {
     return this.snapshotReader.getVersion(datasetId, versionId)
   }
 
+  async listLatestVersionsSince(
+    input: ListLatestVersionsSinceInput
+  ): Promise<LatestVersionsSince | null> {
+    return this.snapshotReader.listLatestVersionsSince(input)
+  }
+
   readRows(input: ReadDatasetRowsInput): AsyncIterable<DatasetRow> {
     return this.rows.readRows(input)
   }

package/src/internal/ducklake-snapshot-reader.ts

@@ -5,6 +5,8 @@ import type {
   DatasetVersion,
   DatasetVersionMode,
   DatasetVersionRef,
+  LatestVersionsSince,
+  ListLatestVersionsSinceInput,
 } from "@sixb/core/lake-storage"
 import { LakeStorageError } from "@sixb/core/lake-storage"
 import type { DuckLakeStorageOptions } from "../types"
@@ -13,8 +15,8 @@ import type { DuckDbQueryRuntime } from "./duckdb-runtime"
 import type { DuckLakeConnectionManager } from "./ducklake-connection-manager"
 import type { DuckLakeDatasetCatalog } from "./ducklake-dataset-catalog"
 import { type DatasetTableRef, resolveDatasetTableRef } from "./ducklake-dataset-table-ref"
-import { encodeDatasetTableName } from "./names"
-import { duckLakeMetadataTableName, quoteSqlString } from "./sql"
+import { decodeDatasetTableName, encodeDatasetTableName } from "./names"
+import { buildDuckLakeMetadataQuery, duckLakeMetadataTableName, quoteSqlString } from "./sql"
 import {
   parseCommitMetadata,
   parseInlineDataChange,
@@ -41,11 +43,24 @@ interface DatasetSnapshotCandidateRow {
   readonly metadata?: SixbCommitMetadata
 }
 
+interface FeedTable {
+  readonly datasetId: string
+  readonly tableId: bigint
+  /** A dataset cannot have a version before its table exists. */
+  readonly startSnapshotId: bigint
+}
+
+interface MissingSnapshotCandidate {
+  readonly snapshotId: string
+  readonly missing: true
+}
+
 interface SnapshotCandidateQueryInput {
+  readonly datasetId: string
   readonly tableId: bigint
   readonly exactSnapshotId?: string
   readonly beforeSnapshotId?: string
-  readonly limit?: number
+  readonly limit: number
 }
 
 interface VisibleSnapshotRowsInput {
@@ -79,6 +94,8 @@ interface FileChangeFlags {
 }
 
 const SNAPSHOT_ROW_BATCH_SIZE = 128
+// One feed read covers at most this many commits; the next read continues from its cursor.
+const FEED_PAGE_SIZE = 512
 
 // The bulk catalog scan shares one descending walk of recent snapshots across
 // all requested datasets, so its cost is bounded by this window rather than by
@@ -189,6 +206,192 @@ export class DuckLakeSnapshotReader {
     })
   }
 
+  /**
+   * Change feed over DuckLake's snapshot table, which is an append-only commit log: snapshot ids
+   * only increase, and a commit becomes visible only after every lower id has (concurrent commits
+   * that pick the same id conflict, and the loser retries with the next one). The cursor is the
+   * last snapshot id read. Every query is bounded by a snapshot-id range, so an idle read is one
+   * indexed query that returns the cursor's own snapshot, whatever the history length.
+   */
+  async listLatestVersionsSince(
+    input: ListLatestVersionsSinceInput
+  ): Promise<LatestVersionsSince | null> {
+    this.connections.assertOpen()
+
+    return this.connections.withAttachedRuntime(async (runtime) => {
+      if (input.cursor === null) {
+        return { cursor: await this.queryHeadSnapshotId(runtime), versions: [] }
+      }
+      if (!/^\d+$/.test(input.cursor)) {
+        return null
+      }
+
+      const snapshots = await this.querySnapshotsFrom(runtime, input.cursor)
+      // The cursor's own snapshot proves the cursor is still a position in this lake's history.
+      if (snapshots[0]?.snapshotId !== input.cursor) {
+        return null
+      }
+      const newSnapshots = snapshots.slice(1)
+      if (newSnapshots.length === 0) {
+        return { cursor: input.cursor, versions: [] }
+      }
+
+      const tables = await this.resolveFeedTables(runtime, input.datasetIds)
+      const latest = await this.collectFeedPage(runtime, tables, newSnapshots)
+      return {
+        cursor: newSnapshots[newSnapshots.length - 1]?.snapshotId ?? input.cursor,
+        versions: [...latest].map(([datasetId, row]) => this.snapshotRowToSummary(datasetId, row)),
+      }
+    })
+  }
+
+  private async queryHeadSnapshotId(runtime: DuckDbQueryRuntime): Promise<string> {
+    const [row] = await runtime.query(
+      buildDuckLakeMetadataQuery(
+        this.options,
+        (table) => `SELECT max(snapshot_id) AS snapshot_id FROM ${table("ducklake_snapshot")}`
+      )
+    )
+    if (row?.snapshot_id === null || row?.snapshot_id === undefined) {
+      throw new LakeStorageError("[SixbDuckLake] DuckLake catalog has no snapshots.")
+    }
+    return String(getBigIntLike(row, "snapshot_id"))
+  }
+
+  /** The cursor's snapshot and up to one feed page of snapshots after it, oldest first. */
+  private async querySnapshotsFrom(
+    runtime: DuckDbQueryRuntime,
+    cursor: string
+  ): Promise<readonly { readonly snapshotId: string; readonly createdAt: Date }[]> {
+    const rows = await runtime.query(
+      buildDuckLakeMetadataQuery(
+        this.options,
+        (table) => `
+          SELECT snapshot_id, snapshot_time
+          FROM ${table("ducklake_snapshot")}
+          WHERE snapshot_id >= ${cursor}
+          ORDER BY snapshot_id
+          LIMIT ${FEED_PAGE_SIZE + 1}
+        `
+      )
+    )
+    return rows.map((row) => ({
+      snapshotId: String(getBigIntLike(row, "snapshot_id")),
+      createdAt: getDate(row, "snapshot_time"),
+    }))
+  }
+
+  /** Current dataset tables, with the first snapshot each table id existed in. */
+  private async resolveFeedTables(
+    runtime: DuckDbQueryRuntime,
+    datasetIds: readonly string[] | undefined
+  ): Promise<readonly FeedTable[]> {
+    const requested = datasetIds === undefined ? null : new Set(datasetIds)
+    const rows = await runtime.query(`
+      SELECT table_id, table_name, begin_snapshot, end_snapshot
+      FROM ${duckLakeMetadataTableName(this.options, "ducklake_table")}
+    `)
+
+    const startByTableId = new Map<bigint, bigint>()
+    const current: { datasetId: string; tableId: bigint }[] = []
+    for (const row of rows) {
+      const tableId = getBigIntLike(row, "table_id")
+      const begin = getBigIntLike(row, "begin_snapshot")
+      const start = startByTableId.get(tableId)
+      if (start === undefined || begin < start) {
+        startByTableId.set(tableId, begin)
+      }
+      if (row.end_snapshot !== null && row.end_snapshot !== undefined) {
+        continue
+      }
+      const datasetId = decodeDatasetTableName(getString(row, "table_name"))
+      if (datasetId !== null && (requested === null || requested.has(datasetId))) {
+        current.push({ datasetId, tableId })
+      }
+    }
+
+    return current.map((table) => ({
+      ...table,
+      startSnapshotId: startByTableId.get(table.tableId) ?? 0n,
+    }))
+  }
+
+  /** Folds a page of snapshots, oldest first, into each dataset's newest data version. */
+  private async collectFeedPage(
+    runtime: DuckDbQueryRuntime,
+    tables: readonly FeedTable[],
+    snapshots: readonly { readonly snapshotId: string; readonly createdAt: Date }[]
+  ): Promise<Map<string, DatasetSnapshotRow>> {
+    const latest = new Map<string, DatasetSnapshotRow>()
+    const first = snapshots[0]?.snapshotId
+    const last = snapshots[snapshots.length - 1]?.snapshotId
+    if (first === undefined || last === undefined || tables.length === 0) {
+      return latest
+    }
+
+    const changeRows = await runtime.query(
+      buildDuckLakeMetadataQuery(
+        this.options,
+        (table) => `
+          SELECT snapshot_id, changes_made, commit_extra_info
+          FROM ${table("ducklake_snapshot_changes")}
+          WHERE snapshot_id >= ${first} AND snapshot_id <= ${last}
+        `
+      )
+    )
+    const fileFlags = await this.queryFileChangeFlags(
+      runtime,
+      snapshots.map((snapshot) => snapshot.snapshotId),
+      tables.map((table) => table.tableId)
+    )
+    const changesBySnapshotId = new Map(
+      changeRows.map((row) => [String(getBigIntLike(row, "snapshot_id")), row])
+    )
+    const tablesByTableId = new Map(tables.map((table) => [table.tableId.toString(), table]))
+    const tablesByDatasetId = new Map(tables.map((table) => [table.datasetId, table]))
+
+    for (const snapshot of snapshots) {
+      const changes = changesBySnapshotId.get(snapshot.snapshotId)
+      if (changes === undefined) {
+        continue
+      }
+      const changesMade = getString(changes, "changes_made")
+      const metadata = parseCommitMetadata(changes.commit_extra_info)
+
+      // Only tables this snapshot names, touches through files, or tags can change visibility.
+      const touched = new Set<FeedTable>()
+      for (const change of changesMade.split(",")) {
+        const table = tablesByTableId.get(change.split(":")[1] ?? "")
+        if (table) touched.add(table)
+      }
+      for (const table of tables) {
+        if (fileFlags.has(fileChangeKey(snapshot.snapshotId, table.tableId))) touched.add(table)
+      }
+      const tagged = metadata === undefined ? undefined : tablesByDatasetId.get(metadata.datasetId)
+      if (tagged) touched.add(tagged)
+
+      for (const table of touched) {
+        if (BigInt(snapshot.snapshotId) < table.startSnapshotId) {
+          continue
+        }
+        const flags = fileFlags.get(fileChangeKey(snapshot.snapshotId, table.tableId))
+        const row = this.candidateToSnapshotRow(table.datasetId, table.tableId, {
+          snapshotId: snapshot.snapshotId,
+          createdAt: snapshot.createdAt,
+          changesMade,
+          hasFileChange: flags?.hasFileChange ?? false,
+          hasFileDeleteChange: flags?.hasFileDeleteChange ?? false,
+          ...(metadata !== undefined ? { metadata } : {}),
+        })
+        if (row && row.mode !== "schema") {
+          latest.set(table.datasetId, row)
+        }
+      }
+    }
+
+    return latest
+  }
+
   private snapshotRowToSummary(
     datasetId: string,
     row: DatasetSnapshotRow
@@ -604,6 +807,7 @@ export class DuckLakeSnapshotReader {
 
     while (input.visibleRowLimit === undefined || snapshots.length < input.visibleRowLimit) {
       const candidates = await this.querySnapshotCandidates(runtime, {
+        datasetId: input.datasetId,
         tableId: input.tableId,
         exactSnapshotId: input.exactSnapshotId,
         beforeSnapshotId,
@@ -614,6 +818,9 @@ export class DuckLakeSnapshotReader {
       }
 
       for (const candidate of candidates) {
+        if ("missing" in candidate) {
+          continue
+        }
         const snapshot = this.candidateToSnapshotRow(input.datasetId, input.tableId, candidate)
         if (snapshot) {
           snapshots.push(snapshot)
@@ -634,104 +841,144 @@ export class DuckLakeSnapshotReader {
     return snapshots
   }
 
+  /**
+   * Hydrates, newest first, the snapshots that may hold a version of one dataset table: those
+   * that added or removed its data or delete files, that DuckLake recorded as changing it inline,
+   * or whose Sixb commit metadata names the dataset. The text matches are a superset;
+   * candidateToSnapshotRow decides visibility. Only this table's snapshots are read, so the cost
+   * follows the dataset's own history rather than every commit in the catalog.
+   *
+   * Returns one row per candidate id, including ids whose snapshot no longer exists, so the caller
+   * can tell a short batch from the end of the history.
+   */
   private async querySnapshotCandidates(
     runtime: DuckDbQueryRuntime,
     input: SnapshotCandidateQueryInput
-  ): Promise<readonly DatasetSnapshotCandidateRow[]> {
-    if (input.limit !== undefined && input.limit <= 0) {
+  ): Promise<readonly (DatasetSnapshotCandidateRow | MissingSnapshotCandidate)[]> {
+    const tableId = input.tableId.toString()
+    const limit = Math.max(0, Math.trunc(input.limit))
+    if (limit === 0) {
       return []
     }
-
-    // A dataset cannot have a version before its table exists. Use the earliest table record:
-    // a rename may create a newer record with the same table id and must not hide older versions.
-    const ducklakeTable = duckLakeMetadataTableName(this.options, "ducklake_table")
-    const where = [
-      `snapshot.snapshot_id >= (SELECT min(begin_snapshot) FROM ${ducklakeTable} WHERE table_id = ${input.tableId})`,
-    ]
     if (input.exactSnapshotId !== undefined) {
       assertDuckLakeSnapshotId(input.exactSnapshotId)
-      where.push(`snapshot.snapshot_id = ${input.exactSnapshotId}`)
     }
     if (input.beforeSnapshotId !== undefined) {
       assertDuckLakeSnapshotId(input.beforeSnapshotId)
-      where.push(`snapshot.snapshot_id < ${input.beforeSnapshotId}`)
     }
 
-    const whereSql = `WHERE ${where.join(" AND ")}`
-    const limitSql =
-      input.limit === undefined ? "" : `LIMIT ${Math.max(0, Math.trunc(input.limit))}`
-    const ducklakeSnapshot = duckLakeMetadataTableName(this.options, "ducklake_snapshot")
-    const ducklakeSnapshotChanges = duckLakeMetadataTableName(
-      this.options,
-      "ducklake_snapshot_changes"
+    const rows = await runtime.query(
+      buildDuckLakeMetadataQuery(this.options, (table) => {
+        // A dataset cannot have a version before its table exists. Use the earliest table record:
+        // a rename may create a newer record with the same table id and must not hide older
+        // versions.
+        const tableStart = `(SELECT min(begin_snapshot) FROM ${table("ducklake_table")} WHERE table_id = ${tableId})`
+        const candidateIds =
+          input.exactSnapshotId !== undefined
+            ? `SELECT CAST(${input.exactSnapshotId} AS BIGINT) AS snapshot_id
+               WHERE ${input.exactSnapshotId} >= ${tableStart}`
+            : `SELECT DISTINCT snapshot_id
+               FROM (
+                 SELECT begin_snapshot AS snapshot_id
+                 FROM ${table("ducklake_data_file")} WHERE table_id = ${tableId}
+                 UNION ALL
+                 SELECT end_snapshot
+                 FROM ${table("ducklake_data_file")}
+                 WHERE table_id = ${tableId} AND end_snapshot IS NOT NULL
+                 UNION ALL
+                 SELECT begin_snapshot
+                 FROM ${table("ducklake_delete_file")} WHERE table_id = ${tableId}
+                 UNION ALL
+                 SELECT end_snapshot
+                 FROM ${table("ducklake_delete_file")}
+                 WHERE table_id = ${tableId} AND end_snapshot IS NOT NULL
+                 UNION ALL
+                 SELECT snapshot_id
+                 FROM ${table("ducklake_snapshot_changes")}
+                 WHERE strpos(',' || changes_made || ',', ${quoteSqlString(`:${tableId},`)}) > 0
+                   OR strpos(
+                     commit_extra_info,
+                     ${quoteSqlString(`"datasetId":${JSON.stringify(input.datasetId)}`)}
+                   ) > 0
+               ) touched
+               WHERE snapshot_id >= ${tableStart}
+                 ${input.beforeSnapshotId === undefined ? "" : `AND snapshot_id < ${input.beforeSnapshotId}`}
+               ORDER BY snapshot_id DESC
+               LIMIT ${limit}`
+
+        return `
+          WITH candidate_ids AS (
+            ${candidateIds}
+          ),
+          candidate_snapshots AS (
+            SELECT
+              snapshot.snapshot_id,
+              snapshot.snapshot_time,
+              changes.changes_made,
+              changes.commit_extra_info
+            FROM ${table("ducklake_snapshot")} snapshot
+            JOIN ${table("ducklake_snapshot_changes")} changes
+              ON changes.snapshot_id = snapshot.snapshot_id
+            WHERE snapshot.snapshot_id IN (SELECT snapshot_id FROM candidate_ids)
+          ),
+          file_changes AS (
+            -- File metadata tells whether this table changed in each candidate snapshot.
+            SELECT begin_snapshot AS snapshot_id, false AS is_delete_change
+            FROM ${table("ducklake_data_file")}
+            WHERE table_id = ${tableId}
+              AND begin_snapshot IN (SELECT snapshot_id FROM candidate_ids)
+            UNION ALL
+            SELECT end_snapshot AS snapshot_id, true AS is_delete_change
+            FROM ${table("ducklake_data_file")}
+            WHERE table_id = ${tableId}
+              AND end_snapshot IN (SELECT snapshot_id FROM candidate_ids)
+            UNION ALL
+            SELECT begin_snapshot AS snapshot_id, true AS is_delete_change
+            FROM ${table("ducklake_delete_file")}
+            WHERE table_id = ${tableId}
+              AND begin_snapshot IN (SELECT snapshot_id FROM candidate_ids)
+            UNION ALL
+            SELECT end_snapshot AS snapshot_id, true AS is_delete_change
+            FROM ${table("ducklake_delete_file")}
+            WHERE table_id = ${tableId}
+              AND end_snapshot IN (SELECT snapshot_id FROM candidate_ids)
+          ),
+          file_changes_by_snapshot AS (
+            -- Collapse file-level changes into one row per snapshot.
+            SELECT
+              snapshot_id,
+              count(*) > 0 AS has_file_change,
+              count(*) FILTER (WHERE is_delete_change) > 0 AS has_file_delete_change
+            FROM file_changes
+            GROUP BY snapshot_id
+          )
+          SELECT
+            -- Keep metadata-only candidates; Sixb filters them with commit_extra_info.
+            candidate_ids.snapshot_id,
+            candidate.snapshot_time,
+            candidate.changes_made,
+            candidate.commit_extra_info,
+            coalesce(file_changes.has_file_change, false) AS has_file_change,
+            coalesce(file_changes.has_file_delete_change, false) AS has_file_delete_change
+          FROM candidate_ids
+          LEFT JOIN candidate_snapshots candidate
+            ON candidate.snapshot_id = candidate_ids.snapshot_id
+          LEFT JOIN file_changes_by_snapshot file_changes
+            ON file_changes.snapshot_id = candidate_ids.snapshot_id
+          ORDER BY candidate_ids.snapshot_id DESC
+        `
+      })
     )
-    const ducklakeDataFile = duckLakeMetadataTableName(this.options, "ducklake_data_file")
-    const ducklakeDeleteFile = duckLakeMetadataTableName(this.options, "ducklake_delete_file")
-
-    const rows = await runtime.query(`
-      WITH candidate_snapshots AS (
-        -- Recent DuckLake snapshots to inspect; Sixb visibility is filtered later.
-        SELECT
-          snapshot.snapshot_id,
-          snapshot.snapshot_time,
-          changes.changes_made,
-          changes.commit_extra_info
-        FROM ${ducklakeSnapshot} snapshot
-        JOIN ${ducklakeSnapshotChanges} changes
-          ON changes.snapshot_id = snapshot.snapshot_id
-        ${whereSql}
-        ORDER BY snapshot.snapshot_id DESC
-        ${limitSql}
-      ),
-      file_changes AS (
-        -- File metadata tells whether this table changed in each candidate snapshot.
-        SELECT begin_snapshot AS snapshot_id, false AS is_delete_change
-        FROM ${ducklakeDataFile}
-        WHERE table_id = ${input.tableId}
-          AND begin_snapshot IN (SELECT snapshot_id FROM candidate_snapshots)
-        UNION ALL
-        SELECT end_snapshot AS snapshot_id, true AS is_delete_change
-        FROM ${ducklakeDataFile}
-        WHERE table_id = ${input.tableId}
-          AND end_snapshot IN (SELECT snapshot_id FROM candidate_snapshots)
-        UNION ALL
-        SELECT begin_snapshot AS snapshot_id, true AS is_delete_change
-        FROM ${ducklakeDeleteFile}
-        WHERE table_id = ${input.tableId}
-          AND begin_snapshot IN (SELECT snapshot_id FROM candidate_snapshots)
-        UNION ALL
-        SELECT end_snapshot AS snapshot_id, true AS is_delete_change
-        FROM ${ducklakeDeleteFile}
-        WHERE table_id = ${input.tableId}
-          AND end_snapshot IN (SELECT snapshot_id FROM candidate_snapshots)
-      ),
-      file_changes_by_snapshot AS (
-        -- Collapse file-level changes into one row per snapshot.
-        SELECT
-          snapshot_id,
-          count(*) > 0 AS has_file_change,
-          count(*) FILTER (WHERE is_delete_change) > 0 AS has_file_delete_change
-        FROM file_changes
-        GROUP BY snapshot_id
-      )
-      SELECT
-        -- Keep metadata-only candidates; Sixb filters them with commit_extra_info.
-        candidate.snapshot_id,
-        candidate.snapshot_time,
-        candidate.changes_made,
-        candidate.commit_extra_info,
-        coalesce(file_changes.has_file_change, false) AS has_file_change,
-        coalesce(file_changes.has_file_delete_change, false) AS has_file_delete_change
-      FROM candidate_snapshots candidate
-      LEFT JOIN file_changes_by_snapshot file_changes
-        ON file_changes.snapshot_id = candidate.snapshot_id
-      ORDER BY candidate.snapshot_id DESC
-    `)
 
     return rows.map((row) => {
+      const snapshotId = String(getBigIntLike(row, "snapshot_id"))
+      // Expired snapshots keep their file records; they are no longer versions.
+      if (row.snapshot_time === null || row.snapshot_time === undefined) {
+        return { snapshotId, missing: true } as const
+      }
       const metadata = parseCommitMetadata(row.commit_extra_info)
       return {
-        snapshotId: String(getBigIntLike(row, "snapshot_id")),
+        snapshotId,
         createdAt: getDate(row, "snapshot_time"),
         changesMade: getString(row, "changes_made"),
         hasFileChange: getBoolean(row, "has_file_change"),