@sixb/ducklake 0.1.5 → 0.1.6

package/dist/internal/ducklake-change-reader.d.ts

@@ -0,0 +1,13 @@
+import { type DatasetChanges, type ReadDatasetChangesInput } from "@sixb/core/lake-storage";
+import type { DuckLakeStorageOptions } from "../types";
+import type { DuckLakeConnectionManager } from "./ducklake-connection-manager";
+import type { DuckLakeSnapshotReader } from "./ducklake-snapshot-reader";
+/** Compare once in DuckDB, then page only the delta without holding the connection between yields. */
+export declare class DuckLakeChangeReader {
+    private readonly options;
+    private readonly connections;
+    private readonly snapshots;
+    constructor(options: DuckLakeStorageOptions, connections: DuckLakeConnectionManager, snapshots: DuckLakeSnapshotReader);
+    readChanges(input: ReadDatasetChangesInput): Promise<DatasetChanges | null>;
+}
+//# sourceMappingURL=ducklake-change-reader.d.ts.map

package/dist/internal/ducklake-change-reader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ducklake-change-reader.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-change-reader.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,KAAK,cAAc,EAGnB,KAAK,uBAAuB,EAE7B,MAAM,yBAAyB,CAAA;AAChC,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAGtD,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAA;AAE9E,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAOxE,sGAAsG;AACtG,qBAAa,oBAAoB;IAE7B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,SAAS;gBAFT,OAAO,EAAE,sBAAsB,EAC/B,WAAW,EAAE,yBAAyB,EACtC,SAAS,EAAE,sBAAsB;IAG9C,WAAW,CAAC,KAAK,EAAE,uBAAuB,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;CA2FlF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sixb/ducklake",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "description": "DuckDB/DuckLake lake storage provider for Sixb",
   "keywords": [
     "sixb",
@@ -36,7 +36,7 @@
     "typecheck": "tsc --noEmit",
     "test": "bun test",
     "test:e2e": "bun run test:e2e:local && bun run test:e2e:remote",
-    "test:e2e:local": "bun test ./tests/ducklake-contract.e2e.ts ./tests/ducklake-file-refs.e2e.ts ./tests/ducklake-catalog-state.e2e.ts ./tests/ducklake-catalogs.e2e.ts ./tests/ducklake-concurrency.e2e.ts ./tests/ducklake-datasets.e2e.ts ./tests/ducklake-driver.e2e.ts ./tests/ducklake-maintenance.e2e.ts ./tests/ducklake-merges.e2e.ts ./tests/ducklake-sql-transforms.e2e.ts ./tests/ducklake-versions.e2e.ts ./tests/ducklake-writes.e2e.ts",
+    "test:e2e:local": "bun test ./tests/ducklake-changes.e2e.ts ./tests/ducklake-contract.e2e.ts ./tests/ducklake-file-refs.e2e.ts ./tests/ducklake-catalog-state.e2e.ts ./tests/ducklake-catalogs.e2e.ts ./tests/ducklake-concurrency.e2e.ts ./tests/ducklake-datasets.e2e.ts ./tests/ducklake-driver.e2e.ts ./tests/ducklake-maintenance.e2e.ts ./tests/ducklake-merges.e2e.ts ./tests/ducklake-sql-transforms.e2e.ts ./tests/ducklake-versions.e2e.ts ./tests/ducklake-writes.e2e.ts",
     "test:e2e:remote": "bun test --preload ./tests/setup.ts ./tests/ducklake-remote.e2e.ts",
     "prepack": "bun run build"
   },
@@ -44,10 +44,10 @@
     "@duckdb/node-api": "^1.5.2-r.1"
   },
   "peerDependencies": {
-    "@sixb/core": "^0.1.9"
+    "@sixb/core": "^0.1.10"
   },
   "devDependencies": {
-    "@sixb/core": "0.1.9",
+    "@sixb/core": "0.1.10",
     "@types/bun": "latest",
     "typescript": "^5.7.0"
   },

package/src/ducklake-storage.ts

@@ -3,11 +3,14 @@ import type {
   BeginDatasetMergeInput,
   BeginDatasetWriteInput,
   DatasetCatalogState,
+  DatasetChanges,
   DatasetVersion,
   LakeMergeSession,
   LakeWriteSession,
+  ReadDatasetChangesInput,
   ReadDatasetRowsInput,
 } from "@sixb/core/lake-storage"
+import { DuckLakeChangeReader } from "./internal/ducklake-change-reader"
 import { DuckLakeConnectionManager } from "./internal/ducklake-connection-manager"
 import { DuckLakeDatasetCatalog } from "./internal/ducklake-dataset-catalog"
 import { DuckLakeMaintenance } from "./internal/ducklake-maintenance"
@@ -36,6 +39,7 @@ export class DuckLakeStorage implements LakeStorageWithSql<"duckdb"> {
   private readonly datasets: DuckLakeDatasetCatalog
   private readonly maintenance: DuckLakeMaintenance
   private readonly rows: DuckLakeRowReader
+  private readonly changes: DuckLakeChangeReader
   private readonly snapshotReader: DuckLakeSnapshotReader
   private readonly writes: DuckLakeWriteCoordinator
 
@@ -59,6 +63,11 @@ export class DuckLakeStorage implements LakeStorageWithSql<"duckdb"> {
       this.datasets
     )
     this.rows = new DuckLakeRowReader(normalizedOptions, this.connections, this.snapshotReader)
+    this.changes = new DuckLakeChangeReader(
+      normalizedOptions,
+      this.connections,
+      this.snapshotReader
+    )
     this.writes = new DuckLakeWriteCoordinator(
       normalizedOptions,
       this.connections,
@@ -122,6 +131,10 @@ export class DuckLakeStorage implements LakeStorageWithSql<"duckdb"> {
     return this.rows.readRows(input)
   }
 
+  readChanges(input: ReadDatasetChangesInput): Promise<DatasetChanges | null> {
+    return this.changes.readChanges(input)
+  }
+
   async runMaintenance(options?: DuckLakeMaintenanceOptions): Promise<DuckLakeMaintenanceReport> {
     return this.maintenance.runMaintenance(options)
   }

package/src/internal/ducklake-change-reader.ts

@@ -0,0 +1,157 @@
+import { randomUUID } from "node:crypto"
+import type { DatasetColumnDefinition, DatasetRow } from "@sixb/core"
+import {
+  type DatasetChanges,
+  type DatasetRowChange,
+  LakeStorageError,
+  type ReadDatasetChangesInput,
+  resolveDatasetChangeColumns,
+} from "@sixb/core/lake-storage"
+import type { DuckLakeStorageOptions } from "../types"
+import { getBigIntLike } from "./duckdb-row"
+import type { DuckDbQueryRuntime } from "./duckdb-runtime"
+import type { DuckLakeConnectionManager } from "./ducklake-connection-manager"
+import { resolveDatasetTableRef } from "./ducklake-dataset-table-ref"
+import type { DuckLakeSnapshotReader } from "./ducklake-snapshot-reader"
+import { normalizeReadValue } from "./schema"
+import { qualifiedTableName, quoteIdentifier } from "./sql"
+import { parseVersionId } from "./versions"
+
+const PAGE_ROWS = 5_000
+
+/** Compare once in DuckDB, then page only the delta without holding the connection between yields. */
+export class DuckLakeChangeReader {
+  constructor(
+    private readonly options: DuckLakeStorageOptions,
+    private readonly connections: DuckLakeConnectionManager,
+    private readonly snapshots: DuckLakeSnapshotReader
+  ) {}
+
+  async readChanges(input: ReadDatasetChangesInput): Promise<DatasetChanges | null> {
+    input.signal?.throwIfAborted()
+    const before = await this.snapshots.getVersion(input.datasetId, input.fromVersionId)
+    const after = await this.snapshots.getVersion(input.datasetId, input.toVersionId)
+    const columns = resolveDatasetChangeColumns(input, before, after)
+    if (!columns) return null
+    const lease = await this.connections.acquireAttachedRuntime()
+    const runtime = lease.runtime
+    const temporary = quoteIdentifier(`sixb_delta_${randomUUID().replaceAll("-", "")}`)
+    let retained = false
+    let created = false
+    try {
+      const ref = await resolveDatasetTableRef(this.options, runtime, input.datasetId)
+      if (!ref) return null
+      const table = qualifiedTableName(this.options, ref.tableName)
+      const oldRelation = `${table} AT (VERSION => ${parseVersionId(input.fromVersionId)})`
+      const newRelation = `${table} AT (VERSION => ${parseVersionId(input.toVersionId)})`
+      const previousCount = await uniqueRowCount(runtime, oldRelation, input.keyColumns)
+      const nextCount = await uniqueRowCount(runtime, newRelation, input.keyColumns)
+      if (previousCount === null || nextCount === null) return null
+      input.signal?.throwIfAborted()
+      const key = quoteIdentifier(input.keyColumns[0]!)
+      const values = columns.flatMap((column, index) => [
+        `b.${quoteIdentifier(column.name)} AS b${index}`,
+        `a.${quoteIdentifier(column.name)} AS a${index}`,
+      ])
+      const join = input.keyColumns
+        .map((name) => `b.${quoteIdentifier(name)} = a.${quoteIdentifier(name)}`)
+        .join(" AND ")
+      const changed = columns
+        .map(
+          (column) =>
+            `b.${quoteIdentifier(column.name)} IS DISTINCT FROM a.${quoteIdentifier(column.name)}`
+        )
+        .join(" OR ")
+      // The native change feed reflects DELETE/INSERT writes: snapshot ingestion can produce 2N
+      // physical changes for one logical edit. Compare the pinned states instead of replaying it.
+      await runtime.run(`CREATE TEMP TABLE ${temporary} AS
+        SELECT b.${key} IS NOT NULL AS before_present, a.${key} IS NOT NULL AS after_present,
+          ${values.join(", ")}
+        FROM (SELECT * FROM ${oldRelation}) b
+        FULL OUTER JOIN (SELECT * FROM ${newRelation}) a ON ${join}
+        WHERE b.${key} IS NULL OR a.${key} IS NULL OR ${changed}`)
+      created = true
+      const [count] = await runtime.query(`SELECT count(*) AS count FROM ${temporary}`)
+      const changeCount = safeCount(count!, "count")
+      input.signal?.throwIfAborted()
+      let closePromise: Promise<void> | undefined
+      const close = () => {
+        closePromise ??= runtime.run(`DROP TABLE IF EXISTS ${temporary}`)
+        return closePromise
+      }
+      const changes = (async function* (): AsyncIterable<DatasetRowChange> {
+        try {
+          let cursor = -1n
+          let delivered = 0
+          while (delivered < changeCount) {
+            input.signal?.throwIfAborted()
+            if (closePromise) throw new LakeStorageError("[SixbDuckLake] Change reader is closed.")
+            const page = await runtime.query(`SELECT rowid AS position, * FROM ${temporary}
+              WHERE rowid > ${cursor} ORDER BY rowid LIMIT ${PAGE_ROWS}`)
+            if (page.length === 0)
+              throw new LakeStorageError(
+                "[SixbDuckLake] Change read ended before its declared count."
+              )
+            for (const row of page) {
+              input.signal?.throwIfAborted()
+              delivered++
+              yield {
+                before: row.before_present === true ? rowImage(row, "b", columns) : null,
+                after: row.after_present === true ? rowImage(row, "a", columns) : null,
+              }
+            }
+            cursor = getBigIntLike(page[page.length - 1]!, "position")
+          }
+        } finally {
+          await close()
+        }
+      })()
+      retained = true
+      return { fromRowCount: previousCount, toRowCount: nextCount, changeCount, changes, close }
+    } finally {
+      // The delta is a DuckDB temp table, independent of the attached DuckLake catalog. Release
+      // the local attachment lock before returning it, otherwise consumer lake operations deadlock.
+      try {
+        if (!retained && created) await runtime.run(`DROP TABLE IF EXISTS ${temporary}`)
+      } finally {
+        await lease.release()
+      }
+    }
+  }
+}
+
+async function uniqueRowCount(
+  runtime: DuckDbQueryRuntime,
+  relation: string,
+  keys: readonly string[]
+): Promise<number | null> {
+  const names = keys.map(quoteIdentifier)
+  const invalid = names.map((name) => `${name} IS NULL OR trim(${name}) = ''`).join(" OR ")
+  const [row] = await runtime.query(`SELECT count(*) AS total,
+    count(DISTINCT (${names.join(", ")})) AS unique_keys,
+    count(*) FILTER (WHERE ${invalid}) AS invalid_keys FROM ${relation}`)
+  const count = safeCount(row!, "total")
+  return safeCount(row!, "unique_keys") === count && safeCount(row!, "invalid_keys") === 0
+    ? count
+    : null
+}
+
+function safeCount(row: Readonly<Record<string, unknown>>, name: string): number {
+  const count = Number(getBigIntLike(row, name))
+  if (!Number.isSafeInteger(count) || count < 0)
+    throw new LakeStorageError("[SixbDuckLake] Change count exceeds the supported integer range.")
+  return count
+}
+
+function rowImage(
+  row: Readonly<Record<string, unknown>>,
+  prefix: "a" | "b",
+  columns: readonly DatasetColumnDefinition[]
+): DatasetRow {
+  return Object.fromEntries(
+    columns.map((column, index) => [
+      column.name,
+      normalizeReadValue(row[`${prefix}${index}`], column),
+    ])
+  )
+}