@sixb/ducklake 0.2.0 → 0.2.1

package/README.md

@@ -102,6 +102,10 @@ budget and PostgreSQL metadata pool. Metadata and writes remain available betwee
 Each read stays pinned to one immutable snapshot. Maintenance waits until no reads remain; it
 blocks new reads only while running, so continuous traffic can postpone it.
 
+`readChanges` compares two versions in DuckDB. When no version between them changed rows
+(`rowsChanged` in the commit metadata, read from at most 32 snapshots), it returns an empty change
+set without reading the table.
+
 `maxStreamingReads` limits native streaming readers per provider (default: `4`); extra reads use
 bounded pages on the main connection. It does not limit pipeline inputs. Raise it when memory and
 catalog capacity allow. Finish the iterator, use `for await` with `break`, or pass `signal` to cancel
@@ -190,8 +194,10 @@ Datasets declaring `sequenceBy` instead compare source revisions: older changes
 identical ties are unchanged, and conflicting ties abort the merge. Deletes require
 `change.delete(key, { sequence })`. Source revisions and deletion tombstones are stored in a
 companion DuckLake table in the same transaction as visible rows, surviving snapshot expiration.
-New tombstone state creates a version even when the visible row count stays zero. Sequenced
-provider writes use merge sessions; snapshot syncs convert rows to ordered upserts.
+New tombstone state creates a version even when the visible rows stay the same; such a version
+records `rowsChanged: false`. So does a newer upsert that differs from the stored row only in its
+`sequenceBy` column: the row is kept and only its source revision advances. Sequenced provider
+writes use merge sessions; snapshot syncs convert rows to ordered upserts.
 Snapshot syncs use `commit({ createInitialVersion: true })` to make a first empty result
 addressable. Later no-op commits reuse the current version, including after a concurrent write.
 Use `commit({ retryOnConflict: true })` to rebase retained changes, with at most 3 attempts.

package/dist/index.js

@@ -647,6 +647,7 @@ function parseCommitMetadata(value) {
   const producer = parseDatasetProducer(parsed.sixb.producer);
   const rowCount = parsed.sixb.rowCount;
   const validatedPrimaryKeyColumns = parsed.sixb.validatedPrimaryKeyColumns;
+  const { rowsChanged, lineageChanged } = parsed.sixb;
   return {
     kind: "datasetVersion",
     datasetId: parsed.sixb.datasetId,
@@ -657,7 +658,9 @@ function parseCommitMetadata(value) {
     ...typeof commitId === "string" && isRowCount(rowCount) ? { rowCount } : {},
     ...typeof commitId === "string" && isValidatedPrimaryKeyColumns(validatedPrimaryKeyColumns) ? { validatedPrimaryKeyColumns } : {},
     ...isSchemaChangeMetadata(parsed.sixb.schemaChange) ? { schemaChange: parsed.sixb.schemaChange } : {},
-    ...parsed.sixb.lineage === undefined ? {} : { lineage: parsed.sixb.lineage }
+    ...parsed.sixb.lineage === undefined ? {} : { lineage: parsed.sixb.lineage },
+    ...typeof rowsChanged === "boolean" ? { rowsChanged } : {},
+    ...typeof lineageChanged === "boolean" ? { lineageChanged } : {}
   };
 }
 function changesTouchTables(changesMade, tableIds) {
@@ -729,6 +732,7 @@ class DuckLakeChangeReader {
     const columns = resolveDatasetChangeColumns(input, before, after);
     if (!columns)
       return null;
+    const unchanged = (input.toVersionId === input.fromVersionId || !after.rowsChanged) && await this.snapshots.holdsRowsOf(input.datasetId, input.toVersionId, input.fromVersionId);
     const lease = await this.connections.acquireAttachedRuntime();
     const runtime = lease.runtime;
     const temporary = quoteIdentifier(`sixb_delta_${randomUUID().replaceAll("-", "")}`);
@@ -741,6 +745,9 @@ class DuckLakeChangeReader {
       const table = qualifiedTableName(this.options, ref.tableName);
       const oldRelation = `${table} AT (VERSION => ${parseVersionId(input.fromVersionId)})`;
       const newRelation = `${table} AT (VERSION => ${parseVersionId(input.toVersionId)})`;
+      if (unchanged) {
+        return noChanges(after.rowCount ?? before.rowCount ?? await countRows(runtime, newRelation));
+      }
       const previousCount = await uniqueRowCount(runtime, oldRelation, input.keyColumns);
       const nextCount = await uniqueRowCount(runtime, newRelation, input.keyColumns);
       if (previousCount === null || nextCount === null)
@@ -806,6 +813,21 @@ class DuckLakeChangeReader {
     }
   }
 }
+function noChanges(rowCount) {
+  return {
+    fromRowCount: rowCount,
+    toRowCount: rowCount,
+    changeCount: 0,
+    changes: async function* () {
+      yield* [];
+    }(),
+    async close() {}
+  };
+}
+async function countRows(runtime, relation) {
+  const [row] = await runtime.query(`SELECT count(*) AS total FROM ${relation}`);
+  return safeCount(row, "total");
+}
 async function uniqueRowCount(runtime, relation, keys) {
   const names = keys.map(quoteIdentifier);
   const invalid = names.map((name) => `${name} IS NULL OR trim(${name}) = ''`).join(" OR ");
@@ -2619,6 +2641,7 @@ import { LakeStorageError as LakeStorageError19 } from "@sixb/core/lake-storage"
 var SNAPSHOT_ROW_BATCH_SIZE = 128;
 var FEED_PAGE_SIZE = 512;
 var SHORT_READ_BATCH_SIZE = 8;
+var UNCHANGED_ROWS_WALK_LIMIT = 32;
 var CATALOG_SNAPSHOT_SCAN_LIMIT = 512;
 
 class DuckLakeSnapshotReader {
@@ -2661,6 +2684,33 @@ class DuckLakeSnapshotReader {
       return this.getVersionForTableRef(runtime, tableRef, snapshotId);
     });
   }
+  async holdsRowsOf(datasetId, versionId, earlierVersionId) {
+    const snapshotId = parseVersionId(versionId);
+    const earlierSnapshotId = parseVersionId(earlierVersionId);
+    if (snapshotId === earlierSnapshotId) {
+      return true;
+    }
+    if (BigInt(snapshotId) < BigInt(earlierSnapshotId)) {
+      return false;
+    }
+    this.connections.assertOpen();
+    return this.connections.withAttachedRuntime(async (runtime) => {
+      const tableRef = await resolveDatasetTableRef(this.options, runtime, datasetId);
+      if (!tableRef) {
+        return false;
+      }
+      const candidates = await this.querySnapshotCandidates(runtime, {
+        table: tableRef,
+        beforeSnapshotId: String(BigInt(snapshotId) + 1n),
+        fromSnapshotId: earlierSnapshotId,
+        limit: UNCHANGED_ROWS_WALK_LIMIT + 1
+      });
+      if (candidates[0]?.snapshotId !== snapshotId || candidates[candidates.length - 1]?.snapshotId !== earlierSnapshotId) {
+        return false;
+      }
+      return candidates.slice(0, -1).every((candidate) => this.candidateToSnapshotRow(datasetId, tableRef.tableId, candidate)?.rowsChanged === false);
+    });
+  }
   async listDatasetCatalogState(datasetIds) {
     this.connections.assertOpen();
     if (datasetIds.length === 0) {
@@ -3101,6 +3151,9 @@ class DuckLakeSnapshotReader {
     if (input.beforeSnapshotId !== undefined) {
       assertDuckLakeSnapshotId2(input.beforeSnapshotId);
     }
+    if (input.fromSnapshotId !== undefined) {
+      assertDuckLakeSnapshotId2(input.fromSnapshotId);
+    }
     const rows = await runtime.query(buildDuckLakeMetadataQuery(this.options, (table) => {
       const bounds = [
         `changes.snapshot_id >= (
@@ -3115,6 +3168,9 @@ class DuckLakeSnapshotReader {
       if (input.beforeSnapshotId !== undefined) {
         bounds.push(`changes.snapshot_id < ${input.beforeSnapshotId}`);
       }
+      if (input.fromSnapshotId !== undefined) {
+        bounds.push(`changes.snapshot_id >= ${input.fromSnapshotId}`);
+      }
       const namesDatasetTable = datasetTableIds(input.table).map((tableId) => `strpos(',' || changes.changes_made || ',', ${quoteSqlString(`:${tableId},`)}) > 0`).join(" OR ");
       return `
           SELECT
@@ -3175,6 +3231,8 @@ class DuckLakeSnapshotReader {
         tableId,
         createdAt: candidate.createdAt,
         mode: metadata.mode ?? "schema",
+        rowsChanged: metadata.rowsChanged ?? true,
+        lineageChanged: metadata.lineageChanged ?? true,
         metadata
       };
     }
@@ -3184,6 +3242,8 @@ class DuckLakeSnapshotReader {
       tableId,
       createdAt: candidate.createdAt,
       mode: candidate.metadata?.mode ?? (hasDeleteChange ? "snapshot" : "append"),
+      rowsChanged: true,
+      lineageChanged: candidate.metadata?.lineageChanged ?? true,
       ...candidate.metadata !== undefined ? { metadata: candidate.metadata } : {}
     };
   }
@@ -3209,6 +3269,8 @@ function snapshotToVersion(tableRef, row, schema) {
     producer: row.metadata?.producer,
     inputs: row.metadata?.inputs,
     ...storedLineage(tableRef.datasetId, row),
+    rowsChanged: row.rowsChanged,
+    lineageChanged: row.lineageChanged,
     ...row.metadata?.rowCount !== undefined ? { rowCount: row.metadata.rowCount } : {}
   };
 }
@@ -3306,7 +3368,7 @@ async function applyDatasetRowsFromRelation(input) {
       changedTableRows
     };
   }
-  const sourceRowCount = await countRows(input.runtime, input.sourceRelationSql);
+  const sourceRowCount = await countRows2(input.runtime, input.sourceRelationSql);
   const knownRowCount = baselineRowCount ?? input.previousRowCount;
   const targetWasEmpty = knownRowCount === undefined ? await isEmptyRelation(input.runtime, table) : knownRowCount === 0;
   await input.runtime.run(`INSERT INTO ${table} (${columnsSql}) SELECT ${columnsSql} FROM ${input.sourceRelationSql}`);
@@ -3376,7 +3438,7 @@ async function fingerprintRelation(runtime, relationSql, columnsSql) {
 function sameFingerprint(left, right) {
   return left.rowCount === right.rowCount && left.hashSum === right.hashSum && left.hashXor === right.hashXor;
 }
-async function countRows(runtime, relationSql) {
+async function countRows2(runtime, relationSql) {
   const [row] = await runtime.query(`SELECT count(*) AS row_count FROM ${relationSql}`);
   if (row === undefined) {
     return 0;
@@ -4707,6 +4769,7 @@ import { randomUUID as randomUUID8 } from "crypto";
 import {
   assertUnsequencedDatasetWrite,
   getDatasetPrimaryKeyColumns as getDatasetPrimaryKeyColumns4,
+  hasDatasetLineageChanges,
   hasDatasetProvenanceChanges,
   LakeConcurrencyError as LakeConcurrencyError2,
   LakeStorageError as LakeStorageError26,
@@ -4727,7 +4790,7 @@ async function applyDatasetMergeFromRelation(input) {
   }
   const stagingTable = quoteIdentifier(input.stagingTableName);
   const targetTable = qualifiedTableName(input.options, encodeDatasetTableName(input.dataset.id));
-  const sourceRowCount = await countRows2(input.runtime, stagingTable);
+  const sourceRowCount = await countRows3(input.runtime, stagingTable);
   const previousRowCount = await currentBaselineRowCount(input, targetTable, primaryKeyColumns);
   return withMaterializedFinalChanges(input, primaryKeyColumns, async (finalChanges, orderingChanged) => {
     const effects = await countMergeEffects(input, targetTable, finalChanges, primaryKeyColumns);
@@ -4827,7 +4890,7 @@ async function applySourceOrdering(input, acceptedTable) {
 }
 async function currentBaselineRowCount(input, targetTable, primaryKeyColumns) {
   if (sameColumns(input.validatedPrimaryKeyColumns, primaryKeyColumns)) {
-    return input.previousRowCount ?? countRows2(input.runtime, targetTable);
+    return input.previousRowCount ?? countRows3(input.runtime, targetTable);
   }
   return inspectUniqueKeyedRelation({
     runtime: input.runtime,
@@ -4920,9 +4983,10 @@ function primaryKeyMatchSql(primaryKeyColumns, leftAlias, rightAlias) {
   return primaryKeyColumns.map((column) => `${leftAlias}.${quoteIdentifier(column)} = ${rightAlias}.${quoteIdentifier(column)}`).join(" AND ");
 }
 function datasetRowDifferenceSql(dataset, leftAlias, rightAlias) {
-  return dataset.schema.columns.map((column) => `${leftAlias}.${quoteIdentifier(column.name)} IS DISTINCT FROM ${rightAlias}.${quoteIdentifier(column.name)}`).join(" OR ");
+  const differences = dataset.schema.columns.filter((column) => column.name !== dataset.sequenceBy).map((column) => `${leftAlias}.${quoteIdentifier(column.name)} IS DISTINCT FROM ${rightAlias}.${quoteIdentifier(column.name)}`);
+  return differences.length > 0 ? differences.join(" OR ") : "FALSE";
 }
-async function countRows2(runtime, relationSql) {
+async function countRows3(runtime, relationSql) {
   const [row] = await runtime.query(`SELECT count(*) AS row_count FROM ${relationSql}`);
   return row === undefined ? 0 : Number(getBigIntLike(row, "row_count"));
 }
@@ -5399,13 +5463,22 @@ class DuckLakeWriteCoordinator {
         validatedPrimaryKeyColumns: latestVersion?.validatedPrimaryKeyColumns
       });
       input.signal?.throwIfAborted();
-      const noOpBase = changeResult.dataChangeExpected ? undefined : latestVersion === undefined ? await this.latestVersionAtReadSnapshot(input) : latestVersion;
+      const parent = latestVersion === undefined ? await this.latestVersionAtReadSnapshot(input) : latestVersion;
+      const noOpBase = changeResult.dataChangeExpected ? undefined : parent;
       const createsVersion = noOpBase === undefined || this.noOpCreatesVersion(input, noOpBase, lineage);
       if (createsVersion && mustClaimTable(input, changeResult, lineage)) {
         await this.claimTable(input);
       }
       const commitId = randomUUID8();
-      await this.setCommitMetadata(input, commitId, commitRowCount(input, changeResult, latestVersion), lineage);
+      await this.setCommitMetadata(input, commitId, commitRowCount(input, changeResult, latestVersion), lineage, {
+        rowsChanged: parent === null || changeResult.changedTableRows,
+        lineageChanged: hasDatasetLineageChanges(parent, {
+          datasetId: input.dataset.id,
+          schema: input.dataset.schema,
+          producer: input.producer,
+          lineage
+        })
+      });
       await commitVersionTransaction(input.runtime, input.dataset.id, createsVersion && mustCommitOnReadSnapshot(input, changeResult) ? { poisonRuntime: () => this.connections.poisonRuntime() } : undefined);
       committed = true;
       const committedWriteSnapshotId = await this.lastCommittedSnapshotId(input.runtime);
@@ -5475,7 +5548,7 @@ class DuckLakeWriteCoordinator {
     }
     return result;
   }
-  async setCommitMetadata(input, commitId, rowCount, lineage) {
+  async setCommitMetadata(input, commitId, rowCount, lineage, changed) {
     const validatedPrimaryKeyColumns = getDatasetPrimaryKeyColumns4(input.dataset);
     const metadata = {
       kind: "datasetVersion",
@@ -5486,7 +5559,9 @@ class DuckLakeWriteCoordinator {
       inputs: input.inputs ? structuredClone(input.inputs) : undefined,
       ...rowCount.kind === "exact" ? { rowCount: rowCount.value } : {},
       ...validatedPrimaryKeyColumns !== null ? { validatedPrimaryKeyColumns: [...validatedPrimaryKeyColumns] } : {},
-      ...lineage === undefined ? {} : { lineage: encodeLineage(lineage) }
+      ...lineage === undefined ? {} : { lineage: encodeLineage(lineage) },
+      rowsChanged: changed.rowsChanged,
+      lineageChanged: changed.lineageChanged
     };
     await input.runtime.run(`CALL ${quoteIdentifier(duckLakeAlias(this.options))}.set_commit_message(${quoteSqlString("Sixb")}, ${quoteSqlString(input.commitMessage)}, extra_info => ${quoteSqlString(JSON.stringify({ sixb: metadata }))})`);
   }
@@ -5737,4 +5812,4 @@ export {
   DuckLakeStorage
 };
 
-//# debugId=9F26B0918A26BCC464756E2164756E21
+//# debugId=8FEE4C4A10B2BA3064756E2164756E21