@sixb/ducklake 0.1.8 → 0.1.9

package/dist/index.js

@@ -2,7 +2,7 @@
 // src/internal/ducklake-change-reader.ts
 import { randomUUID } from "crypto";
 import {
-  LakeStorageError as LakeStorageError7,
+  LakeStorageError as LakeStorageError8,
   resolveDatasetChangeColumns
 } from "@sixb/core/lake-storage";
 
@@ -63,6 +63,9 @@ function getDate(row, key) {
   throw new LakeStorageError(`[SixbDuckLake] Expected DuckDB column '${key}' to be a date, got ${typeof value}.`);
 }
 
+// src/internal/ducklake-dataset-table-ref.ts
+import { LakeStorageError as LakeStorageError4 } from "@sixb/core/lake-storage";
+
 // src/internal/names.ts
 import { LakeStorageError as LakeStorageError2 } from "@sixb/core/lake-storage";
 var DATASET_TABLE_PREFIX = "sixb__ds__";
@@ -412,24 +415,56 @@ function pathScheme(path) {
 
 // src/internal/ducklake-dataset-table-ref.ts
 async function resolveDatasetTableRef(options, runtime, datasetId) {
-  const tableName = encodeDatasetTableName(datasetId);
-  const ducklakeTable = duckLakeMetadataTableName(options, "ducklake_table");
-  const [row] = await runtime.query(`
-    SELECT table_id
-    FROM ${ducklakeTable}
-    WHERE table_name = ${quoteSqlString(tableName)}
-      AND end_snapshot IS NULL
-    LIMIT 1
+  return tableRefFromRows(datasetId, await runtime.query(datasetTablesSql(options, datasetId)));
+}
+async function resolveDatasetTableRefAtCurrentSnapshot(options, runtime, datasetId) {
+  const rows = await runtime.query(`
+    SELECT current.id AS current_snapshot, tables.table_id, tables.table_name
+    FROM ${quoteIdentifier(duckLakeAlias(options))}.current_snapshot() current
+    LEFT JOIN (${datasetTablesSql(options, datasetId)}) tables ON true
   `);
-  return row === undefined ? null : {
+  const [first] = rows;
+  if (first === undefined) {
+    throw new LakeStorageError4("[SixbDuckLake] DuckLake did not return a current snapshot.");
+  }
+  return {
+    snapshotId: String(getBigIntLike(first, "current_snapshot")),
+    table: tableRefFromRows(datasetId, rows.filter((row) => row.table_id !== null && row.table_id !== undefined))
+  };
+}
+function datasetTableIds(table) {
+  return table.sequenceTableId === undefined ? [table.tableId] : [table.tableId, table.sequenceTableId];
+}
+function datasetTablesSql(options, datasetId) {
+  const tableNames = [encodeDatasetTableName(datasetId), encodeDatasetSequenceTableName(datasetId)];
+  return `
+    SELECT table_id, table_name
+    FROM ${duckLakeMetadataTableName(options, "ducklake_table")}
+    WHERE table_name IN (${tableNames.map((name) => quoteSqlString(name)).join(", ")})
+      AND end_snapshot IS NULL
+  `;
+}
+function tableRefFromRows(datasetId, rows) {
+  const tableName = encodeDatasetTableName(datasetId);
+  const tableIdOf = (name) => {
+    const row = rows.find((candidate) => candidate.table_name === name);
+    return row === undefined ? undefined : getBigIntLike(row, "table_id");
+  };
+  const tableId = tableIdOf(tableName);
+  if (tableId === undefined) {
+    return null;
+  }
+  const sequenceTableId = tableIdOf(encodeDatasetSequenceTableName(datasetId));
+  return {
     datasetId,
     tableName,
-    tableId: getBigIntLike(row, "table_id")
+    tableId,
+    ...sequenceTableId === undefined ? {} : { sequenceTableId }
   };
 }
 
 // src/internal/schema.ts
-import { LakeStorageError as LakeStorageError5 } from "@sixb/core/lake-storage";
+import { LakeStorageError as LakeStorageError6 } from "@sixb/core/lake-storage";
 
 // src/internal/duckdb-column-types.ts
 function normalizeDuckDbTypeSql(typeSql) {
@@ -471,23 +506,23 @@ function findDatasetColumnTypeForDuckDbSql(typeSql) {
 
 // src/internal/duckdb-decimal.ts
 import { normalizeDecimalValue } from "@sixb/core";
-import { LakeStorageError as LakeStorageError4 } from "@sixb/core/lake-storage";
+import { LakeStorageError as LakeStorageError5 } from "@sixb/core/lake-storage";
 var decimalType = DUCKDB_COLUMN_TYPES.decimal;
 var maxIntegerDigits = decimalType.precision - decimalType.scale;
 function normalizeDuckDbDecimalValue(value, columnName) {
   if (typeof value !== "string") {
-    throw new LakeStorageError4(`[SixbDuckLake] Decimal column '${columnName}' must be an exact decimal string.`);
+    throw new LakeStorageError5(`[SixbDuckLake] Decimal column '${columnName}' must be an exact decimal string.`);
   }
   let normalized;
   try {
     normalized = normalizeDecimalValue(value);
   } catch {
-    throw new LakeStorageError4(`[SixbDuckLake] Decimal column '${columnName}' must be an exact decimal string.`);
+    throw new LakeStorageError5(`[SixbDuckLake] Decimal column '${columnName}' must be an exact decimal string.`);
   }
   const unsigned = normalized.startsWith("-") ? normalized.slice(1) : normalized;
   const [integer, fraction = ""] = unsigned.split(".");
   if (integer.length > maxIntegerDigits || fraction.length > decimalType.scale) {
-    throw new LakeStorageError4(`[SixbDuckLake] Decimal column '${columnName}' cannot be represented exactly as ${decimalType.sql}; expected at most ${maxIntegerDigits} integer digits and ${decimalType.scale} fractional digits.`);
+    throw new LakeStorageError5(`[SixbDuckLake] Decimal column '${columnName}' cannot be represented exactly as ${decimalType.sql}; expected at most ${maxIntegerDigits} integer digits and ${decimalType.scale} fractional digits.`);
   }
   return normalized;
 }
@@ -501,7 +536,7 @@ function duckDbTypeToDatasetColumnType(typeSql) {
   if (type) {
     return type;
   }
-  throw new LakeStorageError5(`[SixbDuckLake] DuckDB column type '${typeSql}' cannot be mapped to a Sixb dataset column type.`);
+  throw new LakeStorageError6(`[SixbDuckLake] DuckDB column type '${typeSql}' cannot be mapped to a Sixb dataset column type.`);
 }
 function datasetColumnToDuckDbSql(column) {
   const nullableSql = column.nullable ? "" : " NOT NULL";
@@ -539,7 +574,7 @@ function normalizeReadValue(value, column) {
 }
 function normalizeFileRef(value) {
   if (typeof value !== "object" || value === null || Array.isArray(value)) {
-    throw new LakeStorageError5("[SixbDuckLake] DuckDB fileRef value must be a struct.");
+    throw new LakeStorageError6("[SixbDuckLake] DuckDB fileRef value must be a struct.");
   }
   const record = value;
   return {
@@ -554,7 +589,7 @@ function normalizeFileRef(value) {
 function getStringField(record, key) {
   const value = record[key];
   if (typeof value !== "string") {
-    throw new LakeStorageError5(`[SixbDuckLake] DuckDB fileRef field '${key}' must be a string.`);
+    throw new LakeStorageError6(`[SixbDuckLake] DuckDB fileRef field '${key}' must be a string.`);
   }
   return value;
 }
@@ -564,7 +599,7 @@ function getOptionalStringField(record, key) {
     return;
   }
   if (typeof value !== "string") {
-    throw new LakeStorageError5(`[SixbDuckLake] DuckDB fileRef field '${key}' must be a string.`);
+    throw new LakeStorageError6(`[SixbDuckLake] DuckDB fileRef field '${key}' must be a string.`);
   }
   return value;
 }
@@ -576,18 +611,18 @@ function getIntegerField(record, key) {
   if (typeof value === "bigint") {
     return Number(value);
   }
-  throw new LakeStorageError5(`[SixbDuckLake] DuckDB fileRef field '${key}' must be an integer.`);
+  throw new LakeStorageError6(`[SixbDuckLake] DuckDB fileRef field '${key}' must be an integer.`);
 }
 
 // src/internal/versions.ts
-import { LakeStorageError as LakeStorageError6 } from "@sixb/core/lake-storage";
+import { LakeStorageError as LakeStorageError7 } from "@sixb/core/lake-storage";
 function parseVersionId(versionId) {
   if (!versionId.startsWith("ducklake:")) {
-    throw new LakeStorageError6(`[SixbDuckLake] Invalid DuckLake version id '${versionId}'.`);
+    throw new LakeStorageError7(`[SixbDuckLake] Invalid DuckLake version id '${versionId}'.`);
   }
   const snapshotId = versionId.slice("ducklake:".length);
   if (!/^\d+$/.test(snapshotId)) {
-    throw new LakeStorageError6(`[SixbDuckLake] Invalid DuckLake version id '${versionId}'.`);
+    throw new LakeStorageError7(`[SixbDuckLake] Invalid DuckLake version id '${versionId}'.`);
   }
   return snapshotId;
 }
@@ -609,6 +644,7 @@ function parseCommitMetadata(value) {
   }
   const mode = parsed.sixb.mode;
   const commitId = parsed.sixb.commitId;
+  const producer = parseDatasetProducer(parsed.sixb.producer);
   const rowCount = parsed.sixb.rowCount;
   const validatedPrimaryKeyColumns = parsed.sixb.validatedPrimaryKeyColumns;
   return {
@@ -616,14 +652,19 @@ function parseCommitMetadata(value) {
     datasetId: parsed.sixb.datasetId,
     ...typeof commitId === "string" ? { commitId } : {},
     ...mode === "snapshot" || mode === "append" || mode === "merge" || mode === "schema" ? { mode } : {},
-    ...isDatasetProducer(parsed.sixb.producer) ? { producer: parsed.sixb.producer } : {},
+    ...producer === undefined ? {} : { producer },
     ...isDatasetVersionRefs(parsed.sixb.inputs) ? { inputs: parsed.sixb.inputs } : {},
     ...typeof commitId === "string" && isRowCount(rowCount) ? { rowCount } : {},
     ...typeof commitId === "string" && isValidatedPrimaryKeyColumns(validatedPrimaryKeyColumns) ? { validatedPrimaryKeyColumns } : {},
-    ...isSchemaChangeMetadata(parsed.sixb.schemaChange) ? { schemaChange: parsed.sixb.schemaChange } : {}
+    ...isSchemaChangeMetadata(parsed.sixb.schemaChange) ? { schemaChange: parsed.sixb.schemaChange } : {},
+    ...parsed.sixb.lineage === undefined ? {} : { lineage: parsed.sixb.lineage }
   };
 }
-function parseInlineDataChange(changesMade, tableId) {
+function changesTouchTables(changesMade, tableIds) {
+  const ids = new Set(tableIds.map((tableId) => tableId.toString()));
+  return changesMade.split(",").some((change) => ids.has(change.split(":")[1] ?? ""));
+}
+function parseTableDataChange(changesMade, tableId) {
   const tableIdText = tableId.toString();
   let hasDataChange = false;
   let hasDeleteChange = false;
@@ -645,8 +686,16 @@ function parseInlineDataChange(changesMade, tableId) {
 function isRecord(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
-function isDatasetProducer(value) {
-  return isRecord(value) && (value.kind === "sync" || value.kind === "pipeline" || value.kind === "ingest") && optionalString(value.id) && optionalString(value.runId);
+function parseDatasetProducer(value) {
+  if (!isRecord(value) || value.kind !== "sync" && value.kind !== "pipeline" && value.kind !== "ingest") {
+    return;
+  }
+  return {
+    kind: value.kind,
+    ...typeof value.id === "string" ? { id: value.id } : {},
+    ...typeof value.runId === "string" ? { runId: value.runId } : {},
+    ...typeof value.stepId === "string" ? { stepId: value.stepId } : {}
+  };
 }
 function isDatasetVersionRefs(value) {
   return Array.isArray(value) && value.every((item) => isRecord(item) && typeof item.datasetId === "string" && typeof item.versionId === "string");
@@ -654,9 +703,6 @@ function isDatasetVersionRefs(value) {
 function isSchemaChangeMetadata(value) {
   return isRecord(value) && (value.addColumns === undefined || Array.isArray(value.addColumns) && value.addColumns.every((columnName) => typeof columnName === "string"));
 }
-function optionalString(value) {
-  return value === undefined || typeof value === "string";
-}
 function isRowCount(value) {
   return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
 }
@@ -729,11 +775,11 @@ class DuckLakeChangeReader {
           while (delivered < changeCount) {
             input.signal?.throwIfAborted();
             if (closePromise)
-              throw new LakeStorageError7("[SixbDuckLake] Change reader is closed.");
+              throw new LakeStorageError8("[SixbDuckLake] Change reader is closed.");
             const page = await runtime.query(`SELECT rowid AS position, * FROM ${temporary}
               WHERE rowid > ${cursor} ORDER BY rowid LIMIT ${PAGE_ROWS}`);
             if (page.length === 0)
-              throw new LakeStorageError7("[SixbDuckLake] Change read ended before its declared count.");
+              throw new LakeStorageError8("[SixbDuckLake] Change read ended before its declared count.");
             for (const row of page) {
               input.signal?.throwIfAborted();
               delivered++;
@@ -772,7 +818,7 @@ async function uniqueRowCount(runtime, relation, keys) {
 function safeCount(row, name) {
   const count = Number(getBigIntLike(row, name));
   if (!Number.isSafeInteger(count) || count < 0)
-    throw new LakeStorageError7("[SixbDuckLake] Change count exceeds the supported integer range.");
+    throw new LakeStorageError8("[SixbDuckLake] Change count exceeds the supported integer range.");
   return count;
 }
 function rowImage(row, prefix, columns) {
@@ -783,7 +829,7 @@ function rowImage(row, prefix, columns) {
 }
 
 // src/internal/ducklake-connection-manager.ts
-import { LakeStorageError as LakeStorageError11 } from "@sixb/core/lake-storage";
+import { LakeStorageError as LakeStorageError12 } from "@sixb/core/lake-storage";
 
 // src/internal/catalog-key.ts
 import { resolve } from "path";
@@ -808,10 +854,10 @@ import { join, resolve as resolve2 } from "path";
 import {
   DuckDBInstance
 } from "@duckdb/node-api";
-import { LakeStorageError as LakeStorageError10 } from "@sixb/core/lake-storage";
+import { LakeStorageError as LakeStorageError11 } from "@sixb/core/lake-storage";
 
 // src/internal/duckdb-reader.ts
-import { LakeStorageError as LakeStorageError9 } from "@sixb/core/lake-storage";
+import { LakeStorageError as LakeStorageError10 } from "@sixb/core/lake-storage";
 
 // src/internal/duckdb-value-converter.ts
 import {
@@ -819,14 +865,14 @@ import {
   DuckDBTypeId,
   JSDuckDBValueConverter
 } from "@duckdb/node-api";
-import { LakeStorageError as LakeStorageError8 } from "@sixb/core/lake-storage";
+import { LakeStorageError as LakeStorageError9 } from "@sixb/core/lake-storage";
 var sixbDuckDbValueConverter = (value, type, converter) => {
   if (value === null) {
     return null;
   }
   if (type.typeId === DuckDBTypeId.DECIMAL) {
     if (!(value instanceof DuckDBDecimalValue)) {
-      throw new LakeStorageError8("[SixbDuckLake] DuckDB returned an invalid DECIMAL value.");
+      throw new LakeStorageError9("[SixbDuckLake] DuckDB returned an invalid DECIMAL value.");
     }
     return value.toString();
   }
@@ -868,7 +914,7 @@ class NodeDuckDbReader {
   }
   async* rows() {
     if (this.consumed)
-      throw new LakeStorageError9("[SixbDuckLake] Reader can only be consumed once.");
+      throw new LakeStorageError10("[SixbDuckLake] Reader can only be consumed once.");
     this.consumed = true;
     try {
       this.assertOpen();
@@ -931,7 +977,7 @@ class NodeDuckDbReader {
   assertOpen() {
     this.options.signal?.throwIfAborted();
     if (this.closePromise)
-      throw new LakeStorageError9("[SixbDuckLake] Reader is closed.");
+      throw new LakeStorageError10("[SixbDuckLake] Reader is closed.");
   }
 }
 
@@ -1042,12 +1088,12 @@ class NodeDuckDbRuntime {
   }
   assertAcceptingOperations() {
     if (this.closing || this.closed) {
-      throw new LakeStorageError10("[SixbDuckLake] DuckDB runtime is closed.");
+      throw new LakeStorageError11("[SixbDuckLake] DuckDB runtime is closed.");
     }
   }
   assertNotClosed() {
     if (this.closed) {
-      throw new LakeStorageError10("[SixbDuckLake] DuckDB runtime is closed.");
+      throw new LakeStorageError11("[SixbDuckLake] DuckDB runtime is closed.");
     }
   }
 }
@@ -1239,7 +1285,7 @@ class DuckLakeConnectionManager {
     this.observedLocalCatalogGeneration = currentLocalCatalogGeneration(options);
     this.maxStreamingReads = options.maxStreamingReads ?? 4;
     if (!Number.isSafeInteger(this.maxStreamingReads) || this.maxStreamingReads < 1) {
-      throw new LakeStorageError11("[SixbDuckLake] maxStreamingReads must be a positive integer.");
+      throw new LakeStorageError12("[SixbDuckLake] maxStreamingReads must be a positive integer.");
     }
   }
   async acquireRead(signal) {
@@ -1293,7 +1339,7 @@ class DuckLakeConnectionManager {
   }
   assertOpen() {
     if (this.closed) {
-      throw new LakeStorageError11("[SixbDuckLake] DuckLakeStorage is closed.");
+      throw new LakeStorageError12("[SixbDuckLake] DuckLakeStorage is closed.");
     }
   }
   async runtime() {
@@ -1481,7 +1527,7 @@ class DuckLakeConnectionManager {
   }
   async finishClose() {
     this.closed = true;
-    this.shutdown.abort(new LakeStorageError11("[SixbDuckLake] DuckLakeStorage is closed."));
+    this.shutdown.abort(new LakeStorageError12("[SixbDuckLake] DuckLakeStorage is closed."));
     const runtimePromise = this.runtimePromise;
     this.runtimePromise = undefined;
     await this.closeRuntimePromise(runtimePromise, {
@@ -1643,12 +1689,13 @@ function incrementLocalCatalogGeneration(options) {
 import { randomUUID as randomUUID2 } from "crypto";
 import {
   assertDatasetDefinition,
-  LakeStorageError as LakeStorageError14,
+  LakeStorageError as LakeStorageError16,
+  lineageAtCommit,
   planDatasetDefinitionUpdate
 } from "@sixb/core/lake-storage";
 
 // src/internal/ducklake-catalog-schema.ts
-import { LakeStorageError as LakeStorageError12 } from "@sixb/core/lake-storage";
+import { LakeStorageError as LakeStorageError13 } from "@sixb/core/lake-storage";
 var SIXB_PRIMARY_KEY_COMMENT_NAMESPACE = "sixb:primary-key:";
 var DATASET_SEQUENCE_COLUMN_COMMENT = "sixb.sequenceBy.v1";
 var SIXB_PRIMARY_KEY_COMMENT_PREFIX = `${SIXB_PRIMARY_KEY_COMMENT_NAMESPACE}v1:`;
@@ -1699,14 +1746,14 @@ function duckLakeCatalogColumnsToDatasetPrimaryKey(tableName, rows) {
       throwInvalidPrimaryKeyMetadata(tableName, column, comment);
     }
     if (seenOrdinals.has(ordinal)) {
-      throw new LakeStorageError12(`[SixbDuckLake] Dataset table '${tableName}' has duplicate primary-key ordinal '${ordinal}'.`);
+      throw new LakeStorageError13(`[SixbDuckLake] Dataset table '${tableName}' has duplicate primary-key ordinal '${ordinal}'.`);
     }
     seenOrdinals.add(ordinal);
     if (duckDbTypeToDatasetColumnType(column.columnType) !== "string") {
-      throw new LakeStorageError12(`[SixbDuckLake] Dataset table '${tableName}' primary-key column '${column.columnName}' must map to type 'string'.`);
+      throw new LakeStorageError13(`[SixbDuckLake] Dataset table '${tableName}' primary-key column '${column.columnName}' must map to type 'string'.`);
     }
     if (column.nullsAllowed) {
-      throw new LakeStorageError12(`[SixbDuckLake] Dataset table '${tableName}' primary-key column '${column.columnName}' must not be nullable.`);
+      throw new LakeStorageError13(`[SixbDuckLake] Dataset table '${tableName}' primary-key column '${column.columnName}' must not be nullable.`);
     }
     keyedColumns.push({ column, ordinal });
   }
@@ -1716,7 +1763,7 @@ function duckLakeCatalogColumnsToDatasetPrimaryKey(tableName, rows) {
   keyedColumns.sort((left, right) => left.ordinal - right.ordinal);
   for (const [expectedOrdinal, keyedColumn] of keyedColumns.entries()) {
     if (keyedColumn.ordinal !== expectedOrdinal) {
-      throw new LakeStorageError12(`[SixbDuckLake] Dataset table '${tableName}' primary-key ordinals must be contiguous from 0.`);
+      throw new LakeStorageError13(`[SixbDuckLake] Dataset table '${tableName}' primary-key ordinals must be contiguous from 0.`);
     }
   }
   const columnNames = keyedColumns.map(({ column }) => column.columnName);
@@ -1726,7 +1773,7 @@ function duckLakeCatalogColumnsToDatasetPrimaryKey(tableName, rows) {
   return columnNames;
 }
 function throwInvalidPrimaryKeyMetadata(tableName, column, comment) {
-  throw new LakeStorageError12(`[SixbDuckLake] Dataset table '${tableName}' has malformed primary-key metadata '${comment}' on column '${column.columnName}'.`);
+  throw new LakeStorageError13(`[SixbDuckLake] Dataset table '${tableName}' has malformed primary-key metadata '${comment}' on column '${column.columnName}'.`);
 }
 function duckLakeCatalogColumnToDefinition(tableName, column, childrenByParentId) {
   const children = [...childrenByParentId.get(column.columnId) ?? []].sort(compareDuckLakeCatalogColumnOrder);
@@ -1743,7 +1790,7 @@ function duckLakeCatalogColumnTypeToDatasetType(tableName, column, children) {
   if (isFileRefStruct(children)) {
     return "fileRef";
   }
-  throw new LakeStorageError12(`[SixbDuckLake] DuckDB column type '${formatDuckLakeCatalogColumnType(column, children)}' in dataset table '${tableName}' cannot be mapped to a Sixb dataset column type.`);
+  throw new LakeStorageError13(`[SixbDuckLake] DuckDB column type '${formatDuckLakeCatalogColumnType(column, children)}' in dataset table '${tableName}' cannot be mapped to a Sixb dataset column type.`);
 }
 function compareDuckLakeCatalogColumnOrder(left, right) {
   return Number(left.columnOrder - right.columnOrder);
@@ -1765,7 +1812,7 @@ function formatDuckLakeCatalogColumnType(column, children) {
 }
 
 // src/internal/ducklake-current-dataset-definitions.ts
-import { LakeStorageError as LakeStorageError13 } from "@sixb/core/lake-storage";
+import { LakeStorageError as LakeStorageError14 } from "@sixb/core/lake-storage";
 async function readCurrentDatasetDefinitions(input) {
   if (input.datasetIds?.length === 0) {
     return new Map;
@@ -1892,7 +1939,7 @@ async function readCurrentDatasetPartitions(input, tableNames) {
       transform: getString(row, "transform")
     };
     if (partition.transform !== "identity") {
-      throw new LakeStorageError13(`[SixbDuckLake] Dataset table '${partition.tableName}' uses unsupported DuckLake partition transform '${partition.transform}'.`);
+      throw new LakeStorageError14(`[SixbDuckLake] Dataset table '${partition.tableName}' uses unsupported DuckLake partition transform '${partition.transform}'.`);
     }
     return partition;
   });
@@ -1916,6 +1963,188 @@ function groupPartitionsByTableName(rows) {
   return result;
 }
 
+// src/internal/ducklake-version-transaction.ts
+import { LakeConcurrencyError, LakeStorageError as LakeStorageError15 } from "@sixb/core/lake-storage";
+var VERSION_RERUNS = { maxAttempts: 16, baseDelayMillis: 10, maxDelayMillis: 1000 };
+
+class CatalogMovedError extends LakeConcurrencyError {
+}
+async function beginTransactionOnLatestVersion(input) {
+  if (!isSqliteCatalog(input.options.catalog)) {
+    await input.runtime.run("BEGIN TRANSACTION");
+    return (await input.readLatestVersion()).latestVersion;
+  }
+  input.signal?.throwIfAborted();
+  const { snapshotId, latestVersion } = await input.readLatestVersion();
+  await input.runtime.run("BEGIN TRANSACTION");
+  if (await currentSnapshotId(input.options, input.runtime) !== snapshotId) {
+    await input.runtime.run("ROLLBACK");
+    throw new CatalogMovedError(`[SixbDuckLake] Catalog changed while a write to dataset '${input.datasetId}' was prepared.`);
+  }
+  return latestVersion;
+}
+async function commitVersionTransaction(runtime, datasetId, onReadSnapshot) {
+  if (onReadSnapshot) {
+    await runtime.run("SET ducklake_max_retry_count = 0");
+  }
+  try {
+    await runtime.run("COMMIT");
+  } catch (error) {
+    throw catalogMovedError(error, datasetId) ?? error;
+  } finally {
+    if (onReadSnapshot) {
+      await runtime.run("RESET ducklake_max_retry_count").catch(() => {
+        onReadSnapshot.poisonRuntime();
+      });
+    }
+  }
+}
+async function rerunWhenCatalogMoves(datasetId, transaction, signal) {
+  for (let attempt = 1;; attempt++) {
+    try {
+      return await transaction(attempt);
+    } catch (error) {
+      if (!(error instanceof CatalogMovedError)) {
+        throw error;
+      }
+      if (attempt === VERSION_RERUNS.maxAttempts) {
+        throw new LakeConcurrencyError(`[SixbDuckLake] Concurrent writers moved the catalog during ${attempt} attempts to commit dataset '${datasetId}'. Retry the write.`, { cause: error });
+      }
+    }
+    await waitBeforeRerun(rerunDelayMillis(attempt), signal);
+  }
+}
+function rerunDelayMillis(rerun) {
+  if (rerun === 1) {
+    return 0;
+  }
+  const window = Math.min(VERSION_RERUNS.maxDelayMillis, VERSION_RERUNS.baseDelayMillis * 2 ** (rerun - 1));
+  return Math.random() * window;
+}
+function waitBeforeRerun(millis, signal) {
+  signal?.throwIfAborted();
+  if (millis === 0) {
+    return Promise.resolve();
+  }
+  return new Promise((resolve, reject) => {
+    const abort = () => {
+      clearTimeout(timer);
+      reject(signal?.reason);
+    };
+    const timer = setTimeout(() => {
+      signal?.removeEventListener("abort", abort);
+      resolve();
+    }, millis);
+    signal?.addEventListener("abort", abort, { once: true });
+  });
+}
+function isSqliteCatalog(catalog) {
+  return catalog.type === "sqlite" || catalog.type === "custom" && catalog.uri.toLowerCase().startsWith("sqlite:");
+}
+function catalogMovedError(error, datasetId) {
+  if (error instanceof Error && error.message.startsWith("TransactionContext Error: Failed to commit: Failed to commit DuckLake transaction.") && (error.message.includes("Transaction conflict - ") || error.message.includes("Exceeded the maximum retry count"))) {
+    return new CatalogMovedError(`[SixbDuckLake] Concurrent catalog commit for dataset '${datasetId}'.`, { cause: error });
+  }
+  return;
+}
+async function currentSnapshotId(options, runtime) {
+  const [row] = await runtime.query(`SELECT id FROM ${quoteIdentifier(duckLakeAlias(options))}.current_snapshot()`);
+  if (row === undefined) {
+    throw new LakeStorageError15("[SixbDuckLake] DuckLake did not return a current snapshot.");
+  }
+  return String(getBigIntLike(row, "id"));
+}
+
+// src/internal/lineage-encoding.ts
+function encodeLineage(lineage) {
+  if ("unknown" in lineage) {
+    return JSON.stringify({ unknown: lineage.unknown });
+  }
+  const strings = [];
+  const indexes = new Map;
+  const indexOf = (value) => {
+    let index = indexes.get(value);
+    if (index === undefined) {
+      index = strings.length;
+      strings.push(value);
+      indexes.set(value, index);
+    }
+    return index;
+  };
+  const encodeRefs = (refs) => refs.flatMap((ref) => [indexOf(ref.datasetId), indexOf(ref.column)]);
+  const encoded = {
+    v: 1,
+    strings,
+    rows: encodeRefs(lineage.rows),
+    columns: Object.entries(lineage.columns).map(([column, refs]) => [
+      indexOf(column),
+      ...encodeRefs(refs)
+    ])
+  };
+  return JSON.stringify(encoded);
+}
+function decodeLineage(datasetId, stored) {
+  const parsed = typeof stored === "string" ? parseJson(stored) : undefined;
+  if (isRecord2(parsed) && Object.hasOwn(parsed, "unknown")) {
+    return typeof parsed.unknown === "string" ? { unknown: parsed.unknown } : unreadable(datasetId);
+  }
+  return decodeColumnLineage(parsed) ?? unreadable(datasetId);
+}
+function decodeColumnLineage(value) {
+  if (!isRecord2(value) || value.v !== 1 || !isStringList(value.strings)) {
+    return;
+  }
+  const { strings } = value;
+  const decodeRefs = (flat, from) => {
+    if ((flat.length - from) % 2 !== 0) {
+      return;
+    }
+    const refs = [];
+    for (let index = from;index < flat.length; index += 2) {
+      const datasetId = stringAt(strings, flat[index]);
+      const column = stringAt(strings, flat[index + 1]);
+      if (datasetId === undefined || column === undefined) {
+        return;
+      }
+      refs.push({ datasetId, column });
+    }
+    return refs;
+  };
+  const rows = Array.isArray(value.rows) ? decodeRefs(value.rows, 0) : undefined;
+  if (rows === undefined || !Array.isArray(value.columns)) {
+    return;
+  }
+  const columns = new Map;
+  for (const entry of value.columns) {
+    const name = Array.isArray(entry) ? stringAt(strings, entry[0]) : undefined;
+    const refs = name === undefined ? undefined : decodeRefs(entry, 1);
+    if (name === undefined || refs === undefined || columns.has(name)) {
+      return;
+    }
+    columns.set(name, refs);
+  }
+  return { rows, columns: Object.fromEntries(columns) };
+}
+function unreadable(datasetId) {
+  return { unknown: `a version of '${datasetId}' records a column lineage Sixb cannot read` };
+}
+function parseJson(text) {
+  try {
+    return JSON.parse(text);
+  } catch {
+    return;
+  }
+}
+function stringAt(strings, index) {
+  return typeof index === "number" && Number.isInteger(index) ? strings[index] : undefined;
+}
+function isStringList(value) {
+  return Array.isArray(value) && value.every((item) => typeof item === "string");
+}
+function isRecord2(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
 // src/internal/ducklake-dataset-catalog.ts
 class DuckLakeDatasetCatalog {
   options;
@@ -1924,13 +2153,22 @@ class DuckLakeDatasetCatalog {
     this.options = options;
     this.connections = connections;
   }
-  async createDataset(definition) {
+  async createDataset(definition, readLatestVersion) {
     this.assertSchema(definition);
     const existing = await this.getDataset(definition.id);
     if (existing) {
-      const plan = planDatasetDefinitionUpdate(existing, definition);
+      const plan = await rerunWhenCatalogMoves(definition.id, async (attempt) => {
+        const current = attempt === 1 ? existing : await this.getDataset(definition.id);
+        if (!current) {
+          throw new LakeStorageError16(`[SixbDuckLake] Dataset '${definition.id}' was removed while its definition was updated.`);
+        }
+        const plan = planDatasetDefinitionUpdate(current, definition);
+        if (plan.changed) {
+          await this.applyExistingDatasetUpdate(plan, readLatestVersion);
+        }
+        return plan;
+      });
       if (plan.changed) {
-        await this.applyExistingDatasetUpdate(plan);
         this.connections.markLocalCatalogChanged();
         await this.connections.resetRuntime();
       }
@@ -1994,7 +2232,7 @@ class DuckLakeDatasetCatalog {
     if (failures.length > 0) {
       const details = failures.join(`
 `);
-      throw new LakeStorageError14(`[SixbLake] Lake dataset definition check failed for ${failures.length} dataset(s).
+      throw new LakeStorageError16(`[SixbLake] Lake dataset definition check failed for ${failures.length} dataset(s).
 ${details}`);
     }
   }
@@ -2026,32 +2264,50 @@ ${details}`);
       datasetIds: options?.datasetIds
     });
   }
-  async getDatasetSchemaAtSnapshot(runtime, datasetId, tableName, tableId, snapshotId) {
-    assertDuckLakeSnapshotId(snapshotId);
-    const columns = await this.readDatasetColumnsAtSnapshot(runtime, tableName, tableId, snapshotId);
-    if (columns.length === 0) {
-      throw new LakeStorageError14(`[SixbDuckLake] Could not reconstruct schema for dataset '${datasetId}' at DuckLake snapshot '${snapshotId}'.`);
+  async getDatasetSchemasAtSnapshots(runtime, requests) {
+    if (requests.length === 0) {
+      return [];
     }
-    return duckLakeCatalogColumnsToDatasetSchema(tableName, columns);
+    for (const request of requests) {
+      assertDuckLakeSnapshotId(request.snapshotId);
+    }
+    const snapshots = requests.map((request) => BigInt(request.snapshotId));
+    const rows = await this.readDatasetColumnsBetweenSnapshots(runtime, requests.map((request) => request.table), snapshots.reduce((oldest, snapshot) => snapshot < oldest ? snapshot : oldest), snapshots.reduce((newest, snapshot) => snapshot > newest ? snapshot : newest));
+    return requests.map(({ table, snapshotId }, index) => {
+      const snapshot = snapshots[index];
+      const columns = rows.filter((row) => row.tableId === table.tableId && row.beginSnapshot <= snapshot && (row.endSnapshot === undefined || row.endSnapshot > snapshot)).map((row) => ({ tableName: table.tableName, ...row.column }));
+      if (columns.length === 0) {
+        throw new LakeStorageError16(`[SixbDuckLake] Could not reconstruct schema for dataset '${table.datasetId}' at DuckLake snapshot '${snapshotId}'.`);
+      }
+      return duckLakeCatalogColumnsToDatasetSchema(table.tableName, columns);
+    });
   }
   assertSchema(definition) {
-    assertDatasetDefinition(definition, (message) => new LakeStorageError14(`[SixbDuckLake] ${message}`));
+    assertDatasetDefinition(definition, (message) => new LakeStorageError16(`[SixbDuckLake] ${message}`));
   }
-  async applyExistingDatasetUpdate(plan) {
+  async applyExistingDatasetUpdate(plan, readLatestVersion) {
     const tableName = encodeDatasetTableName(plan.definition.id);
     const qualifiedName = qualifiedTableName(this.options, tableName);
+    const { schema } = plan;
+    if (schema.kind === "none") {
+      await this.connections.withExclusiveAttached((runtime) => this.applyMetadataDelta(runtime, qualifiedName, plan));
+      return;
+    }
     await this.connections.withExclusiveAttached(async (runtime) => {
-      if (plan.schema.kind === "none") {
-        await this.applyMetadataDelta(runtime, qualifiedName, plan);
-        return;
-      }
-      await runtime.run("BEGIN TRANSACTION");
       let committed = false;
       try {
-        await this.applySchemaEvolutionDdl(runtime, qualifiedName, plan.schema);
+        const parent = await beginTransactionOnLatestVersion({
+          options: this.options,
+          runtime,
+          datasetId: plan.definition.id,
+          readLatestVersion: () => readLatestVersion(runtime, plan.definition)
+        });
+        await this.applySchemaEvolutionDdl(runtime, qualifiedName, schema);
         await this.applyMetadataDelta(runtime, qualifiedName, plan);
-        await this.setSchemaEvolutionCommitMetadata(runtime, plan.definition.id, plan.schema);
-        await runtime.run("COMMIT");
+        await this.setSchemaEvolutionCommitMetadata(runtime, plan.definition.id, schema, lineageAtCommit({ mode: "schema", schema: plan.definition.schema, parent }));
+        await commitVersionTransaction(runtime, plan.definition.id, {
+          poisonRuntime: () => this.connections.poisonRuntime()
+        });
         committed = true;
       } catch (error) {
         if (!committed) {
@@ -2082,7 +2338,7 @@ ${details}`);
       await runtime.run(`ALTER TABLE ${qualifiedName} SET PARTITIONED BY (${definition.partitionBy.map((columnName) => quoteIdentifier(columnName)).join(", ")})`);
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
-      throw new LakeStorageError14(`[SixbDuckLake] Dataset '${definition.id}' cannot apply partitionBy because DuckLake rejected ALTER TABLE SET PARTITIONED BY: ${message}`);
+      throw new LakeStorageError16(`[SixbDuckLake] Dataset '${definition.id}' cannot apply partitionBy because DuckLake rejected ALTER TABLE SET PARTITIONED BY: ${message}`);
     }
   }
   async applyPrimaryKey(runtime, qualifiedName, definition) {
@@ -2094,7 +2350,7 @@ ${details}`);
       await runtime.run(`COMMENT ON COLUMN ${qualifiedName}.${quoteIdentifier(columnName)} IS ${quoteSqlString(duckLakePrimaryKeyColumnComment(ordinal))}`);
     }
   }
-  async setSchemaEvolutionCommitMetadata(runtime, datasetId, plan) {
+  async setSchemaEvolutionCommitMetadata(runtime, datasetId, plan, lineage) {
     await runtime.run(`CALL ${quoteIdentifier(duckLakeAlias(this.options))}.set_commit_message(${quoteSqlString("Sixb")}, ${quoteSqlString(`evolve dataset ${datasetId} schema`)}, extra_info => ${quoteSqlString(JSON.stringify({
       sixb: {
         kind: "datasetVersion",
@@ -2103,7 +2359,8 @@ ${details}`);
         mode: "schema",
         schemaChange: {
           addColumns: plan.columns.map((column) => column.name)
-        }
+        },
+        ...lineage === undefined ? {} : { lineage: encodeLineage(lineage) }
       }
     }))})`);
   }
@@ -2112,63 +2369,71 @@ ${details}`);
       await runtime.run("ROLLBACK");
     } catch {}
   }
-  async readDatasetColumnsAtSnapshot(runtime, tableName, tableId, snapshotId) {
+  async readDatasetColumnsBetweenSnapshots(runtime, tables, oldestSnapshot, newestSnapshot) {
+    const tableIds = new Set(tables.map((table) => table.tableId));
     const ducklakeColumn = duckLakeMetadataTableName(this.options, "ducklake_column");
     const rows = await runtime.query(`
       SELECT
+        table_id,
         column_id,
         column_order,
         column_name,
         column_type,
         CAST(nulls_allowed AS BOOLEAN) AS nulls_allowed,
-        parent_column
+        parent_column,
+        begin_snapshot,
+        end_snapshot
       FROM ${ducklakeColumn}
-      WHERE table_id = ${tableId}
-        AND begin_snapshot <= ${snapshotId}
-        AND (end_snapshot IS NULL OR end_snapshot > ${snapshotId})
-      ORDER BY column_order
+      WHERE table_id IN (${[...tableIds].join(", ")})
+        AND begin_snapshot <= ${newestSnapshot}
+        AND (end_snapshot IS NULL OR end_snapshot > ${oldestSnapshot})
+      ORDER BY column_order, column_id
     `);
     return rows.map((row) => ({
-      tableName,
-      columnId: getBigIntLike(row, "column_id"),
-      columnOrder: getBigIntLike(row, "column_order"),
-      columnName: getString(row, "column_name"),
-      columnType: getString(row, "column_type"),
-      nullsAllowed: getBoolean(row, "nulls_allowed"),
-      parentColumnId: getOptionalBigIntLike(row, "parent_column")
+      tableId: getBigIntLike(row, "table_id"),
+      beginSnapshot: getBigIntLike(row, "begin_snapshot"),
+      endSnapshot: getOptionalBigIntLike(row, "end_snapshot"),
+      column: {
+        columnId: getBigIntLike(row, "column_id"),
+        columnOrder: getBigIntLike(row, "column_order"),
+        columnName: getString(row, "column_name"),
+        columnType: getString(row, "column_type"),
+        nullsAllowed: getBoolean(row, "nulls_allowed"),
+        parentColumnId: getOptionalBigIntLike(row, "parent_column")
+      }
     }));
   }
 }
 function assertDuckLakeSnapshotId(snapshotId) {
   if (!/^\d+$/.test(snapshotId)) {
-    throw new LakeStorageError14(`[SixbDuckLake] Invalid DuckLake snapshot id '${snapshotId}'.`);
+    throw new LakeStorageError16(`[SixbDuckLake] Invalid DuckLake snapshot id '${snapshotId}'.`);
   }
 }
 
 // src/internal/ducklake-maintenance.ts
-import { LakeStorageError as LakeStorageError15 } from "@sixb/core/lake-storage";
+import { LakeStorageError as LakeStorageError17 } from "@sixb/core/lake-storage";
 var DEFAULT_RETENTION = "7 days";
 
 class DuckLakeMaintenance {
   options;
   connections;
-  constructor(options, connections) {
+  snapshots;
+  constructor(options, connections, snapshots) {
     this.options = options;
     this.connections = connections;
+    this.snapshots = snapshots;
   }
   async runMaintenance(options = {}) {
     this.connections.assertOpen();
     const expireOlderThan = options.expireOlderThan ?? DEFAULT_RETENTION;
     const deleteOlderThan = options.deleteOlderThan ?? expireOlderThan;
     const dryRun = options.dryRun ?? false;
+    const protect = (options.protect ?? []).map(protectedSnapshotId);
     if (this.options.readOnly && !dryRun) {
-      throw new LakeStorageError15("[SixbDuckLake] DuckLake maintenance cannot run in read-only mode.");
+      throw new LakeStorageError17("[SixbDuckLake] DuckLake maintenance cannot run in read-only mode.");
     }
     return this.connections.withMaintenanceRuntime(async (runtime) => {
-      const counts = await runMaintenanceProcedures(runtime, this.options, {
-        expireOlderThan,
-        deleteOlderThan
-      }, dryRun);
+      const counts = await this.runMaintenanceProcedures(runtime, { expireOlderThan, deleteOlderThan }, protect, dryRun);
       if (!dryRun) {
         this.connections.markLocalCatalogChanged();
       }
@@ -2180,26 +2445,55 @@ class DuckLakeMaintenance {
       };
     });
   }
+  async runMaintenanceProcedures(runtime, retention, protect, dryRun) {
+    const aliasSql = quoteSqlString(duckLakeAlias(this.options));
+    const oldSnapshotIds = await this.snapshotsOlderThan(runtime, retention.expireOlderThan);
+    const needed = new Set([
+      ...await this.snapshots.latestVersionSnapshotIds(runtime),
+      ...protect
+    ]);
+    const expirable = oldSnapshotIds.filter((snapshotId) => !needed.has(snapshotId));
+    const snapshots = expirable.length === 0 ? [] : await runtime.query(maintenanceProcedureSql("ducklake_expire_snapshots", aliasSql, `versions => [${expirable.join(", ")}]`, dryRun));
+    const oldFiles = await runtime.query(maintenanceProcedureSql("ducklake_cleanup_old_files", aliasSql, `older_than => ${olderThanSql(retention.deleteOlderThan)}`, dryRun));
+    const orphanedFiles = await runtime.query(maintenanceProcedureSql("ducklake_delete_orphaned_files", aliasSql, `older_than => ${olderThanSql(retention.deleteOlderThan)}`, dryRun));
+    return {
+      snapshots: snapshots.length,
+      keptSnapshots: oldSnapshotIds.length - expirable.length,
+      oldFiles: oldFiles.length,
+      orphanedFiles: orphanedFiles.length
+    };
+  }
+  async snapshotsOlderThan(runtime, olderThan) {
+    const ducklakeSnapshot = duckLakeMetadataTableName(this.options, "ducklake_snapshot");
+    const rows = await runtime.query(`
+      SELECT snapshot_id
+      FROM ${ducklakeSnapshot}
+      WHERE CAST(snapshot_time AS TIMESTAMPTZ) < ${olderThanSql(olderThan)}
+        AND snapshot_id < (
+          SELECT snapshot_id FROM ${ducklakeSnapshot} ORDER BY snapshot_id DESC LIMIT 1
+        )
+      ORDER BY snapshot_id
+    `);
+    return rows.map((row) => String(getBigIntLike(row, "snapshot_id")));
+  }
 }
-async function runMaintenanceProcedures(runtime, options, retention, dryRun) {
-  const aliasSql = quoteSqlString(duckLakeAlias(options));
-  const snapshots = await runtime.query(maintenanceProcedureSql("ducklake_expire_snapshots", aliasSql, retention.expireOlderThan, dryRun));
-  const oldFiles = await runtime.query(maintenanceProcedureSql("ducklake_cleanup_old_files", aliasSql, retention.deleteOlderThan, dryRun));
-  const orphanedFiles = await runtime.query(maintenanceProcedureSql("ducklake_delete_orphaned_files", aliasSql, retention.deleteOlderThan, dryRun));
-  return {
-    snapshots: snapshots.length,
-    oldFiles: oldFiles.length,
-    orphanedFiles: orphanedFiles.length
-  };
+function protectedSnapshotId(ref) {
+  try {
+    return parseVersionId(ref.versionId);
+  } catch (error) {
+    throw new LakeStorageError17(`[SixbDuckLake] Cannot protect version '${ref.versionId}' of dataset '${ref.datasetId}' from maintenance: it is not a DuckLake version id.`, { cause: error });
+  }
+}
+function olderThanSql(olderThan) {
+  return `now() - INTERVAL ${quoteSqlString(olderThan)}`;
 }
-function maintenanceProcedureSql(procedure, aliasSql, olderThan, dryRun) {
-  const olderThanSql = quoteSqlString(olderThan);
+function maintenanceProcedureSql(procedure, aliasSql, selection, dryRun) {
   const dryRunSql = dryRun ? "dry_run => true, " : "";
-  return `CALL ${procedure}(${aliasSql}, ${dryRunSql}` + `older_than => now() - INTERVAL ${olderThanSql})`;
+  return `CALL ${procedure}(${aliasSql}, ${dryRunSql}${selection})`;
 }
 
 // src/internal/ducklake-row-reader.ts
-import { LakeStorageError as LakeStorageError16 } from "@sixb/core/lake-storage";
+import { LakeStorageError as LakeStorageError18 } from "@sixb/core/lake-storage";
 var READ_PAGE_ROWS = 5000;
 var PHYSICAL_ROW_ID_ALIAS = "__sixb_physical_row_id";
 
@@ -2220,7 +2514,7 @@ class DuckLakeRowReader {
         read.signal.throwIfAborted();
         const tableRef = await resolveDatasetTableRef(this.options, runtime, input.datasetId);
         if (!tableRef) {
-          throw new LakeStorageError16(`[SixbDuckLake] Unknown dataset '${input.datasetId}'.`);
+          throw new LakeStorageError18(`[SixbDuckLake] Unknown dataset '${input.datasetId}'.`);
         }
         const version = await this.resolveVersion(runtime, tableRef, input.versionId);
         const columns = this.resolveReadColumns(tableRef.datasetId, version.schema, input.columns);
@@ -2305,13 +2599,13 @@ class DuckLakeRowReader {
     return columns.map((columnName) => {
       const column = columnsByName.get(columnName);
       if (!column) {
-        throw new LakeStorageError16(`[SixbDuckLake] Dataset '${datasetId}' does not have column '${columnName}' at the requested version.`);
+        throw new LakeStorageError18(`[SixbDuckLake] Dataset '${datasetId}' does not have column '${columnName}' at the requested version.`);
       }
       return column;
     });
   }
   throwNoCommittedVersion(datasetId) {
-    throw new LakeStorageError16(`[SixbDuckLake] No committed version found for dataset '${datasetId}'.`);
+    throw new LakeStorageError18(`[SixbDuckLake] No committed version found for dataset '${datasetId}'.`);
   }
 }
 function pageSql(read, limit, offset, cursor) {
@@ -2321,9 +2615,10 @@ function pageSql(read, limit, offset, cursor) {
 }
 
 // src/internal/ducklake-snapshot-reader.ts
-import { LakeStorageError as LakeStorageError17 } from "@sixb/core/lake-storage";
+import { LakeStorageError as LakeStorageError19 } from "@sixb/core/lake-storage";
 var SNAPSHOT_ROW_BATCH_SIZE = 128;
 var FEED_PAGE_SIZE = 512;
+var SHORT_READ_BATCH_SIZE = 8;
 var CATALOG_SNAPSHOT_SCAN_LIMIT = 512;
 
 class DuckLakeSnapshotReader {
@@ -2375,10 +2670,10 @@ class DuckLakeSnapshotReader {
   }
   async collectDatasetCatalogState(runtime, datasetIds) {
     const uniqueIds = [...new Set(datasetIds)];
-    const tableIdsByDatasetId = await this.resolveDatasetTableIds(runtime, uniqueIds);
-    const latestRowByDatasetId = await this.resolveLatestSnapshotRows(runtime, tableIdsByDatasetId);
+    const tablesByDatasetId = await this.resolveDatasetTables(runtime, uniqueIds);
+    const latestRowByDatasetId = await this.resolveLatestSnapshotRows(runtime, tablesByDatasetId);
     return uniqueIds.map((datasetId) => {
-      if (!tableIdsByDatasetId.has(datasetId)) {
+      if (!tablesByDatasetId.has(datasetId)) {
         return { datasetId, materialized: false, latestVersion: null };
       }
       const row = latestRowByDatasetId.get(datasetId);
@@ -2417,7 +2712,7 @@ class DuckLakeSnapshotReader {
   async queryHeadSnapshotId(runtime) {
     const [row] = await runtime.query(buildDuckLakeMetadataQuery(this.options, (table) => `SELECT max(snapshot_id) AS snapshot_id FROM ${table("ducklake_snapshot")}`));
     if (row?.snapshot_id === null || row?.snapshot_id === undefined) {
-      throw new LakeStorageError17("[SixbDuckLake] DuckLake catalog has no snapshots.");
+      throw new LakeStorageError19("[SixbDuckLake] DuckLake catalog has no snapshots.");
     }
     return String(getBigIntLike(row, "snapshot_id"));
   }
@@ -2474,10 +2769,8 @@ class DuckLakeSnapshotReader {
           FROM ${table("ducklake_snapshot_changes")}
           WHERE snapshot_id >= ${first} AND snapshot_id <= ${last}
         `));
-    const fileFlags = await this.queryFileChangeFlags(runtime, snapshots.map((snapshot) => snapshot.snapshotId), tables.map((table) => table.tableId));
     const changesBySnapshotId = new Map(changeRows.map((row) => [String(getBigIntLike(row, "snapshot_id")), row]));
     const tablesByTableId = new Map(tables.map((table) => [table.tableId.toString(), table]));
-    const tablesByDatasetId = new Map(tables.map((table) => [table.datasetId, table]));
     for (const snapshot of snapshots) {
       const changes = changesBySnapshotId.get(snapshot.snapshotId);
       if (changes === undefined) {
@@ -2491,24 +2784,14 @@ class DuckLakeSnapshotReader {
         if (table)
           touched.add(table);
       }
-      for (const table of tables) {
-        if (fileFlags.has(fileChangeKey(snapshot.snapshotId, table.tableId)))
-          touched.add(table);
-      }
-      const tagged = metadata === undefined ? undefined : tablesByDatasetId.get(metadata.datasetId);
-      if (tagged)
-        touched.add(tagged);
       for (const table of touched) {
         if (BigInt(snapshot.snapshotId) < table.startSnapshotId) {
           continue;
         }
-        const flags = fileFlags.get(fileChangeKey(snapshot.snapshotId, table.tableId));
         const row = this.candidateToSnapshotRow(table.datasetId, table.tableId, {
           snapshotId: snapshot.snapshotId,
           createdAt: snapshot.createdAt,
           changesMade,
-          hasFileChange: flags?.hasFileChange ?? false,
-          hasFileDeleteChange: flags?.hasFileDeleteChange ?? false,
           ...metadata !== undefined ? { metadata } : {}
         });
         if (row && row.mode !== "schema") {
@@ -2518,6 +2801,22 @@ class DuckLakeSnapshotReader {
     }
     return latest;
   }
+  async latestVersionSnapshotIds(runtime) {
+    const tables = await runtime.query(`
+      SELECT table_name
+      FROM ${duckLakeMetadataTableName(this.options, "ducklake_table")}
+      WHERE end_snapshot IS NULL
+    `);
+    const datasetIds = tables.flatMap((table) => {
+      const datasetId = decodeDatasetTableName(getString(table, "table_name"));
+      return datasetId === null ? [] : [datasetId];
+    });
+    if (datasetIds.length === 0) {
+      return [];
+    }
+    const latest = await this.resolveLatestSnapshotRows(runtime, await this.resolveDatasetTables(runtime, datasetIds), Number.POSITIVE_INFINITY);
+    return [...latest.values()].map((row) => row.snapshotId);
+  }
   snapshotRowToSummary(datasetId, row) {
     return {
       datasetId,
@@ -2527,54 +2826,69 @@ class DuckLakeSnapshotReader {
       ...row.metadata?.rowCount !== undefined ? { rowCount: row.metadata.rowCount } : {}
     };
   }
-  async resolveDatasetTableIds(runtime, datasetIds) {
+  async resolveDatasetTables(runtime, datasetIds) {
     const datasetIdByTableName = new Map;
+    const datasetIdBySequenceTableName = new Map;
     for (const datasetId of datasetIds) {
       datasetIdByTableName.set(encodeDatasetTableName(datasetId), datasetId);
+      datasetIdBySequenceTableName.set(encodeDatasetSequenceTableName(datasetId), datasetId);
     }
     const ducklakeTable = duckLakeMetadataTableName(this.options, "ducklake_table");
-    const tableNameList = [...datasetIdByTableName.keys()].map((name) => quoteSqlString(name)).join(", ");
+    const tableNameList = [...datasetIdByTableName.keys(), ...datasetIdBySequenceTableName.keys()].map((name) => quoteSqlString(name)).join(", ");
     const rows = await runtime.query(`
       SELECT table_id, table_name
       FROM ${ducklakeTable}
       WHERE table_name IN (${tableNameList})
         AND end_snapshot IS NULL
     `);
-    const tableIdsByDatasetId = new Map;
+    const tablesByDatasetId = new Map;
     for (const row of rows) {
-      const datasetId = datasetIdByTableName.get(getString(row, "table_name"));
+      const tableName = getString(row, "table_name");
+      const datasetId = datasetIdByTableName.get(tableName);
       if (datasetId !== undefined) {
-        tableIdsByDatasetId.set(datasetId, getBigIntLike(row, "table_id"));
+        tablesByDatasetId.set(datasetId, {
+          datasetId,
+          tableName,
+          tableId: getBigIntLike(row, "table_id")
+        });
+      }
+    }
+    for (const row of rows) {
+      const datasetId = datasetIdBySequenceTableName.get(getString(row, "table_name"));
+      const table = datasetId === undefined ? undefined : tablesByDatasetId.get(datasetId);
+      if (datasetId !== undefined && table !== undefined) {
+        tablesByDatasetId.set(datasetId, {
+          ...table,
+          sequenceTableId: getBigIntLike(row, "table_id")
+        });
       }
     }
-    return tableIdsByDatasetId;
+    return tablesByDatasetId;
   }
-  async resolveLatestSnapshotRows(runtime, tableIdsByDatasetId) {
+  async resolveLatestSnapshotRows(runtime, tablesByDatasetId, scanLimit = CATALOG_SNAPSHOT_SCAN_LIMIT) {
     const result = new Map;
-    if (tableIdsByDatasetId.size === 0) {
+    if (tablesByDatasetId.size === 0) {
       return result;
     }
-    const unresolved = new Map(tableIdsByDatasetId);
-    const tableIds = [...new Set(tableIdsByDatasetId.values())];
+    const unresolved = new Map(tablesByDatasetId);
     let beforeSnapshotId;
     let scanned = 0;
-    while (unresolved.size > 0 && scanned < CATALOG_SNAPSHOT_SCAN_LIMIT) {
+    while (unresolved.size > 0 && scanned < scanLimit) {
       const candidates = await this.queryCatalogSnapshotBatch(runtime, beforeSnapshotId, SNAPSHOT_ROW_BATCH_SIZE);
       if (candidates.length === 0) {
         break;
       }
-      const fileFlags = await this.queryFileChangeFlags(runtime, candidates.map((candidate) => candidate.snapshotId), tableIds);
+      const tables = [...unresolved.values()];
+      const metadata = await this.readCommitMetadata(runtime, candidates.filter((candidate) => tables.some((table) => changesTouchTables(candidate.changesMade, datasetTableIds(table)))).map((candidate) => candidate.snapshotId));
       for (const candidate of candidates) {
         scanned += 1;
-        for (const [datasetId, tableId] of [...unresolved]) {
-          const flags = fileFlags.get(fileChangeKey(candidate.snapshotId, tableId));
-          const row = this.candidateToSnapshotRow(datasetId, tableId, {
+        const candidateMetadata = metadata.get(candidate.snapshotId);
+        for (const [datasetId, table] of [...unresolved]) {
+          const row = this.candidateToSnapshotRow(datasetId, table.tableId, {
             snapshotId: candidate.snapshotId,
             createdAt: candidate.createdAt,
             changesMade: candidate.changesMade,
-            hasFileChange: flags?.hasFileChange ?? false,
-            hasFileDeleteChange: flags?.hasFileDeleteChange ?? false,
-            ...candidate.metadata !== undefined ? { metadata: candidate.metadata } : {}
+            ...candidateMetadata !== undefined ? { metadata: candidateMetadata } : {}
           });
           if (row) {
             result.set(datasetId, row);
@@ -2604,67 +2918,22 @@ class DuckLakeSnapshotReader {
       SELECT
         snapshot.snapshot_id,
         snapshot.snapshot_time,
-        changes.changes_made,
-        changes.commit_extra_info
+        changes.changes_made
       FROM ${ducklakeSnapshot} snapshot
       JOIN ${ducklakeSnapshotChanges} changes ON changes.snapshot_id = snapshot.snapshot_id
       ${whereSql}
       ORDER BY snapshot.snapshot_id DESC
       LIMIT ${Math.max(0, Math.trunc(limit))}
     `);
-    return rows.map((row) => {
-      const metadata = parseCommitMetadata(getOptionalString(row, "commit_extra_info"));
-      return {
-        snapshotId: String(getBigIntLike(row, "snapshot_id")),
-        createdAt: getDate(row, "snapshot_time"),
-        changesMade: getString(row, "changes_made"),
-        ...metadata !== undefined ? { metadata } : {}
-      };
-    });
-  }
-  async queryFileChangeFlags(runtime, snapshotIds, tableIds) {
-    const flags = new Map;
-    if (snapshotIds.length === 0 || tableIds.length === 0) {
-      return flags;
-    }
-    for (const snapshotId of snapshotIds) {
-      assertDuckLakeSnapshotId2(snapshotId);
-    }
-    const snapshotIdList = snapshotIds.join(", ");
-    const tableIdList = tableIds.map((tableId) => tableId.toString()).join(", ");
-    const ducklakeDataFile = duckLakeMetadataTableName(this.options, "ducklake_data_file");
-    const ducklakeDeleteFile = duckLakeMetadataTableName(this.options, "ducklake_delete_file");
-    const rows = await runtime.query(`
-      WITH file_changes AS (
-        SELECT begin_snapshot AS snapshot_id, table_id, false AS is_delete
-        FROM ${ducklakeDataFile}
-        WHERE table_id IN (${tableIdList}) AND begin_snapshot IN (${snapshotIdList})
-        UNION ALL
-        SELECT end_snapshot, table_id, true
-        FROM ${ducklakeDataFile}
-        WHERE table_id IN (${tableIdList}) AND end_snapshot IN (${snapshotIdList})
-        UNION ALL
-        SELECT begin_snapshot, table_id, true
-        FROM ${ducklakeDeleteFile}
-        WHERE table_id IN (${tableIdList}) AND begin_snapshot IN (${snapshotIdList})
-        UNION ALL
-        SELECT end_snapshot, table_id, true
-        FROM ${ducklakeDeleteFile}
-        WHERE table_id IN (${tableIdList}) AND end_snapshot IN (${snapshotIdList})
-      )
-      SELECT snapshot_id, table_id, bool_or(is_delete) AS has_delete
-      FROM file_changes
-      GROUP BY snapshot_id, table_id
-    `);
-    for (const row of rows) {
-      const key = fileChangeKey(String(getBigIntLike(row, "snapshot_id")), getBigIntLike(row, "table_id"));
-      flags.set(key, { hasFileChange: true, hasFileDeleteChange: getBoolean(row, "has_delete") });
-    }
-    return flags;
+    return rows.map((row) => ({
+      snapshotId: String(getBigIntLike(row, "snapshot_id")),
+      createdAt: getDate(row, "snapshot_time"),
+      changesMade: getString(row, "changes_made")
+    }));
   }
   assertNoMetadataConflict(snapshotId, datasetId, metadata) {
     if (metadata !== undefined && metadata.datasetId !== datasetId) {
-      throw new LakeStorageError17(`[SixbDuckLake] DuckLake snapshot '${snapshotId}' changed dataset '${datasetId}' but Sixb commit metadata references dataset '${metadata.datasetId}'.`);
+      throw new LakeStorageError19(`[SixbDuckLake] DuckLake snapshot '${snapshotId}' changed dataset '${datasetId}' but Sixb commit metadata references dataset '${metadata.datasetId}'.`);
     }
   }
   async getLatestVersionForDefinition(runtime, dataset) {
@@ -2672,22 +2941,39 @@ class DuckLakeSnapshotReader {
     return tableRef ? this.getLatestVersionForTableRef(runtime, tableRef) : null;
   }
   async getLatestVersionRefForDefinition(runtime, dataset) {
-    const summary = await this.getLatestVersionSummaryForDefinition(runtime, dataset);
-    return summary ? { datasetId: summary.datasetId, versionId: summary.versionId } : null;
-  }
-  async getLatestVersionSummaryForDefinition(runtime, dataset) {
     const tableRef = await resolveDatasetTableRef(this.options, runtime, dataset.id);
     if (!tableRef) {
       return null;
     }
+    const row = await this.getLatestSnapshotRowForTableRef(runtime, tableRef);
+    return row ? tableVersionRef(tableRef, row) : null;
+  }
+  async getLatestVersionSummaryForDefinition(runtime, dataset) {
+    const tableRef = await resolveDatasetTableRef(this.options, runtime, dataset.id);
+    return tableRef ? this.getLatestVersionSummaryForTableRef(runtime, tableRef) : null;
+  }
+  async getLatestVersionSummaryAtCurrentSnapshot(runtime, dataset) {
+    const { snapshotId, table } = await resolveDatasetTableRefAtCurrentSnapshot(this.options, runtime, dataset.id);
+    return {
+      snapshotId,
+      latestVersion: table ? await this.getLatestVersionSummaryForTableRef(runtime, table) : null
+    };
+  }
+  async getLatestVersionSummaryForTableRef(runtime, tableRef) {
     const row = await this.getLatestSnapshotRowForTableRef(runtime, tableRef);
     if (!row) {
       return null;
     }
+    const [schema] = await this.datasets.getDatasetSchemasAtSnapshots(runtime, [
+      { table: tableRef, snapshotId: row.snapshotId }
+    ]);
     return {
       datasetId: tableRef.datasetId,
       versionId: toVersionId(row.snapshotId),
+      schema,
+      producer: row.metadata?.producer,
       inputs: row.metadata?.inputs,
+      ...storedLineage(tableRef.datasetId, row),
       ...row.metadata?.rowCount !== undefined ? { rowCount: row.metadata.rowCount } : {},
       ...row.metadata?.validatedPrimaryKeyColumns !== undefined ? { validatedPrimaryKeyColumns: row.metadata.validatedPrimaryKeyColumns } : {}
     };
@@ -2701,7 +2987,7 @@ class DuckLakeSnapshotReader {
     const match = await this.getSnapshotRowForTableRef(runtime, tableRef, snapshotId, {
       includeParent: true
     });
-    return match ? this.snapshotToVersion(runtime, tableRef, match) : null;
+    return match ? (await this.snapshotsToVersions(runtime, tableRef, [match]))[0] : null;
   }
   async getVersionRefForSnapshot(runtime, dataset, snapshotId) {
     assertDuckLakeSnapshotId2(snapshotId);
@@ -2712,7 +2998,11 @@ class DuckLakeSnapshotReader {
     const row = await this.getSnapshotRowForTableRef(runtime, tableRef, snapshotId, {
       includeParent: false
     });
-    return row ? { datasetId: tableRef.datasetId, versionId: toVersionId(row.snapshotId) } : null;
+    return row ? tableVersionRef(tableRef, row) : null;
+  }
+  async getVersionsForRefs(runtime, refs) {
+    const schemas = await this.datasets.getDatasetSchemasAtSnapshots(runtime, refs.map((ref) => ({ table: ref.table, snapshotId: ref.snapshot.snapshotId })));
+    return refs.map((ref, index) => snapshotToVersion(ref.table, ref.snapshot, schemas[index]));
   }
   async listVersionsForDefinition(runtime, dataset, limit) {
     const tableRef = await resolveDatasetTableRef(this.options, runtime, dataset.id);
@@ -2720,11 +3010,7 @@ class DuckLakeSnapshotReader {
   }
   async listVersionsForTableRef(runtime, tableRef, limit) {
     const limitedRows = await this.getSnapshotRowsForTableRef(runtime, tableRef, limit);
-    const versions = [];
-    for (const row of limitedRows) {
-      versions.push(await this.snapshotToVersion(runtime, tableRef, row));
-    }
-    return versions;
+    return this.snapshotsToVersions(runtime, tableRef, limitedRows);
   }
   async getLatestVersionForTableRef(runtime, tableRef) {
     const [latest] = await this.listVersionsForTableRef(runtime, tableRef, 1);
@@ -2732,8 +3018,7 @@ class DuckLakeSnapshotReader {
   }
   async getLatestSnapshotRowForTableRef(runtime, tableRef) {
     const [row] = await this.collectVisibleSnapshotRows(runtime, {
-      datasetId: tableRef.datasetId,
-      tableId: tableRef.tableId,
+      table: tableRef,
       visibleRowLimit: 1
     });
     return row ?? null;
@@ -2741,8 +3026,7 @@ class DuckLakeSnapshotReader {
   async getSnapshotRowForTableRef(runtime, tableRef, snapshotId, options) {
     assertDuckLakeSnapshotId2(snapshotId);
     const [row] = await this.collectVisibleSnapshotRows(runtime, {
-      datasetId: tableRef.datasetId,
-      tableId: tableRef.tableId,
+      table: tableRef,
       exactSnapshotId: snapshotId,
       visibleRowLimit: 1
     });
@@ -2753,8 +3037,7 @@ class DuckLakeSnapshotReader {
       return row;
     }
     const [parent] = await this.collectVisibleSnapshotRows(runtime, {
-      datasetId: tableRef.datasetId,
-      tableId: tableRef.tableId,
+      table: tableRef,
       beforeSnapshotId: snapshotId,
       visibleRowLimit: 1
     });
@@ -2766,8 +3049,7 @@ class DuckLakeSnapshotReader {
       return [];
     }
     const rows = await this.collectVisibleSnapshotRows(runtime, {
-      datasetId: tableRef.datasetId,
-      tableId: tableRef.tableId,
+      table: tableRef,
       visibleRowLimit: visibleLimit === undefined ? undefined : visibleLimit + 1
     });
     const rowsWithParents = this.withParentSnapshotIds(rows);
@@ -2779,22 +3061,21 @@ class DuckLakeSnapshotReader {
     }
     const snapshots = [];
     let beforeSnapshotId = input.beforeSnapshotId;
+    let shortRead = input.exactSnapshotId !== undefined || input.visibleRowLimit !== undefined && input.visibleRowLimit <= 2;
     while (input.visibleRowLimit === undefined || snapshots.length < input.visibleRowLimit) {
+      const batchSize = shortRead ? SHORT_READ_BATCH_SIZE : SNAPSHOT_ROW_BATCH_SIZE;
       const candidates = await this.querySnapshotCandidates(runtime, {
-        datasetId: input.datasetId,
-        tableId: input.tableId,
+        table: input.table,
         exactSnapshotId: input.exactSnapshotId,
         beforeSnapshotId,
-        limit: input.exactSnapshotId === undefined ? SNAPSHOT_ROW_BATCH_SIZE : 1
+        limit: input.exactSnapshotId === undefined ? batchSize : 1
       });
+      shortRead = false;
       if (candidates.length === 0) {
         break;
       }
       for (const candidate of candidates) {
-        if ("missing" in candidate) {
-          continue;
-        }
-        const snapshot = this.candidateToSnapshotRow(input.datasetId, input.tableId, candidate);
+        const snapshot = this.candidateToSnapshotRow(input.table.datasetId, input.table.tableId, candidate);
         if (snapshot) {
           snapshots.push(snapshot);
         }
@@ -2802,7 +3083,7 @@ class DuckLakeSnapshotReader {
           break;
         }
       }
-      if (input.exactSnapshotId !== undefined || candidates.length < SNAPSHOT_ROW_BATCH_SIZE) {
+      if (input.exactSnapshotId !== undefined || candidates.length < batchSize) {
         break;
       }
       beforeSnapshotId = candidates[candidates.length - 1]?.snapshotId;
@@ -2810,7 +3091,6 @@ class DuckLakeSnapshotReader {
     return snapshots;
   }
   async querySnapshotCandidates(runtime, input) {
-    const tableId = input.tableId.toString();
     const limit = Math.max(0, Math.trunc(input.limit));
     if (limit === 0) {
       return [];
@@ -2822,118 +3102,69 @@ class DuckLakeSnapshotReader {
       assertDuckLakeSnapshotId2(input.beforeSnapshotId);
     }
     const rows = await runtime.query(buildDuckLakeMetadataQuery(this.options, (table) => {
-      const tableStart = `(SELECT min(begin_snapshot) FROM ${table("ducklake_table")} WHERE table_id = ${tableId})`;
-      const candidateIds = input.exactSnapshotId !== undefined ? `SELECT CAST(${input.exactSnapshotId} AS BIGINT) AS snapshot_id
-               WHERE ${input.exactSnapshotId} >= ${tableStart}` : `SELECT DISTINCT snapshot_id
-               FROM (
-                 SELECT begin_snapshot AS snapshot_id
-                 FROM ${table("ducklake_data_file")} WHERE table_id = ${tableId}
-                 UNION ALL
-                 SELECT end_snapshot
-                 FROM ${table("ducklake_data_file")}
-                 WHERE table_id = ${tableId} AND end_snapshot IS NOT NULL
-                 UNION ALL
-                 SELECT begin_snapshot
-                 FROM ${table("ducklake_delete_file")} WHERE table_id = ${tableId}
-                 UNION ALL
-                 SELECT end_snapshot
-                 FROM ${table("ducklake_delete_file")}
-                 WHERE table_id = ${tableId} AND end_snapshot IS NOT NULL
-                 UNION ALL
-                 SELECT snapshot_id
-                 FROM ${table("ducklake_snapshot_changes")}
-                 WHERE strpos(',' || changes_made || ',', ${quoteSqlString(`:${tableId},`)}) > 0
-                   OR strpos(
-                     commit_extra_info,
-                     ${quoteSqlString(`"datasetId":${JSON.stringify(input.datasetId)}`)}
-                   ) > 0
-               ) touched
-               WHERE snapshot_id >= ${tableStart}
-                 ${input.beforeSnapshotId === undefined ? "" : `AND snapshot_id < ${input.beforeSnapshotId}`}
-               ORDER BY snapshot_id DESC
-               LIMIT ${limit}`;
+      const bounds = [
+        `changes.snapshot_id >= (
+            SELECT min(begin_snapshot)
+            FROM ${table("ducklake_table")}
+            WHERE table_id = ${input.table.tableId}
+          )`
+      ];
+      if (input.exactSnapshotId !== undefined) {
+        bounds.push(`changes.snapshot_id = ${input.exactSnapshotId}`);
+      }
+      if (input.beforeSnapshotId !== undefined) {
+        bounds.push(`changes.snapshot_id < ${input.beforeSnapshotId}`);
+      }
+      const namesDatasetTable = datasetTableIds(input.table).map((tableId) => `strpos(',' || changes.changes_made || ',', ${quoteSqlString(`:${tableId},`)}) > 0`).join(" OR ");
       return `
-          WITH candidate_ids AS (
-            ${candidateIds}
-          ),
-          candidate_snapshots AS (
-            SELECT
-              snapshot.snapshot_id,
-              snapshot.snapshot_time,
-              changes.changes_made,
-              changes.commit_extra_info
-            FROM ${table("ducklake_snapshot")} snapshot
-            JOIN ${table("ducklake_snapshot_changes")} changes
-              ON changes.snapshot_id = snapshot.snapshot_id
-            WHERE snapshot.snapshot_id IN (SELECT snapshot_id FROM candidate_ids)
-          ),
-          file_changes AS (
-            -- File metadata tells whether this table changed in each candidate snapshot.
-            SELECT begin_snapshot AS snapshot_id, false AS is_delete_change
-            FROM ${table("ducklake_data_file")}
-            WHERE table_id = ${tableId}
-              AND begin_snapshot IN (SELECT snapshot_id FROM candidate_ids)
-            UNION ALL
-            SELECT end_snapshot AS snapshot_id, true AS is_delete_change
-            FROM ${table("ducklake_data_file")}
-            WHERE table_id = ${tableId}
-              AND end_snapshot IN (SELECT snapshot_id FROM candidate_ids)
-            UNION ALL
-            SELECT begin_snapshot AS snapshot_id, true AS is_delete_change
-            FROM ${table("ducklake_delete_file")}
-            WHERE table_id = ${tableId}
-              AND begin_snapshot IN (SELECT snapshot_id FROM candidate_ids)
-            UNION ALL
-            SELECT end_snapshot AS snapshot_id, true AS is_delete_change
-            FROM ${table("ducklake_delete_file")}
-            WHERE table_id = ${tableId}
-              AND end_snapshot IN (SELECT snapshot_id FROM candidate_ids)
-          ),
-          file_changes_by_snapshot AS (
-            -- Collapse file-level changes into one row per snapshot.
-            SELECT
-              snapshot_id,
-              count(*) > 0 AS has_file_change,
-              count(*) FILTER (WHERE is_delete_change) > 0 AS has_file_delete_change
-            FROM file_changes
-            GROUP BY snapshot_id
-          )
           SELECT
-            -- Keep metadata-only candidates; Sixb filters them with commit_extra_info.
-            candidate_ids.snapshot_id,
-            candidate.snapshot_time,
-            candidate.changes_made,
-            candidate.commit_extra_info,
-            coalesce(file_changes.has_file_change, false) AS has_file_change,
-            coalesce(file_changes.has_file_delete_change, false) AS has_file_delete_change
-          FROM candidate_ids
-          LEFT JOIN candidate_snapshots candidate
-            ON candidate.snapshot_id = candidate_ids.snapshot_id
-          LEFT JOIN file_changes_by_snapshot file_changes
-            ON file_changes.snapshot_id = candidate_ids.snapshot_id
-          ORDER BY candidate_ids.snapshot_id DESC
+            changes.snapshot_id,
+            snapshot.snapshot_time,
+            changes.changes_made,
+            changes.commit_extra_info
+          FROM ${table("ducklake_snapshot_changes")} changes
+          JOIN ${table("ducklake_snapshot")} snapshot
+            ON snapshot.snapshot_id = changes.snapshot_id
+          WHERE ${bounds.join(" AND ")}
+            AND (${namesDatasetTable})
+          ORDER BY changes.snapshot_id DESC
+          LIMIT ${limit}
         `;
     }));
     return rows.map((row) => {
-      const snapshotId = String(getBigIntLike(row, "snapshot_id"));
-      if (row.snapshot_time === null || row.snapshot_time === undefined) {
-        return { snapshotId, missing: true };
-      }
       const metadata = parseCommitMetadata(row.commit_extra_info);
       return {
-        snapshotId,
+        snapshotId: String(getBigIntLike(row, "snapshot_id")),
         createdAt: getDate(row, "snapshot_time"),
         changesMade: getString(row, "changes_made"),
-        hasFileChange: getBoolean(row, "has_file_change"),
-        hasFileDeleteChange: getBoolean(row, "has_file_delete_change"),
         ...metadata !== undefined ? { metadata } : {}
       };
     });
   }
+  async readCommitMetadata(runtime, snapshotIds) {
+    const metadata = new Map;
+    if (snapshotIds.length === 0) {
+      return metadata;
+    }
+    for (const snapshotId of snapshotIds) {
+      assertDuckLakeSnapshotId2(snapshotId);
+    }
+    const ducklakeSnapshotChanges = duckLakeMetadataTableName(this.options, "ducklake_snapshot_changes");
+    const rows = await runtime.query(`
+      SELECT snapshot_id, commit_extra_info
+      FROM ${ducklakeSnapshotChanges}
+      WHERE snapshot_id IN (${snapshotIds.join(", ")})
+    `);
+    for (const row of rows) {
+      const parsed = parseCommitMetadata(getOptionalString(row, "commit_extra_info"));
+      if (parsed !== undefined) {
+        metadata.set(String(getBigIntLike(row, "snapshot_id")), parsed);
+      }
+    }
+    return metadata;
+  }
   candidateToSnapshotRow(datasetId, tableId, candidate) {
-    const inlineChange = parseInlineDataChange(candidate.changesMade, tableId);
-    const hasDataChange = inlineChange.hasDataChange || candidate.hasFileChange;
-    const hasDeleteChange = inlineChange.hasDeleteChange || candidate.hasFileDeleteChange;
+    const { hasDataChange, hasDeleteChange } = parseTableDataChange(candidate.changesMade, tableId);
     if (!hasDataChange) {
       const metadata = candidate.metadata;
       if (!metadata || metadata.datasetId !== datasetId) {
@@ -2956,19 +3187,9 @@ class DuckLakeSnapshotReader {
       ...candidate.metadata !== undefined ? { metadata: candidate.metadata } : {}
     };
   }
-  async snapshotToVersion(runtime, tableRef, row) {
-    const schemaAtSnapshot = await this.datasets.getDatasetSchemaAtSnapshot(runtime, tableRef.datasetId, tableRef.tableName, tableRef.tableId, row.snapshotId);
-    return {
-      datasetId: tableRef.datasetId,
-      versionId: toVersionId(row.snapshotId),
-      parentVersionId: row.parentSnapshotId === undefined ? undefined : toVersionId(row.parentSnapshotId),
-      mode: row.mode,
-      createdAt: row.createdAt,
-      schema: schemaAtSnapshot,
-      producer: row.metadata?.producer,
-      inputs: row.metadata?.inputs,
-      ...row.metadata?.rowCount !== undefined ? { rowCount: row.metadata.rowCount } : {}
-    };
+  async snapshotsToVersions(runtime, tableRef, rows) {
+    const schemas = await this.datasets.getDatasetSchemasAtSnapshots(runtime, rows.map((row) => ({ table: tableRef, snapshotId: row.snapshotId })));
+    return rows.map((row, index) => snapshotToVersion(tableRef, row, schemas[index]));
   }
   withParentSnapshotIds(snapshots) {
     return snapshots.map((snapshot, index) => {
@@ -2977,29 +3198,60 @@ class DuckLakeSnapshotReader {
     });
   }
 }
+function snapshotToVersion(tableRef, row, schema) {
+  return {
+    datasetId: tableRef.datasetId,
+    versionId: toVersionId(row.snapshotId),
+    parentVersionId: row.parentSnapshotId === undefined ? undefined : toVersionId(row.parentSnapshotId),
+    mode: row.mode,
+    createdAt: row.createdAt,
+    schema,
+    producer: row.metadata?.producer,
+    inputs: row.metadata?.inputs,
+    ...storedLineage(tableRef.datasetId, row),
+    ...row.metadata?.rowCount !== undefined ? { rowCount: row.metadata.rowCount } : {}
+  };
+}
+function storedLineage(datasetId, row) {
+  const metadata = row.metadata;
+  if (metadata === undefined) {
+    return {
+      lineage: {
+        unknown: `version '${toVersionId(row.snapshotId)}' of '${datasetId}' was written outside Sixb`
+      }
+    };
+  }
+  return metadata.lineage === undefined ? {} : { lineage: decodeLineage(metadata.datasetId, metadata.lineage) };
+}
+function tableVersionRef(table, snapshot) {
+  return {
+    datasetId: table.datasetId,
+    versionId: toVersionId(snapshot.snapshotId),
+    table,
+    snapshot
+  };
+}
 function assertDuckLakeSnapshotId2(snapshotId) {
   if (!/^\d+$/.test(snapshotId)) {
-    throw new LakeStorageError17(`[SixbDuckLake] Invalid DuckLake snapshot id '${snapshotId}'.`);
+    throw new LakeStorageError19(`[SixbDuckLake] Invalid DuckLake snapshot id '${snapshotId}'.`);
   }
 }
-function fileChangeKey(snapshotId, tableId) {
-  return `${snapshotId}|${tableId}`;
-}
 
 // src/internal/ducklake-sql-executor.ts
-import { randomUUID as randomUUID4 } from "crypto";
+import { randomUUID as randomUUID5 } from "crypto";
 import {
-  LakeStorageError as LakeStorageError20
+  LakeStorageError as LakeStorageError22,
+  resolveStepLineage
 } from "@sixb/core/lake-storage";
 
 // src/internal/dataset-row-commit.ts
 import {
   getDatasetPrimaryKeyColumns,
-  LakeStorageError as LakeStorageError18
+  LakeStorageError as LakeStorageError20
 } from "@sixb/core/lake-storage";
 function assertDatasetWriteMode(mode, operation) {
   if (mode !== "snapshot" && mode !== "append") {
-    throw new LakeStorageError18(`[SixbDuckLake] Invalid ${operation} mode '${String(mode)}'. Expected 'snapshot' or 'append'.`);
+    throw new LakeStorageError20(`[SixbDuckLake] Invalid ${operation} mode '${String(mode)}'. Expected 'snapshot' or 'append'.`);
   }
 }
 async function applyDatasetRowsFromRelation(input) {
@@ -3007,6 +3259,7 @@ async function applyDatasetRowsFromRelation(input) {
   const table = qualifiedTableName(input.options, tableName);
   const columnsSql = datasetSchemaColumnNamesSql(input.dataset.schema);
   const primaryKeyColumns = getDatasetPrimaryKeyColumns(input.dataset);
+  let baselineRowCount;
   if (primaryKeyColumns !== null) {
     await inspectUniqueKeyedRelation({
       runtime: input.runtime,
@@ -3015,7 +3268,7 @@ async function applyDatasetRowsFromRelation(input) {
       context: `${input.mode} source`
     });
     if (input.mode === "append") {
-      await inspectUniqueKeyedRelation({
+      baselineRowCount = await inspectUniqueKeyedRelation({
         runtime: input.runtime,
         dataset: input.dataset,
         relationSql: table,
@@ -3037,21 +3290,30 @@ async function applyDatasetRowsFromRelation(input) {
       return {
         dataChangeExpected: false,
         sourceRowCount: source.rowCount,
-        resultingRowCount: { kind: "exact", value: target.rowCount }
+        resultingRowCount: { kind: "exact", value: target.rowCount },
+        targetWasEmpty: target.rowCount === 0,
+        changedTableRows: false
       };
     }
     await input.runtime.run(`DELETE FROM ${table}`);
     await input.runtime.run(`INSERT INTO ${table} (${columnsSql}) SELECT ${columnsSql} FROM ${input.sourceRelationSql}`);
+    const changedTableRows = source.rowCount > 0 || target.rowCount > 0;
     return {
-      dataChangeExpected: source.rowCount > 0 || target.rowCount > 0,
+      dataChangeExpected: changedTableRows,
       sourceRowCount: source.rowCount,
-      resultingRowCount: { kind: "exact", value: source.rowCount }
+      resultingRowCount: { kind: "exact", value: source.rowCount },
+      targetWasEmpty: target.rowCount === 0,
+      changedTableRows
     };
   }
   const sourceRowCount = await countRows(input.runtime, input.sourceRelationSql);
+  const knownRowCount = baselineRowCount ?? input.previousRowCount;
+  const targetWasEmpty = knownRowCount === undefined ? await isEmptyRelation(input.runtime, table) : knownRowCount === 0;
   await input.runtime.run(`INSERT INTO ${table} (${columnsSql}) SELECT ${columnsSql} FROM ${input.sourceRelationSql}`);
   return {
     dataChangeExpected: sourceRowCount > 0,
+    targetWasEmpty,
+    changedTableRows: sourceRowCount > 0,
     sourceRowCount,
     resultingRowCount: input.previousRowCount === undefined ? { kind: "unknown" } : { kind: "exact", value: input.previousRowCount + sourceRowCount }
   };
@@ -3059,7 +3321,7 @@ async function applyDatasetRowsFromRelation(input) {
 async function inspectUniqueKeyedRelation(input) {
   const primaryKeyColumns = getDatasetPrimaryKeyColumns(input.dataset);
   if (primaryKeyColumns === null) {
-    throw new LakeStorageError18(`[SixbDuckLake] Dataset '${input.dataset.id}' must define a primaryKey before rows can be keyed.`);
+    throw new LakeStorageError20(`[SixbDuckLake] Dataset '${input.dataset.id}' must define a primaryKey before rows can be keyed.`);
   }
   const keysSql = primaryKeyColumns.map((column) => quoteIdentifier(column)).join(", ");
   const [row] = await input.runtime.query(`
@@ -3074,7 +3336,7 @@ async function inspectUniqueKeyedRelation(input) {
   const rowCount = row === undefined ? 0 : Number(getBigIntLike(row, "row_count"));
   const maxKeyCount = row === undefined ? 0 : Number(getBigIntLike(row, "max_key_count"));
   if (maxKeyCount > 1) {
-    throw new LakeStorageError18(`[SixbDuckLake] Dataset '${input.dataset.id}' ${input.context} contains duplicate primary key.`);
+    throw new LakeStorageError20(`[SixbDuckLake] Dataset '${input.dataset.id}' ${input.context} contains duplicate primary key.`);
   }
   return rowCount;
 }
@@ -3091,7 +3353,7 @@ async function assertNoPrimaryKeyOverlap(input) {
     LIMIT 1
   `);
   if (collision !== undefined) {
-    throw new LakeStorageError18(`[SixbDuckLake] Dataset '${input.dataset.id}' ${input.context} contains duplicate primary key.`);
+    throw new LakeStorageError20(`[SixbDuckLake] Dataset '${input.dataset.id}' ${input.context} contains duplicate primary key.`);
   }
 }
 async function fingerprintRelation(runtime, relationSql, columnsSql) {
@@ -3121,11 +3383,19 @@ async function countRows(runtime, relationSql) {
   }
   return Number(getBigIntLike(row, "row_count"));
 }
+async function isEmptyRelation(runtime, relationSql) {
+  const rows = await runtime.query(`SELECT 1 AS present FROM ${relationSql} LIMIT 1`);
+  return rows.length === 0;
+}
+
+// src/internal/sql-lineage/analysis.ts
+import { randomUUID as randomUUID4 } from "crypto";
+import { DuckDBInstance as DuckDBInstance2 } from "@duckdb/node-api";
 
 // src/internal/sql-transform-relations.ts
 import { randomUUID as randomUUID3 } from "crypto";
 import {
-  LakeStorageError as LakeStorageError19
+  LakeStorageError as LakeStorageError21
 } from "@sixb/core/lake-storage";
 var RELATION_PLACEHOLDER_PREFIX = "__sixb_sql_transform_relation_";
 var RELATION_PLACEHOLDER_PATTERN = /__sixb_sql_transform_relation_[0-9a-f]{32}_[0-9]+__/g;
@@ -3139,34 +3409,28 @@ class DuckLakeSqlTransformRelation {
     return this.placeholder;
   }
 }
-function renderDuckLakeSqlTransformSql(input) {
-  const context = createDuckLakeSqlTransformRelationContext(input);
-  const sql = input.sql(context.relations);
-  if (typeof sql !== "string") {
-    throw new LakeStorageError19("[SixbDuckLake] SQL transform body must return a SQL string.");
-  }
-  return context.render(sql);
-}
-function createDuckLakeSqlTransformRelationContext(input) {
+function createDuckLakeSqlTransformTemplate(sourceNames, body) {
   const token = randomUUID3().replaceAll("-", "");
   const relations = Object.create(null);
-  const replacements = new Map;
-  let index = 0;
-  for (const [sourceName, source] of Object.entries(input.sources)) {
+  const sourceNamesByPlaceholder = new Map;
+  sourceNames.forEach((sourceName, index) => {
     const placeholder = `${RELATION_PLACEHOLDER_PREFIX}${token}_${index}__`;
     relations[sourceName] = Object.freeze(new DuckLakeSqlTransformRelation(placeholder));
-    replacements.set(placeholder, renderDuckLakeSourceRelation(input.options, source));
-    index += 1;
+    sourceNamesByPlaceholder.set(placeholder, sourceName);
+  });
+  const sql = body(Object.freeze(relations));
+  if (typeof sql !== "string") {
+    throw new LakeStorageError21("[SixbDuckLake] SQL transform body must return a SQL string.");
   }
   return {
-    relations: Object.freeze(relations),
-    render(sql) {
+    render(relationSql) {
       const rendered = sql.replace(RELATION_PLACEHOLDER_PATTERN, (placeholder) => {
-        const replacement = replacements.get(placeholder);
-        if (replacement === undefined) {
+        const sourceName = sourceNamesByPlaceholder.get(placeholder);
+        const relation = sourceName === undefined ? undefined : relationSql.get(sourceName);
+        if (relation === undefined) {
           throwUnresolvedPlaceholder();
         }
-        return replacement;
+        return relation;
       });
       if (rendered.includes(RELATION_PLACEHOLDER_PREFIX)) {
         throwUnresolvedPlaceholder();
@@ -3175,13 +3439,967 @@ function createDuckLakeSqlTransformRelationContext(input) {
     }
   };
 }
-function renderDuckLakeSourceRelation(options, source) {
-  const snapshotId = parseVersionId(source.versionId);
-  const tableName = encodeDatasetTableName(source.datasetId);
-  return `(SELECT * FROM ${qualifiedTableName(options, tableName)} AT (VERSION => ${snapshotId}))`;
+function renderDuckLakeSourceRelations(options, sources) {
+  return new Map(Object.entries(sources).map(([sourceName, source]) => {
+    const snapshotId = parseVersionId(source.versionId);
+    const table = qualifiedTableName(options, encodeDatasetTableName(source.datasetId));
+    return [
+      sourceName,
+      renderSourceRelation(`${table} AT (VERSION => ${snapshotId})`, source.columns)
+    ];
+  }));
+}
+function renderSourceRelation(tableSql, columns) {
+  const columnsSql = columns === undefined ? "*" : columns.map((column) => quoteIdentifier(column)).join(", ");
+  return `(SELECT ${columnsSql} FROM ${tableSql})`;
+}
+function pickSourceColumns(sourceName, dataset, columns) {
+  if (columns === undefined) {
+    return dataset.schema.columns;
+  }
+  if (columns.length === 0) {
+    throw new LakeStorageError21(`[SixbDuckLake] SQL transform source '${sourceName}' must pick at least one column.`);
+  }
+  const columnsByName = new Map(dataset.schema.columns.map((column) => [column.name, column]));
+  const picked = new Set;
+  return columns.map((columnName) => {
+    const column = columnsByName.get(columnName);
+    if (column === undefined) {
+      throw new LakeStorageError21(`[SixbDuckLake] SQL transform source '${sourceName}' picks column '${columnName}', which dataset '${dataset.id}' does not have.`);
+    }
+    if (picked.has(columnName)) {
+      throw new LakeStorageError21(`[SixbDuckLake] SQL transform source '${sourceName}' picks column '${columnName}' more than once.`);
+    }
+    picked.add(columnName);
+    return column;
+  });
 }
 function throwUnresolvedPlaceholder() {
-  throw new LakeStorageError19("[SixbDuckLake] SQL transform contains an unresolved relation placeholder.");
+  throw new LakeStorageError21("[SixbDuckLake] SQL transform contains an unresolved relation placeholder.");
+}
+
+// src/internal/sql-lineage/rejection.ts
+class SqlLineageRejected extends Error {
+  name = "SqlLineageRejected";
+}
+function reject(reason) {
+  throw new SqlLineageRejected(reason);
+}
+function rejectPlanFormat(detail) {
+  reject(`unexpected DuckDB plan format: ${detail}`);
+}
+
+// src/internal/sql-lineage/plan-json.ts
+function isJsonObject(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function asObject(value, what) {
+  if (!isJsonObject(value)) {
+    rejectPlanFormat(`${what} is not an object`);
+  }
+  return value;
+}
+function field(object, key) {
+  if (!Object.hasOwn(object, key)) {
+    rejectPlanFormat(`missing field '${key}'`);
+  }
+  return object[key];
+}
+function objectField(object, key) {
+  return asObject(field(object, key), `field '${key}'`);
+}
+function nullableObjectField(object, key) {
+  const value = field(object, key);
+  return value === null ? null : asObject(value, `field '${key}'`);
+}
+function stringField(object, key) {
+  const value = field(object, key);
+  if (typeof value !== "string") {
+    rejectPlanFormat(`field '${key}' is not a string`);
+  }
+  return value;
+}
+function booleanField(object, key) {
+  const value = field(object, key);
+  if (typeof value !== "boolean") {
+    rejectPlanFormat(`field '${key}' is not a boolean`);
+  }
+  return value;
+}
+function indexField(object, key) {
+  return asIndex(field(object, key), `field '${key}'`);
+}
+function asIndex(value, what) {
+  if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
+    rejectPlanFormat(`${what} is not an index`);
+  }
+  return value;
+}
+function listField(object, key) {
+  return asList(field(object, key), `field '${key}'`);
+}
+function asList(value, what) {
+  if (value === null) {
+    return [];
+  }
+  if (!Array.isArray(value)) {
+    rejectPlanFormat(`${what} is not a list`);
+  }
+  return value;
+}
+function objectListField(object, key) {
+  return listField(object, key).map((item) => asObject(item, `an item of '${key}'`));
+}
+function indexListField(object, key) {
+  return asIndexList(field(object, key), `field '${key}'`);
+}
+function asIndexList(value, what) {
+  return asList(value, what).map((item) => asIndex(item, `an item of ${what}`));
+}
+function stringListField(object, key) {
+  return listField(object, key).map((item) => {
+    if (typeof item !== "string") {
+      rejectPlanFormat(`an item of '${key}' is not a string`);
+    }
+    return item;
+  });
+}
+function parseSerializerOutput(text, listKey, failure) {
+  const output = asObject(JSON.parse(text), "serializer output");
+  if (booleanField(output, "error")) {
+    reject(`${failure}: ${stringField(output, "error_message")}`);
+  }
+  return listField(output, listKey);
+}
+
+// src/internal/sql-lineage/plan-relation.ts
+var VIRTUAL_COLUMN = -1;
+function bindingKey(tableIndex, columnIndex) {
+  return `${tableIndex}.${columnIndex}`;
+}
+function itemAt(items, index, what) {
+  const item = items[index];
+  if (item === undefined) {
+    rejectPlanFormat(`${what} ${index} is out of range`);
+  }
+  return item;
+}
+function unionOf(...sets) {
+  const union = new Set;
+  for (const set of sets) {
+    addAll(union, set);
+  }
+  return union;
+}
+function addAll(target, source) {
+  for (const item of source) {
+    target.add(item);
+  }
+}
+function sameDependencies(left, right) {
+  return left.size === right.size && [...left].every((item) => right.has(item));
+}
+
+// src/internal/sql-lineage/expressions.ts
+function expressionDependencies(expression, input) {
+  const dependencies = new Set;
+  addExpression(dependencies, expression, input);
+  return dependencies;
+}
+function orderExpressions(orders) {
+  return orders.map((order) => field(asObject(order, "sort order"), "expression"));
+}
+function nestedOrderExpressions(object, key) {
+  const orderBy = nullableObjectField(object, key);
+  return orderBy === null ? [] : orderExpressions(listField(orderBy, "orders"));
+}
+function addExpression(target, value, input) {
+  const expression = asObject(value, "expression");
+  const expressionClass = stringField(expression, "expression_class");
+  switch (expressionClass) {
+    case "BOUND_REF":
+      addAll(target, itemAt(input, indexField(expression, "index"), "column").dependencies);
+      return;
+    case "BOUND_COLUMN_REF":
+      addAll(target, columnByBinding(expression, input).dependencies);
+      return;
+    case "BOUND_CONSTANT":
+      return;
+    case "BOUND_CAST":
+    case "BOUND_UNNEST":
+      addExpression(target, field(expression, "child"), input);
+      return;
+    case "BOUND_FUNCTION":
+      assertNoHiddenExpressions(expression, "lambda_expr");
+      addExpressions(target, listField(expression, "children"), input);
+      return;
+    case "BOUND_AGGREGATE":
+      assertNoHiddenExpressions(expression);
+      addExpressions(target, listField(expression, "children"), input);
+      addOptionalExpression(target, field(expression, "filter"), input);
+      addExpressions(target, nestedOrderExpressions(expression, "order_bys"), input);
+      return;
+    case "BOUND_WINDOW":
+      addWindowExpression(target, expression, input);
+      return;
+    case "BOUND_COMPARISON":
+      addExpression(target, field(expression, "left"), input);
+      addExpression(target, field(expression, "right"), input);
+      return;
+    case "BOUND_BETWEEN":
+      addExpression(target, field(expression, "input"), input);
+      addExpression(target, field(expression, "lower"), input);
+      addExpression(target, field(expression, "upper"), input);
+      return;
+    case "BOUND_CONJUNCTION":
+    case "BOUND_OPERATOR":
+      addExpressions(target, listField(expression, "children"), input);
+      return;
+    case "BOUND_CASE":
+      for (const check of objectListField(expression, "case_checks")) {
+        addExpression(target, field(check, "when_expr"), input);
+        addExpression(target, field(check, "then_expr"), input);
+      }
+      addExpression(target, field(expression, "else_expr"), input);
+      return;
+    default:
+      reject(`expression '${expressionClass}' is not supported`);
+  }
+}
+function addWindowExpression(target, expression, input) {
+  if (stringField(expression, "type") === "WINDOW_AGGREGATE") {
+    assertNoHiddenExpressions(expression);
+  }
+  addExpressions(target, listField(expression, "children"), input);
+  addExpressions(target, listField(expression, "partitions"), input);
+  addExpressions(target, orderExpressions(listField(expression, "orders")), input);
+  addExpressions(target, orderExpressions(listField(expression, "arg_orders")), input);
+  for (const key of ["filters", "start_expr", "end_expr", "offset_expr", "default_expr"]) {
+    addOptionalExpression(target, field(expression, key), input);
+  }
+}
+function addExpressions(target, expressions, input) {
+  for (const expression of expressions) {
+    addExpression(target, expression, input);
+  }
+}
+function addOptionalExpression(target, expression, input) {
+  if (expression !== null) {
+    addExpression(target, expression, input);
+  }
+}
+function columnByBinding(expression, input) {
+  if (indexField(expression, "depth") !== 0) {
+    reject("correlated column references that the planner did not flatten are not supported");
+  }
+  const binding = objectField(expression, "binding");
+  const key = bindingKey(indexField(binding, "table_index"), indexField(binding, "column_index"));
+  const column = input.find((candidate) => candidate.binding === key);
+  if (column === undefined) {
+    rejectPlanFormat(`column binding ${key} does not resolve`);
+  }
+  return column;
+}
+function assertNoHiddenExpressions(expression, allowedKey) {
+  if (!booleanField(expression, "has_serialize")) {
+    return;
+  }
+  const bindData = field(expression, "function_data");
+  const inspected = isJsonObject(bindData) && allowedKey !== undefined ? Object.entries(bindData).flatMap(([key, value]) => key === allowedKey ? [] : [value]) : [bindData];
+  if (inspected.some(containsExpression)) {
+    reject(`function '${stringField(expression, "name")}' keeps expressions in its bind data, which is not supported`);
+  }
+}
+function containsExpression(value) {
+  if (Array.isArray(value)) {
+    return value.some((item) => containsExpression(item));
+  }
+  if (!isJsonObject(value)) {
+    return false;
+  }
+  return Object.hasOwn(value, "expression_class") || Object.values(value).some(containsExpression);
+}
+
+// src/internal/sql-lineage/table-functions.ts
+var PURE_TABLE_FUNCTIONS = new Set([
+  "range",
+  "generate_series",
+  "unnest",
+  "json_each",
+  "json_tree"
+]);
+
+// src/internal/sql-lineage/plan-walker.ts
+var MAX_RECURSIVE_CTE_ROUNDS = 64;
+function planLineage(plan, tables) {
+  const root = evaluate(asObject(plan, "plan"), { tables, ctes: new Map, delimColumns: [] });
+  const rows = sortedIds(root.rows);
+  const columns = root.columns.map((column) => sortedIds(column.dependencies));
+  if ([rows, ...columns].some((ids) => ids.includes(VIRTUAL_COLUMN))) {
+    reject("reading virtual columns such as rowid is not supported");
+  }
+  return { rows, columns };
+}
+function evaluate(node, context) {
+  const type = stringField(node, "type");
+  switch (type) {
+    case "LOGICAL_GET":
+      return evaluateGet(node, context);
+    case "LOGICAL_DUMMY_SCAN":
+      return evaluateDummyScan(node);
+    case "LOGICAL_EXPRESSION_GET":
+      return evaluateExpressionGet(node, context);
+    case "LOGICAL_PROJECTION":
+      return evaluateProjection(node, context);
+    case "LOGICAL_FILTER":
+      return evaluateFilter(node, context);
+    case "LOGICAL_AGGREGATE_AND_GROUP_BY":
+      return evaluateAggregate(node, context);
+    case "LOGICAL_WINDOW":
+      return evaluateWindow(node, context);
+    case "LOGICAL_UNNEST":
+      return evaluateUnnest(node, context);
+    case "LOGICAL_ORDER_BY":
+      return evaluateOrderBy(node, context);
+    case "LOGICAL_LIMIT":
+      return evaluateLimit(node, context);
+    case "LOGICAL_SAMPLE":
+      return evaluateOnlyChild(node, context);
+    case "LOGICAL_DISTINCT":
+      return evaluateDistinct(node, context);
+    case "LOGICAL_CROSS_PRODUCT":
+    case "LOGICAL_POSITIONAL_JOIN":
+      return evaluateCrossProduct(node, context);
+    case "LOGICAL_COMPARISON_JOIN":
+    case "LOGICAL_ASOF_JOIN":
+      return evaluateComparisonJoin(node, context);
+    case "LOGICAL_DELIM_JOIN":
+      return evaluateDelimJoin(node, context);
+    case "LOGICAL_ANY_JOIN":
+      return evaluateAnyJoin(node, context);
+    case "LOGICAL_DELIM_GET":
+      return evaluateDelimGet(node, context);
+    case "LOGICAL_UNION":
+    case "LOGICAL_EXCEPT":
+    case "LOGICAL_INTERSECT":
+      return evaluateSetOperation(node, context);
+    case "LOGICAL_MATERIALIZED_CTE":
+      return evaluateMaterializedCte(node, context);
+    case "LOGICAL_RECURSIVE_CTE":
+      return evaluateRecursiveCte(node, context);
+    case "LOGICAL_CTE_REF":
+      return evaluateCteRef(node, context);
+    default:
+      reject(`plan node '${type}' is not supported`);
+  }
+}
+function evaluateGet(node, context) {
+  if (objectListField(objectField(node, "table_filters"), "filters").length > 0) {
+    rejectPlanFormat("an unoptimized scan has pushed-down filters");
+  }
+  const inputs = objectListField(node, "children");
+  if (inputs.length > 1) {
+    rejectPlanFormat("a scan has several inputs");
+  }
+  const input = inputs[0] === undefined ? undefined : evaluate(inputs[0], context);
+  const functionName = stringField(node, "name");
+  const source = functionName === "seq_scan" ? tableScanSource(node, context, input) : tableFunctionSource(functionName, input);
+  const scanned = objectListField(node, "column_indexes").map((entry) => source.column(indexField(entry, "index")));
+  const tableIndex = indexField(node, "table_index");
+  const projectionIds = indexListField(node, "projection_ids");
+  const positions = projectionIds.length > 0 ? projectionIds : scanned.map((_, index) => index);
+  const columns = positions.map((position) => ({
+    dependencies: itemAt(scanned, position, "scanned column"),
+    binding: bindingKey(tableIndex, position)
+  }));
+  for (const position of indexListField(node, "projected_input")) {
+    if (input === undefined) {
+      rejectPlanFormat("a scan passes through input columns without an input");
+    }
+    columns.push(itemAt(input.columns, position, "input column"));
+  }
+  return { columns, rows: unionOf(input?.rows ?? new Set, source.rows) };
+}
+function tableScanSource(node, context, input) {
+  if (input !== undefined) {
+    rejectPlanFormat("a table scan has an input");
+  }
+  const table = stringField(objectField(node, "function_data"), "table");
+  const columnIds = context.tables.get(table);
+  if (columnIds === undefined) {
+    reject(`table '${table}' is not a source of the transform`);
+  }
+  const names = stringListField(node, "names");
+  return {
+    rows: new Set,
+    column: (index) => {
+      const name = names[index];
+      if (name === undefined) {
+        return new Set([VIRTUAL_COLUMN]);
+      }
+      const id = columnIds.get(name);
+      if (id === undefined) {
+        rejectPlanFormat(`table '${table}' has no column '${name}'`);
+      }
+      return new Set([id]);
+    }
+  };
+}
+function tableFunctionSource(name, input) {
+  if (!PURE_TABLE_FUNCTIONS.has(name)) {
+    reject(`table function '${name}' is not supported`);
+  }
+  const dependencies = input === undefined ? new Set : unionOf(...input.columns.map((column) => column.dependencies));
+  return { rows: dependencies, column: () => dependencies };
+}
+function evaluateDummyScan(node) {
+  assertLeaf(node);
+  return {
+    columns: [{ dependencies: new Set, binding: bindingKey(indexField(node, "table_index"), 0) }],
+    rows: new Set
+  };
+}
+function evaluateExpressionGet(node, context) {
+  const input = evaluateOnlyChild(node, context);
+  const width = listField(node, "expr_types").length;
+  const values = Array.from({ length: width }, () => new Set);
+  for (const row of listField(node, "expressions")) {
+    const expressions = asList(row, "a VALUES row");
+    if (expressions.length !== width) {
+      rejectPlanFormat("a VALUES row does not match the VALUES width");
+    }
+    expressions.forEach((expression, index) => {
+      addAll(itemAt(values, index, "VALUES column"), expressionDependencies(expression, input.columns));
+    });
+  }
+  const tableIndex = indexField(node, "table_index");
+  return {
+    columns: values.map((dependencies, index) => ({
+      dependencies,
+      binding: bindingKey(tableIndex, index)
+    })),
+    rows: input.rows
+  };
+}
+function evaluateProjection(node, context) {
+  const input = evaluateOnlyChild(node, context);
+  const tableIndex = indexField(node, "table_index");
+  return {
+    columns: listField(node, "expressions").map((expression, index) => ({
+      dependencies: expressionDependencies(expression, input.columns),
+      binding: bindingKey(tableIndex, index)
+    })),
+    rows: input.rows
+  };
+}
+function evaluateWindow(node, context) {
+  const input = evaluateOnlyChild(node, context);
+  const windowIndex = indexField(node, "window_index");
+  const rows = new Set(input.rows);
+  const windows = objectListField(node, "expressions").map((expression, index) => {
+    if (stringField(expression, "expression_class") !== "BOUND_WINDOW") {
+      rejectPlanFormat("a window operator computes something other than a window function");
+    }
+    addAll(rows, dependenciesOf(listField(expression, "partitions"), input.columns));
+    addAll(rows, dependenciesOf(orderExpressions(listField(expression, "orders")), input.columns));
+    return {
+      dependencies: expressionDependencies(expression, input.columns),
+      binding: bindingKey(windowIndex, index)
+    };
+  });
+  return { columns: [...input.columns, ...windows], rows };
+}
+function evaluateFilter(node, context) {
+  const input = evaluateOnlyChild(node, context);
+  return {
+    columns: projectColumns(input.columns, indexListField(node, "projection_map")),
+    rows: unionOf(input.rows, dependenciesOf(listField(node, "expressions"), input.columns))
+  };
+}
+function evaluateAggregate(node, context) {
+  const input = evaluateOnlyChild(node, context);
+  const groups = listField(node, "groups").map((expression) => expressionDependencies(expression, input.columns));
+  const groupIndex = indexField(node, "group_index");
+  const aggregateIndex = indexField(node, "aggregate_index");
+  const groupingsIndex = indexField(node, "groupings_index");
+  const groupColumns = groups.map((dependencies, index) => ({
+    dependencies,
+    binding: bindingKey(groupIndex, index)
+  }));
+  const aggregateColumns = listField(node, "expressions").map((expression, index) => ({
+    dependencies: expressionDependencies(expression, input.columns),
+    binding: bindingKey(aggregateIndex, index)
+  }));
+  const groupingColumns = listField(node, "grouping_functions").map((groupIndexes, index) => ({
+    dependencies: unionOf(...asIndexList(groupIndexes, "a grouping function").map((group) => itemAt(groups, group, "group"))),
+    binding: bindingKey(groupingsIndex, index)
+  }));
+  return {
+    columns: [...groupColumns, ...aggregateColumns, ...groupingColumns],
+    rows: unionOf(input.rows, ...groups)
+  };
+}
+function evaluateUnnest(node, context) {
+  const input = evaluateOnlyChild(node, context);
+  const unnestIndex = indexField(node, "unnest_index");
+  const rows = new Set(input.rows);
+  const unnested = listField(node, "expressions").map((expression, index) => {
+    const dependencies = expressionDependencies(expression, input.columns);
+    addAll(rows, dependencies);
+    return { dependencies, binding: bindingKey(unnestIndex, index) };
+  });
+  return { columns: [...input.columns, ...unnested], rows };
+}
+function evaluateOrderBy(node, context) {
+  const input = evaluateOnlyChild(node, context);
+  return {
+    columns: projectColumns(input.columns, indexListField(node, "projections")),
+    rows: unionOf(input.rows, dependenciesOf(orderExpressions(listField(node, "orders")), input.columns))
+  };
+}
+function evaluateLimit(node, context) {
+  const input = evaluateOnlyChild(node, context);
+  return {
+    columns: input.columns,
+    rows: unionOf(input.rows, limitDependencies(objectField(node, "limit_val"), input.columns), limitDependencies(objectField(node, "offset_val"), input.columns))
+  };
+}
+function limitDependencies(limit, input) {
+  const kind = stringField(limit, "type");
+  switch (kind) {
+    case "UNSET":
+    case "CONSTANT_VALUE":
+    case "CONSTANT_PERCENTAGE":
+      return new Set;
+    case "EXPRESSION_VALUE":
+    case "EXPRESSION_PERCENTAGE":
+      return expressionDependencies(field(limit, "expression"), input);
+    default:
+      reject(`LIMIT kind '${kind}' is not supported`);
+  }
+}
+function evaluateDistinct(node, context) {
+  const input = evaluateOnlyChild(node, context);
+  return {
+    columns: input.columns,
+    rows: unionOf(input.rows, dependenciesOf(listField(node, "distinct_targets"), input.columns), dependenciesOf(nestedOrderExpressions(node, "order_by"), input.columns))
+  };
+}
+function evaluateCrossProduct(node, context) {
+  const [left, right] = evaluateTwoChildren(node, context);
+  return { columns: [...left.columns, ...right.columns], rows: unionOf(left.rows, right.rows) };
+}
+function evaluateComparisonJoin(node, context) {
+  const [left, right] = evaluateTwoChildren(node, context);
+  return comparisonJoinOutput(node, left, right);
+}
+function evaluateDelimJoin(node, context) {
+  const [first, second] = twoChildren(node);
+  const flipped = booleanField(node, "delim_flipped");
+  const outer = evaluate(flipped ? second : first, context);
+  context.delimColumns.push(listField(node, "duplicate_eliminated_columns").map((expression) => ({
+    dependencies: expressionDependencies(expression, outer.columns)
+  })));
+  const inner = evaluate(flipped ? first : second, context);
+  context.delimColumns.pop();
+  return flipped ? comparisonJoinOutput(node, inner, outer) : comparisonJoinOutput(node, outer, inner);
+}
+function comparisonJoinOutput(node, left, right) {
+  const condition = new Set;
+  for (const comparison of objectListField(node, "conditions")) {
+    addAll(condition, expressionDependencies(field(comparison, "left"), left.columns));
+    addAll(condition, expressionDependencies(field(comparison, "right"), right.columns));
+  }
+  const predicate = field(node, "predicate");
+  if (predicate !== null) {
+    addAll(condition, expressionDependencies(predicate, [...left.columns, ...right.columns]));
+  }
+  return joinOutput(node, left, right, condition);
+}
+function evaluateAnyJoin(node, context) {
+  const [left, right] = evaluateTwoChildren(node, context);
+  const condition = expressionDependencies(field(node, "condition"), [
+    ...left.columns,
+    ...right.columns
+  ]);
+  return joinOutput(node, left, right, condition);
+}
+function joinOutput(node, left, right, condition) {
+  const rows = unionOf(left.rows, right.rows, condition);
+  const leftColumns = projectColumns(left.columns, indexListField(node, "left_projection_map"));
+  const rightColumns = projectColumns(right.columns, indexListField(node, "right_projection_map"));
+  const joinType = stringField(node, "join_type");
+  switch (joinType) {
+    case "INNER":
+    case "LEFT":
+    case "RIGHT":
+    case "FULL":
+    case "SINGLE":
+      return { columns: [...leftColumns, ...rightColumns], rows };
+    case "SEMI":
+    case "ANTI":
+      return { columns: leftColumns, rows };
+    case "RIGHT_SEMI":
+    case "RIGHT_ANTI":
+      return { columns: rightColumns, rows };
+    case "MARK": {
+      const mark = {
+        dependencies: condition,
+        binding: bindingKey(indexField(node, "mark_index"), 0)
+      };
+      return { columns: [...leftColumns, mark], rows };
+    }
+    default:
+      reject(`join type '${joinType}' is not supported`);
+  }
+}
+function evaluateDelimGet(node, context) {
+  assertLeaf(node);
+  const delimColumns = context.delimColumns.at(-1);
+  if (delimColumns === undefined) {
+    rejectPlanFormat("a DELIM_GET is outside any DELIM_JOIN");
+  }
+  if (listField(node, "chunk_types").length !== delimColumns.length) {
+    rejectPlanFormat("a DELIM_GET does not match its DELIM_JOIN");
+  }
+  return { columns: rebind(delimColumns, indexField(node, "table_index")), rows: new Set };
+}
+function evaluateSetOperation(node, context) {
+  const inputs = objectListField(node, "children").map((child) => evaluate(child, context));
+  const width = indexField(node, "column_count");
+  if (inputs.length < 2 || inputs.some((input) => input.columns.length !== width)) {
+    rejectPlanFormat("set operation inputs do not match");
+  }
+  const tableIndex = indexField(node, "table_index");
+  const rows = unionOf(...inputs.map((input) => input.rows));
+  if (stringField(node, "type") === "LOGICAL_UNION") {
+    const columns = Array.from({ length: width }, (_, index) => ({
+      dependencies: unionOf(...inputs.map((input) => itemAt(input.columns, index, "column").dependencies)),
+      binding: bindingKey(tableIndex, index)
+    }));
+    if (!booleanField(node, "setop_all")) {
+      addAll(rows, unionOf(...columns.map((column) => column.dependencies)));
+    }
+    return { columns, rows };
+  }
+  for (const input of inputs) {
+    addAll(rows, unionOf(...input.columns.map((column) => column.dependencies)));
+  }
+  return { columns: rebind(itemAt(inputs, 0, "input").columns, tableIndex), rows };
+}
+function evaluateMaterializedCte(node, context) {
+  const [definition, query] = twoChildren(node);
+  context.ctes.set(indexField(node, "table_index"), evaluate(definition, context));
+  return evaluate(query, context);
+}
+function evaluateRecursiveCte(node, context) {
+  const [anchorNode, stepNode] = twoChildren(node);
+  const tableIndex = indexField(node, "table_index");
+  const unionAll = booleanField(node, "union_all");
+  const keyTargets = listField(node, "key_targets");
+  let current = evaluate(anchorNode, context);
+  for (let round = 0;round < MAX_RECURSIVE_CTE_ROUNDS; round += 1) {
+    context.ctes.set(tableIndex, current);
+    const step = evaluate(stepNode, context);
+    if (step.columns.length !== current.columns.length) {
+      rejectPlanFormat("recursive CTE branches have different widths");
+    }
+    const columns = current.columns.map((column, index) => ({
+      dependencies: unionOf(column.dependencies, itemAt(step.columns, index, "column").dependencies),
+      binding: bindingKey(tableIndex, index)
+    }));
+    const rows = unionOf(current.rows, step.rows, dependenciesOf(keyTargets, columns));
+    if (!unionAll) {
+      addAll(rows, unionOf(...columns.map((column) => column.dependencies)));
+    }
+    const next = { columns, rows };
+    if (sameRelation(current, next)) {
+      return next;
+    }
+    current = next;
+  }
+  reject("recursive CTE lineage did not converge");
+}
+function evaluateCteRef(node, context) {
+  assertLeaf(node);
+  const definition = context.ctes.get(indexField(node, "cte_index"));
+  if (definition === undefined) {
+    rejectPlanFormat("a CTE_REF reads an undefined CTE");
+  }
+  if (listField(node, "chunk_types").length !== definition.columns.length) {
+    rejectPlanFormat("a CTE_REF does not match its CTE");
+  }
+  return {
+    columns: rebind(definition.columns, indexField(node, "table_index")),
+    rows: definition.rows
+  };
+}
+function twoChildren(node) {
+  const children = objectListField(node, "children");
+  const [first, second] = children;
+  if (children.length !== 2 || first === undefined || second === undefined) {
+    rejectPlanFormat(`'${stringField(node, "type")}' should have two inputs`);
+  }
+  return [first, second];
+}
+function evaluateTwoChildren(node, context) {
+  const [left, right] = twoChildren(node);
+  return [evaluate(left, context), evaluate(right, context)];
+}
+function evaluateOnlyChild(node, context) {
+  const children = objectListField(node, "children");
+  const [child] = children;
+  if (children.length !== 1 || child === undefined) {
+    rejectPlanFormat(`'${stringField(node, "type")}' should have one input`);
+  }
+  return evaluate(child, context);
+}
+function assertLeaf(node) {
+  if (objectListField(node, "children").length > 0) {
+    rejectPlanFormat(`'${stringField(node, "type")}' should have no input`);
+  }
+}
+function projectColumns(columns, projectionMap) {
+  return projectionMap.length === 0 ? columns : projectionMap.map((index) => itemAt(columns, index, "projected column"));
+}
+function rebind(columns, tableIndex) {
+  return columns.map((column, index) => ({
+    dependencies: column.dependencies,
+    binding: bindingKey(tableIndex, index)
+  }));
+}
+function dependenciesOf(expressions, input) {
+  return unionOf(...expressions.map((expression) => expressionDependencies(expression, input)));
+}
+function sameRelation(left, right) {
+  return sameDependencies(left.rows, right.rows) && left.columns.every((column, index) => sameDependencies(column.dependencies, itemAt(right.columns, index, "column").dependencies));
+}
+function sortedIds(ids) {
+  return [...ids].sort((left, right) => left - right);
+}
+
+// src/internal/sql-lineage/statement-gate.ts
+function assertSingleSelect(statements) {
+  const [statement] = statements;
+  if (statements.length !== 1 || statement === undefined) {
+    reject("the transform must be exactly one SELECT statement");
+  }
+  visit(statement, (node) => {
+    if (node.type === "PIVOT") {
+      const unpivot = listField(node, "unpivot_names").length > 0;
+      reject(`${unpivot ? "UNPIVOT" : "PIVOT"} is not supported`);
+    }
+    const unread = readsBeyondSources(node);
+    if (unread)
+      reject(unread);
+  });
+}
+function readsOnlySources(statements, sourceTables, builtinFunctions) {
+  const [statement] = statements;
+  if (statements.length !== 1 || statement === undefined)
+    return false;
+  let only = true;
+  visit(statement, (node, ctes) => {
+    if (readsBeyondSources(node))
+      only = false;
+    else if (node.type === "BASE_TABLE" && !readsSourceOrCte(node, sourceTables, ctes))
+      only = false;
+    else if (callsUnknownFunction(node, builtinFunctions))
+      only = false;
+  });
+  return only;
+}
+function readsBeyondSources(node) {
+  if (node.type === "SHOW_REF") {
+    return "DESCRIBE, SHOW and SUMMARIZE are not supported";
+  }
+  if (node.type === "TABLE_FUNCTION") {
+    const name = stringField(objectField(node, "function"), "function_name");
+    if (!PURE_TABLE_FUNCTIONS.has(name))
+      return `table function '${name}' is not supported`;
+  }
+  if (node.class === "PARAMETER") {
+    return "query parameters are not supported";
+  }
+  return;
+}
+function readsSourceOrCte(table, sourceTables, ctes) {
+  if (qualified(table, "schema_name") || qualified(table, "catalog_name"))
+    return false;
+  const name = stringField(table, "table_name");
+  return sourceTables.has(name) || ctes.has(name.toLowerCase());
+}
+function callsUnknownFunction(node, builtinFunctions) {
+  if (node.class !== "FUNCTION" && node.class !== "WINDOW")
+    return false;
+  if (qualified(node, "catalog") || qualified(node, "schema") && node.schema !== "main") {
+    return true;
+  }
+  const name = node.function_name;
+  return typeof name !== "string" || !builtinFunctions.has(name.toLowerCase());
+}
+function qualified(node, key) {
+  const value = node[key];
+  return typeof value === "string" && value !== "";
+}
+function visit(value, check, ctes = new Set) {
+  if (Array.isArray(value)) {
+    for (const item of value)
+      visit(item, check, ctes);
+    return;
+  }
+  if (!isJsonObject(value))
+    return;
+  const entries = cteEntries(value);
+  const defined = new Set(ctes);
+  for (const { name, query } of entries) {
+    visit(query, check, new Set(defined));
+    defined.add(name);
+  }
+  check(value, defined);
+  const recursive = value.type === "RECURSIVE_CTE_NODE" && typeof value.cte_name === "string" ? new Set([...defined, value.cte_name.toLowerCase()]) : undefined;
+  for (const [key, child] of Object.entries(value)) {
+    if (key === "cte_map" && entries.length > 0)
+      continue;
+    visit(child, check, key === "right" && recursive ? recursive : defined);
+  }
+}
+function cteEntries(node) {
+  const map = node.cte_map;
+  if (!isJsonObject(map) || !Array.isArray(map.map))
+    return [];
+  const entries = [];
+  for (const entry of map.map) {
+    if (!isJsonObject(entry) || typeof entry.key !== "string" || !isJsonObject(entry.value)) {
+      return [];
+    }
+    entries.push({ name: entry.key.toLowerCase(), query: entry.value.query });
+  }
+  return entries;
+}
+
+// src/internal/sql-lineage/analysis.ts
+var ISOLATED_INSTANCE_CONFIG = {
+  enable_external_access: "false",
+  autoinstall_known_extensions: "false",
+  autoload_known_extensions: "false"
+};
+async function analyzeSqlTransformLineage(input) {
+  const tables = analysisTables(input.sources);
+  const sql = input.template.render(new Map(tables.map((table) => [table.sourceName, table.relationSql])));
+  try {
+    return await withAnalysisConnection(tables, async (connection) => {
+      const statements = await serialize(connection, `json_serialize_sql(${quoteSqlString(sql)}, skip_null := false, skip_empty := false)`, "statements", "DuckDB could not parse the transform");
+      try {
+        return { supported: true, ...await analyzeSql(connection, sql, statements, tables) };
+      } catch (error) {
+        const sourceTables = new Set(tables.map((table) => table.tableName));
+        const functions = await builtinFunctions(connection).catch(() => {
+          return;
+        });
+        return unsupported(error, functions !== undefined && readsOnlySources(statements, sourceTables, functions));
+      }
+    });
+  } catch (error) {
+    return unsupported(error, false);
+  }
+}
+var builtins;
+function builtinFunctions(connection) {
+  if (builtins)
+    return builtins;
+  const read = connection.runAndReadAll("SELECT DISTINCT lower(function_name) FROM duckdb_functions()").then((reader) => new Set(reader.getRows().map(([name]) => String(name))));
+  builtins = read;
+  read.catch(() => {
+    if (builtins === read)
+      builtins = undefined;
+  });
+  return read;
+}
+function unsupported(error, readsOnlySources) {
+  return {
+    supported: false,
+    reason: error instanceof Error ? error.message : String(error),
+    readsOnlySources
+  };
+}
+function analysisTables(sources) {
+  const token = randomUUID4().replaceAll("-", "");
+  return Object.entries(sources).map(([sourceName, source], index) => {
+    const tableName = `sixb_lineage_${token}_${index}`;
+    return {
+      sourceName,
+      tableName,
+      columns: pickSourceColumns(sourceName, source.dataset, source.columns),
+      relationSql: renderSourceRelation(quoteIdentifier(tableName), source.columns)
+    };
+  });
+}
+async function withAnalysisConnection(tables, use) {
+  const instance = await DuckDBInstance2.create(":memory:", ISOLATED_INSTANCE_CONFIG);
+  try {
+    const connection = await instance.connect();
+    try {
+      for (const table of tables) {
+        const columnsSql = table.columns.map((column) => datasetColumnToDuckDbSql(column));
+        await connection.run(`CREATE TABLE ${quoteIdentifier(table.tableName)} (${columnsSql.join(", ")})`);
+      }
+      await connection.run("SET lock_configuration = true");
+      return await use(connection);
+    } finally {
+      connection.closeSync();
+    }
+  } finally {
+    instance.closeSync();
+  }
+}
+async function analyzeSql(connection, sql, statements, tables) {
+  assertSingleSelect(statements);
+  const plans = await serialize(connection, `json_serialize_plan(${quoteSqlString(sql)}, skip_null := false, skip_empty := false, optimize := false)`, "plans", "DuckDB could not bind the transform");
+  if (plans.length !== 1) {
+    rejectPlanFormat(`expected one plan, got ${plans.length}`);
+  }
+  const sourceColumns = numberSourceColumns(tables);
+  const lineage = planLineage(plans[0], sourceColumns.ids);
+  const names = await outputColumnNames(connection, sql);
+  if (lineage.columns.length !== names.length) {
+    rejectPlanFormat(`the plan has ${lineage.columns.length} columns, the result ${names.length}`);
+  }
+  const refs = (ids) => ids.map((id) => sourceColumns.refs[id]);
+  return {
+    lineage: {
+      rows: refs(lineage.rows),
+      columns: Object.fromEntries(names.map((name, index) => [name, refs(lineage.columns[index])]))
+    },
+    outputColumns: names
+  };
+}
+function numberSourceColumns(tables) {
+  const refs = [];
+  const ids = new Map(tables.map((table) => {
+    const columnIds = table.columns.map((column) => {
+      refs.push({ source: table.sourceName, column: column.name });
+      return [column.name, refs.length - 1];
+    });
+    return [table.tableName, new Map(columnIds)];
+  }));
+  return { ids, refs };
+}
+async function outputColumnNames(connection, sql) {
+  const prepared = await connection.prepare(`SELECT * FROM (${sql}) AS sixb_lineage_result`);
+  try {
+    return Array.from({ length: prepared.columnCount }, (_, index) => prepared.columnName(index));
+  } finally {
+    prepared.destroySync();
+  }
+}
+async function serialize(connection, serializerSql, listKey, failure) {
+  const reader = await connection.runAndReadAll(`SELECT ${serializerSql}`);
+  const text = reader.getRows()[0]?.[0];
+  if (typeof text !== "string") {
+    rejectPlanFormat("the serializer returned no JSON");
+  }
+  return parseSerializerOutput(text, listKey, failure);
 }
 
 // src/internal/ducklake-sql-executor.ts
@@ -3209,13 +4427,10 @@ class DuckLakeSqlExecutor {
   }
   async* preview(input) {
     this.connections.assertOpen();
+    const template = createDuckLakeSqlTransformTemplate(Object.keys(input.sources), input.sql);
     const rows = await this.connections.withAttachedRuntime(async (runtime) => {
       const sources = await this.resolveSources(runtime, input.sources);
-      const sql = renderDuckLakeSqlTransformSql({
-        options: this.options,
-        sources: sources.relations,
-        sql: input.sql
-      });
+      const sql = template.render(renderDuckLakeSourceRelations(this.options, sources.relations));
       return runtime.query(buildPreviewSql(sql, previewLimit(input.limit)));
     });
     for (const row of rows) {
@@ -3225,15 +4440,16 @@ class DuckLakeSqlExecutor {
   async execute(input) {
     this.connections.assertOpen();
     assertDatasetWriteMode(input.mode, "SQL transform");
+    const template = createDuckLakeSqlTransformTemplate(Object.keys(input.sources), input.sql);
     return this.writes.withCommitRuntime(async (runtime) => {
       const target = await this.resolveTargetDataset(runtime, input.target);
       const sources = await this.resolveSources(runtime, input.sources);
-      const sql = renderDuckLakeSqlTransformSql({
-        options: this.options,
-        sources: sources.relations,
-        sql: input.sql
-      });
-      const checkedCasts = await this.assertResultSchemaMatchesTarget(runtime, target, sql);
+      const sql = template.render(renderDuckLakeSourceRelations(this.options, sources.relations));
+      const sourceVersions = await this.readSourceVersions(runtime, sources).catch((error) => skipLineage(target, error));
+      const [checkedCasts, stepLineage] = await Promise.all([
+        this.assertResultSchemaMatchesTarget(runtime, target, sql),
+        sourceVersions === undefined ? undefined : analyzeExecutedLineage(template, pinnedSources(sources, sourceVersions), sources.relations, target)
+      ]);
       return this.writes.commitDatasetVersionOnExclusiveRuntime(runtime, {
         dataset: target,
         mode: input.mode,
@@ -3241,6 +4457,7 @@ class DuckLakeSqlExecutor {
         commitMessage: input.commitMessage ?? `execute SQL transform for dataset ${target.id}`,
         producer: input.producer,
         inputs: sources.inputs,
+        lineage: stepLineage === undefined || sourceVersions === undefined ? unanalyzedLineage(target, input.producer) : resolveStepLineage(stepLineage, [...sourceVersions.values()]),
         applyChanges: (runtime, context) => this.applySqlTransform({
           runtime,
           target,
@@ -3252,16 +4469,30 @@ class DuckLakeSqlExecutor {
       });
     });
   }
+  async analyze(input) {
+    for (const source of Object.values(input.sources)) {
+      this.datasets.assertSchema(source.dataset);
+    }
+    const analysis = await analyzeSqlTransformLineage({
+      sources: input.sources,
+      template: createDuckLakeSqlTransformTemplate(Object.keys(input.sources), input.sql)
+    });
+    return analysis.supported ? { supported: true, lineage: analysis.lineage } : { supported: false, reason: analysis.reason };
+  }
   async resolveSources(runtime, sources) {
     const relations = Object.create(null);
+    const tables = Object.create(null);
     const inputs = [];
     for (const [sourceName, source] of Object.entries(sources)) {
       const definition = await this.resolveSourceDataset(runtime, source.dataset);
+      pickSourceColumns(sourceName, definition, source.columns);
       const versionRef = await this.resolveSourceVersion(runtime, definition, source.versionId);
       relations[sourceName] = {
         datasetId: definition.id,
-        versionId: versionRef.versionId
+        versionId: versionRef.versionId,
+        ...source.columns === undefined ? {} : { columns: source.columns }
       };
+      tables[sourceName] = { definition, version: versionRef };
       inputs.push({
         datasetId: definition.id,
         versionId: versionRef.versionId
@@ -3269,24 +4500,30 @@ class DuckLakeSqlExecutor {
     }
     return {
       relations: Object.freeze(relations),
+      tables: Object.freeze(tables),
       inputs: Object.freeze(inputs)
     };
   }
   async resolveSourceDataset(runtime, dataset) {
     const definition = await this.datasets.getDatasetOnRuntime(runtime, dataset.id);
     if (!definition) {
-      throw new LakeStorageError20(`[SixbDuckLake] Unknown SQL transform source dataset '${dataset.id}'.`);
+      throw new LakeStorageError22(`[SixbDuckLake] Unknown SQL transform source dataset '${dataset.id}'.`);
     }
     return definition;
   }
   async resolveTargetDataset(runtime, dataset) {
     const definition = await this.datasets.getDatasetOnRuntime(runtime, dataset.id);
     if (!definition) {
-      throw new LakeStorageError20(`[SixbDuckLake] Unknown SQL transform target dataset '${dataset.id}'.`);
+      throw new LakeStorageError22(`[SixbDuckLake] Unknown SQL transform target dataset '${dataset.id}'.`);
     }
     this.datasets.assertSchema(definition);
     return definition;
   }
+  async readSourceVersions(runtime, sources) {
+    const names = Object.keys(sources.tables);
+    const versions = await this.snapshots.getVersionsForRefs(runtime, names.map((sourceName) => sources.tables[sourceName].version));
+    return new Map(names.map((sourceName, index) => [sourceName, versions[index]]));
+  }
   async resolveSourceVersion(runtime, dataset, versionId) {
     if (versionId !== undefined) {
       const snapshotId = parseVersionId(versionId);
@@ -3332,7 +4569,7 @@ class DuckLakeSqlExecutor {
     return checkedCasts;
   }
   async applySqlTransform(input) {
-    const tempTableName = `sixb_sql_transform_${randomUUID4().replaceAll("-", "")}`;
+    const tempTableName = `sixb_sql_transform_${randomUUID5().replaceAll("-", "")}`;
     const tempTable = quoteIdentifier(tempTableName);
     await input.runtime.run(`CREATE TEMP TABLE ${tempTable} AS SELECT * FROM (${input.sql}) AS sixb_sql_transform_result`);
     let result;
@@ -3358,6 +4595,62 @@ class DuckLakeSqlExecutor {
     return result;
   }
 }
+function pinnedSources(sources, versions) {
+  const pinned = Object.create(null);
+  for (const [sourceName, version] of versions) {
+    pinned[sourceName] = {
+      dataset: { ...sources.tables[sourceName].definition, schema: version.schema },
+      columns: sources.relations[sourceName].columns
+    };
+  }
+  return pinned;
+}
+async function analyzeExecutedLineage(template, pinnedSources, relations, target) {
+  let analysis;
+  try {
+    analysis = await analyzeSqlTransformLineage({ sources: pinnedSources, template });
+  } catch (error) {
+    return skipLineage(target, error);
+  }
+  if (!analysis.supported) {
+    return analysis.readsOnlySources ? toDatasetColumnLineage(everySourceColumn(pinnedSources, target), relations) : undefined;
+  }
+  if (!describesTarget(analysis.outputColumns, target))
+    return;
+  return toDatasetColumnLineage(analysis.lineage, relations);
+}
+function everySourceColumn(sources, target) {
+  const rows = Object.entries(sources).flatMap(([source, { dataset, columns }]) => (columns ?? dataset.schema.columns.map((column) => column.name)).map((column) => ({
+    source,
+    column
+  })));
+  return { rows, columns: Object.fromEntries(target.schema.columns.map(({ name }) => [name, []])) };
+}
+function unanalyzedLineage(target, producer) {
+  return producer?.kind === "pipeline" ? undefined : { unknown: `a SQL transform into '${target.id}' could not be analyzed` };
+}
+function skipLineage(target, error) {
+  const message = error instanceof Error ? error.message : String(error);
+  console.warn(`[SixbDuckLake] SQL transform into dataset '${target.id}' records no column lineage: ${message}`);
+  return;
+}
+function describesTarget(outputColumns, target) {
+  return outputColumns.length === target.schema.columns.length && target.schema.columns.every((column, index) => outputColumns[index] === column.name);
+}
+function toDatasetColumnLineage(lineage, relations) {
+  const toRefs = (refs) => {
+    const unique = new Map;
+    for (const { source, column } of refs) {
+      const datasetId = relations[source].datasetId;
+      unique.set(JSON.stringify([datasetId, column]), { datasetId, column });
+    }
+    return [...unique.values()];
+  };
+  return {
+    rows: toRefs(lineage.rows),
+    columns: Object.fromEntries(Object.entries(lineage.columns).map(([column, refs]) => [column, toRefs(refs)]))
+  };
+}
 function previewLimit(limit) {
   if (limit === undefined || !Number.isFinite(limit)) {
     return DEFAULT_PREVIEW_LIMIT;
@@ -3403,33 +4696,34 @@ function normalizePreviewValue(value) {
   return value;
 }
 function throwNoCommittedSourceVersion(datasetId) {
-  throw new LakeStorageError20(`[SixbDuckLake] No committed version found for SQL transform source dataset '${datasetId}'.`);
+  throw new LakeStorageError22(`[SixbDuckLake] No committed version found for SQL transform source dataset '${datasetId}'.`);
 }
 function throwResultSchemaMismatch(datasetId, detail) {
-  throw new LakeStorageError20(`[SixbDuckLake] SQL transform result schema does not match target dataset '${datasetId}': ${detail}.`);
+  throw new LakeStorageError22(`[SixbDuckLake] SQL transform result schema does not match target dataset '${datasetId}': ${detail}.`);
 }
 
 // src/internal/ducklake-write-coordinator.ts
-import { randomUUID as randomUUID7 } from "crypto";
+import { randomUUID as randomUUID8 } from "crypto";
 import {
   assertUnsequencedDatasetWrite,
   getDatasetPrimaryKeyColumns as getDatasetPrimaryKeyColumns4,
-  hasDatasetInputChanges,
-  LakeConcurrencyError,
-  LakeStorageError as LakeStorageError24,
+  hasDatasetProvenanceChanges,
+  LakeConcurrencyError as LakeConcurrencyError2,
+  LakeStorageError as LakeStorageError26,
+  lineageAtCommit as lineageAtCommit2,
   retryDatasetMergeCommit
 } from "@sixb/core/lake-storage";
 
 // src/internal/dataset-row-merge.ts
 import {
   getDatasetPrimaryKeyColumns as getDatasetPrimaryKeyColumns2,
-  LakeStorageError as LakeStorageError21,
+  LakeStorageError as LakeStorageError23,
   reconcileDatasetSequences
 } from "@sixb/core/lake-storage";
 async function applyDatasetMergeFromRelation(input) {
   const primaryKeyColumns = getDatasetPrimaryKeyColumns2(input.dataset);
   if (primaryKeyColumns === null) {
-    throw new LakeStorageError21(`[SixbDuckLake] Dataset '${input.dataset.id}' must define a primaryKey before it can be merged.`);
+    throw new LakeStorageError23(`[SixbDuckLake] Dataset '${input.dataset.id}' must define a primaryKey before it can be merged.`);
   }
   const stagingTable = quoteIdentifier(input.stagingTableName);
   const targetTable = qualifiedTableName(input.options, encodeDatasetTableName(input.dataset.id));
@@ -3442,7 +4736,9 @@ async function applyDatasetMergeFromRelation(input) {
       return {
         dataChangeExpected: orderingChanged,
         sourceRowCount,
-        resultingRowCount: { kind: "exact", value: previousRowCount }
+        resultingRowCount: { kind: "exact", value: previousRowCount },
+        targetWasEmpty: previousRowCount === 0,
+        changedTableRows: false
       };
     }
     const keyMatchSql = primaryKeyMatchSql(primaryKeyColumns, "target", "source");
@@ -3481,7 +4777,9 @@ async function applyDatasetMergeFromRelation(input) {
       resultingRowCount: {
         kind: "exact",
         value: previousRowCount + effects.inserts - effects.deletes
-      }
+      },
+      targetWasEmpty: previousRowCount === 0,
+      changedTableRows: true
     };
   });
 }
@@ -3590,7 +4888,7 @@ function finalChangesSelectSql(input, primaryKeyColumns) {
 async function countMergeEffects(input, targetTable, finalChanges, primaryKeyColumns) {
   const matchColumn = primaryKeyColumns[0];
   if (matchColumn === undefined) {
-    throw new LakeStorageError21(`[SixbDuckLake] Dataset '${input.dataset.id}' primaryKey must contain at least one column.`);
+    throw new LakeStorageError23(`[SixbDuckLake] Dataset '${input.dataset.id}' primaryKey must contain at least one column.`);
   }
   const keyMatchSql = primaryKeyMatchSql(primaryKeyColumns, "target", "source");
   const targetMatch = `target.${quoteIdentifier(matchColumn)} IS NOT NULL`;
@@ -3639,12 +4937,12 @@ function sameColumns(left, right) {
 }
 
 // src/internal/ducklake-merge-session.ts
-import { randomUUID as randomUUID5 } from "crypto";
+import { randomUUID as randomUUID6 } from "crypto";
 import {
   cloneDatasetMergeChange,
   datasetSequenceChange,
   getDatasetMergeChangeValidationError,
-  LakeStorageError as LakeStorageError22
+  LakeStorageError as LakeStorageError24
 } from "@sixb/core/lake-storage";
 
 // src/internal/row-appender.ts
@@ -3703,7 +5001,7 @@ function fileRefToDuckDbStruct(fileRef) {
 // src/internal/ducklake-merge-session.ts
 var MERGE_CHANGE_BATCH_SIZE = 1000;
 async function createDuckLakeMergeSession(input) {
-  const suffix = randomUUID5().replaceAll("-", "");
+  const suffix = randomUUID6().replaceAll("-", "");
   const stagingTableName = `sixb_merge_${suffix}`;
   const sequenceColumnName = `sixb_merge_sequence_${suffix}`;
   const kindColumnName = `sixb_merge_kind_${suffix}`;
@@ -3745,7 +5043,7 @@ class DuckLakeMergeSession {
     for await (const change of changes) {
       const validationError = getDatasetMergeChangeValidationError(change, this.input.dataset);
       if (validationError) {
-        throw new LakeStorageError22(`[SixbDuckLake] ${validationError}`);
+        throw new LakeStorageError24(`[SixbDuckLake] ${validationError}`);
       }
       batch.push({
         sequence: this.nextSequence,
@@ -3808,7 +5106,7 @@ class DuckLakeMergeSession {
   }
   assertOpen() {
     if (this.closed) {
-      throw new LakeStorageError22("[SixbDuckLake] Merge session is already closed.");
+      throw new LakeStorageError24("[SixbDuckLake] Merge session is already closed.");
     }
   }
   async cleanup() {
@@ -3823,16 +5121,16 @@ class DuckLakeMergeSession {
 }
 
 // src/internal/ducklake-write-session.ts
-import { randomUUID as randomUUID6 } from "crypto";
+import { randomUUID as randomUUID7 } from "crypto";
 import { getDatasetRowValidationError } from "@sixb/core";
 import {
   encodeDatasetPrimaryKey,
   getDatasetPrimaryKeyColumns as getDatasetPrimaryKeyColumns3,
-  LakeStorageError as LakeStorageError23
+  LakeStorageError as LakeStorageError25
 } from "@sixb/core/lake-storage";
 var WRITE_ROW_BATCH_SIZE = 1000;
 async function createDuckLakeWriteSession(input) {
-  const stagingTableName = `sixb_write_${randomUUID6().replaceAll("-", "")}`;
+  const stagingTableName = `sixb_write_${randomUUID7().replaceAll("-", "")}`;
   await input.runtime.run(`CREATE TEMP TABLE ${quoteIdentifier(stagingTableName)} (${datasetSchemaToDuckDbColumnsSql(input.write.dataset.schema)})`);
   return new DuckLakeWriteSession(input.commitWrite, input.runtime, input.write, stagingTableName);
 }
@@ -3859,13 +5157,13 @@ class DuckLakeWriteSession {
     for await (const row of rows) {
       const validationError = getDatasetRowValidationError(row, this.input.dataset);
       if (validationError) {
-        throw new LakeStorageError23(`[SixbDuckLake] ${validationError}`);
+        throw new LakeStorageError25(`[SixbDuckLake] ${validationError}`);
       }
       const clonedRow = structuredClone(row);
       if (getDatasetPrimaryKeyColumns3(this.input.dataset) !== null) {
         const primaryKey = encodeDatasetPrimaryKey(this.input.dataset, clonedRow);
         if (this.stagedPrimaryKeys.has(primaryKey) || batchPrimaryKeys.has(primaryKey)) {
-          throw new LakeStorageError23(`[SixbDuckLake] Dataset '${this.input.dataset.id}' ${this.input.mode ?? "snapshot"} source contains duplicate primary key.`);
+          throw new LakeStorageError25(`[SixbDuckLake] Dataset '${this.input.dataset.id}' ${this.input.mode ?? "snapshot"} source contains duplicate primary key.`);
         }
         batchPrimaryKeys.add(primaryKey);
       }
@@ -3922,7 +5220,7 @@ class DuckLakeWriteSession {
   }
   assertOpen() {
     if (this.closed) {
-      throw new LakeStorageError23("[SixbDuckLake] Write session is already closed.");
+      throw new LakeStorageError25("[SixbDuckLake] Write session is already closed.");
     }
   }
   async cleanup() {
@@ -3956,7 +5254,7 @@ class DuckLakeWriteCoordinator {
     this.connections.assertOpen();
     const definition = await this.datasets.getDataset(input.dataset.id);
     if (!definition) {
-      throw new LakeStorageError24(`[SixbDuckLake] Unknown dataset '${input.dataset.id}'.`);
+      throw new LakeStorageError26(`[SixbDuckLake] Unknown dataset '${input.dataset.id}'.`);
     }
     this.datasets.assertSchema(definition);
     assertUnsequencedDatasetWrite(definition);
@@ -3977,15 +5275,15 @@ class DuckLakeWriteCoordinator {
     this.connections.assertOpen();
     const definition = await this.datasets.getDataset(input.dataset.id);
     if (!definition) {
-      throw new LakeStorageError24(`[SixbDuckLake] Unknown dataset '${input.dataset.id}'.`);
+      throw new LakeStorageError26(`[SixbDuckLake] Unknown dataset '${input.dataset.id}'.`);
     }
     if (getDatasetPrimaryKeyColumns4(definition) === null) {
-      throw new LakeStorageError24(`[SixbDuckLake] Dataset '${definition.id}' must define a primaryKey before it can be merged.`);
+      throw new LakeStorageError26(`[SixbDuckLake] Dataset '${definition.id}' must define a primaryKey before it can be merged.`);
     }
     this.datasets.assertSchema(definition);
     const latestVersion = await this.connections.withAttachedRuntime((runtime) => this.snapshots.getLatestVersionForDefinition(runtime, definition));
     if (input.expectedLatestVersionId !== undefined && latestVersion?.versionId !== input.expectedLatestVersionId) {
-      throw new LakeConcurrencyError(`[SixbDuckLake] Optimistic merge start failed for dataset '${definition.id}': expected latest version '${input.expectedLatestVersionId}', found '${latestVersion?.versionId ?? "none"}'.`);
+      throw new LakeConcurrencyError2(`[SixbDuckLake] Optimistic merge start failed for dataset '${definition.id}': expected latest version '${input.expectedLatestVersionId}', found '${latestVersion?.versionId ?? "none"}'.`);
     }
     const runtime = await this.connections.stagingRuntime();
     return createDuckLakeMergeSession({
@@ -3998,7 +5296,7 @@ class DuckLakeWriteCoordinator {
   async commitWrite(input) {
     const definition = await this.datasets.getDataset(input.write.dataset.id);
     if (!definition) {
-      throw new LakeStorageError24(`[SixbDuckLake] Unknown dataset '${input.write.dataset.id}'.`);
+      throw new LakeStorageError26(`[SixbDuckLake] Unknown dataset '${input.write.dataset.id}'.`);
     }
     const mode = input.write.mode ?? "snapshot";
     assertDatasetWriteMode(mode, "write");
@@ -4009,6 +5307,7 @@ class DuckLakeWriteCoordinator {
       commitMessage: input.commit?.commitMessage ?? `write dataset ${input.write.dataset.id}`,
       producer: input.write.producer,
       inputs: input.write.inputs,
+      lineage: input.write.lineage,
       applyChanges: (runtime, context) => this.applyStagedRows({
         ...input,
         runtime,
@@ -4019,10 +5318,10 @@ class DuckLakeWriteCoordinator {
   async commitMerge(input) {
     const definition = await this.datasets.getDataset(input.merge.dataset.id);
     if (!definition) {
-      throw new LakeStorageError24(`[SixbDuckLake] Unknown dataset '${input.merge.dataset.id}'.`);
+      throw new LakeStorageError26(`[SixbDuckLake] Unknown dataset '${input.merge.dataset.id}'.`);
     }
     if (getDatasetPrimaryKeyColumns4(definition) === null) {
-      throw new LakeStorageError24(`[SixbDuckLake] Dataset '${definition.id}' must define a primaryKey before it can be merged.`);
+      throw new LakeStorageError26(`[SixbDuckLake] Dataset '${definition.id}' must define a primaryKey before it can be merged.`);
     }
     return retryDatasetMergeCommit(input.merge, input.commit, (rebase) => this.withCommitRuntime(async (runtime) => {
       input.commit?.signal?.throwIfAborted();
@@ -4033,7 +5332,6 @@ class DuckLakeWriteCoordinator {
         expectedLatestVersionId: baseVersionId,
         allowInitialNoOp: !input.commit?.createInitialVersion,
         signal: input.commit?.signal,
-        disableRetries: true,
         optimisticOperation: "merge commit",
         commitMessage: input.commit?.commitMessage ?? `merge dataset ${definition.id}`,
         producer: input.merge.producer,
@@ -4050,7 +5348,7 @@ class DuckLakeWriteCoordinator {
             validatedPrimaryKeyColumns: context.validatedPrimaryKeyColumns
           });
           if (result.sourceRowCount !== input.changesWritten) {
-            throw new LakeStorageError24(`[SixbDuckLake] Staged merge for dataset '${definition.id}' accepted ${input.changesWritten} change(s), but DuckLake saw ${result.sourceRowCount} source change(s) at commit time.`);
+            throw new LakeStorageError26(`[SixbDuckLake] Staged merge for dataset '${definition.id}' accepted ${input.changesWritten} change(s), but DuckLake saw ${result.sourceRowCount} source change(s) at commit time.`);
           }
           return result;
         }
@@ -4067,39 +5365,14 @@ class DuckLakeWriteCoordinator {
   }
   async commitDatasetVersionOnExclusiveRuntime(runtime, input) {
     assertUnsequencedDatasetWrite(input.dataset);
-    const result = await this.commitVersionOutcomeOnExclusiveRuntime(runtime, {
-      ...input,
-      disableRetries: input.mode === "append" && getDatasetPrimaryKeyColumns4(input.dataset) !== null
-    });
+    const result = await this.commitVersionOutcomeOnExclusiveRuntime(runtime, input);
     if (result.version === null) {
-      throw new LakeStorageError24(`[SixbDuckLake] No DuckLake changes were committed for dataset '${input.dataset.id}', and no previous version exists.`);
+      throw new LakeStorageError26(`[SixbDuckLake] No DuckLake changes were committed for dataset '${input.dataset.id}', and no previous version exists.`);
     }
     return { ...result.version, outcome: result.outcome };
   }
   async commitVersionOutcomeOnExclusiveRuntime(runtime, input) {
-    const runtimeInput = { ...input, runtime };
-    return this.withGuardedRetrySetting(runtimeInput, () => this.commitDatasetVersionUnlocked(runtimeInput));
-  }
-  async withGuardedRetrySetting(input, run) {
-    if (input.expectedLatestVersionId === undefined && !input.disableRetries) {
-      return run();
-    }
-    await input.runtime.run("SET ducklake_max_retry_count = 0");
-    let outcome;
-    try {
-      outcome = { kind: "success", value: await run() };
-    } catch (error) {
-      outcome = { kind: "error", error };
-    }
-    try {
-      await input.runtime.run("RESET ducklake_max_retry_count");
-    } catch {
-      this.connections.poisonRuntime();
-    }
-    if (outcome.kind === "error") {
-      throw outcome.error;
-    }
-    return outcome.value;
+    return rerunWhenCatalogMoves(input.dataset.id, () => this.commitDatasetVersionUnlocked({ ...input, runtime }), input.signal);
   }
   async commitDatasetVersionUnlocked(input) {
     const previousWriteSnapshotId = await this.lastCommittedSnapshotId(input.runtime);
@@ -4114,15 +5387,26 @@ class DuckLakeWriteCoordinator {
           operation: input.optimisticOperation ?? "commit"
         });
       }
+      const lineage = lineageAtCommit2({
+        mode: input.mode,
+        schema: input.dataset.schema,
+        producer: input.producer,
+        written: input.lineage,
+        parent: latestVersion
+      });
       const changeResult = await input.applyChanges(input.runtime, {
         previousRowCount: latestVersion?.rowCount,
         validatedPrimaryKeyColumns: latestVersion?.validatedPrimaryKeyColumns
       });
       input.signal?.throwIfAborted();
-      await this.ensureNoOpHasSnapshot(input, changeResult, latestVersion);
-      const commitId = randomUUID7();
-      await this.setCommitMetadata(input, commitId, commitRowCount(input, changeResult, latestVersion));
-      await input.runtime.run("COMMIT");
+      const noOpBase = changeResult.dataChangeExpected ? undefined : latestVersion === undefined ? await this.latestVersionAtReadSnapshot(input) : latestVersion;
+      const createsVersion = noOpBase === undefined || this.noOpCreatesVersion(input, noOpBase, lineage);
+      if (createsVersion && mustClaimTable(input, changeResult, lineage)) {
+        await this.claimTable(input);
+      }
+      const commitId = randomUUID8();
+      await this.setCommitMetadata(input, commitId, commitRowCount(input, changeResult, latestVersion), lineage);
+      await commitVersionTransaction(input.runtime, input.dataset.id, createsVersion && mustCommitOnReadSnapshot(input, changeResult) ? { poisonRuntime: () => this.connections.poisonRuntime() } : undefined);
       committed = true;
       const committedWriteSnapshotId = await this.lastCommittedSnapshotId(input.runtime);
       try {
@@ -4131,7 +5415,8 @@ class DuckLakeWriteCoordinator {
           commitId,
           previousWriteSnapshotId,
           committedWriteSnapshotId,
-          changeResult
+          changeResult,
+          noOpBaseVersionId: noOpBase?.versionId ?? null
         });
       } finally {
         await this.connections.detachLocalCatalogAfterCommit(input.runtime);
@@ -4139,9 +5424,6 @@ class DuckLakeWriteCoordinator {
     } catch (error) {
       if (!committed) {
         await this.rollbackTransaction(input.runtime);
-        if (error instanceof Error && error.message.startsWith("TransactionContext Error: Failed to commit: Failed to commit DuckLake transaction.") && error.message.includes("Exceeded the maximum retry count of 0") && error.message.includes('duplicate key value violates unique constraint "ducklake_snapshot_pkey"')) {
-          throw new LakeConcurrencyError(`[SixbDuckLake] Concurrent catalog commit for dataset '${input.dataset.id}'.`, { cause: error });
-        }
       }
       throw error;
     }
@@ -4153,7 +5435,7 @@ class DuckLakeWriteCoordinator {
       if (version) {
         return { outcome: "created", version };
       }
-      throw new LakeStorageError24(`[SixbDuckLake] DuckLake committed snapshot '${ownSnapshotId}' for dataset '${input.dataset.id}', but Sixb could not hydrate it as a dataset version.`);
+      throw new LakeStorageError26(`[SixbDuckLake] DuckLake committed snapshot '${ownSnapshotId}' for dataset '${input.dataset.id}', but Sixb could not hydrate it as a dataset version.`);
     }
     if (!input.changeResult.dataChangeExpected) {
       const version = await this.versionForNoOpCommit(input.runtime, input.dataset, input.allowInitialNoOp ?? false);
@@ -4165,15 +5447,18 @@ class DuckLakeWriteCoordinator {
           operation: input.optimisticOperation ?? "commit"
         });
       }
+      if ((version?.versionId ?? null) !== input.noOpBaseVersionId) {
+        throw new CatalogMovedError(`[SixbDuckLake] Dataset '${input.dataset.id}' changed while an unchanged write was compared with it.`);
+      }
       return {
         outcome: "unchanged",
         version
       };
     }
     if (input.committedWriteSnapshotId === input.previousWriteSnapshotId) {
-      throw new LakeStorageError24(`[SixbDuckLake] DuckLake commit for dataset '${input.dataset.id}' completed without producing a snapshot for this non-empty write.`);
+      throw new LakeStorageError26(`[SixbDuckLake] DuckLake commit for dataset '${input.dataset.id}' completed without producing a snapshot for this non-empty write.`);
     }
-    throw new LakeStorageError24(`[SixbDuckLake] DuckLake commit for dataset '${input.dataset.id}' completed, but Sixb could not find a matching snapshot for this write.`);
+    throw new LakeStorageError26(`[SixbDuckLake] DuckLake commit for dataset '${input.dataset.id}' completed, but Sixb could not find a matching snapshot for this write.`);
   }
   async applyStagedRows(input) {
     const mode = input.write.mode ?? "snapshot";
@@ -4186,11 +5471,11 @@ class DuckLakeWriteCoordinator {
       previousRowCount: input.previousRowCount
     });
     if (result.sourceRowCount !== input.rowsWritten) {
-      throw new LakeStorageError24(`[SixbDuckLake] Staged write for dataset '${input.write.dataset.id}' accepted ${input.rowsWritten} row(s), but DuckLake saw ${result.sourceRowCount} source row(s) at commit time.`);
+      throw new LakeStorageError26(`[SixbDuckLake] Staged write for dataset '${input.write.dataset.id}' accepted ${input.rowsWritten} row(s), but DuckLake saw ${result.sourceRowCount} source row(s) at commit time.`);
     }
     return result;
   }
-  async setCommitMetadata(input, commitId, rowCount) {
+  async setCommitMetadata(input, commitId, rowCount, lineage) {
     const validatedPrimaryKeyColumns = getDatasetPrimaryKeyColumns4(input.dataset);
     const metadata = {
       kind: "datasetVersion",
@@ -4200,22 +5485,35 @@ class DuckLakeWriteCoordinator {
       producer: input.producer ? structuredClone(input.producer) : undefined,
       inputs: input.inputs ? structuredClone(input.inputs) : undefined,
       ...rowCount.kind === "exact" ? { rowCount: rowCount.value } : {},
-      ...validatedPrimaryKeyColumns !== null ? { validatedPrimaryKeyColumns: [...validatedPrimaryKeyColumns] } : {}
+      ...validatedPrimaryKeyColumns !== null ? { validatedPrimaryKeyColumns: [...validatedPrimaryKeyColumns] } : {},
+      ...lineage === undefined ? {} : { lineage: encodeLineage(lineage) }
     };
     await input.runtime.run(`CALL ${quoteIdentifier(duckLakeAlias(this.options))}.set_commit_message(${quoteSqlString("Sixb")}, ${quoteSqlString(input.commitMessage)}, extra_info => ${quoteSqlString(JSON.stringify({ sixb: metadata }))})`);
   }
-  async ensureNoOpHasSnapshot(input, changeResult, knownLatestVersion) {
-    if (changeResult.dataChangeExpected) {
-      return;
-    }
-    const latestVersion = knownLatestVersion === undefined ? await this.snapshots.getLatestVersionSummaryForDefinition(input.runtime, input.dataset) : knownLatestVersion;
-    if (latestVersion ? input.mode === "merge" || !hasDatasetInputChanges(latestVersion.inputs, input.inputs) : input.allowInitialNoOp) {
-      return;
-    }
+  async claimTable(input) {
     const table = qualifiedTableName(this.options, encodeDatasetTableName(input.dataset.id));
     const descriptionSql = input.dataset.description === undefined ? "NULL" : quoteSqlString(input.dataset.description);
     await input.runtime.run(`COMMENT ON TABLE ${table} IS ${descriptionSql}`);
   }
+  noOpCreatesVersion(input, latestVersion, lineage) {
+    if (!latestVersion) {
+      return !input.allowInitialNoOp;
+    }
+    return input.mode !== "merge" && hasDatasetProvenanceChanges(latestVersion, {
+      datasetId: input.dataset.id,
+      schema: input.dataset.schema,
+      producer: input.producer,
+      inputs: input.inputs,
+      lineage
+    });
+  }
+  async latestVersionAtReadSnapshot(input) {
+    const { snapshotId, latestVersion } = await this.snapshots.getLatestVersionSummaryAtCurrentSnapshot(input.runtime, input.dataset);
+    if (latestVersion && BigInt(parseVersionId(latestVersion.versionId)) > BigInt(snapshotId)) {
+      throw new CatalogMovedError(`[SixbDuckLake] Dataset '${input.dataset.id}' changed while an unchanged write was compared with it.`);
+    }
+    return latestVersion;
+  }
   async versionForNoOpCommit(runtime, definition, allowInitialNoOp) {
     const latestVersion = await this.snapshots.getLatestVersionForDefinition(runtime, definition);
     if (latestVersion) {
@@ -4224,7 +5522,7 @@ class DuckLakeWriteCoordinator {
     if (allowInitialNoOp) {
       return null;
     }
-    throw new LakeStorageError24(`[SixbDuckLake] No DuckLake changes were committed for dataset '${definition.id}', and no previous version exists.`);
+    throw new LakeStorageError26(`[SixbDuckLake] No DuckLake changes were committed for dataset '${definition.id}', and no previous version exists.`);
   }
   async rollbackTransaction(runtime) {
     try {
@@ -4232,31 +5530,17 @@ class DuckLakeWriteCoordinator {
     } catch {}
   }
   async beginVersionTransaction(input) {
-    if (!isSqliteCatalog(this.options.catalog)) {
+    if (input.mode === "snapshot" && input.expectedLatestVersionId === undefined && !isSqliteCatalog(this.options.catalog)) {
       await input.runtime.run("BEGIN TRANSACTION");
-      if (input.expectedLatestVersionId === undefined && input.mode !== "append") {
-        return;
-      }
-      return this.snapshots.getLatestVersionSummaryForDefinition(input.runtime, input.dataset);
-    }
-    for (let attempt = 0;attempt < 3; attempt++) {
-      input.signal?.throwIfAborted();
-      const snapshotId = await this.currentSnapshotId(input.runtime);
-      const latestVersion = await this.snapshots.getLatestVersionSummaryForDefinition(input.runtime, input.dataset);
-      await input.runtime.run("BEGIN TRANSACTION");
-      if (await this.currentSnapshotId(input.runtime) === snapshotId) {
-        return latestVersion;
-      }
-      await input.runtime.run("ROLLBACK");
-    }
-    throw new LakeConcurrencyError(`[SixbDuckLake] Catalog changed during three write preparations for dataset '${input.dataset.id}'. Retry the write.`);
-  }
-  async currentSnapshotId(runtime) {
-    const [row] = await runtime.query(`SELECT id FROM ${quoteIdentifier(duckLakeAlias(this.options))}.current_snapshot()`);
-    if (row === undefined) {
-      throw new LakeStorageError24("[SixbDuckLake] DuckLake did not return a current snapshot.");
+      return;
     }
-    return String(getBigIntLike(row, "id"));
+    return beginTransactionOnLatestVersion({
+      options: this.options,
+      runtime: input.runtime,
+      datasetId: input.dataset.id,
+      signal: input.signal,
+      readLatestVersion: () => this.snapshots.getLatestVersionSummaryAtCurrentSnapshot(input.runtime, input.dataset)
+    });
   }
   async lastCommittedSnapshotId(runtime) {
     const [row] = await runtime.query(`SELECT id FROM ${quoteIdentifier(duckLakeAlias(this.options))}.last_committed_snapshot()`);
@@ -4301,12 +5585,33 @@ class DuckLakeWriteCoordinator {
       return;
     }
     if (input.actualLatestVersionId !== input.expectedLatestVersionId) {
-      throw new LakeConcurrencyError(`[SixbDuckLake] Optimistic ${input.operation} failed for dataset '${input.datasetId}': expected latest version '${input.expectedLatestVersionId ?? "none"}', found '${input.actualLatestVersionId ?? "none"}'.`);
+      throw new LakeConcurrencyError2(`[SixbDuckLake] Optimistic ${input.operation} failed for dataset '${input.datasetId}': expected latest version '${input.expectedLatestVersionId ?? "none"}', found '${input.actualLatestVersionId ?? "none"}'.`);
     }
   }
 }
-function isSqliteCatalog(catalog) {
-  return catalog.type === "sqlite" || catalog.type === "custom" && catalog.uri.toLowerCase().startsWith("sqlite:");
+function mustClaimTable(input, changeResult, lineage) {
+  if (!changeResult.dataChangeExpected || changeResult.targetWasEmpty) {
+    return true;
+  }
+  if (input.mode === "snapshot") {
+    return false;
+  }
+  return getDatasetPrimaryKeyColumns4(input.dataset) !== null || carriesDependencies(input.producer, lineage);
+}
+function mustCommitOnReadSnapshot(input, changeResult) {
+  if (!changeResult.changedTableRows) {
+    return true;
+  }
+  return input.expectedLatestVersionId !== undefined && input.mode === "append" && getDatasetPrimaryKeyColumns4(input.dataset) === null;
+}
+function carriesDependencies(producer, lineage) {
+  if (lineage === undefined) {
+    return producer?.kind === "pipeline";
+  }
+  if ("unknown" in lineage) {
+    return true;
+  }
+  return lineage.rows.length > 0 || Object.values(lineage.columns).some((refs) => refs.length > 0);
 }
 function commitRowCount(input, changeResult, latestVersion) {
   if (input.mode === "append" && input.expectedLatestVersionId === undefined) {
@@ -4322,7 +5627,7 @@ function commitRowCount(input, changeResult, latestVersion) {
 }
 function assertDuckLakeSnapshotId3(snapshotId) {
   if (!/^\d+$/.test(snapshotId)) {
-    throw new LakeStorageError24(`[SixbDuckLake] Invalid DuckLake snapshot id '${snapshotId}'.`);
+    throw new LakeStorageError26(`[SixbDuckLake] Invalid DuckLake snapshot id '${snapshotId}'.`);
   }
 }
 var localCatalogCommitLocks = new Map;
@@ -4353,6 +5658,7 @@ async function withCatalogCommitLock(options, run) {
 // src/ducklake-storage.ts
 class DuckLakeStorage {
   standard = { id: "ducklake", version: "1.0" };
+  recordsLineage = true;
   sql;
   connections;
   datasets;
@@ -4374,15 +5680,15 @@ class DuckLakeStorage {
     };
     this.connections = new DuckLakeConnectionManager(normalizedOptions);
     this.datasets = new DuckLakeDatasetCatalog(normalizedOptions, this.connections);
-    this.maintenance = new DuckLakeMaintenance(normalizedOptions, this.connections);
     this.snapshotReader = new DuckLakeSnapshotReader(normalizedOptions, this.connections, this.datasets);
+    this.maintenance = new DuckLakeMaintenance(normalizedOptions, this.connections, this.snapshotReader);
     this.rows = new DuckLakeRowReader(normalizedOptions, this.connections, this.snapshotReader);
     this.changes = new DuckLakeChangeReader(normalizedOptions, this.connections, this.snapshotReader);
     this.writes = new DuckLakeWriteCoordinator(normalizedOptions, this.connections, this.datasets, this.snapshotReader);
     this.sql = new DuckLakeSqlExecutor(normalizedOptions, this.connections, this.datasets, this.snapshotReader, this.writes);
   }
   async createDataset(definition) {
-    return this.datasets.createDataset(definition);
+    return this.datasets.createDataset(definition, (runtime, dataset) => this.snapshotReader.getLatestVersionSummaryAtCurrentSnapshot(runtime, dataset));
   }
   async assertDatasetDefinitionsCompatible(definitions) {
     await this.datasets.assertDatasetDefinitionsCompatible(definitions);
@@ -4431,4 +5737,4 @@ export {
   DuckLakeStorage
 };
 
-//# debugId=99E61447F4C74C9E64756E2164756E21
+//# debugId=9F26B0918A26BCC464756E2164756E21