@sixb/ducklake 0.0.1 → 0.1.1

package/dist/index.js

@@ -989,6 +989,9 @@ function datasetColumnToDuckDbSql(column) {
 function datasetSchemaToDuckDbColumnsSql(schema) {
   return schema.columns.map((column) => datasetColumnToDuckDbSql(column)).join(", ");
 }
+function datasetSchemaToDuckDbNullableColumnsSql(schema) {
+  return schema.columns.map((column) => `${quoteIdentifier(column.name)} ${datasetColumnTypeToDuckDbSql(column.type)}`).join(", ");
+}
 function datasetSchemaColumnNamesSql(schema) {
   return schema.columns.map((column) => quoteIdentifier(column.name)).join(", ");
 }
@@ -1056,6 +1059,8 @@ function getIntegerField(record, key) {
 }
 
 // src/internal/ducklake-catalog-schema.ts
+var SIXB_PRIMARY_KEY_COMMENT_NAMESPACE = "sixb:primary-key:";
+var SIXB_PRIMARY_KEY_COMMENT_PREFIX = `${SIXB_PRIMARY_KEY_COMMENT_NAMESPACE}v1:`;
 var FILE_REF_STRUCT_CHILDREN = [
   { name: "blobId", type: "string" },
   { name: "digest", type: "string" },
@@ -1080,6 +1085,58 @@ function duckLakeCatalogColumnsToDatasetSchema(tableName, rows) {
     columns: topLevelColumns.sort(compareDuckLakeCatalogColumnOrder).map((column) => duckLakeCatalogColumnToDefinition(tableName, column, childrenByParentId))
   };
 }
+function duckLakePrimaryKeyColumnComment(ordinal) {
+  return `${SIXB_PRIMARY_KEY_COMMENT_PREFIX}${ordinal}`;
+}
+function duckLakeCatalogColumnsToDatasetPrimaryKey(tableName, rows) {
+  const keyedColumns = [];
+  const seenOrdinals = new Set;
+  for (const column of rows) {
+    const comment = column.comment;
+    if (comment === undefined || !comment.startsWith(SIXB_PRIMARY_KEY_COMMENT_NAMESPACE)) {
+      continue;
+    }
+    if (column.parentColumnId !== undefined || !comment.startsWith(SIXB_PRIMARY_KEY_COMMENT_PREFIX)) {
+      throwInvalidPrimaryKeyMetadata(tableName, column, comment);
+    }
+    const ordinalText = comment.slice(SIXB_PRIMARY_KEY_COMMENT_PREFIX.length);
+    if (!/^(0|[1-9]\d*)$/.test(ordinalText)) {
+      throwInvalidPrimaryKeyMetadata(tableName, column, comment);
+    }
+    const ordinal = Number(ordinalText);
+    if (!Number.isSafeInteger(ordinal)) {
+      throwInvalidPrimaryKeyMetadata(tableName, column, comment);
+    }
+    if (seenOrdinals.has(ordinal)) {
+      throw new LakeStorageError8(`[SixbDuckLake] Dataset table '${tableName}' has duplicate primary-key ordinal '${ordinal}'.`);
+    }
+    seenOrdinals.add(ordinal);
+    if (duckDbTypeToDatasetColumnType(column.columnType) !== "string") {
+      throw new LakeStorageError8(`[SixbDuckLake] Dataset table '${tableName}' primary-key column '${column.columnName}' must map to type 'string'.`);
+    }
+    if (column.nullsAllowed) {
+      throw new LakeStorageError8(`[SixbDuckLake] Dataset table '${tableName}' primary-key column '${column.columnName}' must not be nullable.`);
+    }
+    keyedColumns.push({ column, ordinal });
+  }
+  if (keyedColumns.length === 0) {
+    return;
+  }
+  keyedColumns.sort((left, right) => left.ordinal - right.ordinal);
+  for (const [expectedOrdinal, keyedColumn] of keyedColumns.entries()) {
+    if (keyedColumn.ordinal !== expectedOrdinal) {
+      throw new LakeStorageError8(`[SixbDuckLake] Dataset table '${tableName}' primary-key ordinals must be contiguous from 0.`);
+    }
+  }
+  const columnNames = keyedColumns.map(({ column }) => column.columnName);
+  if (columnNames.length === 1) {
+    return columnNames[0];
+  }
+  return columnNames;
+}
+function throwInvalidPrimaryKeyMetadata(tableName, column, comment) {
+  throw new LakeStorageError8(`[SixbDuckLake] Dataset table '${tableName}' has malformed primary-key metadata '${comment}' on column '${column.columnName}'.`);
+}
 function duckLakeCatalogColumnToDefinition(tableName, column, childrenByParentId) {
   const children = [...childrenByParentId.get(column.columnId) ?? []].sort(compareDuckLakeCatalogColumnOrder);
   return {
@@ -1205,11 +1262,14 @@ async function readCurrentDatasetDefinitions(input) {
   const partitionByTableName = groupPartitionsByTableName(await readCurrentDatasetPartitions(input, tableNames));
   const definitions = new Map;
   for (const table of tables) {
+    const columns = columnsByTableName.get(table.tableName) ?? [];
+    const primaryKey = duckLakeCatalogColumnsToDatasetPrimaryKey(table.tableName, columns);
     const partitionBy = partitionByTableName.get(table.tableName) ?? [];
     definitions.set(table.datasetId, {
       kind: "dataset",
       id: table.datasetId,
-      schema: duckLakeCatalogColumnsToDatasetSchema(table.tableName, columnsByTableName.get(table.tableName) ?? []),
+      schema: duckLakeCatalogColumnsToDatasetSchema(table.tableName, columns),
+      ...primaryKey !== undefined ? { primaryKey } : {},
       ...partitionBy.length > 0 ? { partitionBy } : {},
       ...table.comment !== undefined ? { description: table.comment } : {}
     });
@@ -1243,6 +1303,7 @@ async function readCurrentDatasetColumns(input, tableNames) {
   }
   const ducklakeTable = duckLakeMetadataTableName(input.options, "ducklake_table");
   const ducklakeColumn = duckLakeMetadataTableName(input.options, "ducklake_column");
+  const ducklakeColumnTag = duckLakeMetadataTableName(input.options, "ducklake_column_tag");
   const rows = await input.runtime.query(`
     SELECT
       table_meta.table_name,
@@ -1251,11 +1312,17 @@ async function readCurrentDatasetColumns(input, tableNames) {
       column_meta.column_name,
       column_meta.column_type,
       CAST(column_meta.nulls_allowed AS BOOLEAN) AS nulls_allowed,
-      column_meta.parent_column
+      column_meta.parent_column,
+      column_tag."value" AS comment
     FROM ${ducklakeTable} table_meta
     JOIN ${ducklakeColumn} column_meta
       ON column_meta.table_id = table_meta.table_id
       AND column_meta.end_snapshot IS NULL
+    LEFT JOIN ${ducklakeColumnTag} column_tag
+      ON column_tag.table_id = column_meta.table_id
+      AND column_tag.column_id = column_meta.column_id
+      AND column_tag."key" = 'comment'
+      AND column_tag.end_snapshot IS NULL
     WHERE table_meta.end_snapshot IS NULL
       AND table_meta.table_name IN (${quoteSqlStringList(tableNames)})
     ORDER BY table_meta.table_name, column_meta.column_order
@@ -1267,7 +1334,8 @@ async function readCurrentDatasetColumns(input, tableNames) {
     columnName: getString(row, "column_name"),
     columnType: getString(row, "column_type"),
     nullsAllowed: getBoolean(row, "nulls_allowed"),
-    parentColumnId: getOptionalBigIntLike(row, "parent_column")
+    parentColumnId: getOptionalBigIntLike(row, "parent_column"),
+    comment: getOptionalString(row, "comment")
   }));
 }
 async function readCurrentDatasetPartitions(input, tableNames) {
@@ -1352,11 +1420,23 @@ class DuckLakeDatasetCatalog {
     const tableName = encodeDatasetTableName(definition.id);
     const qualifiedName = qualifiedTableName(this.options, tableName);
     await this.connections.withExclusiveAttached(async (runtime) => {
-      await runtime.run(`CREATE TABLE ${qualifiedName} (${datasetSchemaToDuckDbColumnsSql(definition.schema)})`);
-      if (definition.description !== undefined) {
-        await runtime.run(`COMMENT ON TABLE ${qualifiedName} IS ${quoteSqlString(definition.description)}`);
+      await runtime.run("BEGIN TRANSACTION");
+      let committed = false;
+      try {
+        await runtime.run(`CREATE TABLE ${qualifiedName} (${datasetSchemaToDuckDbColumnsSql(definition.schema)})`);
+        if (definition.description !== undefined) {
+          await runtime.run(`COMMENT ON TABLE ${qualifiedName} IS ${quoteSqlString(definition.description)}`);
+        }
+        await this.applyPrimaryKey(runtime, qualifiedName, definition);
+        await this.applyPartitionBy(runtime, qualifiedName, definition);
+        await runtime.run("COMMIT");
+        committed = true;
+      } catch (error) {
+        if (!committed) {
+          await this.rollbackTransaction(runtime);
+        }
+        throw error;
       }
-      await this.applyPartitionBy(runtime, qualifiedName, definition);
     });
     this.connections.markLocalCatalogChanged();
     return structuredClone(definition);
@@ -1484,6 +1564,15 @@ ${details}`);
       throw new LakeStorageError11(`[SixbDuckLake] Dataset '${definition.id}' cannot apply partitionBy because DuckLake rejected ALTER TABLE SET PARTITIONED BY: ${message}`);
     }
   }
+  async applyPrimaryKey(runtime, qualifiedName, definition) {
+    if (definition.primaryKey === undefined) {
+      return;
+    }
+    const columnNames = typeof definition.primaryKey === "string" ? [definition.primaryKey] : definition.primaryKey;
+    for (const [ordinal, columnName] of columnNames.entries()) {
+      await runtime.run(`COMMENT ON COLUMN ${qualifiedName}.${quoteIdentifier(columnName)} IS ${quoteSqlString(duckLakePrimaryKeyColumnComment(ordinal))}`);
+    }
+  }
   async setSchemaEvolutionCommitMetadata(runtime, datasetId, plan) {
     await runtime.run(`CALL ${quoteIdentifier(duckLakeAlias(this.options))}.set_commit_message(${quoteSqlString("Sixb")}, ${quoteSqlString(`evolve dataset ${datasetId} schema`)}, extra_info => ${quoteSqlString(JSON.stringify({
       sixb: {
@@ -1640,14 +1729,16 @@ function parseCommitMetadata(value) {
   const mode = parsed.sixb.mode;
   const commitId = parsed.sixb.commitId;
   const rowCount = parsed.sixb.rowCount;
+  const validatedPrimaryKeyColumns = parsed.sixb.validatedPrimaryKeyColumns;
   return {
     kind: "datasetVersion",
     datasetId: parsed.sixb.datasetId,
     ...typeof commitId === "string" ? { commitId } : {},
-    ...mode === "snapshot" || mode === "append" || mode === "schema" ? { mode } : {},
+    ...mode === "snapshot" || mode === "append" || mode === "merge" || mode === "schema" ? { mode } : {},
     ...isDatasetProducer(parsed.sixb.producer) ? { producer: parsed.sixb.producer } : {},
     ...isDatasetVersionRefs(parsed.sixb.inputs) ? { inputs: parsed.sixb.inputs } : {},
     ...typeof commitId === "string" && isRowCount(rowCount) ? { rowCount } : {},
+    ...typeof commitId === "string" && isValidatedPrimaryKeyColumns(validatedPrimaryKeyColumns) ? { validatedPrimaryKeyColumns } : {},
     ...isSchemaChangeMetadata(parsed.sixb.schemaChange) ? { schemaChange: parsed.sixb.schemaChange } : {}
   };
 }
@@ -1688,6 +1779,9 @@ function optionalString(value) {
 function isRowCount(value) {
   return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
 }
+function isValidatedPrimaryKeyColumns(value) {
+  return Array.isArray(value) && value.length > 0 && value.every((column) => typeof column === "string" && column.trim().length > 0) && new Set(value).size === value.length;
+}
 
 // src/internal/ducklake-row-reader.ts
 class DuckLakeRowReader {
@@ -2002,7 +2096,8 @@ class DuckLakeSnapshotReader {
     return {
       datasetId: tableRef.datasetId,
       versionId: toVersionId(row.snapshotId),
-      ...row.metadata?.rowCount !== undefined ? { rowCount: row.metadata.rowCount } : {}
+      ...row.metadata?.rowCount !== undefined ? { rowCount: row.metadata.rowCount } : {},
+      ...row.metadata?.validatedPrimaryKeyColumns !== undefined ? { validatedPrimaryKeyColumns: row.metadata.validatedPrimaryKeyColumns } : {}
     };
   }
   async getVersionForSnapshot(runtime, dataset, snapshotId) {
@@ -2062,7 +2157,7 @@ class DuckLakeSnapshotReader {
     if (!row) {
       return null;
     }
-    if (!options.includeParent || row.mode !== "append" && row.mode !== "schema") {
+    if (!options.includeParent || row.mode !== "append" && row.mode !== "merge" && row.mode !== "schema") {
       return row;
     }
     const [parent] = await this.collectVisibleSnapshotRows(runtime, {
@@ -2250,7 +2345,7 @@ class DuckLakeSnapshotReader {
   }
   withParentSnapshotIds(snapshots) {
     return snapshots.map((snapshot, index) => {
-      const parentSnapshotId = snapshot.mode === "append" || snapshot.mode === "schema" ? snapshots[index + 1]?.snapshotId : undefined;
+      const parentSnapshotId = snapshot.mode === "append" || snapshot.mode === "merge" || snapshot.mode === "schema" ? snapshots[index + 1]?.snapshotId : undefined;
       return parentSnapshotId === undefined ? snapshot : { ...snapshot, parentSnapshotId };
     });
   }
@@ -2271,7 +2366,10 @@ import {
 } from "@sixb/core/lake-storage";
 
 // src/internal/dataset-row-commit.ts
-import { LakeStorageError as LakeStorageError16 } from "@sixb/core/lake-storage";
+import {
+  getDatasetPrimaryKeyColumns,
+  LakeStorageError as LakeStorageError16
+} from "@sixb/core/lake-storage";
 function assertDatasetWriteMode(mode, operation) {
   if (mode !== "snapshot" && mode !== "append") {
     throw new LakeStorageError16(`[SixbDuckLake] Invalid ${operation} mode '${String(mode)}'. Expected 'snapshot' or 'append'.`);
@@ -2281,6 +2379,30 @@ async function applyDatasetRowsFromRelation(input) {
   const tableName = encodeDatasetTableName(input.dataset.id);
   const table = qualifiedTableName(input.options, tableName);
   const columnsSql = datasetSchemaColumnNamesSql(input.dataset.schema);
+  const primaryKeyColumns = getDatasetPrimaryKeyColumns(input.dataset);
+  if (primaryKeyColumns !== null) {
+    await inspectUniqueKeyedRelation({
+      runtime: input.runtime,
+      dataset: input.dataset,
+      relationSql: input.sourceRelationSql,
+      context: `${input.mode} source`
+    });
+    if (input.mode === "append") {
+      await inspectUniqueKeyedRelation({
+        runtime: input.runtime,
+        dataset: input.dataset,
+        relationSql: table,
+        context: "current baseline"
+      });
+      await assertNoPrimaryKeyOverlap({
+        runtime: input.runtime,
+        dataset: input.dataset,
+        leftRelationSql: input.sourceRelationSql,
+        rightRelationSql: table,
+        context: "append"
+      });
+    }
+  }
   if (input.mode === "snapshot") {
     const source = await fingerprintRelation(input.runtime, input.sourceRelationSql, columnsSql);
     const target = await fingerprintRelation(input.runtime, table, columnsSql);
@@ -2307,6 +2429,44 @@ async function applyDatasetRowsFromRelation(input) {
     resultingRowCount: input.previousRowCount === undefined ? { kind: "unknown" } : { kind: "exact", value: input.previousRowCount + sourceRowCount }
   };
 }
+async function inspectUniqueKeyedRelation(input) {
+  const primaryKeyColumns = getDatasetPrimaryKeyColumns(input.dataset);
+  if (primaryKeyColumns === null) {
+    throw new LakeStorageError16(`[SixbDuckLake] Dataset '${input.dataset.id}' must define a primaryKey before rows can be keyed.`);
+  }
+  const keysSql = primaryKeyColumns.map((column) => quoteIdentifier(column)).join(", ");
+  const [row] = await input.runtime.query(`
+    SELECT
+      count(*) AS row_count,
+      coalesce(max(key_count), 0) AS max_key_count
+    FROM (
+      SELECT count(*) OVER (PARTITION BY ${keysSql}) AS key_count
+      FROM ${input.relationSql}
+    ) keyed_rows
+  `);
+  const rowCount = row === undefined ? 0 : Number(getBigIntLike(row, "row_count"));
+  const maxKeyCount = row === undefined ? 0 : Number(getBigIntLike(row, "max_key_count"));
+  if (maxKeyCount > 1) {
+    throw new LakeStorageError16(`[SixbDuckLake] Dataset '${input.dataset.id}' ${input.context} contains duplicate primary key.`);
+  }
+  return rowCount;
+}
+async function assertNoPrimaryKeyOverlap(input) {
+  const primaryKeyColumns = getDatasetPrimaryKeyColumns(input.dataset);
+  if (primaryKeyColumns === null) {
+    return;
+  }
+  const matchSql = primaryKeyColumns.map((column) => `left_rows.${quoteIdentifier(column)} = right_rows.${quoteIdentifier(column)}`).join(" AND ");
+  const [collision] = await input.runtime.query(`
+    SELECT 1
+    FROM ${input.leftRelationSql} left_rows
+    JOIN ${input.rightRelationSql} right_rows ON ${matchSql}
+    LIMIT 1
+  `);
+  if (collision !== undefined) {
+    throw new LakeStorageError16(`[SixbDuckLake] Dataset '${input.dataset.id}' ${input.context} contains duplicate primary key.`);
+  }
+}
 async function fingerprintRelation(runtime, relationSql, columnsSql) {
   const [row] = await runtime.query(`
     SELECT
@@ -2604,13 +2764,183 @@ function throwResultSchemaMismatch(datasetId, detail) {
 }
 
 // src/internal/ducklake-write-coordinator.ts
-import { randomUUID as randomUUID5 } from "crypto";
-import { LakeStorageError as LakeStorageError20 } from "@sixb/core/lake-storage";
+import { randomUUID as randomUUID6 } from "crypto";
+import { getDatasetPrimaryKeyColumns as getDatasetPrimaryKeyColumns4, LakeStorageError as LakeStorageError22 } from "@sixb/core/lake-storage";
 
-// src/internal/ducklake-write-session.ts
+// src/internal/dataset-row-merge.ts
+import { getDatasetPrimaryKeyColumns as getDatasetPrimaryKeyColumns2, LakeStorageError as LakeStorageError19 } from "@sixb/core/lake-storage";
+async function applyDatasetMergeFromRelation(input) {
+  const primaryKeyColumns = getDatasetPrimaryKeyColumns2(input.dataset);
+  if (primaryKeyColumns === null) {
+    throw new LakeStorageError19(`[SixbDuckLake] Dataset '${input.dataset.id}' must define a primaryKey before it can be merged.`);
+  }
+  const stagingTable = quoteIdentifier(input.stagingTableName);
+  const targetTable = qualifiedTableName(input.options, encodeDatasetTableName(input.dataset.id));
+  const sourceRowCount = await countRows2(input.runtime, stagingTable);
+  const previousRowCount = await currentBaselineRowCount(input, targetTable, primaryKeyColumns);
+  return withMaterializedFinalChanges(input, primaryKeyColumns, async (finalChanges) => {
+    const effects = await countMergeEffects(input, targetTable, finalChanges, primaryKeyColumns);
+    const effectiveChangeCount = effects.inserts + effects.updates + effects.deletes;
+    if (effectiveChangeCount === 0) {
+      return {
+        dataChangeExpected: false,
+        sourceRowCount,
+        resultingRowCount: { kind: "exact", value: previousRowCount }
+      };
+    }
+    const keyMatchSql = primaryKeyMatchSql(primaryKeyColumns, "target", "source");
+    const rowDifferenceSql = datasetRowDifferenceSql(input.dataset, "target", "source");
+    const kindColumn = `source.${quoteIdentifier(input.kindColumnName)}`;
+    await input.runtime.run(`
+      DELETE FROM ${targetTable} AS target
+      USING ${finalChanges} AS source
+      WHERE ${keyMatchSql}
+        AND (
+          ${kindColumn} = ${quoteSqlString("delete")}
+          OR (
+            ${kindColumn} = ${quoteSqlString("upsert")}
+            AND (${rowDifferenceSql})
+          )
+        )
+    `);
+    const columnsSql = datasetSchemaColumnNamesSql(input.dataset.schema);
+    const selectedColumnsSql = input.dataset.schema.columns.map((column) => `source.${quoteIdentifier(column.name)}`).join(", ");
+    const firstSequenceColumnName = firstSequenceName(input.sequenceColumnName);
+    await input.runtime.run(`
+      INSERT INTO ${targetTable} (${columnsSql})
+      SELECT ${selectedColumnsSql}
+      FROM ${finalChanges} AS source
+      WHERE ${kindColumn} = ${quoteSqlString("upsert")}
+        AND NOT EXISTS (
+          SELECT 1
+          FROM ${targetTable} AS target
+          WHERE ${keyMatchSql}
+        )
+      ORDER BY source.${quoteIdentifier(firstSequenceColumnName)}
+    `);
+    return {
+      dataChangeExpected: true,
+      sourceRowCount,
+      resultingRowCount: {
+        kind: "exact",
+        value: previousRowCount + effects.inserts - effects.deletes
+      }
+    };
+  });
+}
+async function currentBaselineRowCount(input, targetTable, primaryKeyColumns) {
+  if (sameColumns(input.validatedPrimaryKeyColumns, primaryKeyColumns)) {
+    return input.previousRowCount ?? countRows2(input.runtime, targetTable);
+  }
+  return inspectUniqueKeyedRelation({
+    runtime: input.runtime,
+    dataset: input.dataset,
+    relationSql: targetTable,
+    context: "current baseline"
+  });
+}
+async function withMaterializedFinalChanges(input, primaryKeyColumns, run) {
+  const table = quoteIdentifier(`${input.stagingTableName}_final`);
+  await input.runtime.run(`
+    CREATE TEMP TABLE ${table} AS
+    ${finalChangesSelectSql(input, primaryKeyColumns)}
+  `);
+  let outcome;
+  try {
+    outcome = { kind: "success", value: await run(table) };
+  } catch (error) {
+    outcome = { kind: "error", error };
+  }
+  try {
+    await input.runtime.run(`DROP TABLE IF EXISTS ${table}`);
+  } catch (cleanupError) {
+    if (outcome.kind === "success") {
+      throw cleanupError;
+    }
+  }
+  if (outcome.kind === "error") {
+    throw outcome.error;
+  }
+  return outcome.value;
+}
+function finalChangesSelectSql(input, primaryKeyColumns) {
+  const sequenceColumn = quoteIdentifier(input.sequenceColumnName);
+  const keyColumnsSql = primaryKeyColumns.map((column) => quoteIdentifier(column)).join(", ");
+  const rankColumn = quoteIdentifier(rankName(input.sequenceColumnName));
+  const firstSequenceColumn = quoteIdentifier(firstSequenceName(input.sequenceColumnName));
+  return `
+    SELECT * EXCLUDE (${rankColumn})
+    FROM (
+      SELECT
+        *,
+        min(${sequenceColumn}) OVER (PARTITION BY ${keyColumnsSql}) AS ${firstSequenceColumn},
+        row_number() OVER (
+          PARTITION BY ${keyColumnsSql}
+          ORDER BY ${sequenceColumn} DESC
+        ) AS ${rankColumn}
+      FROM ${quoteIdentifier(input.stagingTableName)}
+    ) ranked_changes
+    WHERE ${rankColumn} = 1
+  `;
+}
+async function countMergeEffects(input, targetTable, finalChanges, primaryKeyColumns) {
+  const matchColumn = primaryKeyColumns[0];
+  if (matchColumn === undefined) {
+    throw new LakeStorageError19(`[SixbDuckLake] Dataset '${input.dataset.id}' primaryKey must contain at least one column.`);
+  }
+  const keyMatchSql = primaryKeyMatchSql(primaryKeyColumns, "target", "source");
+  const targetMatch = `target.${quoteIdentifier(matchColumn)} IS NOT NULL`;
+  const kindColumn = `source.${quoteIdentifier(input.kindColumnName)}`;
+  const rowDifferenceSql = datasetRowDifferenceSql(input.dataset, "target", "source");
+  const [row] = await input.runtime.query(`
+    SELECT
+      count(*) FILTER (
+        WHERE ${kindColumn} = ${quoteSqlString("upsert")} AND NOT (${targetMatch})
+      ) AS insert_count,
+      count(*) FILTER (
+        WHERE ${kindColumn} = ${quoteSqlString("upsert")}
+          AND ${targetMatch}
+          AND (${rowDifferenceSql})
+      ) AS update_count,
+      count(*) FILTER (
+        WHERE ${kindColumn} = ${quoteSqlString("delete")} AND ${targetMatch}
+      ) AS delete_count
+    FROM ${finalChanges} AS source
+    LEFT JOIN ${targetTable} AS target ON ${keyMatchSql}
+  `);
+  return {
+    inserts: row === undefined ? 0 : Number(getBigIntLike(row, "insert_count")),
+    updates: row === undefined ? 0 : Number(getBigIntLike(row, "update_count")),
+    deletes: row === undefined ? 0 : Number(getBigIntLike(row, "delete_count"))
+  };
+}
+function primaryKeyMatchSql(primaryKeyColumns, leftAlias, rightAlias) {
+  return primaryKeyColumns.map((column) => `${leftAlias}.${quoteIdentifier(column)} = ${rightAlias}.${quoteIdentifier(column)}`).join(" AND ");
+}
+function datasetRowDifferenceSql(dataset, leftAlias, rightAlias) {
+  return dataset.schema.columns.map((column) => `${leftAlias}.${quoteIdentifier(column.name)} IS DISTINCT FROM ${rightAlias}.${quoteIdentifier(column.name)}`).join(" OR ");
+}
+async function countRows2(runtime, relationSql) {
+  const [row] = await runtime.query(`SELECT count(*) AS row_count FROM ${relationSql}`);
+  return row === undefined ? 0 : Number(getBigIntLike(row, "row_count"));
+}
+function rankName(sequenceColumnName) {
+  return `${sequenceColumnName}_rank`;
+}
+function firstSequenceName(sequenceColumnName) {
+  return `${sequenceColumnName}_first`;
+}
+function sameColumns(left, right) {
+  return left !== undefined && left.length === right.length && left.every((value, index) => value === right[index]);
+}
+
+// src/internal/ducklake-merge-session.ts
 import { randomUUID as randomUUID4 } from "crypto";
-import { getDatasetRowValidationError } from "@sixb/core";
-import { LakeStorageError as LakeStorageError19 } from "@sixb/core/lake-storage";
+import {
+  cloneDatasetMergeChange,
+  getDatasetMergeChangeValidationError,
+  LakeStorageError as LakeStorageError20
+} from "@sixb/core/lake-storage";
 
 // src/internal/row-appender.ts
 function appendDatasetRow(appender, schema, row) {
@@ -2665,10 +2995,134 @@ function fileRefToDuckDbStruct(fileRef) {
   };
 }
 
+// src/internal/ducklake-merge-session.ts
+var MERGE_CHANGE_BATCH_SIZE = 1000;
+async function createDuckLakeMergeSession(input) {
+  const suffix = randomUUID4().replaceAll("-", "");
+  const stagingTableName = `sixb_merge_${suffix}`;
+  const sequenceColumnName = `sixb_merge_sequence_${suffix}`;
+  const kindColumnName = `sixb_merge_kind_${suffix}`;
+  await input.runtime.run(`
+    CREATE TEMP TABLE ${quoteIdentifier(stagingTableName)} (
+      ${quoteIdentifier(sequenceColumnName)} UBIGINT NOT NULL,
+      ${quoteIdentifier(kindColumnName)} VARCHAR NOT NULL,
+      ${datasetSchemaToDuckDbNullableColumnsSql(input.merge.dataset.schema)}
+    )
+  `);
+  return new DuckLakeMergeSession(input.commitMerge, input.runtime, input.merge, input.baseVersionId, stagingTableName, sequenceColumnName, kindColumnName);
+}
+
+class DuckLakeMergeSession {
+  commitMerge;
+  runtime;
+  input;
+  baseVersionId;
+  stagingTableName;
+  sequenceColumnName;
+  kindColumnName;
+  closed = false;
+  cleanedUp = false;
+  nextSequence = 0n;
+  changesWritten = 0;
+  constructor(commitMerge, runtime, input, baseVersionId, stagingTableName, sequenceColumnName, kindColumnName) {
+    this.commitMerge = commitMerge;
+    this.runtime = runtime;
+    this.input = input;
+    this.baseVersionId = baseVersionId;
+    this.stagingTableName = stagingTableName;
+    this.sequenceColumnName = sequenceColumnName;
+    this.kindColumnName = kindColumnName;
+  }
+  async writeChanges(changes) {
+    this.assertOpen();
+    let batch = [];
+    for await (const change of changes) {
+      const validationError = getDatasetMergeChangeValidationError(change, this.input.dataset);
+      if (validationError) {
+        throw new LakeStorageError20(`[SixbDuckLake] ${validationError}`);
+      }
+      batch.push({
+        sequence: this.nextSequence,
+        change: cloneDatasetMergeChange(change)
+      });
+      this.nextSequence += 1n;
+      if (batch.length >= MERGE_CHANGE_BATCH_SIZE) {
+        await this.appendBatchToStagingTable(batch);
+        batch = [];
+      }
+    }
+    if (batch.length > 0) {
+      await this.appendBatchToStagingTable(batch);
+    }
+  }
+  async appendBatchToStagingTable(batch) {
+    await this.runtime.withAppender(this.stagingTableName, (appender) => {
+      for (const staged of batch) {
+        appender.appendBigInt(staged.sequence);
+        appender.appendVarchar(staged.change.kind);
+        appendDatasetRow(appender, this.input.dataset.schema, staged.change.kind === "upsert" ? staged.change.row : staged.change.key);
+      }
+    });
+    this.changesWritten += batch.length;
+  }
+  async commit(input) {
+    this.assertOpen();
+    this.closed = true;
+    let committed = false;
+    try {
+      const result = await this.commitMerge({
+        merge: this.input,
+        baseVersionId: this.baseVersionId,
+        commit: input,
+        stagingTableName: this.stagingTableName,
+        sequenceColumnName: this.sequenceColumnName,
+        kindColumnName: this.kindColumnName,
+        changesWritten: this.changesWritten
+      });
+      committed = true;
+      await this.cleanup();
+      return result;
+    } catch (error) {
+      if (!committed) {
+        await this.cleanup();
+      }
+      throw error;
+    }
+  }
+  async abort() {
+    if (this.closed) {
+      return;
+    }
+    this.closed = true;
+    await this.cleanup();
+  }
+  assertOpen() {
+    if (this.closed) {
+      throw new LakeStorageError20("[SixbDuckLake] Merge session is already closed.");
+    }
+  }
+  async cleanup() {
+    if (this.cleanedUp) {
+      return;
+    }
+    this.cleanedUp = true;
+    try {
+      await this.runtime.run(`DROP TABLE IF EXISTS ${quoteIdentifier(this.stagingTableName)}`);
+    } catch {}
+  }
+}
+
 // src/internal/ducklake-write-session.ts
+import { randomUUID as randomUUID5 } from "crypto";
+import { getDatasetRowValidationError } from "@sixb/core";
+import {
+  encodeDatasetPrimaryKey,
+  getDatasetPrimaryKeyColumns as getDatasetPrimaryKeyColumns3,
+  LakeStorageError as LakeStorageError21
+} from "@sixb/core/lake-storage";
 var WRITE_ROW_BATCH_SIZE = 1000;
 async function createDuckLakeWriteSession(input) {
-  const stagingTableName = `sixb_write_${randomUUID4().replaceAll("-", "")}`;
+  const stagingTableName = `sixb_write_${randomUUID5().replaceAll("-", "")}`;
   await input.runtime.run(`CREATE TEMP TABLE ${quoteIdentifier(stagingTableName)} (${datasetSchemaToDuckDbColumnsSql(input.write.dataset.schema)})`);
   return new DuckLakeWriteSession(input.commitWrite, input.runtime, input.write, stagingTableName);
 }
@@ -2681,6 +3135,7 @@ class DuckLakeWriteSession {
   closed = false;
   cleanedUp = false;
   rowsWritten = 0;
+  stagedPrimaryKeys = new Set;
   constructor(commitWrite, runtime, input, stagingTableName) {
     this.commitWrite = commitWrite;
     this.runtime = runtime;
@@ -2690,28 +3145,41 @@ class DuckLakeWriteSession {
   async writeRows(rows) {
     this.assertOpen();
     let batch = [];
+    let batchPrimaryKeys = new Set;
     for await (const row of rows) {
       const validationError = getDatasetRowValidationError(row, this.input.dataset);
       if (validationError) {
-        throw new LakeStorageError19(`[SixbDuckLake] ${validationError}`);
+        throw new LakeStorageError21(`[SixbDuckLake] ${validationError}`);
       }
-      batch.push(structuredClone(row));
+      const clonedRow = structuredClone(row);
+      if (getDatasetPrimaryKeyColumns3(this.input.dataset) !== null) {
+        const primaryKey = encodeDatasetPrimaryKey(this.input.dataset, clonedRow);
+        if (this.stagedPrimaryKeys.has(primaryKey) || batchPrimaryKeys.has(primaryKey)) {
+          throw new LakeStorageError21(`[SixbDuckLake] Dataset '${this.input.dataset.id}' ${this.input.mode ?? "snapshot"} source contains duplicate primary key.`);
+        }
+        batchPrimaryKeys.add(primaryKey);
+      }
+      batch.push(clonedRow);
       if (batch.length >= WRITE_ROW_BATCH_SIZE) {
-        await this.appendBatchToStagingTable(batch);
+        await this.appendBatchToStagingTable(batch, batchPrimaryKeys);
         batch = [];
+        batchPrimaryKeys = new Set;
       }
     }
     if (batch.length > 0) {
-      await this.appendBatchToStagingTable(batch);
+      await this.appendBatchToStagingTable(batch, batchPrimaryKeys);
     }
   }
-  async appendBatchToStagingTable(batch) {
+  async appendBatchToStagingTable(batch, primaryKeys) {
     await this.runtime.withAppender(this.stagingTableName, (appender) => {
       for (const row of batch) {
         appendDatasetRow(appender, this.input.dataset.schema, row);
       }
     });
     this.rowsWritten += batch.length;
+    for (const primaryKey of primaryKeys) {
+      this.stagedPrimaryKeys.add(primaryKey);
+    }
   }
   async commit(input) {
     this.assertOpen();
@@ -2743,7 +3211,7 @@ class DuckLakeWriteSession {
   }
   assertOpen() {
     if (this.closed) {
-      throw new LakeStorageError19("[SixbDuckLake] Write session is already closed.");
+      throw new LakeStorageError21("[SixbDuckLake] Write session is already closed.");
     }
   }
   async cleanup() {
@@ -2776,7 +3244,7 @@ class DuckLakeWriteCoordinator {
     this.connections.assertOpen();
     const definition = await this.datasets.getDataset(input.dataset.id);
     if (!definition) {
-      throw new LakeStorageError20(`[SixbDuckLake] Unknown dataset '${input.dataset.id}'.`);
+      throw new LakeStorageError22(`[SixbDuckLake] Unknown dataset '${input.dataset.id}'.`);
     }
     this.datasets.assertSchema(definition);
     const mode = input.mode ?? "snapshot";
@@ -2792,10 +3260,32 @@ class DuckLakeWriteCoordinator {
       }
     });
   }
+  async beginMerge(input) {
+    this.connections.assertOpen();
+    const definition = await this.datasets.getDataset(input.dataset.id);
+    if (!definition) {
+      throw new LakeStorageError22(`[SixbDuckLake] Unknown dataset '${input.dataset.id}'.`);
+    }
+    if (getDatasetPrimaryKeyColumns4(definition) === null) {
+      throw new LakeStorageError22(`[SixbDuckLake] Dataset '${definition.id}' must define a primaryKey before it can be merged.`);
+    }
+    this.datasets.assertSchema(definition);
+    const latestVersion = await this.connections.withAttachedRuntime((runtime2) => this.snapshots.getLatestVersionForDefinition(runtime2, definition));
+    if (input.expectedLatestVersionId !== undefined && latestVersion?.versionId !== input.expectedLatestVersionId) {
+      throw new LakeStorageError22(`[SixbDuckLake] Optimistic merge start failed for dataset '${definition.id}': expected latest version '${input.expectedLatestVersionId}', found '${latestVersion?.versionId ?? "none"}'.`);
+    }
+    const runtime = await this.connections.stagingRuntime();
+    return createDuckLakeMergeSession({
+      commitMerge: (options) => this.commitMerge(options),
+      runtime,
+      merge: { ...input, dataset: definition },
+      baseVersionId: latestVersion?.versionId ?? null
+    });
+  }
   async commitWrite(input) {
     const definition = await this.datasets.getDataset(input.write.dataset.id);
     if (!definition) {
-      throw new LakeStorageError20(`[SixbDuckLake] Unknown dataset '${input.write.dataset.id}'.`);
+      throw new LakeStorageError22(`[SixbDuckLake] Unknown dataset '${input.write.dataset.id}'.`);
     }
     const mode = input.write.mode ?? "snapshot";
     assertDatasetWriteMode(mode, "write");
@@ -2813,6 +3303,42 @@ class DuckLakeWriteCoordinator {
       })
     });
   }
+  async commitMerge(input) {
+    const definition = await this.datasets.getDataset(input.merge.dataset.id);
+    if (!definition) {
+      throw new LakeStorageError22(`[SixbDuckLake] Unknown dataset '${input.merge.dataset.id}'.`);
+    }
+    if (getDatasetPrimaryKeyColumns4(definition) === null) {
+      throw new LakeStorageError22(`[SixbDuckLake] Dataset '${definition.id}' must define a primaryKey before it can be merged.`);
+    }
+    return this.withCommitRuntime((runtime) => this.commitVersionOutcomeOnExclusiveRuntime(runtime, {
+      dataset: definition,
+      mode: "merge",
+      expectedLatestVersionId: input.baseVersionId,
+      allowInitialNoOp: true,
+      disableRetries: true,
+      optimisticOperation: "merge commit",
+      commitMessage: input.commit?.commitMessage ?? `merge dataset ${definition.id}`,
+      producer: input.merge.producer,
+      inputs: input.merge.inputs,
+      applyChanges: async (commitRuntime, context) => {
+        const result = await applyDatasetMergeFromRelation({
+          options: this.options,
+          runtime: commitRuntime,
+          dataset: definition,
+          stagingTableName: input.stagingTableName,
+          sequenceColumnName: input.sequenceColumnName,
+          kindColumnName: input.kindColumnName,
+          previousRowCount: context.previousRowCount,
+          validatedPrimaryKeyColumns: context.validatedPrimaryKeyColumns
+        });
+        if (result.sourceRowCount !== input.changesWritten) {
+          throw new LakeStorageError22(`[SixbDuckLake] Staged merge for dataset '${definition.id}' accepted ${input.changesWritten} change(s), but DuckLake saw ${result.sourceRowCount} source change(s) at commit time.`);
+        }
+        return result;
+      }
+    }));
+  }
   async commitDatasetVersion(input) {
     return this.withCommitRuntime((runtime) => this.commitDatasetVersionOnExclusiveRuntime(runtime, input));
   }
@@ -2822,11 +3348,21 @@ class DuckLakeWriteCoordinator {
     });
   }
   async commitDatasetVersionOnExclusiveRuntime(runtime, input) {
+    const result = await this.commitVersionOutcomeOnExclusiveRuntime(runtime, {
+      ...input,
+      disableRetries: input.mode === "append" && getDatasetPrimaryKeyColumns4(input.dataset) !== null
+    });
+    if (result.version === null) {
+      throw new LakeStorageError22(`[SixbDuckLake] No DuckLake changes were committed for dataset '${input.dataset.id}', and no previous version exists.`);
+    }
+    return { ...result.version, outcome: result.outcome };
+  }
+  async commitVersionOutcomeOnExclusiveRuntime(runtime, input) {
     const runtimeInput = { ...input, runtime };
     return this.withGuardedRetrySetting(runtimeInput, () => this.commitDatasetVersionUnlocked(runtimeInput));
   }
   async withGuardedRetrySetting(input, run) {
-    if (input.expectedLatestVersionId === undefined) {
+    if (input.expectedLatestVersionId === undefined && !input.disableRetries) {
       return run();
     }
     await input.runtime.run("SET ducklake_max_retry_count = 0");
@@ -2856,13 +3392,15 @@ class DuckLakeWriteCoordinator {
         this.assertExpectedLatestVersion({
           datasetId: input.dataset.id,
           expectedLatestVersionId: input.expectedLatestVersionId,
-          actualLatestVersionId: latestVersion?.versionId ?? null
+          actualLatestVersionId: latestVersion?.versionId ?? null,
+          operation: input.optimisticOperation ?? "commit"
         });
       }
       const changeResult = await input.applyChanges(input.runtime, {
-        previousRowCount: latestVersion?.rowCount
+        previousRowCount: latestVersion?.rowCount,
+        validatedPrimaryKeyColumns: latestVersion?.validatedPrimaryKeyColumns
       });
-      const commitId = randomUUID5();
+      const commitId = randomUUID6();
       await this.setCommitMetadata(input, commitId, commitRowCount(input, changeResult, latestVersion));
       await input.runtime.run("COMMIT");
       committed = true;
@@ -2890,20 +3428,29 @@ class DuckLakeWriteCoordinator {
     if (ownSnapshotId !== null) {
       const version = await this.snapshots.getVersionForSnapshot(input.runtime, input.dataset, ownSnapshotId);
       if (version) {
-        return { ...version, outcome: "created" };
+        return { outcome: "created", version };
       }
-      throw new LakeStorageError20(`[SixbDuckLake] DuckLake committed snapshot '${ownSnapshotId}' for dataset '${input.dataset.id}', but Sixb could not hydrate it as a dataset version.`);
+      throw new LakeStorageError22(`[SixbDuckLake] DuckLake committed snapshot '${ownSnapshotId}' for dataset '${input.dataset.id}', but Sixb could not hydrate it as a dataset version.`);
     }
     if (!input.changeResult.dataChangeExpected) {
+      const version = await this.versionForNoOpCommit(input.runtime, input.dataset, input.allowInitialNoOp ?? false);
+      if (input.expectedLatestVersionId !== undefined) {
+        this.assertExpectedLatestVersion({
+          datasetId: input.dataset.id,
+          expectedLatestVersionId: input.expectedLatestVersionId,
+          actualLatestVersionId: version?.versionId ?? null,
+          operation: input.optimisticOperation ?? "commit"
+        });
+      }
       return {
-        ...await this.versionForNoOpCommit(input.runtime, input.dataset),
-        outcome: "unchanged"
+        outcome: "unchanged",
+        version
       };
     }
     if (input.committedWriteSnapshotId === input.previousWriteSnapshotId) {
-      throw new LakeStorageError20(`[SixbDuckLake] DuckLake commit for dataset '${input.dataset.id}' completed without producing a snapshot for this non-empty write.`);
+      throw new LakeStorageError22(`[SixbDuckLake] DuckLake commit for dataset '${input.dataset.id}' completed without producing a snapshot for this non-empty write.`);
     }
-    throw new LakeStorageError20(`[SixbDuckLake] DuckLake commit for dataset '${input.dataset.id}' completed, but Sixb could not find a matching snapshot for this write.`);
+    throw new LakeStorageError22(`[SixbDuckLake] DuckLake commit for dataset '${input.dataset.id}' completed, but Sixb could not find a matching snapshot for this write.`);
   }
   async applyStagedRows(input) {
     const mode = input.write.mode ?? "snapshot";
@@ -2916,11 +3463,12 @@ class DuckLakeWriteCoordinator {
       previousRowCount: input.previousRowCount
     });
     if (result.sourceRowCount !== input.rowsWritten) {
-      throw new LakeStorageError20(`[SixbDuckLake] Staged write for dataset '${input.write.dataset.id}' accepted ${input.rowsWritten} row(s), but DuckLake saw ${result.sourceRowCount} source row(s) at commit time.`);
+      throw new LakeStorageError22(`[SixbDuckLake] Staged write for dataset '${input.write.dataset.id}' accepted ${input.rowsWritten} row(s), but DuckLake saw ${result.sourceRowCount} source row(s) at commit time.`);
     }
     return result;
   }
   async setCommitMetadata(input, commitId, rowCount) {
+    const validatedPrimaryKeyColumns = getDatasetPrimaryKeyColumns4(input.dataset);
     const metadata = {
       kind: "datasetVersion",
       datasetId: input.dataset.id,
@@ -2928,16 +3476,20 @@ class DuckLakeWriteCoordinator {
       mode: input.mode,
       producer: input.producer ? structuredClone(input.producer) : undefined,
       inputs: input.inputs ? structuredClone(input.inputs) : undefined,
-      ...rowCount.kind === "exact" ? { rowCount: rowCount.value } : {}
+      ...rowCount.kind === "exact" ? { rowCount: rowCount.value } : {},
+      ...validatedPrimaryKeyColumns !== null ? { validatedPrimaryKeyColumns: [...validatedPrimaryKeyColumns] } : {}
     };
     await input.runtime.run(`CALL ${quoteIdentifier(duckLakeAlias(this.options))}.set_commit_message(${quoteSqlString("Sixb")}, ${quoteSqlString(input.commitMessage)}, extra_info => ${quoteSqlString(JSON.stringify({ sixb: metadata }))})`);
   }
-  async versionForNoOpCommit(runtime, definition) {
+  async versionForNoOpCommit(runtime, definition, allowInitialNoOp) {
     const latestVersion = await this.snapshots.getLatestVersionForDefinition(runtime, definition);
     if (latestVersion) {
       return latestVersion;
     }
-    throw new LakeStorageError20(`[SixbDuckLake] No DuckLake changes were committed for dataset '${definition.id}', and no previous version exists.`);
+    if (allowInitialNoOp) {
+      return null;
+    }
+    throw new LakeStorageError22(`[SixbDuckLake] No DuckLake changes were committed for dataset '${definition.id}', and no previous version exists.`);
   }
   async rollbackTransaction(runtime) {
     try {
@@ -2993,7 +3545,7 @@ class DuckLakeWriteCoordinator {
       return;
     }
     if (input.actualLatestVersionId !== input.expectedLatestVersionId) {
-      throw new LakeStorageError20(`[SixbDuckLake] Optimistic commit failed for dataset '${input.datasetId}': expected latest version '${input.expectedLatestVersionId}', found '${input.actualLatestVersionId ?? "none"}'.`);
+      throw new LakeStorageError22(`[SixbDuckLake] Optimistic ${input.operation} failed for dataset '${input.datasetId}': expected latest version '${input.expectedLatestVersionId ?? "none"}', found '${input.actualLatestVersionId ?? "none"}'.`);
     }
   }
 }
@@ -3011,7 +3563,7 @@ function commitRowCount(input, changeResult, latestVersion) {
 }
 function assertDuckLakeSnapshotId3(snapshotId) {
   if (!/^\d+$/.test(snapshotId)) {
-    throw new LakeStorageError20(`[SixbDuckLake] Invalid DuckLake snapshot id '${snapshotId}'.`);
+    throw new LakeStorageError22(`[SixbDuckLake] Invalid DuckLake snapshot id '${snapshotId}'.`);
   }
 }
 var localCatalogCommitLocks = new Map;
@@ -3089,6 +3641,9 @@ class DuckLakeStorage {
   async beginWrite(input) {
     return this.writes.beginWrite(input);
   }
+  async beginMerge(input) {
+    return this.writes.beginMerge(input);
+  }
   async getLatestVersion(datasetId) {
     return this.snapshotReader.getLatestVersion(datasetId);
   }
@@ -3109,4 +3664,4 @@ export {
   DuckLakeStorage
 };
 
-//# debugId=0B56C838FAF193D864756E2164756E21
+//# debugId=8AEF2C0D635BEB8A64756E2164756E21