@sixb/ducklake 0.1.0 → 0.1.1

package/src/internal/ducklake-snapshot-reader.ts

@@ -60,6 +60,7 @@ export interface DuckLakeVersionSummary {
   readonly datasetId: string
   readonly versionId: string
   readonly rowCount?: number
+  readonly validatedPrimaryKeyColumns?: readonly string[]
 }
 
 /** One snapshot row from the shared catalog scan, before per-dataset filtering. */
@@ -444,6 +445,9 @@ export class DuckLakeSnapshotReader {
       datasetId: tableRef.datasetId,
       versionId: toVersionId(row.snapshotId),
       ...(row.metadata?.rowCount !== undefined ? { rowCount: row.metadata.rowCount } : {}),
+      ...(row.metadata?.validatedPrimaryKeyColumns !== undefined
+        ? { validatedPrimaryKeyColumns: row.metadata.validatedPrimaryKeyColumns }
+        : {}),
     }
   }
 
@@ -549,7 +553,10 @@ export class DuckLakeSnapshotReader {
       return null
     }
 
-    if (!options.includeParent || (row.mode !== "append" && row.mode !== "schema")) {
+    if (
+      !options.includeParent ||
+      (row.mode !== "append" && row.mode !== "merge" && row.mode !== "schema")
+    ) {
       return row
     }
 
@@ -807,7 +814,7 @@ export class DuckLakeSnapshotReader {
       // Parent ids track versions that reuse the previous dataset state.
       // Snapshot versions replace the rows, so time travel stands on the snapshot id.
       const parentSnapshotId =
-        snapshot.mode === "append" || snapshot.mode === "schema"
+        snapshot.mode === "append" || snapshot.mode === "merge" || snapshot.mode === "schema"
           ? snapshots[index + 1]?.snapshotId
           : undefined
       return parentSnapshotId === undefined ? snapshot : { ...snapshot, parentSnapshotId }

package/src/internal/ducklake-write-coordinator.ts

@@ -1,13 +1,16 @@
 import { randomUUID } from "node:crypto"
 import type { DatasetDefinition } from "@sixb/core"
 import type {
+  BeginDatasetMergeInput,
   BeginDatasetWriteInput,
+  DatasetMergeCommitResult,
   DatasetVersion,
   DatasetWriteCommitResult,
   DatasetWriteMode,
+  LakeMergeSession,
   LakeWriteSession,
 } from "@sixb/core/lake-storage"
-import { LakeStorageError } from "@sixb/core/lake-storage"
+import { getDatasetPrimaryKeyColumns, LakeStorageError } from "@sixb/core/lake-storage"
 import type { DuckLakeStorageOptions } from "../types"
 import { localCatalogCoordinationKey } from "./catalog-key"
 import {
@@ -16,10 +19,12 @@ import {
   assertDatasetWriteMode,
   type CommitRowCount,
 } from "./dataset-row-commit"
+import { applyDatasetMergeFromRelation } from "./dataset-row-merge"
 import { getBigIntLike } from "./duckdb-row"
 import type { DuckDbQueryRuntime } from "./duckdb-runtime"
 import type { DuckLakeConnectionManager } from "./ducklake-connection-manager"
 import type { DuckLakeDatasetCatalog } from "./ducklake-dataset-catalog"
+import { createDuckLakeMergeSession, type DuckLakeCommitMergeInput } from "./ducklake-merge-session"
 import type { DuckLakeSnapshotReader, DuckLakeVersionSummary } from "./ducklake-snapshot-reader"
 import { createDuckLakeWriteSession, type DuckLakeCommitWriteInput } from "./ducklake-write-session"
 import { duckLakeAlias, duckLakeMetadataTableName, quoteIdentifier, quoteSqlString } from "./sql"
@@ -38,12 +43,22 @@ export interface DuckLakeCommitDatasetVersionInput {
   ): Promise<ApplyDatasetRowsResult>
 }
 
-interface DuckLakeCommitDatasetVersionRuntimeInput extends DuckLakeCommitDatasetVersionInput {
+interface DuckLakeCommitVersionOutcomeInput
+  extends Omit<DuckLakeCommitDatasetVersionInput, "expectedLatestVersionId" | "mode"> {
+  readonly mode: DatasetWriteMode | "merge"
+  readonly expectedLatestVersionId?: string | null
+  readonly allowInitialNoOp?: boolean
+  readonly disableRetries?: boolean
+  readonly optimisticOperation?: "commit" | "merge commit"
+}
+
+interface DuckLakeCommitVersionOutcomeRuntimeInput extends DuckLakeCommitVersionOutcomeInput {
   readonly runtime: DuckDbQueryRuntime
 }
 
 export interface DuckLakeApplyChangesContext {
   readonly previousRowCount?: number
+  readonly validatedPrimaryKeyColumns?: readonly string[]
 }
 
 /**
@@ -90,6 +105,40 @@ export class DuckLakeWriteCoordinator {
     })
   }
 
+  async beginMerge(input: BeginDatasetMergeInput): Promise<LakeMergeSession> {
+    this.connections.assertOpen()
+
+    const definition = await this.datasets.getDataset(input.dataset.id)
+    if (!definition) {
+      throw new LakeStorageError(`[SixbDuckLake] Unknown dataset '${input.dataset.id}'.`)
+    }
+    if (getDatasetPrimaryKeyColumns(definition) === null) {
+      throw new LakeStorageError(
+        `[SixbDuckLake] Dataset '${definition.id}' must define a primaryKey before it can be merged.`
+      )
+    }
+
+    this.datasets.assertSchema(definition)
+    const latestVersion = await this.connections.withAttachedRuntime((runtime) =>
+      this.snapshots.getLatestVersionForDefinition(runtime, definition)
+    )
+    if (
+      input.expectedLatestVersionId !== undefined &&
+      latestVersion?.versionId !== input.expectedLatestVersionId
+    ) {
+      throw new LakeStorageError(
+        `[SixbDuckLake] Optimistic merge start failed for dataset '${definition.id}': expected latest version '${input.expectedLatestVersionId}', found '${latestVersion?.versionId ?? "none"}'.`
+      )
+    }
+    const runtime = await this.connections.stagingRuntime()
+    return createDuckLakeMergeSession({
+      commitMerge: (options) => this.commitMerge(options),
+      runtime,
+      merge: { ...input, dataset: definition },
+      baseVersionId: latestVersion?.versionId ?? null,
+    })
+  }
+
   private async commitWrite(input: DuckLakeCommitWriteInput): Promise<DatasetWriteCommitResult> {
     const definition = await this.datasets.getDataset(input.write.dataset.id)
     if (!definition) {
@@ -115,6 +164,50 @@ export class DuckLakeWriteCoordinator {
     })
   }
 
+  private async commitMerge(input: DuckLakeCommitMergeInput): Promise<DatasetMergeCommitResult> {
+    const definition = await this.datasets.getDataset(input.merge.dataset.id)
+    if (!definition) {
+      throw new LakeStorageError(`[SixbDuckLake] Unknown dataset '${input.merge.dataset.id}'.`)
+    }
+    if (getDatasetPrimaryKeyColumns(definition) === null) {
+      throw new LakeStorageError(
+        `[SixbDuckLake] Dataset '${definition.id}' must define a primaryKey before it can be merged.`
+      )
+    }
+
+    return this.withCommitRuntime((runtime) =>
+      this.commitVersionOutcomeOnExclusiveRuntime(runtime, {
+        dataset: definition,
+        mode: "merge",
+        expectedLatestVersionId: input.baseVersionId,
+        allowInitialNoOp: true,
+        disableRetries: true,
+        optimisticOperation: "merge commit",
+        commitMessage: input.commit?.commitMessage ?? `merge dataset ${definition.id}`,
+        producer: input.merge.producer,
+        inputs: input.merge.inputs,
+        applyChanges: async (commitRuntime, context) => {
+          const result = await applyDatasetMergeFromRelation({
+            options: this.options,
+            runtime: commitRuntime,
+            dataset: definition,
+            stagingTableName: input.stagingTableName,
+            sequenceColumnName: input.sequenceColumnName,
+            kindColumnName: input.kindColumnName,
+            previousRowCount: context.previousRowCount,
+            validatedPrimaryKeyColumns: context.validatedPrimaryKeyColumns,
+          })
+          if (result.sourceRowCount !== input.changesWritten) {
+            throw new LakeStorageError(
+              `[SixbDuckLake] Staged merge for dataset '${definition.id}' accepted ${input.changesWritten} change(s), but DuckLake saw ${result.sourceRowCount} source change(s) at commit time.`
+            )
+          }
+          return result
+        },
+      })
+    )
+  }
+
   async commitDatasetVersion(
     input: DuckLakeCommitDatasetVersionInput
   ): Promise<DatasetWriteCommitResult> {
@@ -135,6 +228,26 @@ export class DuckLakeWriteCoordinator {
     runtime: DuckDbQueryRuntime,
     input: DuckLakeCommitDatasetVersionInput
   ): Promise<DatasetWriteCommitResult> {
+    const result = await this.commitVersionOutcomeOnExclusiveRuntime(runtime, {
+      ...input,
+      // DuckLake has no primary-key constraint. Do not let its automatic retry replay a keyed
+      // append after the collision checks ran against an older transaction snapshot.
+      disableRetries:
+        input.mode === "append" && getDatasetPrimaryKeyColumns(input.dataset) !== null,
+    })
+    if (result.version === null) {
+      throw new LakeStorageError(
+        `[SixbDuckLake] No DuckLake changes were committed for dataset '${input.dataset.id}', and no previous version exists.`
+      )
+    }
+
+    return { ...result.version, outcome: result.outcome }
+  }
+
+  private async commitVersionOutcomeOnExclusiveRuntime(
+    runtime: DuckDbQueryRuntime,
+    input: DuckLakeCommitVersionOutcomeInput
+  ): Promise<DatasetMergeCommitResult> {
     const runtimeInput = { ...input, runtime }
     return this.withGuardedRetrySetting(runtimeInput, () =>
       this.commitDatasetVersionUnlocked(runtimeInput)
@@ -142,16 +255,16 @@ export class DuckLakeWriteCoordinator {
   }
 
   private async withGuardedRetrySetting<T>(
-    input: DuckLakeCommitDatasetVersionRuntimeInput,
+    input: DuckLakeCommitVersionOutcomeRuntimeInput,
     run: () => Promise<T>
   ): Promise<T> {
-    if (input.expectedLatestVersionId === undefined) {
+    if (input.expectedLatestVersionId === undefined && !input.disableRetries) {
       return run()
     }
 
-    // DuckLake can retry transactions internally. Guarded Sixb commits need a
-    // strict compare-and-swap check, so disable retries only for this exclusive
-    // commit and always restore the runtime setting before releasing the queue.
+    // DuckLake can retry transactions internally. Optimistic guards and keyed append constraint
+    // checks must not be replayed against a different snapshot, so disable retries for this
+    // exclusive commit and restore the runtime setting before releasing the queue.
     await input.runtime.run("SET ducklake_max_retry_count = 0")
     let outcome:
       | { readonly kind: "success"; readonly value: T }
@@ -185,8 +298,8 @@ export class DuckLakeWriteCoordinator {
   }
 
   private async commitDatasetVersionUnlocked(
-    input: DuckLakeCommitDatasetVersionRuntimeInput
-  ): Promise<DatasetWriteCommitResult> {
+    input: DuckLakeCommitVersionOutcomeRuntimeInput
+  ): Promise<DatasetMergeCommitResult> {
     // Capture the write connection's last committed DuckLake snapshot before
     // and after COMMIT. The commitId in DuckLake commit_extra_info proves
     // which snapshot belongs to this transaction.
@@ -204,11 +317,13 @@ export class DuckLakeWriteCoordinator {
           datasetId: input.dataset.id,
           expectedLatestVersionId: input.expectedLatestVersionId,
           actualLatestVersionId: latestVersion?.versionId ?? null,
+          operation: input.optimisticOperation ?? "commit",
         })
       }
 
       const changeResult = await input.applyChanges(input.runtime, {
         previousRowCount: latestVersion?.rowCount,
+        validatedPrimaryKeyColumns: latestVersion?.validatedPrimaryKeyColumns,
       })
       const commitId = randomUUID()
       await this.setCommitMetadata(
@@ -241,13 +356,13 @@ export class DuckLakeWriteCoordinator {
   }
 
   private async versionForCommittedWrite(
-    input: DuckLakeCommitDatasetVersionRuntimeInput & {
+    input: DuckLakeCommitVersionOutcomeRuntimeInput & {
       readonly commitId: string
       readonly previousWriteSnapshotId: string | null
       readonly committedWriteSnapshotId: string | null
       readonly changeResult: ApplyDatasetRowsResult
     }
-  ): Promise<DatasetWriteCommitResult> {
+  ): Promise<DatasetMergeCommitResult> {
     // Hydrate the committed version on the same exclusive runtime. That keeps
     // snapshot matching and row-count fallback inside the same serialized
     // connection state that just committed.
@@ -266,7 +381,7 @@ export class DuckLakeWriteCoordinator {
         ownSnapshotId
       )
       if (version) {
-        return { ...version, outcome: "created" }
+        return { outcome: "created", version }
       }
 
       throw new LakeStorageError(
@@ -275,9 +390,22 @@ export class DuckLakeWriteCoordinator {
     }
 
     if (!input.changeResult.dataChangeExpected) {
+      const version = await this.versionForNoOpCommit(
+        input.runtime,
+        input.dataset,
+        input.allowInitialNoOp ?? false
+      )
+      if (input.expectedLatestVersionId !== undefined) {
+        this.assertExpectedLatestVersion({
+          datasetId: input.dataset.id,
+          expectedLatestVersionId: input.expectedLatestVersionId,
+          actualLatestVersionId: version?.versionId ?? null,
+          operation: input.optimisticOperation ?? "commit",
+        })
+      }
       return {
-        ...(await this.versionForNoOpCommit(input.runtime, input.dataset)),
         outcome: "unchanged",
+        version,
       }
     }
 
@@ -318,7 +446,7 @@ export class DuckLakeWriteCoordinator {
   }
 
   private async setCommitMetadata(
-    input: DuckLakeCommitDatasetVersionRuntimeInput,
+    input: DuckLakeCommitVersionOutcomeRuntimeInput,
     commitId: string,
     rowCount: CommitRowCount
   ): Promise<void> {
@@ -326,6 +454,9 @@ export class DuckLakeWriteCoordinator {
     // transaction. Version id, timestamp, and data-change detection still come
     // from DuckLake itself. commitId is only a transaction correlation token so
     // concurrent writers do not accidentally hydrate each other's snapshots.
+    // Every keyed data path either replaces the baseline or validates its uniqueness before this
+    // marker is persisted. Merge commits can therefore avoid repeating that full-table audit.
+    const validatedPrimaryKeyColumns = getDatasetPrimaryKeyColumns(input.dataset)
     const metadata: SixbCommitMetadata = {
       kind: "datasetVersion",
       datasetId: input.dataset.id,
@@ -334,6 +465,9 @@ export class DuckLakeWriteCoordinator {
       producer: input.producer ? structuredClone(input.producer) : undefined,
       inputs: input.inputs ? structuredClone(input.inputs) : undefined,
       ...(rowCount.kind === "exact" ? { rowCount: rowCount.value } : {}),
+      ...(validatedPrimaryKeyColumns !== null
+        ? { validatedPrimaryKeyColumns: [...validatedPrimaryKeyColumns] }
+        : {}),
     }
 
     await input.runtime.run(
@@ -347,13 +481,18 @@ export class DuckLakeWriteCoordinator {
 
   private async versionForNoOpCommit(
     runtime: DuckDbQueryRuntime,
-    definition: DatasetDefinition
-  ): Promise<DatasetVersion> {
+    definition: DatasetDefinition,
+    allowInitialNoOp: boolean
+  ): Promise<DatasetVersion | null> {
     const latestVersion = await this.snapshots.getLatestVersionForDefinition(runtime, definition)
     if (latestVersion) {
       return latestVersion
     }
 
+    if (allowInitialNoOp) {
+      return null
+    }
+
     throw new LakeStorageError(
       `[SixbDuckLake] No DuckLake changes were committed for dataset '${definition.id}', and no previous version exists.`
     )
@@ -369,7 +508,7 @@ export class DuckLakeWriteCoordinator {
   }
 
   private async latestVersionForCommit(
-    input: DuckLakeCommitDatasetVersionRuntimeInput
+    input: DuckLakeCommitVersionOutcomeRuntimeInput
   ): Promise<DuckLakeVersionSummary | null> {
     if (input.expectedLatestVersionId === undefined && input.mode !== "append") {
       return null
@@ -451,8 +590,9 @@ export class DuckLakeWriteCoordinator {
 
   private assertExpectedLatestVersion(input: {
     readonly datasetId: string
-    readonly expectedLatestVersionId?: string
+    readonly expectedLatestVersionId?: string | null
     readonly actualLatestVersionId: string | null
+    readonly operation: "commit" | "merge commit"
   }): void {
     if (input.expectedLatestVersionId === undefined) {
       return
@@ -460,14 +600,14 @@ export class DuckLakeWriteCoordinator {
 
     if (input.actualLatestVersionId !== input.expectedLatestVersionId) {
       throw new LakeStorageError(
-        `[SixbDuckLake] Optimistic commit failed for dataset '${input.datasetId}': expected latest version '${input.expectedLatestVersionId}', found '${input.actualLatestVersionId ?? "none"}'.`
+        `[SixbDuckLake] Optimistic ${input.operation} failed for dataset '${input.datasetId}': expected latest version '${input.expectedLatestVersionId ?? "none"}', found '${input.actualLatestVersionId ?? "none"}'.`
       )
     }
   }
 }
 
 function commitRowCount(
-  input: Pick<DuckLakeCommitDatasetVersionInput, "expectedLatestVersionId" | "mode">,
+  input: Pick<DuckLakeCommitVersionOutcomeInput, "expectedLatestVersionId" | "mode">,
   changeResult: ApplyDatasetRowsResult,
   latestVersion: DuckLakeVersionSummary | null
 ): CommitRowCount {

package/src/internal/ducklake-write-session.ts

@@ -7,7 +7,11 @@ import type {
   DatasetWriteCommitResult,
   LakeWriteSession,
 } from "@sixb/core/lake-storage"
-import { LakeStorageError } from "@sixb/core/lake-storage"
+import {
+  encodeDatasetPrimaryKey,
+  getDatasetPrimaryKeyColumns,
+  LakeStorageError,
+} from "@sixb/core/lake-storage"
 import type { DuckDbRuntime } from "./duckdb-runtime"
 import { appendDatasetRow } from "./row-appender"
 import { datasetSchemaToDuckDbColumnsSql } from "./schema"
@@ -61,6 +65,7 @@ class DuckLakeWriteSession implements LakeWriteSession {
   private closed = false
   private cleanedUp = false
   private rowsWritten = 0
+  private readonly stagedPrimaryKeys = new Set<string>()
 
   constructor(
     private readonly commitWrite: DuckLakeCommitWrite,
@@ -76,6 +81,7 @@ class DuckLakeWriteSession implements LakeWriteSession {
     // reads (pagination, APIs, SFTP, retries) never hold a runtime slot. Only
     // appendBatchToStagingTable below briefly enters the queue.
     let batch: DatasetRow[] = []
+    let batchPrimaryKeys = new Set<string>()
     for await (const row of rows) {
       const validationError = getDatasetRowValidationError(row, this.input.dataset)
       if (validationError) {
@@ -84,20 +90,34 @@ class DuckLakeWriteSession implements LakeWriteSession {
 
       // Snapshot the validated row. Callers may reuse and mutate one row object
       // between yields, so the batch must not retain a live reference.
-      batch.push(structuredClone(row))
+      const clonedRow = structuredClone(row)
+      if (getDatasetPrimaryKeyColumns(this.input.dataset) !== null) {
+        const primaryKey = encodeDatasetPrimaryKey(this.input.dataset, clonedRow)
+        if (this.stagedPrimaryKeys.has(primaryKey) || batchPrimaryKeys.has(primaryKey)) {
+          throw new LakeStorageError(
+            `[SixbDuckLake] Dataset '${this.input.dataset.id}' ${this.input.mode ?? "snapshot"} source contains duplicate primary key.`
+          )
+        }
+        batchPrimaryKeys.add(primaryKey)
+      }
+      batch.push(clonedRow)
 
       if (batch.length >= WRITE_ROW_BATCH_SIZE) {
-        await this.appendBatchToStagingTable(batch)
+        await this.appendBatchToStagingTable(batch, batchPrimaryKeys)
         batch = []
+        batchPrimaryKeys = new Set()
       }
     }
 
     if (batch.length > 0) {
-      await this.appendBatchToStagingTable(batch)
+      await this.appendBatchToStagingTable(batch, batchPrimaryKeys)
     }
   }
 
-  private async appendBatchToStagingTable(batch: readonly DatasetRow[]): Promise<void> {
+  private async appendBatchToStagingTable(
+    batch: readonly DatasetRow[],
+    primaryKeys: ReadonlySet<string>
+  ): Promise<void> {
     // Synchronous callback: no awaits inside the queue slot. The appender
     // buffers and flushes the whole batch into the staging temp table on close.
     await this.runtime.withAppender(this.stagingTableName, (appender) => {
@@ -108,6 +128,9 @@ class DuckLakeWriteSession implements LakeWriteSession {
     // Count only rows actually appended -- commit asserts rowsWritten ===
     // sourceRowCount in the write coordinator.
     this.rowsWritten += batch.length
+    for (const primaryKey of primaryKeys) {
+      this.stagedPrimaryKeys.add(primaryKey)
+    }
   }
 
   async commit(input?: CommitDatasetWriteInput): Promise<DatasetWriteCommitResult> {

package/src/internal/schema.ts

@@ -49,6 +49,13 @@ export function datasetSchemaToDuckDbColumnsSql(schema: DatasetSchema): string {
   return schema.columns.map((column) => datasetColumnToDuckDbSql(column)).join(", ")
 }
 
+/** Render a dataset schema for merge staging, where delete rows leave non-key values null. */
+export function datasetSchemaToDuckDbNullableColumnsSql(schema: DatasetSchema): string {
+  return schema.columns
+    .map((column) => `${quoteIdentifier(column.name)} ${datasetColumnTypeToDuckDbSql(column.type)}`)
+    .join(", ")
+}
+
 /**
  * Render the column list used by INSERT ... SELECT operations.
  */

package/src/internal/versions.ts

@@ -13,6 +13,7 @@ export interface SixbCommitMetadata {
   readonly producer?: DatasetVersion["producer"]
   readonly inputs?: DatasetVersion["inputs"]
   readonly rowCount?: number
+  readonly validatedPrimaryKeyColumns?: readonly string[]
   readonly schemaChange?: SixbSchemaChangeMetadata
 }
 
@@ -57,15 +58,21 @@ export function parseCommitMetadata(value: unknown): SixbCommitMetadata | undefi
   const mode = parsed.sixb.mode
   const commitId = parsed.sixb.commitId
   const rowCount = parsed.sixb.rowCount
+  const validatedPrimaryKeyColumns = parsed.sixb.validatedPrimaryKeyColumns
 
   return {
     kind: "datasetVersion",
     datasetId: parsed.sixb.datasetId,
     ...(typeof commitId === "string" ? { commitId } : {}),
-    ...(mode === "snapshot" || mode === "append" || mode === "schema" ? { mode } : {}),
+    ...(mode === "snapshot" || mode === "append" || mode === "merge" || mode === "schema"
+      ? { mode }
+      : {}),
     ...(isDatasetProducer(parsed.sixb.producer) ? { producer: parsed.sixb.producer } : {}),
     ...(isDatasetVersionRefs(parsed.sixb.inputs) ? { inputs: parsed.sixb.inputs } : {}),
     ...(typeof commitId === "string" && isRowCount(rowCount) ? { rowCount } : {}),
+    ...(typeof commitId === "string" && isValidatedPrimaryKeyColumns(validatedPrimaryKeyColumns)
+      ? { validatedPrimaryKeyColumns }
+      : {}),
     ...(isSchemaChangeMetadata(parsed.sixb.schemaChange)
       ? { schemaChange: parsed.sixb.schemaChange }
       : {}),
@@ -138,3 +145,12 @@ function optionalString(value: unknown): boolean {
 function isRowCount(value: unknown): value is number {
   return typeof value === "number" && Number.isSafeInteger(value) && value >= 0
 }
+
+function isValidatedPrimaryKeyColumns(value: unknown): value is readonly string[] {
+  return (
+    Array.isArray(value) &&
+    value.length > 0 &&
+    value.every((column) => typeof column === "string" && column.trim().length > 0) &&
+    new Set(value).size === value.length
+  )
+}