@sixb/ducklake 0.1.3 → 0.1.5

package/src/internal/ducklake-write-coordinator.ts

@@ -10,7 +10,14 @@ import type {
   LakeMergeSession,
   LakeWriteSession,
 } from "@sixb/core/lake-storage"
-import { getDatasetPrimaryKeyColumns, LakeStorageError } from "@sixb/core/lake-storage"
+import {
+  assertUnsequencedDatasetWrite,
+  getDatasetPrimaryKeyColumns,
+  hasDatasetInputChanges,
+  LakeConcurrencyError,
+  LakeStorageError,
+  retryDatasetMergeCommit,
+} from "@sixb/core/lake-storage"
 import type { DuckLakeStorageOptions } from "../types"
 import { localCatalogCoordinationKey } from "./catalog-key"
 import {
@@ -40,7 +47,7 @@ import { parseCommitMetadata, type SixbCommitMetadata } from "./versions"
 export interface DuckLakeCommitDatasetVersionInput {
   readonly dataset: DatasetDefinition
   readonly mode: DatasetWriteMode
-  readonly expectedLatestVersionId?: string
+  readonly expectedLatestVersionId?: string | null
   readonly commitMessage: string
   readonly producer?: BeginDatasetWriteInput["producer"]
   readonly inputs?: BeginDatasetWriteInput["inputs"]
@@ -55,6 +62,7 @@ interface DuckLakeCommitVersionOutcomeInput
   readonly mode: DatasetWriteMode | "merge"
   readonly expectedLatestVersionId?: string | null
   readonly allowInitialNoOp?: boolean
+  readonly signal?: AbortSignal
   readonly disableRetries?: boolean
   readonly optimisticOperation?: "commit" | "merge commit"
 }
@@ -93,6 +101,7 @@ export class DuckLakeWriteCoordinator {
     }
 
     this.datasets.assertSchema(definition)
+    assertUnsequencedDatasetWrite(definition)
     const mode = input.mode ?? "snapshot"
     assertDatasetWriteMode(mode, "write")
 
@@ -133,7 +142,7 @@ export class DuckLakeWriteCoordinator {
       input.expectedLatestVersionId !== undefined &&
       latestVersion?.versionId !== input.expectedLatestVersionId
     ) {
-      throw new LakeStorageError(
+      throw new LakeConcurrencyError(
         `[SixbDuckLake] Optimistic merge start failed for dataset '${definition.id}': expected latest version '${input.expectedLatestVersionId}', found '${latestVersion?.versionId ?? "none"}'.`
       )
     }
@@ -182,35 +191,43 @@ export class DuckLakeWriteCoordinator {
       )
     }
 
-    return this.withCommitRuntime((runtime) =>
-      this.commitVersionOutcomeOnExclusiveRuntime(runtime, {
-        dataset: definition,
-        mode: "merge",
-        expectedLatestVersionId: input.baseVersionId,
-        allowInitialNoOp: true,
-        disableRetries: true,
-        optimisticOperation: "merge commit",
-        commitMessage: input.commit?.commitMessage ?? `merge dataset ${definition.id}`,
-        producer: input.merge.producer,
-        inputs: input.merge.inputs,
-        applyChanges: async (commitRuntime, context) => {
-          const result = await applyDatasetMergeFromRelation({
-            options: this.options,
-            runtime: commitRuntime,
-            dataset: definition,
-            stagingTableName: input.stagingTableName,
-            sequenceColumnName: input.sequenceColumnName,
-            kindColumnName: input.kindColumnName,
-            previousRowCount: context.previousRowCount,
-            validatedPrimaryKeyColumns: context.validatedPrimaryKeyColumns,
-          })
-          if (result.sourceRowCount !== input.changesWritten) {
-            throw new LakeStorageError(
-              `[SixbDuckLake] Staged merge for dataset '${definition.id}' accepted ${input.changesWritten} change(s), but DuckLake saw ${result.sourceRowCount} source change(s) at commit time.`
-            )
-          }
-          return result
-        },
+    return retryDatasetMergeCommit(input.merge, input.commit, (rebase) =>
+      this.withCommitRuntime(async (runtime) => {
+        input.commit?.signal?.throwIfAborted()
+        const baseVersionId = rebase
+          ? ((await this.snapshots.getLatestVersionForDefinition(runtime, definition))?.versionId ??
+            null)
+          : input.baseVersionId
+        return this.commitVersionOutcomeOnExclusiveRuntime(runtime, {
+          dataset: definition,
+          mode: "merge",
+          expectedLatestVersionId: baseVersionId,
+          allowInitialNoOp: !input.commit?.createInitialVersion,
+          signal: input.commit?.signal,
+          disableRetries: true,
+          optimisticOperation: "merge commit",
+          commitMessage: input.commit?.commitMessage ?? `merge dataset ${definition.id}`,
+          producer: input.merge.producer,
+          inputs: input.merge.inputs,
+          applyChanges: async (commitRuntime, context) => {
+            const result = await applyDatasetMergeFromRelation({
+              options: this.options,
+              runtime: commitRuntime,
+              dataset: definition,
+              stagingTableName: input.stagingTableName,
+              sequenceColumnName: input.sequenceColumnName,
+              kindColumnName: input.kindColumnName,
+              previousRowCount: context.previousRowCount,
+              validatedPrimaryKeyColumns: context.validatedPrimaryKeyColumns,
+            })
+            if (result.sourceRowCount !== input.changesWritten) {
+              throw new LakeStorageError(
+                `[SixbDuckLake] Staged merge for dataset '${definition.id}' accepted ${input.changesWritten} change(s), but DuckLake saw ${result.sourceRowCount} source change(s) at commit time.`
+              )
+            }
+            return result
+          },
+        })
       })
     )
   }
@@ -235,6 +252,7 @@ export class DuckLakeWriteCoordinator {
     runtime: DuckDbQueryRuntime,
     input: DuckLakeCommitDatasetVersionInput
   ): Promise<DatasetWriteCommitResult> {
+    assertUnsequencedDatasetWrite(input.dataset)
     const result = await this.commitVersionOutcomeOnExclusiveRuntime(runtime, {
       ...input,
       // DuckLake has no primary-key constraint. Do not let its automatic retry replay a keyed
@@ -332,7 +350,8 @@ export class DuckLakeWriteCoordinator {
         previousRowCount: latestVersion?.rowCount,
         validatedPrimaryKeyColumns: latestVersion?.validatedPrimaryKeyColumns,
       })
-      await this.ensureInitialNoOpHasSnapshot(input, changeResult, latestVersion)
+      input.signal?.throwIfAborted()
+      await this.ensureNoOpHasSnapshot(input, changeResult, latestVersion)
       const commitId = randomUUID()
       await this.setCommitMetadata(
         input,
@@ -358,6 +377,24 @@ export class DuckLakeWriteCoordinator {
     } catch (error) {
       if (!committed) {
         await this.rollbackTransaction(input.runtime)
+        // DuckLake's snapshot-id uniqueness constraint is its catalog commit CAS. With native
+        // retries disabled, this error proves the transaction lost to another writer. Do not
+        // classify generic connection/COMMIT errors: their durable outcome may be ambiguous.
+        if (
+          error instanceof Error &&
+          error.message.startsWith(
+            "TransactionContext Error: Failed to commit: Failed to commit DuckLake transaction."
+          ) &&
+          error.message.includes("Exceeded the maximum retry count of 0") &&
+          error.message.includes(
+            'duplicate key value violates unique constraint "ducklake_snapshot_pkey"'
+          )
+        ) {
+          throw new LakeConcurrencyError(
+            `[SixbDuckLake] Concurrent catalog commit for dataset '${input.dataset.id}'.`,
+            { cause: error }
+          )
+        }
       }
       throw error
     }
@@ -487,26 +524,30 @@ export class DuckLakeWriteCoordinator {
     )
   }
 
-  private async ensureInitialNoOpHasSnapshot(
+  private async ensureNoOpHasSnapshot(
     input: DuckLakeCommitVersionOutcomeRuntimeInput,
     changeResult: ApplyDatasetRowsResult,
     knownLatestVersion: DuckLakeVersionSummary | null
   ): Promise<void> {
-    if (changeResult.dataChangeExpected || input.allowInitialNoOp) {
+    if (changeResult.dataChangeExpected) {
       return
     }
 
     const latestVersion =
       knownLatestVersion ??
       (await this.snapshots.getLatestVersionSummaryForDefinition(input.runtime, input.dataset))
-    if (latestVersion) {
+    if (
+      latestVersion
+        ? input.mode === "merge" || !hasDatasetInputChanges(latestVersion.inputs, input.inputs)
+        : input.allowInitialNoOp
+    ) {
       return
     }
 
     // DuckLake 1.5.2 does not create a snapshot for a transaction whose data and metadata are
     // unchanged, and set_commit_message alone cannot force one. Reapplying the current table
     // comment is an idempotent metadata write: it leaves the visible DatasetDefinition unchanged
-    // while giving the first empty write a real DuckLake snapshot for time travel and lineage.
+    // while giving initial empty writes and changed input lineage a real DuckLake snapshot.
     const table = qualifiedTableName(this.options, encodeDatasetTableName(input.dataset.id))
     const descriptionSql =
       input.dataset.description === undefined ? "NULL" : quoteSqlString(input.dataset.description)
@@ -633,7 +674,7 @@ export class DuckLakeWriteCoordinator {
     }
 
     if (input.actualLatestVersionId !== input.expectedLatestVersionId) {
-      throw new LakeStorageError(
+      throw new LakeConcurrencyError(
         `[SixbDuckLake] Optimistic ${input.operation} failed for dataset '${input.datasetId}': expected latest version '${input.expectedLatestVersionId ?? "none"}', found '${input.actualLatestVersionId ?? "none"}'.`
       )
     }

package/src/internal/names.ts

@@ -99,6 +99,11 @@ export function encodeDatasetTableName(datasetId: string): string {
   return `${DATASET_TABLE_PREFIX}${encodeDatasetId(datasetId)}`
 }
 
+/** Current source-ordering state, deliberately outside the dataset table namespace. */
+export function encodeDatasetSequenceTableName(datasetId: string): string {
+  return `sixb__seq__${encodeDatasetTableName(datasetId)}`
+}
+
 /**
  * Decode a physical DuckLake table name back to a Sixb dataset id.
  */

package/src/internal/versions.ts

@@ -113,7 +113,7 @@ function isRecord(value: unknown): value is Readonly<Record<string, unknown>> {
 function isDatasetProducer(value: unknown): value is NonNullable<DatasetVersion["producer"]> {
   return (
     isRecord(value) &&
-    (value.kind === "sync" || value.kind === "pipeline") &&
+    (value.kind === "sync" || value.kind === "pipeline" || value.kind === "ingest") &&
     optionalString(value.id) &&
     optionalString(value.runId)
   )