@sixb/ducklake 0.1.2 → 0.1.4

package/src/internal/ducklake-write-coordinator.ts

@@ -10,7 +10,14 @@ import type {
   LakeMergeSession,
   LakeWriteSession,
 } from "@sixb/core/lake-storage"
-import { getDatasetPrimaryKeyColumns, LakeStorageError } from "@sixb/core/lake-storage"
+import {
+  assertUnsequencedDatasetWrite,
+  getDatasetPrimaryKeyColumns,
+  hasDatasetInputChanges,
+  LakeConcurrencyError,
+  LakeStorageError,
+  retryDatasetMergeCommit,
+} from "@sixb/core/lake-storage"
 import type { DuckLakeStorageOptions } from "../types"
 import { localCatalogCoordinationKey } from "./catalog-key"
 import {
@@ -27,13 +34,20 @@ import type { DuckLakeDatasetCatalog } from "./ducklake-dataset-catalog"
 import { createDuckLakeMergeSession, type DuckLakeCommitMergeInput } from "./ducklake-merge-session"
 import type { DuckLakeSnapshotReader, DuckLakeVersionSummary } from "./ducklake-snapshot-reader"
 import { createDuckLakeWriteSession, type DuckLakeCommitWriteInput } from "./ducklake-write-session"
-import { duckLakeAlias, duckLakeMetadataTableName, quoteIdentifier, quoteSqlString } from "./sql"
+import { encodeDatasetTableName } from "./names"
+import {
+  duckLakeAlias,
+  duckLakeMetadataTableName,
+  qualifiedTableName,
+  quoteIdentifier,
+  quoteSqlString,
+} from "./sql"
 import { parseCommitMetadata, type SixbCommitMetadata } from "./versions"
 
 export interface DuckLakeCommitDatasetVersionInput {
   readonly dataset: DatasetDefinition
   readonly mode: DatasetWriteMode
-  readonly expectedLatestVersionId?: string
+  readonly expectedLatestVersionId?: string | null
   readonly commitMessage: string
   readonly producer?: BeginDatasetWriteInput["producer"]
   readonly inputs?: BeginDatasetWriteInput["inputs"]
@@ -48,6 +62,7 @@ interface DuckLakeCommitVersionOutcomeInput
   readonly mode: DatasetWriteMode | "merge"
   readonly expectedLatestVersionId?: string | null
   readonly allowInitialNoOp?: boolean
+  readonly signal?: AbortSignal
   readonly disableRetries?: boolean
   readonly optimisticOperation?: "commit" | "merge commit"
 }
@@ -86,6 +101,7 @@ export class DuckLakeWriteCoordinator {
     }
 
     this.datasets.assertSchema(definition)
+    assertUnsequencedDatasetWrite(definition)
     const mode = input.mode ?? "snapshot"
     assertDatasetWriteMode(mode, "write")
 
@@ -126,7 +142,7 @@ export class DuckLakeWriteCoordinator {
       input.expectedLatestVersionId !== undefined &&
       latestVersion?.versionId !== input.expectedLatestVersionId
     ) {
-      throw new LakeStorageError(
+      throw new LakeConcurrencyError(
         `[SixbDuckLake] Optimistic merge start failed for dataset '${definition.id}': expected latest version '${input.expectedLatestVersionId}', found '${latestVersion?.versionId ?? "none"}'.`
       )
     }
@@ -175,35 +191,43 @@ export class DuckLakeWriteCoordinator {
       )
     }
 
-    return this.withCommitRuntime((runtime) =>
-      this.commitVersionOutcomeOnExclusiveRuntime(runtime, {
-        dataset: definition,
-        mode: "merge",
-        expectedLatestVersionId: input.baseVersionId,
-        allowInitialNoOp: true,
-        disableRetries: true,
-        optimisticOperation: "merge commit",
-        commitMessage: input.commit?.commitMessage ?? `merge dataset ${definition.id}`,
-        producer: input.merge.producer,
-        inputs: input.merge.inputs,
-        applyChanges: async (commitRuntime, context) => {
-          const result = await applyDatasetMergeFromRelation({
-            options: this.options,
-            runtime: commitRuntime,
-            dataset: definition,
-            stagingTableName: input.stagingTableName,
-            sequenceColumnName: input.sequenceColumnName,
-            kindColumnName: input.kindColumnName,
-            previousRowCount: context.previousRowCount,
-            validatedPrimaryKeyColumns: context.validatedPrimaryKeyColumns,
-          })
-          if (result.sourceRowCount !== input.changesWritten) {
-            throw new LakeStorageError(
-              `[SixbDuckLake] Staged merge for dataset '${definition.id}' accepted ${input.changesWritten} change(s), but DuckLake saw ${result.sourceRowCount} source change(s) at commit time.`
-            )
-          }
-          return result
-        },
+    return retryDatasetMergeCommit(input.merge, input.commit, (rebase) =>
+      this.withCommitRuntime(async (runtime) => {
+        input.commit?.signal?.throwIfAborted()
+        const baseVersionId = rebase
+          ? ((await this.snapshots.getLatestVersionForDefinition(runtime, definition))?.versionId ??
+            null)
+          : input.baseVersionId
+        return this.commitVersionOutcomeOnExclusiveRuntime(runtime, {
+          dataset: definition,
+          mode: "merge",
+          expectedLatestVersionId: baseVersionId,
+          allowInitialNoOp: !input.commit?.createInitialVersion,
+          signal: input.commit?.signal,
+          disableRetries: true,
+          optimisticOperation: "merge commit",
+          commitMessage: input.commit?.commitMessage ?? `merge dataset ${definition.id}`,
+          producer: input.merge.producer,
+          inputs: input.merge.inputs,
+          applyChanges: async (commitRuntime, context) => {
+            const result = await applyDatasetMergeFromRelation({
+              options: this.options,
+              runtime: commitRuntime,
+              dataset: definition,
+              stagingTableName: input.stagingTableName,
+              sequenceColumnName: input.sequenceColumnName,
+              kindColumnName: input.kindColumnName,
+              previousRowCount: context.previousRowCount,
+              validatedPrimaryKeyColumns: context.validatedPrimaryKeyColumns,
+            })
+            if (result.sourceRowCount !== input.changesWritten) {
+              throw new LakeStorageError(
+                `[SixbDuckLake] Staged merge for dataset '${definition.id}' accepted ${input.changesWritten} change(s), but DuckLake saw ${result.sourceRowCount} source change(s) at commit time.`
+              )
+            }
+            return result
+          },
+        })
       })
     )
   }
@@ -228,6 +252,7 @@ export class DuckLakeWriteCoordinator {
     runtime: DuckDbQueryRuntime,
     input: DuckLakeCommitDatasetVersionInput
   ): Promise<DatasetWriteCommitResult> {
+    assertUnsequencedDatasetWrite(input.dataset)
     const result = await this.commitVersionOutcomeOnExclusiveRuntime(runtime, {
       ...input,
       // DuckLake has no primary-key constraint. Do not let its automatic retry replay a keyed
@@ -325,6 +350,8 @@ export class DuckLakeWriteCoordinator {
         previousRowCount: latestVersion?.rowCount,
         validatedPrimaryKeyColumns: latestVersion?.validatedPrimaryKeyColumns,
       })
+      input.signal?.throwIfAborted()
+      await this.ensureNoOpHasSnapshot(input, changeResult, latestVersion)
       const commitId = randomUUID()
       await this.setCommitMetadata(
         input,
@@ -350,6 +377,24 @@ export class DuckLakeWriteCoordinator {
     } catch (error) {
       if (!committed) {
         await this.rollbackTransaction(input.runtime)
+        // DuckLake's snapshot-id uniqueness constraint is its catalog commit CAS. With native
+        // retries disabled, this error proves the transaction lost to another writer. Do not
+        // classify generic connection/COMMIT errors: their durable outcome may be ambiguous.
+        if (
+          error instanceof Error &&
+          error.message.startsWith(
+            "TransactionContext Error: Failed to commit: Failed to commit DuckLake transaction."
+          ) &&
+          error.message.includes("Exceeded the maximum retry count of 0") &&
+          error.message.includes(
+            'duplicate key value violates unique constraint "ducklake_snapshot_pkey"'
+          )
+        ) {
+          throw new LakeConcurrencyError(
+            `[SixbDuckLake] Concurrent catalog commit for dataset '${input.dataset.id}'.`,
+            { cause: error }
+          )
+        }
       }
       throw error
     }
@@ -479,6 +524,36 @@ export class DuckLakeWriteCoordinator {
     )
   }
 
+  private async ensureNoOpHasSnapshot(
+    input: DuckLakeCommitVersionOutcomeRuntimeInput,
+    changeResult: ApplyDatasetRowsResult,
+    knownLatestVersion: DuckLakeVersionSummary | null
+  ): Promise<void> {
+    if (changeResult.dataChangeExpected) {
+      return
+    }
+
+    const latestVersion =
+      knownLatestVersion ??
+      (await this.snapshots.getLatestVersionSummaryForDefinition(input.runtime, input.dataset))
+    if (
+      latestVersion
+        ? input.mode === "merge" || !hasDatasetInputChanges(latestVersion.inputs, input.inputs)
+        : input.allowInitialNoOp
+    ) {
+      return
+    }
+
+    // DuckLake 1.5.2 does not create a snapshot for a transaction whose data and metadata are
+    // unchanged, and set_commit_message alone cannot force one. Reapplying the current table
+    // comment is an idempotent metadata write: it leaves the visible DatasetDefinition unchanged
+    // while giving initial empty writes and changed input lineage a real DuckLake snapshot.
+    const table = qualifiedTableName(this.options, encodeDatasetTableName(input.dataset.id))
+    const descriptionSql =
+      input.dataset.description === undefined ? "NULL" : quoteSqlString(input.dataset.description)
+    await input.runtime.run(`COMMENT ON TABLE ${table} IS ${descriptionSql}`)
+  }
+
   private async versionForNoOpCommit(
     runtime: DuckDbQueryRuntime,
     definition: DatasetDefinition,
@@ -599,7 +674,7 @@ export class DuckLakeWriteCoordinator {
     }
 
     if (input.actualLatestVersionId !== input.expectedLatestVersionId) {
-      throw new LakeStorageError(
+      throw new LakeConcurrencyError(
         `[SixbDuckLake] Optimistic ${input.operation} failed for dataset '${input.datasetId}': expected latest version '${input.expectedLatestVersionId ?? "none"}', found '${input.actualLatestVersionId ?? "none"}'.`
       )
     }

package/src/internal/names.ts

@@ -99,6 +99,11 @@ export function encodeDatasetTableName(datasetId: string): string {
   return `${DATASET_TABLE_PREFIX}${encodeDatasetId(datasetId)}`
 }
 
+/** Current source-ordering state, deliberately outside the dataset table namespace. */
+export function encodeDatasetSequenceTableName(datasetId: string): string {
+  return `sixb__seq__${encodeDatasetTableName(datasetId)}`
+}
+
 /**
  * Decode a physical DuckLake table name back to a Sixb dataset id.
  */

package/src/internal/versions.ts

@@ -113,7 +113,7 @@ function isRecord(value: unknown): value is Readonly<Record<string, unknown>> {
 function isDatasetProducer(value: unknown): value is NonNullable<DatasetVersion["producer"]> {
   return (
     isRecord(value) &&
-    (value.kind === "sync" || value.kind === "pipeline") &&
+    (value.kind === "sync" || value.kind === "pipeline" || value.kind === "ingest") &&
     optionalString(value.id) &&
     optionalString(value.runId)
   )