@sixb/ducklake 0.1.3 → 0.1.4

package/README.md

@@ -180,12 +180,28 @@ await merge.writeChanges([
 const result = await merge.commit()
 ```
 
-Changes remain ordered across `writeChanges(...)` calls, and the final change for a repeated key
+For datasets without `sequenceBy`, changes remain ordered across `writeChanges(...)` calls, and the final change for a repeated key
 wins. Upserts are complete rows rather than patches. Commit checks the dataset version captured by
 `beginMerge`, applies effective changes atomically, and creates a version only when visible rows
 change. An initial no-op returns `{ outcome: "unchanged", version: null }`; a later no-op returns the
 existing latest version.
 
+Datasets declaring `sequenceBy` instead compare source revisions: older changes are ignored,
+identical ties are unchanged, and conflicting ties abort the merge. Deletes require
+`change.delete(key, { sequence })`. Source revisions and deletion tombstones are stored in a
+companion DuckLake table in the same transaction as visible rows, surviving snapshot expiration.
+New tombstone state creates a version even when the visible row count stays zero. Sequenced
+provider writes use merge sessions; snapshot syncs convert rows to ordered upserts.
+Snapshot syncs use `commit({ createInitialVersion: true })` to make a first empty result
+addressable. Later no-op commits reuse the current version, including after a concurrent write.
+Use `commit({ retryOnConflict: true })` to rebase retained changes, with at most 3 attempts.
+Explicit `expectedLatestVersionId` guards are never relaxed.
+See [source ordering](../../docs/data/datasets.md#source-ordering).
+
+Local catalogs serialize commits within one process. PostgreSQL catalogs support concurrent
+writer processes; known snapshot-ID conflicts are retried from the retained staging table.
+Other provider/connection failures propagate without an automatic retry.
+
 Snapshot, append, and SQL-transform writes to keyed datasets also enforce unique keys so a later
 merge never starts from an ambiguous baseline. Application authors normally use
 `defineSync(..., { mode: "merge" })`; the sync worker stages those changes through this contract.

package/dist/index.js

@@ -845,7 +845,11 @@ function incrementLocalCatalogGeneration(options) {
 
 // src/internal/ducklake-dataset-catalog.ts
 import { randomUUID } from "crypto";
-import { LakeStorageError as LakeStorageError11, planDatasetDefinitionUpdate } from "@sixb/core/lake-storage";
+import {
+  assertDatasetDefinition,
+  LakeStorageError as LakeStorageError11,
+  planDatasetDefinitionUpdate
+} from "@sixb/core/lake-storage";
 
 // src/internal/duckdb-row.ts
 import { LakeStorageError as LakeStorageError5 } from "@sixb/core/lake-storage";
@@ -1060,6 +1064,7 @@ function getIntegerField(record, key) {
 
 // src/internal/ducklake-catalog-schema.ts
 var SIXB_PRIMARY_KEY_COMMENT_NAMESPACE = "sixb:primary-key:";
+var DATASET_SEQUENCE_COLUMN_COMMENT = "sixb.sequenceBy.v1";
 var SIXB_PRIMARY_KEY_COMMENT_PREFIX = `${SIXB_PRIMARY_KEY_COMMENT_NAMESPACE}v1:`;
 var FILE_REF_STRUCT_CHILDREN = [
   { name: "blobId", type: "string" },
@@ -1241,6 +1246,9 @@ function decodeDatasetId(encoded) {
 function encodeDatasetTableName(datasetId) {
   return `${DATASET_TABLE_PREFIX}${encodeDatasetId(datasetId)}`;
 }
+function encodeDatasetSequenceTableName(datasetId) {
+  return `sixb__seq__${encodeDatasetTableName(datasetId)}`;
+}
 function decodeDatasetTableName(tableName) {
   if (!tableName.startsWith(DATASET_TABLE_PREFIX)) {
     return null;
@@ -1264,12 +1272,14 @@ async function readCurrentDatasetDefinitions(input) {
   for (const table of tables) {
     const columns = columnsByTableName.get(table.tableName) ?? [];
     const primaryKey = duckLakeCatalogColumnsToDatasetPrimaryKey(table.tableName, columns);
+    const sequenceBy = columns.find((column) => column.comment === DATASET_SEQUENCE_COLUMN_COMMENT)?.columnName;
     const partitionBy = partitionByTableName.get(table.tableName) ?? [];
     definitions.set(table.datasetId, {
       kind: "dataset",
       id: table.datasetId,
       schema: duckLakeCatalogColumnsToDatasetSchema(table.tableName, columns),
       ...primaryKey !== undefined ? { primaryKey } : {},
+      ...sequenceBy !== undefined ? { sequenceBy } : {},
       ...partitionBy.length > 0 ? { partitionBy } : {},
       ...table.comment !== undefined ? { description: table.comment } : {}
     });
@@ -1428,6 +1438,10 @@ class DuckLakeDatasetCatalog {
           await runtime.run(`COMMENT ON TABLE ${qualifiedName} IS ${quoteSqlString(definition.description)}`);
         }
         await this.applyPrimaryKey(runtime, qualifiedName, definition);
+        if (definition.sequenceBy !== undefined) {
+          await runtime.run(`COMMENT ON COLUMN ${qualifiedName}.${quoteIdentifier(definition.sequenceBy)} IS ${quoteSqlString(DATASET_SEQUENCE_COLUMN_COMMENT)}`);
+          await runtime.run(`CREATE TABLE ${qualifiedTableName(this.options, encodeDatasetSequenceTableName(definition.id))} (source_key VARCHAR NOT NULL, sequence VARCHAR NOT NULL, content VARCHAR)`);
+        }
         await this.applyPartitionBy(runtime, qualifiedName, definition);
         await runtime.run("COMMIT");
         committed = true;
@@ -1512,9 +1526,7 @@ ${details}`);
     return duckLakeCatalogColumnsToDatasetSchema(tableName, columns);
   }
   assertSchema(definition) {
-    if (!definition.schema) {
-      throw new LakeStorageError11(`[SixbDuckLake] Dataset '${definition.id}' requires a schema for DuckLake storage.`);
-    }
+    assertDatasetDefinition(definition, (message) => new LakeStorageError11(`[SixbDuckLake] ${message}`));
   }
   async applyExistingDatasetUpdate(plan) {
     const tableName = encodeDatasetTableName(plan.definition.id);
@@ -1765,7 +1777,7 @@ function isRecord(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 function isDatasetProducer(value) {
-  return isRecord(value) && (value.kind === "sync" || value.kind === "pipeline") && optionalString(value.id) && optionalString(value.runId);
+  return isRecord(value) && (value.kind === "sync" || value.kind === "pipeline" || value.kind === "ingest") && optionalString(value.id) && optionalString(value.runId);
 }
 function isDatasetVersionRefs(value) {
   return Array.isArray(value) && value.every((item) => isRecord(item) && typeof item.datasetId === "string" && typeof item.versionId === "string");
@@ -2126,6 +2138,7 @@ class DuckLakeSnapshotReader {
     return {
       datasetId: tableRef.datasetId,
       versionId: toVersionId(row.snapshotId),
+      inputs: row.metadata?.inputs,
       ...row.metadata?.rowCount !== undefined ? { rowCount: row.metadata.rowCount } : {},
       ...row.metadata?.validatedPrimaryKeyColumns !== undefined ? { validatedPrimaryKeyColumns: row.metadata.validatedPrimaryKeyColumns } : {}
     };
@@ -2814,10 +2827,21 @@ function throwResultSchemaMismatch(datasetId, detail) {
 
 // src/internal/ducklake-write-coordinator.ts
 import { randomUUID as randomUUID6 } from "crypto";
-import { getDatasetPrimaryKeyColumns as getDatasetPrimaryKeyColumns4, LakeStorageError as LakeStorageError22 } from "@sixb/core/lake-storage";
+import {
+  assertUnsequencedDatasetWrite,
+  getDatasetPrimaryKeyColumns as getDatasetPrimaryKeyColumns4,
+  hasDatasetInputChanges,
+  LakeConcurrencyError,
+  LakeStorageError as LakeStorageError22,
+  retryDatasetMergeCommit
+} from "@sixb/core/lake-storage";
 
 // src/internal/dataset-row-merge.ts
-import { getDatasetPrimaryKeyColumns as getDatasetPrimaryKeyColumns2, LakeStorageError as LakeStorageError19 } from "@sixb/core/lake-storage";
+import {
+  getDatasetPrimaryKeyColumns as getDatasetPrimaryKeyColumns2,
+  LakeStorageError as LakeStorageError19,
+  reconcileDatasetSequences
+} from "@sixb/core/lake-storage";
 async function applyDatasetMergeFromRelation(input) {
   const primaryKeyColumns = getDatasetPrimaryKeyColumns2(input.dataset);
   if (primaryKeyColumns === null) {
@@ -2827,12 +2851,12 @@ async function applyDatasetMergeFromRelation(input) {
   const targetTable = qualifiedTableName(input.options, encodeDatasetTableName(input.dataset.id));
   const sourceRowCount = await countRows2(input.runtime, stagingTable);
   const previousRowCount = await currentBaselineRowCount(input, targetTable, primaryKeyColumns);
-  return withMaterializedFinalChanges(input, primaryKeyColumns, async (finalChanges) => {
+  return withMaterializedFinalChanges(input, primaryKeyColumns, async (finalChanges, orderingChanged) => {
     const effects = await countMergeEffects(input, targetTable, finalChanges, primaryKeyColumns);
     const effectiveChangeCount = effects.inserts + effects.updates + effects.deletes;
     if (effectiveChangeCount === 0) {
       return {
-        dataChangeExpected: false,
+        dataChangeExpected: orderingChanged,
         sourceRowCount,
         resultingRowCount: { kind: "exact", value: previousRowCount }
       };
@@ -2877,6 +2901,48 @@ async function applyDatasetMergeFromRelation(input) {
     };
   });
 }
+async function applySourceOrdering(input, acceptedTable) {
+  const source = quoteIdentifier(`${input.sequenceColumnName}_source`);
+  const ordinal = quoteIdentifier(input.sequenceColumnName);
+  const staging = quoteIdentifier(input.stagingTableName);
+  const stateTable = qualifiedTableName(input.options, encodeDatasetSequenceTableName(input.dataset.id));
+  await input.runtime.run(`CREATE TEMP TABLE ${acceptedTable} (source_key VARCHAR, ordinal UBIGINT)`);
+  let lastOrdinal = -1n;
+  let changed = false;
+  while (true) {
+    const staged = await input.runtime.query(`SELECT ${ordinal} AS ordinal, ${source} AS source FROM ${staging}
+       WHERE ${ordinal} > ${lastOrdinal} ORDER BY ${ordinal} LIMIT 1000`);
+    if (staged.length === 0)
+      return changed;
+    const changes = staged.map((row) => JSON.parse(getString(row, "source")));
+    lastOrdinal = getBigIntLike(staged[staged.length - 1], "ordinal");
+    const batchKeys = changes.map((change) => quoteSqlString(change.key)).join(",");
+    const current = await input.runtime.query(`SELECT * FROM ${stateTable} WHERE source_key IN (${batchKeys})`);
+    const previous = new Map;
+    for (const row of current) {
+      previous.set(getString(row, "source_key"), {
+        sequence: getString(row, "sequence"),
+        content: getOptionalString(row, "content") ?? null
+      });
+    }
+    const { states, accepted } = reconcileDatasetSequences(input.dataset, previous, changes);
+    if (accepted.size === 0)
+      continue;
+    changed = true;
+    const stateValues = [];
+    const acceptedValues = [];
+    for (const [key, index] of accepted) {
+      const state = states.get(key);
+      stateValues.push(`(${quoteSqlString(key)}, ${quoteSqlString(state.sequence)}, ${state.content === null ? "NULL" : quoteSqlString(state.content)})`);
+      acceptedValues.push(`(${quoteSqlString(key)}, ${getBigIntLike(staged[index], "ordinal")})`);
+    }
+    const keys = [...accepted.keys()].map(quoteSqlString).join(",");
+    await input.runtime.run(`DELETE FROM ${stateTable} WHERE source_key IN (${keys})`);
+    await input.runtime.run(`INSERT INTO ${stateTable} VALUES ${stateValues.join(",")}`);
+    await input.runtime.run(`DELETE FROM ${acceptedTable} WHERE source_key IN (${keys})`);
+    await input.runtime.run(`INSERT INTO ${acceptedTable} VALUES ${acceptedValues.join(",")}`);
+  }
+}
 async function currentBaselineRowCount(input, targetTable, primaryKeyColumns) {
   if (sameColumns(input.validatedPrimaryKeyColumns, primaryKeyColumns)) {
     return input.previousRowCount ?? countRows2(input.runtime, targetTable);
@@ -2890,18 +2956,23 @@ async function currentBaselineRowCount(input, targetTable, primaryKeyColumns) {
 }
 async function withMaterializedFinalChanges(input, primaryKeyColumns, run) {
   const table = quoteIdentifier(`${input.stagingTableName}_final`);
-  await input.runtime.run(`
-    CREATE TEMP TABLE ${table} AS
-    ${finalChangesSelectSql(input, primaryKeyColumns)}
-  `);
+  const acceptedTable = quoteIdentifier(`${input.stagingTableName}_accepted`);
   let outcome;
   try {
-    outcome = { kind: "success", value: await run(table) };
+    const sequenced = input.dataset.sequenceBy !== undefined;
+    const orderingChanged = sequenced ? await applySourceOrdering(input, acceptedTable) : false;
+    const ordinal = quoteIdentifier(input.sequenceColumnName);
+    const selectSql = sequenced ? `SELECT source.*, source.${ordinal} AS ${quoteIdentifier(firstSequenceName(input.sequenceColumnName))}
+         FROM ${quoteIdentifier(input.stagingTableName)} AS source
+         JOIN ${acceptedTable} AS accepted ON source.${ordinal} = accepted.ordinal` : finalChangesSelectSql(input, primaryKeyColumns);
+    await input.runtime.run(`CREATE TEMP TABLE ${table} AS ${selectSql}`);
+    outcome = { kind: "success", value: await run(table, orderingChanged) };
   } catch (error) {
     outcome = { kind: "error", error };
   }
   try {
     await input.runtime.run(`DROP TABLE IF EXISTS ${table}`);
+    await input.runtime.run(`DROP TABLE IF EXISTS ${acceptedTable}`);
   } catch (cleanupError) {
     if (outcome.kind === "success") {
       throw cleanupError;
@@ -2987,6 +3058,7 @@ function sameColumns(left, right) {
 import { randomUUID as randomUUID4 } from "crypto";
 import {
   cloneDatasetMergeChange,
+  datasetSequenceChange,
   getDatasetMergeChangeValidationError,
   LakeStorageError as LakeStorageError20
 } from "@sixb/core/lake-storage";
@@ -3055,6 +3127,7 @@ async function createDuckLakeMergeSession(input) {
     CREATE TEMP TABLE ${quoteIdentifier(stagingTableName)} (
       ${quoteIdentifier(sequenceColumnName)} UBIGINT NOT NULL,
       ${quoteIdentifier(kindColumnName)} VARCHAR NOT NULL,
+      ${input.merge.dataset.sequenceBy === undefined ? "" : `${quoteIdentifier(`${sequenceColumnName}_source`)} VARCHAR NOT NULL,`}
       ${datasetSchemaToDuckDbNullableColumnsSql(input.merge.dataset.schema)}
     )
   `);
@@ -3110,6 +3183,9 @@ class DuckLakeMergeSession {
       for (const staged of batch) {
         appender.appendBigInt(staged.sequence);
         appender.appendVarchar(staged.change.kind);
+        if (this.input.dataset.sequenceBy !== undefined) {
+          appender.appendVarchar(JSON.stringify(datasetSequenceChange(this.input.dataset, staged.change)));
+        }
         appendDatasetRow(appender, this.input.dataset.schema, staged.change.kind === "upsert" ? staged.change.row : staged.change.key);
       }
     });
@@ -3298,6 +3374,7 @@ class DuckLakeWriteCoordinator {
       throw new LakeStorageError22(`[SixbDuckLake] Unknown dataset '${input.dataset.id}'.`);
     }
     this.datasets.assertSchema(definition);
+    assertUnsequencedDatasetWrite(definition);
     const mode = input.mode ?? "snapshot";
     assertDatasetWriteMode(mode, "write");
     const runtime = await this.connections.stagingRuntime();
@@ -3323,7 +3400,7 @@ class DuckLakeWriteCoordinator {
     this.datasets.assertSchema(definition);
     const latestVersion = await this.connections.withAttachedRuntime((runtime2) => this.snapshots.getLatestVersionForDefinition(runtime2, definition));
     if (input.expectedLatestVersionId !== undefined && latestVersion?.versionId !== input.expectedLatestVersionId) {
-      throw new LakeStorageError22(`[SixbDuckLake] Optimistic merge start failed for dataset '${definition.id}': expected latest version '${input.expectedLatestVersionId}', found '${latestVersion?.versionId ?? "none"}'.`);
+      throw new LakeConcurrencyError(`[SixbDuckLake] Optimistic merge start failed for dataset '${definition.id}': expected latest version '${input.expectedLatestVersionId}', found '${latestVersion?.versionId ?? "none"}'.`);
     }
     const runtime = await this.connections.stagingRuntime();
     return createDuckLakeMergeSession({
@@ -3362,32 +3439,37 @@ class DuckLakeWriteCoordinator {
     if (getDatasetPrimaryKeyColumns4(definition) === null) {
       throw new LakeStorageError22(`[SixbDuckLake] Dataset '${definition.id}' must define a primaryKey before it can be merged.`);
     }
-    return this.withCommitRuntime((runtime) => this.commitVersionOutcomeOnExclusiveRuntime(runtime, {
-      dataset: definition,
-      mode: "merge",
-      expectedLatestVersionId: input.baseVersionId,
-      allowInitialNoOp: true,
-      disableRetries: true,
-      optimisticOperation: "merge commit",
-      commitMessage: input.commit?.commitMessage ?? `merge dataset ${definition.id}`,
-      producer: input.merge.producer,
-      inputs: input.merge.inputs,
-      applyChanges: async (commitRuntime, context) => {
-        const result = await applyDatasetMergeFromRelation({
-          options: this.options,
-          runtime: commitRuntime,
-          dataset: definition,
-          stagingTableName: input.stagingTableName,
-          sequenceColumnName: input.sequenceColumnName,
-          kindColumnName: input.kindColumnName,
-          previousRowCount: context.previousRowCount,
-          validatedPrimaryKeyColumns: context.validatedPrimaryKeyColumns
-        });
-        if (result.sourceRowCount !== input.changesWritten) {
-          throw new LakeStorageError22(`[SixbDuckLake] Staged merge for dataset '${definition.id}' accepted ${input.changesWritten} change(s), but DuckLake saw ${result.sourceRowCount} source change(s) at commit time.`);
+    return retryDatasetMergeCommit(input.merge, input.commit, (rebase) => this.withCommitRuntime(async (runtime) => {
+      input.commit?.signal?.throwIfAborted();
+      const baseVersionId = rebase ? (await this.snapshots.getLatestVersionForDefinition(runtime, definition))?.versionId ?? null : input.baseVersionId;
+      return this.commitVersionOutcomeOnExclusiveRuntime(runtime, {
+        dataset: definition,
+        mode: "merge",
+        expectedLatestVersionId: baseVersionId,
+        allowInitialNoOp: !input.commit?.createInitialVersion,
+        signal: input.commit?.signal,
+        disableRetries: true,
+        optimisticOperation: "merge commit",
+        commitMessage: input.commit?.commitMessage ?? `merge dataset ${definition.id}`,
+        producer: input.merge.producer,
+        inputs: input.merge.inputs,
+        applyChanges: async (commitRuntime, context) => {
+          const result = await applyDatasetMergeFromRelation({
+            options: this.options,
+            runtime: commitRuntime,
+            dataset: definition,
+            stagingTableName: input.stagingTableName,
+            sequenceColumnName: input.sequenceColumnName,
+            kindColumnName: input.kindColumnName,
+            previousRowCount: context.previousRowCount,
+            validatedPrimaryKeyColumns: context.validatedPrimaryKeyColumns
+          });
+          if (result.sourceRowCount !== input.changesWritten) {
+            throw new LakeStorageError22(`[SixbDuckLake] Staged merge for dataset '${definition.id}' accepted ${input.changesWritten} change(s), but DuckLake saw ${result.sourceRowCount} source change(s) at commit time.`);
+          }
+          return result;
         }
-        return result;
-      }
+      });
     }));
   }
   async commitDatasetVersion(input) {
@@ -3399,6 +3481,7 @@ class DuckLakeWriteCoordinator {
     });
   }
   async commitDatasetVersionOnExclusiveRuntime(runtime, input) {
+    assertUnsequencedDatasetWrite(input.dataset);
     const result = await this.commitVersionOutcomeOnExclusiveRuntime(runtime, {
       ...input,
       disableRetries: input.mode === "append" && getDatasetPrimaryKeyColumns4(input.dataset) !== null
@@ -3451,7 +3534,8 @@ class DuckLakeWriteCoordinator {
         previousRowCount: latestVersion?.rowCount,
         validatedPrimaryKeyColumns: latestVersion?.validatedPrimaryKeyColumns
       });
-      await this.ensureInitialNoOpHasSnapshot(input, changeResult, latestVersion);
+      input.signal?.throwIfAborted();
+      await this.ensureNoOpHasSnapshot(input, changeResult, latestVersion);
       const commitId = randomUUID6();
       await this.setCommitMetadata(input, commitId, commitRowCount(input, changeResult, latestVersion));
       await input.runtime.run("COMMIT");
@@ -3471,6 +3555,9 @@ class DuckLakeWriteCoordinator {
     } catch (error) {
       if (!committed) {
         await this.rollbackTransaction(input.runtime);
+        if (error instanceof Error && error.message.startsWith("TransactionContext Error: Failed to commit: Failed to commit DuckLake transaction.") && error.message.includes("Exceeded the maximum retry count of 0") && error.message.includes('duplicate key value violates unique constraint "ducklake_snapshot_pkey"')) {
+          throw new LakeConcurrencyError(`[SixbDuckLake] Concurrent catalog commit for dataset '${input.dataset.id}'.`, { cause: error });
+        }
       }
       throw error;
     }
@@ -3533,12 +3620,12 @@ class DuckLakeWriteCoordinator {
     };
     await input.runtime.run(`CALL ${quoteIdentifier(duckLakeAlias(this.options))}.set_commit_message(${quoteSqlString("Sixb")}, ${quoteSqlString(input.commitMessage)}, extra_info => ${quoteSqlString(JSON.stringify({ sixb: metadata }))})`);
   }
-  async ensureInitialNoOpHasSnapshot(input, changeResult, knownLatestVersion) {
-    if (changeResult.dataChangeExpected || input.allowInitialNoOp) {
+  async ensureNoOpHasSnapshot(input, changeResult, knownLatestVersion) {
+    if (changeResult.dataChangeExpected) {
       return;
     }
     const latestVersion = knownLatestVersion ?? await this.snapshots.getLatestVersionSummaryForDefinition(input.runtime, input.dataset);
-    if (latestVersion) {
+    if (latestVersion ? input.mode === "merge" || !hasDatasetInputChanges(latestVersion.inputs, input.inputs) : input.allowInitialNoOp) {
       return;
     }
     const table = qualifiedTableName(this.options, encodeDatasetTableName(input.dataset.id));
@@ -3609,7 +3696,7 @@ class DuckLakeWriteCoordinator {
       return;
     }
     if (input.actualLatestVersionId !== input.expectedLatestVersionId) {
-      throw new LakeStorageError22(`[SixbDuckLake] Optimistic ${input.operation} failed for dataset '${input.datasetId}': expected latest version '${input.expectedLatestVersionId ?? "none"}', found '${input.actualLatestVersionId ?? "none"}'.`);
+      throw new LakeConcurrencyError(`[SixbDuckLake] Optimistic ${input.operation} failed for dataset '${input.datasetId}': expected latest version '${input.expectedLatestVersionId ?? "none"}', found '${input.actualLatestVersionId ?? "none"}'.`);
     }
   }
 }
@@ -3728,4 +3815,4 @@ export {
   DuckLakeStorage
 };
 
-//# debugId=85151472B319A47E64756E2164756E21
+//# debugId=2A2E6355256FAE5864756E2164756E21