@sixb/ducklake 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -1
- package/dist/index.js +142 -55
- package/dist/index.js.map +11 -11
- package/dist/internal/dataset-row-merge.d.ts.map +1 -1
- package/dist/internal/ducklake-catalog-schema.d.ts +1 -0
- package/dist/internal/ducklake-catalog-schema.d.ts.map +1 -1
- package/dist/internal/ducklake-current-dataset-definitions.d.ts.map +1 -1
- package/dist/internal/ducklake-dataset-catalog.d.ts.map +1 -1
- package/dist/internal/ducklake-merge-session.d.ts.map +1 -1
- package/dist/internal/ducklake-snapshot-reader.d.ts +1 -0
- package/dist/internal/ducklake-snapshot-reader.d.ts.map +1 -1
- package/dist/internal/ducklake-write-coordinator.d.ts +2 -2
- package/dist/internal/ducklake-write-coordinator.d.ts.map +1 -1
- package/dist/internal/names.d.ts +2 -0
- package/dist/internal/names.d.ts.map +1 -1
- package/package.json +4 -4
- package/src/internal/dataset-row-merge.ts +114 -36
- package/src/internal/ducklake-catalog-schema.ts +1 -0
- package/src/internal/ducklake-current-dataset-definitions.ts +5 -0
- package/src/internal/ducklake-dataset-catalog.ts +19 -7
- package/src/internal/ducklake-merge-session.ts +7 -0
- package/src/internal/ducklake-snapshot-reader.ts +2 -0
- package/src/internal/ducklake-write-coordinator.ts +79 -38
- package/src/internal/names.ts +5 -0
- package/src/internal/versions.ts +1 -1
package/README.md
CHANGED
|
@@ -180,12 +180,28 @@ await merge.writeChanges([
|
|
|
180
180
|
const result = await merge.commit()
|
|
181
181
|
```
|
|
182
182
|
|
|
183
|
-
|
|
183
|
+
For datasets without `sequenceBy`, changes remain ordered across `writeChanges(...)` calls, and the final change for a repeated key
|
|
184
184
|
wins. Upserts are complete rows rather than patches. Commit checks the dataset version captured by
|
|
185
185
|
`beginMerge`, applies effective changes atomically, and creates a version only when visible rows
|
|
186
186
|
change. An initial no-op returns `{ outcome: "unchanged", version: null }`; a later no-op returns the
|
|
187
187
|
existing latest version.
|
|
188
188
|
|
|
189
|
+
Datasets declaring `sequenceBy` instead compare source revisions: older changes are ignored,
|
|
190
|
+
identical ties are unchanged, and conflicting ties abort the merge. Deletes require
|
|
191
|
+
`change.delete(key, { sequence })`. Source revisions and deletion tombstones are stored in a
|
|
192
|
+
companion DuckLake table in the same transaction as visible rows, surviving snapshot expiration.
|
|
193
|
+
New tombstone state creates a version even when the visible row count stays zero. Sequenced
|
|
194
|
+
provider writes use merge sessions; snapshot syncs convert rows to ordered upserts.
|
|
195
|
+
Snapshot syncs use `commit({ createInitialVersion: true })` to make a first empty result
|
|
196
|
+
addressable. Later no-op commits reuse the current version, including after a concurrent write.
|
|
197
|
+
Use `commit({ retryOnConflict: true })` to rebase retained changes, with at most 3 attempts.
|
|
198
|
+
Explicit `expectedLatestVersionId` guards are never relaxed.
|
|
199
|
+
See [source ordering](../../docs/data/datasets.md#source-ordering).
|
|
200
|
+
|
|
201
|
+
Local catalogs serialize commits within one process. PostgreSQL catalogs support concurrent
|
|
202
|
+
writer processes; known snapshot-ID conflicts are retried from the retained staging table.
|
|
203
|
+
Other provider/connection failures propagate without an automatic retry.
|
|
204
|
+
|
|
189
205
|
Snapshot, append, and SQL-transform writes to keyed datasets also enforce unique keys so a later
|
|
190
206
|
merge never starts from an ambiguous baseline. Application authors normally use
|
|
191
207
|
`defineSync(..., { mode: "merge" })`; the sync worker stages those changes through this contract.
|
package/dist/index.js
CHANGED
|
@@ -337,8 +337,8 @@ class NodeDuckDbRuntime {
|
|
|
337
337
|
async* streamRows(sql, values) {
|
|
338
338
|
this.assertAcceptingOperations();
|
|
339
339
|
let releaseOperation;
|
|
340
|
-
const operationFinished = new Promise((
|
|
341
|
-
releaseOperation =
|
|
340
|
+
const operationFinished = new Promise((resolve) => {
|
|
341
|
+
releaseOperation = resolve;
|
|
342
342
|
});
|
|
343
343
|
const waitForTurn = this.operations.then(() => {
|
|
344
344
|
this.assertNotClosed();
|
|
@@ -793,8 +793,8 @@ class DuckLakeConnectionManager {
|
|
|
793
793
|
}
|
|
794
794
|
const previous = this.localAttachmentLock.catch(() => {});
|
|
795
795
|
let release;
|
|
796
|
-
const current = new Promise((
|
|
797
|
-
release =
|
|
796
|
+
const current = new Promise((resolve) => {
|
|
797
|
+
release = resolve;
|
|
798
798
|
});
|
|
799
799
|
this.localAttachmentLock = previous.then(() => current);
|
|
800
800
|
await previous;
|
|
@@ -845,7 +845,11 @@ function incrementLocalCatalogGeneration(options) {
|
|
|
845
845
|
|
|
846
846
|
// src/internal/ducklake-dataset-catalog.ts
|
|
847
847
|
import { randomUUID } from "crypto";
|
|
848
|
-
import {
|
|
848
|
+
import {
|
|
849
|
+
assertDatasetDefinition,
|
|
850
|
+
LakeStorageError as LakeStorageError11,
|
|
851
|
+
planDatasetDefinitionUpdate
|
|
852
|
+
} from "@sixb/core/lake-storage";
|
|
849
853
|
|
|
850
854
|
// src/internal/duckdb-row.ts
|
|
851
855
|
import { LakeStorageError as LakeStorageError5 } from "@sixb/core/lake-storage";
|
|
@@ -1060,6 +1064,7 @@ function getIntegerField(record, key) {
|
|
|
1060
1064
|
|
|
1061
1065
|
// src/internal/ducklake-catalog-schema.ts
|
|
1062
1066
|
var SIXB_PRIMARY_KEY_COMMENT_NAMESPACE = "sixb:primary-key:";
|
|
1067
|
+
var DATASET_SEQUENCE_COLUMN_COMMENT = "sixb.sequenceBy.v1";
|
|
1063
1068
|
var SIXB_PRIMARY_KEY_COMMENT_PREFIX = `${SIXB_PRIMARY_KEY_COMMENT_NAMESPACE}v1:`;
|
|
1064
1069
|
var FILE_REF_STRUCT_CHILDREN = [
|
|
1065
1070
|
{ name: "blobId", type: "string" },
|
|
@@ -1241,6 +1246,9 @@ function decodeDatasetId(encoded) {
|
|
|
1241
1246
|
function encodeDatasetTableName(datasetId) {
|
|
1242
1247
|
return `${DATASET_TABLE_PREFIX}${encodeDatasetId(datasetId)}`;
|
|
1243
1248
|
}
|
|
1249
|
+
function encodeDatasetSequenceTableName(datasetId) {
|
|
1250
|
+
return `sixb__seq__${encodeDatasetTableName(datasetId)}`;
|
|
1251
|
+
}
|
|
1244
1252
|
function decodeDatasetTableName(tableName) {
|
|
1245
1253
|
if (!tableName.startsWith(DATASET_TABLE_PREFIX)) {
|
|
1246
1254
|
return null;
|
|
@@ -1264,12 +1272,14 @@ async function readCurrentDatasetDefinitions(input) {
|
|
|
1264
1272
|
for (const table of tables) {
|
|
1265
1273
|
const columns = columnsByTableName.get(table.tableName) ?? [];
|
|
1266
1274
|
const primaryKey = duckLakeCatalogColumnsToDatasetPrimaryKey(table.tableName, columns);
|
|
1275
|
+
const sequenceBy = columns.find((column) => column.comment === DATASET_SEQUENCE_COLUMN_COMMENT)?.columnName;
|
|
1267
1276
|
const partitionBy = partitionByTableName.get(table.tableName) ?? [];
|
|
1268
1277
|
definitions.set(table.datasetId, {
|
|
1269
1278
|
kind: "dataset",
|
|
1270
1279
|
id: table.datasetId,
|
|
1271
1280
|
schema: duckLakeCatalogColumnsToDatasetSchema(table.tableName, columns),
|
|
1272
1281
|
...primaryKey !== undefined ? { primaryKey } : {},
|
|
1282
|
+
...sequenceBy !== undefined ? { sequenceBy } : {},
|
|
1273
1283
|
...partitionBy.length > 0 ? { partitionBy } : {},
|
|
1274
1284
|
...table.comment !== undefined ? { description: table.comment } : {}
|
|
1275
1285
|
});
|
|
@@ -1428,6 +1438,10 @@ class DuckLakeDatasetCatalog {
|
|
|
1428
1438
|
await runtime.run(`COMMENT ON TABLE ${qualifiedName} IS ${quoteSqlString(definition.description)}`);
|
|
1429
1439
|
}
|
|
1430
1440
|
await this.applyPrimaryKey(runtime, qualifiedName, definition);
|
|
1441
|
+
if (definition.sequenceBy !== undefined) {
|
|
1442
|
+
await runtime.run(`COMMENT ON COLUMN ${qualifiedName}.${quoteIdentifier(definition.sequenceBy)} IS ${quoteSqlString(DATASET_SEQUENCE_COLUMN_COMMENT)}`);
|
|
1443
|
+
await runtime.run(`CREATE TABLE ${qualifiedTableName(this.options, encodeDatasetSequenceTableName(definition.id))} (source_key VARCHAR NOT NULL, sequence VARCHAR NOT NULL, content VARCHAR)`);
|
|
1444
|
+
}
|
|
1431
1445
|
await this.applyPartitionBy(runtime, qualifiedName, definition);
|
|
1432
1446
|
await runtime.run("COMMIT");
|
|
1433
1447
|
committed = true;
|
|
@@ -1512,9 +1526,7 @@ ${details}`);
|
|
|
1512
1526
|
return duckLakeCatalogColumnsToDatasetSchema(tableName, columns);
|
|
1513
1527
|
}
|
|
1514
1528
|
assertSchema(definition) {
|
|
1515
|
-
|
|
1516
|
-
throw new LakeStorageError11(`[SixbDuckLake] Dataset '${definition.id}' requires a schema for DuckLake storage.`);
|
|
1517
|
-
}
|
|
1529
|
+
assertDatasetDefinition(definition, (message) => new LakeStorageError11(`[SixbDuckLake] ${message}`));
|
|
1518
1530
|
}
|
|
1519
1531
|
async applyExistingDatasetUpdate(plan) {
|
|
1520
1532
|
const tableName = encodeDatasetTableName(plan.definition.id);
|
|
@@ -1765,7 +1777,7 @@ function isRecord(value) {
|
|
|
1765
1777
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1766
1778
|
}
|
|
1767
1779
|
function isDatasetProducer(value) {
|
|
1768
|
-
return isRecord(value) && (value.kind === "sync" || value.kind === "pipeline") && optionalString(value.id) && optionalString(value.runId);
|
|
1780
|
+
return isRecord(value) && (value.kind === "sync" || value.kind === "pipeline" || value.kind === "ingest") && optionalString(value.id) && optionalString(value.runId);
|
|
1769
1781
|
}
|
|
1770
1782
|
function isDatasetVersionRefs(value) {
|
|
1771
1783
|
return Array.isArray(value) && value.every((item) => isRecord(item) && typeof item.datasetId === "string" && typeof item.versionId === "string");
|
|
@@ -2126,6 +2138,7 @@ class DuckLakeSnapshotReader {
|
|
|
2126
2138
|
return {
|
|
2127
2139
|
datasetId: tableRef.datasetId,
|
|
2128
2140
|
versionId: toVersionId(row.snapshotId),
|
|
2141
|
+
inputs: row.metadata?.inputs,
|
|
2129
2142
|
...row.metadata?.rowCount !== undefined ? { rowCount: row.metadata.rowCount } : {},
|
|
2130
2143
|
...row.metadata?.validatedPrimaryKeyColumns !== undefined ? { validatedPrimaryKeyColumns: row.metadata.validatedPrimaryKeyColumns } : {}
|
|
2131
2144
|
};
|
|
@@ -2644,8 +2657,8 @@ class DuckLakeSqlExecutor {
|
|
|
2644
2657
|
commitMessage: input.commitMessage ?? `execute SQL transform for dataset ${target.id}`,
|
|
2645
2658
|
producer: input.producer,
|
|
2646
2659
|
inputs: sources.inputs,
|
|
2647
|
-
applyChanges: (
|
|
2648
|
-
runtime
|
|
2660
|
+
applyChanges: (runtime, context) => this.applySqlTransform({
|
|
2661
|
+
runtime,
|
|
2649
2662
|
target,
|
|
2650
2663
|
mode: input.mode,
|
|
2651
2664
|
sql,
|
|
@@ -2814,10 +2827,21 @@ function throwResultSchemaMismatch(datasetId, detail) {
|
|
|
2814
2827
|
|
|
2815
2828
|
// src/internal/ducklake-write-coordinator.ts
|
|
2816
2829
|
import { randomUUID as randomUUID6 } from "crypto";
|
|
2817
|
-
import {
|
|
2830
|
+
import {
|
|
2831
|
+
assertUnsequencedDatasetWrite,
|
|
2832
|
+
getDatasetPrimaryKeyColumns as getDatasetPrimaryKeyColumns4,
|
|
2833
|
+
hasDatasetInputChanges,
|
|
2834
|
+
LakeConcurrencyError,
|
|
2835
|
+
LakeStorageError as LakeStorageError22,
|
|
2836
|
+
retryDatasetMergeCommit
|
|
2837
|
+
} from "@sixb/core/lake-storage";
|
|
2818
2838
|
|
|
2819
2839
|
// src/internal/dataset-row-merge.ts
|
|
2820
|
-
import {
|
|
2840
|
+
import {
|
|
2841
|
+
getDatasetPrimaryKeyColumns as getDatasetPrimaryKeyColumns2,
|
|
2842
|
+
LakeStorageError as LakeStorageError19,
|
|
2843
|
+
reconcileDatasetSequences
|
|
2844
|
+
} from "@sixb/core/lake-storage";
|
|
2821
2845
|
async function applyDatasetMergeFromRelation(input) {
|
|
2822
2846
|
const primaryKeyColumns = getDatasetPrimaryKeyColumns2(input.dataset);
|
|
2823
2847
|
if (primaryKeyColumns === null) {
|
|
@@ -2827,12 +2851,12 @@ async function applyDatasetMergeFromRelation(input) {
|
|
|
2827
2851
|
const targetTable = qualifiedTableName(input.options, encodeDatasetTableName(input.dataset.id));
|
|
2828
2852
|
const sourceRowCount = await countRows2(input.runtime, stagingTable);
|
|
2829
2853
|
const previousRowCount = await currentBaselineRowCount(input, targetTable, primaryKeyColumns);
|
|
2830
|
-
return withMaterializedFinalChanges(input, primaryKeyColumns, async (finalChanges) => {
|
|
2854
|
+
return withMaterializedFinalChanges(input, primaryKeyColumns, async (finalChanges, orderingChanged) => {
|
|
2831
2855
|
const effects = await countMergeEffects(input, targetTable, finalChanges, primaryKeyColumns);
|
|
2832
2856
|
const effectiveChangeCount = effects.inserts + effects.updates + effects.deletes;
|
|
2833
2857
|
if (effectiveChangeCount === 0) {
|
|
2834
2858
|
return {
|
|
2835
|
-
dataChangeExpected:
|
|
2859
|
+
dataChangeExpected: orderingChanged,
|
|
2836
2860
|
sourceRowCount,
|
|
2837
2861
|
resultingRowCount: { kind: "exact", value: previousRowCount }
|
|
2838
2862
|
};
|
|
@@ -2877,6 +2901,48 @@ async function applyDatasetMergeFromRelation(input) {
|
|
|
2877
2901
|
};
|
|
2878
2902
|
});
|
|
2879
2903
|
}
|
|
2904
|
+
async function applySourceOrdering(input, acceptedTable) {
|
|
2905
|
+
const source = quoteIdentifier(`${input.sequenceColumnName}_source`);
|
|
2906
|
+
const ordinal = quoteIdentifier(input.sequenceColumnName);
|
|
2907
|
+
const staging = quoteIdentifier(input.stagingTableName);
|
|
2908
|
+
const stateTable = qualifiedTableName(input.options, encodeDatasetSequenceTableName(input.dataset.id));
|
|
2909
|
+
await input.runtime.run(`CREATE TEMP TABLE ${acceptedTable} (source_key VARCHAR, ordinal UBIGINT)`);
|
|
2910
|
+
let lastOrdinal = -1n;
|
|
2911
|
+
let changed = false;
|
|
2912
|
+
while (true) {
|
|
2913
|
+
const staged = await input.runtime.query(`SELECT ${ordinal} AS ordinal, ${source} AS source FROM ${staging}
|
|
2914
|
+
WHERE ${ordinal} > ${lastOrdinal} ORDER BY ${ordinal} LIMIT 1000`);
|
|
2915
|
+
if (staged.length === 0)
|
|
2916
|
+
return changed;
|
|
2917
|
+
const changes = staged.map((row) => JSON.parse(getString(row, "source")));
|
|
2918
|
+
lastOrdinal = getBigIntLike(staged[staged.length - 1], "ordinal");
|
|
2919
|
+
const batchKeys = changes.map((change) => quoteSqlString(change.key)).join(",");
|
|
2920
|
+
const current = await input.runtime.query(`SELECT * FROM ${stateTable} WHERE source_key IN (${batchKeys})`);
|
|
2921
|
+
const previous = new Map;
|
|
2922
|
+
for (const row of current) {
|
|
2923
|
+
previous.set(getString(row, "source_key"), {
|
|
2924
|
+
sequence: getString(row, "sequence"),
|
|
2925
|
+
content: getOptionalString(row, "content") ?? null
|
|
2926
|
+
});
|
|
2927
|
+
}
|
|
2928
|
+
const { states, accepted } = reconcileDatasetSequences(input.dataset, previous, changes);
|
|
2929
|
+
if (accepted.size === 0)
|
|
2930
|
+
continue;
|
|
2931
|
+
changed = true;
|
|
2932
|
+
const stateValues = [];
|
|
2933
|
+
const acceptedValues = [];
|
|
2934
|
+
for (const [key, index] of accepted) {
|
|
2935
|
+
const state = states.get(key);
|
|
2936
|
+
stateValues.push(`(${quoteSqlString(key)}, ${quoteSqlString(state.sequence)}, ${state.content === null ? "NULL" : quoteSqlString(state.content)})`);
|
|
2937
|
+
acceptedValues.push(`(${quoteSqlString(key)}, ${getBigIntLike(staged[index], "ordinal")})`);
|
|
2938
|
+
}
|
|
2939
|
+
const keys = [...accepted.keys()].map(quoteSqlString).join(",");
|
|
2940
|
+
await input.runtime.run(`DELETE FROM ${stateTable} WHERE source_key IN (${keys})`);
|
|
2941
|
+
await input.runtime.run(`INSERT INTO ${stateTable} VALUES ${stateValues.join(",")}`);
|
|
2942
|
+
await input.runtime.run(`DELETE FROM ${acceptedTable} WHERE source_key IN (${keys})`);
|
|
2943
|
+
await input.runtime.run(`INSERT INTO ${acceptedTable} VALUES ${acceptedValues.join(",")}`);
|
|
2944
|
+
}
|
|
2945
|
+
}
|
|
2880
2946
|
async function currentBaselineRowCount(input, targetTable, primaryKeyColumns) {
|
|
2881
2947
|
if (sameColumns(input.validatedPrimaryKeyColumns, primaryKeyColumns)) {
|
|
2882
2948
|
return input.previousRowCount ?? countRows2(input.runtime, targetTable);
|
|
@@ -2890,18 +2956,23 @@ async function currentBaselineRowCount(input, targetTable, primaryKeyColumns) {
|
|
|
2890
2956
|
}
|
|
2891
2957
|
async function withMaterializedFinalChanges(input, primaryKeyColumns, run) {
|
|
2892
2958
|
const table = quoteIdentifier(`${input.stagingTableName}_final`);
|
|
2893
|
-
|
|
2894
|
-
CREATE TEMP TABLE ${table} AS
|
|
2895
|
-
${finalChangesSelectSql(input, primaryKeyColumns)}
|
|
2896
|
-
`);
|
|
2959
|
+
const acceptedTable = quoteIdentifier(`${input.stagingTableName}_accepted`);
|
|
2897
2960
|
let outcome;
|
|
2898
2961
|
try {
|
|
2899
|
-
|
|
2962
|
+
const sequenced = input.dataset.sequenceBy !== undefined;
|
|
2963
|
+
const orderingChanged = sequenced ? await applySourceOrdering(input, acceptedTable) : false;
|
|
2964
|
+
const ordinal = quoteIdentifier(input.sequenceColumnName);
|
|
2965
|
+
const selectSql = sequenced ? `SELECT source.*, source.${ordinal} AS ${quoteIdentifier(firstSequenceName(input.sequenceColumnName))}
|
|
2966
|
+
FROM ${quoteIdentifier(input.stagingTableName)} AS source
|
|
2967
|
+
JOIN ${acceptedTable} AS accepted ON source.${ordinal} = accepted.ordinal` : finalChangesSelectSql(input, primaryKeyColumns);
|
|
2968
|
+
await input.runtime.run(`CREATE TEMP TABLE ${table} AS ${selectSql}`);
|
|
2969
|
+
outcome = { kind: "success", value: await run(table, orderingChanged) };
|
|
2900
2970
|
} catch (error) {
|
|
2901
2971
|
outcome = { kind: "error", error };
|
|
2902
2972
|
}
|
|
2903
2973
|
try {
|
|
2904
2974
|
await input.runtime.run(`DROP TABLE IF EXISTS ${table}`);
|
|
2975
|
+
await input.runtime.run(`DROP TABLE IF EXISTS ${acceptedTable}`);
|
|
2905
2976
|
} catch (cleanupError) {
|
|
2906
2977
|
if (outcome.kind === "success") {
|
|
2907
2978
|
throw cleanupError;
|
|
@@ -2987,6 +3058,7 @@ function sameColumns(left, right) {
|
|
|
2987
3058
|
import { randomUUID as randomUUID4 } from "crypto";
|
|
2988
3059
|
import {
|
|
2989
3060
|
cloneDatasetMergeChange,
|
|
3061
|
+
datasetSequenceChange,
|
|
2990
3062
|
getDatasetMergeChangeValidationError,
|
|
2991
3063
|
LakeStorageError as LakeStorageError20
|
|
2992
3064
|
} from "@sixb/core/lake-storage";
|
|
@@ -3055,6 +3127,7 @@ async function createDuckLakeMergeSession(input) {
|
|
|
3055
3127
|
CREATE TEMP TABLE ${quoteIdentifier(stagingTableName)} (
|
|
3056
3128
|
${quoteIdentifier(sequenceColumnName)} UBIGINT NOT NULL,
|
|
3057
3129
|
${quoteIdentifier(kindColumnName)} VARCHAR NOT NULL,
|
|
3130
|
+
${input.merge.dataset.sequenceBy === undefined ? "" : `${quoteIdentifier(`${sequenceColumnName}_source`)} VARCHAR NOT NULL,`}
|
|
3058
3131
|
${datasetSchemaToDuckDbNullableColumnsSql(input.merge.dataset.schema)}
|
|
3059
3132
|
)
|
|
3060
3133
|
`);
|
|
@@ -3110,6 +3183,9 @@ class DuckLakeMergeSession {
|
|
|
3110
3183
|
for (const staged of batch) {
|
|
3111
3184
|
appender.appendBigInt(staged.sequence);
|
|
3112
3185
|
appender.appendVarchar(staged.change.kind);
|
|
3186
|
+
if (this.input.dataset.sequenceBy !== undefined) {
|
|
3187
|
+
appender.appendVarchar(JSON.stringify(datasetSequenceChange(this.input.dataset, staged.change)));
|
|
3188
|
+
}
|
|
3113
3189
|
appendDatasetRow(appender, this.input.dataset.schema, staged.change.kind === "upsert" ? staged.change.row : staged.change.key);
|
|
3114
3190
|
}
|
|
3115
3191
|
});
|
|
@@ -3298,6 +3374,7 @@ class DuckLakeWriteCoordinator {
|
|
|
3298
3374
|
throw new LakeStorageError22(`[SixbDuckLake] Unknown dataset '${input.dataset.id}'.`);
|
|
3299
3375
|
}
|
|
3300
3376
|
this.datasets.assertSchema(definition);
|
|
3377
|
+
assertUnsequencedDatasetWrite(definition);
|
|
3301
3378
|
const mode = input.mode ?? "snapshot";
|
|
3302
3379
|
assertDatasetWriteMode(mode, "write");
|
|
3303
3380
|
const runtime = await this.connections.stagingRuntime();
|
|
@@ -3321,9 +3398,9 @@ class DuckLakeWriteCoordinator {
|
|
|
3321
3398
|
throw new LakeStorageError22(`[SixbDuckLake] Dataset '${definition.id}' must define a primaryKey before it can be merged.`);
|
|
3322
3399
|
}
|
|
3323
3400
|
this.datasets.assertSchema(definition);
|
|
3324
|
-
const latestVersion = await this.connections.withAttachedRuntime((
|
|
3401
|
+
const latestVersion = await this.connections.withAttachedRuntime((runtime) => this.snapshots.getLatestVersionForDefinition(runtime, definition));
|
|
3325
3402
|
if (input.expectedLatestVersionId !== undefined && latestVersion?.versionId !== input.expectedLatestVersionId) {
|
|
3326
|
-
throw new
|
|
3403
|
+
throw new LakeConcurrencyError(`[SixbDuckLake] Optimistic merge start failed for dataset '${definition.id}': expected latest version '${input.expectedLatestVersionId}', found '${latestVersion?.versionId ?? "none"}'.`);
|
|
3327
3404
|
}
|
|
3328
3405
|
const runtime = await this.connections.stagingRuntime();
|
|
3329
3406
|
return createDuckLakeMergeSession({
|
|
@@ -3362,32 +3439,37 @@ class DuckLakeWriteCoordinator {
|
|
|
3362
3439
|
if (getDatasetPrimaryKeyColumns4(definition) === null) {
|
|
3363
3440
|
throw new LakeStorageError22(`[SixbDuckLake] Dataset '${definition.id}' must define a primaryKey before it can be merged.`);
|
|
3364
3441
|
}
|
|
3365
|
-
return
|
|
3366
|
-
|
|
3367
|
-
|
|
3368
|
-
|
|
3369
|
-
|
|
3370
|
-
|
|
3371
|
-
|
|
3372
|
-
|
|
3373
|
-
|
|
3374
|
-
|
|
3375
|
-
|
|
3376
|
-
|
|
3377
|
-
|
|
3378
|
-
|
|
3379
|
-
|
|
3380
|
-
|
|
3381
|
-
|
|
3382
|
-
|
|
3383
|
-
|
|
3384
|
-
|
|
3385
|
-
|
|
3386
|
-
|
|
3387
|
-
|
|
3442
|
+
return retryDatasetMergeCommit(input.merge, input.commit, (rebase) => this.withCommitRuntime(async (runtime) => {
|
|
3443
|
+
input.commit?.signal?.throwIfAborted();
|
|
3444
|
+
const baseVersionId = rebase ? (await this.snapshots.getLatestVersionForDefinition(runtime, definition))?.versionId ?? null : input.baseVersionId;
|
|
3445
|
+
return this.commitVersionOutcomeOnExclusiveRuntime(runtime, {
|
|
3446
|
+
dataset: definition,
|
|
3447
|
+
mode: "merge",
|
|
3448
|
+
expectedLatestVersionId: baseVersionId,
|
|
3449
|
+
allowInitialNoOp: !input.commit?.createInitialVersion,
|
|
3450
|
+
signal: input.commit?.signal,
|
|
3451
|
+
disableRetries: true,
|
|
3452
|
+
optimisticOperation: "merge commit",
|
|
3453
|
+
commitMessage: input.commit?.commitMessage ?? `merge dataset ${definition.id}`,
|
|
3454
|
+
producer: input.merge.producer,
|
|
3455
|
+
inputs: input.merge.inputs,
|
|
3456
|
+
applyChanges: async (commitRuntime, context) => {
|
|
3457
|
+
const result = await applyDatasetMergeFromRelation({
|
|
3458
|
+
options: this.options,
|
|
3459
|
+
runtime: commitRuntime,
|
|
3460
|
+
dataset: definition,
|
|
3461
|
+
stagingTableName: input.stagingTableName,
|
|
3462
|
+
sequenceColumnName: input.sequenceColumnName,
|
|
3463
|
+
kindColumnName: input.kindColumnName,
|
|
3464
|
+
previousRowCount: context.previousRowCount,
|
|
3465
|
+
validatedPrimaryKeyColumns: context.validatedPrimaryKeyColumns
|
|
3466
|
+
});
|
|
3467
|
+
if (result.sourceRowCount !== input.changesWritten) {
|
|
3468
|
+
throw new LakeStorageError22(`[SixbDuckLake] Staged merge for dataset '${definition.id}' accepted ${input.changesWritten} change(s), but DuckLake saw ${result.sourceRowCount} source change(s) at commit time.`);
|
|
3469
|
+
}
|
|
3470
|
+
return result;
|
|
3388
3471
|
}
|
|
3389
|
-
|
|
3390
|
-
}
|
|
3472
|
+
});
|
|
3391
3473
|
}));
|
|
3392
3474
|
}
|
|
3393
3475
|
async commitDatasetVersion(input) {
|
|
@@ -3399,6 +3481,7 @@ class DuckLakeWriteCoordinator {
|
|
|
3399
3481
|
});
|
|
3400
3482
|
}
|
|
3401
3483
|
async commitDatasetVersionOnExclusiveRuntime(runtime, input) {
|
|
3484
|
+
assertUnsequencedDatasetWrite(input.dataset);
|
|
3402
3485
|
const result = await this.commitVersionOutcomeOnExclusiveRuntime(runtime, {
|
|
3403
3486
|
...input,
|
|
3404
3487
|
disableRetries: input.mode === "append" && getDatasetPrimaryKeyColumns4(input.dataset) !== null
|
|
@@ -3451,7 +3534,8 @@ class DuckLakeWriteCoordinator {
|
|
|
3451
3534
|
previousRowCount: latestVersion?.rowCount,
|
|
3452
3535
|
validatedPrimaryKeyColumns: latestVersion?.validatedPrimaryKeyColumns
|
|
3453
3536
|
});
|
|
3454
|
-
|
|
3537
|
+
input.signal?.throwIfAborted();
|
|
3538
|
+
await this.ensureNoOpHasSnapshot(input, changeResult, latestVersion);
|
|
3455
3539
|
const commitId = randomUUID6();
|
|
3456
3540
|
await this.setCommitMetadata(input, commitId, commitRowCount(input, changeResult, latestVersion));
|
|
3457
3541
|
await input.runtime.run("COMMIT");
|
|
@@ -3471,6 +3555,9 @@ class DuckLakeWriteCoordinator {
|
|
|
3471
3555
|
} catch (error) {
|
|
3472
3556
|
if (!committed) {
|
|
3473
3557
|
await this.rollbackTransaction(input.runtime);
|
|
3558
|
+
if (error instanceof Error && error.message.startsWith("TransactionContext Error: Failed to commit: Failed to commit DuckLake transaction.") && error.message.includes("Exceeded the maximum retry count of 0") && error.message.includes('duplicate key value violates unique constraint "ducklake_snapshot_pkey"')) {
|
|
3559
|
+
throw new LakeConcurrencyError(`[SixbDuckLake] Concurrent catalog commit for dataset '${input.dataset.id}'.`, { cause: error });
|
|
3560
|
+
}
|
|
3474
3561
|
}
|
|
3475
3562
|
throw error;
|
|
3476
3563
|
}
|
|
@@ -3533,12 +3620,12 @@ class DuckLakeWriteCoordinator {
|
|
|
3533
3620
|
};
|
|
3534
3621
|
await input.runtime.run(`CALL ${quoteIdentifier(duckLakeAlias(this.options))}.set_commit_message(${quoteSqlString("Sixb")}, ${quoteSqlString(input.commitMessage)}, extra_info => ${quoteSqlString(JSON.stringify({ sixb: metadata }))})`);
|
|
3535
3622
|
}
|
|
3536
|
-
async
|
|
3537
|
-
if (changeResult.dataChangeExpected
|
|
3623
|
+
async ensureNoOpHasSnapshot(input, changeResult, knownLatestVersion) {
|
|
3624
|
+
if (changeResult.dataChangeExpected) {
|
|
3538
3625
|
return;
|
|
3539
3626
|
}
|
|
3540
3627
|
const latestVersion = knownLatestVersion ?? await this.snapshots.getLatestVersionSummaryForDefinition(input.runtime, input.dataset);
|
|
3541
|
-
if (latestVersion) {
|
|
3628
|
+
if (latestVersion ? input.mode === "merge" || !hasDatasetInputChanges(latestVersion.inputs, input.inputs) : input.allowInitialNoOp) {
|
|
3542
3629
|
return;
|
|
3543
3630
|
}
|
|
3544
3631
|
const table = qualifiedTableName(this.options, encodeDatasetTableName(input.dataset.id));
|
|
@@ -3609,7 +3696,7 @@ class DuckLakeWriteCoordinator {
|
|
|
3609
3696
|
return;
|
|
3610
3697
|
}
|
|
3611
3698
|
if (input.actualLatestVersionId !== input.expectedLatestVersionId) {
|
|
3612
|
-
throw new
|
|
3699
|
+
throw new LakeConcurrencyError(`[SixbDuckLake] Optimistic ${input.operation} failed for dataset '${input.datasetId}': expected latest version '${input.expectedLatestVersionId ?? "none"}', found '${input.actualLatestVersionId ?? "none"}'.`);
|
|
3613
3700
|
}
|
|
3614
3701
|
}
|
|
3615
3702
|
}
|
|
@@ -3639,8 +3726,8 @@ async function withCatalogCommitLock(options, run) {
|
|
|
3639
3726
|
const previous = localCatalogCommitLocks.get(key) ?? Promise.resolve();
|
|
3640
3727
|
const ready = previous.catch(() => {});
|
|
3641
3728
|
let release;
|
|
3642
|
-
const next = new Promise((
|
|
3643
|
-
release =
|
|
3729
|
+
const next = new Promise((resolve) => {
|
|
3730
|
+
release = resolve;
|
|
3644
3731
|
});
|
|
3645
3732
|
const current = ready.then(() => next);
|
|
3646
3733
|
localCatalogCommitLocks.set(key, current);
|
|
@@ -3728,4 +3815,4 @@ export {
|
|
|
3728
3815
|
DuckLakeStorage
|
|
3729
3816
|
};
|
|
3730
3817
|
|
|
3731
|
-
//# debugId=
|
|
3818
|
+
//# debugId=75A88322BA3CC9E264756E2164756E21
|