@sixb/ducklake 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -1
- package/dist/index.js +166 -47
- package/dist/index.js.map +12 -12
- package/dist/internal/dataset-row-merge.d.ts.map +1 -1
- package/dist/internal/ducklake-catalog-schema.d.ts +1 -0
- package/dist/internal/ducklake-catalog-schema.d.ts.map +1 -1
- package/dist/internal/ducklake-current-dataset-definitions.d.ts.map +1 -1
- package/dist/internal/ducklake-dataset-catalog.d.ts.map +1 -1
- package/dist/internal/ducklake-merge-session.d.ts.map +1 -1
- package/dist/internal/ducklake-snapshot-reader.d.ts +1 -0
- package/dist/internal/ducklake-snapshot-reader.d.ts.map +1 -1
- package/dist/internal/ducklake-sql-executor.d.ts.map +1 -1
- package/dist/internal/ducklake-write-coordinator.d.ts +2 -1
- package/dist/internal/ducklake-write-coordinator.d.ts.map +1 -1
- package/dist/internal/names.d.ts +2 -0
- package/dist/internal/names.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/internal/dataset-row-merge.ts +114 -36
- package/src/internal/ducklake-catalog-schema.ts +1 -0
- package/src/internal/ducklake-current-dataset-definitions.ts +5 -0
- package/src/internal/ducklake-dataset-catalog.ts +19 -7
- package/src/internal/ducklake-merge-session.ts +7 -0
- package/src/internal/ducklake-snapshot-reader.ts +2 -0
- package/src/internal/ducklake-sql-executor.ts +54 -11
- package/src/internal/ducklake-write-coordinator.ts +109 -34
- package/src/internal/names.ts +5 -0
- package/src/internal/versions.ts +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset-row-merge.d.ts","sourceRoot":"","sources":["../../src/internal/dataset-row-merge.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;
|
|
1
|
+
{"version":3,"file":"dataset-row-merge.d.ts","sourceRoot":"","sources":["../../src/internal/dataset-row-merge.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAQnD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AACtD,OAAO,EAAE,KAAK,sBAAsB,EAA8B,MAAM,sBAAsB,CAAA;AAE9F,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAK1D,MAAM,WAAW,kCAAkC;IACjD,QAAQ,CAAC,OAAO,EAAE,sBAAsB,CAAA;IACxC,QAAQ,CAAC,OAAO,EAAE,kBAAkB,CAAA;IACpC,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAA;IACnC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAA;IACjC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAA;IACnC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;IAC/B,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAA;IAClC,QAAQ,CAAC,0BAA0B,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACxD;AAQD,wBAAsB,6BAA6B,CACjD,KAAK,EAAE,kCAAkC,GACxC,OAAO,CAAC,sBAAsB,CAAC,CAyEjC"}
|
|
@@ -9,6 +9,7 @@ export interface DuckLakeCatalogColumn {
|
|
|
9
9
|
readonly parentColumnId?: bigint;
|
|
10
10
|
readonly comment?: string;
|
|
11
11
|
}
|
|
12
|
+
export declare const DATASET_SEQUENCE_COLUMN_COMMENT = "sixb.sequenceBy.v1";
|
|
12
13
|
export declare function duckLakeCatalogColumnsToDatasetSchema(tableName: string, rows: readonly DuckLakeCatalogColumn[]): DatasetSchema;
|
|
13
14
|
export declare function duckLakePrimaryKeyColumnComment(ordinal: number): string;
|
|
14
15
|
export declare function duckLakeCatalogColumnsToDatasetPrimaryKey(tableName: string, rows: readonly DuckLakeCatalogColumn[]): DatasetPrimaryKey | undefined;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ducklake-catalog-schema.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-catalog-schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAA2B,iBAAiB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAI3F,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAA;IAC9B,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAA;IAChC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAC1B;
|
|
1
|
+
{"version":3,"file":"ducklake-catalog-schema.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-catalog-schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAA2B,iBAAiB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAI3F,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAA;IAC9B,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAA;IAChC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAC1B;AAGD,eAAO,MAAM,+BAA+B,uBAAuB,CAAA;AAYnE,wBAAgB,qCAAqC,CACnD,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,SAAS,qBAAqB,EAAE,GACrC,aAAa,CAoBf;AAED,wBAAgB,+BAA+B,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAEvE;AAED,wBAAgB,yCAAyC,CACvD,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,SAAS,qBAAqB,EAAE,GACrC,iBAAiB,GAAG,SAAS,CAkE/B"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ducklake-current-dataset-definitions.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-current-dataset-definitions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAEnD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAQtD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;
|
|
1
|
+
{"version":3,"file":"ducklake-current-dataset-definitions.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-current-dataset-definitions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAEnD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAQtD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAsB1D,MAAM,WAAW,kCAAkC;IACjD,QAAQ,CAAC,OAAO,EAAE,sBAAsB,CAAA;IACxC,QAAQ,CAAC,OAAO,EAAE,kBAAkB,CAAA;IACpC,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACxC;AAED,wBAAsB,6BAA6B,CACjD,KAAK,EAAE,kCAAkC,GACxC,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC,CAsCzC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ducklake-dataset-catalog.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-dataset-catalog.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;
|
|
1
|
+
{"version":3,"file":"ducklake-dataset-catalog.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-dataset-catalog.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAOlE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAEtD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAO1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAA;AAY9E;;;;;;GAMG;AACH,qBAAa,sBAAsB;IAE/B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;gBADX,OAAO,EAAE,sBAAsB,EAC/B,WAAW,EAAE,yBAAyB;IAGnD,aAAa,CAAC,UAAU,EAAE,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAiExE,kCAAkC,CACtC,WAAW,EAAE,SAAS,iBAAiB,EAAE,GACxC,OAAO,CAAC,IAAI,CAAC;IAwCV,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAQhE,mBAAmB,CACvB,OAAO,EAAE,kBAAkB,EAC3B,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAU9B,YAAY,IAAI,OAAO,CAAC,SAAS,iBAAiB,EAAE,CAAC;YAU7C,6BAA6B;YAY7B,sCAAsC;IAW9C,0BAA0B,CAC9B,OAAO,EAAE,kBAAkB,EAC3B,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,EACf,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,aAAa,CAAC;IAazB,YAAY,CAAC,UAAU,EAAE,iBAAiB,GAAG,IAAI;YAOnC,0BAA0B;YA+B1B,uBAAuB;YAYvB,kBAAkB;YAmBlB,gBAAgB;YA0BhB,eAAe;YAoBf,gCAAgC;YAwBhC,mBAAmB;YASnB,4BAA4B;CAgC3C"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ducklake-merge-session.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-merge-session.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,sBAAsB,EACtB,uBAAuB,EACvB,wBAAwB,EACxB,gBAAgB,EACjB,MAAM,yBAAyB,CAAA;
|
|
1
|
+
{"version":3,"file":"ducklake-merge-session.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-merge-session.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,sBAAsB,EACtB,uBAAuB,EACvB,wBAAwB,EACxB,gBAAgB,EACjB,MAAM,yBAAyB,CAAA;AAOhC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAKrD,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,KAAK,EAAE,sBAAsB,CAAA;IACtC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;IACrC,QAAQ,CAAC,MAAM,CAAC,EAAE,uBAAuB,CAAA;IACzC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAA;IACjC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAA;IACnC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;IAC/B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;CAChC;AAED,KAAK,mBAAmB,GAAG,CAAC,KAAK,EAAE,wBAAwB,KAAK,OAAO,CAAC,wBAAwB,CAAC,CAAA;AASjG,UAAU,+BAA+B;IACvC,QAAQ,CAAC,WAAW,EAAE,mBAAmB,CAAA;IACzC,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAA;IAC/B,QAAQ,CAAC,KAAK,EAAE,sBAAsB,CAAA;IACtC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;CACtC;AAED,wBAAsB,0BAA0B,CAC9C,KAAK,EAAE,+BAA+B,GACrC,OAAO,CAAC,gBAAgB,CAAC,CAwB3B"}
|
|
@@ -8,6 +8,7 @@ import { type DatasetTableRef } from "./ducklake-dataset-table-ref";
|
|
|
8
8
|
export interface DuckLakeVersionSummary {
|
|
9
9
|
readonly datasetId: string;
|
|
10
10
|
readonly versionId: string;
|
|
11
|
+
readonly inputs?: readonly DatasetVersionRef[];
|
|
11
12
|
readonly rowCount?: number;
|
|
12
13
|
readonly validatedPrimaryKeyColumns?: readonly string[];
|
|
13
14
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ducklake-snapshot-reader.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-snapshot-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,KAAK,EACV,mBAAmB,EAEnB,cAAc,EAEd,iBAAiB,EAClB,MAAM,yBAAyB,CAAA;AAEhC,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAEtD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAC1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAA;AAC9E,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACxE,OAAO,EAAE,KAAK,eAAe,EAA0B,MAAM,8BAA8B,CAAA;AA4C3F,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,0BAA0B,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACxD;AAyBD;;;;;;;;;;;GAWG;AACH,qBAAa,sBAAsB;IAE/B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAFR,OAAO,EAAE,sBAAsB,EAC/B,WAAW,EAAE,yBAAyB,EACtC,QAAQ,EAAE,sBAAsB;IAG7C,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC;IAanF,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAanE,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IActF;;;;;;OAMG;IACG,uBAAuB,CAC3B,UAAU,EAAE,SAAS,MAAM,EAAE,GAC5B,OAAO,CAAC,SAAS,mBAAmB,EAAE,CAAC;YAY5B,0BAA0B;IAsBxC,OAAO,CAAC,oBAAoB;YAad,sBAAsB;IAiCpC;;;;;;OAMG;YACW,yBAAyB;YA8DzB,yBAAyB;YAyCzB,oBAAoB;IAuDlC,OAAO,CAAC,wBAAwB;IAY1B,6BAA6B,CACjC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAK3B,gCAAgC,CACpC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAK9B,oCAAoC,CACxC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"ducklake-snapshot-reader.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-snapshot-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,KAAK,EACV,mBAAmB,EAEnB,cAAc,EAEd,iBAAiB,EAClB,MAAM,yBAAyB,CAAA;AAEhC,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAEtD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAC1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAA;AAC9E,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACxE,OAAO,EAAE,KAAK,eAAe,EAA0B,MAAM,8BAA8B,CAAA;AA4C3F,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,iBAAiB,EAAE,CAAA;IAC9C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,0BAA0B,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACxD;AAyBD;;;;;;;;;;;GAWG;AACH,qBAAa,sBAAsB;IAE/B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAFR,OAAO,EAAE,sBAAsB,EAC/B,WAAW,EAAE,yBAAyB,EACtC,QAAQ,EAAE,sBAAsB;IAG7C,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC;IAanF,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAanE,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IActF;;;;;;OAMG;IACG,uBAAuB,CAC3B,UAAU,EAAE,SAAS,MAAM,EAAE,GAC5B,OAAO,CAAC,SAAS,mBAAmB,EAAE,CAAC;YAY5B,0BAA0B;IAsBxC,OAAO,CAAC,oBAAoB;YAad,sBAAsB;IAiCpC;;;;;;OAMG;YACW,yBAAyB;YA8DzB,yBAAyB;YAyCzB,oBAAoB;IAuDlC,OAAO,CAAC,wBAAwB;IAY1B,6BAA6B,CACjC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAK3B,gCAAgC,CACpC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAK9B,oCAAoC,CACxC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC;IAsBnC,qBAAqB,CACzB,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,EAC1B,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAO3B,qBAAqB,CACzB,OAAO,EAAE,kBAAkB,EAC3B,QAAQ,EAAE,eAAe,EACzB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAO3B,wBAAwB,CAC5B,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,EAC1B,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAc9B,yBAAyB,CAC7B,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,EAC1B,KAAK,CAAC,EAAE,MAAM,GACb,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC;YAKvB,uBAAuB;IAe/B,2BAA2B,CAC/B,OAAO,EAAE,kBAAkB,EAC3B,QAAQ,EAAE,eAAe,GACxB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;YAKnB,+BAA+B;YAY/B,yBAAyB;YAmCzB,0BAA0B;YAmB1B,0BAA0B;YA2C1B,uBAAuB;IAsGrC,OAAO,CAAC,sBAAsB;YAyChB,iBAAiB;IAgC/B,OAAO,CAAC,qBAAqB;CAa9B"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ducklake-sql-executor.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-sql-executor.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAA8C,UAAU,EAAE,MAAM,YAAY,CAAA;AACxF,OAAO,EAEL,KAAK,wBAAwB,EAE7B,KAAK,wBAAwB,EAC7B,KAAK,eAAe,EAEpB,KAAK,wBAAwB,EAC9B,MAAM,yBAAyB,CAAA;AAChC,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;
|
|
1
|
+
{"version":3,"file":"ducklake-sql-executor.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-sql-executor.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAA8C,UAAU,EAAE,MAAM,YAAY,CAAA;AACxF,OAAO,EAEL,KAAK,wBAAwB,EAE7B,KAAK,wBAAwB,EAC7B,KAAK,eAAe,EAEpB,KAAK,wBAAwB,EAC9B,MAAM,yBAAyB,CAAA;AAChC,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAStD,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAA;AAC9E,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACxE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACxE,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAoC5E;;GAEG;AACH,qBAAa,mBAAoB,YAAW,eAAe,CAAC,QAAQ,CAAC;IASjE,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,MAAM;IAZzB,QAAQ,CAAC,OAAO,EAAG,QAAQ,CAAS;IACpC,QAAQ,CAAC,YAAY;;;;MAIpB;gBAGkB,OAAO,EAAE,sBAAsB,EAC/B,WAAW,EAAE,yBAAyB,EACtC,QAAQ,EAAE,sBAAsB,EAChC,SAAS,EAAE,sBAAsB,EACjC,MAAM,EAAE,wBAAwB;IAG5C,OAAO,CAAC,KAAK,EAAE,wBAAwB,CAAC,QAAQ,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC;IAkB9E,OAAO,CAAC,KAAK,EAAE,wBAAwB,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC,wBAAwB,CAAC;YAmC7E,cAAc;YA2Bd,oBAAoB;YAcpB,oBAAoB;YAepB,oBAAoB;YAuBpB,+BAA+B;YAoD/B,iBAAiB;CAuChC"}
|
|
@@ -9,7 +9,7 @@ import type { DuckLakeSnapshotReader } from "./ducklake-snapshot-reader";
|
|
|
9
9
|
export interface DuckLakeCommitDatasetVersionInput {
|
|
10
10
|
readonly dataset: DatasetDefinition;
|
|
11
11
|
readonly mode: DatasetWriteMode;
|
|
12
|
-
readonly expectedLatestVersionId?: string;
|
|
12
|
+
readonly expectedLatestVersionId?: string | null;
|
|
13
13
|
readonly commitMessage: string;
|
|
14
14
|
readonly producer?: BeginDatasetWriteInput["producer"];
|
|
15
15
|
readonly inputs?: BeginDatasetWriteInput["inputs"];
|
|
@@ -46,6 +46,7 @@ export declare class DuckLakeWriteCoordinator {
|
|
|
46
46
|
private versionForCommittedWrite;
|
|
47
47
|
private applyStagedRows;
|
|
48
48
|
private setCommitMetadata;
|
|
49
|
+
private ensureNoOpHasSnapshot;
|
|
49
50
|
private versionForNoOpCommit;
|
|
50
51
|
private rollbackTransaction;
|
|
51
52
|
private latestVersionForCommit;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ducklake-write-coordinator.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-write-coordinator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,KAAK,EACV,sBAAsB,EACtB,sBAAsB,EAGtB,wBAAwB,EACxB,gBAAgB,EAChB,gBAAgB,EAChB,gBAAgB,EACjB,MAAM,yBAAyB,CAAA;
|
|
1
|
+
{"version":3,"file":"ducklake-write-coordinator.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-write-coordinator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,KAAK,EACV,sBAAsB,EACtB,sBAAsB,EAGtB,wBAAwB,EACxB,gBAAgB,EAChB,gBAAgB,EAChB,gBAAgB,EACjB,MAAM,yBAAyB,CAAA;AAShC,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAEtD,OAAO,EACL,KAAK,sBAAsB,EAI5B,MAAM,sBAAsB,CAAA;AAG7B,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAC1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAA;AAC9E,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAExE,OAAO,KAAK,EAAE,sBAAsB,EAA0B,MAAM,4BAA4B,CAAA;AAYhG,MAAM,WAAW,iCAAiC;IAChD,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAA;IACnC,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAA;IAC/B,QAAQ,CAAC,uBAAuB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IAChD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;IAC9B,QAAQ,CAAC,QAAQ,CAAC,EAAE,sBAAsB,CAAC,UAAU,CAAC,CAAA;IACtD,QAAQ,CAAC,MAAM,CAAC,EAAE,sBAAsB,CAAC,QAAQ,CAAC,CAAA;IAClD,YAAY,CACV,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,2BAA2B,GACnC,OAAO,CAAC,sBAAsB,CAAC,CAAA;CACnC;AAgBD,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAA;IAClC,QAAQ,CAAC,0BAA0B,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACxD;AAED;;;;;;;GAOG;AACH,qBAAa,wBAAwB;IAEjC,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,SAAS;gBAHT,OAAO,EAAE,sBAAsB,EAC/B,WAAW,EAAE,yBAAyB,EACtC,QAAQ,EAAE,sBAAsB,EAChC,SAAS,EAAE,sBAAsB;IAG9C,UAAU,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA6BpE,UAAU,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;YAkC5D,WAAW;YAyBX,WAAW;IAoDnB,oBAAoB,CACxB,KAAK,EAAE,iCAAiC,GACvC,OAAO,CAAC,wBAAwB,CAAC;IAM9B,iBAAiB,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAQlF,sCAAsC,CAC1C,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,iCAAiC,GACvC,OAAO,CAAC,wBAAwB,CAAC;YAkBtB,sCAAsC;YAUtC,uBAAuB;YA2CvB,4BAA4B;YA8E5B,wBAAwB;YAiExB,eAAe;YAyBf,iBAAiB;YAkCjB,qBAAqB;YA8BrB,oBAAoB;YAmBpB,mBAAmB;YASnB,sBAAsB;YAUtB,uBAAuB;YAYvB,wBAAwB;YAiCxB,iBAAiB;IA0B/B,OAAO,CAAC,2BAA2B;CAgBpC"}
|
package/dist/internal/names.d.ts
CHANGED
|
@@ -6,6 +6,8 @@ export declare const DATASET_TABLE_PREFIX = "sixb__ds__";
|
|
|
6
6
|
* Convert a Sixb dataset id to its physical DuckLake table name.
|
|
7
7
|
*/
|
|
8
8
|
export declare function encodeDatasetTableName(datasetId: string): string;
|
|
9
|
+
/** Current source-ordering state, deliberately outside the dataset table namespace. */
|
|
10
|
+
export declare function encodeDatasetSequenceTableName(datasetId: string): string;
|
|
9
11
|
/**
|
|
10
12
|
* Decode a physical DuckLake table name back to a Sixb dataset id.
|
|
11
13
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"names.d.ts","sourceRoot":"","sources":["../../src/internal/names.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,eAAO,MAAM,oBAAoB,eAAe,CAAA;AAyFhD;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEhE;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAMvE"}
|
|
1
|
+
{"version":3,"file":"names.d.ts","sourceRoot":"","sources":["../../src/internal/names.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,eAAO,MAAM,oBAAoB,eAAe,CAAA;AAyFhD;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEhE;AAED,uFAAuF;AACvF,wBAAgB,8BAA8B,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAExE;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAMvE"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sixb/ducklake",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.4",
|
|
4
4
|
"description": "DuckDB/DuckLake lake storage provider for Sixb",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"sixb",
|
|
@@ -44,10 +44,10 @@
|
|
|
44
44
|
"@duckdb/node-api": "^1.5.2-r.1"
|
|
45
45
|
},
|
|
46
46
|
"peerDependencies": {
|
|
47
|
-
"@sixb/core": "^0.1.
|
|
47
|
+
"@sixb/core": "^0.1.8"
|
|
48
48
|
},
|
|
49
49
|
"devDependencies": {
|
|
50
|
-
"@sixb/core": "0.1.
|
|
50
|
+
"@sixb/core": "0.1.8",
|
|
51
51
|
"@types/bun": "latest",
|
|
52
52
|
"typescript": "^5.7.0"
|
|
53
53
|
},
|
|
@@ -1,10 +1,16 @@
|
|
|
1
1
|
import type { DatasetDefinition } from "@sixb/core"
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
type DatasetSequenceChange,
|
|
4
|
+
type DatasetSequenceState,
|
|
5
|
+
getDatasetPrimaryKeyColumns,
|
|
6
|
+
LakeStorageError,
|
|
7
|
+
reconcileDatasetSequences,
|
|
8
|
+
} from "@sixb/core/lake-storage"
|
|
3
9
|
import type { DuckLakeStorageOptions } from "../types"
|
|
4
10
|
import { type ApplyDatasetRowsResult, inspectUniqueKeyedRelation } from "./dataset-row-commit"
|
|
5
|
-
import { getBigIntLike } from "./duckdb-row"
|
|
11
|
+
import { getBigIntLike, getOptionalString, getString } from "./duckdb-row"
|
|
6
12
|
import type { DuckDbQueryRuntime } from "./duckdb-runtime"
|
|
7
|
-
import { encodeDatasetTableName } from "./names"
|
|
13
|
+
import { encodeDatasetSequenceTableName, encodeDatasetTableName } from "./names"
|
|
8
14
|
import { datasetSchemaColumnNamesSql } from "./schema"
|
|
9
15
|
import { qualifiedTableName, quoteIdentifier, quoteSqlString } from "./sql"
|
|
10
16
|
|
|
@@ -40,23 +46,26 @@ export async function applyDatasetMergeFromRelation(
|
|
|
40
46
|
const sourceRowCount = await countRows(input.runtime, stagingTable)
|
|
41
47
|
const previousRowCount = await currentBaselineRowCount(input, targetTable, primaryKeyColumns)
|
|
42
48
|
|
|
43
|
-
return withMaterializedFinalChanges(
|
|
44
|
-
|
|
45
|
-
|
|
49
|
+
return withMaterializedFinalChanges(
|
|
50
|
+
input,
|
|
51
|
+
primaryKeyColumns,
|
|
52
|
+
async (finalChanges, orderingChanged) => {
|
|
53
|
+
const effects = await countMergeEffects(input, targetTable, finalChanges, primaryKeyColumns)
|
|
54
|
+
const effectiveChangeCount = effects.inserts + effects.updates + effects.deletes
|
|
46
55
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
56
|
+
if (effectiveChangeCount === 0) {
|
|
57
|
+
return {
|
|
58
|
+
dataChangeExpected: orderingChanged,
|
|
59
|
+
sourceRowCount,
|
|
60
|
+
resultingRowCount: { kind: "exact", value: previousRowCount },
|
|
61
|
+
}
|
|
52
62
|
}
|
|
53
|
-
}
|
|
54
63
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
64
|
+
const keyMatchSql = primaryKeyMatchSql(primaryKeyColumns, "target", "source")
|
|
65
|
+
const rowDifferenceSql = datasetRowDifferenceSql(input.dataset, "target", "source")
|
|
66
|
+
const kindColumn = `source.${quoteIdentifier(input.kindColumnName)}`
|
|
58
67
|
|
|
59
|
-
|
|
68
|
+
await input.runtime.run(`
|
|
60
69
|
DELETE FROM ${targetTable} AS target
|
|
61
70
|
USING ${finalChanges} AS source
|
|
62
71
|
WHERE ${keyMatchSql}
|
|
@@ -69,12 +78,12 @@ export async function applyDatasetMergeFromRelation(
|
|
|
69
78
|
)
|
|
70
79
|
`)
|
|
71
80
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
81
|
+
const columnsSql = datasetSchemaColumnNamesSql(input.dataset.schema)
|
|
82
|
+
const selectedColumnsSql = input.dataset.schema.columns
|
|
83
|
+
.map((column) => `source.${quoteIdentifier(column.name)}`)
|
|
84
|
+
.join(", ")
|
|
85
|
+
const firstSequenceColumnName = firstSequenceName(input.sequenceColumnName)
|
|
86
|
+
await input.runtime.run(`
|
|
78
87
|
INSERT INTO ${targetTable} (${columnsSql})
|
|
79
88
|
SELECT ${selectedColumnsSql}
|
|
80
89
|
FROM ${finalChanges} AS source
|
|
@@ -87,15 +96,77 @@ export async function applyDatasetMergeFromRelation(
|
|
|
87
96
|
ORDER BY source.${quoteIdentifier(firstSequenceColumnName)}
|
|
88
97
|
`)
|
|
89
98
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
99
|
+
return {
|
|
100
|
+
dataChangeExpected: true,
|
|
101
|
+
sourceRowCount,
|
|
102
|
+
resultingRowCount: {
|
|
103
|
+
kind: "exact",
|
|
104
|
+
value: previousRowCount + effects.inserts - effects.deletes,
|
|
105
|
+
},
|
|
106
|
+
}
|
|
97
107
|
}
|
|
98
|
-
|
|
108
|
+
)
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** The companion table is current state, committed in the same DuckLake transaction as rows.
|
|
112
|
+
* It is not recovered from expirable snapshot history. Only keys present in staging are read.
|
|
113
|
+
*/
|
|
114
|
+
async function applySourceOrdering(
|
|
115
|
+
input: ApplyDatasetMergeFromRelationInput,
|
|
116
|
+
acceptedTable: string
|
|
117
|
+
): Promise<boolean> {
|
|
118
|
+
const source = quoteIdentifier(`${input.sequenceColumnName}_source`)
|
|
119
|
+
const ordinal = quoteIdentifier(input.sequenceColumnName)
|
|
120
|
+
const staging = quoteIdentifier(input.stagingTableName)
|
|
121
|
+
const stateTable = qualifiedTableName(
|
|
122
|
+
input.options,
|
|
123
|
+
encodeDatasetSequenceTableName(input.dataset.id)
|
|
124
|
+
)
|
|
125
|
+
await input.runtime.run(
|
|
126
|
+
`CREATE TEMP TABLE ${acceptedTable} (source_key VARCHAR, ordinal UBIGINT)`
|
|
127
|
+
)
|
|
128
|
+
let lastOrdinal = -1n
|
|
129
|
+
let changed = false
|
|
130
|
+
while (true) {
|
|
131
|
+
const staged = await input.runtime.query(
|
|
132
|
+
`SELECT ${ordinal} AS ordinal, ${source} AS source FROM ${staging}
|
|
133
|
+
WHERE ${ordinal} > ${lastOrdinal} ORDER BY ${ordinal} LIMIT 1000`
|
|
134
|
+
)
|
|
135
|
+
if (staged.length === 0) return changed
|
|
136
|
+
const changes = staged.map(
|
|
137
|
+
(row) => JSON.parse(getString(row, "source")) as DatasetSequenceChange
|
|
138
|
+
)
|
|
139
|
+
lastOrdinal = getBigIntLike(staged[staged.length - 1]!, "ordinal")
|
|
140
|
+
const batchKeys = changes.map((change) => quoteSqlString(change.key)).join(",")
|
|
141
|
+
const current = await input.runtime.query(
|
|
142
|
+
`SELECT * FROM ${stateTable} WHERE source_key IN (${batchKeys})`
|
|
143
|
+
)
|
|
144
|
+
const previous = new Map<string, DatasetSequenceState>()
|
|
145
|
+
for (const row of current) {
|
|
146
|
+
previous.set(getString(row, "source_key"), {
|
|
147
|
+
sequence: getString(row, "sequence"),
|
|
148
|
+
content: getOptionalString(row, "content") ?? null,
|
|
149
|
+
})
|
|
150
|
+
}
|
|
151
|
+
// Earlier batches are visible inside this transaction. A later conflict rolls them all back.
|
|
152
|
+
const { states, accepted } = reconcileDatasetSequences(input.dataset, previous, changes)
|
|
153
|
+
if (accepted.size === 0) continue
|
|
154
|
+
changed = true
|
|
155
|
+
const stateValues: string[] = []
|
|
156
|
+
const acceptedValues: string[] = []
|
|
157
|
+
for (const [key, index] of accepted) {
|
|
158
|
+
const state = states.get(key)!
|
|
159
|
+
stateValues.push(
|
|
160
|
+
`(${quoteSqlString(key)}, ${quoteSqlString(state.sequence)}, ${state.content === null ? "NULL" : quoteSqlString(state.content)})`
|
|
161
|
+
)
|
|
162
|
+
acceptedValues.push(`(${quoteSqlString(key)}, ${getBigIntLike(staged[index]!, "ordinal")})`)
|
|
163
|
+
}
|
|
164
|
+
const keys = [...accepted.keys()].map(quoteSqlString).join(",")
|
|
165
|
+
await input.runtime.run(`DELETE FROM ${stateTable} WHERE source_key IN (${keys})`)
|
|
166
|
+
await input.runtime.run(`INSERT INTO ${stateTable} VALUES ${stateValues.join(",")}`)
|
|
167
|
+
await input.runtime.run(`DELETE FROM ${acceptedTable} WHERE source_key IN (${keys})`)
|
|
168
|
+
await input.runtime.run(`INSERT INTO ${acceptedTable} VALUES ${acceptedValues.join(",")}`)
|
|
169
|
+
}
|
|
99
170
|
}
|
|
100
171
|
|
|
101
172
|
async function currentBaselineRowCount(
|
|
@@ -120,25 +191,32 @@ async function currentBaselineRowCount(
|
|
|
120
191
|
async function withMaterializedFinalChanges<T>(
|
|
121
192
|
input: ApplyDatasetMergeFromRelationInput,
|
|
122
193
|
primaryKeyColumns: readonly string[],
|
|
123
|
-
run: (finalChangesTable: string) => Promise<T>
|
|
194
|
+
run: (finalChangesTable: string, orderingChanged: boolean) => Promise<T>
|
|
124
195
|
): Promise<T> {
|
|
125
196
|
const table = quoteIdentifier(`${input.stagingTableName}_final`)
|
|
126
|
-
|
|
127
|
-
CREATE TEMP TABLE ${table} AS
|
|
128
|
-
${finalChangesSelectSql(input, primaryKeyColumns)}
|
|
129
|
-
`)
|
|
197
|
+
const acceptedTable = quoteIdentifier(`${input.stagingTableName}_accepted`)
|
|
130
198
|
|
|
131
199
|
let outcome:
|
|
132
200
|
| { readonly kind: "success"; readonly value: T }
|
|
133
201
|
| { readonly kind: "error"; readonly error: unknown }
|
|
134
202
|
try {
|
|
135
|
-
|
|
203
|
+
const sequenced = input.dataset.sequenceBy !== undefined
|
|
204
|
+
const orderingChanged = sequenced ? await applySourceOrdering(input, acceptedTable) : false
|
|
205
|
+
const ordinal = quoteIdentifier(input.sequenceColumnName)
|
|
206
|
+
const selectSql = sequenced
|
|
207
|
+
? `SELECT source.*, source.${ordinal} AS ${quoteIdentifier(firstSequenceName(input.sequenceColumnName))}
|
|
208
|
+
FROM ${quoteIdentifier(input.stagingTableName)} AS source
|
|
209
|
+
JOIN ${acceptedTable} AS accepted ON source.${ordinal} = accepted.ordinal`
|
|
210
|
+
: finalChangesSelectSql(input, primaryKeyColumns)
|
|
211
|
+
await input.runtime.run(`CREATE TEMP TABLE ${table} AS ${selectSql}`)
|
|
212
|
+
outcome = { kind: "success", value: await run(table, orderingChanged) }
|
|
136
213
|
} catch (error) {
|
|
137
214
|
outcome = { kind: "error", error }
|
|
138
215
|
}
|
|
139
216
|
|
|
140
217
|
try {
|
|
141
218
|
await input.runtime.run(`DROP TABLE IF EXISTS ${table}`)
|
|
219
|
+
await input.runtime.run(`DROP TABLE IF EXISTS ${acceptedTable}`)
|
|
142
220
|
} catch (cleanupError) {
|
|
143
221
|
if (outcome.kind === "success") {
|
|
144
222
|
throw cleanupError
|
|
@@ -14,6 +14,7 @@ export interface DuckLakeCatalogColumn {
|
|
|
14
14
|
}
|
|
15
15
|
|
|
16
16
|
const SIXB_PRIMARY_KEY_COMMENT_NAMESPACE = "sixb:primary-key:"
|
|
17
|
+
export const DATASET_SEQUENCE_COLUMN_COMMENT = "sixb.sequenceBy.v1"
|
|
17
18
|
const SIXB_PRIMARY_KEY_COMMENT_PREFIX = `${SIXB_PRIMARY_KEY_COMMENT_NAMESPACE}v1:`
|
|
18
19
|
|
|
19
20
|
const FILE_REF_STRUCT_CHILDREN = [
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
} from "./duckdb-row"
|
|
11
11
|
import type { DuckDbQueryRuntime } from "./duckdb-runtime"
|
|
12
12
|
import {
|
|
13
|
+
DATASET_SEQUENCE_COLUMN_COMMENT,
|
|
13
14
|
type DuckLakeCatalogColumn,
|
|
14
15
|
duckLakeCatalogColumnsToDatasetPrimaryKey,
|
|
15
16
|
duckLakeCatalogColumnsToDatasetSchema,
|
|
@@ -59,12 +60,16 @@ export async function readCurrentDatasetDefinitions(
|
|
|
59
60
|
for (const table of tables) {
|
|
60
61
|
const columns = columnsByTableName.get(table.tableName) ?? []
|
|
61
62
|
const primaryKey = duckLakeCatalogColumnsToDatasetPrimaryKey(table.tableName, columns)
|
|
63
|
+
const sequenceBy = columns.find(
|
|
64
|
+
(column) => column.comment === DATASET_SEQUENCE_COLUMN_COMMENT
|
|
65
|
+
)?.columnName
|
|
62
66
|
const partitionBy = partitionByTableName.get(table.tableName) ?? []
|
|
63
67
|
definitions.set(table.datasetId, {
|
|
64
68
|
kind: "dataset",
|
|
65
69
|
id: table.datasetId,
|
|
66
70
|
schema: duckLakeCatalogColumnsToDatasetSchema(table.tableName, columns),
|
|
67
71
|
...(primaryKey !== undefined ? { primaryKey } : {}),
|
|
72
|
+
...(sequenceBy !== undefined ? { sequenceBy } : {}),
|
|
68
73
|
...(partitionBy.length > 0 ? { partitionBy } : {}),
|
|
69
74
|
...(table.comment !== undefined ? { description: table.comment } : {}),
|
|
70
75
|
})
|
|
@@ -1,18 +1,23 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto"
|
|
2
2
|
import type { DatasetDefinition, DatasetSchema } from "@sixb/core"
|
|
3
3
|
import type { DatasetDefinitionUpdatePlan, DatasetSchemaUpdatePlan } from "@sixb/core/lake-storage"
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
assertDatasetDefinition,
|
|
6
|
+
LakeStorageError,
|
|
7
|
+
planDatasetDefinitionUpdate,
|
|
8
|
+
} from "@sixb/core/lake-storage"
|
|
5
9
|
import type { DuckLakeStorageOptions } from "../types"
|
|
6
10
|
import { getBigIntLike, getBoolean, getOptionalBigIntLike, getString } from "./duckdb-row"
|
|
7
11
|
import type { DuckDbQueryRuntime } from "./duckdb-runtime"
|
|
8
12
|
import {
|
|
13
|
+
DATASET_SEQUENCE_COLUMN_COMMENT,
|
|
9
14
|
type DuckLakeCatalogColumn,
|
|
10
15
|
duckLakeCatalogColumnsToDatasetSchema,
|
|
11
16
|
duckLakePrimaryKeyColumnComment,
|
|
12
17
|
} from "./ducklake-catalog-schema"
|
|
13
18
|
import type { DuckLakeConnectionManager } from "./ducklake-connection-manager"
|
|
14
19
|
import { readCurrentDatasetDefinitions } from "./ducklake-current-dataset-definitions"
|
|
15
|
-
import { encodeDatasetTableName } from "./names"
|
|
20
|
+
import { encodeDatasetSequenceTableName, encodeDatasetTableName } from "./names"
|
|
16
21
|
import { datasetColumnToDuckDbSql, datasetSchemaToDuckDbColumnsSql } from "./schema"
|
|
17
22
|
import {
|
|
18
23
|
duckLakeAlias,
|
|
@@ -76,6 +81,14 @@ export class DuckLakeDatasetCatalog {
|
|
|
76
81
|
}
|
|
77
82
|
|
|
78
83
|
await this.applyPrimaryKey(runtime, qualifiedName, definition)
|
|
84
|
+
if (definition.sequenceBy !== undefined) {
|
|
85
|
+
await runtime.run(
|
|
86
|
+
`COMMENT ON COLUMN ${qualifiedName}.${quoteIdentifier(definition.sequenceBy)} IS ${quoteSqlString(DATASET_SEQUENCE_COLUMN_COMMENT)}`
|
|
87
|
+
)
|
|
88
|
+
await runtime.run(
|
|
89
|
+
`CREATE TABLE ${qualifiedTableName(this.options, encodeDatasetSequenceTableName(definition.id))} (source_key VARCHAR NOT NULL, sequence VARCHAR NOT NULL, content VARCHAR)`
|
|
90
|
+
)
|
|
91
|
+
}
|
|
79
92
|
await this.applyPartitionBy(runtime, qualifiedName, definition)
|
|
80
93
|
|
|
81
94
|
await runtime.run("COMMIT")
|
|
@@ -208,11 +221,10 @@ export class DuckLakeDatasetCatalog {
|
|
|
208
221
|
}
|
|
209
222
|
|
|
210
223
|
assertSchema(definition: DatasetDefinition): void {
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
}
|
|
224
|
+
assertDatasetDefinition(
|
|
225
|
+
definition,
|
|
226
|
+
(message) => new LakeStorageError(`[SixbDuckLake] ${message}`)
|
|
227
|
+
)
|
|
216
228
|
}
|
|
217
229
|
|
|
218
230
|
private async applyExistingDatasetUpdate(plan: DatasetDefinitionUpdatePlan): Promise<void> {
|
|
@@ -8,6 +8,7 @@ import type {
|
|
|
8
8
|
} from "@sixb/core/lake-storage"
|
|
9
9
|
import {
|
|
10
10
|
cloneDatasetMergeChange,
|
|
11
|
+
datasetSequenceChange,
|
|
11
12
|
getDatasetMergeChangeValidationError,
|
|
12
13
|
LakeStorageError,
|
|
13
14
|
} from "@sixb/core/lake-storage"
|
|
@@ -54,6 +55,7 @@ export async function createDuckLakeMergeSession(
|
|
|
54
55
|
CREATE TEMP TABLE ${quoteIdentifier(stagingTableName)} (
|
|
55
56
|
${quoteIdentifier(sequenceColumnName)} UBIGINT NOT NULL,
|
|
56
57
|
${quoteIdentifier(kindColumnName)} VARCHAR NOT NULL,
|
|
58
|
+
${input.merge.dataset.sequenceBy === undefined ? "" : `${quoteIdentifier(`${sequenceColumnName}_source`)} VARCHAR NOT NULL,`}
|
|
57
59
|
${datasetSchemaToDuckDbNullableColumnsSql(input.merge.dataset.schema)}
|
|
58
60
|
)
|
|
59
61
|
`)
|
|
@@ -122,6 +124,11 @@ class DuckLakeMergeSession implements LakeMergeSession {
|
|
|
122
124
|
for (const staged of batch) {
|
|
123
125
|
appender.appendBigInt(staged.sequence)
|
|
124
126
|
appender.appendVarchar(staged.change.kind)
|
|
127
|
+
if (this.input.dataset.sequenceBy !== undefined) {
|
|
128
|
+
appender.appendVarchar(
|
|
129
|
+
JSON.stringify(datasetSequenceChange(this.input.dataset, staged.change))
|
|
130
|
+
)
|
|
131
|
+
}
|
|
125
132
|
appendDatasetRow(
|
|
126
133
|
appender,
|
|
127
134
|
this.input.dataset.schema,
|
|
@@ -59,6 +59,7 @@ interface VisibleSnapshotRowsInput {
|
|
|
59
59
|
export interface DuckLakeVersionSummary {
|
|
60
60
|
readonly datasetId: string
|
|
61
61
|
readonly versionId: string
|
|
62
|
+
readonly inputs?: readonly DatasetVersionRef[]
|
|
62
63
|
readonly rowCount?: number
|
|
63
64
|
readonly validatedPrimaryKeyColumns?: readonly string[]
|
|
64
65
|
}
|
|
@@ -444,6 +445,7 @@ export class DuckLakeSnapshotReader {
|
|
|
444
445
|
return {
|
|
445
446
|
datasetId: tableRef.datasetId,
|
|
446
447
|
versionId: toVersionId(row.snapshotId),
|
|
448
|
+
inputs: row.metadata?.inputs,
|
|
447
449
|
...(row.metadata?.rowCount !== undefined ? { rowCount: row.metadata.rowCount } : {}),
|
|
448
450
|
...(row.metadata?.validatedPrimaryKeyColumns !== undefined
|
|
449
451
|
? { validatedPrimaryKeyColumns: row.metadata.validatedPrimaryKeyColumns }
|
|
@@ -15,13 +15,14 @@ import {
|
|
|
15
15
|
applyDatasetRowsFromRelation,
|
|
16
16
|
assertDatasetWriteMode,
|
|
17
17
|
} from "./dataset-row-commit"
|
|
18
|
+
import { findDatasetColumnTypeForDuckDbSql } from "./duckdb-column-types"
|
|
18
19
|
import { getString } from "./duckdb-row"
|
|
19
20
|
import type { DuckDbQueryRuntime } from "./duckdb-runtime"
|
|
20
21
|
import type { DuckLakeConnectionManager } from "./ducklake-connection-manager"
|
|
21
22
|
import type { DuckLakeDatasetCatalog } from "./ducklake-dataset-catalog"
|
|
22
23
|
import type { DuckLakeSnapshotReader } from "./ducklake-snapshot-reader"
|
|
23
24
|
import type { DuckLakeWriteCoordinator } from "./ducklake-write-coordinator"
|
|
24
|
-
import {
|
|
25
|
+
import { datasetColumnTypeToDuckDbSql } from "./schema"
|
|
25
26
|
import { quoteIdentifier } from "./sql"
|
|
26
27
|
import {
|
|
27
28
|
type DuckLakeSqlTransformSourceRelation,
|
|
@@ -42,9 +43,20 @@ interface ApplySqlTransformInput {
|
|
|
42
43
|
readonly target: DatasetDefinition
|
|
43
44
|
readonly mode: DatasetWriteMode
|
|
44
45
|
readonly sql: string
|
|
46
|
+
readonly checkedCasts: readonly SqlTransformCheckedCast[]
|
|
45
47
|
readonly previousRowCount?: number
|
|
46
48
|
}
|
|
47
49
|
|
|
50
|
+
interface SqlTransformResultColumn {
|
|
51
|
+
readonly name: string
|
|
52
|
+
readonly typeSql: string
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
interface SqlTransformCheckedCast {
|
|
56
|
+
readonly columnName: string
|
|
57
|
+
readonly targetTypeSql: string
|
|
58
|
+
}
|
|
59
|
+
|
|
48
60
|
/**
|
|
49
61
|
* DuckDB-backed SQL transform executor for DuckLakeStorage.
|
|
50
62
|
*/
|
|
@@ -95,7 +107,7 @@ export class DuckLakeSqlExecutor implements LakeSqlExecutor<"duckdb"> {
|
|
|
95
107
|
sql: input.sql,
|
|
96
108
|
})
|
|
97
109
|
|
|
98
|
-
await this.assertResultSchemaMatchesTarget(runtime, target, sql)
|
|
110
|
+
const checkedCasts = await this.assertResultSchemaMatchesTarget(runtime, target, sql)
|
|
99
111
|
|
|
100
112
|
return this.writes.commitDatasetVersionOnExclusiveRuntime(runtime, {
|
|
101
113
|
dataset: target,
|
|
@@ -110,6 +122,7 @@ export class DuckLakeSqlExecutor implements LakeSqlExecutor<"duckdb"> {
|
|
|
110
122
|
target,
|
|
111
123
|
mode: input.mode,
|
|
112
124
|
sql,
|
|
125
|
+
checkedCasts,
|
|
113
126
|
previousRowCount: context.previousRowCount,
|
|
114
127
|
}),
|
|
115
128
|
})
|
|
@@ -199,12 +212,13 @@ export class DuckLakeSqlExecutor implements LakeSqlExecutor<"duckdb"> {
|
|
|
199
212
|
runtime: DuckDbQueryRuntime,
|
|
200
213
|
target: DatasetDefinition,
|
|
201
214
|
sql: string
|
|
202
|
-
): Promise<
|
|
215
|
+
): Promise<readonly SqlTransformCheckedCast[]> {
|
|
203
216
|
const rows = await runtime.query(
|
|
204
217
|
`DESCRIBE SELECT * FROM (${sql}) AS sixb_sql_transform_result_schema`
|
|
205
218
|
)
|
|
206
219
|
const actualColumns = rows.map((row) => resultColumnFromDescribeRow(row))
|
|
207
220
|
const expectedColumns = target.schema.columns
|
|
221
|
+
const checkedCasts: SqlTransformCheckedCast[] = []
|
|
208
222
|
|
|
209
223
|
if (actualColumns.length !== expectedColumns.length) {
|
|
210
224
|
throwResultSchemaMismatch(
|
|
@@ -224,13 +238,26 @@ export class DuckLakeSqlExecutor implements LakeSqlExecutor<"duckdb"> {
|
|
|
224
238
|
)
|
|
225
239
|
}
|
|
226
240
|
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
`column '${expected.name}' should have type '${expected.type}', got '${actual.type}'`
|
|
231
|
-
)
|
|
241
|
+
const actualDatasetType = findDatasetColumnTypeForDuckDbSql(actual.typeSql)
|
|
242
|
+
if (actualDatasetType === expected.type) {
|
|
243
|
+
continue
|
|
232
244
|
}
|
|
245
|
+
|
|
246
|
+
if (isCheckedInt64Cast(actual.typeSql, expected.type)) {
|
|
247
|
+
checkedCasts.push({
|
|
248
|
+
columnName: expected.name,
|
|
249
|
+
targetTypeSql: datasetColumnTypeToDuckDbSql(expected.type),
|
|
250
|
+
})
|
|
251
|
+
continue
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
throwResultSchemaMismatch(
|
|
255
|
+
target.id,
|
|
256
|
+
`column '${expected.name}' should have type '${expected.type}', got '${actualDatasetType ?? actual.typeSql}'`
|
|
257
|
+
)
|
|
233
258
|
}
|
|
259
|
+
|
|
260
|
+
return checkedCasts
|
|
234
261
|
}
|
|
235
262
|
|
|
236
263
|
private async applySqlTransform(input: ApplySqlTransformInput): Promise<ApplyDatasetRowsResult> {
|
|
@@ -243,6 +270,14 @@ export class DuckLakeSqlExecutor implements LakeSqlExecutor<"duckdb"> {
|
|
|
243
270
|
|
|
244
271
|
let result: ApplyDatasetRowsResult
|
|
245
272
|
try {
|
|
273
|
+
for (const checkedCast of input.checkedCasts) {
|
|
274
|
+
await input.runtime.run(
|
|
275
|
+
`ALTER TABLE ${tempTable} ALTER COLUMN ${quoteIdentifier(
|
|
276
|
+
checkedCast.columnName
|
|
277
|
+
)} SET DATA TYPE ${checkedCast.targetTypeSql}`
|
|
278
|
+
)
|
|
279
|
+
}
|
|
280
|
+
|
|
246
281
|
result = await applyDatasetRowsFromRelation({
|
|
247
282
|
options: this.options,
|
|
248
283
|
runtime: input.runtime,
|
|
@@ -280,11 +315,19 @@ function buildPreviewSql(sql: string, limit: number): string {
|
|
|
280
315
|
|
|
281
316
|
function resultColumnFromDescribeRow(
|
|
282
317
|
row: Readonly<Record<string, unknown>>
|
|
283
|
-
):
|
|
318
|
+
): SqlTransformResultColumn {
|
|
284
319
|
const name = getString(row, "column_name")
|
|
285
|
-
const
|
|
320
|
+
const typeSql = getString(row, "column_type")
|
|
321
|
+
|
|
322
|
+
return { name, typeSql }
|
|
323
|
+
}
|
|
286
324
|
|
|
287
|
-
|
|
325
|
+
function isCheckedInt64Cast(
|
|
326
|
+
actualTypeSql: string,
|
|
327
|
+
expectedType: DatasetColumnDefinition["type"]
|
|
328
|
+
): boolean {
|
|
329
|
+
const normalizedType = actualTypeSql.trim().toUpperCase()
|
|
330
|
+
return expectedType === "int64" && (normalizedType === "HUGEINT" || normalizedType === "INT128")
|
|
288
331
|
}
|
|
289
332
|
|
|
290
333
|
function normalizePreviewRow(row: Readonly<Record<string, unknown>>): DatasetRow {
|