@sixb/ducklake 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -1
- package/dist/index.js +133 -46
- package/dist/index.js.map +11 -11
- package/dist/internal/dataset-row-merge.d.ts.map +1 -1
- package/dist/internal/ducklake-catalog-schema.d.ts +1 -0
- package/dist/internal/ducklake-catalog-schema.d.ts.map +1 -1
- package/dist/internal/ducklake-current-dataset-definitions.d.ts.map +1 -1
- package/dist/internal/ducklake-dataset-catalog.d.ts.map +1 -1
- package/dist/internal/ducklake-merge-session.d.ts.map +1 -1
- package/dist/internal/ducklake-snapshot-reader.d.ts +1 -0
- package/dist/internal/ducklake-snapshot-reader.d.ts.map +1 -1
- package/dist/internal/ducklake-write-coordinator.d.ts +2 -2
- package/dist/internal/ducklake-write-coordinator.d.ts.map +1 -1
- package/dist/internal/names.d.ts +2 -0
- package/dist/internal/names.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/internal/dataset-row-merge.ts +114 -36
- package/src/internal/ducklake-catalog-schema.ts +1 -0
- package/src/internal/ducklake-current-dataset-definitions.ts +5 -0
- package/src/internal/ducklake-dataset-catalog.ts +19 -7
- package/src/internal/ducklake-merge-session.ts +7 -0
- package/src/internal/ducklake-snapshot-reader.ts +2 -0
- package/src/internal/ducklake-write-coordinator.ts +79 -38
- package/src/internal/names.ts +5 -0
- package/src/internal/versions.ts +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset-row-merge.d.ts","sourceRoot":"","sources":["../../src/internal/dataset-row-merge.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;
|
|
1
|
+
{"version":3,"file":"dataset-row-merge.d.ts","sourceRoot":"","sources":["../../src/internal/dataset-row-merge.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAQnD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AACtD,OAAO,EAAE,KAAK,sBAAsB,EAA8B,MAAM,sBAAsB,CAAA;AAE9F,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAK1D,MAAM,WAAW,kCAAkC;IACjD,QAAQ,CAAC,OAAO,EAAE,sBAAsB,CAAA;IACxC,QAAQ,CAAC,OAAO,EAAE,kBAAkB,CAAA;IACpC,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAA;IACnC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAA;IACjC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAA;IACnC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;IAC/B,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAA;IAClC,QAAQ,CAAC,0BAA0B,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACxD;AAQD,wBAAsB,6BAA6B,CACjD,KAAK,EAAE,kCAAkC,GACxC,OAAO,CAAC,sBAAsB,CAAC,CAyEjC"}
|
|
@@ -9,6 +9,7 @@ export interface DuckLakeCatalogColumn {
|
|
|
9
9
|
readonly parentColumnId?: bigint;
|
|
10
10
|
readonly comment?: string;
|
|
11
11
|
}
|
|
12
|
+
export declare const DATASET_SEQUENCE_COLUMN_COMMENT = "sixb.sequenceBy.v1";
|
|
12
13
|
export declare function duckLakeCatalogColumnsToDatasetSchema(tableName: string, rows: readonly DuckLakeCatalogColumn[]): DatasetSchema;
|
|
13
14
|
export declare function duckLakePrimaryKeyColumnComment(ordinal: number): string;
|
|
14
15
|
export declare function duckLakeCatalogColumnsToDatasetPrimaryKey(tableName: string, rows: readonly DuckLakeCatalogColumn[]): DatasetPrimaryKey | undefined;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ducklake-catalog-schema.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-catalog-schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAA2B,iBAAiB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAI3F,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAA;IAC9B,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAA;IAChC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAC1B;
|
|
1
|
+
{"version":3,"file":"ducklake-catalog-schema.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-catalog-schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAA2B,iBAAiB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAI3F,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAA;IAC9B,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAA;IAChC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAC1B;AAGD,eAAO,MAAM,+BAA+B,uBAAuB,CAAA;AAYnE,wBAAgB,qCAAqC,CACnD,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,SAAS,qBAAqB,EAAE,GACrC,aAAa,CAoBf;AAED,wBAAgB,+BAA+B,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAEvE;AAED,wBAAgB,yCAAyC,CACvD,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,SAAS,qBAAqB,EAAE,GACrC,iBAAiB,GAAG,SAAS,CAkE/B"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ducklake-current-dataset-definitions.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-current-dataset-definitions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAEnD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAQtD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;
|
|
1
|
+
{"version":3,"file":"ducklake-current-dataset-definitions.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-current-dataset-definitions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAEnD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAQtD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAsB1D,MAAM,WAAW,kCAAkC;IACjD,QAAQ,CAAC,OAAO,EAAE,sBAAsB,CAAA;IACxC,QAAQ,CAAC,OAAO,EAAE,kBAAkB,CAAA;IACpC,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACxC;AAED,wBAAsB,6BAA6B,CACjD,KAAK,EAAE,kCAAkC,GACxC,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC,CAsCzC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ducklake-dataset-catalog.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-dataset-catalog.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;
|
|
1
|
+
{"version":3,"file":"ducklake-dataset-catalog.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-dataset-catalog.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAOlE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAEtD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAO1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAA;AAY9E;;;;;;GAMG;AACH,qBAAa,sBAAsB;IAE/B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;gBADX,OAAO,EAAE,sBAAsB,EAC/B,WAAW,EAAE,yBAAyB;IAGnD,aAAa,CAAC,UAAU,EAAE,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAiExE,kCAAkC,CACtC,WAAW,EAAE,SAAS,iBAAiB,EAAE,GACxC,OAAO,CAAC,IAAI,CAAC;IAwCV,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAQhE,mBAAmB,CACvB,OAAO,EAAE,kBAAkB,EAC3B,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAU9B,YAAY,IAAI,OAAO,CAAC,SAAS,iBAAiB,EAAE,CAAC;YAU7C,6BAA6B;YAY7B,sCAAsC;IAW9C,0BAA0B,CAC9B,OAAO,EAAE,kBAAkB,EAC3B,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,EACf,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,aAAa,CAAC;IAazB,YAAY,CAAC,UAAU,EAAE,iBAAiB,GAAG,IAAI;YAOnC,0BAA0B;YA+B1B,uBAAuB;YAYvB,kBAAkB;YAmBlB,gBAAgB;YA0BhB,eAAe;YAoBf,gCAAgC;YAwBhC,mBAAmB;YASnB,4BAA4B;CAgC3C"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ducklake-merge-session.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-merge-session.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,sBAAsB,EACtB,uBAAuB,EACvB,wBAAwB,EACxB,gBAAgB,EACjB,MAAM,yBAAyB,CAAA;
|
|
1
|
+
{"version":3,"file":"ducklake-merge-session.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-merge-session.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,sBAAsB,EACtB,uBAAuB,EACvB,wBAAwB,EACxB,gBAAgB,EACjB,MAAM,yBAAyB,CAAA;AAOhC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAKrD,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,KAAK,EAAE,sBAAsB,CAAA;IACtC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;IACrC,QAAQ,CAAC,MAAM,CAAC,EAAE,uBAAuB,CAAA;IACzC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAA;IACjC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAA;IACnC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;IAC/B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;CAChC;AAED,KAAK,mBAAmB,GAAG,CAAC,KAAK,EAAE,wBAAwB,KAAK,OAAO,CAAC,wBAAwB,CAAC,CAAA;AASjG,UAAU,+BAA+B;IACvC,QAAQ,CAAC,WAAW,EAAE,mBAAmB,CAAA;IACzC,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAA;IAC/B,QAAQ,CAAC,KAAK,EAAE,sBAAsB,CAAA;IACtC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;CACtC;AAED,wBAAsB,0BAA0B,CAC9C,KAAK,EAAE,+BAA+B,GACrC,OAAO,CAAC,gBAAgB,CAAC,CAwB3B"}
|
|
@@ -8,6 +8,7 @@ import { type DatasetTableRef } from "./ducklake-dataset-table-ref";
|
|
|
8
8
|
export interface DuckLakeVersionSummary {
|
|
9
9
|
readonly datasetId: string;
|
|
10
10
|
readonly versionId: string;
|
|
11
|
+
readonly inputs?: readonly DatasetVersionRef[];
|
|
11
12
|
readonly rowCount?: number;
|
|
12
13
|
readonly validatedPrimaryKeyColumns?: readonly string[];
|
|
13
14
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ducklake-snapshot-reader.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-snapshot-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,KAAK,EACV,mBAAmB,EAEnB,cAAc,EAEd,iBAAiB,EAClB,MAAM,yBAAyB,CAAA;AAEhC,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAEtD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAC1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAA;AAC9E,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACxE,OAAO,EAAE,KAAK,eAAe,EAA0B,MAAM,8BAA8B,CAAA;AA4C3F,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,0BAA0B,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACxD;AAyBD;;;;;;;;;;;GAWG;AACH,qBAAa,sBAAsB;IAE/B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAFR,OAAO,EAAE,sBAAsB,EAC/B,WAAW,EAAE,yBAAyB,EACtC,QAAQ,EAAE,sBAAsB;IAG7C,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC;IAanF,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAanE,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IActF;;;;;;OAMG;IACG,uBAAuB,CAC3B,UAAU,EAAE,SAAS,MAAM,EAAE,GAC5B,OAAO,CAAC,SAAS,mBAAmB,EAAE,CAAC;YAY5B,0BAA0B;IAsBxC,OAAO,CAAC,oBAAoB;YAad,sBAAsB;IAiCpC;;;;;;OAMG;YACW,yBAAyB;YA8DzB,yBAAyB;YAyCzB,oBAAoB;IAuDlC,OAAO,CAAC,wBAAwB;IAY1B,6BAA6B,CACjC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAK3B,gCAAgC,CACpC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAK9B,oCAAoC,CACxC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"ducklake-snapshot-reader.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-snapshot-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,KAAK,EACV,mBAAmB,EAEnB,cAAc,EAEd,iBAAiB,EAClB,MAAM,yBAAyB,CAAA;AAEhC,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAEtD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAC1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAA;AAC9E,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AACxE,OAAO,EAAE,KAAK,eAAe,EAA0B,MAAM,8BAA8B,CAAA;AA4C3F,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,iBAAiB,EAAE,CAAA;IAC9C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,0BAA0B,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACxD;AAyBD;;;;;;;;;;;GAWG;AACH,qBAAa,sBAAsB;IAE/B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAFR,OAAO,EAAE,sBAAsB,EAC/B,WAAW,EAAE,yBAAyB,EACtC,QAAQ,EAAE,sBAAsB;IAG7C,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC;IAanF,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAanE,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IActF;;;;;;OAMG;IACG,uBAAuB,CAC3B,UAAU,EAAE,SAAS,MAAM,EAAE,GAC5B,OAAO,CAAC,SAAS,mBAAmB,EAAE,CAAC;YAY5B,0BAA0B;IAsBxC,OAAO,CAAC,oBAAoB;YAad,sBAAsB;IAiCpC;;;;;;OAMG;YACW,yBAAyB;YA8DzB,yBAAyB;YAyCzB,oBAAoB;IAuDlC,OAAO,CAAC,wBAAwB;IAY1B,6BAA6B,CACjC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAK3B,gCAAgC,CACpC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAK9B,oCAAoC,CACxC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC;IAsBnC,qBAAqB,CACzB,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,EAC1B,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAO3B,qBAAqB,CACzB,OAAO,EAAE,kBAAkB,EAC3B,QAAQ,EAAE,eAAe,EACzB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAO3B,wBAAwB,CAC5B,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,EAC1B,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAc9B,yBAAyB,CAC7B,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,iBAAiB,EAC1B,KAAK,CAAC,EAAE,MAAM,GACb,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC;YAKvB,uBAAuB;IAe/B,2BAA2B,CAC/B,OAAO,EAAE,kBAAkB,EAC3B,QAAQ,EAAE,eAAe,GACxB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;YAKnB,+BAA+B;YAY/B,yBAAyB;YAmCzB,0BAA0B;YAmB1B,0BAA0B;YA2C1B,uBAAuB;IAsGrC,OAAO,CAAC,sBAAsB;YAyChB,iBAAiB;IAgC/B,OAAO,CAAC,qBAAqB;CAa9B"}
|
|
@@ -9,7 +9,7 @@ import type { DuckLakeSnapshotReader } from "./ducklake-snapshot-reader";
|
|
|
9
9
|
export interface DuckLakeCommitDatasetVersionInput {
|
|
10
10
|
readonly dataset: DatasetDefinition;
|
|
11
11
|
readonly mode: DatasetWriteMode;
|
|
12
|
-
readonly expectedLatestVersionId?: string;
|
|
12
|
+
readonly expectedLatestVersionId?: string | null;
|
|
13
13
|
readonly commitMessage: string;
|
|
14
14
|
readonly producer?: BeginDatasetWriteInput["producer"];
|
|
15
15
|
readonly inputs?: BeginDatasetWriteInput["inputs"];
|
|
@@ -46,7 +46,7 @@ export declare class DuckLakeWriteCoordinator {
|
|
|
46
46
|
private versionForCommittedWrite;
|
|
47
47
|
private applyStagedRows;
|
|
48
48
|
private setCommitMetadata;
|
|
49
|
-
private
|
|
49
|
+
private ensureNoOpHasSnapshot;
|
|
50
50
|
private versionForNoOpCommit;
|
|
51
51
|
private rollbackTransaction;
|
|
52
52
|
private latestVersionForCommit;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ducklake-write-coordinator.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-write-coordinator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,KAAK,EACV,sBAAsB,EACtB,sBAAsB,EAGtB,wBAAwB,EACxB,gBAAgB,EAChB,gBAAgB,EAChB,gBAAgB,EACjB,MAAM,yBAAyB,CAAA;
|
|
1
|
+
{"version":3,"file":"ducklake-write-coordinator.d.ts","sourceRoot":"","sources":["../../src/internal/ducklake-write-coordinator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,KAAK,EACV,sBAAsB,EACtB,sBAAsB,EAGtB,wBAAwB,EACxB,gBAAgB,EAChB,gBAAgB,EAChB,gBAAgB,EACjB,MAAM,yBAAyB,CAAA;AAShC,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAEtD,OAAO,EACL,KAAK,sBAAsB,EAI5B,MAAM,sBAAsB,CAAA;AAG7B,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAC1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAA;AAC9E,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAExE,OAAO,KAAK,EAAE,sBAAsB,EAA0B,MAAM,4BAA4B,CAAA;AAYhG,MAAM,WAAW,iCAAiC;IAChD,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAA;IACnC,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAA;IAC/B,QAAQ,CAAC,uBAAuB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IAChD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;IAC9B,QAAQ,CAAC,QAAQ,CAAC,EAAE,sBAAsB,CAAC,UAAU,CAAC,CAAA;IACtD,QAAQ,CAAC,MAAM,CAAC,EAAE,sBAAsB,CAAC,QAAQ,CAAC,CAAA;IAClD,YAAY,CACV,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,2BAA2B,GACnC,OAAO,CAAC,sBAAsB,CAAC,CAAA;CACnC;AAgBD,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAA;IAClC,QAAQ,CAAC,0BAA0B,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACxD;AAED;;;;;;;GAOG;AACH,qBAAa,wBAAwB;IAEjC,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,SAAS;gBAHT,OAAO,EAAE,sBAAsB,EAC/B,WAAW,EAAE,yBAAyB,EACtC,QAAQ,EAAE,sBAAsB,EAChC,SAAS,EAAE,sBAAsB;IAG9C,UAAU,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA6BpE,UAAU,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;YAkC5D,WAAW;YAyBX,WAAW;IAoDnB,oBAAoB,CACxB,KAAK,EAAE,iCAAiC,GACvC,OAAO,CAAC,wBAAwB,CAAC;IAM9B,iBAAiB,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAQlF,sCAAsC,CAC1C,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,iCAAiC,GACvC,OAAO,CAAC,wBAAwB,CAAC;YAkBtB,sCAAsC;YAUtC,uBAAuB;YA2CvB,4BAA4B;YA8E5B,wBAAwB;YAiExB,eAAe;YAyBf,iBAAiB;YAkCjB,qBAAqB;YA8BrB,oBAAoB;YAmBpB,mBAAmB;YASnB,sBAAsB;YAUtB,uBAAuB;YAYvB,wBAAwB;YAiCxB,iBAAiB;IA0B/B,OAAO,CAAC,2BAA2B;CAgBpC"}
|
package/dist/internal/names.d.ts
CHANGED
|
@@ -6,6 +6,8 @@ export declare const DATASET_TABLE_PREFIX = "sixb__ds__";
|
|
|
6
6
|
* Convert a Sixb dataset id to its physical DuckLake table name.
|
|
7
7
|
*/
|
|
8
8
|
export declare function encodeDatasetTableName(datasetId: string): string;
|
|
9
|
+
/** Current source-ordering state, deliberately outside the dataset table namespace. */
|
|
10
|
+
export declare function encodeDatasetSequenceTableName(datasetId: string): string;
|
|
9
11
|
/**
|
|
10
12
|
* Decode a physical DuckLake table name back to a Sixb dataset id.
|
|
11
13
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"names.d.ts","sourceRoot":"","sources":["../../src/internal/names.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,eAAO,MAAM,oBAAoB,eAAe,CAAA;AAyFhD;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEhE;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAMvE"}
|
|
1
|
+
{"version":3,"file":"names.d.ts","sourceRoot":"","sources":["../../src/internal/names.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,eAAO,MAAM,oBAAoB,eAAe,CAAA;AAyFhD;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEhE;AAED,uFAAuF;AACvF,wBAAgB,8BAA8B,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAExE;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAMvE"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sixb/ducklake",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.4",
|
|
4
4
|
"description": "DuckDB/DuckLake lake storage provider for Sixb",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"sixb",
|
|
@@ -44,10 +44,10 @@
|
|
|
44
44
|
"@duckdb/node-api": "^1.5.2-r.1"
|
|
45
45
|
},
|
|
46
46
|
"peerDependencies": {
|
|
47
|
-
"@sixb/core": "^0.1.
|
|
47
|
+
"@sixb/core": "^0.1.8"
|
|
48
48
|
},
|
|
49
49
|
"devDependencies": {
|
|
50
|
-
"@sixb/core": "0.1.
|
|
50
|
+
"@sixb/core": "0.1.8",
|
|
51
51
|
"@types/bun": "latest",
|
|
52
52
|
"typescript": "^5.7.0"
|
|
53
53
|
},
|
|
@@ -1,10 +1,16 @@
|
|
|
1
1
|
import type { DatasetDefinition } from "@sixb/core"
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
type DatasetSequenceChange,
|
|
4
|
+
type DatasetSequenceState,
|
|
5
|
+
getDatasetPrimaryKeyColumns,
|
|
6
|
+
LakeStorageError,
|
|
7
|
+
reconcileDatasetSequences,
|
|
8
|
+
} from "@sixb/core/lake-storage"
|
|
3
9
|
import type { DuckLakeStorageOptions } from "../types"
|
|
4
10
|
import { type ApplyDatasetRowsResult, inspectUniqueKeyedRelation } from "./dataset-row-commit"
|
|
5
|
-
import { getBigIntLike } from "./duckdb-row"
|
|
11
|
+
import { getBigIntLike, getOptionalString, getString } from "./duckdb-row"
|
|
6
12
|
import type { DuckDbQueryRuntime } from "./duckdb-runtime"
|
|
7
|
-
import { encodeDatasetTableName } from "./names"
|
|
13
|
+
import { encodeDatasetSequenceTableName, encodeDatasetTableName } from "./names"
|
|
8
14
|
import { datasetSchemaColumnNamesSql } from "./schema"
|
|
9
15
|
import { qualifiedTableName, quoteIdentifier, quoteSqlString } from "./sql"
|
|
10
16
|
|
|
@@ -40,23 +46,26 @@ export async function applyDatasetMergeFromRelation(
|
|
|
40
46
|
const sourceRowCount = await countRows(input.runtime, stagingTable)
|
|
41
47
|
const previousRowCount = await currentBaselineRowCount(input, targetTable, primaryKeyColumns)
|
|
42
48
|
|
|
43
|
-
return withMaterializedFinalChanges(
|
|
44
|
-
|
|
45
|
-
|
|
49
|
+
return withMaterializedFinalChanges(
|
|
50
|
+
input,
|
|
51
|
+
primaryKeyColumns,
|
|
52
|
+
async (finalChanges, orderingChanged) => {
|
|
53
|
+
const effects = await countMergeEffects(input, targetTable, finalChanges, primaryKeyColumns)
|
|
54
|
+
const effectiveChangeCount = effects.inserts + effects.updates + effects.deletes
|
|
46
55
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
56
|
+
if (effectiveChangeCount === 0) {
|
|
57
|
+
return {
|
|
58
|
+
dataChangeExpected: orderingChanged,
|
|
59
|
+
sourceRowCount,
|
|
60
|
+
resultingRowCount: { kind: "exact", value: previousRowCount },
|
|
61
|
+
}
|
|
52
62
|
}
|
|
53
|
-
}
|
|
54
63
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
64
|
+
const keyMatchSql = primaryKeyMatchSql(primaryKeyColumns, "target", "source")
|
|
65
|
+
const rowDifferenceSql = datasetRowDifferenceSql(input.dataset, "target", "source")
|
|
66
|
+
const kindColumn = `source.${quoteIdentifier(input.kindColumnName)}`
|
|
58
67
|
|
|
59
|
-
|
|
68
|
+
await input.runtime.run(`
|
|
60
69
|
DELETE FROM ${targetTable} AS target
|
|
61
70
|
USING ${finalChanges} AS source
|
|
62
71
|
WHERE ${keyMatchSql}
|
|
@@ -69,12 +78,12 @@ export async function applyDatasetMergeFromRelation(
|
|
|
69
78
|
)
|
|
70
79
|
`)
|
|
71
80
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
81
|
+
const columnsSql = datasetSchemaColumnNamesSql(input.dataset.schema)
|
|
82
|
+
const selectedColumnsSql = input.dataset.schema.columns
|
|
83
|
+
.map((column) => `source.${quoteIdentifier(column.name)}`)
|
|
84
|
+
.join(", ")
|
|
85
|
+
const firstSequenceColumnName = firstSequenceName(input.sequenceColumnName)
|
|
86
|
+
await input.runtime.run(`
|
|
78
87
|
INSERT INTO ${targetTable} (${columnsSql})
|
|
79
88
|
SELECT ${selectedColumnsSql}
|
|
80
89
|
FROM ${finalChanges} AS source
|
|
@@ -87,15 +96,77 @@ export async function applyDatasetMergeFromRelation(
|
|
|
87
96
|
ORDER BY source.${quoteIdentifier(firstSequenceColumnName)}
|
|
88
97
|
`)
|
|
89
98
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
99
|
+
return {
|
|
100
|
+
dataChangeExpected: true,
|
|
101
|
+
sourceRowCount,
|
|
102
|
+
resultingRowCount: {
|
|
103
|
+
kind: "exact",
|
|
104
|
+
value: previousRowCount + effects.inserts - effects.deletes,
|
|
105
|
+
},
|
|
106
|
+
}
|
|
97
107
|
}
|
|
98
|
-
|
|
108
|
+
)
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** The companion table is current state, committed in the same DuckLake transaction as rows.
|
|
112
|
+
* It is not recovered from expirable snapshot history. Only keys present in staging are read.
|
|
113
|
+
*/
|
|
114
|
+
async function applySourceOrdering(
|
|
115
|
+
input: ApplyDatasetMergeFromRelationInput,
|
|
116
|
+
acceptedTable: string
|
|
117
|
+
): Promise<boolean> {
|
|
118
|
+
const source = quoteIdentifier(`${input.sequenceColumnName}_source`)
|
|
119
|
+
const ordinal = quoteIdentifier(input.sequenceColumnName)
|
|
120
|
+
const staging = quoteIdentifier(input.stagingTableName)
|
|
121
|
+
const stateTable = qualifiedTableName(
|
|
122
|
+
input.options,
|
|
123
|
+
encodeDatasetSequenceTableName(input.dataset.id)
|
|
124
|
+
)
|
|
125
|
+
await input.runtime.run(
|
|
126
|
+
`CREATE TEMP TABLE ${acceptedTable} (source_key VARCHAR, ordinal UBIGINT)`
|
|
127
|
+
)
|
|
128
|
+
let lastOrdinal = -1n
|
|
129
|
+
let changed = false
|
|
130
|
+
while (true) {
|
|
131
|
+
const staged = await input.runtime.query(
|
|
132
|
+
`SELECT ${ordinal} AS ordinal, ${source} AS source FROM ${staging}
|
|
133
|
+
WHERE ${ordinal} > ${lastOrdinal} ORDER BY ${ordinal} LIMIT 1000`
|
|
134
|
+
)
|
|
135
|
+
if (staged.length === 0) return changed
|
|
136
|
+
const changes = staged.map(
|
|
137
|
+
(row) => JSON.parse(getString(row, "source")) as DatasetSequenceChange
|
|
138
|
+
)
|
|
139
|
+
lastOrdinal = getBigIntLike(staged[staged.length - 1]!, "ordinal")
|
|
140
|
+
const batchKeys = changes.map((change) => quoteSqlString(change.key)).join(",")
|
|
141
|
+
const current = await input.runtime.query(
|
|
142
|
+
`SELECT * FROM ${stateTable} WHERE source_key IN (${batchKeys})`
|
|
143
|
+
)
|
|
144
|
+
const previous = new Map<string, DatasetSequenceState>()
|
|
145
|
+
for (const row of current) {
|
|
146
|
+
previous.set(getString(row, "source_key"), {
|
|
147
|
+
sequence: getString(row, "sequence"),
|
|
148
|
+
content: getOptionalString(row, "content") ?? null,
|
|
149
|
+
})
|
|
150
|
+
}
|
|
151
|
+
// Earlier batches are visible inside this transaction. A later conflict rolls them all back.
|
|
152
|
+
const { states, accepted } = reconcileDatasetSequences(input.dataset, previous, changes)
|
|
153
|
+
if (accepted.size === 0) continue
|
|
154
|
+
changed = true
|
|
155
|
+
const stateValues: string[] = []
|
|
156
|
+
const acceptedValues: string[] = []
|
|
157
|
+
for (const [key, index] of accepted) {
|
|
158
|
+
const state = states.get(key)!
|
|
159
|
+
stateValues.push(
|
|
160
|
+
`(${quoteSqlString(key)}, ${quoteSqlString(state.sequence)}, ${state.content === null ? "NULL" : quoteSqlString(state.content)})`
|
|
161
|
+
)
|
|
162
|
+
acceptedValues.push(`(${quoteSqlString(key)}, ${getBigIntLike(staged[index]!, "ordinal")})`)
|
|
163
|
+
}
|
|
164
|
+
const keys = [...accepted.keys()].map(quoteSqlString).join(",")
|
|
165
|
+
await input.runtime.run(`DELETE FROM ${stateTable} WHERE source_key IN (${keys})`)
|
|
166
|
+
await input.runtime.run(`INSERT INTO ${stateTable} VALUES ${stateValues.join(",")}`)
|
|
167
|
+
await input.runtime.run(`DELETE FROM ${acceptedTable} WHERE source_key IN (${keys})`)
|
|
168
|
+
await input.runtime.run(`INSERT INTO ${acceptedTable} VALUES ${acceptedValues.join(",")}`)
|
|
169
|
+
}
|
|
99
170
|
}
|
|
100
171
|
|
|
101
172
|
async function currentBaselineRowCount(
|
|
@@ -120,25 +191,32 @@ async function currentBaselineRowCount(
|
|
|
120
191
|
async function withMaterializedFinalChanges<T>(
|
|
121
192
|
input: ApplyDatasetMergeFromRelationInput,
|
|
122
193
|
primaryKeyColumns: readonly string[],
|
|
123
|
-
run: (finalChangesTable: string) => Promise<T>
|
|
194
|
+
run: (finalChangesTable: string, orderingChanged: boolean) => Promise<T>
|
|
124
195
|
): Promise<T> {
|
|
125
196
|
const table = quoteIdentifier(`${input.stagingTableName}_final`)
|
|
126
|
-
|
|
127
|
-
CREATE TEMP TABLE ${table} AS
|
|
128
|
-
${finalChangesSelectSql(input, primaryKeyColumns)}
|
|
129
|
-
`)
|
|
197
|
+
const acceptedTable = quoteIdentifier(`${input.stagingTableName}_accepted`)
|
|
130
198
|
|
|
131
199
|
let outcome:
|
|
132
200
|
| { readonly kind: "success"; readonly value: T }
|
|
133
201
|
| { readonly kind: "error"; readonly error: unknown }
|
|
134
202
|
try {
|
|
135
|
-
|
|
203
|
+
const sequenced = input.dataset.sequenceBy !== undefined
|
|
204
|
+
const orderingChanged = sequenced ? await applySourceOrdering(input, acceptedTable) : false
|
|
205
|
+
const ordinal = quoteIdentifier(input.sequenceColumnName)
|
|
206
|
+
const selectSql = sequenced
|
|
207
|
+
? `SELECT source.*, source.${ordinal} AS ${quoteIdentifier(firstSequenceName(input.sequenceColumnName))}
|
|
208
|
+
FROM ${quoteIdentifier(input.stagingTableName)} AS source
|
|
209
|
+
JOIN ${acceptedTable} AS accepted ON source.${ordinal} = accepted.ordinal`
|
|
210
|
+
: finalChangesSelectSql(input, primaryKeyColumns)
|
|
211
|
+
await input.runtime.run(`CREATE TEMP TABLE ${table} AS ${selectSql}`)
|
|
212
|
+
outcome = { kind: "success", value: await run(table, orderingChanged) }
|
|
136
213
|
} catch (error) {
|
|
137
214
|
outcome = { kind: "error", error }
|
|
138
215
|
}
|
|
139
216
|
|
|
140
217
|
try {
|
|
141
218
|
await input.runtime.run(`DROP TABLE IF EXISTS ${table}`)
|
|
219
|
+
await input.runtime.run(`DROP TABLE IF EXISTS ${acceptedTable}`)
|
|
142
220
|
} catch (cleanupError) {
|
|
143
221
|
if (outcome.kind === "success") {
|
|
144
222
|
throw cleanupError
|
|
@@ -14,6 +14,7 @@ export interface DuckLakeCatalogColumn {
|
|
|
14
14
|
}
|
|
15
15
|
|
|
16
16
|
const SIXB_PRIMARY_KEY_COMMENT_NAMESPACE = "sixb:primary-key:"
|
|
17
|
+
export const DATASET_SEQUENCE_COLUMN_COMMENT = "sixb.sequenceBy.v1"
|
|
17
18
|
const SIXB_PRIMARY_KEY_COMMENT_PREFIX = `${SIXB_PRIMARY_KEY_COMMENT_NAMESPACE}v1:`
|
|
18
19
|
|
|
19
20
|
const FILE_REF_STRUCT_CHILDREN = [
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
} from "./duckdb-row"
|
|
11
11
|
import type { DuckDbQueryRuntime } from "./duckdb-runtime"
|
|
12
12
|
import {
|
|
13
|
+
DATASET_SEQUENCE_COLUMN_COMMENT,
|
|
13
14
|
type DuckLakeCatalogColumn,
|
|
14
15
|
duckLakeCatalogColumnsToDatasetPrimaryKey,
|
|
15
16
|
duckLakeCatalogColumnsToDatasetSchema,
|
|
@@ -59,12 +60,16 @@ export async function readCurrentDatasetDefinitions(
|
|
|
59
60
|
for (const table of tables) {
|
|
60
61
|
const columns = columnsByTableName.get(table.tableName) ?? []
|
|
61
62
|
const primaryKey = duckLakeCatalogColumnsToDatasetPrimaryKey(table.tableName, columns)
|
|
63
|
+
const sequenceBy = columns.find(
|
|
64
|
+
(column) => column.comment === DATASET_SEQUENCE_COLUMN_COMMENT
|
|
65
|
+
)?.columnName
|
|
62
66
|
const partitionBy = partitionByTableName.get(table.tableName) ?? []
|
|
63
67
|
definitions.set(table.datasetId, {
|
|
64
68
|
kind: "dataset",
|
|
65
69
|
id: table.datasetId,
|
|
66
70
|
schema: duckLakeCatalogColumnsToDatasetSchema(table.tableName, columns),
|
|
67
71
|
...(primaryKey !== undefined ? { primaryKey } : {}),
|
|
72
|
+
...(sequenceBy !== undefined ? { sequenceBy } : {}),
|
|
68
73
|
...(partitionBy.length > 0 ? { partitionBy } : {}),
|
|
69
74
|
...(table.comment !== undefined ? { description: table.comment } : {}),
|
|
70
75
|
})
|
|
@@ -1,18 +1,23 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto"
|
|
2
2
|
import type { DatasetDefinition, DatasetSchema } from "@sixb/core"
|
|
3
3
|
import type { DatasetDefinitionUpdatePlan, DatasetSchemaUpdatePlan } from "@sixb/core/lake-storage"
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
assertDatasetDefinition,
|
|
6
|
+
LakeStorageError,
|
|
7
|
+
planDatasetDefinitionUpdate,
|
|
8
|
+
} from "@sixb/core/lake-storage"
|
|
5
9
|
import type { DuckLakeStorageOptions } from "../types"
|
|
6
10
|
import { getBigIntLike, getBoolean, getOptionalBigIntLike, getString } from "./duckdb-row"
|
|
7
11
|
import type { DuckDbQueryRuntime } from "./duckdb-runtime"
|
|
8
12
|
import {
|
|
13
|
+
DATASET_SEQUENCE_COLUMN_COMMENT,
|
|
9
14
|
type DuckLakeCatalogColumn,
|
|
10
15
|
duckLakeCatalogColumnsToDatasetSchema,
|
|
11
16
|
duckLakePrimaryKeyColumnComment,
|
|
12
17
|
} from "./ducklake-catalog-schema"
|
|
13
18
|
import type { DuckLakeConnectionManager } from "./ducklake-connection-manager"
|
|
14
19
|
import { readCurrentDatasetDefinitions } from "./ducklake-current-dataset-definitions"
|
|
15
|
-
import { encodeDatasetTableName } from "./names"
|
|
20
|
+
import { encodeDatasetSequenceTableName, encodeDatasetTableName } from "./names"
|
|
16
21
|
import { datasetColumnToDuckDbSql, datasetSchemaToDuckDbColumnsSql } from "./schema"
|
|
17
22
|
import {
|
|
18
23
|
duckLakeAlias,
|
|
@@ -76,6 +81,14 @@ export class DuckLakeDatasetCatalog {
|
|
|
76
81
|
}
|
|
77
82
|
|
|
78
83
|
await this.applyPrimaryKey(runtime, qualifiedName, definition)
|
|
84
|
+
if (definition.sequenceBy !== undefined) {
|
|
85
|
+
await runtime.run(
|
|
86
|
+
`COMMENT ON COLUMN ${qualifiedName}.${quoteIdentifier(definition.sequenceBy)} IS ${quoteSqlString(DATASET_SEQUENCE_COLUMN_COMMENT)}`
|
|
87
|
+
)
|
|
88
|
+
await runtime.run(
|
|
89
|
+
`CREATE TABLE ${qualifiedTableName(this.options, encodeDatasetSequenceTableName(definition.id))} (source_key VARCHAR NOT NULL, sequence VARCHAR NOT NULL, content VARCHAR)`
|
|
90
|
+
)
|
|
91
|
+
}
|
|
79
92
|
await this.applyPartitionBy(runtime, qualifiedName, definition)
|
|
80
93
|
|
|
81
94
|
await runtime.run("COMMIT")
|
|
@@ -208,11 +221,10 @@ export class DuckLakeDatasetCatalog {
|
|
|
208
221
|
}
|
|
209
222
|
|
|
210
223
|
assertSchema(definition: DatasetDefinition): void {
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
}
|
|
224
|
+
assertDatasetDefinition(
|
|
225
|
+
definition,
|
|
226
|
+
(message) => new LakeStorageError(`[SixbDuckLake] ${message}`)
|
|
227
|
+
)
|
|
216
228
|
}
|
|
217
229
|
|
|
218
230
|
private async applyExistingDatasetUpdate(plan: DatasetDefinitionUpdatePlan): Promise<void> {
|
|
@@ -8,6 +8,7 @@ import type {
|
|
|
8
8
|
} from "@sixb/core/lake-storage"
|
|
9
9
|
import {
|
|
10
10
|
cloneDatasetMergeChange,
|
|
11
|
+
datasetSequenceChange,
|
|
11
12
|
getDatasetMergeChangeValidationError,
|
|
12
13
|
LakeStorageError,
|
|
13
14
|
} from "@sixb/core/lake-storage"
|
|
@@ -54,6 +55,7 @@ export async function createDuckLakeMergeSession(
|
|
|
54
55
|
CREATE TEMP TABLE ${quoteIdentifier(stagingTableName)} (
|
|
55
56
|
${quoteIdentifier(sequenceColumnName)} UBIGINT NOT NULL,
|
|
56
57
|
${quoteIdentifier(kindColumnName)} VARCHAR NOT NULL,
|
|
58
|
+
${input.merge.dataset.sequenceBy === undefined ? "" : `${quoteIdentifier(`${sequenceColumnName}_source`)} VARCHAR NOT NULL,`}
|
|
57
59
|
${datasetSchemaToDuckDbNullableColumnsSql(input.merge.dataset.schema)}
|
|
58
60
|
)
|
|
59
61
|
`)
|
|
@@ -122,6 +124,11 @@ class DuckLakeMergeSession implements LakeMergeSession {
|
|
|
122
124
|
for (const staged of batch) {
|
|
123
125
|
appender.appendBigInt(staged.sequence)
|
|
124
126
|
appender.appendVarchar(staged.change.kind)
|
|
127
|
+
if (this.input.dataset.sequenceBy !== undefined) {
|
|
128
|
+
appender.appendVarchar(
|
|
129
|
+
JSON.stringify(datasetSequenceChange(this.input.dataset, staged.change))
|
|
130
|
+
)
|
|
131
|
+
}
|
|
125
132
|
appendDatasetRow(
|
|
126
133
|
appender,
|
|
127
134
|
this.input.dataset.schema,
|
|
@@ -59,6 +59,7 @@ interface VisibleSnapshotRowsInput {
|
|
|
59
59
|
export interface DuckLakeVersionSummary {
|
|
60
60
|
readonly datasetId: string
|
|
61
61
|
readonly versionId: string
|
|
62
|
+
readonly inputs?: readonly DatasetVersionRef[]
|
|
62
63
|
readonly rowCount?: number
|
|
63
64
|
readonly validatedPrimaryKeyColumns?: readonly string[]
|
|
64
65
|
}
|
|
@@ -444,6 +445,7 @@ export class DuckLakeSnapshotReader {
|
|
|
444
445
|
return {
|
|
445
446
|
datasetId: tableRef.datasetId,
|
|
446
447
|
versionId: toVersionId(row.snapshotId),
|
|
448
|
+
inputs: row.metadata?.inputs,
|
|
447
449
|
...(row.metadata?.rowCount !== undefined ? { rowCount: row.metadata.rowCount } : {}),
|
|
448
450
|
...(row.metadata?.validatedPrimaryKeyColumns !== undefined
|
|
449
451
|
? { validatedPrimaryKeyColumns: row.metadata.validatedPrimaryKeyColumns }
|