lakeql 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +96 -17
- package/dist/bin.js +8822 -0
- package/dist/{chunk-VRJ72DGA.js → chunk-D3A4VN3U.js} +246 -1393
- package/dist/chunk-MUTXXFOW.js +1379 -0
- package/dist/cloudflare.d.ts +1 -1
- package/dist/cloudflare.js +3 -2
- package/dist/index.d.ts +41 -8
- package/dist/index.js +2 -1
- package/dist/node.d.ts +1 -1
- package/dist/node.js +3 -2
- package/package.json +20 -17
|
@@ -4,9 +4,36 @@ import { gridDisk, latLngToCell, cellToParent, isValidCell } from 'h3-js';
|
|
|
4
4
|
import { parquetMetadataAsync, parquetReadObjects } from 'hyparquet';
|
|
5
5
|
import { parquetWriteBuffer } from 'hyparquet-writer';
|
|
6
6
|
|
|
7
|
+
var __create = Object.create;
|
|
8
|
+
var __defProp = Object.defineProperty;
|
|
9
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
10
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
11
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
12
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
13
|
+
var __commonJS = (cb, mod2) => function __require() {
|
|
14
|
+
return mod2 || (0, cb[__getOwnPropNames(cb)[0]])((mod2 = { exports: {} }).exports, mod2), mod2.exports;
|
|
15
|
+
};
|
|
16
|
+
var __copyProps = (to, from, except, desc) => {
|
|
17
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
18
|
+
for (let key of __getOwnPropNames(from))
|
|
19
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
20
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
21
|
+
}
|
|
22
|
+
return to;
|
|
23
|
+
};
|
|
24
|
+
var __toESM = (mod2, isNodeMode, target) => (target = mod2 != null ? __create(__getProtoOf(mod2)) : {}, __copyProps(
|
|
25
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
26
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
27
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
28
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
29
|
+
isNodeMode || !mod2 || !mod2.__esModule ? __defProp(target, "default", { value: mod2, enumerable: true }) : target,
|
|
30
|
+
mod2
|
|
31
|
+
));
|
|
32
|
+
|
|
7
33
|
// ../core/dist/errors.js
|
|
8
34
|
var ERROR_CODES = [
|
|
9
35
|
"LAQL_PARSE_ERROR",
|
|
36
|
+
"LAQL_SQL_UNSUPPORTED",
|
|
10
37
|
"LAQL_TYPE_ERROR",
|
|
11
38
|
"LAQL_UNKNOWN_TABLE",
|
|
12
39
|
"LAQL_UNKNOWN_COLUMN",
|
|
@@ -71,6 +98,14 @@ function evaluate(expr, row) {
|
|
|
71
98
|
return likeMatch(evaluate(expr.target, row), expr.pattern, expr.caseInsensitive);
|
|
72
99
|
case "call":
|
|
73
100
|
return callFunction(expr.fn, expr.args.map((arg) => evaluate(arg, row)));
|
|
101
|
+
case "arithmetic":
|
|
102
|
+
return arithmetic(expr.op, evaluate(expr.left, row), evaluate(expr.right, row));
|
|
103
|
+
case "case":
|
|
104
|
+
for (const branch of expr.whens) {
|
|
105
|
+
if (toSqlBoolean(evaluate(branch.when, row)) === true)
|
|
106
|
+
return evaluate(branch.value, row);
|
|
107
|
+
}
|
|
108
|
+
return expr.else === void 0 ? null : evaluate(expr.else, row);
|
|
74
109
|
}
|
|
75
110
|
}
|
|
76
111
|
function matches(expr, row) {
|
|
@@ -144,6 +179,28 @@ function compare(op, left, right) {
|
|
|
144
179
|
return order >= 0;
|
|
145
180
|
}
|
|
146
181
|
}
|
|
182
|
+
function arithmetic(op, left, right) {
|
|
183
|
+
if (left === null || right === null)
|
|
184
|
+
return null;
|
|
185
|
+
if (typeof left !== "number" || typeof right !== "number") {
|
|
186
|
+
throw new LaQLError("LAQL_TYPE_ERROR", "Arithmetic expressions require numeric values", {
|
|
187
|
+
leftType: typeof left,
|
|
188
|
+
rightType: typeof right
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
switch (op) {
|
|
192
|
+
case "add":
|
|
193
|
+
return left + right;
|
|
194
|
+
case "sub":
|
|
195
|
+
return left - right;
|
|
196
|
+
case "mul":
|
|
197
|
+
return left * right;
|
|
198
|
+
case "div":
|
|
199
|
+
return left / right;
|
|
200
|
+
case "mod":
|
|
201
|
+
return left % right;
|
|
202
|
+
}
|
|
203
|
+
}
|
|
147
204
|
function isNumberLike(value) {
|
|
148
205
|
return typeof value === "number" || typeof value === "bigint";
|
|
149
206
|
}
|
|
@@ -865,6 +922,21 @@ function ilike(column, pattern) {
|
|
|
865
922
|
function fn(name, ...args) {
|
|
866
923
|
return { kind: "call", fn: name, args: args.map(toValue) };
|
|
867
924
|
}
|
|
925
|
+
function add(left, right) {
|
|
926
|
+
return { kind: "arithmetic", op: "add", left: toValue(left), right: toValue(right) };
|
|
927
|
+
}
|
|
928
|
+
function sub(left, right) {
|
|
929
|
+
return { kind: "arithmetic", op: "sub", left: toValue(left), right: toValue(right) };
|
|
930
|
+
}
|
|
931
|
+
function mul(left, right) {
|
|
932
|
+
return { kind: "arithmetic", op: "mul", left: toValue(left), right: toValue(right) };
|
|
933
|
+
}
|
|
934
|
+
function div(left, right) {
|
|
935
|
+
return { kind: "arithmetic", op: "div", left: toValue(left), right: toValue(right) };
|
|
936
|
+
}
|
|
937
|
+
function mod(left, right) {
|
|
938
|
+
return { kind: "arithmetic", op: "mod", left: toValue(left), right: toValue(right) };
|
|
939
|
+
}
|
|
868
940
|
|
|
869
941
|
// ../core/dist/manifest.js
|
|
870
942
|
function createTaskManifest(input) {
|
|
@@ -1661,7 +1733,15 @@ async function collectRightRows(rows, maxRightRows) {
|
|
|
1661
1733
|
return out;
|
|
1662
1734
|
}
|
|
1663
1735
|
function validateJoinOptions(options, strategy) {
|
|
1664
|
-
|
|
1736
|
+
const leftKeys = normalizeJoinKeys(options.leftKey, "leftKey", strategy);
|
|
1737
|
+
const rightKeys = normalizeJoinKeys(options.rightKey, "rightKey", strategy);
|
|
1738
|
+
if (leftKeys.length !== rightKeys.length) {
|
|
1739
|
+
throw new LaQLError("LAQL_TYPE_ERROR", `${strategy} join key counts must match`, {
|
|
1740
|
+
leftKey: options.leftKey,
|
|
1741
|
+
rightKey: options.rightKey
|
|
1742
|
+
});
|
|
1743
|
+
}
|
|
1744
|
+
if (leftKeys.length === 0) {
|
|
1665
1745
|
throw new LaQLError("LAQL_TYPE_ERROR", `${strategy} join requires leftKey and rightKey`);
|
|
1666
1746
|
}
|
|
1667
1747
|
if (!Number.isInteger(options.maxRightRows) || options.maxRightRows < 1) {
|
|
@@ -1673,10 +1753,24 @@ function validateJoinOptions(options, strategy) {
|
|
|
1673
1753
|
});
|
|
1674
1754
|
}
|
|
1675
1755
|
}
|
|
1756
|
+
function normalizeJoinKeys(key, label, strategy) {
|
|
1757
|
+
const keys = Array.isArray(key) ? key : [key];
|
|
1758
|
+
if (keys.some((column) => typeof column !== "string" || column.length === 0)) {
|
|
1759
|
+
throw new LaQLError("LAQL_TYPE_ERROR", `${strategy} join ${label} must contain column names`, {
|
|
1760
|
+
[label]: key
|
|
1761
|
+
});
|
|
1762
|
+
}
|
|
1763
|
+
return keys;
|
|
1764
|
+
}
|
|
1676
1765
|
function joinKey(row, column) {
|
|
1677
1766
|
return stableStringify(joinValue(row, column));
|
|
1678
1767
|
}
|
|
1679
1768
|
function joinValue(row, column) {
|
|
1769
|
+
if (Array.isArray(column))
|
|
1770
|
+
return column.map((key) => scalarJoinValue(row, key));
|
|
1771
|
+
return scalarJoinValue(row, column);
|
|
1772
|
+
}
|
|
1773
|
+
function scalarJoinValue(row, column) {
|
|
1680
1774
|
if (!(column in row)) {
|
|
1681
1775
|
throw new LaQLError("LAQL_UNKNOWN_COLUMN", `Unknown join key ${column}`, { column });
|
|
1682
1776
|
}
|
|
@@ -1691,8 +1785,10 @@ function joinValue(row, column) {
|
|
|
1691
1785
|
function mergeRows(left, right, options) {
|
|
1692
1786
|
const out = { ...left };
|
|
1693
1787
|
const prefix = options.rightPrefix ?? "right.";
|
|
1788
|
+
const leftKeys = normalizeJoinKeys(options.leftKey, "leftKey", "Merge");
|
|
1789
|
+
const rightKeys = normalizeJoinKeys(options.rightKey, "rightKey", "Merge");
|
|
1694
1790
|
for (const [key, value] of Object.entries(right)) {
|
|
1695
|
-
if (key
|
|
1791
|
+
if (rightKeys.includes(key) && leftKeys.includes(key))
|
|
1696
1792
|
continue;
|
|
1697
1793
|
const outKey = key in out ? `${prefix}${key}` : key;
|
|
1698
1794
|
out[outKey] = value;
|
|
@@ -1907,6 +2003,18 @@ function collectColumns(expr, columns) {
|
|
|
1907
2003
|
for (const arg of expr.args)
|
|
1908
2004
|
collectColumns(arg, columns);
|
|
1909
2005
|
return;
|
|
2006
|
+
case "arithmetic":
|
|
2007
|
+
collectColumns(expr.left, columns);
|
|
2008
|
+
collectColumns(expr.right, columns);
|
|
2009
|
+
return;
|
|
2010
|
+
case "case":
|
|
2011
|
+
for (const branch of expr.whens) {
|
|
2012
|
+
collectColumns(branch.when, columns);
|
|
2013
|
+
collectColumns(branch.value, columns);
|
|
2014
|
+
}
|
|
2015
|
+
if (expr.else !== void 0)
|
|
2016
|
+
collectColumns(expr.else, columns);
|
|
2017
|
+
return;
|
|
1910
2018
|
}
|
|
1911
2019
|
}
|
|
1912
2020
|
|
|
@@ -1982,6 +2090,8 @@ function fileMayMatch(file, expr) {
|
|
|
1982
2090
|
case "column":
|
|
1983
2091
|
case "null-check":
|
|
1984
2092
|
case "like":
|
|
2093
|
+
case "arithmetic":
|
|
2094
|
+
case "case":
|
|
1985
2095
|
case "call":
|
|
1986
2096
|
return callMayMatch(file, expr);
|
|
1987
2097
|
case "not":
|
|
@@ -2239,9 +2349,15 @@ var QueryBuilder = class _QueryBuilder {
|
|
|
2239
2349
|
select(columns) {
|
|
2240
2350
|
return new _QueryBuilder(this.lake, { ...this.init, select: columns });
|
|
2241
2351
|
}
|
|
2352
|
+
project(projections) {
|
|
2353
|
+
return new _QueryBuilder(this.lake, { ...this.init, projections });
|
|
2354
|
+
}
|
|
2242
2355
|
where(expr) {
|
|
2243
2356
|
return new _QueryBuilder(this.lake, { ...this.init, where: expr });
|
|
2244
2357
|
}
|
|
2358
|
+
distinct(enabled = true) {
|
|
2359
|
+
return new _QueryBuilder(this.lake, { ...this.init, distinct: enabled });
|
|
2360
|
+
}
|
|
2245
2361
|
orderBy(terms) {
|
|
2246
2362
|
return new _QueryBuilder(this.lake, { ...this.init, orderBy: normalizeOrderBy(terms) });
|
|
2247
2363
|
}
|
|
@@ -2385,9 +2501,10 @@ var QueryResult = class {
|
|
|
2385
2501
|
const startedAt = config.now();
|
|
2386
2502
|
let offsetSkipped = 0;
|
|
2387
2503
|
let returned = 0;
|
|
2504
|
+
const distinct = config.distinct === true ? /* @__PURE__ */ new Set() : void 0;
|
|
2388
2505
|
const { planned: paths, skipped: skippedFiles } = await this.planObjects();
|
|
2389
2506
|
stats.filesSkipped = skippedFiles;
|
|
2390
|
-
const columns = projectedReadColumns(config.select, config.where);
|
|
2507
|
+
const columns = projectedReadColumns(config.select, config.where, void 0, config.projections);
|
|
2391
2508
|
for (const object of paths) {
|
|
2392
2509
|
stats.filesPlanned += 1;
|
|
2393
2510
|
stats.filesRead += 1;
|
|
@@ -2416,13 +2533,17 @@ var QueryResult = class {
|
|
|
2416
2533
|
if (!matches(config.where, row))
|
|
2417
2534
|
continue;
|
|
2418
2535
|
stats.rowsMatched += 1;
|
|
2536
|
+
const projected = project(row, config.select, config.projections);
|
|
2537
|
+
if (distinct !== void 0 && !addDistinctRow(distinct, projected, config.budget)) {
|
|
2538
|
+
continue;
|
|
2539
|
+
}
|
|
2419
2540
|
if (offsetSkipped < (config.offset ?? 0)) {
|
|
2420
2541
|
offsetSkipped += 1;
|
|
2421
2542
|
continue;
|
|
2422
2543
|
}
|
|
2423
2544
|
if (config.limit !== void 0 && returned >= config.limit)
|
|
2424
2545
|
break;
|
|
2425
|
-
out.push(
|
|
2546
|
+
out.push(projected);
|
|
2426
2547
|
returned += 1;
|
|
2427
2548
|
stats.rowsReturned += 1;
|
|
2428
2549
|
enforceBudget(config.budget, stats, config.now, startedAt);
|
|
@@ -2451,6 +2572,9 @@ var QueryResult = class {
|
|
|
2451
2572
|
if (config.limit === void 0) {
|
|
2452
2573
|
throw new LaQLError("LAQL_TYPE_ERROR", "topKWithState requires limit");
|
|
2453
2574
|
}
|
|
2575
|
+
if (config.distinct === true) {
|
|
2576
|
+
throw new LaQLError("LAQL_TYPE_ERROR", "topKWithState does not support distinct queries");
|
|
2577
|
+
}
|
|
2454
2578
|
const orderBy = normalizeOrderBy(config.orderBy);
|
|
2455
2579
|
const topK = (config.offset ?? 0) + config.limit;
|
|
2456
2580
|
const matched = await topKRowsFromState(orderBy, config.offset ?? 0, config.limit, options);
|
|
@@ -2461,7 +2585,7 @@ var QueryResult = class {
|
|
|
2461
2585
|
matched.sort((left, right) => compareRows(left, right, orderBy));
|
|
2462
2586
|
const start = config.offset ?? 0;
|
|
2463
2587
|
const end = start + config.limit;
|
|
2464
|
-
const rows = matched.slice(start, end).map((row) => project(row, config.select));
|
|
2588
|
+
const rows = matched.slice(start, end).map((row) => project(row, config.select, config.projections));
|
|
2465
2589
|
for (const _row of rows) {
|
|
2466
2590
|
this.stats.rowsReturned += 1;
|
|
2467
2591
|
enforceBudget(config.budget, this.stats, config.now, startedAt);
|
|
@@ -2480,6 +2604,9 @@ var QueryResult = class {
|
|
|
2480
2604
|
if (config.orderBy === void 0) {
|
|
2481
2605
|
throw new LaQLError("LAQL_TYPE_ERROR", "sortWithState requires orderBy");
|
|
2482
2606
|
}
|
|
2607
|
+
if (config.distinct === true) {
|
|
2608
|
+
throw new LaQLError("LAQL_TYPE_ERROR", "sortWithState does not support distinct queries");
|
|
2609
|
+
}
|
|
2483
2610
|
const orderBy = normalizeOrderBy(config.orderBy);
|
|
2484
2611
|
const runs = await sortRunsFromState(orderBy, options);
|
|
2485
2612
|
const startedAt = config.now();
|
|
@@ -2487,7 +2614,7 @@ var QueryResult = class {
|
|
|
2487
2614
|
const matched = mergeSortRuns(runs, orderBy);
|
|
2488
2615
|
const start = config.offset ?? 0;
|
|
2489
2616
|
const end = config.limit === void 0 ? matched.length : start + config.limit;
|
|
2490
|
-
const rows = matched.slice(start, end).map((row) => project(row, config.select));
|
|
2617
|
+
const rows = matched.slice(start, end).map((row) => project(row, config.select, config.projections));
|
|
2491
2618
|
for (const _row of rows) {
|
|
2492
2619
|
this.stats.rowsReturned += 1;
|
|
2493
2620
|
enforceBudget(config.budget, this.stats, config.now, startedAt);
|
|
@@ -2599,7 +2726,7 @@ var QueryResult = class {
|
|
|
2599
2726
|
group.add(row);
|
|
2600
2727
|
enforceOperatorMemoryBudget(this.config.budget, estimateAggregateOperatorMemoryBytes(groupColumns, spec, groups));
|
|
2601
2728
|
}
|
|
2602
|
-
const rows = [...groups.values()].map((group) => group.finish());
|
|
2729
|
+
const rows = applyAggregateResultOptions([...groups.values()].map((group) => group.finish()), options);
|
|
2603
2730
|
for (const _row of rows) {
|
|
2604
2731
|
this.stats.rowsReturned += 1;
|
|
2605
2732
|
enforceBudget(this.config.budget, this.stats, this.config.now, startedAt);
|
|
@@ -2621,16 +2748,27 @@ var QueryResult = class {
|
|
|
2621
2748
|
const config = this.config;
|
|
2622
2749
|
const { stats } = this;
|
|
2623
2750
|
const matched = [];
|
|
2624
|
-
const topK = config.limit === void 0 ? void 0 : (config.offset ?? 0) + config.limit;
|
|
2751
|
+
const topK = config.distinct === true || config.limit === void 0 ? void 0 : (config.offset ?? 0) + config.limit;
|
|
2625
2752
|
const startedAt = config.now();
|
|
2626
2753
|
await this.collectOrderedMatches(matched, topK, startedAt);
|
|
2627
2754
|
matched.sort((left, right) => compareRows(left, right, config.orderBy ?? []));
|
|
2628
|
-
const start = config.offset ?? 0;
|
|
2629
|
-
const end = config.limit === void 0 ? matched.length : start + config.limit;
|
|
2630
2755
|
const batchSize = config.batchSize ?? 4096;
|
|
2631
2756
|
let batch = [];
|
|
2632
|
-
|
|
2633
|
-
|
|
2757
|
+
let offsetSkipped = 0;
|
|
2758
|
+
let returned = 0;
|
|
2759
|
+
const distinct = config.distinct === true ? /* @__PURE__ */ new Set() : void 0;
|
|
2760
|
+
for (const row of matched) {
|
|
2761
|
+
const projected = project(row, config.select, config.projections);
|
|
2762
|
+
if (distinct !== void 0 && !addDistinctRow(distinct, projected, config.budget))
|
|
2763
|
+
continue;
|
|
2764
|
+
if (offsetSkipped < (config.offset ?? 0)) {
|
|
2765
|
+
offsetSkipped += 1;
|
|
2766
|
+
continue;
|
|
2767
|
+
}
|
|
2768
|
+
if (config.limit !== void 0 && returned >= config.limit)
|
|
2769
|
+
break;
|
|
2770
|
+
batch.push(projected);
|
|
2771
|
+
returned += 1;
|
|
2634
2772
|
stats.rowsReturned += 1;
|
|
2635
2773
|
enforceBudget(config.budget, stats, config.now, startedAt);
|
|
2636
2774
|
if (batch.length >= batchSize) {
|
|
@@ -2649,7 +2787,7 @@ var QueryResult = class {
|
|
|
2649
2787
|
const { planned: paths, skipped: skippedFiles } = await this.planObjects();
|
|
2650
2788
|
stats.filesSkipped = skippedFiles;
|
|
2651
2789
|
const orderBy = config.orderBy ?? [];
|
|
2652
|
-
const columns = projectedReadColumns(config.select, config.where, orderBy);
|
|
2790
|
+
const columns = projectedReadColumns(config.select, config.where, orderBy, config.projections);
|
|
2653
2791
|
for (const object of paths) {
|
|
2654
2792
|
stats.filesPlanned += 1;
|
|
2655
2793
|
stats.filesRead += 1;
|
|
@@ -2691,7 +2829,7 @@ var QueryResult = class {
|
|
|
2691
2829
|
const { stats } = this;
|
|
2692
2830
|
const { planned: paths, skipped: skippedFiles } = await this.planObjects();
|
|
2693
2831
|
stats.filesSkipped = skippedFiles;
|
|
2694
|
-
const columns = projectedReadColumns(config.select, config.where, orderBy);
|
|
2832
|
+
const columns = projectedReadColumns(config.select, config.where, orderBy, config.projections);
|
|
2695
2833
|
const runCapacity = config.budget.maxBufferedRows ?? Number.POSITIVE_INFINITY;
|
|
2696
2834
|
const currentRun = [];
|
|
2697
2835
|
for (const object of paths) {
|
|
@@ -2736,7 +2874,7 @@ var QueryResult = class {
|
|
|
2736
2874
|
async explain() {
|
|
2737
2875
|
const { planned, skipped } = await this.planObjects();
|
|
2738
2876
|
const tasks = await this.tasksFromObjects(planned);
|
|
2739
|
-
const projectedColumns = projectedReadColumns(this.config.select, this.config.where) ?? [];
|
|
2877
|
+
const projectedColumns = projectedReadColumns(this.config.select, this.config.where, void 0, this.config.projections) ?? [];
|
|
2740
2878
|
const json = {
|
|
2741
2879
|
queryId: this.stats.queryId,
|
|
2742
2880
|
filesPlanned: tasks.length,
|
|
@@ -2759,7 +2897,7 @@ var QueryResult = class {
|
|
|
2759
2897
|
}
|
|
2760
2898
|
async tasksFromObjects(objects) {
|
|
2761
2899
|
const config = this.config;
|
|
2762
|
-
const projectedColumns = projectedReadColumns(config.select, config.where, config.orderBy);
|
|
2900
|
+
const projectedColumns = projectedReadColumns(config.select, config.where, config.orderBy, config.projections);
|
|
2763
2901
|
const tasks = [];
|
|
2764
2902
|
for (const object of objects) {
|
|
2765
2903
|
const partitionValues = config.hive ? parseHivePartitions(object.path) : {};
|
|
@@ -2973,7 +3111,9 @@ var CountState = class {
|
|
|
2973
3111
|
constructor(count = 0) {
|
|
2974
3112
|
this.count = count;
|
|
2975
3113
|
}
|
|
2976
|
-
add(
|
|
3114
|
+
add(value) {
|
|
3115
|
+
if (value === null || value === void 0)
|
|
3116
|
+
return;
|
|
2977
3117
|
this.count += 1;
|
|
2978
3118
|
}
|
|
2979
3119
|
finish() {
|
|
@@ -3409,8 +3549,11 @@ function snapshotValue(value) {
|
|
|
3409
3549
|
function aggregateValue(row, alias, aggregate) {
|
|
3410
3550
|
if (!aggregate)
|
|
3411
3551
|
throw new LaQLError("LAQL_VALIDATION_ERROR", `Missing aggregate ${alias}`);
|
|
3412
|
-
if (aggregate.op === "count" && aggregate.column === void 0)
|
|
3552
|
+
if (aggregate.op === "count" && aggregate.column === void 0 && aggregate.expr === void 0) {
|
|
3413
3553
|
return true;
|
|
3554
|
+
}
|
|
3555
|
+
if (aggregate.expr !== void 0)
|
|
3556
|
+
return evaluate(aggregate.expr, row);
|
|
3414
3557
|
if (aggregate.column === void 0) {
|
|
3415
3558
|
throw new LaQLError("LAQL_TYPE_ERROR", `${aggregate.op} requires a column`, { aggregate });
|
|
3416
3559
|
}
|
|
@@ -3432,9 +3575,32 @@ function validateAggregateRequest(groupColumns, spec, options) {
|
|
|
3432
3575
|
if (options.maxGroups !== void 0 && (!Number.isInteger(options.maxGroups) || options.maxGroups < 1)) {
|
|
3433
3576
|
throw new LaQLError("LAQL_TYPE_ERROR", "maxGroups must be a positive integer");
|
|
3434
3577
|
}
|
|
3578
|
+
if (options.orderBy !== void 0)
|
|
3579
|
+
normalizeOrderBy(options.orderBy);
|
|
3580
|
+
if (options.limit !== void 0 && (!Number.isInteger(options.limit) || options.limit < 0)) {
|
|
3581
|
+
throw new LaQLError("LAQL_TYPE_ERROR", "aggregate limit must be a non-negative integer");
|
|
3582
|
+
}
|
|
3583
|
+
if (options.offset !== void 0 && (!Number.isInteger(options.offset) || options.offset < 0)) {
|
|
3584
|
+
throw new LaQLError("LAQL_TYPE_ERROR", "aggregate offset must be a non-negative integer");
|
|
3585
|
+
}
|
|
3435
3586
|
for (const aggregate of Object.values(spec))
|
|
3436
3587
|
validateAggregateExpr(aggregate);
|
|
3437
3588
|
}
|
|
3589
|
+
function applyAggregateResultOptions(rows, options) {
|
|
3590
|
+
let out = rows;
|
|
3591
|
+
if (options.having !== void 0)
|
|
3592
|
+
out = out.filter((row) => matches(options.having, row));
|
|
3593
|
+
if (options.orderBy !== void 0) {
|
|
3594
|
+
const orderBy = normalizeOrderBy(options.orderBy);
|
|
3595
|
+
out = [...out].sort((left, right) => compareRows(left, right, orderBy));
|
|
3596
|
+
}
|
|
3597
|
+
const offset = options.offset ?? 0;
|
|
3598
|
+
if (options.limit !== void 0)
|
|
3599
|
+
return out.slice(offset, offset + options.limit);
|
|
3600
|
+
if (offset > 0)
|
|
3601
|
+
return out.slice(offset);
|
|
3602
|
+
return out;
|
|
3603
|
+
}
|
|
3438
3604
|
function validateAggregateExpr(aggregate) {
|
|
3439
3605
|
const ops = [
|
|
3440
3606
|
"count",
|
|
@@ -3451,8 +3617,15 @@ function validateAggregateExpr(aggregate) {
|
|
|
3451
3617
|
if (!ops.includes(aggregate.op)) {
|
|
3452
3618
|
throw new LaQLError("LAQL_TYPE_ERROR", `Unsupported aggregate ${aggregate.op}`, { aggregate });
|
|
3453
3619
|
}
|
|
3454
|
-
if (aggregate.
|
|
3455
|
-
throw new LaQLError("LAQL_TYPE_ERROR",
|
|
3620
|
+
if (aggregate.column !== void 0 && aggregate.expr !== void 0) {
|
|
3621
|
+
throw new LaQLError("LAQL_TYPE_ERROR", "aggregate cannot specify both column and expr", {
|
|
3622
|
+
aggregate
|
|
3623
|
+
});
|
|
3624
|
+
}
|
|
3625
|
+
if (aggregate.op !== "count" && aggregate.column === void 0 && aggregate.expr === void 0) {
|
|
3626
|
+
throw new LaQLError("LAQL_TYPE_ERROR", `${aggregate.op} requires a column or expression`, {
|
|
3627
|
+
aggregate
|
|
3628
|
+
});
|
|
3456
3629
|
}
|
|
3457
3630
|
}
|
|
3458
3631
|
function throwAggregateType(op) {
|
|
@@ -3474,6 +3647,11 @@ function parseJsonQuery(input) {
|
|
|
3474
3647
|
}
|
|
3475
3648
|
if (input.where !== void 0)
|
|
3476
3649
|
init.where = parseJsonExpr(input.where);
|
|
3650
|
+
if (input.distinct !== void 0) {
|
|
3651
|
+
if (typeof input.distinct !== "boolean")
|
|
3652
|
+
throwParse("JSON query distinct must be a boolean");
|
|
3653
|
+
init.distinct = input.distinct;
|
|
3654
|
+
}
|
|
3477
3655
|
if (input.orderBy !== void 0) {
|
|
3478
3656
|
if (!Array.isArray(input.orderBy))
|
|
3479
3657
|
throwParse("JSON query orderBy must be an array");
|
|
@@ -3615,6 +3793,9 @@ function validateQueryInit(init) {
|
|
|
3615
3793
|
}
|
|
3616
3794
|
if (init.orderBy !== void 0)
|
|
3617
3795
|
normalizeOrderBy(init.orderBy);
|
|
3796
|
+
if (init.distinct !== void 0 && typeof init.distinct !== "boolean") {
|
|
3797
|
+
throw new LaQLError("LAQL_TYPE_ERROR", "distinct must be a boolean");
|
|
3798
|
+
}
|
|
3618
3799
|
}
|
|
3619
3800
|
function applyQueryPolicy(init, policy) {
|
|
3620
3801
|
const allowedColumns = policy.allowedColumns === void 0 ? void 0 : normalizeAllowedColumns(policy.allowedColumns);
|
|
@@ -3641,8 +3822,12 @@ function cloneBookmarkQuery(init) {
|
|
|
3641
3822
|
const query = { source: init.source };
|
|
3642
3823
|
if (init.select !== void 0)
|
|
3643
3824
|
query.select = [...init.select];
|
|
3825
|
+
if (init.projections !== void 0)
|
|
3826
|
+
query.projections = init.projections;
|
|
3644
3827
|
if (init.where !== void 0)
|
|
3645
3828
|
query.where = init.where;
|
|
3829
|
+
if (init.distinct !== void 0)
|
|
3830
|
+
query.distinct = init.distinct;
|
|
3646
3831
|
if (init.orderBy !== void 0) {
|
|
3647
3832
|
query.orderBy = init.orderBy.map((term) => {
|
|
3648
3833
|
const queryTerm = { column: term.column };
|
|
@@ -3700,6 +3885,8 @@ function validatePolicyColumns(init, effectiveWhere, allowedColumns) {
|
|
|
3700
3885
|
requested.add(column);
|
|
3701
3886
|
for (const term of init.orderBy ?? [])
|
|
3702
3887
|
requested.add(term.column);
|
|
3888
|
+
for (const expr of Object.values(init.projections ?? {}))
|
|
3889
|
+
collectExprColumns(expr, requested);
|
|
3703
3890
|
collectExprColumns(effectiveWhere, requested);
|
|
3704
3891
|
for (const column of requested) {
|
|
3705
3892
|
if (!allowed.has(column)) {
|
|
@@ -3760,12 +3947,14 @@ function globRegex(pattern) {
|
|
|
3760
3947
|
}
|
|
3761
3948
|
return new RegExp(`${source}$`, "u");
|
|
3762
3949
|
}
|
|
3763
|
-
function projectedReadColumns(select, where, orderBy = void 0) {
|
|
3950
|
+
function projectedReadColumns(select, where, orderBy = void 0, projections = void 0) {
|
|
3764
3951
|
const columns = /* @__PURE__ */ new Set();
|
|
3765
3952
|
for (const column of select ?? [])
|
|
3766
3953
|
columns.add(column);
|
|
3767
3954
|
for (const term of orderBy ?? [])
|
|
3768
3955
|
columns.add(term.column);
|
|
3956
|
+
for (const expr of Object.values(projections ?? {}))
|
|
3957
|
+
collectExprColumns(expr, columns);
|
|
3769
3958
|
collectExprColumns(where, columns);
|
|
3770
3959
|
return columns.size === 0 ? void 0 : [...columns].sort();
|
|
3771
3960
|
}
|
|
@@ -3776,6 +3965,8 @@ function aggregateReadColumns(groupColumns, spec, where) {
|
|
|
3776
3965
|
for (const aggregate of Object.values(spec)) {
|
|
3777
3966
|
if (aggregate.column !== void 0)
|
|
3778
3967
|
columns.add(aggregate.column);
|
|
3968
|
+
if (aggregate.expr !== void 0)
|
|
3969
|
+
collectExprColumns(aggregate.expr, columns);
|
|
3779
3970
|
}
|
|
3780
3971
|
collectExprColumns(where, columns);
|
|
3781
3972
|
return columns.size === 0 ? void 0 : [...columns].sort();
|
|
@@ -3820,20 +4011,42 @@ function collectExprColumns(expr, columns) {
|
|
|
3820
4011
|
for (const arg of expr.args)
|
|
3821
4012
|
collectExprColumns(arg, columns);
|
|
3822
4013
|
return;
|
|
4014
|
+
case "arithmetic":
|
|
4015
|
+
collectExprColumns(expr.left, columns);
|
|
4016
|
+
collectExprColumns(expr.right, columns);
|
|
4017
|
+
return;
|
|
4018
|
+
case "case":
|
|
4019
|
+
for (const branch of expr.whens) {
|
|
4020
|
+
collectExprColumns(branch.when, columns);
|
|
4021
|
+
collectExprColumns(branch.value, columns);
|
|
4022
|
+
}
|
|
4023
|
+
collectExprColumns(expr.else, columns);
|
|
4024
|
+
return;
|
|
3823
4025
|
}
|
|
3824
4026
|
}
|
|
3825
|
-
function project(row, select) {
|
|
3826
|
-
if (!select)
|
|
4027
|
+
function project(row, select, projections) {
|
|
4028
|
+
if (!select && !projections)
|
|
3827
4029
|
return row;
|
|
3828
4030
|
const out = {};
|
|
3829
|
-
for (const column of select) {
|
|
4031
|
+
for (const column of select ?? []) {
|
|
3830
4032
|
if (!(column in row)) {
|
|
3831
4033
|
throw new LaQLError("LAQL_UNKNOWN_COLUMN", `Unknown column ${column}`, { column });
|
|
3832
4034
|
}
|
|
3833
4035
|
out[column] = row[column];
|
|
3834
4036
|
}
|
|
4037
|
+
for (const [alias, expr] of Object.entries(projections ?? {}))
|
|
4038
|
+
out[alias] = evaluate(expr, row);
|
|
3835
4039
|
return out;
|
|
3836
4040
|
}
|
|
4041
|
+
function addDistinctRow(seen, row, budget) {
|
|
4042
|
+
const key = stableStringify(jsonSafeValue(row));
|
|
4043
|
+
if (seen.has(key))
|
|
4044
|
+
return false;
|
|
4045
|
+
seen.add(key);
|
|
4046
|
+
enforceBufferedRowsBudget(budget, seen.size);
|
|
4047
|
+
enforceOperatorMemoryBudget(budget, estimateOperatorMemoryBytes([...seen]));
|
|
4048
|
+
return true;
|
|
4049
|
+
}
|
|
3837
4050
|
function normalizeOrderBy(terms) {
|
|
3838
4051
|
if (!Array.isArray(terms) || terms.length === 0) {
|
|
3839
4052
|
throw new LaQLError("LAQL_TYPE_ERROR", "orderBy must contain at least one term");
|
|
@@ -3983,6 +4196,8 @@ function partitionEval(expr, partitions) {
|
|
|
3983
4196
|
case "null-check":
|
|
3984
4197
|
case "like":
|
|
3985
4198
|
case "call":
|
|
4199
|
+
case "arithmetic":
|
|
4200
|
+
case "case":
|
|
3986
4201
|
return expressionIsPartitionOnly(expr, partitions) ? matches(expr, partitions) : "unknown";
|
|
3987
4202
|
case "not": {
|
|
3988
4203
|
const value = partitionEval(expr.operand, partitions);
|
|
@@ -4323,1296 +4538,6 @@ var ReadSemaphore = class {
|
|
|
4323
4538
|
this.queue.shift()?.();
|
|
4324
4539
|
}
|
|
4325
4540
|
};
|
|
4326
|
-
|
|
4327
|
-
// ../iceberg/dist/index.js
|
|
4328
|
-
var PACKAGE = "@laql/iceberg";
|
|
4329
|
-
var IcebergTable = class {
|
|
4330
|
-
store;
|
|
4331
|
-
metadataPath;
|
|
4332
|
-
metadata;
|
|
4333
|
-
constructor(store, metadataPath, metadata) {
|
|
4334
|
-
this.store = store;
|
|
4335
|
-
this.metadataPath = metadataPath;
|
|
4336
|
-
this.metadata = metadata;
|
|
4337
|
-
}
|
|
4338
|
-
snapshot(options = {}) {
|
|
4339
|
-
if (options.snapshotId !== void 0)
|
|
4340
|
-
return this.snapshotById(options.snapshotId);
|
|
4341
|
-
if (options.ref !== void 0) {
|
|
4342
|
-
const ref = this.metadata.refs?.[options.ref];
|
|
4343
|
-
if (!ref) {
|
|
4344
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", `Unknown Iceberg ref ${options.ref}`, {
|
|
4345
|
-
ref: options.ref
|
|
4346
|
-
});
|
|
4347
|
-
}
|
|
4348
|
-
return this.snapshotById(ref["snapshot-id"]);
|
|
4349
|
-
}
|
|
4350
|
-
if (options.asOfTimestampMs !== void 0) {
|
|
4351
|
-
const snapshot = [...this.metadata.snapshots].filter((candidate) => candidate["timestamp-ms"] <= options.asOfTimestampMs).sort((a, b) => b["timestamp-ms"] - a["timestamp-ms"])[0];
|
|
4352
|
-
if (!snapshot) {
|
|
4353
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "No Iceberg snapshot at requested timestamp", {
|
|
4354
|
-
asOfTimestampMs: options.asOfTimestampMs
|
|
4355
|
-
});
|
|
4356
|
-
}
|
|
4357
|
-
return snapshot;
|
|
4358
|
-
}
|
|
4359
|
-
return this.snapshotById(this.metadata["current-snapshot-id"]);
|
|
4360
|
-
}
|
|
4361
|
-
schema(schemaId) {
|
|
4362
|
-
const schema = this.metadata.schemas.find((candidate) => candidate["schema-id"] === schemaId);
|
|
4363
|
-
if (!schema) {
|
|
4364
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", `Unknown Iceberg schema ${schemaId}`, { schemaId });
|
|
4365
|
-
}
|
|
4366
|
-
return schema.fields;
|
|
4367
|
-
}
|
|
4368
|
-
projectRow(row, options = {}) {
|
|
4369
|
-
const snapshot = this.snapshot(options);
|
|
4370
|
-
const fields = this.schema(snapshot["schema-id"]);
|
|
4371
|
-
projectedIds(fields, options.select);
|
|
4372
|
-
const selected = options.select === void 0 ? void 0 : new Set(options.select);
|
|
4373
|
-
const out = {};
|
|
4374
|
-
for (const field of fields) {
|
|
4375
|
-
if (selected !== void 0 && !selected.has(field.name))
|
|
4376
|
-
continue;
|
|
4377
|
-
const sourceName = field.name in row ? field.name : field.sourceId !== void 0 ? this.fieldNameById(field.sourceId) : field.name;
|
|
4378
|
-
out[field.name] = sourceName !== void 0 && sourceName in row ? row[sourceName] : null;
|
|
4379
|
-
}
|
|
4380
|
-
return out;
|
|
4381
|
-
}
|
|
4382
|
-
planFiles(options = {}) {
|
|
4383
|
-
const snapshot = this.snapshot(options);
|
|
4384
|
-
const fields = this.schema(snapshot["schema-id"]);
|
|
4385
|
-
const projectedFieldIds = projectedIds(fields, options.select);
|
|
4386
|
-
const readMode = options.readMode ?? "strict";
|
|
4387
|
-
const files = [];
|
|
4388
|
-
let manifestsSkipped = 0;
|
|
4389
|
-
let filesSkipped = 0;
|
|
4390
|
-
let deleteFilesPlanned = 0;
|
|
4391
|
-
let deleteFilesIgnored = 0;
|
|
4392
|
-
const manifests = snapshotManifests(snapshot);
|
|
4393
|
-
for (const manifest of manifests) {
|
|
4394
|
-
const manifestMayMatch = manifest.files.some((file) => partitionMayMatch2(options.where, file.partition ?? {}));
|
|
4395
|
-
if (!manifestMayMatch) {
|
|
4396
|
-
manifestsSkipped += 1;
|
|
4397
|
-
filesSkipped += manifest.files.length;
|
|
4398
|
-
continue;
|
|
4399
|
-
}
|
|
4400
|
-
for (const file of manifest.files) {
|
|
4401
|
-
if (!partitionMayMatch2(options.where, file.partition ?? {})) {
|
|
4402
|
-
filesSkipped += 1;
|
|
4403
|
-
continue;
|
|
4404
|
-
}
|
|
4405
|
-
const supportedDeleteFiles = supportedIcebergDeleteFiles(file.deleteFiles);
|
|
4406
|
-
const unsupportedDeleteFiles = unsupportedIcebergDeleteFiles(file.deleteFiles);
|
|
4407
|
-
if (unsupportedDeleteFiles.length > 0 && readMode === "strict") {
|
|
4408
|
-
throw new LaQLError("LAQL_UNSUPPORTED_DELETE_FILES", "Snapshot contains delete files unsupported by strict Iceberg planning", {
|
|
4409
|
-
path: file.path,
|
|
4410
|
-
deleteFiles: file.deleteFiles,
|
|
4411
|
-
supportedDeleteFiles,
|
|
4412
|
-
unsupportedDeleteFiles
|
|
4413
|
-
});
|
|
4414
|
-
}
|
|
4415
|
-
const planned = {
|
|
4416
|
-
path: file.path,
|
|
4417
|
-
sequenceNumber: file.sequenceNumber,
|
|
4418
|
-
partition: file.partition ?? {},
|
|
4419
|
-
recordCount: file.recordCount,
|
|
4420
|
-
projectedFieldIds,
|
|
4421
|
-
snapshotId: snapshot["snapshot-id"]
|
|
4422
|
-
};
|
|
4423
|
-
if (file.fileSizeInBytes !== void 0)
|
|
4424
|
-
planned.fileSizeInBytes = file.fileSizeInBytes;
|
|
4425
|
-
if ((readMode === "strict" || readMode === "ignore-unsupported-deletes") && supportedDeleteFiles.length > 0) {
|
|
4426
|
-
planned.deleteFiles = supportedDeleteFiles;
|
|
4427
|
-
deleteFilesPlanned += supportedDeleteFiles.length;
|
|
4428
|
-
}
|
|
4429
|
-
if (readMode === "ignore-unsupported-deletes") {
|
|
4430
|
-
deleteFilesIgnored += unsupportedDeleteFiles.length;
|
|
4431
|
-
} else if (readMode === "ignore-deletes") {
|
|
4432
|
-
deleteFilesIgnored += (file.deleteFiles ?? []).length;
|
|
4433
|
-
}
|
|
4434
|
-
files.push(planned);
|
|
4435
|
-
}
|
|
4436
|
-
}
|
|
4437
|
-
files.sort((a, b) => a.sequenceNumber - b.sequenceNumber || a.path.localeCompare(b.path));
|
|
4438
|
-
return {
|
|
4439
|
-
snapshotId: snapshot["snapshot-id"],
|
|
4440
|
-
schemaId: snapshot["schema-id"],
|
|
4441
|
-
manifestsRead: manifests.length - manifestsSkipped,
|
|
4442
|
-
manifestsSkipped,
|
|
4443
|
-
filesPlanned: files.length,
|
|
4444
|
-
filesSkipped,
|
|
4445
|
-
deleteFilesPlanned,
|
|
4446
|
-
deleteFilesIgnored,
|
|
4447
|
-
files
|
|
4448
|
-
};
|
|
4449
|
-
}
|
|
4450
|
-
async appendFiles(options) {
|
|
4451
|
-
if (options.files.length === 0) {
|
|
4452
|
-
throw new LaQLError("LAQL_VALIDATION_ERROR", "Iceberg append requires at least one file");
|
|
4453
|
-
}
|
|
4454
|
-
if (this.metadata["format-version"] !== 2) {
|
|
4455
|
-
throw new LaQLError("LAQL_VALIDATION_ERROR", "Iceberg append requires format-version 2 metadata", { formatVersion: this.metadata["format-version"] });
|
|
4456
|
-
}
|
|
4457
|
-
const currentSnapshot = this.snapshot();
|
|
4458
|
-
const nextSnapshotId = options.nextSnapshotId ?? randomSnapshotId(this.metadata.snapshots.map(snapshotIdOf));
|
|
4459
|
-
validateNewSnapshotId(nextSnapshotId, this.metadata.snapshots);
|
|
4460
|
-
const nextSequenceNumber = maxSequenceNumber(this.metadata) + 1;
|
|
4461
|
-
const manifestPath = appendManifestPath(this.metadataPath, options.jobId, nextSnapshotId);
|
|
4462
|
-
const manifest = {
|
|
4463
|
-
path: manifestPath,
|
|
4464
|
-
files: options.files.map((file, index) => ({
|
|
4465
|
-
path: file.path,
|
|
4466
|
-
sequenceNumber: nextSequenceNumber + index,
|
|
4467
|
-
partition: sortStringRecord(file.partition ?? {}),
|
|
4468
|
-
recordCount: file.recordCount,
|
|
4469
|
-
fileSizeInBytes: file.fileSizeInBytes
|
|
4470
|
-
}))
|
|
4471
|
-
};
|
|
4472
|
-
const nextSnapshot = {
|
|
4473
|
-
"snapshot-id": nextSnapshotId,
|
|
4474
|
-
"timestamp-ms": options.nowMs ?? Date.now(),
|
|
4475
|
-
"schema-id": currentSnapshot["schema-id"],
|
|
4476
|
-
manifests: [...snapshotManifests(currentSnapshot).map(cloneManifest), manifest]
|
|
4477
|
-
};
|
|
4478
|
-
const metadata = cloneMetadata(this.metadata);
|
|
4479
|
-
metadata["current-snapshot-id"] = nextSnapshotId;
|
|
4480
|
-
metadata.snapshots.push(nextSnapshot);
|
|
4481
|
-
metadata.refs = {
|
|
4482
|
-
...metadata.refs ?? {},
|
|
4483
|
-
main: { type: "branch", "snapshot-id": nextSnapshotId }
|
|
4484
|
-
};
|
|
4485
|
-
const nextMetadataPath = nextMetadataPathFor(this.metadataPath, nextSnapshotId);
|
|
4486
|
-
const catalog = options.catalog ?? new ObjectStoreIcebergCommitCatalog();
|
|
4487
|
-
const commit = await catalog.commitAppend({
|
|
4488
|
-
store: this.store,
|
|
4489
|
-
currentMetadataPath: this.metadataPath,
|
|
4490
|
-
nextMetadataPath,
|
|
4491
|
-
expectedSnapshotId: currentSnapshot["snapshot-id"],
|
|
4492
|
-
nextSnapshotId,
|
|
4493
|
-
manifestPath,
|
|
4494
|
-
manifest,
|
|
4495
|
-
metadata
|
|
4496
|
-
});
|
|
4497
|
-
const committed = typeof commit === "boolean" ? commit : commit.committed;
|
|
4498
|
-
if (!committed) {
|
|
4499
|
-
throw new LaQLError("LAQL_ICEBERG_COMMIT_CONFLICT", "Iceberg append commit conflict", {
|
|
4500
|
-
metadataPath: this.metadataPath,
|
|
4501
|
-
expectedSnapshotId: currentSnapshot["snapshot-id"],
|
|
4502
|
-
nextSnapshotId
|
|
4503
|
-
});
|
|
4504
|
-
}
|
|
4505
|
-
const committedMetadataPath = typeof commit === "boolean" ? nextMetadataPath : commit.metadataPath ?? nextMetadataPath;
|
|
4506
|
-
return {
|
|
4507
|
-
snapshotId: nextSnapshotId,
|
|
4508
|
-
previousSnapshotId: currentSnapshot["snapshot-id"],
|
|
4509
|
-
metadataPath: committedMetadataPath,
|
|
4510
|
-
manifestPath,
|
|
4511
|
-
files: manifest.files.map((file) => ({
|
|
4512
|
-
path: file.path,
|
|
4513
|
-
sequenceNumber: file.sequenceNumber,
|
|
4514
|
-
partition: file.partition ?? {},
|
|
4515
|
-
recordCount: file.recordCount,
|
|
4516
|
-
projectedFieldIds: [],
|
|
4517
|
-
snapshotId: nextSnapshotId
|
|
4518
|
-
}))
|
|
4519
|
-
};
|
|
4520
|
-
}
|
|
4521
|
-
async appendOutputManifest(options) {
|
|
4522
|
-
const files = options.manifest.entries.map((entry) => {
|
|
4523
|
-
if (entry.iceberg === void 0) {
|
|
4524
|
-
throw new LaQLError("LAQL_VALIDATION_ERROR", "Output manifest entry is missing Iceberg file metadata", {
|
|
4525
|
-
taskId: entry.taskId,
|
|
4526
|
-
outputPath: entry.outputPath
|
|
4527
|
-
});
|
|
4528
|
-
}
|
|
4529
|
-
return {
|
|
4530
|
-
path: entry.outputPath,
|
|
4531
|
-
partition: entry.iceberg.partitionValues,
|
|
4532
|
-
recordCount: entry.iceberg.recordCount,
|
|
4533
|
-
fileSizeInBytes: entry.iceberg.fileSizeInBytes
|
|
4534
|
-
};
|
|
4535
|
-
});
|
|
4536
|
-
return await this.appendFiles({
|
|
4537
|
-
files,
|
|
4538
|
-
jobId: options.manifest.jobId,
|
|
4539
|
-
...options.nowMs !== void 0 ? { nowMs: options.nowMs } : {},
|
|
4540
|
-
...options.nextSnapshotId !== void 0 ? { nextSnapshotId: options.nextSnapshotId } : {},
|
|
4541
|
-
...options.catalog !== void 0 ? { catalog: options.catalog } : {}
|
|
4542
|
-
});
|
|
4543
|
-
}
|
|
4544
|
-
snapshotById(snapshotId) {
|
|
4545
|
-
const snapshot = this.metadata.snapshots.find((candidate) => candidate["snapshot-id"] === snapshotId);
|
|
4546
|
-
if (!snapshot) {
|
|
4547
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", `Unknown Iceberg snapshot ${snapshotId}`, {
|
|
4548
|
-
snapshotId
|
|
4549
|
-
});
|
|
4550
|
-
}
|
|
4551
|
-
return snapshot;
|
|
4552
|
-
}
|
|
4553
|
-
fieldNameById(fieldId) {
|
|
4554
|
-
for (const schema of this.metadata.schemas) {
|
|
4555
|
-
const field = schema.fields.find((candidate) => candidate.id === fieldId);
|
|
4556
|
-
if (field)
|
|
4557
|
-
return field.name;
|
|
4558
|
-
}
|
|
4559
|
-
return void 0;
|
|
4560
|
-
}
|
|
4561
|
-
};
|
|
4562
|
-
function planFiles(table, options = {}) {
|
|
4563
|
-
return table.planFiles(options);
|
|
4564
|
-
}
|
|
4565
|
-
var ObjectStoreIcebergCommitCatalog = class {
|
|
4566
|
-
async commitAppend(input) {
|
|
4567
|
-
if (!supportsConditionalPut(input.store)) {
|
|
4568
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "Object-store Iceberg append requires conditional put support", { metadataPath: input.currentMetadataPath });
|
|
4569
|
-
}
|
|
4570
|
-
const currentBytes = await input.store.get(input.currentMetadataPath);
|
|
4571
|
-
if (!currentBytes) {
|
|
4572
|
-
throw new LaQLError("LAQL_OBJECT_NOT_FOUND", `No object at ${input.currentMetadataPath}`, {
|
|
4573
|
-
path: input.currentMetadataPath
|
|
4574
|
-
});
|
|
4575
|
-
}
|
|
4576
|
-
const current = validateMetadata(JSON.parse(new TextDecoder().decode(currentBytes)));
|
|
4577
|
-
if (current["current-snapshot-id"] !== input.expectedSnapshotId) {
|
|
4578
|
-
return { committed: false };
|
|
4579
|
-
}
|
|
4580
|
-
const versionHintPath = metadataVersionHintPath(input.nextMetadataPath);
|
|
4581
|
-
const versionHintHead = await input.store.head(versionHintPath);
|
|
4582
|
-
await input.store.put(input.manifestPath, new TextEncoder().encode(`${stableStringify(input.manifest)}
|
|
4583
|
-
`), { contentType: "application/json" });
|
|
4584
|
-
await input.store.put(input.nextMetadataPath, new TextEncoder().encode(`${JSON.stringify(input.metadata, null, 2)}
|
|
4585
|
-
`), { contentType: "application/json" });
|
|
4586
|
-
const updated = await input.store.conditionalPut(versionHintPath, new TextEncoder().encode(`${input.nextSnapshotId}
|
|
4587
|
-
`), { contentType: "text/plain", expectedEtag: versionHintHead?.etag ?? null });
|
|
4588
|
-
if (!updated)
|
|
4589
|
-
return { committed: false };
|
|
4590
|
-
return { committed: true, metadataPath: input.nextMetadataPath };
|
|
4591
|
-
}
|
|
4592
|
-
};
|
|
4593
|
-
var IcebergRestCatalog = class {
|
|
4594
|
-
url;
|
|
4595
|
-
namespace;
|
|
4596
|
-
table;
|
|
4597
|
-
prefix;
|
|
4598
|
-
token;
|
|
4599
|
-
fetchFn;
|
|
4600
|
-
constructor(options) {
|
|
4601
|
-
this.url = options.url;
|
|
4602
|
-
this.namespace = namespaceParts(options.namespace);
|
|
4603
|
-
this.table = requiredNonEmptyString(options.table, "table");
|
|
4604
|
-
this.prefix = catalogPrefixParts(options.prefix);
|
|
4605
|
-
this.token = options.token;
|
|
4606
|
-
this.fetchFn = options.fetch ?? fetch;
|
|
4607
|
-
}
|
|
4608
|
-
async loadTable(store) {
|
|
4609
|
-
const response = await this.requestJson(this.tableUrl(), { method: "GET" });
|
|
4610
|
-
if (!isRecord4(response) || typeof response["metadata-location"] !== "string") {
|
|
4611
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "Invalid Iceberg REST load table response", {
|
|
4612
|
-
url: this.tableUrl()
|
|
4613
|
-
});
|
|
4614
|
-
}
|
|
4615
|
-
return new IcebergTable(store, response["metadata-location"], await hydrateMetadataManifests(store, validateMetadata(response.metadata)));
|
|
4616
|
-
}
|
|
4617
|
-
async listTables() {
|
|
4618
|
-
return validateListTablesResponse(await this.requestJson(this.namespaceTablesUrl(), {
|
|
4619
|
-
method: "GET"
|
|
4620
|
-
}));
|
|
4621
|
-
}
|
|
4622
|
-
async commitAppend(input) {
|
|
4623
|
-
await input.store.put(input.manifestPath, new TextEncoder().encode(`${stableStringify(input.manifest)}
|
|
4624
|
-
`), { contentType: "application/json" });
|
|
4625
|
-
await input.store.put(input.nextMetadataPath, new TextEncoder().encode(`${JSON.stringify(input.metadata, null, 2)}
|
|
4626
|
-
`), { contentType: "application/json" });
|
|
4627
|
-
const response = await this.fetchFn(this.tableUrl(), {
|
|
4628
|
-
method: "POST",
|
|
4629
|
-
headers: this.headers(true),
|
|
4630
|
-
body: JSON.stringify({
|
|
4631
|
-
identifier: { namespace: this.namespace, name: this.table },
|
|
4632
|
-
requirements: [
|
|
4633
|
-
{
|
|
4634
|
-
type: "assert-ref-snapshot-id",
|
|
4635
|
-
ref: "main",
|
|
4636
|
-
"snapshot-id": input.expectedSnapshotId
|
|
4637
|
-
}
|
|
4638
|
-
],
|
|
4639
|
-
updates: [
|
|
4640
|
-
{
|
|
4641
|
-
action: "add-snapshot",
|
|
4642
|
-
snapshot: restAppendSnapshot(input)
|
|
4643
|
-
},
|
|
4644
|
-
{
|
|
4645
|
-
action: "set-snapshot-ref",
|
|
4646
|
-
"ref-name": "main",
|
|
4647
|
-
type: "branch",
|
|
4648
|
-
"snapshot-id": input.nextSnapshotId
|
|
4649
|
-
}
|
|
4650
|
-
]
|
|
4651
|
-
})
|
|
4652
|
-
});
|
|
4653
|
-
if (response.status === 409)
|
|
4654
|
-
return { committed: false };
|
|
4655
|
-
if (!response.ok) {
|
|
4656
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg REST table commit failed", {
|
|
4657
|
-
url: this.tableUrl(),
|
|
4658
|
-
status: response.status,
|
|
4659
|
-
statusText: response.statusText
|
|
4660
|
-
});
|
|
4661
|
-
}
|
|
4662
|
-
const metadataPath = await commitResponseMetadataPath(response);
|
|
4663
|
-
return {
|
|
4664
|
-
committed: true,
|
|
4665
|
-
...metadataPath !== void 0 ? { metadataPath } : {}
|
|
4666
|
-
};
|
|
4667
|
-
}
|
|
4668
|
-
async requestJson(url, init) {
|
|
4669
|
-
const response = await this.fetchFn(url, {
|
|
4670
|
-
...init,
|
|
4671
|
-
headers: this.headers(init.body !== void 0)
|
|
4672
|
-
});
|
|
4673
|
-
if (!response.ok) {
|
|
4674
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg REST catalog request failed", {
|
|
4675
|
-
url,
|
|
4676
|
-
status: response.status,
|
|
4677
|
-
statusText: response.statusText
|
|
4678
|
-
});
|
|
4679
|
-
}
|
|
4680
|
-
try {
|
|
4681
|
-
return await response.json();
|
|
4682
|
-
} catch (cause) {
|
|
4683
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg REST catalog response is not JSON", {
|
|
4684
|
-
url,
|
|
4685
|
-
cause
|
|
4686
|
-
});
|
|
4687
|
-
}
|
|
4688
|
-
}
|
|
4689
|
-
headers(hasBody) {
|
|
4690
|
-
const headers = new Headers({ accept: "application/json" });
|
|
4691
|
-
if (hasBody)
|
|
4692
|
-
headers.set("content-type", "application/json");
|
|
4693
|
-
if (this.token !== void 0)
|
|
4694
|
-
headers.set("authorization", `Bearer ${this.token}`);
|
|
4695
|
-
return headers;
|
|
4696
|
-
}
|
|
4697
|
-
tableUrl() {
|
|
4698
|
-
return restCatalogUrl(this.url, [...this.namespaceTableSegments(), this.table]);
|
|
4699
|
-
}
|
|
4700
|
-
namespaceTablesUrl() {
|
|
4701
|
-
return restCatalogUrl(this.url, this.namespaceTableSegments());
|
|
4702
|
-
}
|
|
4703
|
-
namespaceTableSegments() {
|
|
4704
|
-
return ["v1", ...this.prefix, "namespaces", this.namespace.join(""), "tables"];
|
|
4705
|
-
}
|
|
4706
|
-
};
|
|
4707
|
-
function icebergRestCatalog(options) {
|
|
4708
|
-
return new IcebergRestCatalog(options);
|
|
4709
|
-
}
|
|
4710
|
-
var IcebergUnsupportedCatalog = class {
|
|
4711
|
-
catalog;
|
|
4712
|
-
namespace;
|
|
4713
|
-
table;
|
|
4714
|
-
constructor(catalog, namespace, table) {
|
|
4715
|
-
this.catalog = requiredNonEmptyString(catalog, "catalog");
|
|
4716
|
-
this.namespace = namespaceParts(namespace);
|
|
4717
|
-
this.table = requiredNonEmptyString(table, "table");
|
|
4718
|
-
}
|
|
4719
|
-
async loadTable(_store) {
|
|
4720
|
-
throw this.unsupported("loadTable");
|
|
4721
|
-
}
|
|
4722
|
-
async listTables() {
|
|
4723
|
-
throw this.unsupported("listTables");
|
|
4724
|
-
}
|
|
4725
|
-
async commitAppend(_input) {
|
|
4726
|
-
throw this.unsupported("commitAppend");
|
|
4727
|
-
}
|
|
4728
|
-
unsupported(operation) {
|
|
4729
|
-
return new LaQLError("LAQL_CATALOG_ERROR", `${this.catalog} Iceberg catalog is not implemented`, {
|
|
4730
|
-
catalog: this.catalog,
|
|
4731
|
-
namespace: this.namespace,
|
|
4732
|
-
table: this.table,
|
|
4733
|
-
operation
|
|
4734
|
-
});
|
|
4735
|
-
}
|
|
4736
|
-
};
|
|
4737
|
-
function icebergGlueCatalog(options) {
|
|
4738
|
-
requiredNonEmptyString(options.region, "region");
|
|
4739
|
-
return new IcebergUnsupportedCatalog("Glue", options.namespace, options.table);
|
|
4740
|
-
}
|
|
4741
|
-
function icebergNessieCatalog(options) {
|
|
4742
|
-
requiredNonEmptyString(options.url, "url");
|
|
4743
|
-
if (options.ref !== void 0)
|
|
4744
|
-
requiredNonEmptyString(options.ref, "ref");
|
|
4745
|
-
return new IcebergUnsupportedCatalog("Nessie", options.namespace, options.table);
|
|
4746
|
-
}
|
|
4747
|
-
async function loadIcebergTable(options) {
|
|
4748
|
-
const readControls = loadReadControls(options);
|
|
4749
|
-
const store = withObjectStoreReadControls(options.store, readControls);
|
|
4750
|
-
const bytes = await store.get(options.metadataPath);
|
|
4751
|
-
if (!bytes) {
|
|
4752
|
-
throw new LaQLError("LAQL_OBJECT_NOT_FOUND", `No object at ${options.metadataPath}`, {
|
|
4753
|
-
path: options.metadataPath
|
|
4754
|
-
});
|
|
4755
|
-
}
|
|
4756
|
-
throwIfAborted(readControls.signal);
|
|
4757
|
-
const text = new TextDecoder().decode(bytes);
|
|
4758
|
-
try {
|
|
4759
|
-
return new IcebergTable(store, options.metadataPath, await hydrateMetadataManifests(store, validateMetadata(JSON.parse(text)), readControls));
|
|
4760
|
-
} catch (cause) {
|
|
4761
|
-
if (cause instanceof LaQLError)
|
|
4762
|
-
throw cause;
|
|
4763
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", `Invalid Iceberg metadata at ${options.metadataPath}`, {
|
|
4764
|
-
path: options.metadataPath,
|
|
4765
|
-
cause
|
|
4766
|
-
});
|
|
4767
|
-
}
|
|
4768
|
-
}
|
|
4769
|
-
async function loadIcebergTableFromObjectStore(options) {
|
|
4770
|
-
const readControls = loadReadControls(options);
|
|
4771
|
-
const store = withObjectStoreReadControls(options.store, readControls);
|
|
4772
|
-
const tableLocation = trimTrailingSlash(options.tableLocation);
|
|
4773
|
-
const metadataPrefix = `${tableLocation}/metadata/`;
|
|
4774
|
-
const versionHintPath = `${metadataPrefix}version-hint.text`;
|
|
4775
|
-
const hintedVersion = await readVersionHint(store, versionHintPath, readControls);
|
|
4776
|
-
const metadataPath = hintedVersion === void 0 ? await latestMetadataPathFromList(store, metadataPrefix, readControls) : `${metadataPrefix}v${hintedVersion}.metadata.json`;
|
|
4777
|
-
return await loadIcebergTable({ store, metadataPath, ...readControls });
|
|
4778
|
-
}
|
|
4779
|
-
async function loadIcebergTableFromRest(options) {
|
|
4780
|
-
return await icebergRestCatalog(options).loadTable(withObjectStoreReadControls(options.store, loadReadControls(options)));
|
|
4781
|
-
}
|
|
4782
|
-
function loadReadControls(options) {
|
|
4783
|
-
const controls = {};
|
|
4784
|
-
if (options.maxConcurrentReads !== void 0)
|
|
4785
|
-
controls.maxConcurrentReads = options.maxConcurrentReads;
|
|
4786
|
-
const signal = readControlSignal(options);
|
|
4787
|
-
if (signal !== void 0)
|
|
4788
|
-
controls.signal = signal;
|
|
4789
|
-
return controls;
|
|
4790
|
-
}
|
|
4791
|
-
function applyIcebergDeletes(options) {
|
|
4792
|
-
const rowOffset = options.rowOffset ?? 0;
|
|
4793
|
-
const positionDeletes = /* @__PURE__ */ new Set();
|
|
4794
|
-
for (const deletion of options.positionDeletes ?? []) {
|
|
4795
|
-
if (deletion.path !== options.dataFilePath)
|
|
4796
|
-
continue;
|
|
4797
|
-
positionDeletes.add(validateDeletePosition(deletion.position, deletion.path));
|
|
4798
|
-
}
|
|
4799
|
-
for (const deletionVector of options.deletionVectors ?? []) {
|
|
4800
|
-
if (deletionVector.path !== options.dataFilePath)
|
|
4801
|
-
continue;
|
|
4802
|
-
for (const position of deletionVector.positions) {
|
|
4803
|
-
positionDeletes.add(validateDeletePosition(position, deletionVector.path));
|
|
4804
|
-
}
|
|
4805
|
-
}
|
|
4806
|
-
const equalityDeleteKeys = /* @__PURE__ */ new Map();
|
|
4807
|
-
for (const deletion of options.equalityDeletes ?? []) {
|
|
4808
|
-
if (deletion.columns.length === 0) {
|
|
4809
|
-
throw new LaQLError("LAQL_VALIDATION_ERROR", "Iceberg equality delete requires columns");
|
|
4810
|
-
}
|
|
4811
|
-
const columnsKey = stableStringify(deletion.columns);
|
|
4812
|
-
let keys = equalityDeleteKeys.get(columnsKey);
|
|
4813
|
-
if (!keys) {
|
|
4814
|
-
keys = /* @__PURE__ */ new Set();
|
|
4815
|
-
equalityDeleteKeys.set(columnsKey, keys);
|
|
4816
|
-
}
|
|
4817
|
-
keys.add(equalityKey(deletion.row, deletion.columns));
|
|
4818
|
-
}
|
|
4819
|
-
return options.rows.filter((row, index) => {
|
|
4820
|
-
if (positionDeletes.has(rowOffset + index))
|
|
4821
|
-
return false;
|
|
4822
|
-
for (const [columnsKey, keys] of equalityDeleteKeys) {
|
|
4823
|
-
const columns = JSON.parse(columnsKey);
|
|
4824
|
-
if (keys.has(equalityKey(row, columns)))
|
|
4825
|
-
return false;
|
|
4826
|
-
}
|
|
4827
|
-
return true;
|
|
4828
|
-
});
|
|
4829
|
-
}
|
|
4830
|
-
async function* scanPlannedIcebergRows(options) {
|
|
4831
|
-
const files = Array.isArray(options.plan) ? options.plan : options.plan.files;
|
|
4832
|
-
const signal = readControlSignal(options);
|
|
4833
|
-
for (const file of files) {
|
|
4834
|
-
throwIfAborted(signal);
|
|
4835
|
-
const deletes = await decodedDeletesForFile(file, options, signal);
|
|
4836
|
-
throwIfAborted(signal);
|
|
4837
|
-
const data = await options.readDataFile(file);
|
|
4838
|
-
throwIfAborted(signal);
|
|
4839
|
-
let rowOffset = 0;
|
|
4840
|
-
for await (const batch of rowBatches(data)) {
|
|
4841
|
-
throwIfAborted(signal);
|
|
4842
|
-
const rows = Array.isArray(batch) ? batch : batch.rows;
|
|
4843
|
-
const absoluteRowOffset = Array.isArray(batch) ? rowOffset : batch.rowOffset;
|
|
4844
|
-
const visibleRows = hasDeletes(deletes) ? applyIcebergDeletes({
|
|
4845
|
-
dataFilePath: file.path,
|
|
4846
|
-
rows,
|
|
4847
|
-
rowOffset: absoluteRowOffset,
|
|
4848
|
-
...deletes
|
|
4849
|
-
}) : rows;
|
|
4850
|
-
rowOffset = absoluteRowOffset + rows.length;
|
|
4851
|
-
if (visibleRows.length > 0)
|
|
4852
|
-
yield visibleRows;
|
|
4853
|
-
throwIfAborted(signal);
|
|
4854
|
-
}
|
|
4855
|
-
}
|
|
4856
|
-
}
|
|
4857
|
-
async function decodedDeletesForFile(file, options, signal) {
|
|
4858
|
-
const out = {};
|
|
4859
|
-
for (const deleteFile of file.deleteFiles ?? []) {
|
|
4860
|
-
throwIfAborted(signal);
|
|
4861
|
-
const decoded = await options.readDeleteFile(deleteFile, file);
|
|
4862
|
-
throwIfAborted(signal);
|
|
4863
|
-
pushDeletes(out, decoded);
|
|
4864
|
-
}
|
|
4865
|
-
return out;
|
|
4866
|
-
}
|
|
4867
|
-
function pushDeletes(target, source) {
|
|
4868
|
-
if (source.positionDeletes !== void 0) {
|
|
4869
|
-
target.positionDeletes = [...target.positionDeletes ?? [], ...source.positionDeletes];
|
|
4870
|
-
}
|
|
4871
|
-
if (source.equalityDeletes !== void 0) {
|
|
4872
|
-
target.equalityDeletes = [...target.equalityDeletes ?? [], ...source.equalityDeletes];
|
|
4873
|
-
}
|
|
4874
|
-
if (source.deletionVectors !== void 0) {
|
|
4875
|
-
target.deletionVectors = [...target.deletionVectors ?? [], ...source.deletionVectors];
|
|
4876
|
-
}
|
|
4877
|
-
}
|
|
4878
|
-
function hasDeletes(deletes) {
|
|
4879
|
-
return (deletes.positionDeletes?.length ?? 0) > 0 || (deletes.equalityDeletes?.length ?? 0) > 0 || (deletes.deletionVectors?.length ?? 0) > 0;
|
|
4880
|
-
}
|
|
4881
|
-
async function* rowBatches(rows) {
|
|
4882
|
-
if (isAsyncIterable(rows)) {
|
|
4883
|
-
yield* rows;
|
|
4884
|
-
} else {
|
|
4885
|
-
yield rows;
|
|
4886
|
-
}
|
|
4887
|
-
}
|
|
4888
|
-
function isAsyncIterable(value) {
|
|
4889
|
-
return typeof value === "object" && value !== null && Symbol.asyncIterator in value && typeof value[Symbol.asyncIterator] === "function";
|
|
4890
|
-
}
|
|
4891
|
-
function validateDeletePosition(position, path) {
|
|
4892
|
-
if (!Number.isInteger(position) || position < 0) {
|
|
4893
|
-
throw new LaQLError("LAQL_VALIDATION_ERROR", "Iceberg delete position must be non-negative", {
|
|
4894
|
-
path,
|
|
4895
|
-
position
|
|
4896
|
-
});
|
|
4897
|
-
}
|
|
4898
|
-
return position;
|
|
4899
|
-
}
|
|
4900
|
-
function nextMetadataPathFor(metadataPath, snapshotId) {
|
|
4901
|
-
const slash = metadataPath.lastIndexOf("/");
|
|
4902
|
-
const prefix = slash === -1 ? "" : `${metadataPath.slice(0, slash + 1)}`;
|
|
4903
|
-
return `${prefix}v${snapshotId}.metadata.json`;
|
|
4904
|
-
}
|
|
4905
|
-
function randomSnapshotId(existingIds) {
|
|
4906
|
-
const existing = new Set(existingIds);
|
|
4907
|
-
for (let attempt = 0; attempt < 100; attempt += 1) {
|
|
4908
|
-
const id = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER) + 1;
|
|
4909
|
-
if (!existing.has(id))
|
|
4910
|
-
return id;
|
|
4911
|
-
}
|
|
4912
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "Unable to allocate unique Iceberg snapshot id");
|
|
4913
|
-
}
|
|
4914
|
-
function validateNewSnapshotId(snapshotId, snapshots) {
|
|
4915
|
-
if (!Number.isSafeInteger(snapshotId) || snapshotId <= 0) {
|
|
4916
|
-
throw new LaQLError("LAQL_VALIDATION_ERROR", "Iceberg snapshot id must be a positive safe integer", { snapshotId });
|
|
4917
|
-
}
|
|
4918
|
-
if (snapshots.some((snapshot) => snapshotIdOf(snapshot) === snapshotId)) {
|
|
4919
|
-
throw new LaQLError("LAQL_VALIDATION_ERROR", "Iceberg snapshot id already exists", {
|
|
4920
|
-
snapshotId
|
|
4921
|
-
});
|
|
4922
|
-
}
|
|
4923
|
-
}
|
|
4924
|
-
function snapshotIdOf(snapshot) {
|
|
4925
|
-
return snapshot["snapshot-id"];
|
|
4926
|
-
}
|
|
4927
|
-
function metadataVersionHintPath(metadataPath) {
|
|
4928
|
-
const slash = metadataPath.lastIndexOf("/");
|
|
4929
|
-
const prefix = slash === -1 ? "" : `${metadataPath.slice(0, slash + 1)}`;
|
|
4930
|
-
return `${prefix}version-hint.text`;
|
|
4931
|
-
}
|
|
4932
|
-
function supportsConditionalPut(store) {
|
|
4933
|
-
return typeof store.conditionalPut === "function";
|
|
4934
|
-
}
|
|
4935
|
-
async function hydrateMetadataManifests(store, metadata, controls = {}) {
|
|
4936
|
-
const hydrated = cloneMetadata(metadata);
|
|
4937
|
-
const tablePrefix = tableLocationObjectPrefix(hydrated.location);
|
|
4938
|
-
for (const snapshot of hydrated.snapshots) {
|
|
4939
|
-
throwIfAborted(controls.signal);
|
|
4940
|
-
const manifestReferences = snapshot.manifests ?? (snapshot["manifest-list"] !== void 0 ? await readManifestList(store, validateManifestSourcedPath(snapshot["manifest-list"], tablePrefix)) : []);
|
|
4941
|
-
const manifests = await Promise.all(manifestReferences.map(async (manifest) => {
|
|
4942
|
-
const manifestPath = validateManifestSourcedPath(manifest.path, tablePrefix);
|
|
4943
|
-
if (Array.isArray(manifest.files))
|
|
4944
|
-
return validateManifestPaths(manifest, manifestPath, tablePrefix);
|
|
4945
|
-
return await readManifest(store, manifestPath, tablePrefix);
|
|
4946
|
-
}));
|
|
4947
|
-
throwIfAborted(controls.signal);
|
|
4948
|
-
snapshot.manifests = mergeDeleteManifests(manifests);
|
|
4949
|
-
}
|
|
4950
|
-
return hydrated;
|
|
4951
|
-
}
|
|
4952
|
-
function mergeDeleteManifests(manifests) {
|
|
4953
|
-
const deleteFiles = manifests.flatMap((manifest) => manifest.deleteFiles ?? []);
|
|
4954
|
-
const dataManifests = manifests.filter((manifest) => manifest.files.length > 0);
|
|
4955
|
-
if (deleteFiles.length === 0)
|
|
4956
|
-
return dataManifests;
|
|
4957
|
-
return dataManifests.map((manifest) => ({
|
|
4958
|
-
...manifest,
|
|
4959
|
-
files: manifest.files.map((file) => {
|
|
4960
|
-
const applicable = deleteFiles.filter((deleteFile) => deleteFileMayApply(deleteFile, file));
|
|
4961
|
-
if (applicable.length === 0)
|
|
4962
|
-
return file;
|
|
4963
|
-
return {
|
|
4964
|
-
...file,
|
|
4965
|
-
deleteFiles: [...file.deleteFiles ?? [], ...applicable.map(publicDeleteFile)]
|
|
4966
|
-
};
|
|
4967
|
-
})
|
|
4968
|
-
}));
|
|
4969
|
-
}
|
|
4970
|
-
function deleteFileMayApply(deleteFile, file) {
|
|
4971
|
-
if (deleteFile.partition === void 0 || Object.keys(deleteFile.partition).length === 0) {
|
|
4972
|
-
return true;
|
|
4973
|
-
}
|
|
4974
|
-
const filePartition = file.partition ?? {};
|
|
4975
|
-
return Object.entries(deleteFile.partition).every(([key, value]) => filePartition[key] === value);
|
|
4976
|
-
}
|
|
4977
|
-
function publicDeleteFile(deleteFile) {
|
|
4978
|
-
return { content: deleteFile.content, path: deleteFile.path };
|
|
4979
|
-
}
|
|
4980
|
-
async function readManifestList(store, path) {
|
|
4981
|
-
const bytes = await store.get(path);
|
|
4982
|
-
if (!bytes) {
|
|
4983
|
-
throw new LaQLError("LAQL_OBJECT_NOT_FOUND", `No Iceberg manifest list at ${path}`, { path });
|
|
4984
|
-
}
|
|
4985
|
-
try {
|
|
4986
|
-
return avroObjectContainer(bytes) ? validateAvroManifestList(await decodeAvroObjectContainer(bytes), path) : validateManifestList(JSON.parse(new TextDecoder().decode(bytes)), path);
|
|
4987
|
-
} catch (cause) {
|
|
4988
|
-
if (cause instanceof LaQLError)
|
|
4989
|
-
throw cause;
|
|
4990
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", `Invalid Iceberg manifest list at ${path}`, {
|
|
4991
|
-
path,
|
|
4992
|
-
cause
|
|
4993
|
-
});
|
|
4994
|
-
}
|
|
4995
|
-
}
|
|
4996
|
-
async function readManifest(store, path, tablePrefix = "") {
|
|
4997
|
-
const bytes = await store.get(path);
|
|
4998
|
-
if (!bytes) {
|
|
4999
|
-
throw new LaQLError("LAQL_OBJECT_NOT_FOUND", `No Iceberg manifest at ${path}`, { path });
|
|
5000
|
-
}
|
|
5001
|
-
try {
|
|
5002
|
-
const manifest = avroObjectContainer(bytes) ? validateAvroManifest(await decodeAvroObjectContainer(bytes), path) : validateManifest(JSON.parse(new TextDecoder().decode(bytes)), path);
|
|
5003
|
-
return validateManifestPaths(manifest, path, tablePrefix);
|
|
5004
|
-
} catch (cause) {
|
|
5005
|
-
if (cause instanceof LaQLError)
|
|
5006
|
-
throw cause;
|
|
5007
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", `Invalid Iceberg manifest at ${path}`, {
|
|
5008
|
-
path,
|
|
5009
|
-
cause
|
|
5010
|
-
});
|
|
5011
|
-
}
|
|
5012
|
-
}
|
|
5013
|
-
function validateAvroManifestList(records, path) {
|
|
5014
|
-
return records.map((record) => {
|
|
5015
|
-
if (!isRecord4(record) || typeof record.manifest_path !== "string") {
|
|
5016
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg Avro manifest list entry is invalid", {
|
|
5017
|
-
path
|
|
5018
|
-
});
|
|
5019
|
-
}
|
|
5020
|
-
validateManifestContent(record.content, path);
|
|
5021
|
-
return { path: record.manifest_path };
|
|
5022
|
-
});
|
|
5023
|
-
}
|
|
5024
|
-
function validateAvroManifest(records, path) {
|
|
5025
|
-
const files = [];
|
|
5026
|
-
const deleteFiles = [];
|
|
5027
|
-
for (const record of records) {
|
|
5028
|
-
if (!isRecord4(record)) {
|
|
5029
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg Avro manifest entry is invalid", {
|
|
5030
|
-
path
|
|
5031
|
-
});
|
|
5032
|
-
}
|
|
5033
|
-
if (typeof record.status === "number" && record.status === 2)
|
|
5034
|
-
continue;
|
|
5035
|
-
const dataFile = record.data_file;
|
|
5036
|
-
if (!isRecord4(dataFile)) {
|
|
5037
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg Avro manifest entry is missing data_file", {
|
|
5038
|
-
path
|
|
5039
|
-
});
|
|
5040
|
-
}
|
|
5041
|
-
const content = typeof dataFile.content === "number" ? dataFile.content : 0;
|
|
5042
|
-
const recordCount = safeAvroNumber(dataFile.record_count);
|
|
5043
|
-
if (typeof dataFile.file_path !== "string" || recordCount === void 0) {
|
|
5044
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg Avro data file has invalid fields", {
|
|
5045
|
-
path
|
|
5046
|
-
});
|
|
5047
|
-
}
|
|
5048
|
-
if (content !== 0) {
|
|
5049
|
-
deleteFiles.push({
|
|
5050
|
-
content: avroDeleteContent(content, dataFile),
|
|
5051
|
-
path: dataFile.file_path,
|
|
5052
|
-
partition: avroPartitionValues(dataFile.partition)
|
|
5053
|
-
});
|
|
5054
|
-
continue;
|
|
5055
|
-
}
|
|
5056
|
-
const sequenceNumber = safeAvroNumber(record.sequence_number) ?? safeAvroNumber(record.file_sequence_number) ?? 0;
|
|
5057
|
-
const file = {
|
|
5058
|
-
path: dataFile.file_path,
|
|
5059
|
-
sequenceNumber,
|
|
5060
|
-
partition: avroPartitionValues(dataFile.partition),
|
|
5061
|
-
recordCount
|
|
5062
|
-
};
|
|
5063
|
-
const fileSizeInBytes = safeAvroNumber(dataFile.file_size_in_bytes);
|
|
5064
|
-
if (fileSizeInBytes !== void 0) {
|
|
5065
|
-
file.fileSizeInBytes = fileSizeInBytes;
|
|
5066
|
-
}
|
|
5067
|
-
files.push(file);
|
|
5068
|
-
}
|
|
5069
|
-
const manifest = { path, files };
|
|
5070
|
-
if (deleteFiles.length > 0)
|
|
5071
|
-
manifest.deleteFiles = deleteFiles;
|
|
5072
|
-
return validateManifestPaths(manifest, path);
|
|
5073
|
-
}
|
|
5074
|
-
function avroDeleteContent(content, dataFile) {
|
|
5075
|
-
if (content === 1 && (String(dataFile.file_format).toLowerCase() === "puffin" || dataFile.content_offset !== void 0 || dataFile.content_size_in_bytes !== void 0)) {
|
|
5076
|
-
return "deletion-vector";
|
|
5077
|
-
}
|
|
5078
|
-
if (content === 1)
|
|
5079
|
-
return "position-delete";
|
|
5080
|
-
if (content === 2)
|
|
5081
|
-
return "equality-delete";
|
|
5082
|
-
return `unsupported-delete-${content}`;
|
|
5083
|
-
}
|
|
5084
|
-
function avroPartitionValues(value) {
|
|
5085
|
-
if (!isRecord4(value))
|
|
5086
|
-
return {};
|
|
5087
|
-
const out = {};
|
|
5088
|
-
for (const [key, inner] of Object.entries(value)) {
|
|
5089
|
-
if (inner === null || inner === void 0)
|
|
5090
|
-
continue;
|
|
5091
|
-
out[key] = String(jsonSafeValue(inner));
|
|
5092
|
-
}
|
|
5093
|
-
return sortStringRecord(out);
|
|
5094
|
-
}
|
|
5095
|
-
function safeAvroNumber(value) {
|
|
5096
|
-
if (typeof value === "number" && Number.isSafeInteger(value))
|
|
5097
|
-
return value;
|
|
5098
|
-
if (typeof value === "bigint") {
|
|
5099
|
-
const numberValue = Number(value);
|
|
5100
|
-
if (Number.isSafeInteger(numberValue) && BigInt(numberValue) === value)
|
|
5101
|
-
return numberValue;
|
|
5102
|
-
}
|
|
5103
|
-
return void 0;
|
|
5104
|
-
}
|
|
5105
|
-
function avroObjectContainer(bytes) {
|
|
5106
|
-
return bytes.length >= 4 && bytes[0] === 79 && bytes[1] === 98 && bytes[2] === 106 && bytes[3] === 1;
|
|
5107
|
-
}
|
|
5108
|
-
async function decodeAvroObjectContainer(bytes) {
|
|
5109
|
-
const avro = await loadAvro();
|
|
5110
|
-
const avroBigIntLongType = avro.types.LongType.__with({
|
|
5111
|
-
fromBuffer: (buffer) => buffer.readBigInt64LE(),
|
|
5112
|
-
toBuffer: (value) => {
|
|
5113
|
-
const buffer = Buffer.alloc(8);
|
|
5114
|
-
buffer.writeBigInt64LE(BigInt(value));
|
|
5115
|
-
return buffer;
|
|
5116
|
-
},
|
|
5117
|
-
fromJSON: (value) => BigInt(value),
|
|
5118
|
-
toJSON: (value) => value.toString(),
|
|
5119
|
-
isValid: (value) => typeof value === "bigint" || typeof value === "number" && Number.isSafeInteger(value),
|
|
5120
|
-
compare: (left, right) => {
|
|
5121
|
-
const leftBigInt = BigInt(left);
|
|
5122
|
-
const rightBigInt = BigInt(right);
|
|
5123
|
-
return leftBigInt === rightBigInt ? 0 : leftBigInt < rightBigInt ? -1 : 1;
|
|
5124
|
-
}
|
|
5125
|
-
});
|
|
5126
|
-
const avroLongTypeHook = (schema) => schema === "long" || isRecord4(schema) && schema.type === "long" ? avroBigIntLongType : void 0;
|
|
5127
|
-
const decoder = new avro.streams.BlockDecoder({
|
|
5128
|
-
parseHook: (schema) => avro.Type.forSchema(schema, { typeHook: avroLongTypeHook })
|
|
5129
|
-
});
|
|
5130
|
-
const records = [];
|
|
5131
|
-
const done = new Promise((resolve, reject) => {
|
|
5132
|
-
decoder.on("data", (record) => records.push(record));
|
|
5133
|
-
decoder.on("end", () => resolve(records));
|
|
5134
|
-
decoder.on("error", reject);
|
|
5135
|
-
});
|
|
5136
|
-
decoder.end(Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength));
|
|
5137
|
-
return done;
|
|
5138
|
-
}
|
|
5139
|
-
async function loadAvro() {
|
|
5140
|
-
const module = await import('avsc');
|
|
5141
|
-
return module.default ?? module;
|
|
5142
|
-
}
|
|
5143
|
-
function validateManifestList(value, path) {
|
|
5144
|
-
const manifests = Array.isArray(value) ? value : isRecord4(value) && Array.isArray(value.manifests) ? value.manifests : void 0;
|
|
5145
|
-
if (manifests === void 0) {
|
|
5146
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg manifest list has invalid required fields", {
|
|
5147
|
-
path
|
|
5148
|
-
});
|
|
5149
|
-
}
|
|
5150
|
-
return manifests.map((manifest) => validateManifestReference(manifest, path));
|
|
5151
|
-
}
|
|
5152
|
-
function validateManifest(value, path) {
|
|
5153
|
-
if (!isRecord4(value) || typeof value.path !== "string" || !Array.isArray(value.files)) {
|
|
5154
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg manifest has invalid required fields", {
|
|
5155
|
-
path
|
|
5156
|
-
});
|
|
5157
|
-
}
|
|
5158
|
-
return value;
|
|
5159
|
-
}
|
|
5160
|
-
function validateManifestReference(value, path) {
|
|
5161
|
-
if (!isRecord4(value) || typeof value.path !== "string") {
|
|
5162
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg manifest list entry is invalid", { path });
|
|
5163
|
-
}
|
|
5164
|
-
validateManifestContent(value.content, path);
|
|
5165
|
-
if (Array.isArray(value.files))
|
|
5166
|
-
return value;
|
|
5167
|
-
return { path: value.path };
|
|
5168
|
-
}
|
|
5169
|
-
function validateManifestContent(content, path) {
|
|
5170
|
-
if (content === void 0 || content === null)
|
|
5171
|
-
return;
|
|
5172
|
-
if (content === 0 || content === 1 || content === "data" || content === "deletes")
|
|
5173
|
-
return;
|
|
5174
|
-
throw new LaQLError("LAQL_UNSUPPORTED_ICEBERG_FEATURE", "Iceberg manifest list contains an unsupported manifest content type", {
|
|
5175
|
-
path,
|
|
5176
|
-
content
|
|
5177
|
-
});
|
|
5178
|
-
}
|
|
5179
|
-
function validateManifestPaths(manifest, path, tablePrefix = "") {
|
|
5180
|
-
validateManifestSourcedPath(manifest.path, tablePrefix);
|
|
5181
|
-
for (const file of manifest.files) {
|
|
5182
|
-
validateManifestSourcedPath(file.path, tablePrefix);
|
|
5183
|
-
for (const deleteFile of file.deleteFiles ?? []) {
|
|
5184
|
-
validateManifestSourcedPath(deleteFile.path, tablePrefix);
|
|
5185
|
-
}
|
|
5186
|
-
}
|
|
5187
|
-
for (const deleteFile of manifest.deleteFiles ?? []) {
|
|
5188
|
-
validateManifestSourcedPath(deleteFile.path, tablePrefix);
|
|
5189
|
-
}
|
|
5190
|
-
return { ...manifest, path };
|
|
5191
|
-
}
|
|
5192
|
-
function validateManifestSourcedPath(path, tablePrefix = "") {
|
|
5193
|
-
if (/^(?:[a-z][a-z0-9+.-]*:)?\/\//iu.test(path) || path.startsWith("/")) {
|
|
5194
|
-
throw new LaQLError("LAQL_VALIDATION_ERROR", `Iceberg manifest path must be relative: ${path}`, {
|
|
5195
|
-
path
|
|
5196
|
-
});
|
|
5197
|
-
}
|
|
5198
|
-
for (const segment of path.split("/")) {
|
|
5199
|
-
let decoded;
|
|
5200
|
-
try {
|
|
5201
|
-
decoded = decodeURIComponent(segment);
|
|
5202
|
-
} catch {
|
|
5203
|
-
throw new LaQLError("LAQL_VALIDATION_ERROR", `Iceberg manifest path is invalid: ${path}`, {
|
|
5204
|
-
path
|
|
5205
|
-
});
|
|
5206
|
-
}
|
|
5207
|
-
if (decoded === "." || decoded === "..") {
|
|
5208
|
-
throw new LaQLError("LAQL_VALIDATION_ERROR", `Iceberg manifest path contains traversal: ${path}`, {
|
|
5209
|
-
path
|
|
5210
|
-
});
|
|
5211
|
-
}
|
|
5212
|
-
}
|
|
5213
|
-
if (tablePrefix !== "" && path !== tablePrefix && !path.startsWith(`${tablePrefix}/`)) {
|
|
5214
|
-
throw new LaQLError("LAQL_VALIDATION_ERROR", `Iceberg manifest path escapes table location: ${path}`, {
|
|
5215
|
-
path,
|
|
5216
|
-
tableLocation: tablePrefix
|
|
5217
|
-
});
|
|
5218
|
-
}
|
|
5219
|
-
return path;
|
|
5220
|
-
}
|
|
5221
|
-
function tableLocationObjectPrefix(location) {
|
|
5222
|
-
const trimmed = trimTrailingSlash(location.trim());
|
|
5223
|
-
if (trimmed === "" || !trimmed.includes("/"))
|
|
5224
|
-
return "";
|
|
5225
|
-
if (/^[a-z][a-z0-9+.-]*:\/\//iu.test(trimmed)) {
|
|
5226
|
-
const url = new URL(trimmed);
|
|
5227
|
-
return trimSlashes(decodeURIComponent(url.pathname));
|
|
5228
|
-
}
|
|
5229
|
-
return trimSlashes(trimmed);
|
|
5230
|
-
}
|
|
5231
|
-
function trimSlashes(value) {
|
|
5232
|
-
return value.replace(/^\/+|\/+$/gu, "");
|
|
5233
|
-
}
|
|
5234
|
-
async function readVersionHint(store, path, controls = {}) {
|
|
5235
|
-
const bytes = await store.get(path);
|
|
5236
|
-
if (!bytes)
|
|
5237
|
-
return void 0;
|
|
5238
|
-
throwIfAborted(controls.signal);
|
|
5239
|
-
const text = new TextDecoder().decode(bytes).trim();
|
|
5240
|
-
const version = Number(text);
|
|
5241
|
-
if (!Number.isInteger(version) || version < 0) {
|
|
5242
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "Invalid Iceberg version hint", {
|
|
5243
|
-
path,
|
|
5244
|
-
versionHint: text
|
|
5245
|
-
});
|
|
5246
|
-
}
|
|
5247
|
-
return version;
|
|
5248
|
-
}
|
|
5249
|
-
async function latestMetadataPathFromList(store, metadataPrefix, controls = {}) {
|
|
5250
|
-
let latest;
|
|
5251
|
-
for await (const object of store.list(metadataPrefix)) {
|
|
5252
|
-
throwIfAborted(controls.signal);
|
|
5253
|
-
const name = object.path.slice(metadataPrefix.length);
|
|
5254
|
-
const match = /^v(\d+)\.metadata\.json$/u.exec(name);
|
|
5255
|
-
if (!match)
|
|
5256
|
-
continue;
|
|
5257
|
-
const version = Number(match[1]);
|
|
5258
|
-
if (!Number.isSafeInteger(version))
|
|
5259
|
-
continue;
|
|
5260
|
-
if (latest === void 0 || version > latest.version)
|
|
5261
|
-
latest = { path: object.path, version };
|
|
5262
|
-
}
|
|
5263
|
-
if (latest === void 0) {
|
|
5264
|
-
throw new LaQLError("LAQL_OBJECT_NOT_FOUND", "No Iceberg metadata files found", {
|
|
5265
|
-
prefix: metadataPrefix
|
|
5266
|
-
});
|
|
5267
|
-
}
|
|
5268
|
-
return latest.path;
|
|
5269
|
-
}
|
|
5270
|
-
function trimTrailingSlash(value) {
|
|
5271
|
-
return value.endsWith("/") ? value.slice(0, -1) : value;
|
|
5272
|
-
}
|
|
5273
|
-
function restCatalogUrl(baseUrl, segments) {
|
|
5274
|
-
const url = new URL(baseUrl);
|
|
5275
|
-
const basePath = url.pathname.replace(/\/+$/u, "");
|
|
5276
|
-
const encodedSegments = segments.map((segment) => encodeURIComponent(segment));
|
|
5277
|
-
url.pathname = [basePath, ...encodedSegments].filter((segment) => segment.length > 0).join("/");
|
|
5278
|
-
return url.toString();
|
|
5279
|
-
}
|
|
5280
|
-
function namespaceParts(namespace) {
|
|
5281
|
-
const parts = Array.isArray(namespace) ? namespace : namespace.split(".");
|
|
5282
|
-
const out = parts.map((part) => requiredNonEmptyString(part, "namespace"));
|
|
5283
|
-
if (out.length === 0) {
|
|
5284
|
-
throw new LaQLError("LAQL_VALIDATION_ERROR", "Iceberg REST namespace is required");
|
|
5285
|
-
}
|
|
5286
|
-
return out;
|
|
5287
|
-
}
|
|
5288
|
-
function catalogPrefixParts(prefix) {
|
|
5289
|
-
if (prefix === void 0 || prefix.trim() === "")
|
|
5290
|
-
return [];
|
|
5291
|
-
return prefix.split("/").filter((part) => part.length > 0).map((part) => requiredNonEmptyString(part, "prefix"));
|
|
5292
|
-
}
|
|
5293
|
-
function requiredNonEmptyString(value, field) {
|
|
5294
|
-
const trimmed = value.trim();
|
|
5295
|
-
if (trimmed.length === 0) {
|
|
5296
|
-
throw new LaQLError("LAQL_VALIDATION_ERROR", `Iceberg REST ${field} is required`);
|
|
5297
|
-
}
|
|
5298
|
-
return trimmed;
|
|
5299
|
-
}
|
|
5300
|
-
async function commitResponseMetadataPath(response) {
|
|
5301
|
-
if (response.status === 204)
|
|
5302
|
-
return void 0;
|
|
5303
|
-
try {
|
|
5304
|
-
const body = await response.json();
|
|
5305
|
-
if (isRecord4(body) && typeof body["metadata-location"] === "string") {
|
|
5306
|
-
return body["metadata-location"];
|
|
5307
|
-
}
|
|
5308
|
-
} catch {
|
|
5309
|
-
return void 0;
|
|
5310
|
-
}
|
|
5311
|
-
return void 0;
|
|
5312
|
-
}
|
|
5313
|
-
function restAppendSnapshot(input) {
|
|
5314
|
-
const snapshot = input.metadata.snapshots.at(-1);
|
|
5315
|
-
if (snapshot === void 0) {
|
|
5316
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg append metadata is missing next snapshot");
|
|
5317
|
-
}
|
|
5318
|
-
return {
|
|
5319
|
-
"snapshot-id": snapshot["snapshot-id"],
|
|
5320
|
-
"parent-snapshot-id": input.expectedSnapshotId,
|
|
5321
|
-
"timestamp-ms": snapshot["timestamp-ms"],
|
|
5322
|
-
"schema-id": snapshot["schema-id"],
|
|
5323
|
-
"manifest-list": input.manifestPath,
|
|
5324
|
-
summary: {
|
|
5325
|
-
operation: "append",
|
|
5326
|
-
"added-data-files": String(input.manifest.files.length),
|
|
5327
|
-
"added-records": String(input.manifest.files.reduce((sum, file) => sum + file.recordCount, 0))
|
|
5328
|
-
}
|
|
5329
|
-
};
|
|
5330
|
-
}
|
|
5331
|
-
function equalityKey(row, columns) {
|
|
5332
|
-
return stableStringify(columns.map((column) => {
|
|
5333
|
-
if (!(column in row)) {
|
|
5334
|
-
throw new LaQLError("LAQL_UNKNOWN_COLUMN", `Unknown Iceberg equality delete column ${column}`, {
|
|
5335
|
-
column
|
|
5336
|
-
});
|
|
5337
|
-
}
|
|
5338
|
-
return jsonSafeValue(row[column]);
|
|
5339
|
-
}));
|
|
5340
|
-
}
|
|
5341
|
-
function appendManifestPath(metadataPath, jobId, snapshotId) {
|
|
5342
|
-
const slash = metadataPath.lastIndexOf("/");
|
|
5343
|
-
const prefix = slash === -1 ? "" : `${metadataPath.slice(0, slash + 1)}`;
|
|
5344
|
-
return `${prefix}${jobId ?? "append"}-${snapshotId}.manifest.json`;
|
|
5345
|
-
}
|
|
5346
|
-
function maxSequenceNumber(metadata) {
|
|
5347
|
-
let max = 0;
|
|
5348
|
-
for (const snapshot of metadata.snapshots) {
|
|
5349
|
-
for (const manifest of snapshotManifests(snapshot)) {
|
|
5350
|
-
for (const file of manifest.files)
|
|
5351
|
-
max = Math.max(max, file.sequenceNumber);
|
|
5352
|
-
}
|
|
5353
|
-
}
|
|
5354
|
-
return max;
|
|
5355
|
-
}
|
|
5356
|
-
function supportedIcebergDeleteFiles(deleteFiles) {
|
|
5357
|
-
const supported = [];
|
|
5358
|
-
for (const deleteFile of deleteFiles ?? []) {
|
|
5359
|
-
if (deleteFile.content === "position-delete" || deleteFile.content === "equality-delete") {
|
|
5360
|
-
supported.push({ content: deleteFile.content, path: deleteFile.path });
|
|
5361
|
-
}
|
|
5362
|
-
}
|
|
5363
|
-
return supported;
|
|
5364
|
-
}
|
|
5365
|
-
function unsupportedIcebergDeleteFiles(deleteFiles) {
|
|
5366
|
-
const supported = /* @__PURE__ */ new Set(["position-delete", "equality-delete"]);
|
|
5367
|
-
return (deleteFiles ?? []).filter((deleteFile) => !supported.has(deleteFile.content));
|
|
5368
|
-
}
|
|
5369
|
-
function cloneMetadata(metadata) {
|
|
5370
|
-
const cloned = {
|
|
5371
|
-
"format-version": metadata["format-version"],
|
|
5372
|
-
"table-uuid": metadata["table-uuid"],
|
|
5373
|
-
location: metadata.location,
|
|
5374
|
-
"current-snapshot-id": metadata["current-snapshot-id"],
|
|
5375
|
-
schemas: metadata.schemas.map((schema) => ({
|
|
5376
|
-
"schema-id": schema["schema-id"],
|
|
5377
|
-
fields: schema.fields.map((field) => ({ ...field }))
|
|
5378
|
-
})),
|
|
5379
|
-
snapshots: metadata.snapshots.map((snapshot) => {
|
|
5380
|
-
const cloned2 = {
|
|
5381
|
-
"snapshot-id": snapshot["snapshot-id"],
|
|
5382
|
-
"timestamp-ms": snapshot["timestamp-ms"],
|
|
5383
|
-
"schema-id": snapshot["schema-id"]
|
|
5384
|
-
};
|
|
5385
|
-
if (snapshot["manifest-list"] !== void 0)
|
|
5386
|
-
cloned2["manifest-list"] = snapshot["manifest-list"];
|
|
5387
|
-
if (snapshot.manifests !== void 0) {
|
|
5388
|
-
cloned2.manifests = snapshot.manifests.map(cloneManifestOrReference);
|
|
5389
|
-
}
|
|
5390
|
-
return cloned2;
|
|
5391
|
-
})
|
|
5392
|
-
};
|
|
5393
|
-
if (metadata.refs)
|
|
5394
|
-
cloned.refs = cloneRefs(metadata.refs);
|
|
5395
|
-
if (metadata["partition-specs"]) {
|
|
5396
|
-
cloned["partition-specs"] = metadata["partition-specs"].map((spec) => ({
|
|
5397
|
-
"spec-id": spec["spec-id"],
|
|
5398
|
-
fields: spec.fields.map((field) => ({ ...field }))
|
|
5399
|
-
}));
|
|
5400
|
-
}
|
|
5401
|
-
if (metadata["sort-orders"]) {
|
|
5402
|
-
cloned["sort-orders"] = metadata["sort-orders"].map((order) => ({
|
|
5403
|
-
"order-id": order["order-id"],
|
|
5404
|
-
fields: order.fields.map((field) => field)
|
|
5405
|
-
}));
|
|
5406
|
-
}
|
|
5407
|
-
return cloned;
|
|
5408
|
-
}
|
|
5409
|
-
function cloneRefs(refs) {
|
|
5410
|
-
const out = {};
|
|
5411
|
-
for (const [name, ref] of Object.entries(refs)) {
|
|
5412
|
-
out[name] = { type: ref.type, "snapshot-id": ref["snapshot-id"] };
|
|
5413
|
-
}
|
|
5414
|
-
return out;
|
|
5415
|
-
}
|
|
5416
|
-
function cloneManifest(manifest) {
|
|
5417
|
-
const cloned = {
|
|
5418
|
-
path: manifest.path,
|
|
5419
|
-
files: manifest.files.map((file) => {
|
|
5420
|
-
const cloned2 = {
|
|
5421
|
-
path: file.path,
|
|
5422
|
-
sequenceNumber: file.sequenceNumber,
|
|
5423
|
-
partition: sortStringRecord(file.partition ?? {}),
|
|
5424
|
-
recordCount: file.recordCount
|
|
5425
|
-
};
|
|
5426
|
-
if (file.fileSizeInBytes !== void 0)
|
|
5427
|
-
cloned2.fileSizeInBytes = file.fileSizeInBytes;
|
|
5428
|
-
if (file.deleteFiles !== void 0) {
|
|
5429
|
-
cloned2.deleteFiles = file.deleteFiles.map((deleteFile) => ({ ...deleteFile }));
|
|
5430
|
-
}
|
|
5431
|
-
return cloned2;
|
|
5432
|
-
})
|
|
5433
|
-
};
|
|
5434
|
-
if (manifest.deleteFiles !== void 0) {
|
|
5435
|
-
cloned.deleteFiles = manifest.deleteFiles.map((deleteFile) => ({ ...deleteFile }));
|
|
5436
|
-
}
|
|
5437
|
-
return cloned;
|
|
5438
|
-
}
|
|
5439
|
-
function cloneManifestOrReference(manifest) {
|
|
5440
|
-
if (!Array.isArray(manifest.files)) {
|
|
5441
|
-
return { path: manifest.path };
|
|
5442
|
-
}
|
|
5443
|
-
return cloneManifest(manifest);
|
|
5444
|
-
}
|
|
5445
|
-
function snapshotManifests(snapshot) {
|
|
5446
|
-
return snapshot.manifests ?? [];
|
|
5447
|
-
}
|
|
5448
|
-
function sortStringRecord(record) {
|
|
5449
|
-
const out = {};
|
|
5450
|
-
for (const key of Object.keys(record).sort())
|
|
5451
|
-
out[key] = record[key] ?? "";
|
|
5452
|
-
return out;
|
|
5453
|
-
}
|
|
5454
|
-
function validateListTablesResponse(value) {
|
|
5455
|
-
if (!isRecord4(value) || !Array.isArray(value.identifiers)) {
|
|
5456
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "Invalid Iceberg REST list tables response");
|
|
5457
|
-
}
|
|
5458
|
-
return value.identifiers.map((identifier) => {
|
|
5459
|
-
if (!isRecord4(identifier) || !Array.isArray(identifier.namespace) || !identifier.namespace.every((part) => typeof part === "string") || typeof identifier.name !== "string") {
|
|
5460
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "Invalid Iceberg REST table identifier");
|
|
5461
|
-
}
|
|
5462
|
-
return { namespace: identifier.namespace, name: identifier.name };
|
|
5463
|
-
});
|
|
5464
|
-
}
|
|
5465
|
-
function validateMetadata(value) {
|
|
5466
|
-
if (!isRecord4(value))
|
|
5467
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg metadata must be an object");
|
|
5468
|
-
if (value["format-version"] !== 1 && value["format-version"] !== 2) {
|
|
5469
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "Only Iceberg format-version 1 and 2 metadata is supported for reads", {
|
|
5470
|
-
formatVersion: value["format-version"]
|
|
5471
|
-
});
|
|
5472
|
-
}
|
|
5473
|
-
if (!Array.isArray(value.snapshots) || !Array.isArray(value.schemas)) {
|
|
5474
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg metadata is missing snapshots or schemas");
|
|
5475
|
-
}
|
|
5476
|
-
if (!isMetadataFile(value)) {
|
|
5477
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg metadata has invalid required fields");
|
|
5478
|
-
}
|
|
5479
|
-
rejectUnsupportedMetadataFeatures(value);
|
|
5480
|
-
return value;
|
|
5481
|
-
}
|
|
5482
|
-
function rejectUnsupportedMetadataFeatures(metadata) {
|
|
5483
|
-
rejectAdvertisedFeatureFlags(metadata);
|
|
5484
|
-
rejectUnsupportedPartitionTransforms(metadata["partition-specs"]);
|
|
5485
|
-
rejectUnsupportedSortOrders(metadata["sort-orders"]);
|
|
5486
|
-
}
|
|
5487
|
-
function rejectAdvertisedFeatureFlags(metadata) {
|
|
5488
|
-
for (const key of ["features", "table-features", "format-version-features"]) {
|
|
5489
|
-
const features = metadata[key];
|
|
5490
|
-
if (features === void 0 || Array.isArray(features) && features.length === 0)
|
|
5491
|
-
continue;
|
|
5492
|
-
throw new LaQLError("LAQL_UNSUPPORTED_ICEBERG_FEATURE", "Iceberg metadata advertises unsupported table-format features", {
|
|
5493
|
-
featureProperty: key,
|
|
5494
|
-
features
|
|
5495
|
-
});
|
|
5496
|
-
}
|
|
5497
|
-
}
|
|
5498
|
-
function rejectUnsupportedPartitionTransforms(partitionSpecs) {
|
|
5499
|
-
if (partitionSpecs === void 0)
|
|
5500
|
-
return;
|
|
5501
|
-
if (!Array.isArray(partitionSpecs)) {
|
|
5502
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg partition-specs must be an array");
|
|
5503
|
-
}
|
|
5504
|
-
for (const spec of partitionSpecs) {
|
|
5505
|
-
if (!isRecord4(spec) || !Array.isArray(spec.fields)) {
|
|
5506
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg partition spec is invalid");
|
|
5507
|
-
}
|
|
5508
|
-
for (const field of spec.fields) {
|
|
5509
|
-
if (!isRecord4(field) || typeof field.transform !== "string") {
|
|
5510
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg partition field is invalid");
|
|
5511
|
-
}
|
|
5512
|
-
if (field.transform !== "identity" && field.transform !== "void") {
|
|
5513
|
-
throw new LaQLError("LAQL_UNSUPPORTED_ICEBERG_FEATURE", "Iceberg partition transform is not supported for strict planning", {
|
|
5514
|
-
specId: spec["spec-id"],
|
|
5515
|
-
fieldName: field.name,
|
|
5516
|
-
transform: field.transform
|
|
5517
|
-
});
|
|
5518
|
-
}
|
|
5519
|
-
}
|
|
5520
|
-
}
|
|
5521
|
-
}
|
|
5522
|
-
function rejectUnsupportedSortOrders(sortOrders) {
|
|
5523
|
-
if (sortOrders === void 0)
|
|
5524
|
-
return;
|
|
5525
|
-
if (!Array.isArray(sortOrders)) {
|
|
5526
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg sort-orders must be an array");
|
|
5527
|
-
}
|
|
5528
|
-
for (const order of sortOrders) {
|
|
5529
|
-
if (!isRecord4(order) || !Array.isArray(order.fields)) {
|
|
5530
|
-
throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg sort order is invalid");
|
|
5531
|
-
}
|
|
5532
|
-
if (order.fields.length > 0) {
|
|
5533
|
-
throw new LaQLError("LAQL_UNSUPPORTED_ICEBERG_FEATURE", "Iceberg sorted table metadata is not supported for strict planning", {
|
|
5534
|
-
orderId: order["order-id"],
|
|
5535
|
-
fields: order.fields
|
|
5536
|
-
});
|
|
5537
|
-
}
|
|
5538
|
-
}
|
|
5539
|
-
}
|
|
5540
|
-
function isMetadataFile(value) {
|
|
5541
|
-
if (!isRecord4(value))
|
|
5542
|
-
return false;
|
|
5543
|
-
return (value["format-version"] === 1 || value["format-version"] === 2) && typeof value["table-uuid"] === "string" && typeof value.location === "string" && typeof value["current-snapshot-id"] === "number" && Array.isArray(value.refs) === false && Array.isArray(value.schemas) && Array.isArray(value.snapshots);
|
|
5544
|
-
}
|
|
5545
|
-
function projectedIds(fields, select) {
|
|
5546
|
-
if (!select)
|
|
5547
|
-
return fields.map((field) => field.id).sort((a, b) => a - b);
|
|
5548
|
-
return select.map((name) => {
|
|
5549
|
-
const field = fields.find((candidate) => candidate.name === name);
|
|
5550
|
-
if (!field) {
|
|
5551
|
-
throw new LaQLError("LAQL_UNKNOWN_COLUMN", `Unknown Iceberg column ${name}`, {
|
|
5552
|
-
column: name
|
|
5553
|
-
});
|
|
5554
|
-
}
|
|
5555
|
-
return field.sourceId ?? field.id;
|
|
5556
|
-
}).sort((a, b) => a - b);
|
|
5557
|
-
}
|
|
5558
|
-
function partitionMayMatch2(expr, partition) {
|
|
5559
|
-
if (!expr)
|
|
5560
|
-
return true;
|
|
5561
|
-
const columns = /* @__PURE__ */ new Set();
|
|
5562
|
-
collectColumns2(expr, columns);
|
|
5563
|
-
if (columns.size === 0 || [...columns].some((column) => !(column in partition)))
|
|
5564
|
-
return true;
|
|
5565
|
-
try {
|
|
5566
|
-
return matches(expr, partition);
|
|
5567
|
-
} catch (cause) {
|
|
5568
|
-
if (cause instanceof LaQLError && cause.code === "LAQL_TYPE_ERROR")
|
|
5569
|
-
return true;
|
|
5570
|
-
throw cause;
|
|
5571
|
-
}
|
|
5572
|
-
}
|
|
5573
|
-
function collectColumns2(expr, columns) {
|
|
5574
|
-
switch (expr.kind) {
|
|
5575
|
-
case "column":
|
|
5576
|
-
columns.add(expr.name);
|
|
5577
|
-
return;
|
|
5578
|
-
case "literal":
|
|
5579
|
-
return;
|
|
5580
|
-
case "compare":
|
|
5581
|
-
collectColumns2(expr.left, columns);
|
|
5582
|
-
collectColumns2(expr.right, columns);
|
|
5583
|
-
return;
|
|
5584
|
-
case "in":
|
|
5585
|
-
collectColumns2(expr.target, columns);
|
|
5586
|
-
for (const value of expr.values)
|
|
5587
|
-
collectColumns2(value, columns);
|
|
5588
|
-
return;
|
|
5589
|
-
case "between":
|
|
5590
|
-
collectColumns2(expr.target, columns);
|
|
5591
|
-
collectColumns2(expr.low, columns);
|
|
5592
|
-
collectColumns2(expr.high, columns);
|
|
5593
|
-
return;
|
|
5594
|
-
case "null-check":
|
|
5595
|
-
collectColumns2(expr.target, columns);
|
|
5596
|
-
return;
|
|
5597
|
-
case "logical":
|
|
5598
|
-
for (const operand of expr.operands)
|
|
5599
|
-
collectColumns2(operand, columns);
|
|
5600
|
-
return;
|
|
5601
|
-
case "not":
|
|
5602
|
-
collectColumns2(expr.operand, columns);
|
|
5603
|
-
return;
|
|
5604
|
-
case "like":
|
|
5605
|
-
collectColumns2(expr.target, columns);
|
|
5606
|
-
return;
|
|
5607
|
-
case "call":
|
|
5608
|
-
for (const arg of expr.args)
|
|
5609
|
-
collectColumns2(arg, columns);
|
|
5610
|
-
return;
|
|
5611
|
-
}
|
|
5612
|
-
}
|
|
5613
|
-
function isRecord4(value) {
|
|
5614
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
5615
|
-
}
|
|
5616
4541
|
async function asyncBufferFromStore(store, path, options = void 0) {
|
|
5617
4542
|
const controlledStore = options === void 0 ? store : withObjectStoreReadControls(store, options.budget);
|
|
5618
4543
|
throwIfAborted(options?.budget.signal);
|
|
@@ -5847,7 +4772,7 @@ function partitionedParquetOutputEntries(result, options) {
|
|
|
5847
4772
|
const entry = {
|
|
5848
4773
|
taskId: typeof options.taskId === "function" ? options.taskId(file, index) : options.taskId,
|
|
5849
4774
|
outputPath: file.path,
|
|
5850
|
-
partitionValues:
|
|
4775
|
+
partitionValues: sortStringRecord(file.partitionValues),
|
|
5851
4776
|
rowCount: file.rowCount,
|
|
5852
4777
|
byteSize: file.byteSize,
|
|
5853
4778
|
contentHash: file.contentHash
|
|
@@ -5858,7 +4783,7 @@ function partitionedParquetOutputEntries(result, options) {
|
|
|
5858
4783
|
entry.iceberg = {
|
|
5859
4784
|
recordCount: file.rowCount,
|
|
5860
4785
|
fileSizeInBytes: file.byteSize,
|
|
5861
|
-
partitionValues:
|
|
4786
|
+
partitionValues: sortStringRecord(file.partitionValues)
|
|
5862
4787
|
};
|
|
5863
4788
|
}
|
|
5864
4789
|
return entry;
|
|
@@ -5872,7 +4797,7 @@ function outputEntriesToPartitionedResult(entries) {
|
|
|
5872
4797
|
byteSize: entry.byteSize,
|
|
5873
4798
|
contentHash: entry.contentHash ?? "",
|
|
5874
4799
|
rowCount: entry.rowCount,
|
|
5875
|
-
partitionValues:
|
|
4800
|
+
partitionValues: sortStringRecord(entry.partitionValues)
|
|
5876
4801
|
};
|
|
5877
4802
|
if (entry.etag !== void 0)
|
|
5878
4803
|
file.etag = entry.etag;
|
|
@@ -6288,7 +5213,7 @@ function compareInsertValues(left, right, column, rowIndex) {
|
|
|
6288
5213
|
return 1;
|
|
6289
5214
|
return 0;
|
|
6290
5215
|
}
|
|
6291
|
-
function
|
|
5216
|
+
function sortStringRecord(record) {
|
|
6292
5217
|
const out = {};
|
|
6293
5218
|
for (const key of Object.keys(record).sort())
|
|
6294
5219
|
out[key] = record[key] ?? "";
|
|
@@ -6472,6 +5397,8 @@ function rowGroupMayMatch(rowGroup, expr) {
|
|
|
6472
5397
|
case "column":
|
|
6473
5398
|
case "null-check":
|
|
6474
5399
|
case "like":
|
|
5400
|
+
case "arithmetic":
|
|
5401
|
+
case "case":
|
|
6475
5402
|
return true;
|
|
6476
5403
|
case "call":
|
|
6477
5404
|
return callMayMatch2(rowGroup, expr);
|
|
@@ -6747,78 +5674,4 @@ function createParquetLake(config) {
|
|
|
6747
5674
|
});
|
|
6748
5675
|
}
|
|
6749
5676
|
|
|
6750
|
-
|
|
6751
|
-
async function loadTable(options) {
|
|
6752
|
-
if (options.format === "parquet") {
|
|
6753
|
-
return { format: "parquet", store: options.store, path: options.path };
|
|
6754
|
-
}
|
|
6755
|
-
const { format: _format, ...icebergOptions } = options;
|
|
6756
|
-
return {
|
|
6757
|
-
format: "iceberg",
|
|
6758
|
-
store: options.store,
|
|
6759
|
-
table: await loadIcebergTable(icebergOptions)
|
|
6760
|
-
};
|
|
6761
|
-
}
|
|
6762
|
-
function planFiles2(table, options = {}) {
|
|
6763
|
-
if (table.format === "parquet") {
|
|
6764
|
-
return { format: "parquet", store: table.store, files: [{ path: table.path }] };
|
|
6765
|
-
}
|
|
6766
|
-
return {
|
|
6767
|
-
format: "iceberg",
|
|
6768
|
-
store: table.store,
|
|
6769
|
-
table: table.table,
|
|
6770
|
-
plan: planFiles(table.table, options),
|
|
6771
|
-
options
|
|
6772
|
-
};
|
|
6773
|
-
}
|
|
6774
|
-
async function* scanBatches(plan, options = {}) {
|
|
6775
|
-
if (plan.format === "parquet") {
|
|
6776
|
-
for (const file of plan.files) {
|
|
6777
|
-
const readOptions = {};
|
|
6778
|
-
if (options.batchSize !== void 0) readOptions.batchSize = options.batchSize;
|
|
6779
|
-
for await (const batch of readParquetObjectBatches(plan.store, file.path, readOptions)) {
|
|
6780
|
-
yield batch.rows;
|
|
6781
|
-
}
|
|
6782
|
-
}
|
|
6783
|
-
return;
|
|
6784
|
-
}
|
|
6785
|
-
for await (const batch of scanPlannedIcebergRows({
|
|
6786
|
-
plan: plan.plan,
|
|
6787
|
-
...options,
|
|
6788
|
-
readDataFile: async (file) => projectIcebergParquetBatches(
|
|
6789
|
-
plan.store,
|
|
6790
|
-
plan.table,
|
|
6791
|
-
file.path,
|
|
6792
|
-
file.partition,
|
|
6793
|
-
file.snapshotId,
|
|
6794
|
-
plan.options,
|
|
6795
|
-
options
|
|
6796
|
-
),
|
|
6797
|
-
readDeleteFile: async (deleteFile) => readIcebergParquetDeletes(plan.store, deleteFile)
|
|
6798
|
-
})) {
|
|
6799
|
-
yield batch;
|
|
6800
|
-
}
|
|
6801
|
-
}
|
|
6802
|
-
async function* scanRows(plan, options = {}) {
|
|
6803
|
-
for await (const batch of scanBatches(plan, options)) {
|
|
6804
|
-
for (const row of batch) {
|
|
6805
|
-
yield row;
|
|
6806
|
-
}
|
|
6807
|
-
}
|
|
6808
|
-
}
|
|
6809
|
-
async function* projectIcebergParquetBatches(store, table, path, partition, snapshotId, planOptions, scanOptions) {
|
|
6810
|
-
const readOptions = {};
|
|
6811
|
-
if (scanOptions.batchSize !== void 0) readOptions.batchSize = scanOptions.batchSize;
|
|
6812
|
-
for await (const batch of readParquetObjectBatches(store, path, readOptions)) {
|
|
6813
|
-
yield {
|
|
6814
|
-
rowOffset: batch.rowOffset,
|
|
6815
|
-
rows: batch.rows.map((row) => {
|
|
6816
|
-
const projectOptions = { snapshotId };
|
|
6817
|
-
if (planOptions.select !== void 0) projectOptions.select = planOptions.select;
|
|
6818
|
-
return table.projectRow({ ...partition, ...row }, projectOptions);
|
|
6819
|
-
})
|
|
6820
|
-
};
|
|
6821
|
-
}
|
|
6822
|
-
}
|
|
6823
|
-
|
|
6824
|
-
export { AggregationBuilder, CacheApiCache, ERROR_CODES, IcebergRestCatalog, IcebergTable, IcebergUnsupportedCatalog, LaQLError, Lake, MemoryCache, MemoryCheckpointAdapter, MemoryObjectStore, MemorySpillAdapter, ObjectStoreIcebergCommitCatalog, PACKAGE, ParquetScanAdapter, QueryBuilder, QueryResult, ResumedQuery, advanceTaskCheckpoint, and, applyIcebergDeletes, assertBookmarkMatches, asyncBufferFromStore, bboxMayIntersect, between, broadcastJoin, buildBBoxIndex, buildMinMaxIndex, cacheApiCache, classifyPredicate, col, createBookmark, createOutputManifest, createOutputManifestFromCheckpoints, createParquetLake, createParquetTableAs, createTaskManifest, deserializeAggregateOperatorState, deserializeSortOperatorState, deserializeTopKOperatorState, encodeJsonLine, eq, evaluate, fingerprint, fn, gt, gte, icebergGlueCatalog, icebergNessieCatalog, icebergRestCatalog, ilike, isIn, isLaQLError, isNotNull, isNull, jsonSafeValue, like, lit, loadIcebergTable, loadIcebergTableFromObjectStore, loadIcebergTableFromRest, loadTable, lookupJoin, lt, lte, matches, memoryCache, memoryCheckpointAdapter, memorySpillAdapter, memoryStore, ne, not, notIn, or, parquetScanner, parseHivePartitions, parseJsonQuery, partitionedParquetOutputEntries, planFiles2 as planFiles, planRowGroups, planRowGroupsFromMetadata, pruneFilesWithIndex, readControlSignal, readIcebergParquetDeletes, readOutputManifest, readParquetMetadata, readParquetObjectBatches, readParquetObjects, rejectUnsupportedParquetSchema, rowGroupMayMatch, scanBatches, scanPlannedIcebergRows, scanRows, serializeAggregateOperatorState, serializeSortOperatorState, serializeTopKOperatorState, signPaginationToken, stableStringify, throwIfAborted, transitionTaskCheckpoint, verifyPaginationToken, withObjectStoreReadControls, writeOutputManifest, writeParquet, writePartitionedParquet, writePartitionedParquetTask };
|
|
5677
|
+
export { AggregationBuilder, CacheApiCache, ERROR_CODES, LaQLError, Lake, MemoryCache, MemoryCheckpointAdapter, MemoryObjectStore, MemorySpillAdapter, ParquetScanAdapter, QueryBuilder, QueryResult, ResumedQuery, __commonJS, __toESM, add, advanceTaskCheckpoint, and, assertBookmarkMatches, asyncBufferFromStore, bboxMayIntersect, between, broadcastJoin, buildBBoxIndex, buildMinMaxIndex, cacheApiCache, classifyPredicate, col, createBookmark, createOutputManifest, createOutputManifestFromCheckpoints, createParquetLake, createParquetTableAs, createTaskManifest, deserializeAggregateOperatorState, deserializeSortOperatorState, deserializeTopKOperatorState, div, encodeJsonLine, eq, evaluate, fingerprint, fn, gt, gte, ilike, isIn, isLaQLError, isNotNull, isNull, jsonSafeValue, like, lit, lookupJoin, lt, lte, matches, memoryCache, memoryCheckpointAdapter, memorySpillAdapter, memoryStore, mod, mul, ne, not, notIn, or, parquetScanner, parseHivePartitions, parseJsonQuery, partitionedParquetOutputEntries, planRowGroups, planRowGroupsFromMetadata, pruneFilesWithIndex, readControlSignal, readIcebergParquetDeletes, readOutputManifest, readParquetMetadata, readParquetObjectBatches, readParquetObjects, rejectUnsupportedParquetSchema, rowGroupMayMatch, serializeAggregateOperatorState, serializeSortOperatorState, serializeTopKOperatorState, signPaginationToken, stableStringify, sub, throwIfAborted, transitionTaskCheckpoint, verifyPaginationToken, withObjectStoreReadControls, writeOutputManifest, writeParquet, writePartitionedParquet, writePartitionedParquetTask };
|