lakeql 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,9 +4,36 @@ import { gridDisk, latLngToCell, cellToParent, isValidCell } from 'h3-js';
4
4
  import { parquetMetadataAsync, parquetReadObjects } from 'hyparquet';
5
5
  import { parquetWriteBuffer } from 'hyparquet-writer';
6
6
 
7
+ var __create = Object.create;
8
+ var __defProp = Object.defineProperty;
9
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
10
+ var __getOwnPropNames = Object.getOwnPropertyNames;
11
+ var __getProtoOf = Object.getPrototypeOf;
12
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
13
+ var __commonJS = (cb, mod2) => function __require() {
14
+ return mod2 || (0, cb[__getOwnPropNames(cb)[0]])((mod2 = { exports: {} }).exports, mod2), mod2.exports;
15
+ };
16
+ var __copyProps = (to, from, except, desc) => {
17
+ if (from && typeof from === "object" || typeof from === "function") {
18
+ for (let key of __getOwnPropNames(from))
19
+ if (!__hasOwnProp.call(to, key) && key !== except)
20
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
21
+ }
22
+ return to;
23
+ };
24
+ var __toESM = (mod2, isNodeMode, target) => (target = mod2 != null ? __create(__getProtoOf(mod2)) : {}, __copyProps(
25
+ // If the importer is in node compatibility mode or this is not an ESM
26
+ // file that has been converted to a CommonJS file using a Babel-
27
+ // compatible transform (i.e. "__esModule" has not been set), then set
28
+ // "default" to the CommonJS "module.exports" for node compatibility.
29
+ isNodeMode || !mod2 || !mod2.__esModule ? __defProp(target, "default", { value: mod2, enumerable: true }) : target,
30
+ mod2
31
+ ));
32
+
7
33
  // ../core/dist/errors.js
8
34
  var ERROR_CODES = [
9
35
  "LAQL_PARSE_ERROR",
36
+ "LAQL_SQL_UNSUPPORTED",
10
37
  "LAQL_TYPE_ERROR",
11
38
  "LAQL_UNKNOWN_TABLE",
12
39
  "LAQL_UNKNOWN_COLUMN",
@@ -71,6 +98,14 @@ function evaluate(expr, row) {
71
98
  return likeMatch(evaluate(expr.target, row), expr.pattern, expr.caseInsensitive);
72
99
  case "call":
73
100
  return callFunction(expr.fn, expr.args.map((arg) => evaluate(arg, row)));
101
+ case "arithmetic":
102
+ return arithmetic(expr.op, evaluate(expr.left, row), evaluate(expr.right, row));
103
+ case "case":
104
+ for (const branch of expr.whens) {
105
+ if (toSqlBoolean(evaluate(branch.when, row)) === true)
106
+ return evaluate(branch.value, row);
107
+ }
108
+ return expr.else === void 0 ? null : evaluate(expr.else, row);
74
109
  }
75
110
  }
76
111
  function matches(expr, row) {
@@ -144,6 +179,28 @@ function compare(op, left, right) {
144
179
  return order >= 0;
145
180
  }
146
181
  }
182
+ function arithmetic(op, left, right) {
183
+ if (left === null || right === null)
184
+ return null;
185
+ if (typeof left !== "number" || typeof right !== "number") {
186
+ throw new LaQLError("LAQL_TYPE_ERROR", "Arithmetic expressions require numeric values", {
187
+ leftType: typeof left,
188
+ rightType: typeof right
189
+ });
190
+ }
191
+ switch (op) {
192
+ case "add":
193
+ return left + right;
194
+ case "sub":
195
+ return left - right;
196
+ case "mul":
197
+ return left * right;
198
+ case "div":
199
+ return left / right;
200
+ case "mod":
201
+ return left % right;
202
+ }
203
+ }
147
204
  function isNumberLike(value) {
148
205
  return typeof value === "number" || typeof value === "bigint";
149
206
  }
@@ -865,6 +922,21 @@ function ilike(column, pattern) {
865
922
  function fn(name, ...args) {
866
923
  return { kind: "call", fn: name, args: args.map(toValue) };
867
924
  }
925
+ function add(left, right) {
926
+ return { kind: "arithmetic", op: "add", left: toValue(left), right: toValue(right) };
927
+ }
928
+ function sub(left, right) {
929
+ return { kind: "arithmetic", op: "sub", left: toValue(left), right: toValue(right) };
930
+ }
931
+ function mul(left, right) {
932
+ return { kind: "arithmetic", op: "mul", left: toValue(left), right: toValue(right) };
933
+ }
934
+ function div(left, right) {
935
+ return { kind: "arithmetic", op: "div", left: toValue(left), right: toValue(right) };
936
+ }
937
+ function mod(left, right) {
938
+ return { kind: "arithmetic", op: "mod", left: toValue(left), right: toValue(right) };
939
+ }
868
940
 
869
941
  // ../core/dist/manifest.js
870
942
  function createTaskManifest(input) {
@@ -1661,7 +1733,15 @@ async function collectRightRows(rows, maxRightRows) {
1661
1733
  return out;
1662
1734
  }
1663
1735
  function validateJoinOptions(options, strategy) {
1664
- if (!options.leftKey || !options.rightKey) {
1736
+ const leftKeys = normalizeJoinKeys(options.leftKey, "leftKey", strategy);
1737
+ const rightKeys = normalizeJoinKeys(options.rightKey, "rightKey", strategy);
1738
+ if (leftKeys.length !== rightKeys.length) {
1739
+ throw new LaQLError("LAQL_TYPE_ERROR", `${strategy} join key counts must match`, {
1740
+ leftKey: options.leftKey,
1741
+ rightKey: options.rightKey
1742
+ });
1743
+ }
1744
+ if (leftKeys.length === 0) {
1665
1745
  throw new LaQLError("LAQL_TYPE_ERROR", `${strategy} join requires leftKey and rightKey`);
1666
1746
  }
1667
1747
  if (!Number.isInteger(options.maxRightRows) || options.maxRightRows < 1) {
@@ -1673,10 +1753,24 @@ function validateJoinOptions(options, strategy) {
1673
1753
  });
1674
1754
  }
1675
1755
  }
1756
+ function normalizeJoinKeys(key, label, strategy) {
1757
+ const keys = Array.isArray(key) ? key : [key];
1758
+ if (keys.some((column) => typeof column !== "string" || column.length === 0)) {
1759
+ throw new LaQLError("LAQL_TYPE_ERROR", `${strategy} join ${label} must contain column names`, {
1760
+ [label]: key
1761
+ });
1762
+ }
1763
+ return keys;
1764
+ }
1676
1765
  function joinKey(row, column) {
1677
1766
  return stableStringify(joinValue(row, column));
1678
1767
  }
1679
1768
  function joinValue(row, column) {
1769
+ if (Array.isArray(column))
1770
+ return column.map((key) => scalarJoinValue(row, key));
1771
+ return scalarJoinValue(row, column);
1772
+ }
1773
+ function scalarJoinValue(row, column) {
1680
1774
  if (!(column in row)) {
1681
1775
  throw new LaQLError("LAQL_UNKNOWN_COLUMN", `Unknown join key ${column}`, { column });
1682
1776
  }
@@ -1691,8 +1785,10 @@ function joinValue(row, column) {
1691
1785
  function mergeRows(left, right, options) {
1692
1786
  const out = { ...left };
1693
1787
  const prefix = options.rightPrefix ?? "right.";
1788
+ const leftKeys = normalizeJoinKeys(options.leftKey, "leftKey", "Merge");
1789
+ const rightKeys = normalizeJoinKeys(options.rightKey, "rightKey", "Merge");
1694
1790
  for (const [key, value] of Object.entries(right)) {
1695
- if (key === options.rightKey && options.leftKey === options.rightKey)
1791
+ if (rightKeys.includes(key) && leftKeys.includes(key))
1696
1792
  continue;
1697
1793
  const outKey = key in out ? `${prefix}${key}` : key;
1698
1794
  out[outKey] = value;
@@ -1907,6 +2003,18 @@ function collectColumns(expr, columns) {
1907
2003
  for (const arg of expr.args)
1908
2004
  collectColumns(arg, columns);
1909
2005
  return;
2006
+ case "arithmetic":
2007
+ collectColumns(expr.left, columns);
2008
+ collectColumns(expr.right, columns);
2009
+ return;
2010
+ case "case":
2011
+ for (const branch of expr.whens) {
2012
+ collectColumns(branch.when, columns);
2013
+ collectColumns(branch.value, columns);
2014
+ }
2015
+ if (expr.else !== void 0)
2016
+ collectColumns(expr.else, columns);
2017
+ return;
1910
2018
  }
1911
2019
  }
1912
2020
 
@@ -1982,6 +2090,8 @@ function fileMayMatch(file, expr) {
1982
2090
  case "column":
1983
2091
  case "null-check":
1984
2092
  case "like":
2093
+ case "arithmetic":
2094
+ case "case":
1985
2095
  case "call":
1986
2096
  return callMayMatch(file, expr);
1987
2097
  case "not":
@@ -2239,9 +2349,15 @@ var QueryBuilder = class _QueryBuilder {
2239
2349
  select(columns) {
2240
2350
  return new _QueryBuilder(this.lake, { ...this.init, select: columns });
2241
2351
  }
2352
+ project(projections) {
2353
+ return new _QueryBuilder(this.lake, { ...this.init, projections });
2354
+ }
2242
2355
  where(expr) {
2243
2356
  return new _QueryBuilder(this.lake, { ...this.init, where: expr });
2244
2357
  }
2358
+ distinct(enabled = true) {
2359
+ return new _QueryBuilder(this.lake, { ...this.init, distinct: enabled });
2360
+ }
2245
2361
  orderBy(terms) {
2246
2362
  return new _QueryBuilder(this.lake, { ...this.init, orderBy: normalizeOrderBy(terms) });
2247
2363
  }
@@ -2385,9 +2501,10 @@ var QueryResult = class {
2385
2501
  const startedAt = config.now();
2386
2502
  let offsetSkipped = 0;
2387
2503
  let returned = 0;
2504
+ const distinct = config.distinct === true ? /* @__PURE__ */ new Set() : void 0;
2388
2505
  const { planned: paths, skipped: skippedFiles } = await this.planObjects();
2389
2506
  stats.filesSkipped = skippedFiles;
2390
- const columns = projectedReadColumns(config.select, config.where);
2507
+ const columns = projectedReadColumns(config.select, config.where, void 0, config.projections);
2391
2508
  for (const object of paths) {
2392
2509
  stats.filesPlanned += 1;
2393
2510
  stats.filesRead += 1;
@@ -2416,13 +2533,17 @@ var QueryResult = class {
2416
2533
  if (!matches(config.where, row))
2417
2534
  continue;
2418
2535
  stats.rowsMatched += 1;
2536
+ const projected = project(row, config.select, config.projections);
2537
+ if (distinct !== void 0 && !addDistinctRow(distinct, projected, config.budget)) {
2538
+ continue;
2539
+ }
2419
2540
  if (offsetSkipped < (config.offset ?? 0)) {
2420
2541
  offsetSkipped += 1;
2421
2542
  continue;
2422
2543
  }
2423
2544
  if (config.limit !== void 0 && returned >= config.limit)
2424
2545
  break;
2425
- out.push(project(row, config.select));
2546
+ out.push(projected);
2426
2547
  returned += 1;
2427
2548
  stats.rowsReturned += 1;
2428
2549
  enforceBudget(config.budget, stats, config.now, startedAt);
@@ -2451,6 +2572,9 @@ var QueryResult = class {
2451
2572
  if (config.limit === void 0) {
2452
2573
  throw new LaQLError("LAQL_TYPE_ERROR", "topKWithState requires limit");
2453
2574
  }
2575
+ if (config.distinct === true) {
2576
+ throw new LaQLError("LAQL_TYPE_ERROR", "topKWithState does not support distinct queries");
2577
+ }
2454
2578
  const orderBy = normalizeOrderBy(config.orderBy);
2455
2579
  const topK = (config.offset ?? 0) + config.limit;
2456
2580
  const matched = await topKRowsFromState(orderBy, config.offset ?? 0, config.limit, options);
@@ -2461,7 +2585,7 @@ var QueryResult = class {
2461
2585
  matched.sort((left, right) => compareRows(left, right, orderBy));
2462
2586
  const start = config.offset ?? 0;
2463
2587
  const end = start + config.limit;
2464
- const rows = matched.slice(start, end).map((row) => project(row, config.select));
2588
+ const rows = matched.slice(start, end).map((row) => project(row, config.select, config.projections));
2465
2589
  for (const _row of rows) {
2466
2590
  this.stats.rowsReturned += 1;
2467
2591
  enforceBudget(config.budget, this.stats, config.now, startedAt);
@@ -2480,6 +2604,9 @@ var QueryResult = class {
2480
2604
  if (config.orderBy === void 0) {
2481
2605
  throw new LaQLError("LAQL_TYPE_ERROR", "sortWithState requires orderBy");
2482
2606
  }
2607
+ if (config.distinct === true) {
2608
+ throw new LaQLError("LAQL_TYPE_ERROR", "sortWithState does not support distinct queries");
2609
+ }
2483
2610
  const orderBy = normalizeOrderBy(config.orderBy);
2484
2611
  const runs = await sortRunsFromState(orderBy, options);
2485
2612
  const startedAt = config.now();
@@ -2487,7 +2614,7 @@ var QueryResult = class {
2487
2614
  const matched = mergeSortRuns(runs, orderBy);
2488
2615
  const start = config.offset ?? 0;
2489
2616
  const end = config.limit === void 0 ? matched.length : start + config.limit;
2490
- const rows = matched.slice(start, end).map((row) => project(row, config.select));
2617
+ const rows = matched.slice(start, end).map((row) => project(row, config.select, config.projections));
2491
2618
  for (const _row of rows) {
2492
2619
  this.stats.rowsReturned += 1;
2493
2620
  enforceBudget(config.budget, this.stats, config.now, startedAt);
@@ -2599,7 +2726,7 @@ var QueryResult = class {
2599
2726
  group.add(row);
2600
2727
  enforceOperatorMemoryBudget(this.config.budget, estimateAggregateOperatorMemoryBytes(groupColumns, spec, groups));
2601
2728
  }
2602
- const rows = [...groups.values()].map((group) => group.finish());
2729
+ const rows = applyAggregateResultOptions([...groups.values()].map((group) => group.finish()), options);
2603
2730
  for (const _row of rows) {
2604
2731
  this.stats.rowsReturned += 1;
2605
2732
  enforceBudget(this.config.budget, this.stats, this.config.now, startedAt);
@@ -2621,16 +2748,27 @@ var QueryResult = class {
2621
2748
  const config = this.config;
2622
2749
  const { stats } = this;
2623
2750
  const matched = [];
2624
- const topK = config.limit === void 0 ? void 0 : (config.offset ?? 0) + config.limit;
2751
+ const topK = config.distinct === true || config.limit === void 0 ? void 0 : (config.offset ?? 0) + config.limit;
2625
2752
  const startedAt = config.now();
2626
2753
  await this.collectOrderedMatches(matched, topK, startedAt);
2627
2754
  matched.sort((left, right) => compareRows(left, right, config.orderBy ?? []));
2628
- const start = config.offset ?? 0;
2629
- const end = config.limit === void 0 ? matched.length : start + config.limit;
2630
2755
  const batchSize = config.batchSize ?? 4096;
2631
2756
  let batch = [];
2632
- for (const row of matched.slice(start, end)) {
2633
- batch.push(project(row, config.select));
2757
+ let offsetSkipped = 0;
2758
+ let returned = 0;
2759
+ const distinct = config.distinct === true ? /* @__PURE__ */ new Set() : void 0;
2760
+ for (const row of matched) {
2761
+ const projected = project(row, config.select, config.projections);
2762
+ if (distinct !== void 0 && !addDistinctRow(distinct, projected, config.budget))
2763
+ continue;
2764
+ if (offsetSkipped < (config.offset ?? 0)) {
2765
+ offsetSkipped += 1;
2766
+ continue;
2767
+ }
2768
+ if (config.limit !== void 0 && returned >= config.limit)
2769
+ break;
2770
+ batch.push(projected);
2771
+ returned += 1;
2634
2772
  stats.rowsReturned += 1;
2635
2773
  enforceBudget(config.budget, stats, config.now, startedAt);
2636
2774
  if (batch.length >= batchSize) {
@@ -2649,7 +2787,7 @@ var QueryResult = class {
2649
2787
  const { planned: paths, skipped: skippedFiles } = await this.planObjects();
2650
2788
  stats.filesSkipped = skippedFiles;
2651
2789
  const orderBy = config.orderBy ?? [];
2652
- const columns = projectedReadColumns(config.select, config.where, orderBy);
2790
+ const columns = projectedReadColumns(config.select, config.where, orderBy, config.projections);
2653
2791
  for (const object of paths) {
2654
2792
  stats.filesPlanned += 1;
2655
2793
  stats.filesRead += 1;
@@ -2691,7 +2829,7 @@ var QueryResult = class {
2691
2829
  const { stats } = this;
2692
2830
  const { planned: paths, skipped: skippedFiles } = await this.planObjects();
2693
2831
  stats.filesSkipped = skippedFiles;
2694
- const columns = projectedReadColumns(config.select, config.where, orderBy);
2832
+ const columns = projectedReadColumns(config.select, config.where, orderBy, config.projections);
2695
2833
  const runCapacity = config.budget.maxBufferedRows ?? Number.POSITIVE_INFINITY;
2696
2834
  const currentRun = [];
2697
2835
  for (const object of paths) {
@@ -2736,7 +2874,7 @@ var QueryResult = class {
2736
2874
  async explain() {
2737
2875
  const { planned, skipped } = await this.planObjects();
2738
2876
  const tasks = await this.tasksFromObjects(planned);
2739
- const projectedColumns = projectedReadColumns(this.config.select, this.config.where) ?? [];
2877
+ const projectedColumns = projectedReadColumns(this.config.select, this.config.where, void 0, this.config.projections) ?? [];
2740
2878
  const json = {
2741
2879
  queryId: this.stats.queryId,
2742
2880
  filesPlanned: tasks.length,
@@ -2759,7 +2897,7 @@ var QueryResult = class {
2759
2897
  }
2760
2898
  async tasksFromObjects(objects) {
2761
2899
  const config = this.config;
2762
- const projectedColumns = projectedReadColumns(config.select, config.where, config.orderBy);
2900
+ const projectedColumns = projectedReadColumns(config.select, config.where, config.orderBy, config.projections);
2763
2901
  const tasks = [];
2764
2902
  for (const object of objects) {
2765
2903
  const partitionValues = config.hive ? parseHivePartitions(object.path) : {};
@@ -2973,7 +3111,9 @@ var CountState = class {
2973
3111
  constructor(count = 0) {
2974
3112
  this.count = count;
2975
3113
  }
2976
- add(_value) {
3114
+ add(value) {
3115
+ if (value === null || value === void 0)
3116
+ return;
2977
3117
  this.count += 1;
2978
3118
  }
2979
3119
  finish() {
@@ -3409,8 +3549,11 @@ function snapshotValue(value) {
3409
3549
  function aggregateValue(row, alias, aggregate) {
3410
3550
  if (!aggregate)
3411
3551
  throw new LaQLError("LAQL_VALIDATION_ERROR", `Missing aggregate ${alias}`);
3412
- if (aggregate.op === "count" && aggregate.column === void 0)
3552
+ if (aggregate.op === "count" && aggregate.column === void 0 && aggregate.expr === void 0) {
3413
3553
  return true;
3554
+ }
3555
+ if (aggregate.expr !== void 0)
3556
+ return evaluate(aggregate.expr, row);
3414
3557
  if (aggregate.column === void 0) {
3415
3558
  throw new LaQLError("LAQL_TYPE_ERROR", `${aggregate.op} requires a column`, { aggregate });
3416
3559
  }
@@ -3432,9 +3575,32 @@ function validateAggregateRequest(groupColumns, spec, options) {
3432
3575
  if (options.maxGroups !== void 0 && (!Number.isInteger(options.maxGroups) || options.maxGroups < 1)) {
3433
3576
  throw new LaQLError("LAQL_TYPE_ERROR", "maxGroups must be a positive integer");
3434
3577
  }
3578
+ if (options.orderBy !== void 0)
3579
+ normalizeOrderBy(options.orderBy);
3580
+ if (options.limit !== void 0 && (!Number.isInteger(options.limit) || options.limit < 0)) {
3581
+ throw new LaQLError("LAQL_TYPE_ERROR", "aggregate limit must be a non-negative integer");
3582
+ }
3583
+ if (options.offset !== void 0 && (!Number.isInteger(options.offset) || options.offset < 0)) {
3584
+ throw new LaQLError("LAQL_TYPE_ERROR", "aggregate offset must be a non-negative integer");
3585
+ }
3435
3586
  for (const aggregate of Object.values(spec))
3436
3587
  validateAggregateExpr(aggregate);
3437
3588
  }
3589
+ function applyAggregateResultOptions(rows, options) {
3590
+ let out = rows;
3591
+ if (options.having !== void 0)
3592
+ out = out.filter((row) => matches(options.having, row));
3593
+ if (options.orderBy !== void 0) {
3594
+ const orderBy = normalizeOrderBy(options.orderBy);
3595
+ out = [...out].sort((left, right) => compareRows(left, right, orderBy));
3596
+ }
3597
+ const offset = options.offset ?? 0;
3598
+ if (options.limit !== void 0)
3599
+ return out.slice(offset, offset + options.limit);
3600
+ if (offset > 0)
3601
+ return out.slice(offset);
3602
+ return out;
3603
+ }
3438
3604
  function validateAggregateExpr(aggregate) {
3439
3605
  const ops = [
3440
3606
  "count",
@@ -3451,8 +3617,15 @@ function validateAggregateExpr(aggregate) {
3451
3617
  if (!ops.includes(aggregate.op)) {
3452
3618
  throw new LaQLError("LAQL_TYPE_ERROR", `Unsupported aggregate ${aggregate.op}`, { aggregate });
3453
3619
  }
3454
- if (aggregate.op !== "count" && aggregate.column === void 0) {
3455
- throw new LaQLError("LAQL_TYPE_ERROR", `${aggregate.op} requires a column`, { aggregate });
3620
+ if (aggregate.column !== void 0 && aggregate.expr !== void 0) {
3621
+ throw new LaQLError("LAQL_TYPE_ERROR", "aggregate cannot specify both column and expr", {
3622
+ aggregate
3623
+ });
3624
+ }
3625
+ if (aggregate.op !== "count" && aggregate.column === void 0 && aggregate.expr === void 0) {
3626
+ throw new LaQLError("LAQL_TYPE_ERROR", `${aggregate.op} requires a column or expression`, {
3627
+ aggregate
3628
+ });
3456
3629
  }
3457
3630
  }
3458
3631
  function throwAggregateType(op) {
@@ -3474,6 +3647,11 @@ function parseJsonQuery(input) {
3474
3647
  }
3475
3648
  if (input.where !== void 0)
3476
3649
  init.where = parseJsonExpr(input.where);
3650
+ if (input.distinct !== void 0) {
3651
+ if (typeof input.distinct !== "boolean")
3652
+ throwParse("JSON query distinct must be a boolean");
3653
+ init.distinct = input.distinct;
3654
+ }
3477
3655
  if (input.orderBy !== void 0) {
3478
3656
  if (!Array.isArray(input.orderBy))
3479
3657
  throwParse("JSON query orderBy must be an array");
@@ -3615,6 +3793,9 @@ function validateQueryInit(init) {
3615
3793
  }
3616
3794
  if (init.orderBy !== void 0)
3617
3795
  normalizeOrderBy(init.orderBy);
3796
+ if (init.distinct !== void 0 && typeof init.distinct !== "boolean") {
3797
+ throw new LaQLError("LAQL_TYPE_ERROR", "distinct must be a boolean");
3798
+ }
3618
3799
  }
3619
3800
  function applyQueryPolicy(init, policy) {
3620
3801
  const allowedColumns = policy.allowedColumns === void 0 ? void 0 : normalizeAllowedColumns(policy.allowedColumns);
@@ -3641,8 +3822,12 @@ function cloneBookmarkQuery(init) {
3641
3822
  const query = { source: init.source };
3642
3823
  if (init.select !== void 0)
3643
3824
  query.select = [...init.select];
3825
+ if (init.projections !== void 0)
3826
+ query.projections = init.projections;
3644
3827
  if (init.where !== void 0)
3645
3828
  query.where = init.where;
3829
+ if (init.distinct !== void 0)
3830
+ query.distinct = init.distinct;
3646
3831
  if (init.orderBy !== void 0) {
3647
3832
  query.orderBy = init.orderBy.map((term) => {
3648
3833
  const queryTerm = { column: term.column };
@@ -3700,6 +3885,8 @@ function validatePolicyColumns(init, effectiveWhere, allowedColumns) {
3700
3885
  requested.add(column);
3701
3886
  for (const term of init.orderBy ?? [])
3702
3887
  requested.add(term.column);
3888
+ for (const expr of Object.values(init.projections ?? {}))
3889
+ collectExprColumns(expr, requested);
3703
3890
  collectExprColumns(effectiveWhere, requested);
3704
3891
  for (const column of requested) {
3705
3892
  if (!allowed.has(column)) {
@@ -3760,12 +3947,14 @@ function globRegex(pattern) {
3760
3947
  }
3761
3948
  return new RegExp(`${source}$`, "u");
3762
3949
  }
3763
- function projectedReadColumns(select, where, orderBy = void 0) {
3950
+ function projectedReadColumns(select, where, orderBy = void 0, projections = void 0) {
3764
3951
  const columns = /* @__PURE__ */ new Set();
3765
3952
  for (const column of select ?? [])
3766
3953
  columns.add(column);
3767
3954
  for (const term of orderBy ?? [])
3768
3955
  columns.add(term.column);
3956
+ for (const expr of Object.values(projections ?? {}))
3957
+ collectExprColumns(expr, columns);
3769
3958
  collectExprColumns(where, columns);
3770
3959
  return columns.size === 0 ? void 0 : [...columns].sort();
3771
3960
  }
@@ -3776,6 +3965,8 @@ function aggregateReadColumns(groupColumns, spec, where) {
3776
3965
  for (const aggregate of Object.values(spec)) {
3777
3966
  if (aggregate.column !== void 0)
3778
3967
  columns.add(aggregate.column);
3968
+ if (aggregate.expr !== void 0)
3969
+ collectExprColumns(aggregate.expr, columns);
3779
3970
  }
3780
3971
  collectExprColumns(where, columns);
3781
3972
  return columns.size === 0 ? void 0 : [...columns].sort();
@@ -3820,20 +4011,42 @@ function collectExprColumns(expr, columns) {
3820
4011
  for (const arg of expr.args)
3821
4012
  collectExprColumns(arg, columns);
3822
4013
  return;
4014
+ case "arithmetic":
4015
+ collectExprColumns(expr.left, columns);
4016
+ collectExprColumns(expr.right, columns);
4017
+ return;
4018
+ case "case":
4019
+ for (const branch of expr.whens) {
4020
+ collectExprColumns(branch.when, columns);
4021
+ collectExprColumns(branch.value, columns);
4022
+ }
4023
+ collectExprColumns(expr.else, columns);
4024
+ return;
3823
4025
  }
3824
4026
  }
3825
- function project(row, select) {
3826
- if (!select)
4027
+ function project(row, select, projections) {
4028
+ if (!select && !projections)
3827
4029
  return row;
3828
4030
  const out = {};
3829
- for (const column of select) {
4031
+ for (const column of select ?? []) {
3830
4032
  if (!(column in row)) {
3831
4033
  throw new LaQLError("LAQL_UNKNOWN_COLUMN", `Unknown column ${column}`, { column });
3832
4034
  }
3833
4035
  out[column] = row[column];
3834
4036
  }
4037
+ for (const [alias, expr] of Object.entries(projections ?? {}))
4038
+ out[alias] = evaluate(expr, row);
3835
4039
  return out;
3836
4040
  }
4041
+ function addDistinctRow(seen, row, budget) {
4042
+ const key = stableStringify(jsonSafeValue(row));
4043
+ if (seen.has(key))
4044
+ return false;
4045
+ seen.add(key);
4046
+ enforceBufferedRowsBudget(budget, seen.size);
4047
+ enforceOperatorMemoryBudget(budget, estimateOperatorMemoryBytes([...seen]));
4048
+ return true;
4049
+ }
3837
4050
  function normalizeOrderBy(terms) {
3838
4051
  if (!Array.isArray(terms) || terms.length === 0) {
3839
4052
  throw new LaQLError("LAQL_TYPE_ERROR", "orderBy must contain at least one term");
@@ -3983,6 +4196,8 @@ function partitionEval(expr, partitions) {
3983
4196
  case "null-check":
3984
4197
  case "like":
3985
4198
  case "call":
4199
+ case "arithmetic":
4200
+ case "case":
3986
4201
  return expressionIsPartitionOnly(expr, partitions) ? matches(expr, partitions) : "unknown";
3987
4202
  case "not": {
3988
4203
  const value = partitionEval(expr.operand, partitions);
@@ -4323,1296 +4538,6 @@ var ReadSemaphore = class {
4323
4538
  this.queue.shift()?.();
4324
4539
  }
4325
4540
  };
4326
-
4327
- // ../iceberg/dist/index.js
4328
- var PACKAGE = "@laql/iceberg";
4329
- var IcebergTable = class {
4330
- store;
4331
- metadataPath;
4332
- metadata;
4333
- constructor(store, metadataPath, metadata) {
4334
- this.store = store;
4335
- this.metadataPath = metadataPath;
4336
- this.metadata = metadata;
4337
- }
4338
- snapshot(options = {}) {
4339
- if (options.snapshotId !== void 0)
4340
- return this.snapshotById(options.snapshotId);
4341
- if (options.ref !== void 0) {
4342
- const ref = this.metadata.refs?.[options.ref];
4343
- if (!ref) {
4344
- throw new LaQLError("LAQL_CATALOG_ERROR", `Unknown Iceberg ref ${options.ref}`, {
4345
- ref: options.ref
4346
- });
4347
- }
4348
- return this.snapshotById(ref["snapshot-id"]);
4349
- }
4350
- if (options.asOfTimestampMs !== void 0) {
4351
- const snapshot = [...this.metadata.snapshots].filter((candidate) => candidate["timestamp-ms"] <= options.asOfTimestampMs).sort((a, b) => b["timestamp-ms"] - a["timestamp-ms"])[0];
4352
- if (!snapshot) {
4353
- throw new LaQLError("LAQL_CATALOG_ERROR", "No Iceberg snapshot at requested timestamp", {
4354
- asOfTimestampMs: options.asOfTimestampMs
4355
- });
4356
- }
4357
- return snapshot;
4358
- }
4359
- return this.snapshotById(this.metadata["current-snapshot-id"]);
4360
- }
4361
- schema(schemaId) {
4362
- const schema = this.metadata.schemas.find((candidate) => candidate["schema-id"] === schemaId);
4363
- if (!schema) {
4364
- throw new LaQLError("LAQL_CATALOG_ERROR", `Unknown Iceberg schema ${schemaId}`, { schemaId });
4365
- }
4366
- return schema.fields;
4367
- }
4368
- projectRow(row, options = {}) {
4369
- const snapshot = this.snapshot(options);
4370
- const fields = this.schema(snapshot["schema-id"]);
4371
- projectedIds(fields, options.select);
4372
- const selected = options.select === void 0 ? void 0 : new Set(options.select);
4373
- const out = {};
4374
- for (const field of fields) {
4375
- if (selected !== void 0 && !selected.has(field.name))
4376
- continue;
4377
- const sourceName = field.name in row ? field.name : field.sourceId !== void 0 ? this.fieldNameById(field.sourceId) : field.name;
4378
- out[field.name] = sourceName !== void 0 && sourceName in row ? row[sourceName] : null;
4379
- }
4380
- return out;
4381
- }
4382
- planFiles(options = {}) {
4383
- const snapshot = this.snapshot(options);
4384
- const fields = this.schema(snapshot["schema-id"]);
4385
- const projectedFieldIds = projectedIds(fields, options.select);
4386
- const readMode = options.readMode ?? "strict";
4387
- const files = [];
4388
- let manifestsSkipped = 0;
4389
- let filesSkipped = 0;
4390
- let deleteFilesPlanned = 0;
4391
- let deleteFilesIgnored = 0;
4392
- const manifests = snapshotManifests(snapshot);
4393
- for (const manifest of manifests) {
4394
- const manifestMayMatch = manifest.files.some((file) => partitionMayMatch2(options.where, file.partition ?? {}));
4395
- if (!manifestMayMatch) {
4396
- manifestsSkipped += 1;
4397
- filesSkipped += manifest.files.length;
4398
- continue;
4399
- }
4400
- for (const file of manifest.files) {
4401
- if (!partitionMayMatch2(options.where, file.partition ?? {})) {
4402
- filesSkipped += 1;
4403
- continue;
4404
- }
4405
- const supportedDeleteFiles = supportedIcebergDeleteFiles(file.deleteFiles);
4406
- const unsupportedDeleteFiles = unsupportedIcebergDeleteFiles(file.deleteFiles);
4407
- if (unsupportedDeleteFiles.length > 0 && readMode === "strict") {
4408
- throw new LaQLError("LAQL_UNSUPPORTED_DELETE_FILES", "Snapshot contains delete files unsupported by strict Iceberg planning", {
4409
- path: file.path,
4410
- deleteFiles: file.deleteFiles,
4411
- supportedDeleteFiles,
4412
- unsupportedDeleteFiles
4413
- });
4414
- }
4415
- const planned = {
4416
- path: file.path,
4417
- sequenceNumber: file.sequenceNumber,
4418
- partition: file.partition ?? {},
4419
- recordCount: file.recordCount,
4420
- projectedFieldIds,
4421
- snapshotId: snapshot["snapshot-id"]
4422
- };
4423
- if (file.fileSizeInBytes !== void 0)
4424
- planned.fileSizeInBytes = file.fileSizeInBytes;
4425
- if ((readMode === "strict" || readMode === "ignore-unsupported-deletes") && supportedDeleteFiles.length > 0) {
4426
- planned.deleteFiles = supportedDeleteFiles;
4427
- deleteFilesPlanned += supportedDeleteFiles.length;
4428
- }
4429
- if (readMode === "ignore-unsupported-deletes") {
4430
- deleteFilesIgnored += unsupportedDeleteFiles.length;
4431
- } else if (readMode === "ignore-deletes") {
4432
- deleteFilesIgnored += (file.deleteFiles ?? []).length;
4433
- }
4434
- files.push(planned);
4435
- }
4436
- }
4437
- files.sort((a, b) => a.sequenceNumber - b.sequenceNumber || a.path.localeCompare(b.path));
4438
- return {
4439
- snapshotId: snapshot["snapshot-id"],
4440
- schemaId: snapshot["schema-id"],
4441
- manifestsRead: manifests.length - manifestsSkipped,
4442
- manifestsSkipped,
4443
- filesPlanned: files.length,
4444
- filesSkipped,
4445
- deleteFilesPlanned,
4446
- deleteFilesIgnored,
4447
- files
4448
- };
4449
- }
4450
- async appendFiles(options) {
4451
- if (options.files.length === 0) {
4452
- throw new LaQLError("LAQL_VALIDATION_ERROR", "Iceberg append requires at least one file");
4453
- }
4454
- if (this.metadata["format-version"] !== 2) {
4455
- throw new LaQLError("LAQL_VALIDATION_ERROR", "Iceberg append requires format-version 2 metadata", { formatVersion: this.metadata["format-version"] });
4456
- }
4457
- const currentSnapshot = this.snapshot();
4458
- const nextSnapshotId = options.nextSnapshotId ?? randomSnapshotId(this.metadata.snapshots.map(snapshotIdOf));
4459
- validateNewSnapshotId(nextSnapshotId, this.metadata.snapshots);
4460
- const nextSequenceNumber = maxSequenceNumber(this.metadata) + 1;
4461
- const manifestPath = appendManifestPath(this.metadataPath, options.jobId, nextSnapshotId);
4462
- const manifest = {
4463
- path: manifestPath,
4464
- files: options.files.map((file, index) => ({
4465
- path: file.path,
4466
- sequenceNumber: nextSequenceNumber + index,
4467
- partition: sortStringRecord(file.partition ?? {}),
4468
- recordCount: file.recordCount,
4469
- fileSizeInBytes: file.fileSizeInBytes
4470
- }))
4471
- };
4472
- const nextSnapshot = {
4473
- "snapshot-id": nextSnapshotId,
4474
- "timestamp-ms": options.nowMs ?? Date.now(),
4475
- "schema-id": currentSnapshot["schema-id"],
4476
- manifests: [...snapshotManifests(currentSnapshot).map(cloneManifest), manifest]
4477
- };
4478
- const metadata = cloneMetadata(this.metadata);
4479
- metadata["current-snapshot-id"] = nextSnapshotId;
4480
- metadata.snapshots.push(nextSnapshot);
4481
- metadata.refs = {
4482
- ...metadata.refs ?? {},
4483
- main: { type: "branch", "snapshot-id": nextSnapshotId }
4484
- };
4485
- const nextMetadataPath = nextMetadataPathFor(this.metadataPath, nextSnapshotId);
4486
- const catalog = options.catalog ?? new ObjectStoreIcebergCommitCatalog();
4487
- const commit = await catalog.commitAppend({
4488
- store: this.store,
4489
- currentMetadataPath: this.metadataPath,
4490
- nextMetadataPath,
4491
- expectedSnapshotId: currentSnapshot["snapshot-id"],
4492
- nextSnapshotId,
4493
- manifestPath,
4494
- manifest,
4495
- metadata
4496
- });
4497
- const committed = typeof commit === "boolean" ? commit : commit.committed;
4498
- if (!committed) {
4499
- throw new LaQLError("LAQL_ICEBERG_COMMIT_CONFLICT", "Iceberg append commit conflict", {
4500
- metadataPath: this.metadataPath,
4501
- expectedSnapshotId: currentSnapshot["snapshot-id"],
4502
- nextSnapshotId
4503
- });
4504
- }
4505
- const committedMetadataPath = typeof commit === "boolean" ? nextMetadataPath : commit.metadataPath ?? nextMetadataPath;
4506
- return {
4507
- snapshotId: nextSnapshotId,
4508
- previousSnapshotId: currentSnapshot["snapshot-id"],
4509
- metadataPath: committedMetadataPath,
4510
- manifestPath,
4511
- files: manifest.files.map((file) => ({
4512
- path: file.path,
4513
- sequenceNumber: file.sequenceNumber,
4514
- partition: file.partition ?? {},
4515
- recordCount: file.recordCount,
4516
- projectedFieldIds: [],
4517
- snapshotId: nextSnapshotId
4518
- }))
4519
- };
4520
- }
4521
- async appendOutputManifest(options) {
4522
- const files = options.manifest.entries.map((entry) => {
4523
- if (entry.iceberg === void 0) {
4524
- throw new LaQLError("LAQL_VALIDATION_ERROR", "Output manifest entry is missing Iceberg file metadata", {
4525
- taskId: entry.taskId,
4526
- outputPath: entry.outputPath
4527
- });
4528
- }
4529
- return {
4530
- path: entry.outputPath,
4531
- partition: entry.iceberg.partitionValues,
4532
- recordCount: entry.iceberg.recordCount,
4533
- fileSizeInBytes: entry.iceberg.fileSizeInBytes
4534
- };
4535
- });
4536
- return await this.appendFiles({
4537
- files,
4538
- jobId: options.manifest.jobId,
4539
- ...options.nowMs !== void 0 ? { nowMs: options.nowMs } : {},
4540
- ...options.nextSnapshotId !== void 0 ? { nextSnapshotId: options.nextSnapshotId } : {},
4541
- ...options.catalog !== void 0 ? { catalog: options.catalog } : {}
4542
- });
4543
- }
4544
- snapshotById(snapshotId) {
4545
- const snapshot = this.metadata.snapshots.find((candidate) => candidate["snapshot-id"] === snapshotId);
4546
- if (!snapshot) {
4547
- throw new LaQLError("LAQL_CATALOG_ERROR", `Unknown Iceberg snapshot ${snapshotId}`, {
4548
- snapshotId
4549
- });
4550
- }
4551
- return snapshot;
4552
- }
4553
- fieldNameById(fieldId) {
4554
- for (const schema of this.metadata.schemas) {
4555
- const field = schema.fields.find((candidate) => candidate.id === fieldId);
4556
- if (field)
4557
- return field.name;
4558
- }
4559
- return void 0;
4560
- }
4561
- };
4562
- function planFiles(table, options = {}) {
4563
- return table.planFiles(options);
4564
- }
4565
- var ObjectStoreIcebergCommitCatalog = class {
4566
- async commitAppend(input) {
4567
- if (!supportsConditionalPut(input.store)) {
4568
- throw new LaQLError("LAQL_CATALOG_ERROR", "Object-store Iceberg append requires conditional put support", { metadataPath: input.currentMetadataPath });
4569
- }
4570
- const currentBytes = await input.store.get(input.currentMetadataPath);
4571
- if (!currentBytes) {
4572
- throw new LaQLError("LAQL_OBJECT_NOT_FOUND", `No object at ${input.currentMetadataPath}`, {
4573
- path: input.currentMetadataPath
4574
- });
4575
- }
4576
- const current = validateMetadata(JSON.parse(new TextDecoder().decode(currentBytes)));
4577
- if (current["current-snapshot-id"] !== input.expectedSnapshotId) {
4578
- return { committed: false };
4579
- }
4580
- const versionHintPath = metadataVersionHintPath(input.nextMetadataPath);
4581
- const versionHintHead = await input.store.head(versionHintPath);
4582
- await input.store.put(input.manifestPath, new TextEncoder().encode(`${stableStringify(input.manifest)}
4583
- `), { contentType: "application/json" });
4584
- await input.store.put(input.nextMetadataPath, new TextEncoder().encode(`${JSON.stringify(input.metadata, null, 2)}
4585
- `), { contentType: "application/json" });
4586
- const updated = await input.store.conditionalPut(versionHintPath, new TextEncoder().encode(`${input.nextSnapshotId}
4587
- `), { contentType: "text/plain", expectedEtag: versionHintHead?.etag ?? null });
4588
- if (!updated)
4589
- return { committed: false };
4590
- return { committed: true, metadataPath: input.nextMetadataPath };
4591
- }
4592
- };
4593
- var IcebergRestCatalog = class {
4594
- url;
4595
- namespace;
4596
- table;
4597
- prefix;
4598
- token;
4599
- fetchFn;
4600
- constructor(options) {
4601
- this.url = options.url;
4602
- this.namespace = namespaceParts(options.namespace);
4603
- this.table = requiredNonEmptyString(options.table, "table");
4604
- this.prefix = catalogPrefixParts(options.prefix);
4605
- this.token = options.token;
4606
- this.fetchFn = options.fetch ?? fetch;
4607
- }
4608
- async loadTable(store) {
4609
- const response = await this.requestJson(this.tableUrl(), { method: "GET" });
4610
- if (!isRecord4(response) || typeof response["metadata-location"] !== "string") {
4611
- throw new LaQLError("LAQL_CATALOG_ERROR", "Invalid Iceberg REST load table response", {
4612
- url: this.tableUrl()
4613
- });
4614
- }
4615
- return new IcebergTable(store, response["metadata-location"], await hydrateMetadataManifests(store, validateMetadata(response.metadata)));
4616
- }
4617
- async listTables() {
4618
- return validateListTablesResponse(await this.requestJson(this.namespaceTablesUrl(), {
4619
- method: "GET"
4620
- }));
4621
- }
4622
- async commitAppend(input) {
4623
- await input.store.put(input.manifestPath, new TextEncoder().encode(`${stableStringify(input.manifest)}
4624
- `), { contentType: "application/json" });
4625
- await input.store.put(input.nextMetadataPath, new TextEncoder().encode(`${JSON.stringify(input.metadata, null, 2)}
4626
- `), { contentType: "application/json" });
4627
- const response = await this.fetchFn(this.tableUrl(), {
4628
- method: "POST",
4629
- headers: this.headers(true),
4630
- body: JSON.stringify({
4631
- identifier: { namespace: this.namespace, name: this.table },
4632
- requirements: [
4633
- {
4634
- type: "assert-ref-snapshot-id",
4635
- ref: "main",
4636
- "snapshot-id": input.expectedSnapshotId
4637
- }
4638
- ],
4639
- updates: [
4640
- {
4641
- action: "add-snapshot",
4642
- snapshot: restAppendSnapshot(input)
4643
- },
4644
- {
4645
- action: "set-snapshot-ref",
4646
- "ref-name": "main",
4647
- type: "branch",
4648
- "snapshot-id": input.nextSnapshotId
4649
- }
4650
- ]
4651
- })
4652
- });
4653
- if (response.status === 409)
4654
- return { committed: false };
4655
- if (!response.ok) {
4656
- throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg REST table commit failed", {
4657
- url: this.tableUrl(),
4658
- status: response.status,
4659
- statusText: response.statusText
4660
- });
4661
- }
4662
- const metadataPath = await commitResponseMetadataPath(response);
4663
- return {
4664
- committed: true,
4665
- ...metadataPath !== void 0 ? { metadataPath } : {}
4666
- };
4667
- }
4668
- async requestJson(url, init) {
4669
- const response = await this.fetchFn(url, {
4670
- ...init,
4671
- headers: this.headers(init.body !== void 0)
4672
- });
4673
- if (!response.ok) {
4674
- throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg REST catalog request failed", {
4675
- url,
4676
- status: response.status,
4677
- statusText: response.statusText
4678
- });
4679
- }
4680
- try {
4681
- return await response.json();
4682
- } catch (cause) {
4683
- throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg REST catalog response is not JSON", {
4684
- url,
4685
- cause
4686
- });
4687
- }
4688
- }
4689
- headers(hasBody) {
4690
- const headers = new Headers({ accept: "application/json" });
4691
- if (hasBody)
4692
- headers.set("content-type", "application/json");
4693
- if (this.token !== void 0)
4694
- headers.set("authorization", `Bearer ${this.token}`);
4695
- return headers;
4696
- }
4697
- tableUrl() {
4698
- return restCatalogUrl(this.url, [...this.namespaceTableSegments(), this.table]);
4699
- }
4700
- namespaceTablesUrl() {
4701
- return restCatalogUrl(this.url, this.namespaceTableSegments());
4702
- }
4703
- namespaceTableSegments() {
4704
- return ["v1", ...this.prefix, "namespaces", this.namespace.join(""), "tables"];
4705
- }
4706
- };
4707
- function icebergRestCatalog(options) {
4708
- return new IcebergRestCatalog(options);
4709
- }
4710
- var IcebergUnsupportedCatalog = class {
4711
- catalog;
4712
- namespace;
4713
- table;
4714
- constructor(catalog, namespace, table) {
4715
- this.catalog = requiredNonEmptyString(catalog, "catalog");
4716
- this.namespace = namespaceParts(namespace);
4717
- this.table = requiredNonEmptyString(table, "table");
4718
- }
4719
- async loadTable(_store) {
4720
- throw this.unsupported("loadTable");
4721
- }
4722
- async listTables() {
4723
- throw this.unsupported("listTables");
4724
- }
4725
- async commitAppend(_input) {
4726
- throw this.unsupported("commitAppend");
4727
- }
4728
- unsupported(operation) {
4729
- return new LaQLError("LAQL_CATALOG_ERROR", `${this.catalog} Iceberg catalog is not implemented`, {
4730
- catalog: this.catalog,
4731
- namespace: this.namespace,
4732
- table: this.table,
4733
- operation
4734
- });
4735
- }
4736
- };
4737
- function icebergGlueCatalog(options) {
4738
- requiredNonEmptyString(options.region, "region");
4739
- return new IcebergUnsupportedCatalog("Glue", options.namespace, options.table);
4740
- }
4741
- function icebergNessieCatalog(options) {
4742
- requiredNonEmptyString(options.url, "url");
4743
- if (options.ref !== void 0)
4744
- requiredNonEmptyString(options.ref, "ref");
4745
- return new IcebergUnsupportedCatalog("Nessie", options.namespace, options.table);
4746
- }
4747
- async function loadIcebergTable(options) {
4748
- const readControls = loadReadControls(options);
4749
- const store = withObjectStoreReadControls(options.store, readControls);
4750
- const bytes = await store.get(options.metadataPath);
4751
- if (!bytes) {
4752
- throw new LaQLError("LAQL_OBJECT_NOT_FOUND", `No object at ${options.metadataPath}`, {
4753
- path: options.metadataPath
4754
- });
4755
- }
4756
- throwIfAborted(readControls.signal);
4757
- const text = new TextDecoder().decode(bytes);
4758
- try {
4759
- return new IcebergTable(store, options.metadataPath, await hydrateMetadataManifests(store, validateMetadata(JSON.parse(text)), readControls));
4760
- } catch (cause) {
4761
- if (cause instanceof LaQLError)
4762
- throw cause;
4763
- throw new LaQLError("LAQL_CATALOG_ERROR", `Invalid Iceberg metadata at ${options.metadataPath}`, {
4764
- path: options.metadataPath,
4765
- cause
4766
- });
4767
- }
4768
- }
4769
- async function loadIcebergTableFromObjectStore(options) {
4770
- const readControls = loadReadControls(options);
4771
- const store = withObjectStoreReadControls(options.store, readControls);
4772
- const tableLocation = trimTrailingSlash(options.tableLocation);
4773
- const metadataPrefix = `${tableLocation}/metadata/`;
4774
- const versionHintPath = `${metadataPrefix}version-hint.text`;
4775
- const hintedVersion = await readVersionHint(store, versionHintPath, readControls);
4776
- const metadataPath = hintedVersion === void 0 ? await latestMetadataPathFromList(store, metadataPrefix, readControls) : `${metadataPrefix}v${hintedVersion}.metadata.json`;
4777
- return await loadIcebergTable({ store, metadataPath, ...readControls });
4778
- }
4779
- async function loadIcebergTableFromRest(options) {
4780
- return await icebergRestCatalog(options).loadTable(withObjectStoreReadControls(options.store, loadReadControls(options)));
4781
- }
4782
- function loadReadControls(options) {
4783
- const controls = {};
4784
- if (options.maxConcurrentReads !== void 0)
4785
- controls.maxConcurrentReads = options.maxConcurrentReads;
4786
- const signal = readControlSignal(options);
4787
- if (signal !== void 0)
4788
- controls.signal = signal;
4789
- return controls;
4790
- }
4791
- function applyIcebergDeletes(options) {
4792
- const rowOffset = options.rowOffset ?? 0;
4793
- const positionDeletes = /* @__PURE__ */ new Set();
4794
- for (const deletion of options.positionDeletes ?? []) {
4795
- if (deletion.path !== options.dataFilePath)
4796
- continue;
4797
- positionDeletes.add(validateDeletePosition(deletion.position, deletion.path));
4798
- }
4799
- for (const deletionVector of options.deletionVectors ?? []) {
4800
- if (deletionVector.path !== options.dataFilePath)
4801
- continue;
4802
- for (const position of deletionVector.positions) {
4803
- positionDeletes.add(validateDeletePosition(position, deletionVector.path));
4804
- }
4805
- }
4806
- const equalityDeleteKeys = /* @__PURE__ */ new Map();
4807
- for (const deletion of options.equalityDeletes ?? []) {
4808
- if (deletion.columns.length === 0) {
4809
- throw new LaQLError("LAQL_VALIDATION_ERROR", "Iceberg equality delete requires columns");
4810
- }
4811
- const columnsKey = stableStringify(deletion.columns);
4812
- let keys = equalityDeleteKeys.get(columnsKey);
4813
- if (!keys) {
4814
- keys = /* @__PURE__ */ new Set();
4815
- equalityDeleteKeys.set(columnsKey, keys);
4816
- }
4817
- keys.add(equalityKey(deletion.row, deletion.columns));
4818
- }
4819
- return options.rows.filter((row, index) => {
4820
- if (positionDeletes.has(rowOffset + index))
4821
- return false;
4822
- for (const [columnsKey, keys] of equalityDeleteKeys) {
4823
- const columns = JSON.parse(columnsKey);
4824
- if (keys.has(equalityKey(row, columns)))
4825
- return false;
4826
- }
4827
- return true;
4828
- });
4829
- }
4830
- async function* scanPlannedIcebergRows(options) {
4831
- const files = Array.isArray(options.plan) ? options.plan : options.plan.files;
4832
- const signal = readControlSignal(options);
4833
- for (const file of files) {
4834
- throwIfAborted(signal);
4835
- const deletes = await decodedDeletesForFile(file, options, signal);
4836
- throwIfAborted(signal);
4837
- const data = await options.readDataFile(file);
4838
- throwIfAborted(signal);
4839
- let rowOffset = 0;
4840
- for await (const batch of rowBatches(data)) {
4841
- throwIfAborted(signal);
4842
- const rows = Array.isArray(batch) ? batch : batch.rows;
4843
- const absoluteRowOffset = Array.isArray(batch) ? rowOffset : batch.rowOffset;
4844
- const visibleRows = hasDeletes(deletes) ? applyIcebergDeletes({
4845
- dataFilePath: file.path,
4846
- rows,
4847
- rowOffset: absoluteRowOffset,
4848
- ...deletes
4849
- }) : rows;
4850
- rowOffset = absoluteRowOffset + rows.length;
4851
- if (visibleRows.length > 0)
4852
- yield visibleRows;
4853
- throwIfAborted(signal);
4854
- }
4855
- }
4856
- }
4857
- async function decodedDeletesForFile(file, options, signal) {
4858
- const out = {};
4859
- for (const deleteFile of file.deleteFiles ?? []) {
4860
- throwIfAborted(signal);
4861
- const decoded = await options.readDeleteFile(deleteFile, file);
4862
- throwIfAborted(signal);
4863
- pushDeletes(out, decoded);
4864
- }
4865
- return out;
4866
- }
4867
- function pushDeletes(target, source) {
4868
- if (source.positionDeletes !== void 0) {
4869
- target.positionDeletes = [...target.positionDeletes ?? [], ...source.positionDeletes];
4870
- }
4871
- if (source.equalityDeletes !== void 0) {
4872
- target.equalityDeletes = [...target.equalityDeletes ?? [], ...source.equalityDeletes];
4873
- }
4874
- if (source.deletionVectors !== void 0) {
4875
- target.deletionVectors = [...target.deletionVectors ?? [], ...source.deletionVectors];
4876
- }
4877
- }
4878
- function hasDeletes(deletes) {
4879
- return (deletes.positionDeletes?.length ?? 0) > 0 || (deletes.equalityDeletes?.length ?? 0) > 0 || (deletes.deletionVectors?.length ?? 0) > 0;
4880
- }
4881
- async function* rowBatches(rows) {
4882
- if (isAsyncIterable(rows)) {
4883
- yield* rows;
4884
- } else {
4885
- yield rows;
4886
- }
4887
- }
4888
- function isAsyncIterable(value) {
4889
- return typeof value === "object" && value !== null && Symbol.asyncIterator in value && typeof value[Symbol.asyncIterator] === "function";
4890
- }
4891
- function validateDeletePosition(position, path) {
4892
- if (!Number.isInteger(position) || position < 0) {
4893
- throw new LaQLError("LAQL_VALIDATION_ERROR", "Iceberg delete position must be non-negative", {
4894
- path,
4895
- position
4896
- });
4897
- }
4898
- return position;
4899
- }
4900
- function nextMetadataPathFor(metadataPath, snapshotId) {
4901
- const slash = metadataPath.lastIndexOf("/");
4902
- const prefix = slash === -1 ? "" : `${metadataPath.slice(0, slash + 1)}`;
4903
- return `${prefix}v${snapshotId}.metadata.json`;
4904
- }
4905
- function randomSnapshotId(existingIds) {
4906
- const existing = new Set(existingIds);
4907
- for (let attempt = 0; attempt < 100; attempt += 1) {
4908
- const id = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER) + 1;
4909
- if (!existing.has(id))
4910
- return id;
4911
- }
4912
- throw new LaQLError("LAQL_CATALOG_ERROR", "Unable to allocate unique Iceberg snapshot id");
4913
- }
4914
- function validateNewSnapshotId(snapshotId, snapshots) {
4915
- if (!Number.isSafeInteger(snapshotId) || snapshotId <= 0) {
4916
- throw new LaQLError("LAQL_VALIDATION_ERROR", "Iceberg snapshot id must be a positive safe integer", { snapshotId });
4917
- }
4918
- if (snapshots.some((snapshot) => snapshotIdOf(snapshot) === snapshotId)) {
4919
- throw new LaQLError("LAQL_VALIDATION_ERROR", "Iceberg snapshot id already exists", {
4920
- snapshotId
4921
- });
4922
- }
4923
- }
4924
- function snapshotIdOf(snapshot) {
4925
- return snapshot["snapshot-id"];
4926
- }
4927
- function metadataVersionHintPath(metadataPath) {
4928
- const slash = metadataPath.lastIndexOf("/");
4929
- const prefix = slash === -1 ? "" : `${metadataPath.slice(0, slash + 1)}`;
4930
- return `${prefix}version-hint.text`;
4931
- }
4932
- function supportsConditionalPut(store) {
4933
- return typeof store.conditionalPut === "function";
4934
- }
4935
- async function hydrateMetadataManifests(store, metadata, controls = {}) {
4936
- const hydrated = cloneMetadata(metadata);
4937
- const tablePrefix = tableLocationObjectPrefix(hydrated.location);
4938
- for (const snapshot of hydrated.snapshots) {
4939
- throwIfAborted(controls.signal);
4940
- const manifestReferences = snapshot.manifests ?? (snapshot["manifest-list"] !== void 0 ? await readManifestList(store, validateManifestSourcedPath(snapshot["manifest-list"], tablePrefix)) : []);
4941
- const manifests = await Promise.all(manifestReferences.map(async (manifest) => {
4942
- const manifestPath = validateManifestSourcedPath(manifest.path, tablePrefix);
4943
- if (Array.isArray(manifest.files))
4944
- return validateManifestPaths(manifest, manifestPath, tablePrefix);
4945
- return await readManifest(store, manifestPath, tablePrefix);
4946
- }));
4947
- throwIfAborted(controls.signal);
4948
- snapshot.manifests = mergeDeleteManifests(manifests);
4949
- }
4950
- return hydrated;
4951
- }
4952
- function mergeDeleteManifests(manifests) {
4953
- const deleteFiles = manifests.flatMap((manifest) => manifest.deleteFiles ?? []);
4954
- const dataManifests = manifests.filter((manifest) => manifest.files.length > 0);
4955
- if (deleteFiles.length === 0)
4956
- return dataManifests;
4957
- return dataManifests.map((manifest) => ({
4958
- ...manifest,
4959
- files: manifest.files.map((file) => {
4960
- const applicable = deleteFiles.filter((deleteFile) => deleteFileMayApply(deleteFile, file));
4961
- if (applicable.length === 0)
4962
- return file;
4963
- return {
4964
- ...file,
4965
- deleteFiles: [...file.deleteFiles ?? [], ...applicable.map(publicDeleteFile)]
4966
- };
4967
- })
4968
- }));
4969
- }
4970
- function deleteFileMayApply(deleteFile, file) {
4971
- if (deleteFile.partition === void 0 || Object.keys(deleteFile.partition).length === 0) {
4972
- return true;
4973
- }
4974
- const filePartition = file.partition ?? {};
4975
- return Object.entries(deleteFile.partition).every(([key, value]) => filePartition[key] === value);
4976
- }
4977
- function publicDeleteFile(deleteFile) {
4978
- return { content: deleteFile.content, path: deleteFile.path };
4979
- }
4980
- async function readManifestList(store, path) {
4981
- const bytes = await store.get(path);
4982
- if (!bytes) {
4983
- throw new LaQLError("LAQL_OBJECT_NOT_FOUND", `No Iceberg manifest list at ${path}`, { path });
4984
- }
4985
- try {
4986
- return avroObjectContainer(bytes) ? validateAvroManifestList(await decodeAvroObjectContainer(bytes), path) : validateManifestList(JSON.parse(new TextDecoder().decode(bytes)), path);
4987
- } catch (cause) {
4988
- if (cause instanceof LaQLError)
4989
- throw cause;
4990
- throw new LaQLError("LAQL_CATALOG_ERROR", `Invalid Iceberg manifest list at ${path}`, {
4991
- path,
4992
- cause
4993
- });
4994
- }
4995
- }
4996
- async function readManifest(store, path, tablePrefix = "") {
4997
- const bytes = await store.get(path);
4998
- if (!bytes) {
4999
- throw new LaQLError("LAQL_OBJECT_NOT_FOUND", `No Iceberg manifest at ${path}`, { path });
5000
- }
5001
- try {
5002
- const manifest = avroObjectContainer(bytes) ? validateAvroManifest(await decodeAvroObjectContainer(bytes), path) : validateManifest(JSON.parse(new TextDecoder().decode(bytes)), path);
5003
- return validateManifestPaths(manifest, path, tablePrefix);
5004
- } catch (cause) {
5005
- if (cause instanceof LaQLError)
5006
- throw cause;
5007
- throw new LaQLError("LAQL_CATALOG_ERROR", `Invalid Iceberg manifest at ${path}`, {
5008
- path,
5009
- cause
5010
- });
5011
- }
5012
- }
5013
- function validateAvroManifestList(records, path) {
5014
- return records.map((record) => {
5015
- if (!isRecord4(record) || typeof record.manifest_path !== "string") {
5016
- throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg Avro manifest list entry is invalid", {
5017
- path
5018
- });
5019
- }
5020
- validateManifestContent(record.content, path);
5021
- return { path: record.manifest_path };
5022
- });
5023
- }
5024
- function validateAvroManifest(records, path) {
5025
- const files = [];
5026
- const deleteFiles = [];
5027
- for (const record of records) {
5028
- if (!isRecord4(record)) {
5029
- throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg Avro manifest entry is invalid", {
5030
- path
5031
- });
5032
- }
5033
- if (typeof record.status === "number" && record.status === 2)
5034
- continue;
5035
- const dataFile = record.data_file;
5036
- if (!isRecord4(dataFile)) {
5037
- throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg Avro manifest entry is missing data_file", {
5038
- path
5039
- });
5040
- }
5041
- const content = typeof dataFile.content === "number" ? dataFile.content : 0;
5042
- const recordCount = safeAvroNumber(dataFile.record_count);
5043
- if (typeof dataFile.file_path !== "string" || recordCount === void 0) {
5044
- throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg Avro data file has invalid fields", {
5045
- path
5046
- });
5047
- }
5048
- if (content !== 0) {
5049
- deleteFiles.push({
5050
- content: avroDeleteContent(content, dataFile),
5051
- path: dataFile.file_path,
5052
- partition: avroPartitionValues(dataFile.partition)
5053
- });
5054
- continue;
5055
- }
5056
- const sequenceNumber = safeAvroNumber(record.sequence_number) ?? safeAvroNumber(record.file_sequence_number) ?? 0;
5057
- const file = {
5058
- path: dataFile.file_path,
5059
- sequenceNumber,
5060
- partition: avroPartitionValues(dataFile.partition),
5061
- recordCount
5062
- };
5063
- const fileSizeInBytes = safeAvroNumber(dataFile.file_size_in_bytes);
5064
- if (fileSizeInBytes !== void 0) {
5065
- file.fileSizeInBytes = fileSizeInBytes;
5066
- }
5067
- files.push(file);
5068
- }
5069
- const manifest = { path, files };
5070
- if (deleteFiles.length > 0)
5071
- manifest.deleteFiles = deleteFiles;
5072
- return validateManifestPaths(manifest, path);
5073
- }
5074
- function avroDeleteContent(content, dataFile) {
5075
- if (content === 1 && (String(dataFile.file_format).toLowerCase() === "puffin" || dataFile.content_offset !== void 0 || dataFile.content_size_in_bytes !== void 0)) {
5076
- return "deletion-vector";
5077
- }
5078
- if (content === 1)
5079
- return "position-delete";
5080
- if (content === 2)
5081
- return "equality-delete";
5082
- return `unsupported-delete-${content}`;
5083
- }
5084
- function avroPartitionValues(value) {
5085
- if (!isRecord4(value))
5086
- return {};
5087
- const out = {};
5088
- for (const [key, inner] of Object.entries(value)) {
5089
- if (inner === null || inner === void 0)
5090
- continue;
5091
- out[key] = String(jsonSafeValue(inner));
5092
- }
5093
- return sortStringRecord(out);
5094
- }
5095
- function safeAvroNumber(value) {
5096
- if (typeof value === "number" && Number.isSafeInteger(value))
5097
- return value;
5098
- if (typeof value === "bigint") {
5099
- const numberValue = Number(value);
5100
- if (Number.isSafeInteger(numberValue) && BigInt(numberValue) === value)
5101
- return numberValue;
5102
- }
5103
- return void 0;
5104
- }
5105
- function avroObjectContainer(bytes) {
5106
- return bytes.length >= 4 && bytes[0] === 79 && bytes[1] === 98 && bytes[2] === 106 && bytes[3] === 1;
5107
- }
5108
- async function decodeAvroObjectContainer(bytes) {
5109
- const avro = await loadAvro();
5110
- const avroBigIntLongType = avro.types.LongType.__with({
5111
- fromBuffer: (buffer) => buffer.readBigInt64LE(),
5112
- toBuffer: (value) => {
5113
- const buffer = Buffer.alloc(8);
5114
- buffer.writeBigInt64LE(BigInt(value));
5115
- return buffer;
5116
- },
5117
- fromJSON: (value) => BigInt(value),
5118
- toJSON: (value) => value.toString(),
5119
- isValid: (value) => typeof value === "bigint" || typeof value === "number" && Number.isSafeInteger(value),
5120
- compare: (left, right) => {
5121
- const leftBigInt = BigInt(left);
5122
- const rightBigInt = BigInt(right);
5123
- return leftBigInt === rightBigInt ? 0 : leftBigInt < rightBigInt ? -1 : 1;
5124
- }
5125
- });
5126
- const avroLongTypeHook = (schema) => schema === "long" || isRecord4(schema) && schema.type === "long" ? avroBigIntLongType : void 0;
5127
- const decoder = new avro.streams.BlockDecoder({
5128
- parseHook: (schema) => avro.Type.forSchema(schema, { typeHook: avroLongTypeHook })
5129
- });
5130
- const records = [];
5131
- const done = new Promise((resolve, reject) => {
5132
- decoder.on("data", (record) => records.push(record));
5133
- decoder.on("end", () => resolve(records));
5134
- decoder.on("error", reject);
5135
- });
5136
- decoder.end(Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength));
5137
- return done;
5138
- }
5139
- async function loadAvro() {
5140
- const module = await import('avsc');
5141
- return module.default ?? module;
5142
- }
5143
- function validateManifestList(value, path) {
5144
- const manifests = Array.isArray(value) ? value : isRecord4(value) && Array.isArray(value.manifests) ? value.manifests : void 0;
5145
- if (manifests === void 0) {
5146
- throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg manifest list has invalid required fields", {
5147
- path
5148
- });
5149
- }
5150
- return manifests.map((manifest) => validateManifestReference(manifest, path));
5151
- }
5152
- function validateManifest(value, path) {
5153
- if (!isRecord4(value) || typeof value.path !== "string" || !Array.isArray(value.files)) {
5154
- throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg manifest has invalid required fields", {
5155
- path
5156
- });
5157
- }
5158
- return value;
5159
- }
5160
- function validateManifestReference(value, path) {
5161
- if (!isRecord4(value) || typeof value.path !== "string") {
5162
- throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg manifest list entry is invalid", { path });
5163
- }
5164
- validateManifestContent(value.content, path);
5165
- if (Array.isArray(value.files))
5166
- return value;
5167
- return { path: value.path };
5168
- }
5169
- function validateManifestContent(content, path) {
5170
- if (content === void 0 || content === null)
5171
- return;
5172
- if (content === 0 || content === 1 || content === "data" || content === "deletes")
5173
- return;
5174
- throw new LaQLError("LAQL_UNSUPPORTED_ICEBERG_FEATURE", "Iceberg manifest list contains an unsupported manifest content type", {
5175
- path,
5176
- content
5177
- });
5178
- }
5179
- function validateManifestPaths(manifest, path, tablePrefix = "") {
5180
- validateManifestSourcedPath(manifest.path, tablePrefix);
5181
- for (const file of manifest.files) {
5182
- validateManifestSourcedPath(file.path, tablePrefix);
5183
- for (const deleteFile of file.deleteFiles ?? []) {
5184
- validateManifestSourcedPath(deleteFile.path, tablePrefix);
5185
- }
5186
- }
5187
- for (const deleteFile of manifest.deleteFiles ?? []) {
5188
- validateManifestSourcedPath(deleteFile.path, tablePrefix);
5189
- }
5190
- return { ...manifest, path };
5191
- }
5192
- function validateManifestSourcedPath(path, tablePrefix = "") {
5193
- if (/^(?:[a-z][a-z0-9+.-]*:)?\/\//iu.test(path) || path.startsWith("/")) {
5194
- throw new LaQLError("LAQL_VALIDATION_ERROR", `Iceberg manifest path must be relative: ${path}`, {
5195
- path
5196
- });
5197
- }
5198
- for (const segment of path.split("/")) {
5199
- let decoded;
5200
- try {
5201
- decoded = decodeURIComponent(segment);
5202
- } catch {
5203
- throw new LaQLError("LAQL_VALIDATION_ERROR", `Iceberg manifest path is invalid: ${path}`, {
5204
- path
5205
- });
5206
- }
5207
- if (decoded === "." || decoded === "..") {
5208
- throw new LaQLError("LAQL_VALIDATION_ERROR", `Iceberg manifest path contains traversal: ${path}`, {
5209
- path
5210
- });
5211
- }
5212
- }
5213
- if (tablePrefix !== "" && path !== tablePrefix && !path.startsWith(`${tablePrefix}/`)) {
5214
- throw new LaQLError("LAQL_VALIDATION_ERROR", `Iceberg manifest path escapes table location: ${path}`, {
5215
- path,
5216
- tableLocation: tablePrefix
5217
- });
5218
- }
5219
- return path;
5220
- }
5221
- function tableLocationObjectPrefix(location) {
5222
- const trimmed = trimTrailingSlash(location.trim());
5223
- if (trimmed === "" || !trimmed.includes("/"))
5224
- return "";
5225
- if (/^[a-z][a-z0-9+.-]*:\/\//iu.test(trimmed)) {
5226
- const url = new URL(trimmed);
5227
- return trimSlashes(decodeURIComponent(url.pathname));
5228
- }
5229
- return trimSlashes(trimmed);
5230
- }
5231
- function trimSlashes(value) {
5232
- return value.replace(/^\/+|\/+$/gu, "");
5233
- }
5234
- async function readVersionHint(store, path, controls = {}) {
5235
- const bytes = await store.get(path);
5236
- if (!bytes)
5237
- return void 0;
5238
- throwIfAborted(controls.signal);
5239
- const text = new TextDecoder().decode(bytes).trim();
5240
- const version = Number(text);
5241
- if (!Number.isInteger(version) || version < 0) {
5242
- throw new LaQLError("LAQL_CATALOG_ERROR", "Invalid Iceberg version hint", {
5243
- path,
5244
- versionHint: text
5245
- });
5246
- }
5247
- return version;
5248
- }
5249
- async function latestMetadataPathFromList(store, metadataPrefix, controls = {}) {
5250
- let latest;
5251
- for await (const object of store.list(metadataPrefix)) {
5252
- throwIfAborted(controls.signal);
5253
- const name = object.path.slice(metadataPrefix.length);
5254
- const match = /^v(\d+)\.metadata\.json$/u.exec(name);
5255
- if (!match)
5256
- continue;
5257
- const version = Number(match[1]);
5258
- if (!Number.isSafeInteger(version))
5259
- continue;
5260
- if (latest === void 0 || version > latest.version)
5261
- latest = { path: object.path, version };
5262
- }
5263
- if (latest === void 0) {
5264
- throw new LaQLError("LAQL_OBJECT_NOT_FOUND", "No Iceberg metadata files found", {
5265
- prefix: metadataPrefix
5266
- });
5267
- }
5268
- return latest.path;
5269
- }
5270
- function trimTrailingSlash(value) {
5271
- return value.endsWith("/") ? value.slice(0, -1) : value;
5272
- }
5273
- function restCatalogUrl(baseUrl, segments) {
5274
- const url = new URL(baseUrl);
5275
- const basePath = url.pathname.replace(/\/+$/u, "");
5276
- const encodedSegments = segments.map((segment) => encodeURIComponent(segment));
5277
- url.pathname = [basePath, ...encodedSegments].filter((segment) => segment.length > 0).join("/");
5278
- return url.toString();
5279
- }
5280
- function namespaceParts(namespace) {
5281
- const parts = Array.isArray(namespace) ? namespace : namespace.split(".");
5282
- const out = parts.map((part) => requiredNonEmptyString(part, "namespace"));
5283
- if (out.length === 0) {
5284
- throw new LaQLError("LAQL_VALIDATION_ERROR", "Iceberg REST namespace is required");
5285
- }
5286
- return out;
5287
- }
5288
- function catalogPrefixParts(prefix) {
5289
- if (prefix === void 0 || prefix.trim() === "")
5290
- return [];
5291
- return prefix.split("/").filter((part) => part.length > 0).map((part) => requiredNonEmptyString(part, "prefix"));
5292
- }
5293
- function requiredNonEmptyString(value, field) {
5294
- const trimmed = value.trim();
5295
- if (trimmed.length === 0) {
5296
- throw new LaQLError("LAQL_VALIDATION_ERROR", `Iceberg REST ${field} is required`);
5297
- }
5298
- return trimmed;
5299
- }
5300
- async function commitResponseMetadataPath(response) {
5301
- if (response.status === 204)
5302
- return void 0;
5303
- try {
5304
- const body = await response.json();
5305
- if (isRecord4(body) && typeof body["metadata-location"] === "string") {
5306
- return body["metadata-location"];
5307
- }
5308
- } catch {
5309
- return void 0;
5310
- }
5311
- return void 0;
5312
- }
5313
- function restAppendSnapshot(input) {
5314
- const snapshot = input.metadata.snapshots.at(-1);
5315
- if (snapshot === void 0) {
5316
- throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg append metadata is missing next snapshot");
5317
- }
5318
- return {
5319
- "snapshot-id": snapshot["snapshot-id"],
5320
- "parent-snapshot-id": input.expectedSnapshotId,
5321
- "timestamp-ms": snapshot["timestamp-ms"],
5322
- "schema-id": snapshot["schema-id"],
5323
- "manifest-list": input.manifestPath,
5324
- summary: {
5325
- operation: "append",
5326
- "added-data-files": String(input.manifest.files.length),
5327
- "added-records": String(input.manifest.files.reduce((sum, file) => sum + file.recordCount, 0))
5328
- }
5329
- };
5330
- }
5331
- function equalityKey(row, columns) {
5332
- return stableStringify(columns.map((column) => {
5333
- if (!(column in row)) {
5334
- throw new LaQLError("LAQL_UNKNOWN_COLUMN", `Unknown Iceberg equality delete column ${column}`, {
5335
- column
5336
- });
5337
- }
5338
- return jsonSafeValue(row[column]);
5339
- }));
5340
- }
5341
- function appendManifestPath(metadataPath, jobId, snapshotId) {
5342
- const slash = metadataPath.lastIndexOf("/");
5343
- const prefix = slash === -1 ? "" : `${metadataPath.slice(0, slash + 1)}`;
5344
- return `${prefix}${jobId ?? "append"}-${snapshotId}.manifest.json`;
5345
- }
5346
- function maxSequenceNumber(metadata) {
5347
- let max = 0;
5348
- for (const snapshot of metadata.snapshots) {
5349
- for (const manifest of snapshotManifests(snapshot)) {
5350
- for (const file of manifest.files)
5351
- max = Math.max(max, file.sequenceNumber);
5352
- }
5353
- }
5354
- return max;
5355
- }
5356
- function supportedIcebergDeleteFiles(deleteFiles) {
5357
- const supported = [];
5358
- for (const deleteFile of deleteFiles ?? []) {
5359
- if (deleteFile.content === "position-delete" || deleteFile.content === "equality-delete") {
5360
- supported.push({ content: deleteFile.content, path: deleteFile.path });
5361
- }
5362
- }
5363
- return supported;
5364
- }
5365
- function unsupportedIcebergDeleteFiles(deleteFiles) {
5366
- const supported = /* @__PURE__ */ new Set(["position-delete", "equality-delete"]);
5367
- return (deleteFiles ?? []).filter((deleteFile) => !supported.has(deleteFile.content));
5368
- }
5369
- function cloneMetadata(metadata) {
5370
- const cloned = {
5371
- "format-version": metadata["format-version"],
5372
- "table-uuid": metadata["table-uuid"],
5373
- location: metadata.location,
5374
- "current-snapshot-id": metadata["current-snapshot-id"],
5375
- schemas: metadata.schemas.map((schema) => ({
5376
- "schema-id": schema["schema-id"],
5377
- fields: schema.fields.map((field) => ({ ...field }))
5378
- })),
5379
- snapshots: metadata.snapshots.map((snapshot) => {
5380
- const cloned2 = {
5381
- "snapshot-id": snapshot["snapshot-id"],
5382
- "timestamp-ms": snapshot["timestamp-ms"],
5383
- "schema-id": snapshot["schema-id"]
5384
- };
5385
- if (snapshot["manifest-list"] !== void 0)
5386
- cloned2["manifest-list"] = snapshot["manifest-list"];
5387
- if (snapshot.manifests !== void 0) {
5388
- cloned2.manifests = snapshot.manifests.map(cloneManifestOrReference);
5389
- }
5390
- return cloned2;
5391
- })
5392
- };
5393
- if (metadata.refs)
5394
- cloned.refs = cloneRefs(metadata.refs);
5395
- if (metadata["partition-specs"]) {
5396
- cloned["partition-specs"] = metadata["partition-specs"].map((spec) => ({
5397
- "spec-id": spec["spec-id"],
5398
- fields: spec.fields.map((field) => ({ ...field }))
5399
- }));
5400
- }
5401
- if (metadata["sort-orders"]) {
5402
- cloned["sort-orders"] = metadata["sort-orders"].map((order) => ({
5403
- "order-id": order["order-id"],
5404
- fields: order.fields.map((field) => field)
5405
- }));
5406
- }
5407
- return cloned;
5408
- }
5409
- function cloneRefs(refs) {
5410
- const out = {};
5411
- for (const [name, ref] of Object.entries(refs)) {
5412
- out[name] = { type: ref.type, "snapshot-id": ref["snapshot-id"] };
5413
- }
5414
- return out;
5415
- }
5416
- function cloneManifest(manifest) {
5417
- const cloned = {
5418
- path: manifest.path,
5419
- files: manifest.files.map((file) => {
5420
- const cloned2 = {
5421
- path: file.path,
5422
- sequenceNumber: file.sequenceNumber,
5423
- partition: sortStringRecord(file.partition ?? {}),
5424
- recordCount: file.recordCount
5425
- };
5426
- if (file.fileSizeInBytes !== void 0)
5427
- cloned2.fileSizeInBytes = file.fileSizeInBytes;
5428
- if (file.deleteFiles !== void 0) {
5429
- cloned2.deleteFiles = file.deleteFiles.map((deleteFile) => ({ ...deleteFile }));
5430
- }
5431
- return cloned2;
5432
- })
5433
- };
5434
- if (manifest.deleteFiles !== void 0) {
5435
- cloned.deleteFiles = manifest.deleteFiles.map((deleteFile) => ({ ...deleteFile }));
5436
- }
5437
- return cloned;
5438
- }
5439
- function cloneManifestOrReference(manifest) {
5440
- if (!Array.isArray(manifest.files)) {
5441
- return { path: manifest.path };
5442
- }
5443
- return cloneManifest(manifest);
5444
- }
5445
- function snapshotManifests(snapshot) {
5446
- return snapshot.manifests ?? [];
5447
- }
5448
- function sortStringRecord(record) {
5449
- const out = {};
5450
- for (const key of Object.keys(record).sort())
5451
- out[key] = record[key] ?? "";
5452
- return out;
5453
- }
5454
- function validateListTablesResponse(value) {
5455
- if (!isRecord4(value) || !Array.isArray(value.identifiers)) {
5456
- throw new LaQLError("LAQL_CATALOG_ERROR", "Invalid Iceberg REST list tables response");
5457
- }
5458
- return value.identifiers.map((identifier) => {
5459
- if (!isRecord4(identifier) || !Array.isArray(identifier.namespace) || !identifier.namespace.every((part) => typeof part === "string") || typeof identifier.name !== "string") {
5460
- throw new LaQLError("LAQL_CATALOG_ERROR", "Invalid Iceberg REST table identifier");
5461
- }
5462
- return { namespace: identifier.namespace, name: identifier.name };
5463
- });
5464
- }
5465
- function validateMetadata(value) {
5466
- if (!isRecord4(value))
5467
- throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg metadata must be an object");
5468
- if (value["format-version"] !== 1 && value["format-version"] !== 2) {
5469
- throw new LaQLError("LAQL_CATALOG_ERROR", "Only Iceberg format-version 1 and 2 metadata is supported for reads", {
5470
- formatVersion: value["format-version"]
5471
- });
5472
- }
5473
- if (!Array.isArray(value.snapshots) || !Array.isArray(value.schemas)) {
5474
- throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg metadata is missing snapshots or schemas");
5475
- }
5476
- if (!isMetadataFile(value)) {
5477
- throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg metadata has invalid required fields");
5478
- }
5479
- rejectUnsupportedMetadataFeatures(value);
5480
- return value;
5481
- }
5482
- function rejectUnsupportedMetadataFeatures(metadata) {
5483
- rejectAdvertisedFeatureFlags(metadata);
5484
- rejectUnsupportedPartitionTransforms(metadata["partition-specs"]);
5485
- rejectUnsupportedSortOrders(metadata["sort-orders"]);
5486
- }
5487
- function rejectAdvertisedFeatureFlags(metadata) {
5488
- for (const key of ["features", "table-features", "format-version-features"]) {
5489
- const features = metadata[key];
5490
- if (features === void 0 || Array.isArray(features) && features.length === 0)
5491
- continue;
5492
- throw new LaQLError("LAQL_UNSUPPORTED_ICEBERG_FEATURE", "Iceberg metadata advertises unsupported table-format features", {
5493
- featureProperty: key,
5494
- features
5495
- });
5496
- }
5497
- }
5498
- function rejectUnsupportedPartitionTransforms(partitionSpecs) {
5499
- if (partitionSpecs === void 0)
5500
- return;
5501
- if (!Array.isArray(partitionSpecs)) {
5502
- throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg partition-specs must be an array");
5503
- }
5504
- for (const spec of partitionSpecs) {
5505
- if (!isRecord4(spec) || !Array.isArray(spec.fields)) {
5506
- throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg partition spec is invalid");
5507
- }
5508
- for (const field of spec.fields) {
5509
- if (!isRecord4(field) || typeof field.transform !== "string") {
5510
- throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg partition field is invalid");
5511
- }
5512
- if (field.transform !== "identity" && field.transform !== "void") {
5513
- throw new LaQLError("LAQL_UNSUPPORTED_ICEBERG_FEATURE", "Iceberg partition transform is not supported for strict planning", {
5514
- specId: spec["spec-id"],
5515
- fieldName: field.name,
5516
- transform: field.transform
5517
- });
5518
- }
5519
- }
5520
- }
5521
- }
5522
- function rejectUnsupportedSortOrders(sortOrders) {
5523
- if (sortOrders === void 0)
5524
- return;
5525
- if (!Array.isArray(sortOrders)) {
5526
- throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg sort-orders must be an array");
5527
- }
5528
- for (const order of sortOrders) {
5529
- if (!isRecord4(order) || !Array.isArray(order.fields)) {
5530
- throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg sort order is invalid");
5531
- }
5532
- if (order.fields.length > 0) {
5533
- throw new LaQLError("LAQL_UNSUPPORTED_ICEBERG_FEATURE", "Iceberg sorted table metadata is not supported for strict planning", {
5534
- orderId: order["order-id"],
5535
- fields: order.fields
5536
- });
5537
- }
5538
- }
5539
- }
5540
- function isMetadataFile(value) {
5541
- if (!isRecord4(value))
5542
- return false;
5543
- return (value["format-version"] === 1 || value["format-version"] === 2) && typeof value["table-uuid"] === "string" && typeof value.location === "string" && typeof value["current-snapshot-id"] === "number" && Array.isArray(value.refs) === false && Array.isArray(value.schemas) && Array.isArray(value.snapshots);
5544
- }
5545
- function projectedIds(fields, select) {
5546
- if (!select)
5547
- return fields.map((field) => field.id).sort((a, b) => a - b);
5548
- return select.map((name) => {
5549
- const field = fields.find((candidate) => candidate.name === name);
5550
- if (!field) {
5551
- throw new LaQLError("LAQL_UNKNOWN_COLUMN", `Unknown Iceberg column ${name}`, {
5552
- column: name
5553
- });
5554
- }
5555
- return field.sourceId ?? field.id;
5556
- }).sort((a, b) => a - b);
5557
- }
5558
- function partitionMayMatch2(expr, partition) {
5559
- if (!expr)
5560
- return true;
5561
- const columns = /* @__PURE__ */ new Set();
5562
- collectColumns2(expr, columns);
5563
- if (columns.size === 0 || [...columns].some((column) => !(column in partition)))
5564
- return true;
5565
- try {
5566
- return matches(expr, partition);
5567
- } catch (cause) {
5568
- if (cause instanceof LaQLError && cause.code === "LAQL_TYPE_ERROR")
5569
- return true;
5570
- throw cause;
5571
- }
5572
- }
5573
- function collectColumns2(expr, columns) {
5574
- switch (expr.kind) {
5575
- case "column":
5576
- columns.add(expr.name);
5577
- return;
5578
- case "literal":
5579
- return;
5580
- case "compare":
5581
- collectColumns2(expr.left, columns);
5582
- collectColumns2(expr.right, columns);
5583
- return;
5584
- case "in":
5585
- collectColumns2(expr.target, columns);
5586
- for (const value of expr.values)
5587
- collectColumns2(value, columns);
5588
- return;
5589
- case "between":
5590
- collectColumns2(expr.target, columns);
5591
- collectColumns2(expr.low, columns);
5592
- collectColumns2(expr.high, columns);
5593
- return;
5594
- case "null-check":
5595
- collectColumns2(expr.target, columns);
5596
- return;
5597
- case "logical":
5598
- for (const operand of expr.operands)
5599
- collectColumns2(operand, columns);
5600
- return;
5601
- case "not":
5602
- collectColumns2(expr.operand, columns);
5603
- return;
5604
- case "like":
5605
- collectColumns2(expr.target, columns);
5606
- return;
5607
- case "call":
5608
- for (const arg of expr.args)
5609
- collectColumns2(arg, columns);
5610
- return;
5611
- }
5612
- }
5613
- function isRecord4(value) {
5614
- return typeof value === "object" && value !== null && !Array.isArray(value);
5615
- }
5616
4541
  async function asyncBufferFromStore(store, path, options = void 0) {
5617
4542
  const controlledStore = options === void 0 ? store : withObjectStoreReadControls(store, options.budget);
5618
4543
  throwIfAborted(options?.budget.signal);
@@ -5847,7 +4772,7 @@ function partitionedParquetOutputEntries(result, options) {
5847
4772
  const entry = {
5848
4773
  taskId: typeof options.taskId === "function" ? options.taskId(file, index) : options.taskId,
5849
4774
  outputPath: file.path,
5850
- partitionValues: sortStringRecord2(file.partitionValues),
4775
+ partitionValues: sortStringRecord(file.partitionValues),
5851
4776
  rowCount: file.rowCount,
5852
4777
  byteSize: file.byteSize,
5853
4778
  contentHash: file.contentHash
@@ -5858,7 +4783,7 @@ function partitionedParquetOutputEntries(result, options) {
5858
4783
  entry.iceberg = {
5859
4784
  recordCount: file.rowCount,
5860
4785
  fileSizeInBytes: file.byteSize,
5861
- partitionValues: sortStringRecord2(file.partitionValues)
4786
+ partitionValues: sortStringRecord(file.partitionValues)
5862
4787
  };
5863
4788
  }
5864
4789
  return entry;
@@ -5872,7 +4797,7 @@ function outputEntriesToPartitionedResult(entries) {
5872
4797
  byteSize: entry.byteSize,
5873
4798
  contentHash: entry.contentHash ?? "",
5874
4799
  rowCount: entry.rowCount,
5875
- partitionValues: sortStringRecord2(entry.partitionValues)
4800
+ partitionValues: sortStringRecord(entry.partitionValues)
5876
4801
  };
5877
4802
  if (entry.etag !== void 0)
5878
4803
  file.etag = entry.etag;
@@ -6288,7 +5213,7 @@ function compareInsertValues(left, right, column, rowIndex) {
6288
5213
  return 1;
6289
5214
  return 0;
6290
5215
  }
6291
- function sortStringRecord2(record) {
5216
+ function sortStringRecord(record) {
6292
5217
  const out = {};
6293
5218
  for (const key of Object.keys(record).sort())
6294
5219
  out[key] = record[key] ?? "";
@@ -6472,6 +5397,8 @@ function rowGroupMayMatch(rowGroup, expr) {
6472
5397
  case "column":
6473
5398
  case "null-check":
6474
5399
  case "like":
5400
+ case "arithmetic":
5401
+ case "case":
6475
5402
  return true;
6476
5403
  case "call":
6477
5404
  return callMayMatch2(rowGroup, expr);
@@ -6747,78 +5674,4 @@ function createParquetLake(config) {
6747
5674
  });
6748
5675
  }
6749
5676
 
6750
- // src/engine.ts
6751
- async function loadTable(options) {
6752
- if (options.format === "parquet") {
6753
- return { format: "parquet", store: options.store, path: options.path };
6754
- }
6755
- const { format: _format, ...icebergOptions } = options;
6756
- return {
6757
- format: "iceberg",
6758
- store: options.store,
6759
- table: await loadIcebergTable(icebergOptions)
6760
- };
6761
- }
6762
- function planFiles2(table, options = {}) {
6763
- if (table.format === "parquet") {
6764
- return { format: "parquet", store: table.store, files: [{ path: table.path }] };
6765
- }
6766
- return {
6767
- format: "iceberg",
6768
- store: table.store,
6769
- table: table.table,
6770
- plan: planFiles(table.table, options),
6771
- options
6772
- };
6773
- }
6774
- async function* scanBatches(plan, options = {}) {
6775
- if (plan.format === "parquet") {
6776
- for (const file of plan.files) {
6777
- const readOptions = {};
6778
- if (options.batchSize !== void 0) readOptions.batchSize = options.batchSize;
6779
- for await (const batch of readParquetObjectBatches(plan.store, file.path, readOptions)) {
6780
- yield batch.rows;
6781
- }
6782
- }
6783
- return;
6784
- }
6785
- for await (const batch of scanPlannedIcebergRows({
6786
- plan: plan.plan,
6787
- ...options,
6788
- readDataFile: async (file) => projectIcebergParquetBatches(
6789
- plan.store,
6790
- plan.table,
6791
- file.path,
6792
- file.partition,
6793
- file.snapshotId,
6794
- plan.options,
6795
- options
6796
- ),
6797
- readDeleteFile: async (deleteFile) => readIcebergParquetDeletes(plan.store, deleteFile)
6798
- })) {
6799
- yield batch;
6800
- }
6801
- }
6802
- async function* scanRows(plan, options = {}) {
6803
- for await (const batch of scanBatches(plan, options)) {
6804
- for (const row of batch) {
6805
- yield row;
6806
- }
6807
- }
6808
- }
6809
- async function* projectIcebergParquetBatches(store, table, path, partition, snapshotId, planOptions, scanOptions) {
6810
- const readOptions = {};
6811
- if (scanOptions.batchSize !== void 0) readOptions.batchSize = scanOptions.batchSize;
6812
- for await (const batch of readParquetObjectBatches(store, path, readOptions)) {
6813
- yield {
6814
- rowOffset: batch.rowOffset,
6815
- rows: batch.rows.map((row) => {
6816
- const projectOptions = { snapshotId };
6817
- if (planOptions.select !== void 0) projectOptions.select = planOptions.select;
6818
- return table.projectRow({ ...partition, ...row }, projectOptions);
6819
- })
6820
- };
6821
- }
6822
- }
6823
-
6824
- export { AggregationBuilder, CacheApiCache, ERROR_CODES, IcebergRestCatalog, IcebergTable, IcebergUnsupportedCatalog, LaQLError, Lake, MemoryCache, MemoryCheckpointAdapter, MemoryObjectStore, MemorySpillAdapter, ObjectStoreIcebergCommitCatalog, PACKAGE, ParquetScanAdapter, QueryBuilder, QueryResult, ResumedQuery, advanceTaskCheckpoint, and, applyIcebergDeletes, assertBookmarkMatches, asyncBufferFromStore, bboxMayIntersect, between, broadcastJoin, buildBBoxIndex, buildMinMaxIndex, cacheApiCache, classifyPredicate, col, createBookmark, createOutputManifest, createOutputManifestFromCheckpoints, createParquetLake, createParquetTableAs, createTaskManifest, deserializeAggregateOperatorState, deserializeSortOperatorState, deserializeTopKOperatorState, encodeJsonLine, eq, evaluate, fingerprint, fn, gt, gte, icebergGlueCatalog, icebergNessieCatalog, icebergRestCatalog, ilike, isIn, isLaQLError, isNotNull, isNull, jsonSafeValue, like, lit, loadIcebergTable, loadIcebergTableFromObjectStore, loadIcebergTableFromRest, loadTable, lookupJoin, lt, lte, matches, memoryCache, memoryCheckpointAdapter, memorySpillAdapter, memoryStore, ne, not, notIn, or, parquetScanner, parseHivePartitions, parseJsonQuery, partitionedParquetOutputEntries, planFiles2 as planFiles, planRowGroups, planRowGroupsFromMetadata, pruneFilesWithIndex, readControlSignal, readIcebergParquetDeletes, readOutputManifest, readParquetMetadata, readParquetObjectBatches, readParquetObjects, rejectUnsupportedParquetSchema, rowGroupMayMatch, scanBatches, scanPlannedIcebergRows, scanRows, serializeAggregateOperatorState, serializeSortOperatorState, serializeTopKOperatorState, signPaginationToken, stableStringify, throwIfAborted, transitionTaskCheckpoint, verifyPaginationToken, withObjectStoreReadControls, writeOutputManifest, writeParquet, writePartitionedParquet, writePartitionedParquetTask };
5677
+ export { AggregationBuilder, CacheApiCache, ERROR_CODES, LaQLError, Lake, MemoryCache, MemoryCheckpointAdapter, MemoryObjectStore, MemorySpillAdapter, ParquetScanAdapter, QueryBuilder, QueryResult, ResumedQuery, __commonJS, __toESM, add, advanceTaskCheckpoint, and, assertBookmarkMatches, asyncBufferFromStore, bboxMayIntersect, between, broadcastJoin, buildBBoxIndex, buildMinMaxIndex, cacheApiCache, classifyPredicate, col, createBookmark, createOutputManifest, createOutputManifestFromCheckpoints, createParquetLake, createParquetTableAs, createTaskManifest, deserializeAggregateOperatorState, deserializeSortOperatorState, deserializeTopKOperatorState, div, encodeJsonLine, eq, evaluate, fingerprint, fn, gt, gte, ilike, isIn, isLaQLError, isNotNull, isNull, jsonSafeValue, like, lit, lookupJoin, lt, lte, matches, memoryCache, memoryCheckpointAdapter, memorySpillAdapter, memoryStore, mod, mul, ne, not, notIn, or, parquetScanner, parseHivePartitions, parseJsonQuery, partitionedParquetOutputEntries, planRowGroups, planRowGroupsFromMetadata, pruneFilesWithIndex, readControlSignal, readIcebergParquetDeletes, readOutputManifest, readParquetMetadata, readParquetObjectBatches, readParquetObjects, rejectUnsupportedParquetSchema, rowGroupMayMatch, serializeAggregateOperatorState, serializeSortOperatorState, serializeTopKOperatorState, signPaginationToken, stableStringify, sub, throwIfAborted, transitionTaskCheckpoint, verifyPaginationToken, withObjectStoreReadControls, writeOutputManifest, writeParquet, writePartitionedParquet, writePartitionedParquetTask };