lakeql 0.1.2 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin.js +205 -81
- package/dist/{chunk-MUTXXFOW.js → chunk-5K5JMJ2M.js} +772 -131
- package/dist/chunk-MXGEAVHL.js +11381 -0
- package/dist/chunk-TFD5RFKB.js +1152 -0
- package/dist/cloudflare.d.ts +21 -5
- package/dist/cloudflare.js +105 -5
- package/dist/geo-backend-TSQJWAAB.js +18 -0
- package/dist/index.d.ts +842 -232
- package/dist/index.js +3 -2
- package/dist/node.d.ts +29 -7
- package/dist/node.js +195 -35
- package/package.json +13 -10
- package/dist/chunk-D3A4VN3U.js +0 -5677
|
@@ -1,7 +1,402 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Lake, stableStringify, vectorValue, scalarVectorValue, batchFromColumns, withObjectStoreReadControls, throwIfAborted, readControlSignal, readParquetObjectBatches, readIcebergParquetDeletes } from './chunk-MXGEAVHL.js';
|
|
2
|
+
import { LakeqlError, jsonSafeValue, matches } from './chunk-TFD5RFKB.js';
|
|
3
|
+
|
|
4
|
+
// ../core/dist/in-memory.js
|
|
5
|
+
var InMemoryRowsScanner = class {
|
|
6
|
+
tables;
|
|
7
|
+
constructor(tables, options = {}) {
|
|
8
|
+
this.tables = normalizeInMemoryTables(tables, options);
|
|
9
|
+
}
|
|
10
|
+
async *scan(path, options) {
|
|
11
|
+
const table = this.table(path);
|
|
12
|
+
const batchSize = options.batchSize;
|
|
13
|
+
for (let start = 0; start < table.rows.length; start += batchSize) {
|
|
14
|
+
const out = [];
|
|
15
|
+
for (const row of table.rows.slice(start, start + batchSize)) {
|
|
16
|
+
out.push(projectPhysicalColumns(row, options.columns));
|
|
17
|
+
}
|
|
18
|
+
yield out;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
async planTask(path) {
|
|
22
|
+
return { rowGroupRanges: [{ start: 0, end: this.table(path).rows.length }] };
|
|
23
|
+
}
|
|
24
|
+
tableInfo(path) {
|
|
25
|
+
const table = this.table(path);
|
|
26
|
+
return {
|
|
27
|
+
path: table.path,
|
|
28
|
+
size: table.size,
|
|
29
|
+
etag: table.etag,
|
|
30
|
+
lastModified: table.lastModified
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
listTables(prefix, options = {}) {
|
|
34
|
+
const out = [];
|
|
35
|
+
for (const table of [...this.tables.values()].sort((left, right) => left.path.localeCompare(right.path))) {
|
|
36
|
+
if (!table.path.startsWith(prefix))
|
|
37
|
+
continue;
|
|
38
|
+
out.push({
|
|
39
|
+
path: table.path,
|
|
40
|
+
size: table.size,
|
|
41
|
+
etag: table.etag,
|
|
42
|
+
lastModified: table.lastModified
|
|
43
|
+
});
|
|
44
|
+
if (options.limit !== void 0 && out.length >= options.limit)
|
|
45
|
+
break;
|
|
46
|
+
}
|
|
47
|
+
return out;
|
|
48
|
+
}
|
|
49
|
+
table(path) {
|
|
50
|
+
const table = this.tables.get(path);
|
|
51
|
+
if (table === void 0) {
|
|
52
|
+
throw new LakeqlError("LAKEQL_UNKNOWN_TABLE", `No in-memory table named ${path}`, { path });
|
|
53
|
+
}
|
|
54
|
+
return table;
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
var InMemoryRowsStore = class {
|
|
58
|
+
scanner;
|
|
59
|
+
constructor(scanner) {
|
|
60
|
+
this.scanner = scanner;
|
|
61
|
+
}
|
|
62
|
+
async get(path) {
|
|
63
|
+
this.scanner.tableInfo(path);
|
|
64
|
+
return null;
|
|
65
|
+
}
|
|
66
|
+
async getRange(path) {
|
|
67
|
+
this.scanner.tableInfo(path);
|
|
68
|
+
return new Uint8Array();
|
|
69
|
+
}
|
|
70
|
+
async put() {
|
|
71
|
+
throw new LakeqlError("LAKEQL_TYPE_ERROR", "In-memory row stores are read-only");
|
|
72
|
+
}
|
|
73
|
+
async delete() {
|
|
74
|
+
throw new LakeqlError("LAKEQL_TYPE_ERROR", "In-memory row stores are read-only");
|
|
75
|
+
}
|
|
76
|
+
async *list(prefix, options) {
|
|
77
|
+
yield* this.scanner.listTables(prefix, options);
|
|
78
|
+
}
|
|
79
|
+
async head(path) {
|
|
80
|
+
try {
|
|
81
|
+
const table = this.scanner.tableInfo(path);
|
|
82
|
+
const head = {
|
|
83
|
+
size: table.size,
|
|
84
|
+
contentType: "application/vnd.lakeql.rows+json"
|
|
85
|
+
};
|
|
86
|
+
if (table.etag !== void 0)
|
|
87
|
+
head.etag = table.etag;
|
|
88
|
+
if (table.lastModified !== void 0)
|
|
89
|
+
head.lastModified = table.lastModified;
|
|
90
|
+
return head;
|
|
91
|
+
} catch (cause) {
|
|
92
|
+
if (cause instanceof LakeqlError && cause.code === "LAKEQL_UNKNOWN_TABLE")
|
|
93
|
+
return null;
|
|
94
|
+
throw cause;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
};
|
|
98
|
+
function inMemoryRowsScanner(tables, options = {}) {
|
|
99
|
+
return new InMemoryRowsScanner(tables, options);
|
|
100
|
+
}
|
|
101
|
+
function createInMemoryLake(tables, options = {}) {
|
|
102
|
+
const { maxRows, maxBytes, ...lakeOptions } = options;
|
|
103
|
+
const tableOptions = {};
|
|
104
|
+
if (maxRows !== void 0)
|
|
105
|
+
tableOptions.maxRows = maxRows;
|
|
106
|
+
if (maxBytes !== void 0)
|
|
107
|
+
tableOptions.maxBytes = maxBytes;
|
|
108
|
+
const scanner = inMemoryRowsScanner(tables, tableOptions);
|
|
109
|
+
return new Lake({
|
|
110
|
+
...lakeOptions,
|
|
111
|
+
store: new InMemoryRowsStore(scanner),
|
|
112
|
+
scanner
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
function normalizeInMemoryTables(tables, options) {
|
|
116
|
+
const normalized = /* @__PURE__ */ new Map();
|
|
117
|
+
let totalRows = 0;
|
|
118
|
+
let totalBytes = 0;
|
|
119
|
+
for (const [path, rows] of Object.entries(tables)) {
|
|
120
|
+
if (path.trim() === "") {
|
|
121
|
+
throw new LakeqlError("LAKEQL_TYPE_ERROR", "In-memory table names must be non-empty");
|
|
122
|
+
}
|
|
123
|
+
const copiedRows = rows.map((row) => ({ ...row }));
|
|
124
|
+
const size = estimateRowsBytes(copiedRows);
|
|
125
|
+
totalRows += copiedRows.length;
|
|
126
|
+
totalBytes += size;
|
|
127
|
+
enforceInMemoryTableBudget(options, totalRows, totalBytes);
|
|
128
|
+
normalized.set(path, {
|
|
129
|
+
path,
|
|
130
|
+
rows: copiedRows,
|
|
131
|
+
size,
|
|
132
|
+
etag: `memory-${stableStringify([path, copiedRows.length, size])}`,
|
|
133
|
+
lastModified: /* @__PURE__ */ new Date(0)
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
return normalized;
|
|
137
|
+
}
|
|
138
|
+
function enforceInMemoryTableBudget(options, rows, bytes) {
|
|
139
|
+
if (options.maxRows !== void 0 && rows > options.maxRows) {
|
|
140
|
+
throw new LakeqlError("LAKEQL_BUDGET_EXCEEDED", `In-memory ingest exceeded row budget (${rows} > ${options.maxRows})`, { metric: "ingest rows", limit: options.maxRows, actual: rows });
|
|
141
|
+
}
|
|
142
|
+
if (options.maxBytes !== void 0 && bytes > options.maxBytes) {
|
|
143
|
+
throw new LakeqlError("LAKEQL_BUDGET_EXCEEDED", `In-memory ingest exceeded byte budget (${bytes} > ${options.maxBytes})`, { metric: "ingest bytes", limit: options.maxBytes, actual: bytes });
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
function estimateRowsBytes(rows) {
|
|
147
|
+
return new TextEncoder().encode(stableStringify(jsonSafeValue(rows))).byteLength;
|
|
148
|
+
}
|
|
149
|
+
function projectPhysicalColumns(row, columns) {
|
|
150
|
+
if (columns === void 0)
|
|
151
|
+
return { ...row };
|
|
152
|
+
const out = {};
|
|
153
|
+
for (const column of columns) {
|
|
154
|
+
if (column in row)
|
|
155
|
+
out[column] = row[column];
|
|
156
|
+
}
|
|
157
|
+
return out;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// ../core/dist/object-store-json-cache.js
|
|
161
|
+
var ObjectStoreJsonCache = class {
|
|
162
|
+
store;
|
|
163
|
+
prefix;
|
|
164
|
+
ttlMs;
|
|
165
|
+
now;
|
|
166
|
+
constructor(options) {
|
|
167
|
+
this.store = options.store;
|
|
168
|
+
this.prefix = normalizeCachePrefix(options.prefix);
|
|
169
|
+
this.ttlMs = options.ttlMs;
|
|
170
|
+
this.now = options.now ?? Date.now;
|
|
171
|
+
}
|
|
172
|
+
async get(key) {
|
|
173
|
+
const path = this.pathFor(key);
|
|
174
|
+
const bytes = await this.store.get(path);
|
|
175
|
+
if (bytes === null)
|
|
176
|
+
return void 0;
|
|
177
|
+
let parsed;
|
|
178
|
+
try {
|
|
179
|
+
parsed = JSON.parse(new TextDecoder().decode(bytes));
|
|
180
|
+
} catch (cause) {
|
|
181
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", `Invalid JSON cache entry at ${path}`, {
|
|
182
|
+
path,
|
|
183
|
+
cause
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
if (!isStoredJsonCacheEntry(parsed)) {
|
|
187
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", `Invalid JSON cache entry at ${path}`, {
|
|
188
|
+
path
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
if (parsed.expiresAt !== void 0 && parsed.expiresAt <= this.now()) {
|
|
192
|
+
await this.store.delete(path);
|
|
193
|
+
return void 0;
|
|
194
|
+
}
|
|
195
|
+
const entry = { value: parsed.value };
|
|
196
|
+
if (parsed.expiresAt !== void 0)
|
|
197
|
+
entry.expiresAt = parsed.expiresAt;
|
|
198
|
+
return entry;
|
|
199
|
+
}
|
|
200
|
+
async set(key, entry) {
|
|
201
|
+
if (entry.value === void 0) {
|
|
202
|
+
throw new LakeqlError("LAKEQL_TYPE_ERROR", "JSON cache value is not JSON serializable", {
|
|
203
|
+
key
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
const stored = { value: entry.value };
|
|
207
|
+
if (entry.expiresAt !== void 0) {
|
|
208
|
+
stored.expiresAt = entry.expiresAt;
|
|
209
|
+
} else if (this.ttlMs !== void 0) {
|
|
210
|
+
stored.expiresAt = this.now() + this.ttlMs;
|
|
211
|
+
}
|
|
212
|
+
let json;
|
|
213
|
+
try {
|
|
214
|
+
json = JSON.stringify(stored);
|
|
215
|
+
} catch (cause) {
|
|
216
|
+
throw new LakeqlError("LAKEQL_TYPE_ERROR", "JSON cache value is not JSON serializable", {
|
|
217
|
+
key,
|
|
218
|
+
cause
|
|
219
|
+
});
|
|
220
|
+
}
|
|
221
|
+
await this.store.put(this.pathFor(key), new TextEncoder().encode(json), {
|
|
222
|
+
contentType: "application/json"
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
async delete(key) {
|
|
226
|
+
await this.store.delete(this.pathFor(key));
|
|
227
|
+
}
|
|
228
|
+
pathFor(key) {
|
|
229
|
+
return `${this.prefix}/${encodeURIComponent(key)}.json`;
|
|
230
|
+
}
|
|
231
|
+
};
|
|
232
|
+
function objectStoreJsonCache(options) {
|
|
233
|
+
return new ObjectStoreJsonCache(options);
|
|
234
|
+
}
|
|
235
|
+
function normalizeCachePrefix(prefix) {
|
|
236
|
+
const trimmed = prefix.replace(/^\/+|\/+$/g, "");
|
|
237
|
+
if (trimmed === "") {
|
|
238
|
+
throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "JSON cache prefix must be non-empty");
|
|
239
|
+
}
|
|
240
|
+
return trimmed;
|
|
241
|
+
}
|
|
242
|
+
function isStoredJsonCacheEntry(value) {
|
|
243
|
+
return typeof value === "object" && value !== null && "value" in value;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// ../core/dist/vector-join.js
|
|
247
|
+
function vectorHashJoin(left, right, options) {
|
|
248
|
+
const normalized = validateVectorJoinOptions(left, right, options);
|
|
249
|
+
const rightIndices = selectedIndices(right.rowCount, options.rightSelection);
|
|
250
|
+
enforceMaxRightRows(rightIndices.length, options.maxRightRows);
|
|
251
|
+
const index = buildRightIndex(right, rightIndices, normalized.rightKeys);
|
|
252
|
+
const output = createOutputColumns(left, right, normalized);
|
|
253
|
+
for (const leftIndex of selectedIndices(left.rowCount, options.leftSelection)) {
|
|
254
|
+
const matches2 = index.get(joinKey(left, leftIndex, normalized.leftKeys));
|
|
255
|
+
if (normalized.type === "semi") {
|
|
256
|
+
if (matches2 !== void 0 && matches2.length > 0)
|
|
257
|
+
appendLeftOnly(output, left, leftIndex);
|
|
258
|
+
continue;
|
|
259
|
+
}
|
|
260
|
+
if (normalized.type === "anti") {
|
|
261
|
+
if (matches2 === void 0 || matches2.length === 0)
|
|
262
|
+
appendLeftOnly(output, left, leftIndex);
|
|
263
|
+
continue;
|
|
264
|
+
}
|
|
265
|
+
if (matches2 === void 0 || matches2.length === 0) {
|
|
266
|
+
if (normalized.type === "left")
|
|
267
|
+
appendLeftWithNullRight(output, left, right, leftIndex, normalized);
|
|
268
|
+
continue;
|
|
269
|
+
}
|
|
270
|
+
for (const rightIndex of matches2)
|
|
271
|
+
appendJoined(output, left, right, leftIndex, rightIndex, normalized);
|
|
272
|
+
}
|
|
273
|
+
return batchFromOutput(output);
|
|
274
|
+
}
|
|
275
|
+
function validateVectorJoinOptions(left, right, options) {
|
|
276
|
+
const leftKeys = normalizeJoinKeys(options.leftKey, "leftKey");
|
|
277
|
+
const rightKeys = normalizeJoinKeys(options.rightKey, "rightKey");
|
|
278
|
+
if (leftKeys.length !== rightKeys.length) {
|
|
279
|
+
throw new LakeqlError("LAKEQL_TYPE_ERROR", "Vector join key counts must match", {
|
|
280
|
+
leftKey: options.leftKey,
|
|
281
|
+
rightKey: options.rightKey
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
if (!Number.isInteger(options.maxRightRows) || options.maxRightRows < 1) {
|
|
285
|
+
throw new LakeqlError("LAKEQL_TYPE_ERROR", "Vector join maxRightRows must be a positive integer");
|
|
286
|
+
}
|
|
287
|
+
const type = options.type ?? "inner";
|
|
288
|
+
if (type !== "inner" && type !== "left" && type !== "semi" && type !== "anti") {
|
|
289
|
+
throw new LakeqlError("LAKEQL_TYPE_ERROR", "Vector join type is not supported", { type });
|
|
290
|
+
}
|
|
291
|
+
for (const key of leftKeys)
|
|
292
|
+
assertColumn(left, key);
|
|
293
|
+
for (const key of rightKeys)
|
|
294
|
+
assertColumn(right, key);
|
|
295
|
+
const rightPrefix = options.rightPrefix ?? "right.";
|
|
296
|
+
return {
|
|
297
|
+
leftKeys,
|
|
298
|
+
rightKeys,
|
|
299
|
+
type,
|
|
300
|
+
rightPrefix,
|
|
301
|
+
outputRightColumns: type === "semi" || type === "anti" ? [] : outputRightColumns(left, right, leftKeys, rightKeys, rightPrefix)
|
|
302
|
+
};
|
|
303
|
+
}
|
|
304
|
+
function buildRightIndex(right, rightIndices, rightKeys) {
|
|
305
|
+
const index = /* @__PURE__ */ new Map();
|
|
306
|
+
for (const rightIndex of rightIndices) {
|
|
307
|
+
const key = joinKey(right, rightIndex, rightKeys);
|
|
308
|
+
const bucket = index.get(key);
|
|
309
|
+
if (bucket === void 0)
|
|
310
|
+
index.set(key, [rightIndex]);
|
|
311
|
+
else
|
|
312
|
+
bucket.push(rightIndex);
|
|
313
|
+
}
|
|
314
|
+
return index;
|
|
315
|
+
}
|
|
316
|
+
function createOutputColumns(left, _right, options) {
|
|
317
|
+
const output = {};
|
|
318
|
+
for (const column of Object.keys(left.columns))
|
|
319
|
+
output[column] = [];
|
|
320
|
+
for (const { output: column } of options.outputRightColumns)
|
|
321
|
+
output[column] = [];
|
|
322
|
+
return output;
|
|
323
|
+
}
|
|
324
|
+
function appendJoined(output, left, right, leftIndex, rightIndex, options) {
|
|
325
|
+
appendLeft(output, left, leftIndex);
|
|
326
|
+
appendRight(output, right, rightIndex, options);
|
|
327
|
+
}
|
|
328
|
+
function appendLeftOnly(output, left, leftIndex) {
|
|
329
|
+
appendLeft(output, left, leftIndex);
|
|
330
|
+
}
|
|
331
|
+
function appendLeftWithNullRight(output, left, right, leftIndex, options) {
|
|
332
|
+
appendLeft(output, left, leftIndex);
|
|
333
|
+
for (const { output: column } of options.outputRightColumns)
|
|
334
|
+
output[column]?.push(null);
|
|
335
|
+
}
|
|
336
|
+
function appendLeft(output, left, leftIndex) {
|
|
337
|
+
for (const [column, vector] of Object.entries(left.columns)) {
|
|
338
|
+
output[column]?.push(vectorValue(vector, leftIndex));
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
function appendRight(output, right, rightIndex, options) {
|
|
342
|
+
for (const { input, output: column } of options.outputRightColumns) {
|
|
343
|
+
const vector = right.columns[input];
|
|
344
|
+
if (vector === void 0)
|
|
345
|
+
throw new LakeqlError("LAKEQL_UNKNOWN_COLUMN", `Unknown column ${input}`, { column: input });
|
|
346
|
+
output[column]?.push(vectorValue(vector, rightIndex));
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
function outputRightColumns(left, right, leftKeys, rightKeys, rightPrefix) {
|
|
350
|
+
return Object.keys(right.columns).flatMap((column) => {
|
|
351
|
+
if (rightKeys.includes(column) && leftKeys.includes(column))
|
|
352
|
+
return [];
|
|
353
|
+
return [{ input: column, output: column in left.columns ? `${rightPrefix}${column}` : column }];
|
|
354
|
+
});
|
|
355
|
+
}
|
|
356
|
+
function joinKey(batch, index, keys) {
|
|
357
|
+
const values = keys.map((key) => scalarJoinValue(batch, index, key));
|
|
358
|
+
return stableStringify(values.length === 1 ? values[0] : values);
|
|
359
|
+
}
|
|
360
|
+
function scalarJoinValue(batch, index, column) {
|
|
361
|
+
const vector = batch.columns[column];
|
|
362
|
+
if (vector === void 0)
|
|
363
|
+
throw new LakeqlError("LAKEQL_UNKNOWN_COLUMN", `Unknown join key ${column}`, { column });
|
|
364
|
+
return scalarVectorValue(vector, index);
|
|
365
|
+
}
|
|
366
|
+
function normalizeJoinKeys(key, label) {
|
|
367
|
+
const keys = Array.isArray(key) ? key : [key];
|
|
368
|
+
if (keys.length === 0 || keys.some((column) => typeof column !== "string" || column.length === 0)) {
|
|
369
|
+
throw new LakeqlError("LAKEQL_TYPE_ERROR", `Vector join ${label} must contain column names`, {
|
|
370
|
+
[label]: key
|
|
371
|
+
});
|
|
372
|
+
}
|
|
373
|
+
return keys;
|
|
374
|
+
}
|
|
375
|
+
function assertColumn(batch, column) {
|
|
376
|
+
if (batch.columns[column] === void 0) {
|
|
377
|
+
throw new LakeqlError("LAKEQL_UNKNOWN_COLUMN", `Unknown join key ${column}`, { column });
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
function selectedIndices(rowCount, selection) {
|
|
381
|
+
const indices = [];
|
|
382
|
+
for (let index = 0; index < rowCount; index += 1) {
|
|
383
|
+
if (selection !== void 0 && selection[index] !== 1)
|
|
384
|
+
continue;
|
|
385
|
+
indices.push(index);
|
|
386
|
+
}
|
|
387
|
+
return indices;
|
|
388
|
+
}
|
|
389
|
+
function enforceMaxRightRows(actual, limit) {
|
|
390
|
+
if (actual <= limit)
|
|
391
|
+
return;
|
|
392
|
+
throw new LakeqlError("LAKEQL_BUDGET_EXCEEDED", `Vector join exceeded maxRightRows (${actual} > ${limit})`, { metric: "maxRightRows", limit, actual });
|
|
393
|
+
}
|
|
394
|
+
function batchFromOutput(output) {
|
|
395
|
+
return batchFromColumns(output);
|
|
396
|
+
}
|
|
2
397
|
|
|
3
398
|
// ../iceberg/dist/index.js
|
|
4
|
-
var PACKAGE = "
|
|
399
|
+
var PACKAGE = "lakeql-iceberg";
|
|
5
400
|
var IcebergTable = class {
|
|
6
401
|
store;
|
|
7
402
|
metadataPath;
|
|
@@ -17,7 +412,7 @@ var IcebergTable = class {
|
|
|
17
412
|
if (options.ref !== void 0) {
|
|
18
413
|
const ref = this.metadata.refs?.[options.ref];
|
|
19
414
|
if (!ref) {
|
|
20
|
-
throw new
|
|
415
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", `Unknown Iceberg ref ${options.ref}`, {
|
|
21
416
|
ref: options.ref
|
|
22
417
|
});
|
|
23
418
|
}
|
|
@@ -26,7 +421,7 @@ var IcebergTable = class {
|
|
|
26
421
|
if (options.asOfTimestampMs !== void 0) {
|
|
27
422
|
const snapshot = [...this.metadata.snapshots].filter((candidate) => candidate["timestamp-ms"] <= options.asOfTimestampMs).sort((a, b) => b["timestamp-ms"] - a["timestamp-ms"])[0];
|
|
28
423
|
if (!snapshot) {
|
|
29
|
-
throw new
|
|
424
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "No Iceberg snapshot at requested timestamp", {
|
|
30
425
|
asOfTimestampMs: options.asOfTimestampMs
|
|
31
426
|
});
|
|
32
427
|
}
|
|
@@ -37,7 +432,9 @@ var IcebergTable = class {
|
|
|
37
432
|
schema(schemaId) {
|
|
38
433
|
const schema = this.metadata.schemas.find((candidate) => candidate["schema-id"] === schemaId);
|
|
39
434
|
if (!schema) {
|
|
40
|
-
throw new
|
|
435
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", `Unknown Iceberg schema ${schemaId}`, {
|
|
436
|
+
schemaId
|
|
437
|
+
});
|
|
41
438
|
}
|
|
42
439
|
return schema.fields;
|
|
43
440
|
}
|
|
@@ -81,7 +478,7 @@ var IcebergTable = class {
|
|
|
81
478
|
const supportedDeleteFiles = supportedIcebergDeleteFiles(file.deleteFiles);
|
|
82
479
|
const unsupportedDeleteFiles = unsupportedIcebergDeleteFiles(file.deleteFiles);
|
|
83
480
|
if (unsupportedDeleteFiles.length > 0 && readMode === "strict") {
|
|
84
|
-
throw new
|
|
481
|
+
throw new LakeqlError("LAKEQL_UNSUPPORTED_DELETE_FILES", "Snapshot contains delete files unsupported by strict Iceberg planning", {
|
|
85
482
|
path: file.path,
|
|
86
483
|
deleteFiles: file.deleteFiles,
|
|
87
484
|
supportedDeleteFiles,
|
|
@@ -125,10 +522,10 @@ var IcebergTable = class {
|
|
|
125
522
|
}
|
|
126
523
|
async appendFiles(options) {
|
|
127
524
|
if (options.files.length === 0) {
|
|
128
|
-
throw new
|
|
525
|
+
throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "Iceberg append requires at least one file");
|
|
129
526
|
}
|
|
130
527
|
if (this.metadata["format-version"] !== 2) {
|
|
131
|
-
throw new
|
|
528
|
+
throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "Iceberg append requires format-version 2 metadata", { formatVersion: this.metadata["format-version"] });
|
|
132
529
|
}
|
|
133
530
|
const currentSnapshot = this.snapshot();
|
|
134
531
|
const nextSnapshotId = options.nextSnapshotId ?? randomSnapshotId(this.metadata.snapshots.map(snapshotIdOf));
|
|
@@ -172,7 +569,7 @@ var IcebergTable = class {
|
|
|
172
569
|
});
|
|
173
570
|
const committed = typeof commit === "boolean" ? commit : commit.committed;
|
|
174
571
|
if (!committed) {
|
|
175
|
-
throw new
|
|
572
|
+
throw new LakeqlError("LAKEQL_ICEBERG_COMMIT_CONFLICT", "Iceberg append commit conflict", {
|
|
176
573
|
metadataPath: this.metadataPath,
|
|
177
574
|
expectedSnapshotId: currentSnapshot["snapshot-id"],
|
|
178
575
|
nextSnapshotId
|
|
@@ -197,7 +594,7 @@ var IcebergTable = class {
|
|
|
197
594
|
async appendOutputManifest(options) {
|
|
198
595
|
const files = options.manifest.entries.map((entry) => {
|
|
199
596
|
if (entry.iceberg === void 0) {
|
|
200
|
-
throw new
|
|
597
|
+
throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "Output manifest entry is missing Iceberg file metadata", {
|
|
201
598
|
taskId: entry.taskId,
|
|
202
599
|
outputPath: entry.outputPath
|
|
203
600
|
});
|
|
@@ -220,7 +617,7 @@ var IcebergTable = class {
|
|
|
220
617
|
snapshotById(snapshotId) {
|
|
221
618
|
const snapshot = this.metadata.snapshots.find((candidate) => candidate["snapshot-id"] === snapshotId);
|
|
222
619
|
if (!snapshot) {
|
|
223
|
-
throw new
|
|
620
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", `Unknown Iceberg snapshot ${snapshotId}`, {
|
|
224
621
|
snapshotId
|
|
225
622
|
});
|
|
226
623
|
}
|
|
@@ -241,11 +638,11 @@ function planFiles(table, options = {}) {
|
|
|
241
638
|
var ObjectStoreIcebergCommitCatalog = class {
|
|
242
639
|
async commitAppend(input) {
|
|
243
640
|
if (!supportsConditionalPut(input.store)) {
|
|
244
|
-
throw new
|
|
641
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Object-store Iceberg append requires conditional put support", { metadataPath: input.currentMetadataPath });
|
|
245
642
|
}
|
|
246
643
|
const currentBytes = await input.store.get(input.currentMetadataPath);
|
|
247
644
|
if (!currentBytes) {
|
|
248
|
-
throw new
|
|
645
|
+
throw new LakeqlError("LAKEQL_OBJECT_NOT_FOUND", `No object at ${input.currentMetadataPath}`, {
|
|
249
646
|
path: input.currentMetadataPath
|
|
250
647
|
});
|
|
251
648
|
}
|
|
@@ -270,37 +667,61 @@ var IcebergRestCatalog = class {
|
|
|
270
667
|
url;
|
|
271
668
|
namespace;
|
|
272
669
|
table;
|
|
273
|
-
|
|
670
|
+
explicitPrefix;
|
|
671
|
+
warehouse;
|
|
672
|
+
accessDelegation;
|
|
274
673
|
token;
|
|
275
674
|
fetchFn;
|
|
675
|
+
configPromise;
|
|
676
|
+
prefixPromise;
|
|
276
677
|
constructor(options) {
|
|
277
678
|
this.url = options.url;
|
|
278
679
|
this.namespace = namespaceParts(options.namespace);
|
|
279
680
|
this.table = requiredNonEmptyString(options.table, "table");
|
|
280
|
-
this.
|
|
681
|
+
this.explicitPrefix = options.prefix === void 0 ? void 0 : catalogPrefixParts(options.prefix);
|
|
682
|
+
this.warehouse = options.warehouse === void 0 ? void 0 : requiredNonEmptyString(options.warehouse, "warehouse");
|
|
683
|
+
this.accessDelegation = options.accessDelegation?.join(",");
|
|
281
684
|
this.token = options.token;
|
|
282
685
|
this.fetchFn = options.fetch ?? fetch;
|
|
283
686
|
}
|
|
284
|
-
async loadTable(store) {
|
|
285
|
-
const response = await this.
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
687
|
+
async loadTable(store, options = {}) {
|
|
688
|
+
const response = await this.loadTableResult();
|
|
689
|
+
const readControls = loadReadControls(options);
|
|
690
|
+
const controlledStore = withObjectStoreReadControls(store, readControls);
|
|
691
|
+
return new IcebergTable(controlledStore, response["metadata-location"], await hydrateMetadataManifests(controlledStore, response.metadata, readControls, options.cache));
|
|
692
|
+
}
|
|
693
|
+
async loadTableResult(options = {}) {
|
|
694
|
+
const url = await this.tableUrl(options.snapshots);
|
|
695
|
+
const headers = this.headers(false);
|
|
696
|
+
if (this.accessDelegation !== void 0)
|
|
697
|
+
headers.set("X-Iceberg-Access-Delegation", this.accessDelegation);
|
|
698
|
+
if (options.ifNoneMatch !== void 0)
|
|
699
|
+
headers.set("If-None-Match", options.ifNoneMatch);
|
|
700
|
+
const response = await this.requestJsonResponse(url, { method: "GET", headers });
|
|
701
|
+
return validateRestLoadTableResult(response.body, url, response.headers.get("etag"));
|
|
702
|
+
}
|
|
703
|
+
async loadConfig() {
|
|
704
|
+
if (this.configPromise === void 0) {
|
|
705
|
+
const url = this.configUrl();
|
|
706
|
+
this.configPromise = this.requestJson(url, { method: "GET" }).then((body) => validateRestCatalogConfig(body, url)).catch((cause) => {
|
|
707
|
+
this.configPromise = void 0;
|
|
708
|
+
throw cause;
|
|
289
709
|
});
|
|
290
710
|
}
|
|
291
|
-
return
|
|
711
|
+
return await this.configPromise;
|
|
292
712
|
}
|
|
293
713
|
async listTables() {
|
|
294
|
-
return validateListTablesResponse(await this.requestJson(this.namespaceTablesUrl(), {
|
|
714
|
+
return validateListTablesResponse(await this.requestJson(await this.namespaceTablesUrl(), {
|
|
295
715
|
method: "GET"
|
|
296
716
|
}));
|
|
297
717
|
}
|
|
298
718
|
async commitAppend(input) {
|
|
719
|
+
const tableUrl = await this.tableUrl();
|
|
299
720
|
await input.store.put(input.manifestPath, new TextEncoder().encode(`${stableStringify(input.manifest)}
|
|
300
721
|
`), { contentType: "application/json" });
|
|
301
722
|
await input.store.put(input.nextMetadataPath, new TextEncoder().encode(`${JSON.stringify(input.metadata, null, 2)}
|
|
302
723
|
`), { contentType: "application/json" });
|
|
303
|
-
const response = await this.fetchFn(
|
|
724
|
+
const response = await this.fetchFn(tableUrl, {
|
|
304
725
|
method: "POST",
|
|
305
726
|
headers: this.headers(true),
|
|
306
727
|
body: JSON.stringify({
|
|
@@ -329,8 +750,8 @@ var IcebergRestCatalog = class {
|
|
|
329
750
|
if (response.status === 409)
|
|
330
751
|
return { committed: false };
|
|
331
752
|
if (!response.ok) {
|
|
332
|
-
throw new
|
|
333
|
-
url:
|
|
753
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Iceberg REST table commit failed", {
|
|
754
|
+
url: tableUrl,
|
|
334
755
|
status: response.status,
|
|
335
756
|
statusText: response.statusText
|
|
336
757
|
});
|
|
@@ -342,21 +763,24 @@ var IcebergRestCatalog = class {
|
|
|
342
763
|
};
|
|
343
764
|
}
|
|
344
765
|
async requestJson(url, init) {
|
|
766
|
+
return (await this.requestJsonResponse(url, init)).body;
|
|
767
|
+
}
|
|
768
|
+
async requestJsonResponse(url, init) {
|
|
345
769
|
const response = await this.fetchFn(url, {
|
|
346
770
|
...init,
|
|
347
|
-
headers: this.headers(init.body !== void 0)
|
|
771
|
+
headers: init.headers ?? this.headers(init.body !== void 0)
|
|
348
772
|
});
|
|
349
773
|
if (!response.ok) {
|
|
350
|
-
throw new
|
|
774
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Iceberg REST catalog request failed", {
|
|
351
775
|
url,
|
|
352
776
|
status: response.status,
|
|
353
777
|
statusText: response.statusText
|
|
354
778
|
});
|
|
355
779
|
}
|
|
356
780
|
try {
|
|
357
|
-
return await response.json();
|
|
781
|
+
return { body: await response.json(), headers: response.headers };
|
|
358
782
|
} catch (cause) {
|
|
359
|
-
throw new
|
|
783
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Iceberg REST catalog response is not JSON", {
|
|
360
784
|
url,
|
|
361
785
|
cause
|
|
362
786
|
});
|
|
@@ -370,14 +794,50 @@ var IcebergRestCatalog = class {
|
|
|
370
794
|
headers.set("authorization", `Bearer ${this.token}`);
|
|
371
795
|
return headers;
|
|
372
796
|
}
|
|
373
|
-
tableUrl() {
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
return
|
|
797
|
+
async tableUrl(snapshots) {
|
|
798
|
+
const url = new URL(restCatalogUrl(this.url, [...await this.namespaceTableSegments(), this.table]));
|
|
799
|
+
if (snapshots !== void 0)
|
|
800
|
+
url.searchParams.set("snapshots", snapshots);
|
|
801
|
+
return url.toString();
|
|
802
|
+
}
|
|
803
|
+
async namespaceTablesUrl() {
|
|
804
|
+
return restCatalogUrl(this.url, await this.namespaceTableSegments());
|
|
805
|
+
}
|
|
806
|
+
async namespaceTableSegments() {
|
|
807
|
+
return [
|
|
808
|
+
"v1",
|
|
809
|
+
...await this.prefixParts(),
|
|
810
|
+
"namespaces",
|
|
811
|
+
this.namespace.join(""),
|
|
812
|
+
"tables"
|
|
813
|
+
];
|
|
814
|
+
}
|
|
815
|
+
async prefixParts() {
|
|
816
|
+
if (this.explicitPrefix !== void 0)
|
|
817
|
+
return this.explicitPrefix;
|
|
818
|
+
if (this.warehouse === void 0)
|
|
819
|
+
return [];
|
|
820
|
+
if (this.prefixPromise === void 0)
|
|
821
|
+
this.prefixPromise = this.computePrefixParts();
|
|
822
|
+
return await this.prefixPromise;
|
|
823
|
+
}
|
|
824
|
+
async computePrefixParts() {
|
|
825
|
+
try {
|
|
826
|
+
const config = await this.loadConfig();
|
|
827
|
+
const serverPrefix = config.overrides.prefix ?? config.defaults.prefix;
|
|
828
|
+
if (serverPrefix !== void 0 && serverPrefix.trim() !== "") {
|
|
829
|
+
return catalogPrefixParts(serverPrefix);
|
|
830
|
+
}
|
|
831
|
+
} catch {
|
|
832
|
+
return [this.warehouse];
|
|
833
|
+
}
|
|
834
|
+
return [this.warehouse];
|
|
378
835
|
}
|
|
379
|
-
|
|
380
|
-
|
|
836
|
+
configUrl() {
|
|
837
|
+
const url = new URL(restCatalogUrl(this.url, ["v1", "config"]));
|
|
838
|
+
if (this.warehouse !== void 0)
|
|
839
|
+
url.searchParams.set("warehouse", this.warehouse);
|
|
840
|
+
return url.toString();
|
|
381
841
|
}
|
|
382
842
|
};
|
|
383
843
|
function icebergRestCatalog(options) {
|
|
@@ -392,7 +852,7 @@ var IcebergUnsupportedCatalog = class {
|
|
|
392
852
|
this.namespace = namespaceParts(namespace);
|
|
393
853
|
this.table = requiredNonEmptyString(table, "table");
|
|
394
854
|
}
|
|
395
|
-
async loadTable(_store) {
|
|
855
|
+
async loadTable(_store, _options = {}) {
|
|
396
856
|
throw this.unsupported("loadTable");
|
|
397
857
|
}
|
|
398
858
|
async listTables() {
|
|
@@ -402,7 +862,7 @@ var IcebergUnsupportedCatalog = class {
|
|
|
402
862
|
throw this.unsupported("commitAppend");
|
|
403
863
|
}
|
|
404
864
|
unsupported(operation) {
|
|
405
|
-
return new
|
|
865
|
+
return new LakeqlError("LAKEQL_CATALOG_ERROR", `${this.catalog} Iceberg catalog is not implemented`, {
|
|
406
866
|
catalog: this.catalog,
|
|
407
867
|
namespace: this.namespace,
|
|
408
868
|
table: this.table,
|
|
@@ -425,18 +885,18 @@ async function loadIcebergTable(options) {
|
|
|
425
885
|
const store = withObjectStoreReadControls(options.store, readControls);
|
|
426
886
|
const bytes = await store.get(options.metadataPath);
|
|
427
887
|
if (!bytes) {
|
|
428
|
-
throw new
|
|
888
|
+
throw new LakeqlError("LAKEQL_OBJECT_NOT_FOUND", `No object at ${options.metadataPath}`, {
|
|
429
889
|
path: options.metadataPath
|
|
430
890
|
});
|
|
431
891
|
}
|
|
432
892
|
throwIfAborted(readControls.signal);
|
|
433
893
|
const text = new TextDecoder().decode(bytes);
|
|
434
894
|
try {
|
|
435
|
-
return new IcebergTable(store, options.metadataPath, await hydrateMetadataManifests(store, validateMetadata(JSON.parse(text)), readControls));
|
|
895
|
+
return new IcebergTable(store, options.metadataPath, await hydrateMetadataManifests(store, validateMetadata(JSON.parse(text)), readControls, options.cache));
|
|
436
896
|
} catch (cause) {
|
|
437
|
-
if (cause instanceof
|
|
897
|
+
if (cause instanceof LakeqlError)
|
|
438
898
|
throw cause;
|
|
439
|
-
throw new
|
|
899
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", `Invalid Iceberg metadata at ${options.metadataPath}`, {
|
|
440
900
|
path: options.metadataPath,
|
|
441
901
|
cause
|
|
442
902
|
});
|
|
@@ -450,10 +910,23 @@ async function loadIcebergTableFromObjectStore(options) {
|
|
|
450
910
|
const versionHintPath = `${metadataPrefix}version-hint.text`;
|
|
451
911
|
const hintedVersion = await readVersionHint(store, versionHintPath, readControls);
|
|
452
912
|
const metadataPath = hintedVersion === void 0 ? await latestMetadataPathFromList(store, metadataPrefix, readControls) : `${metadataPrefix}v${hintedVersion}.metadata.json`;
|
|
453
|
-
return await loadIcebergTable({
|
|
913
|
+
return await loadIcebergTable({
|
|
914
|
+
store,
|
|
915
|
+
metadataPath,
|
|
916
|
+
...readControls,
|
|
917
|
+
...options.cache !== void 0 ? { cache: options.cache } : {}
|
|
918
|
+
});
|
|
454
919
|
}
|
|
455
920
|
async function loadIcebergTableFromRest(options) {
|
|
456
|
-
|
|
921
|
+
const catalog = icebergRestCatalog(options);
|
|
922
|
+
const response = await catalog.loadTableResult();
|
|
923
|
+
const baseStore = options.store ?? (options.storeFactory === void 0 ? void 0 : await options.storeFactory({ ...response, catalog }));
|
|
924
|
+
if (baseStore === void 0) {
|
|
925
|
+
throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "Iceberg REST table loading requires a store or storeFactory");
|
|
926
|
+
}
|
|
927
|
+
const readControls = loadReadControls(options);
|
|
928
|
+
const store = withObjectStoreReadControls(baseStore, readControls);
|
|
929
|
+
return new IcebergTable(store, response["metadata-location"], await hydrateMetadataManifests(store, response.metadata, readControls, options.cache));
|
|
457
930
|
}
|
|
458
931
|
function loadReadControls(options) {
|
|
459
932
|
const controls = {};
|
|
@@ -482,7 +955,7 @@ function applyIcebergDeletes(options) {
|
|
|
482
955
|
const equalityDeleteKeys = /* @__PURE__ */ new Map();
|
|
483
956
|
for (const deletion of options.equalityDeletes ?? []) {
|
|
484
957
|
if (deletion.columns.length === 0) {
|
|
485
|
-
throw new
|
|
958
|
+
throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "Iceberg equality delete requires columns");
|
|
486
959
|
}
|
|
487
960
|
const columnsKey = stableStringify(deletion.columns);
|
|
488
961
|
let keys = equalityDeleteKeys.get(columnsKey);
|
|
@@ -566,7 +1039,7 @@ function isAsyncIterable(value) {
|
|
|
566
1039
|
}
|
|
567
1040
|
function validateDeletePosition(position, path) {
|
|
568
1041
|
if (!Number.isInteger(position) || position < 0) {
|
|
569
|
-
throw new
|
|
1042
|
+
throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "Iceberg delete position must be non-negative", {
|
|
570
1043
|
path,
|
|
571
1044
|
position
|
|
572
1045
|
});
|
|
@@ -585,14 +1058,14 @@ function randomSnapshotId(existingIds) {
|
|
|
585
1058
|
if (!existing.has(id))
|
|
586
1059
|
return id;
|
|
587
1060
|
}
|
|
588
|
-
throw new
|
|
1061
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Unable to allocate unique Iceberg snapshot id");
|
|
589
1062
|
}
|
|
590
1063
|
function validateNewSnapshotId(snapshotId, snapshots) {
|
|
591
1064
|
if (!Number.isSafeInteger(snapshotId) || snapshotId <= 0) {
|
|
592
|
-
throw new
|
|
1065
|
+
throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "Iceberg snapshot id must be a positive safe integer", { snapshotId });
|
|
593
1066
|
}
|
|
594
1067
|
if (snapshots.some((snapshot) => snapshotIdOf(snapshot) === snapshotId)) {
|
|
595
|
-
throw new
|
|
1068
|
+
throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "Iceberg snapshot id already exists", {
|
|
596
1069
|
snapshotId
|
|
597
1070
|
});
|
|
598
1071
|
}
|
|
@@ -608,23 +1081,81 @@ function metadataVersionHintPath(metadataPath) {
|
|
|
608
1081
|
function supportsConditionalPut(store) {
|
|
609
1082
|
return typeof store.conditionalPut === "function";
|
|
610
1083
|
}
|
|
611
|
-
async function hydrateMetadataManifests(store, metadata, controls = {}) {
|
|
1084
|
+
async function hydrateMetadataManifests(store, metadata, controls = {}, persistentCache) {
|
|
612
1085
|
const hydrated = cloneMetadata(metadata);
|
|
613
|
-
const
|
|
614
|
-
|
|
1086
|
+
const tableLocation = tableLocationRef(hydrated.location);
|
|
1087
|
+
const cache = {
|
|
1088
|
+
lists: /* @__PURE__ */ new Map(),
|
|
1089
|
+
manifests: /* @__PURE__ */ new Map(),
|
|
1090
|
+
tableCacheKey: icebergTableCacheKey(hydrated)
|
|
1091
|
+
};
|
|
1092
|
+
if (persistentCache !== void 0)
|
|
1093
|
+
cache.persistent = persistentCache;
|
|
1094
|
+
await Promise.all(hydrated.snapshots.map(async (snapshot) => {
|
|
615
1095
|
throwIfAborted(controls.signal);
|
|
616
|
-
|
|
617
|
-
const manifests = await Promise.all(manifestReferences.map(async (manifest) => {
|
|
618
|
-
const manifestPath = validateManifestSourcedPath(manifest.path, tablePrefix);
|
|
619
|
-
if (Array.isArray(manifest.files))
|
|
620
|
-
return validateManifestPaths(manifest, manifestPath, tablePrefix);
|
|
621
|
-
return await readManifest(store, manifestPath, tablePrefix);
|
|
622
|
-
}));
|
|
1096
|
+
snapshot.manifests = mergeDeleteManifests(await hydrateSnapshotManifests(store, snapshot, tableLocation, cache, controls));
|
|
623
1097
|
throwIfAborted(controls.signal);
|
|
624
|
-
|
|
625
|
-
}
|
|
1098
|
+
}));
|
|
626
1099
|
return hydrated;
|
|
627
1100
|
}
|
|
1101
|
+
async function hydrateSnapshotManifests(store, snapshot, tableLocation, cache, controls) {
|
|
1102
|
+
const manifestReferences = snapshot.manifests ?? (snapshot["manifest-list"] !== void 0 ? await cachedManifestList(store, validateManifestSourcedPath(snapshot["manifest-list"], tableLocation), cache) : []);
|
|
1103
|
+
return await Promise.all(manifestReferences.map(async (manifest) => {
|
|
1104
|
+
throwIfAborted(controls.signal);
|
|
1105
|
+
const manifestPath = validateManifestSourcedPath(manifest.path, tableLocation);
|
|
1106
|
+
if (Array.isArray(manifest.files))
|
|
1107
|
+
return validateManifestPaths(manifest, manifestPath, tableLocation);
|
|
1108
|
+
return await cachedManifest(store, manifestPath, tableLocation, cache);
|
|
1109
|
+
}));
|
|
1110
|
+
}
|
|
1111
|
+
function cachedManifestList(store, path, cache) {
|
|
1112
|
+
const cached = cache.lists.get(path);
|
|
1113
|
+
if (cached !== void 0)
|
|
1114
|
+
return cached;
|
|
1115
|
+
const key = icebergMetadataCacheKey(cache, "manifest-list", path);
|
|
1116
|
+
const promise = readPersistentCache(cache.persistent, key, cloneManifestReferences).then(async (persistent) => {
|
|
1117
|
+
if (persistent !== void 0)
|
|
1118
|
+
return persistent;
|
|
1119
|
+
const manifests = await readManifestList(store, path);
|
|
1120
|
+
await cache.persistent?.set(key, { value: manifests.map(cloneManifestOrReference) });
|
|
1121
|
+
return manifests;
|
|
1122
|
+
}).catch((cause) => {
|
|
1123
|
+
cache.lists.delete(path);
|
|
1124
|
+
throw cause;
|
|
1125
|
+
});
|
|
1126
|
+
cache.lists.set(path, promise);
|
|
1127
|
+
return promise;
|
|
1128
|
+
}
|
|
1129
|
+
function cachedManifest(store, path, tableLocation, cache) {
|
|
1130
|
+
const cached = cache.manifests.get(path);
|
|
1131
|
+
if (cached !== void 0)
|
|
1132
|
+
return cached;
|
|
1133
|
+
const key = icebergMetadataCacheKey(cache, "manifest", path);
|
|
1134
|
+
const promise = readPersistentCache(cache.persistent, key, cloneManifest).then(async (persistent) => {
|
|
1135
|
+
if (persistent !== void 0)
|
|
1136
|
+
return validateManifestPaths(persistent, path, tableLocation);
|
|
1137
|
+
const manifest = await readManifest(store, path, tableLocation);
|
|
1138
|
+
await cache.persistent?.set(key, { value: cloneManifest(manifest) });
|
|
1139
|
+
return manifest;
|
|
1140
|
+
}).catch((cause) => {
|
|
1141
|
+
cache.manifests.delete(path);
|
|
1142
|
+
throw cause;
|
|
1143
|
+
});
|
|
1144
|
+
cache.manifests.set(path, promise);
|
|
1145
|
+
return promise;
|
|
1146
|
+
}
|
|
1147
|
+
async function readPersistentCache(cache, key, clone) {
|
|
1148
|
+
const entry = await cache?.get(key);
|
|
1149
|
+
if (entry === void 0)
|
|
1150
|
+
return void 0;
|
|
1151
|
+
return clone(entry.value);
|
|
1152
|
+
}
|
|
1153
|
+
function icebergTableCacheKey(metadata) {
|
|
1154
|
+
return `${metadata["table-uuid"]}:${metadata.location}`;
|
|
1155
|
+
}
|
|
1156
|
+
function icebergMetadataCacheKey(cache, kind, path) {
|
|
1157
|
+
return `iceberg:${cache.tableCacheKey}:${kind}:${path}`;
|
|
1158
|
+
}
|
|
628
1159
|
function mergeDeleteManifests(manifests) {
|
|
629
1160
|
const deleteFiles = manifests.flatMap((manifest) => manifest.deleteFiles ?? []);
|
|
630
1161
|
const dataManifests = manifests.filter((manifest) => manifest.files.length > 0);
|
|
@@ -656,31 +1187,33 @@ function publicDeleteFile(deleteFile) {
|
|
|
656
1187
|
async function readManifestList(store, path) {
|
|
657
1188
|
const bytes = await store.get(path);
|
|
658
1189
|
if (!bytes) {
|
|
659
|
-
throw new
|
|
1190
|
+
throw new LakeqlError("LAKEQL_OBJECT_NOT_FOUND", `No Iceberg manifest list at ${path}`, {
|
|
1191
|
+
path
|
|
1192
|
+
});
|
|
660
1193
|
}
|
|
661
1194
|
try {
|
|
662
1195
|
return avroObjectContainer(bytes) ? validateAvroManifestList(await decodeAvroObjectContainer(bytes), path) : validateManifestList(JSON.parse(new TextDecoder().decode(bytes)), path);
|
|
663
1196
|
} catch (cause) {
|
|
664
|
-
if (cause instanceof
|
|
1197
|
+
if (cause instanceof LakeqlError)
|
|
665
1198
|
throw cause;
|
|
666
|
-
throw new
|
|
1199
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", `Invalid Iceberg manifest list at ${path}`, {
|
|
667
1200
|
path,
|
|
668
1201
|
cause
|
|
669
1202
|
});
|
|
670
1203
|
}
|
|
671
1204
|
}
|
|
672
|
-
async function readManifest(store, path,
|
|
1205
|
+
async function readManifest(store, path, tableLocation = tableLocationRef("")) {
|
|
673
1206
|
const bytes = await store.get(path);
|
|
674
1207
|
if (!bytes) {
|
|
675
|
-
throw new
|
|
1208
|
+
throw new LakeqlError("LAKEQL_OBJECT_NOT_FOUND", `No Iceberg manifest at ${path}`, { path });
|
|
676
1209
|
}
|
|
677
1210
|
try {
|
|
678
1211
|
const manifest = avroObjectContainer(bytes) ? validateAvroManifest(await decodeAvroObjectContainer(bytes), path) : validateManifest(JSON.parse(new TextDecoder().decode(bytes)), path);
|
|
679
|
-
return validateManifestPaths(manifest, path,
|
|
1212
|
+
return validateManifestPaths(manifest, path, tableLocation);
|
|
680
1213
|
} catch (cause) {
|
|
681
|
-
if (cause instanceof
|
|
1214
|
+
if (cause instanceof LakeqlError)
|
|
682
1215
|
throw cause;
|
|
683
|
-
throw new
|
|
1216
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", `Invalid Iceberg manifest at ${path}`, {
|
|
684
1217
|
path,
|
|
685
1218
|
cause
|
|
686
1219
|
});
|
|
@@ -689,7 +1222,7 @@ async function readManifest(store, path, tablePrefix = "") {
|
|
|
689
1222
|
function validateAvroManifestList(records, path) {
|
|
690
1223
|
return records.map((record) => {
|
|
691
1224
|
if (!isRecord(record) || typeof record.manifest_path !== "string") {
|
|
692
|
-
throw new
|
|
1225
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Iceberg Avro manifest list entry is invalid", {
|
|
693
1226
|
path
|
|
694
1227
|
});
|
|
695
1228
|
}
|
|
@@ -702,7 +1235,7 @@ function validateAvroManifest(records, path) {
|
|
|
702
1235
|
const deleteFiles = [];
|
|
703
1236
|
for (const record of records) {
|
|
704
1237
|
if (!isRecord(record)) {
|
|
705
|
-
throw new
|
|
1238
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Iceberg Avro manifest entry is invalid", {
|
|
706
1239
|
path
|
|
707
1240
|
});
|
|
708
1241
|
}
|
|
@@ -710,14 +1243,14 @@ function validateAvroManifest(records, path) {
|
|
|
710
1243
|
continue;
|
|
711
1244
|
const dataFile = record.data_file;
|
|
712
1245
|
if (!isRecord(dataFile)) {
|
|
713
|
-
throw new
|
|
1246
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Iceberg Avro manifest entry is missing data_file", {
|
|
714
1247
|
path
|
|
715
1248
|
});
|
|
716
1249
|
}
|
|
717
1250
|
const content = typeof dataFile.content === "number" ? dataFile.content : 0;
|
|
718
1251
|
const recordCount = safeAvroNumber(dataFile.record_count);
|
|
719
1252
|
if (typeof dataFile.file_path !== "string" || recordCount === void 0) {
|
|
720
|
-
throw new
|
|
1253
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Iceberg Avro data file has invalid fields", {
|
|
721
1254
|
path
|
|
722
1255
|
});
|
|
723
1256
|
}
|
|
@@ -745,7 +1278,7 @@ function validateAvroManifest(records, path) {
|
|
|
745
1278
|
const manifest = { path, files };
|
|
746
1279
|
if (deleteFiles.length > 0)
|
|
747
1280
|
manifest.deleteFiles = deleteFiles;
|
|
748
|
-
return
|
|
1281
|
+
return manifest;
|
|
749
1282
|
}
|
|
750
1283
|
function avroDeleteContent(content, dataFile) {
|
|
751
1284
|
if (content === 1 && (String(dataFile.file_format).toLowerCase() === "puffin" || dataFile.content_offset !== void 0 || dataFile.content_size_in_bytes !== void 0)) {
|
|
@@ -819,7 +1352,7 @@ async function loadAvro() {
|
|
|
819
1352
|
function validateManifestList(value, path) {
|
|
820
1353
|
const manifests = Array.isArray(value) ? value : isRecord(value) && Array.isArray(value.manifests) ? value.manifests : void 0;
|
|
821
1354
|
if (manifests === void 0) {
|
|
822
|
-
throw new
|
|
1355
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Iceberg manifest list has invalid required fields", {
|
|
823
1356
|
path
|
|
824
1357
|
});
|
|
825
1358
|
}
|
|
@@ -827,7 +1360,7 @@ function validateManifestList(value, path) {
|
|
|
827
1360
|
}
|
|
828
1361
|
function validateManifest(value, path) {
|
|
829
1362
|
if (!isRecord(value) || typeof value.path !== "string" || !Array.isArray(value.files)) {
|
|
830
|
-
throw new
|
|
1363
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Iceberg manifest has invalid required fields", {
|
|
831
1364
|
path
|
|
832
1365
|
});
|
|
833
1366
|
}
|
|
@@ -835,7 +1368,9 @@ function validateManifest(value, path) {
|
|
|
835
1368
|
}
|
|
836
1369
|
function validateManifestReference(value, path) {
|
|
837
1370
|
if (!isRecord(value) || typeof value.path !== "string") {
|
|
838
|
-
throw new
|
|
1371
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Iceberg manifest list entry is invalid", {
|
|
1372
|
+
path
|
|
1373
|
+
});
|
|
839
1374
|
}
|
|
840
1375
|
validateManifestContent(value.content, path);
|
|
841
1376
|
if (Array.isArray(value.files))
|
|
@@ -847,62 +1382,100 @@ function validateManifestContent(content, path) {
|
|
|
847
1382
|
return;
|
|
848
1383
|
if (content === 0 || content === 1 || content === "data" || content === "deletes")
|
|
849
1384
|
return;
|
|
850
|
-
throw new
|
|
1385
|
+
throw new LakeqlError("LAKEQL_UNSUPPORTED_ICEBERG_FEATURE", "Iceberg manifest list contains an unsupported manifest content type", {
|
|
851
1386
|
path,
|
|
852
1387
|
content
|
|
853
1388
|
});
|
|
854
1389
|
}
|
|
855
|
-
function validateManifestPaths(manifest, path,
|
|
856
|
-
validateManifestSourcedPath(manifest.path,
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
1390
|
+
function validateManifestPaths(manifest, path, tableLocation = tableLocationRef("")) {
|
|
1391
|
+
validateManifestSourcedPath(manifest.path, tableLocation);
|
|
1392
|
+
return {
|
|
1393
|
+
...manifest,
|
|
1394
|
+
path,
|
|
1395
|
+
files: manifest.files.map((file) => ({
|
|
1396
|
+
...file,
|
|
1397
|
+
path: validateManifestSourcedPath(file.path, tableLocation),
|
|
1398
|
+
...file.deleteFiles !== void 0 ? {
|
|
1399
|
+
deleteFiles: file.deleteFiles.map((deleteFile) => ({
|
|
1400
|
+
...deleteFile,
|
|
1401
|
+
path: validateManifestSourcedPath(deleteFile.path, tableLocation)
|
|
1402
|
+
}))
|
|
1403
|
+
} : {}
|
|
1404
|
+
})),
|
|
1405
|
+
...manifest.deleteFiles !== void 0 ? {
|
|
1406
|
+
deleteFiles: manifest.deleteFiles.map((deleteFile) => ({
|
|
1407
|
+
...deleteFile,
|
|
1408
|
+
path: validateManifestSourcedPath(deleteFile.path, tableLocation)
|
|
1409
|
+
}))
|
|
1410
|
+
} : {}
|
|
1411
|
+
};
|
|
1412
|
+
}
|
|
1413
|
+
function validateManifestSourcedPath(path, tableLocation = tableLocationRef("")) {
|
|
1414
|
+
const normalized = normalizeManifestSourcedPath(path, tableLocation);
|
|
1415
|
+
validateRelativeObjectPath(normalized, path);
|
|
1416
|
+
if (tableLocation.prefix !== "" && normalized !== tableLocation.prefix && !normalized.startsWith(`${tableLocation.prefix}/`)) {
|
|
1417
|
+
throw new LakeqlError("LAKEQL_VALIDATION_ERROR", `Iceberg manifest path escapes table location: ${path}`, {
|
|
1418
|
+
path,
|
|
1419
|
+
tableLocation: tableLocation.prefix
|
|
1420
|
+
});
|
|
865
1421
|
}
|
|
866
|
-
return
|
|
1422
|
+
return normalized;
|
|
867
1423
|
}
|
|
868
|
-
function
|
|
1424
|
+
function normalizeManifestSourcedPath(path, tableLocation) {
|
|
869
1425
|
if (/^(?:[a-z][a-z0-9+.-]*:)?\/\//iu.test(path) || path.startsWith("/")) {
|
|
870
|
-
|
|
871
|
-
path
|
|
872
|
-
|
|
1426
|
+
if (tableLocation.uriAuthority === void 0) {
|
|
1427
|
+
throw new LakeqlError("LAKEQL_VALIDATION_ERROR", `Iceberg manifest path must be relative: ${path}`, {
|
|
1428
|
+
path
|
|
1429
|
+
});
|
|
1430
|
+
}
|
|
1431
|
+
let url;
|
|
1432
|
+
try {
|
|
1433
|
+
url = new URL(path);
|
|
1434
|
+
} catch {
|
|
1435
|
+
throw new LakeqlError("LAKEQL_VALIDATION_ERROR", `Iceberg manifest path is invalid: ${path}`, {
|
|
1436
|
+
path
|
|
1437
|
+
});
|
|
1438
|
+
}
|
|
1439
|
+
const authority = `${url.protocol}//${url.host}`;
|
|
1440
|
+
if (authority !== tableLocation.uriAuthority) {
|
|
1441
|
+
throw new LakeqlError("LAKEQL_VALIDATION_ERROR", `Iceberg manifest path escapes table location: ${path}`, {
|
|
1442
|
+
path,
|
|
1443
|
+
tableLocation: tableLocation.uriAuthority
|
|
1444
|
+
});
|
|
1445
|
+
}
|
|
1446
|
+
return trimSlashes(decodeURIComponent(url.pathname));
|
|
873
1447
|
}
|
|
1448
|
+
return path;
|
|
1449
|
+
}
|
|
1450
|
+
function validateRelativeObjectPath(path, originalPath) {
|
|
874
1451
|
for (const segment of path.split("/")) {
|
|
875
1452
|
let decoded;
|
|
876
1453
|
try {
|
|
877
1454
|
decoded = decodeURIComponent(segment);
|
|
878
1455
|
} catch {
|
|
879
|
-
throw new
|
|
880
|
-
path
|
|
1456
|
+
throw new LakeqlError("LAKEQL_VALIDATION_ERROR", `Iceberg manifest path is invalid: ${originalPath}`, {
|
|
1457
|
+
path: originalPath
|
|
881
1458
|
});
|
|
882
1459
|
}
|
|
883
1460
|
if (decoded === "." || decoded === "..") {
|
|
884
|
-
throw new
|
|
885
|
-
path
|
|
1461
|
+
throw new LakeqlError("LAKEQL_VALIDATION_ERROR", `Iceberg manifest path contains traversal: ${originalPath}`, {
|
|
1462
|
+
path: originalPath
|
|
886
1463
|
});
|
|
887
1464
|
}
|
|
888
1465
|
}
|
|
889
|
-
if (tablePrefix !== "" && path !== tablePrefix && !path.startsWith(`${tablePrefix}/`)) {
|
|
890
|
-
throw new LaQLError("LAQL_VALIDATION_ERROR", `Iceberg manifest path escapes table location: ${path}`, {
|
|
891
|
-
path,
|
|
892
|
-
tableLocation: tablePrefix
|
|
893
|
-
});
|
|
894
|
-
}
|
|
895
|
-
return path;
|
|
896
1466
|
}
|
|
897
|
-
function
|
|
1467
|
+
function tableLocationRef(location) {
|
|
898
1468
|
const trimmed = trimTrailingSlash(location.trim());
|
|
899
1469
|
if (trimmed === "" || !trimmed.includes("/"))
|
|
900
|
-
return "";
|
|
1470
|
+
return { prefix: "" };
|
|
901
1471
|
if (/^[a-z][a-z0-9+.-]*:\/\//iu.test(trimmed)) {
|
|
902
1472
|
const url = new URL(trimmed);
|
|
903
|
-
return
|
|
1473
|
+
return {
|
|
1474
|
+
prefix: trimSlashes(decodeURIComponent(url.pathname)),
|
|
1475
|
+
uriAuthority: `${url.protocol}//${url.host}`
|
|
1476
|
+
};
|
|
904
1477
|
}
|
|
905
|
-
return trimSlashes(trimmed);
|
|
1478
|
+
return { prefix: trimSlashes(trimmed) };
|
|
906
1479
|
}
|
|
907
1480
|
function trimSlashes(value) {
|
|
908
1481
|
return value.replace(/^\/+|\/+$/gu, "");
|
|
@@ -915,7 +1488,7 @@ async function readVersionHint(store, path, controls = {}) {
|
|
|
915
1488
|
const text = new TextDecoder().decode(bytes).trim();
|
|
916
1489
|
const version = Number(text);
|
|
917
1490
|
if (!Number.isInteger(version) || version < 0) {
|
|
918
|
-
throw new
|
|
1491
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Invalid Iceberg version hint", {
|
|
919
1492
|
path,
|
|
920
1493
|
versionHint: text
|
|
921
1494
|
});
|
|
@@ -937,7 +1510,7 @@ async function latestMetadataPathFromList(store, metadataPrefix, controls = {})
|
|
|
937
1510
|
latest = { path: object.path, version };
|
|
938
1511
|
}
|
|
939
1512
|
if (latest === void 0) {
|
|
940
|
-
throw new
|
|
1513
|
+
throw new LakeqlError("LAKEQL_OBJECT_NOT_FOUND", "No Iceberg metadata files found", {
|
|
941
1514
|
prefix: metadataPrefix
|
|
942
1515
|
});
|
|
943
1516
|
}
|
|
@@ -957,7 +1530,7 @@ function namespaceParts(namespace) {
|
|
|
957
1530
|
const parts = Array.isArray(namespace) ? namespace : namespace.split(".");
|
|
958
1531
|
const out = parts.map((part) => requiredNonEmptyString(part, "namespace"));
|
|
959
1532
|
if (out.length === 0) {
|
|
960
|
-
throw new
|
|
1533
|
+
throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "Iceberg REST namespace is required");
|
|
961
1534
|
}
|
|
962
1535
|
return out;
|
|
963
1536
|
}
|
|
@@ -969,7 +1542,7 @@ function catalogPrefixParts(prefix) {
|
|
|
969
1542
|
function requiredNonEmptyString(value, field) {
|
|
970
1543
|
const trimmed = value.trim();
|
|
971
1544
|
if (trimmed.length === 0) {
|
|
972
|
-
throw new
|
|
1545
|
+
throw new LakeqlError("LAKEQL_VALIDATION_ERROR", `Iceberg REST ${field} is required`);
|
|
973
1546
|
}
|
|
974
1547
|
return trimmed;
|
|
975
1548
|
}
|
|
@@ -989,7 +1562,7 @@ async function commitResponseMetadataPath(response) {
|
|
|
989
1562
|
function restAppendSnapshot(input) {
|
|
990
1563
|
const snapshot = input.metadata.snapshots.at(-1);
|
|
991
1564
|
if (snapshot === void 0) {
|
|
992
|
-
throw new
|
|
1565
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Iceberg append metadata is missing next snapshot");
|
|
993
1566
|
}
|
|
994
1567
|
return {
|
|
995
1568
|
"snapshot-id": snapshot["snapshot-id"],
|
|
@@ -1007,7 +1580,7 @@ function restAppendSnapshot(input) {
|
|
|
1007
1580
|
function equalityKey(row, columns) {
|
|
1008
1581
|
return stableStringify(columns.map((column) => {
|
|
1009
1582
|
if (!(column in row)) {
|
|
1010
|
-
throw new
|
|
1583
|
+
throw new LakeqlError("LAKEQL_UNKNOWN_COLUMN", `Unknown Iceberg equality delete column ${column}`, {
|
|
1011
1584
|
column
|
|
1012
1585
|
});
|
|
1013
1586
|
}
|
|
@@ -1118,6 +1691,9 @@ function cloneManifestOrReference(manifest) {
|
|
|
1118
1691
|
}
|
|
1119
1692
|
return cloneManifest(manifest);
|
|
1120
1693
|
}
|
|
1694
|
+
function cloneManifestReferences(manifests) {
|
|
1695
|
+
return manifests.map(cloneManifestOrReference);
|
|
1696
|
+
}
|
|
1121
1697
|
function snapshotManifests(snapshot) {
|
|
1122
1698
|
return snapshot.manifests ?? [];
|
|
1123
1699
|
}
|
|
@@ -1129,28 +1705,93 @@ function sortStringRecord(record) {
|
|
|
1129
1705
|
}
|
|
1130
1706
|
function validateListTablesResponse(value) {
|
|
1131
1707
|
if (!isRecord(value) || !Array.isArray(value.identifiers)) {
|
|
1132
|
-
throw new
|
|
1708
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Invalid Iceberg REST list tables response");
|
|
1133
1709
|
}
|
|
1134
1710
|
return value.identifiers.map((identifier) => {
|
|
1135
1711
|
if (!isRecord(identifier) || !Array.isArray(identifier.namespace) || !identifier.namespace.every((part) => typeof part === "string") || typeof identifier.name !== "string") {
|
|
1136
|
-
throw new
|
|
1712
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Invalid Iceberg REST table identifier");
|
|
1137
1713
|
}
|
|
1138
1714
|
return { namespace: identifier.namespace, name: identifier.name };
|
|
1139
1715
|
});
|
|
1140
1716
|
}
|
|
1717
|
+
function validateRestLoadTableResult(value, url, etag) {
|
|
1718
|
+
if (!isRecord(value) || typeof value["metadata-location"] !== "string") {
|
|
1719
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Invalid Iceberg REST load table response", {
|
|
1720
|
+
url
|
|
1721
|
+
});
|
|
1722
|
+
}
|
|
1723
|
+
return {
|
|
1724
|
+
"metadata-location": value["metadata-location"],
|
|
1725
|
+
metadata: validateMetadata(value.metadata),
|
|
1726
|
+
config: validateStringRecord(value.config, "Iceberg REST load table config"),
|
|
1727
|
+
"storage-credentials": validateStorageCredentials(value["storage-credentials"]),
|
|
1728
|
+
etag
|
|
1729
|
+
};
|
|
1730
|
+
}
|
|
1731
|
+
function validateRestCatalogConfig(value, url) {
|
|
1732
|
+
if (!isRecord(value)) {
|
|
1733
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Invalid Iceberg REST catalog config response", {
|
|
1734
|
+
url
|
|
1735
|
+
});
|
|
1736
|
+
}
|
|
1737
|
+
const config = {
|
|
1738
|
+
defaults: validateStringRecord(value.defaults, "Iceberg REST catalog defaults"),
|
|
1739
|
+
overrides: validateStringRecord(value.overrides, "Iceberg REST catalog overrides")
|
|
1740
|
+
};
|
|
1741
|
+
if (Array.isArray(value.endpoints) && value.endpoints.every((endpoint) => typeof endpoint === "string")) {
|
|
1742
|
+
config.endpoints = value.endpoints;
|
|
1743
|
+
}
|
|
1744
|
+
if (typeof value["idempotency-key-lifetime"] === "string") {
|
|
1745
|
+
config["idempotency-key-lifetime"] = value["idempotency-key-lifetime"];
|
|
1746
|
+
}
|
|
1747
|
+
return config;
|
|
1748
|
+
}
|
|
1749
|
+
function validateStorageCredentials(value) {
|
|
1750
|
+
if (value === void 0)
|
|
1751
|
+
return [];
|
|
1752
|
+
if (!Array.isArray(value)) {
|
|
1753
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Invalid Iceberg REST storage credentials");
|
|
1754
|
+
}
|
|
1755
|
+
return value.map((credential) => {
|
|
1756
|
+
if (!isRecord(credential) || typeof credential.prefix !== "string") {
|
|
1757
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Invalid Iceberg REST storage credential");
|
|
1758
|
+
}
|
|
1759
|
+
return {
|
|
1760
|
+
prefix: credential.prefix,
|
|
1761
|
+
config: validateStringRecord(credential.config, "Iceberg REST storage credential config")
|
|
1762
|
+
};
|
|
1763
|
+
});
|
|
1764
|
+
}
|
|
1765
|
+
function validateStringRecord(value, label) {
|
|
1766
|
+
if (value === void 0)
|
|
1767
|
+
return {};
|
|
1768
|
+
if (!isRecord(value)) {
|
|
1769
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", `${label} must be an object`);
|
|
1770
|
+
}
|
|
1771
|
+
const out = {};
|
|
1772
|
+
for (const [key, entry] of Object.entries(value)) {
|
|
1773
|
+
if (typeof entry !== "string") {
|
|
1774
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", `${label} values must be strings`, {
|
|
1775
|
+
key
|
|
1776
|
+
});
|
|
1777
|
+
}
|
|
1778
|
+
out[key] = entry;
|
|
1779
|
+
}
|
|
1780
|
+
return out;
|
|
1781
|
+
}
|
|
1141
1782
|
function validateMetadata(value) {
|
|
1142
1783
|
if (!isRecord(value))
|
|
1143
|
-
throw new
|
|
1784
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Iceberg metadata must be an object");
|
|
1144
1785
|
if (value["format-version"] !== 1 && value["format-version"] !== 2) {
|
|
1145
|
-
throw new
|
|
1786
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Only Iceberg format-version 1 and 2 metadata is supported for reads", {
|
|
1146
1787
|
formatVersion: value["format-version"]
|
|
1147
1788
|
});
|
|
1148
1789
|
}
|
|
1149
1790
|
if (!Array.isArray(value.snapshots) || !Array.isArray(value.schemas)) {
|
|
1150
|
-
throw new
|
|
1791
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Iceberg metadata is missing snapshots or schemas");
|
|
1151
1792
|
}
|
|
1152
1793
|
if (!isMetadataFile(value)) {
|
|
1153
|
-
throw new
|
|
1794
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Iceberg metadata has invalid required fields");
|
|
1154
1795
|
}
|
|
1155
1796
|
rejectUnsupportedMetadataFeatures(value);
|
|
1156
1797
|
return value;
|
|
@@ -1165,7 +1806,7 @@ function rejectAdvertisedFeatureFlags(metadata) {
|
|
|
1165
1806
|
const features = metadata[key];
|
|
1166
1807
|
if (features === void 0 || Array.isArray(features) && features.length === 0)
|
|
1167
1808
|
continue;
|
|
1168
|
-
throw new
|
|
1809
|
+
throw new LakeqlError("LAKEQL_UNSUPPORTED_ICEBERG_FEATURE", "Iceberg metadata advertises unsupported table-format features", {
|
|
1169
1810
|
featureProperty: key,
|
|
1170
1811
|
features
|
|
1171
1812
|
});
|
|
@@ -1175,18 +1816,18 @@ function rejectUnsupportedPartitionTransforms(partitionSpecs) {
|
|
|
1175
1816
|
if (partitionSpecs === void 0)
|
|
1176
1817
|
return;
|
|
1177
1818
|
if (!Array.isArray(partitionSpecs)) {
|
|
1178
|
-
throw new
|
|
1819
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Iceberg partition-specs must be an array");
|
|
1179
1820
|
}
|
|
1180
1821
|
for (const spec of partitionSpecs) {
|
|
1181
1822
|
if (!isRecord(spec) || !Array.isArray(spec.fields)) {
|
|
1182
|
-
throw new
|
|
1823
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Iceberg partition spec is invalid");
|
|
1183
1824
|
}
|
|
1184
1825
|
for (const field of spec.fields) {
|
|
1185
1826
|
if (!isRecord(field) || typeof field.transform !== "string") {
|
|
1186
|
-
throw new
|
|
1827
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Iceberg partition field is invalid");
|
|
1187
1828
|
}
|
|
1188
1829
|
if (field.transform !== "identity" && field.transform !== "void") {
|
|
1189
|
-
throw new
|
|
1830
|
+
throw new LakeqlError("LAKEQL_UNSUPPORTED_ICEBERG_FEATURE", "Iceberg partition transform is not supported for strict planning", {
|
|
1190
1831
|
specId: spec["spec-id"],
|
|
1191
1832
|
fieldName: field.name,
|
|
1192
1833
|
transform: field.transform
|
|
@@ -1199,14 +1840,14 @@ function rejectUnsupportedSortOrders(sortOrders) {
|
|
|
1199
1840
|
if (sortOrders === void 0)
|
|
1200
1841
|
return;
|
|
1201
1842
|
if (!Array.isArray(sortOrders)) {
|
|
1202
|
-
throw new
|
|
1843
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Iceberg sort-orders must be an array");
|
|
1203
1844
|
}
|
|
1204
1845
|
for (const order of sortOrders) {
|
|
1205
1846
|
if (!isRecord(order) || !Array.isArray(order.fields)) {
|
|
1206
|
-
throw new
|
|
1847
|
+
throw new LakeqlError("LAKEQL_CATALOG_ERROR", "Iceberg sort order is invalid");
|
|
1207
1848
|
}
|
|
1208
1849
|
if (order.fields.length > 0) {
|
|
1209
|
-
throw new
|
|
1850
|
+
throw new LakeqlError("LAKEQL_UNSUPPORTED_ICEBERG_FEATURE", "Iceberg sorted table metadata is not supported for strict planning", {
|
|
1210
1851
|
orderId: order["order-id"],
|
|
1211
1852
|
fields: order.fields
|
|
1212
1853
|
});
|
|
@@ -1224,7 +1865,7 @@ function projectedIds(fields, select) {
|
|
|
1224
1865
|
return select.map((name) => {
|
|
1225
1866
|
const field = fields.find((candidate) => candidate.name === name);
|
|
1226
1867
|
if (!field) {
|
|
1227
|
-
throw new
|
|
1868
|
+
throw new LakeqlError("LAKEQL_UNKNOWN_COLUMN", `Unknown Iceberg column ${name}`, {
|
|
1228
1869
|
column: name
|
|
1229
1870
|
});
|
|
1230
1871
|
}
|
|
@@ -1241,7 +1882,7 @@ function partitionMayMatch(expr, partition) {
|
|
|
1241
1882
|
try {
|
|
1242
1883
|
return matches(expr, partition);
|
|
1243
1884
|
} catch (cause) {
|
|
1244
|
-
if (cause instanceof
|
|
1885
|
+
if (cause instanceof LakeqlError && cause.code === "LAKEQL_TYPE_ERROR")
|
|
1245
1886
|
return true;
|
|
1246
1887
|
throw cause;
|
|
1247
1888
|
}
|
|
@@ -1376,4 +2017,4 @@ async function* projectIcebergParquetBatches(store, table, path, partition, snap
|
|
|
1376
2017
|
}
|
|
1377
2018
|
}
|
|
1378
2019
|
|
|
1379
|
-
export { IcebergRestCatalog, IcebergTable, IcebergUnsupportedCatalog, ObjectStoreIcebergCommitCatalog, PACKAGE, applyIcebergDeletes, icebergGlueCatalog, icebergNessieCatalog, icebergRestCatalog, loadIcebergTable, loadIcebergTableFromObjectStore, loadIcebergTableFromRest, loadTable, planFiles2 as planFiles, scanBatches, scanPlannedIcebergRows, scanRows };
|
|
2020
|
+
export { IcebergRestCatalog, IcebergTable, IcebergUnsupportedCatalog, InMemoryRowsScanner, InMemoryRowsStore, ObjectStoreIcebergCommitCatalog, ObjectStoreJsonCache, PACKAGE, applyIcebergDeletes, createInMemoryLake, icebergGlueCatalog, icebergNessieCatalog, icebergRestCatalog, inMemoryRowsScanner, loadIcebergTable, loadIcebergTableFromObjectStore, loadIcebergTableFromRest, loadTable, objectStoreJsonCache, planFiles2 as planFiles, scanBatches, scanPlannedIcebergRows, scanRows, vectorHashJoin };
|