lakeql 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1379 @@
1
+ import { LaQLError, stableStringify, withObjectStoreReadControls, throwIfAborted, readControlSignal, jsonSafeValue, matches, readParquetObjectBatches, readIcebergParquetDeletes } from './chunk-D3A4VN3U.js';
2
+
3
+ // ../iceberg/dist/index.js
4
+ var PACKAGE = "@laql/iceberg";
5
+ var IcebergTable = class {
6
+ store;
7
+ metadataPath;
8
+ metadata;
9
+ constructor(store, metadataPath, metadata) {
10
+ this.store = store;
11
+ this.metadataPath = metadataPath;
12
+ this.metadata = metadata;
13
+ }
14
+ snapshot(options = {}) {
15
+ if (options.snapshotId !== void 0)
16
+ return this.snapshotById(options.snapshotId);
17
+ if (options.ref !== void 0) {
18
+ const ref = this.metadata.refs?.[options.ref];
19
+ if (!ref) {
20
+ throw new LaQLError("LAQL_CATALOG_ERROR", `Unknown Iceberg ref ${options.ref}`, {
21
+ ref: options.ref
22
+ });
23
+ }
24
+ return this.snapshotById(ref["snapshot-id"]);
25
+ }
26
+ if (options.asOfTimestampMs !== void 0) {
27
+ const snapshot = [...this.metadata.snapshots].filter((candidate) => candidate["timestamp-ms"] <= options.asOfTimestampMs).sort((a, b) => b["timestamp-ms"] - a["timestamp-ms"])[0];
28
+ if (!snapshot) {
29
+ throw new LaQLError("LAQL_CATALOG_ERROR", "No Iceberg snapshot at requested timestamp", {
30
+ asOfTimestampMs: options.asOfTimestampMs
31
+ });
32
+ }
33
+ return snapshot;
34
+ }
35
+ return this.snapshotById(this.metadata["current-snapshot-id"]);
36
+ }
37
+ schema(schemaId) {
38
+ const schema = this.metadata.schemas.find((candidate) => candidate["schema-id"] === schemaId);
39
+ if (!schema) {
40
+ throw new LaQLError("LAQL_CATALOG_ERROR", `Unknown Iceberg schema ${schemaId}`, { schemaId });
41
+ }
42
+ return schema.fields;
43
+ }
44
+ projectRow(row, options = {}) {
45
+ const snapshot = this.snapshot(options);
46
+ const fields = this.schema(snapshot["schema-id"]);
47
+ projectedIds(fields, options.select);
48
+ const selected = options.select === void 0 ? void 0 : new Set(options.select);
49
+ const out = {};
50
+ for (const field of fields) {
51
+ if (selected !== void 0 && !selected.has(field.name))
52
+ continue;
53
+ const sourceName = field.name in row ? field.name : field.sourceId !== void 0 ? this.fieldNameById(field.sourceId) : field.name;
54
+ out[field.name] = sourceName !== void 0 && sourceName in row ? row[sourceName] : null;
55
+ }
56
+ return out;
57
+ }
58
+ planFiles(options = {}) {
59
+ const snapshot = this.snapshot(options);
60
+ const fields = this.schema(snapshot["schema-id"]);
61
+ const projectedFieldIds = projectedIds(fields, options.select);
62
+ const readMode = options.readMode ?? "strict";
63
+ const files = [];
64
+ let manifestsSkipped = 0;
65
+ let filesSkipped = 0;
66
+ let deleteFilesPlanned = 0;
67
+ let deleteFilesIgnored = 0;
68
+ const manifests = snapshotManifests(snapshot);
69
+ for (const manifest of manifests) {
70
+ const manifestMayMatch = manifest.files.some((file) => partitionMayMatch(options.where, file.partition ?? {}));
71
+ if (!manifestMayMatch) {
72
+ manifestsSkipped += 1;
73
+ filesSkipped += manifest.files.length;
74
+ continue;
75
+ }
76
+ for (const file of manifest.files) {
77
+ if (!partitionMayMatch(options.where, file.partition ?? {})) {
78
+ filesSkipped += 1;
79
+ continue;
80
+ }
81
+ const supportedDeleteFiles = supportedIcebergDeleteFiles(file.deleteFiles);
82
+ const unsupportedDeleteFiles = unsupportedIcebergDeleteFiles(file.deleteFiles);
83
+ if (unsupportedDeleteFiles.length > 0 && readMode === "strict") {
84
+ throw new LaQLError("LAQL_UNSUPPORTED_DELETE_FILES", "Snapshot contains delete files unsupported by strict Iceberg planning", {
85
+ path: file.path,
86
+ deleteFiles: file.deleteFiles,
87
+ supportedDeleteFiles,
88
+ unsupportedDeleteFiles
89
+ });
90
+ }
91
+ const planned = {
92
+ path: file.path,
93
+ sequenceNumber: file.sequenceNumber,
94
+ partition: file.partition ?? {},
95
+ recordCount: file.recordCount,
96
+ projectedFieldIds,
97
+ snapshotId: snapshot["snapshot-id"]
98
+ };
99
+ if (file.fileSizeInBytes !== void 0)
100
+ planned.fileSizeInBytes = file.fileSizeInBytes;
101
+ if ((readMode === "strict" || readMode === "ignore-unsupported-deletes") && supportedDeleteFiles.length > 0) {
102
+ planned.deleteFiles = supportedDeleteFiles;
103
+ deleteFilesPlanned += supportedDeleteFiles.length;
104
+ }
105
+ if (readMode === "ignore-unsupported-deletes") {
106
+ deleteFilesIgnored += unsupportedDeleteFiles.length;
107
+ } else if (readMode === "ignore-deletes") {
108
+ deleteFilesIgnored += (file.deleteFiles ?? []).length;
109
+ }
110
+ files.push(planned);
111
+ }
112
+ }
113
+ files.sort((a, b) => a.sequenceNumber - b.sequenceNumber || a.path.localeCompare(b.path));
114
+ return {
115
+ snapshotId: snapshot["snapshot-id"],
116
+ schemaId: snapshot["schema-id"],
117
+ manifestsRead: manifests.length - manifestsSkipped,
118
+ manifestsSkipped,
119
+ filesPlanned: files.length,
120
+ filesSkipped,
121
+ deleteFilesPlanned,
122
+ deleteFilesIgnored,
123
+ files
124
+ };
125
+ }
126
+ async appendFiles(options) {
127
+ if (options.files.length === 0) {
128
+ throw new LaQLError("LAQL_VALIDATION_ERROR", "Iceberg append requires at least one file");
129
+ }
130
+ if (this.metadata["format-version"] !== 2) {
131
+ throw new LaQLError("LAQL_VALIDATION_ERROR", "Iceberg append requires format-version 2 metadata", { formatVersion: this.metadata["format-version"] });
132
+ }
133
+ const currentSnapshot = this.snapshot();
134
+ const nextSnapshotId = options.nextSnapshotId ?? randomSnapshotId(this.metadata.snapshots.map(snapshotIdOf));
135
+ validateNewSnapshotId(nextSnapshotId, this.metadata.snapshots);
136
+ const nextSequenceNumber = maxSequenceNumber(this.metadata) + 1;
137
+ const manifestPath = appendManifestPath(this.metadataPath, options.jobId, nextSnapshotId);
138
+ const manifest = {
139
+ path: manifestPath,
140
+ files: options.files.map((file, index) => ({
141
+ path: file.path,
142
+ sequenceNumber: nextSequenceNumber + index,
143
+ partition: sortStringRecord(file.partition ?? {}),
144
+ recordCount: file.recordCount,
145
+ fileSizeInBytes: file.fileSizeInBytes
146
+ }))
147
+ };
148
+ const nextSnapshot = {
149
+ "snapshot-id": nextSnapshotId,
150
+ "timestamp-ms": options.nowMs ?? Date.now(),
151
+ "schema-id": currentSnapshot["schema-id"],
152
+ manifests: [...snapshotManifests(currentSnapshot).map(cloneManifest), manifest]
153
+ };
154
+ const metadata = cloneMetadata(this.metadata);
155
+ metadata["current-snapshot-id"] = nextSnapshotId;
156
+ metadata.snapshots.push(nextSnapshot);
157
+ metadata.refs = {
158
+ ...metadata.refs ?? {},
159
+ main: { type: "branch", "snapshot-id": nextSnapshotId }
160
+ };
161
+ const nextMetadataPath = nextMetadataPathFor(this.metadataPath, nextSnapshotId);
162
+ const catalog = options.catalog ?? new ObjectStoreIcebergCommitCatalog();
163
+ const commit = await catalog.commitAppend({
164
+ store: this.store,
165
+ currentMetadataPath: this.metadataPath,
166
+ nextMetadataPath,
167
+ expectedSnapshotId: currentSnapshot["snapshot-id"],
168
+ nextSnapshotId,
169
+ manifestPath,
170
+ manifest,
171
+ metadata
172
+ });
173
+ const committed = typeof commit === "boolean" ? commit : commit.committed;
174
+ if (!committed) {
175
+ throw new LaQLError("LAQL_ICEBERG_COMMIT_CONFLICT", "Iceberg append commit conflict", {
176
+ metadataPath: this.metadataPath,
177
+ expectedSnapshotId: currentSnapshot["snapshot-id"],
178
+ nextSnapshotId
179
+ });
180
+ }
181
+ const committedMetadataPath = typeof commit === "boolean" ? nextMetadataPath : commit.metadataPath ?? nextMetadataPath;
182
+ return {
183
+ snapshotId: nextSnapshotId,
184
+ previousSnapshotId: currentSnapshot["snapshot-id"],
185
+ metadataPath: committedMetadataPath,
186
+ manifestPath,
187
+ files: manifest.files.map((file) => ({
188
+ path: file.path,
189
+ sequenceNumber: file.sequenceNumber,
190
+ partition: file.partition ?? {},
191
+ recordCount: file.recordCount,
192
+ projectedFieldIds: [],
193
+ snapshotId: nextSnapshotId
194
+ }))
195
+ };
196
+ }
197
+ async appendOutputManifest(options) {
198
+ const files = options.manifest.entries.map((entry) => {
199
+ if (entry.iceberg === void 0) {
200
+ throw new LaQLError("LAQL_VALIDATION_ERROR", "Output manifest entry is missing Iceberg file metadata", {
201
+ taskId: entry.taskId,
202
+ outputPath: entry.outputPath
203
+ });
204
+ }
205
+ return {
206
+ path: entry.outputPath,
207
+ partition: entry.iceberg.partitionValues,
208
+ recordCount: entry.iceberg.recordCount,
209
+ fileSizeInBytes: entry.iceberg.fileSizeInBytes
210
+ };
211
+ });
212
+ return await this.appendFiles({
213
+ files,
214
+ jobId: options.manifest.jobId,
215
+ ...options.nowMs !== void 0 ? { nowMs: options.nowMs } : {},
216
+ ...options.nextSnapshotId !== void 0 ? { nextSnapshotId: options.nextSnapshotId } : {},
217
+ ...options.catalog !== void 0 ? { catalog: options.catalog } : {}
218
+ });
219
+ }
220
+ snapshotById(snapshotId) {
221
+ const snapshot = this.metadata.snapshots.find((candidate) => candidate["snapshot-id"] === snapshotId);
222
+ if (!snapshot) {
223
+ throw new LaQLError("LAQL_CATALOG_ERROR", `Unknown Iceberg snapshot ${snapshotId}`, {
224
+ snapshotId
225
+ });
226
+ }
227
+ return snapshot;
228
+ }
229
+ fieldNameById(fieldId) {
230
+ for (const schema of this.metadata.schemas) {
231
+ const field = schema.fields.find((candidate) => candidate.id === fieldId);
232
+ if (field)
233
+ return field.name;
234
+ }
235
+ return void 0;
236
+ }
237
+ };
238
+ function planFiles(table, options = {}) {
239
+ return table.planFiles(options);
240
+ }
241
+ var ObjectStoreIcebergCommitCatalog = class {
242
+ async commitAppend(input) {
243
+ if (!supportsConditionalPut(input.store)) {
244
+ throw new LaQLError("LAQL_CATALOG_ERROR", "Object-store Iceberg append requires conditional put support", { metadataPath: input.currentMetadataPath });
245
+ }
246
+ const currentBytes = await input.store.get(input.currentMetadataPath);
247
+ if (!currentBytes) {
248
+ throw new LaQLError("LAQL_OBJECT_NOT_FOUND", `No object at ${input.currentMetadataPath}`, {
249
+ path: input.currentMetadataPath
250
+ });
251
+ }
252
+ const current = validateMetadata(JSON.parse(new TextDecoder().decode(currentBytes)));
253
+ if (current["current-snapshot-id"] !== input.expectedSnapshotId) {
254
+ return { committed: false };
255
+ }
256
+ const versionHintPath = metadataVersionHintPath(input.nextMetadataPath);
257
+ const versionHintHead = await input.store.head(versionHintPath);
258
+ await input.store.put(input.manifestPath, new TextEncoder().encode(`${stableStringify(input.manifest)}
259
+ `), { contentType: "application/json" });
260
+ await input.store.put(input.nextMetadataPath, new TextEncoder().encode(`${JSON.stringify(input.metadata, null, 2)}
261
+ `), { contentType: "application/json" });
262
+ const updated = await input.store.conditionalPut(versionHintPath, new TextEncoder().encode(`${input.nextSnapshotId}
263
+ `), { contentType: "text/plain", expectedEtag: versionHintHead?.etag ?? null });
264
+ if (!updated)
265
+ return { committed: false };
266
+ return { committed: true, metadataPath: input.nextMetadataPath };
267
+ }
268
+ };
269
+ var IcebergRestCatalog = class {
270
+ url;
271
+ namespace;
272
+ table;
273
+ prefix;
274
+ token;
275
+ fetchFn;
276
+ constructor(options) {
277
+ this.url = options.url;
278
+ this.namespace = namespaceParts(options.namespace);
279
+ this.table = requiredNonEmptyString(options.table, "table");
280
+ this.prefix = catalogPrefixParts(options.prefix);
281
+ this.token = options.token;
282
+ this.fetchFn = options.fetch ?? fetch;
283
+ }
284
+ async loadTable(store) {
285
+ const response = await this.requestJson(this.tableUrl(), { method: "GET" });
286
+ if (!isRecord(response) || typeof response["metadata-location"] !== "string") {
287
+ throw new LaQLError("LAQL_CATALOG_ERROR", "Invalid Iceberg REST load table response", {
288
+ url: this.tableUrl()
289
+ });
290
+ }
291
+ return new IcebergTable(store, response["metadata-location"], await hydrateMetadataManifests(store, validateMetadata(response.metadata)));
292
+ }
293
+ async listTables() {
294
+ return validateListTablesResponse(await this.requestJson(this.namespaceTablesUrl(), {
295
+ method: "GET"
296
+ }));
297
+ }
298
+ async commitAppend(input) {
299
+ await input.store.put(input.manifestPath, new TextEncoder().encode(`${stableStringify(input.manifest)}
300
+ `), { contentType: "application/json" });
301
+ await input.store.put(input.nextMetadataPath, new TextEncoder().encode(`${JSON.stringify(input.metadata, null, 2)}
302
+ `), { contentType: "application/json" });
303
+ const response = await this.fetchFn(this.tableUrl(), {
304
+ method: "POST",
305
+ headers: this.headers(true),
306
+ body: JSON.stringify({
307
+ identifier: { namespace: this.namespace, name: this.table },
308
+ requirements: [
309
+ {
310
+ type: "assert-ref-snapshot-id",
311
+ ref: "main",
312
+ "snapshot-id": input.expectedSnapshotId
313
+ }
314
+ ],
315
+ updates: [
316
+ {
317
+ action: "add-snapshot",
318
+ snapshot: restAppendSnapshot(input)
319
+ },
320
+ {
321
+ action: "set-snapshot-ref",
322
+ "ref-name": "main",
323
+ type: "branch",
324
+ "snapshot-id": input.nextSnapshotId
325
+ }
326
+ ]
327
+ })
328
+ });
329
+ if (response.status === 409)
330
+ return { committed: false };
331
+ if (!response.ok) {
332
+ throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg REST table commit failed", {
333
+ url: this.tableUrl(),
334
+ status: response.status,
335
+ statusText: response.statusText
336
+ });
337
+ }
338
+ const metadataPath = await commitResponseMetadataPath(response);
339
+ return {
340
+ committed: true,
341
+ ...metadataPath !== void 0 ? { metadataPath } : {}
342
+ };
343
+ }
344
+ async requestJson(url, init) {
345
+ const response = await this.fetchFn(url, {
346
+ ...init,
347
+ headers: this.headers(init.body !== void 0)
348
+ });
349
+ if (!response.ok) {
350
+ throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg REST catalog request failed", {
351
+ url,
352
+ status: response.status,
353
+ statusText: response.statusText
354
+ });
355
+ }
356
+ try {
357
+ return await response.json();
358
+ } catch (cause) {
359
+ throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg REST catalog response is not JSON", {
360
+ url,
361
+ cause
362
+ });
363
+ }
364
+ }
365
+ headers(hasBody) {
366
+ const headers = new Headers({ accept: "application/json" });
367
+ if (hasBody)
368
+ headers.set("content-type", "application/json");
369
+ if (this.token !== void 0)
370
+ headers.set("authorization", `Bearer ${this.token}`);
371
+ return headers;
372
+ }
373
+ tableUrl() {
374
+ return restCatalogUrl(this.url, [...this.namespaceTableSegments(), this.table]);
375
+ }
376
+ namespaceTablesUrl() {
377
+ return restCatalogUrl(this.url, this.namespaceTableSegments());
378
+ }
379
+ namespaceTableSegments() {
380
+ return ["v1", ...this.prefix, "namespaces", this.namespace.join(""), "tables"];
381
+ }
382
+ };
383
+ function icebergRestCatalog(options) {
384
+ return new IcebergRestCatalog(options);
385
+ }
386
+ var IcebergUnsupportedCatalog = class {
387
+ catalog;
388
+ namespace;
389
+ table;
390
+ constructor(catalog, namespace, table) {
391
+ this.catalog = requiredNonEmptyString(catalog, "catalog");
392
+ this.namespace = namespaceParts(namespace);
393
+ this.table = requiredNonEmptyString(table, "table");
394
+ }
395
+ async loadTable(_store) {
396
+ throw this.unsupported("loadTable");
397
+ }
398
+ async listTables() {
399
+ throw this.unsupported("listTables");
400
+ }
401
+ async commitAppend(_input) {
402
+ throw this.unsupported("commitAppend");
403
+ }
404
+ unsupported(operation) {
405
+ return new LaQLError("LAQL_CATALOG_ERROR", `${this.catalog} Iceberg catalog is not implemented`, {
406
+ catalog: this.catalog,
407
+ namespace: this.namespace,
408
+ table: this.table,
409
+ operation
410
+ });
411
+ }
412
+ };
413
+ function icebergGlueCatalog(options) {
414
+ requiredNonEmptyString(options.region, "region");
415
+ return new IcebergUnsupportedCatalog("Glue", options.namespace, options.table);
416
+ }
417
+ function icebergNessieCatalog(options) {
418
+ requiredNonEmptyString(options.url, "url");
419
+ if (options.ref !== void 0)
420
+ requiredNonEmptyString(options.ref, "ref");
421
+ return new IcebergUnsupportedCatalog("Nessie", options.namespace, options.table);
422
+ }
423
+ async function loadIcebergTable(options) {
424
+ const readControls = loadReadControls(options);
425
+ const store = withObjectStoreReadControls(options.store, readControls);
426
+ const bytes = await store.get(options.metadataPath);
427
+ if (!bytes) {
428
+ throw new LaQLError("LAQL_OBJECT_NOT_FOUND", `No object at ${options.metadataPath}`, {
429
+ path: options.metadataPath
430
+ });
431
+ }
432
+ throwIfAborted(readControls.signal);
433
+ const text = new TextDecoder().decode(bytes);
434
+ try {
435
+ return new IcebergTable(store, options.metadataPath, await hydrateMetadataManifests(store, validateMetadata(JSON.parse(text)), readControls));
436
+ } catch (cause) {
437
+ if (cause instanceof LaQLError)
438
+ throw cause;
439
+ throw new LaQLError("LAQL_CATALOG_ERROR", `Invalid Iceberg metadata at ${options.metadataPath}`, {
440
+ path: options.metadataPath,
441
+ cause
442
+ });
443
+ }
444
+ }
445
+ async function loadIcebergTableFromObjectStore(options) {
446
+ const readControls = loadReadControls(options);
447
+ const store = withObjectStoreReadControls(options.store, readControls);
448
+ const tableLocation = trimTrailingSlash(options.tableLocation);
449
+ const metadataPrefix = `${tableLocation}/metadata/`;
450
+ const versionHintPath = `${metadataPrefix}version-hint.text`;
451
+ const hintedVersion = await readVersionHint(store, versionHintPath, readControls);
452
+ const metadataPath = hintedVersion === void 0 ? await latestMetadataPathFromList(store, metadataPrefix, readControls) : `${metadataPrefix}v${hintedVersion}.metadata.json`;
453
+ return await loadIcebergTable({ store, metadataPath, ...readControls });
454
+ }
455
+ async function loadIcebergTableFromRest(options) {
456
+ return await icebergRestCatalog(options).loadTable(withObjectStoreReadControls(options.store, loadReadControls(options)));
457
+ }
458
+ function loadReadControls(options) {
459
+ const controls = {};
460
+ if (options.maxConcurrentReads !== void 0)
461
+ controls.maxConcurrentReads = options.maxConcurrentReads;
462
+ const signal = readControlSignal(options);
463
+ if (signal !== void 0)
464
+ controls.signal = signal;
465
+ return controls;
466
+ }
467
+ function applyIcebergDeletes(options) {
468
+ const rowOffset = options.rowOffset ?? 0;
469
+ const positionDeletes = /* @__PURE__ */ new Set();
470
+ for (const deletion of options.positionDeletes ?? []) {
471
+ if (deletion.path !== options.dataFilePath)
472
+ continue;
473
+ positionDeletes.add(validateDeletePosition(deletion.position, deletion.path));
474
+ }
475
+ for (const deletionVector of options.deletionVectors ?? []) {
476
+ if (deletionVector.path !== options.dataFilePath)
477
+ continue;
478
+ for (const position of deletionVector.positions) {
479
+ positionDeletes.add(validateDeletePosition(position, deletionVector.path));
480
+ }
481
+ }
482
+ const equalityDeleteKeys = /* @__PURE__ */ new Map();
483
+ for (const deletion of options.equalityDeletes ?? []) {
484
+ if (deletion.columns.length === 0) {
485
+ throw new LaQLError("LAQL_VALIDATION_ERROR", "Iceberg equality delete requires columns");
486
+ }
487
+ const columnsKey = stableStringify(deletion.columns);
488
+ let keys = equalityDeleteKeys.get(columnsKey);
489
+ if (!keys) {
490
+ keys = /* @__PURE__ */ new Set();
491
+ equalityDeleteKeys.set(columnsKey, keys);
492
+ }
493
+ keys.add(equalityKey(deletion.row, deletion.columns));
494
+ }
495
+ return options.rows.filter((row, index) => {
496
+ if (positionDeletes.has(rowOffset + index))
497
+ return false;
498
+ for (const [columnsKey, keys] of equalityDeleteKeys) {
499
+ const columns = JSON.parse(columnsKey);
500
+ if (keys.has(equalityKey(row, columns)))
501
+ return false;
502
+ }
503
+ return true;
504
+ });
505
+ }
506
+ async function* scanPlannedIcebergRows(options) {
507
+ const files = Array.isArray(options.plan) ? options.plan : options.plan.files;
508
+ const signal = readControlSignal(options);
509
+ for (const file of files) {
510
+ throwIfAborted(signal);
511
+ const deletes = await decodedDeletesForFile(file, options, signal);
512
+ throwIfAborted(signal);
513
+ const data = await options.readDataFile(file);
514
+ throwIfAborted(signal);
515
+ let rowOffset = 0;
516
+ for await (const batch of rowBatches(data)) {
517
+ throwIfAborted(signal);
518
+ const rows = Array.isArray(batch) ? batch : batch.rows;
519
+ const absoluteRowOffset = Array.isArray(batch) ? rowOffset : batch.rowOffset;
520
+ const visibleRows = hasDeletes(deletes) ? applyIcebergDeletes({
521
+ dataFilePath: file.path,
522
+ rows,
523
+ rowOffset: absoluteRowOffset,
524
+ ...deletes
525
+ }) : rows;
526
+ rowOffset = absoluteRowOffset + rows.length;
527
+ if (visibleRows.length > 0)
528
+ yield visibleRows;
529
+ throwIfAborted(signal);
530
+ }
531
+ }
532
+ }
533
+ async function decodedDeletesForFile(file, options, signal) {
534
+ const out = {};
535
+ for (const deleteFile of file.deleteFiles ?? []) {
536
+ throwIfAborted(signal);
537
+ const decoded = await options.readDeleteFile(deleteFile, file);
538
+ throwIfAborted(signal);
539
+ pushDeletes(out, decoded);
540
+ }
541
+ return out;
542
+ }
543
+ function pushDeletes(target, source) {
544
+ if (source.positionDeletes !== void 0) {
545
+ target.positionDeletes = [...target.positionDeletes ?? [], ...source.positionDeletes];
546
+ }
547
+ if (source.equalityDeletes !== void 0) {
548
+ target.equalityDeletes = [...target.equalityDeletes ?? [], ...source.equalityDeletes];
549
+ }
550
+ if (source.deletionVectors !== void 0) {
551
+ target.deletionVectors = [...target.deletionVectors ?? [], ...source.deletionVectors];
552
+ }
553
+ }
554
+ function hasDeletes(deletes) {
555
+ return (deletes.positionDeletes?.length ?? 0) > 0 || (deletes.equalityDeletes?.length ?? 0) > 0 || (deletes.deletionVectors?.length ?? 0) > 0;
556
+ }
557
+ async function* rowBatches(rows) {
558
+ if (isAsyncIterable(rows)) {
559
+ yield* rows;
560
+ } else {
561
+ yield rows;
562
+ }
563
+ }
564
+ function isAsyncIterable(value) {
565
+ return typeof value === "object" && value !== null && Symbol.asyncIterator in value && typeof value[Symbol.asyncIterator] === "function";
566
+ }
567
+ function validateDeletePosition(position, path) {
568
+ if (!Number.isInteger(position) || position < 0) {
569
+ throw new LaQLError("LAQL_VALIDATION_ERROR", "Iceberg delete position must be non-negative", {
570
+ path,
571
+ position
572
+ });
573
+ }
574
+ return position;
575
+ }
576
+ function nextMetadataPathFor(metadataPath, snapshotId) {
577
+ const slash = metadataPath.lastIndexOf("/");
578
+ const prefix = slash === -1 ? "" : `${metadataPath.slice(0, slash + 1)}`;
579
+ return `${prefix}v${snapshotId}.metadata.json`;
580
+ }
581
+ function randomSnapshotId(existingIds) {
582
+ const existing = new Set(existingIds);
583
+ for (let attempt = 0; attempt < 100; attempt += 1) {
584
+ const id = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER) + 1;
585
+ if (!existing.has(id))
586
+ return id;
587
+ }
588
+ throw new LaQLError("LAQL_CATALOG_ERROR", "Unable to allocate unique Iceberg snapshot id");
589
+ }
590
+ function validateNewSnapshotId(snapshotId, snapshots) {
591
+ if (!Number.isSafeInteger(snapshotId) || snapshotId <= 0) {
592
+ throw new LaQLError("LAQL_VALIDATION_ERROR", "Iceberg snapshot id must be a positive safe integer", { snapshotId });
593
+ }
594
+ if (snapshots.some((snapshot) => snapshotIdOf(snapshot) === snapshotId)) {
595
+ throw new LaQLError("LAQL_VALIDATION_ERROR", "Iceberg snapshot id already exists", {
596
+ snapshotId
597
+ });
598
+ }
599
+ }
600
+ function snapshotIdOf(snapshot) {
601
+ return snapshot["snapshot-id"];
602
+ }
603
+ function metadataVersionHintPath(metadataPath) {
604
+ const slash = metadataPath.lastIndexOf("/");
605
+ const prefix = slash === -1 ? "" : `${metadataPath.slice(0, slash + 1)}`;
606
+ return `${prefix}version-hint.text`;
607
+ }
608
+ function supportsConditionalPut(store) {
609
+ return typeof store.conditionalPut === "function";
610
+ }
611
+ async function hydrateMetadataManifests(store, metadata, controls = {}) {
612
+ const hydrated = cloneMetadata(metadata);
613
+ const tablePrefix = tableLocationObjectPrefix(hydrated.location);
614
+ for (const snapshot of hydrated.snapshots) {
615
+ throwIfAborted(controls.signal);
616
+ const manifestReferences = snapshot.manifests ?? (snapshot["manifest-list"] !== void 0 ? await readManifestList(store, validateManifestSourcedPath(snapshot["manifest-list"], tablePrefix)) : []);
617
+ const manifests = await Promise.all(manifestReferences.map(async (manifest) => {
618
+ const manifestPath = validateManifestSourcedPath(manifest.path, tablePrefix);
619
+ if (Array.isArray(manifest.files))
620
+ return validateManifestPaths(manifest, manifestPath, tablePrefix);
621
+ return await readManifest(store, manifestPath, tablePrefix);
622
+ }));
623
+ throwIfAborted(controls.signal);
624
+ snapshot.manifests = mergeDeleteManifests(manifests);
625
+ }
626
+ return hydrated;
627
+ }
628
+ function mergeDeleteManifests(manifests) {
629
+ const deleteFiles = manifests.flatMap((manifest) => manifest.deleteFiles ?? []);
630
+ const dataManifests = manifests.filter((manifest) => manifest.files.length > 0);
631
+ if (deleteFiles.length === 0)
632
+ return dataManifests;
633
+ return dataManifests.map((manifest) => ({
634
+ ...manifest,
635
+ files: manifest.files.map((file) => {
636
+ const applicable = deleteFiles.filter((deleteFile) => deleteFileMayApply(deleteFile, file));
637
+ if (applicable.length === 0)
638
+ return file;
639
+ return {
640
+ ...file,
641
+ deleteFiles: [...file.deleteFiles ?? [], ...applicable.map(publicDeleteFile)]
642
+ };
643
+ })
644
+ }));
645
+ }
646
+ function deleteFileMayApply(deleteFile, file) {
647
+ if (deleteFile.partition === void 0 || Object.keys(deleteFile.partition).length === 0) {
648
+ return true;
649
+ }
650
+ const filePartition = file.partition ?? {};
651
+ return Object.entries(deleteFile.partition).every(([key, value]) => filePartition[key] === value);
652
+ }
653
+ function publicDeleteFile(deleteFile) {
654
+ return { content: deleteFile.content, path: deleteFile.path };
655
+ }
656
+ async function readManifestList(store, path) {
657
+ const bytes = await store.get(path);
658
+ if (!bytes) {
659
+ throw new LaQLError("LAQL_OBJECT_NOT_FOUND", `No Iceberg manifest list at ${path}`, { path });
660
+ }
661
+ try {
662
+ return avroObjectContainer(bytes) ? validateAvroManifestList(await decodeAvroObjectContainer(bytes), path) : validateManifestList(JSON.parse(new TextDecoder().decode(bytes)), path);
663
+ } catch (cause) {
664
+ if (cause instanceof LaQLError)
665
+ throw cause;
666
+ throw new LaQLError("LAQL_CATALOG_ERROR", `Invalid Iceberg manifest list at ${path}`, {
667
+ path,
668
+ cause
669
+ });
670
+ }
671
+ }
672
+ async function readManifest(store, path, tablePrefix = "") {
673
+ const bytes = await store.get(path);
674
+ if (!bytes) {
675
+ throw new LaQLError("LAQL_OBJECT_NOT_FOUND", `No Iceberg manifest at ${path}`, { path });
676
+ }
677
+ try {
678
+ const manifest = avroObjectContainer(bytes) ? validateAvroManifest(await decodeAvroObjectContainer(bytes), path) : validateManifest(JSON.parse(new TextDecoder().decode(bytes)), path);
679
+ return validateManifestPaths(manifest, path, tablePrefix);
680
+ } catch (cause) {
681
+ if (cause instanceof LaQLError)
682
+ throw cause;
683
+ throw new LaQLError("LAQL_CATALOG_ERROR", `Invalid Iceberg manifest at ${path}`, {
684
+ path,
685
+ cause
686
+ });
687
+ }
688
+ }
689
+ function validateAvroManifestList(records, path) {
690
+ return records.map((record) => {
691
+ if (!isRecord(record) || typeof record.manifest_path !== "string") {
692
+ throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg Avro manifest list entry is invalid", {
693
+ path
694
+ });
695
+ }
696
+ validateManifestContent(record.content, path);
697
+ return { path: record.manifest_path };
698
+ });
699
+ }
700
+ function validateAvroManifest(records, path) {
701
+ const files = [];
702
+ const deleteFiles = [];
703
+ for (const record of records) {
704
+ if (!isRecord(record)) {
705
+ throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg Avro manifest entry is invalid", {
706
+ path
707
+ });
708
+ }
709
+ if (typeof record.status === "number" && record.status === 2)
710
+ continue;
711
+ const dataFile = record.data_file;
712
+ if (!isRecord(dataFile)) {
713
+ throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg Avro manifest entry is missing data_file", {
714
+ path
715
+ });
716
+ }
717
+ const content = typeof dataFile.content === "number" ? dataFile.content : 0;
718
+ const recordCount = safeAvroNumber(dataFile.record_count);
719
+ if (typeof dataFile.file_path !== "string" || recordCount === void 0) {
720
+ throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg Avro data file has invalid fields", {
721
+ path
722
+ });
723
+ }
724
+ if (content !== 0) {
725
+ deleteFiles.push({
726
+ content: avroDeleteContent(content, dataFile),
727
+ path: dataFile.file_path,
728
+ partition: avroPartitionValues(dataFile.partition)
729
+ });
730
+ continue;
731
+ }
732
+ const sequenceNumber = safeAvroNumber(record.sequence_number) ?? safeAvroNumber(record.file_sequence_number) ?? 0;
733
+ const file = {
734
+ path: dataFile.file_path,
735
+ sequenceNumber,
736
+ partition: avroPartitionValues(dataFile.partition),
737
+ recordCount
738
+ };
739
+ const fileSizeInBytes = safeAvroNumber(dataFile.file_size_in_bytes);
740
+ if (fileSizeInBytes !== void 0) {
741
+ file.fileSizeInBytes = fileSizeInBytes;
742
+ }
743
+ files.push(file);
744
+ }
745
+ const manifest = { path, files };
746
+ if (deleteFiles.length > 0)
747
+ manifest.deleteFiles = deleteFiles;
748
+ return validateManifestPaths(manifest, path);
749
+ }
750
+ function avroDeleteContent(content, dataFile) {
751
+ if (content === 1 && (String(dataFile.file_format).toLowerCase() === "puffin" || dataFile.content_offset !== void 0 || dataFile.content_size_in_bytes !== void 0)) {
752
+ return "deletion-vector";
753
+ }
754
+ if (content === 1)
755
+ return "position-delete";
756
+ if (content === 2)
757
+ return "equality-delete";
758
+ return `unsupported-delete-${content}`;
759
+ }
760
+ function avroPartitionValues(value) {
761
+ if (!isRecord(value))
762
+ return {};
763
+ const out = {};
764
+ for (const [key, inner] of Object.entries(value)) {
765
+ if (inner === null || inner === void 0)
766
+ continue;
767
+ out[key] = String(jsonSafeValue(inner));
768
+ }
769
+ return sortStringRecord(out);
770
+ }
771
+ function safeAvroNumber(value) {
772
+ if (typeof value === "number" && Number.isSafeInteger(value))
773
+ return value;
774
+ if (typeof value === "bigint") {
775
+ const numberValue = Number(value);
776
+ if (Number.isSafeInteger(numberValue) && BigInt(numberValue) === value)
777
+ return numberValue;
778
+ }
779
+ return void 0;
780
+ }
781
+ function avroObjectContainer(bytes) {
782
+ return bytes.length >= 4 && bytes[0] === 79 && bytes[1] === 98 && bytes[2] === 106 && bytes[3] === 1;
783
+ }
784
+ async function decodeAvroObjectContainer(bytes) {
785
+ const avro = await loadAvro();
786
+ const avroBigIntLongType = avro.types.LongType.__with({
787
+ fromBuffer: (buffer) => buffer.readBigInt64LE(),
788
+ toBuffer: (value) => {
789
+ const buffer = Buffer.alloc(8);
790
+ buffer.writeBigInt64LE(BigInt(value));
791
+ return buffer;
792
+ },
793
+ fromJSON: (value) => BigInt(value),
794
+ toJSON: (value) => value.toString(),
795
+ isValid: (value) => typeof value === "bigint" || typeof value === "number" && Number.isSafeInteger(value),
796
+ compare: (left, right) => {
797
+ const leftBigInt = BigInt(left);
798
+ const rightBigInt = BigInt(right);
799
+ return leftBigInt === rightBigInt ? 0 : leftBigInt < rightBigInt ? -1 : 1;
800
+ }
801
+ });
802
+ const avroLongTypeHook = (schema) => schema === "long" || isRecord(schema) && schema.type === "long" ? avroBigIntLongType : void 0;
803
+ const decoder = new avro.streams.BlockDecoder({
804
+ parseHook: (schema) => avro.Type.forSchema(schema, { typeHook: avroLongTypeHook })
805
+ });
806
+ const records = [];
807
+ const done = new Promise((resolve, reject) => {
808
+ decoder.on("data", (record) => records.push(record));
809
+ decoder.on("end", () => resolve(records));
810
+ decoder.on("error", reject);
811
+ });
812
+ decoder.end(Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength));
813
+ return done;
814
+ }
815
+ async function loadAvro() {
816
+ const module = await import('avsc');
817
+ return module.default ?? module;
818
+ }
819
+ function validateManifestList(value, path) {
820
+ const manifests = Array.isArray(value) ? value : isRecord(value) && Array.isArray(value.manifests) ? value.manifests : void 0;
821
+ if (manifests === void 0) {
822
+ throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg manifest list has invalid required fields", {
823
+ path
824
+ });
825
+ }
826
+ return manifests.map((manifest) => validateManifestReference(manifest, path));
827
+ }
828
+ function validateManifest(value, path) {
829
+ if (!isRecord(value) || typeof value.path !== "string" || !Array.isArray(value.files)) {
830
+ throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg manifest has invalid required fields", {
831
+ path
832
+ });
833
+ }
834
+ return value;
835
+ }
836
+ function validateManifestReference(value, path) {
837
+ if (!isRecord(value) || typeof value.path !== "string") {
838
+ throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg manifest list entry is invalid", { path });
839
+ }
840
+ validateManifestContent(value.content, path);
841
+ if (Array.isArray(value.files))
842
+ return value;
843
+ return { path: value.path };
844
+ }
845
+ function validateManifestContent(content, path) {
846
+ if (content === void 0 || content === null)
847
+ return;
848
+ if (content === 0 || content === 1 || content === "data" || content === "deletes")
849
+ return;
850
+ throw new LaQLError("LAQL_UNSUPPORTED_ICEBERG_FEATURE", "Iceberg manifest list contains an unsupported manifest content type", {
851
+ path,
852
+ content
853
+ });
854
+ }
855
+ function validateManifestPaths(manifest, path, tablePrefix = "") {
856
+ validateManifestSourcedPath(manifest.path, tablePrefix);
857
+ for (const file of manifest.files) {
858
+ validateManifestSourcedPath(file.path, tablePrefix);
859
+ for (const deleteFile of file.deleteFiles ?? []) {
860
+ validateManifestSourcedPath(deleteFile.path, tablePrefix);
861
+ }
862
+ }
863
+ for (const deleteFile of manifest.deleteFiles ?? []) {
864
+ validateManifestSourcedPath(deleteFile.path, tablePrefix);
865
+ }
866
+ return { ...manifest, path };
867
+ }
868
+ function validateManifestSourcedPath(path, tablePrefix = "") {
869
+ if (/^(?:[a-z][a-z0-9+.-]*:)?\/\//iu.test(path) || path.startsWith("/")) {
870
+ throw new LaQLError("LAQL_VALIDATION_ERROR", `Iceberg manifest path must be relative: ${path}`, {
871
+ path
872
+ });
873
+ }
874
+ for (const segment of path.split("/")) {
875
+ let decoded;
876
+ try {
877
+ decoded = decodeURIComponent(segment);
878
+ } catch {
879
+ throw new LaQLError("LAQL_VALIDATION_ERROR", `Iceberg manifest path is invalid: ${path}`, {
880
+ path
881
+ });
882
+ }
883
+ if (decoded === "." || decoded === "..") {
884
+ throw new LaQLError("LAQL_VALIDATION_ERROR", `Iceberg manifest path contains traversal: ${path}`, {
885
+ path
886
+ });
887
+ }
888
+ }
889
+ if (tablePrefix !== "" && path !== tablePrefix && !path.startsWith(`${tablePrefix}/`)) {
890
+ throw new LaQLError("LAQL_VALIDATION_ERROR", `Iceberg manifest path escapes table location: ${path}`, {
891
+ path,
892
+ tableLocation: tablePrefix
893
+ });
894
+ }
895
+ return path;
896
+ }
897
+ function tableLocationObjectPrefix(location) {
898
+ const trimmed = trimTrailingSlash(location.trim());
899
+ if (trimmed === "" || !trimmed.includes("/"))
900
+ return "";
901
+ if (/^[a-z][a-z0-9+.-]*:\/\//iu.test(trimmed)) {
902
+ const url = new URL(trimmed);
903
+ return trimSlashes(decodeURIComponent(url.pathname));
904
+ }
905
+ return trimSlashes(trimmed);
906
+ }
907
+ function trimSlashes(value) {
908
+ return value.replace(/^\/+|\/+$/gu, "");
909
+ }
910
+ async function readVersionHint(store, path, controls = {}) {
911
+ const bytes = await store.get(path);
912
+ if (!bytes)
913
+ return void 0;
914
+ throwIfAborted(controls.signal);
915
+ const text = new TextDecoder().decode(bytes).trim();
916
+ const version = Number(text);
917
+ if (!Number.isInteger(version) || version < 0) {
918
+ throw new LaQLError("LAQL_CATALOG_ERROR", "Invalid Iceberg version hint", {
919
+ path,
920
+ versionHint: text
921
+ });
922
+ }
923
+ return version;
924
+ }
925
+ async function latestMetadataPathFromList(store, metadataPrefix, controls = {}) {
926
+ let latest;
927
+ for await (const object of store.list(metadataPrefix)) {
928
+ throwIfAborted(controls.signal);
929
+ const name = object.path.slice(metadataPrefix.length);
930
+ const match = /^v(\d+)\.metadata\.json$/u.exec(name);
931
+ if (!match)
932
+ continue;
933
+ const version = Number(match[1]);
934
+ if (!Number.isSafeInteger(version))
935
+ continue;
936
+ if (latest === void 0 || version > latest.version)
937
+ latest = { path: object.path, version };
938
+ }
939
+ if (latest === void 0) {
940
+ throw new LaQLError("LAQL_OBJECT_NOT_FOUND", "No Iceberg metadata files found", {
941
+ prefix: metadataPrefix
942
+ });
943
+ }
944
+ return latest.path;
945
+ }
946
+ function trimTrailingSlash(value) {
947
+ return value.endsWith("/") ? value.slice(0, -1) : value;
948
+ }
949
+ function restCatalogUrl(baseUrl, segments) {
950
+ const url = new URL(baseUrl);
951
+ const basePath = url.pathname.replace(/\/+$/u, "");
952
+ const encodedSegments = segments.map((segment) => encodeURIComponent(segment));
953
+ url.pathname = [basePath, ...encodedSegments].filter((segment) => segment.length > 0).join("/");
954
+ return url.toString();
955
+ }
956
+ function namespaceParts(namespace) {
957
+ const parts = Array.isArray(namespace) ? namespace : namespace.split(".");
958
+ const out = parts.map((part) => requiredNonEmptyString(part, "namespace"));
959
+ if (out.length === 0) {
960
+ throw new LaQLError("LAQL_VALIDATION_ERROR", "Iceberg REST namespace is required");
961
+ }
962
+ return out;
963
+ }
964
+ function catalogPrefixParts(prefix) {
965
+ if (prefix === void 0 || prefix.trim() === "")
966
+ return [];
967
+ return prefix.split("/").filter((part) => part.length > 0).map((part) => requiredNonEmptyString(part, "prefix"));
968
+ }
969
+ function requiredNonEmptyString(value, field) {
970
+ const trimmed = value.trim();
971
+ if (trimmed.length === 0) {
972
+ throw new LaQLError("LAQL_VALIDATION_ERROR", `Iceberg REST ${field} is required`);
973
+ }
974
+ return trimmed;
975
+ }
976
+ async function commitResponseMetadataPath(response) {
977
+ if (response.status === 204)
978
+ return void 0;
979
+ try {
980
+ const body = await response.json();
981
+ if (isRecord(body) && typeof body["metadata-location"] === "string") {
982
+ return body["metadata-location"];
983
+ }
984
+ } catch {
985
+ return void 0;
986
+ }
987
+ return void 0;
988
+ }
989
+ function restAppendSnapshot(input) {
990
+ const snapshot = input.metadata.snapshots.at(-1);
991
+ if (snapshot === void 0) {
992
+ throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg append metadata is missing next snapshot");
993
+ }
994
+ return {
995
+ "snapshot-id": snapshot["snapshot-id"],
996
+ "parent-snapshot-id": input.expectedSnapshotId,
997
+ "timestamp-ms": snapshot["timestamp-ms"],
998
+ "schema-id": snapshot["schema-id"],
999
+ "manifest-list": input.manifestPath,
1000
+ summary: {
1001
+ operation: "append",
1002
+ "added-data-files": String(input.manifest.files.length),
1003
+ "added-records": String(input.manifest.files.reduce((sum, file) => sum + file.recordCount, 0))
1004
+ }
1005
+ };
1006
+ }
1007
+ function equalityKey(row, columns) {
1008
+ return stableStringify(columns.map((column) => {
1009
+ if (!(column in row)) {
1010
+ throw new LaQLError("LAQL_UNKNOWN_COLUMN", `Unknown Iceberg equality delete column ${column}`, {
1011
+ column
1012
+ });
1013
+ }
1014
+ return jsonSafeValue(row[column]);
1015
+ }));
1016
+ }
1017
+ function appendManifestPath(metadataPath, jobId, snapshotId) {
1018
+ const slash = metadataPath.lastIndexOf("/");
1019
+ const prefix = slash === -1 ? "" : `${metadataPath.slice(0, slash + 1)}`;
1020
+ return `${prefix}${jobId ?? "append"}-${snapshotId}.manifest.json`;
1021
+ }
1022
+ function maxSequenceNumber(metadata) {
1023
+ let max = 0;
1024
+ for (const snapshot of metadata.snapshots) {
1025
+ for (const manifest of snapshotManifests(snapshot)) {
1026
+ for (const file of manifest.files)
1027
+ max = Math.max(max, file.sequenceNumber);
1028
+ }
1029
+ }
1030
+ return max;
1031
+ }
1032
+ function supportedIcebergDeleteFiles(deleteFiles) {
1033
+ const supported = [];
1034
+ for (const deleteFile of deleteFiles ?? []) {
1035
+ if (deleteFile.content === "position-delete" || deleteFile.content === "equality-delete") {
1036
+ supported.push({ content: deleteFile.content, path: deleteFile.path });
1037
+ }
1038
+ }
1039
+ return supported;
1040
+ }
1041
+ function unsupportedIcebergDeleteFiles(deleteFiles) {
1042
+ const supported = /* @__PURE__ */ new Set(["position-delete", "equality-delete"]);
1043
+ return (deleteFiles ?? []).filter((deleteFile) => !supported.has(deleteFile.content));
1044
+ }
1045
+ function cloneMetadata(metadata) {
1046
+ const cloned = {
1047
+ "format-version": metadata["format-version"],
1048
+ "table-uuid": metadata["table-uuid"],
1049
+ location: metadata.location,
1050
+ "current-snapshot-id": metadata["current-snapshot-id"],
1051
+ schemas: metadata.schemas.map((schema) => ({
1052
+ "schema-id": schema["schema-id"],
1053
+ fields: schema.fields.map((field) => ({ ...field }))
1054
+ })),
1055
+ snapshots: metadata.snapshots.map((snapshot) => {
1056
+ const cloned2 = {
1057
+ "snapshot-id": snapshot["snapshot-id"],
1058
+ "timestamp-ms": snapshot["timestamp-ms"],
1059
+ "schema-id": snapshot["schema-id"]
1060
+ };
1061
+ if (snapshot["manifest-list"] !== void 0)
1062
+ cloned2["manifest-list"] = snapshot["manifest-list"];
1063
+ if (snapshot.manifests !== void 0) {
1064
+ cloned2.manifests = snapshot.manifests.map(cloneManifestOrReference);
1065
+ }
1066
+ return cloned2;
1067
+ })
1068
+ };
1069
+ if (metadata.refs)
1070
+ cloned.refs = cloneRefs(metadata.refs);
1071
+ if (metadata["partition-specs"]) {
1072
+ cloned["partition-specs"] = metadata["partition-specs"].map((spec) => ({
1073
+ "spec-id": spec["spec-id"],
1074
+ fields: spec.fields.map((field) => ({ ...field }))
1075
+ }));
1076
+ }
1077
+ if (metadata["sort-orders"]) {
1078
+ cloned["sort-orders"] = metadata["sort-orders"].map((order) => ({
1079
+ "order-id": order["order-id"],
1080
+ fields: order.fields.map((field) => field)
1081
+ }));
1082
+ }
1083
+ return cloned;
1084
+ }
1085
+ function cloneRefs(refs) {
1086
+ const out = {};
1087
+ for (const [name, ref] of Object.entries(refs)) {
1088
+ out[name] = { type: ref.type, "snapshot-id": ref["snapshot-id"] };
1089
+ }
1090
+ return out;
1091
+ }
1092
+ function cloneManifest(manifest) {
1093
+ const cloned = {
1094
+ path: manifest.path,
1095
+ files: manifest.files.map((file) => {
1096
+ const cloned2 = {
1097
+ path: file.path,
1098
+ sequenceNumber: file.sequenceNumber,
1099
+ partition: sortStringRecord(file.partition ?? {}),
1100
+ recordCount: file.recordCount
1101
+ };
1102
+ if (file.fileSizeInBytes !== void 0)
1103
+ cloned2.fileSizeInBytes = file.fileSizeInBytes;
1104
+ if (file.deleteFiles !== void 0) {
1105
+ cloned2.deleteFiles = file.deleteFiles.map((deleteFile) => ({ ...deleteFile }));
1106
+ }
1107
+ return cloned2;
1108
+ })
1109
+ };
1110
+ if (manifest.deleteFiles !== void 0) {
1111
+ cloned.deleteFiles = manifest.deleteFiles.map((deleteFile) => ({ ...deleteFile }));
1112
+ }
1113
+ return cloned;
1114
+ }
1115
+ function cloneManifestOrReference(manifest) {
1116
+ if (!Array.isArray(manifest.files)) {
1117
+ return { path: manifest.path };
1118
+ }
1119
+ return cloneManifest(manifest);
1120
+ }
1121
+ function snapshotManifests(snapshot) {
1122
+ return snapshot.manifests ?? [];
1123
+ }
1124
+ function sortStringRecord(record) {
1125
+ const out = {};
1126
+ for (const key of Object.keys(record).sort())
1127
+ out[key] = record[key] ?? "";
1128
+ return out;
1129
+ }
1130
+ function validateListTablesResponse(value) {
1131
+ if (!isRecord(value) || !Array.isArray(value.identifiers)) {
1132
+ throw new LaQLError("LAQL_CATALOG_ERROR", "Invalid Iceberg REST list tables response");
1133
+ }
1134
+ return value.identifiers.map((identifier) => {
1135
+ if (!isRecord(identifier) || !Array.isArray(identifier.namespace) || !identifier.namespace.every((part) => typeof part === "string") || typeof identifier.name !== "string") {
1136
+ throw new LaQLError("LAQL_CATALOG_ERROR", "Invalid Iceberg REST table identifier");
1137
+ }
1138
+ return { namespace: identifier.namespace, name: identifier.name };
1139
+ });
1140
+ }
1141
+ function validateMetadata(value) {
1142
+ if (!isRecord(value))
1143
+ throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg metadata must be an object");
1144
+ if (value["format-version"] !== 1 && value["format-version"] !== 2) {
1145
+ throw new LaQLError("LAQL_CATALOG_ERROR", "Only Iceberg format-version 1 and 2 metadata is supported for reads", {
1146
+ formatVersion: value["format-version"]
1147
+ });
1148
+ }
1149
+ if (!Array.isArray(value.snapshots) || !Array.isArray(value.schemas)) {
1150
+ throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg metadata is missing snapshots or schemas");
1151
+ }
1152
+ if (!isMetadataFile(value)) {
1153
+ throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg metadata has invalid required fields");
1154
+ }
1155
+ rejectUnsupportedMetadataFeatures(value);
1156
+ return value;
1157
+ }
1158
+ function rejectUnsupportedMetadataFeatures(metadata) {
1159
+ rejectAdvertisedFeatureFlags(metadata);
1160
+ rejectUnsupportedPartitionTransforms(metadata["partition-specs"]);
1161
+ rejectUnsupportedSortOrders(metadata["sort-orders"]);
1162
+ }
1163
+ function rejectAdvertisedFeatureFlags(metadata) {
1164
+ for (const key of ["features", "table-features", "format-version-features"]) {
1165
+ const features = metadata[key];
1166
+ if (features === void 0 || Array.isArray(features) && features.length === 0)
1167
+ continue;
1168
+ throw new LaQLError("LAQL_UNSUPPORTED_ICEBERG_FEATURE", "Iceberg metadata advertises unsupported table-format features", {
1169
+ featureProperty: key,
1170
+ features
1171
+ });
1172
+ }
1173
+ }
1174
+ function rejectUnsupportedPartitionTransforms(partitionSpecs) {
1175
+ if (partitionSpecs === void 0)
1176
+ return;
1177
+ if (!Array.isArray(partitionSpecs)) {
1178
+ throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg partition-specs must be an array");
1179
+ }
1180
+ for (const spec of partitionSpecs) {
1181
+ if (!isRecord(spec) || !Array.isArray(spec.fields)) {
1182
+ throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg partition spec is invalid");
1183
+ }
1184
+ for (const field of spec.fields) {
1185
+ if (!isRecord(field) || typeof field.transform !== "string") {
1186
+ throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg partition field is invalid");
1187
+ }
1188
+ if (field.transform !== "identity" && field.transform !== "void") {
1189
+ throw new LaQLError("LAQL_UNSUPPORTED_ICEBERG_FEATURE", "Iceberg partition transform is not supported for strict planning", {
1190
+ specId: spec["spec-id"],
1191
+ fieldName: field.name,
1192
+ transform: field.transform
1193
+ });
1194
+ }
1195
+ }
1196
+ }
1197
+ }
1198
+ function rejectUnsupportedSortOrders(sortOrders) {
1199
+ if (sortOrders === void 0)
1200
+ return;
1201
+ if (!Array.isArray(sortOrders)) {
1202
+ throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg sort-orders must be an array");
1203
+ }
1204
+ for (const order of sortOrders) {
1205
+ if (!isRecord(order) || !Array.isArray(order.fields)) {
1206
+ throw new LaQLError("LAQL_CATALOG_ERROR", "Iceberg sort order is invalid");
1207
+ }
1208
+ if (order.fields.length > 0) {
1209
+ throw new LaQLError("LAQL_UNSUPPORTED_ICEBERG_FEATURE", "Iceberg sorted table metadata is not supported for strict planning", {
1210
+ orderId: order["order-id"],
1211
+ fields: order.fields
1212
+ });
1213
+ }
1214
+ }
1215
+ }
1216
+ function isMetadataFile(value) {
1217
+ if (!isRecord(value))
1218
+ return false;
1219
+ return (value["format-version"] === 1 || value["format-version"] === 2) && typeof value["table-uuid"] === "string" && typeof value.location === "string" && typeof value["current-snapshot-id"] === "number" && Array.isArray(value.refs) === false && Array.isArray(value.schemas) && Array.isArray(value.snapshots);
1220
+ }
1221
+ function projectedIds(fields, select) {
1222
+ if (!select)
1223
+ return fields.map((field) => field.id).sort((a, b) => a - b);
1224
+ return select.map((name) => {
1225
+ const field = fields.find((candidate) => candidate.name === name);
1226
+ if (!field) {
1227
+ throw new LaQLError("LAQL_UNKNOWN_COLUMN", `Unknown Iceberg column ${name}`, {
1228
+ column: name
1229
+ });
1230
+ }
1231
+ return field.sourceId ?? field.id;
1232
+ }).sort((a, b) => a - b);
1233
+ }
1234
+ function partitionMayMatch(expr, partition) {
1235
+ if (!expr)
1236
+ return true;
1237
+ const columns = /* @__PURE__ */ new Set();
1238
+ collectColumns(expr, columns);
1239
+ if (columns.size === 0 || [...columns].some((column) => !(column in partition)))
1240
+ return true;
1241
+ try {
1242
+ return matches(expr, partition);
1243
+ } catch (cause) {
1244
+ if (cause instanceof LaQLError && cause.code === "LAQL_TYPE_ERROR")
1245
+ return true;
1246
+ throw cause;
1247
+ }
1248
+ }
1249
+ function collectColumns(expr, columns) {
1250
+ switch (expr.kind) {
1251
+ case "column":
1252
+ columns.add(expr.name);
1253
+ return;
1254
+ case "literal":
1255
+ return;
1256
+ case "compare":
1257
+ collectColumns(expr.left, columns);
1258
+ collectColumns(expr.right, columns);
1259
+ return;
1260
+ case "in":
1261
+ collectColumns(expr.target, columns);
1262
+ for (const value of expr.values)
1263
+ collectColumns(value, columns);
1264
+ return;
1265
+ case "between":
1266
+ collectColumns(expr.target, columns);
1267
+ collectColumns(expr.low, columns);
1268
+ collectColumns(expr.high, columns);
1269
+ return;
1270
+ case "null-check":
1271
+ collectColumns(expr.target, columns);
1272
+ return;
1273
+ case "logical":
1274
+ for (const operand of expr.operands)
1275
+ collectColumns(operand, columns);
1276
+ return;
1277
+ case "not":
1278
+ collectColumns(expr.operand, columns);
1279
+ return;
1280
+ case "like":
1281
+ collectColumns(expr.target, columns);
1282
+ return;
1283
+ case "call":
1284
+ for (const arg of expr.args)
1285
+ collectColumns(arg, columns);
1286
+ return;
1287
+ case "arithmetic":
1288
+ collectColumns(expr.left, columns);
1289
+ collectColumns(expr.right, columns);
1290
+ return;
1291
+ case "case":
1292
+ for (const branch of expr.whens) {
1293
+ collectColumns(branch.when, columns);
1294
+ collectColumns(branch.value, columns);
1295
+ }
1296
+ if (expr.else !== void 0)
1297
+ collectColumns(expr.else, columns);
1298
+ return;
1299
+ }
1300
+ }
1301
+ function isRecord(value) {
1302
+ return typeof value === "object" && value !== null && !Array.isArray(value);
1303
+ }
1304
+
1305
+ // src/engine.ts
1306
+ async function loadTable(options) {
1307
+ if (options.format === "parquet") {
1308
+ return { format: "parquet", store: options.store, path: options.path };
1309
+ }
1310
+ const { format: _format, ...icebergOptions } = options;
1311
+ return {
1312
+ format: "iceberg",
1313
+ store: options.store,
1314
+ table: await loadIcebergTable(icebergOptions)
1315
+ };
1316
+ }
1317
+ function planFiles2(table, options = {}) {
1318
+ if (table.format === "parquet") {
1319
+ return { format: "parquet", store: table.store, files: [{ path: table.path }] };
1320
+ }
1321
+ return {
1322
+ format: "iceberg",
1323
+ store: table.store,
1324
+ table: table.table,
1325
+ plan: planFiles(table.table, options),
1326
+ options
1327
+ };
1328
+ }
1329
+ async function* scanBatches(plan, options = {}) {
1330
+ if (plan.format === "parquet") {
1331
+ for (const file of plan.files) {
1332
+ const readOptions = {};
1333
+ if (options.batchSize !== void 0) readOptions.batchSize = options.batchSize;
1334
+ for await (const batch of readParquetObjectBatches(plan.store, file.path, readOptions)) {
1335
+ yield batch.rows;
1336
+ }
1337
+ }
1338
+ return;
1339
+ }
1340
+ for await (const batch of scanPlannedIcebergRows({
1341
+ plan: plan.plan,
1342
+ ...options,
1343
+ readDataFile: async (file) => projectIcebergParquetBatches(
1344
+ plan.store,
1345
+ plan.table,
1346
+ file.path,
1347
+ file.partition,
1348
+ file.snapshotId,
1349
+ plan.options,
1350
+ options
1351
+ ),
1352
+ readDeleteFile: async (deleteFile) => readIcebergParquetDeletes(plan.store, deleteFile)
1353
+ })) {
1354
+ yield batch;
1355
+ }
1356
+ }
1357
+ async function* scanRows(plan, options = {}) {
1358
+ for await (const batch of scanBatches(plan, options)) {
1359
+ for (const row of batch) {
1360
+ yield row;
1361
+ }
1362
+ }
1363
+ }
1364
+ async function* projectIcebergParquetBatches(store, table, path, partition, snapshotId, planOptions, scanOptions) {
1365
+ const readOptions = {};
1366
+ if (scanOptions.batchSize !== void 0) readOptions.batchSize = scanOptions.batchSize;
1367
+ for await (const batch of readParquetObjectBatches(store, path, readOptions)) {
1368
+ yield {
1369
+ rowOffset: batch.rowOffset,
1370
+ rows: batch.rows.map((row) => {
1371
+ const projectOptions = { snapshotId };
1372
+ if (planOptions.select !== void 0) projectOptions.select = planOptions.select;
1373
+ return table.projectRow({ ...partition, ...row }, projectOptions);
1374
+ })
1375
+ };
1376
+ }
1377
+ }
1378
+
1379
+ export { IcebergRestCatalog, IcebergTable, IcebergUnsupportedCatalog, ObjectStoreIcebergCommitCatalog, PACKAGE, applyIcebergDeletes, icebergGlueCatalog, icebergNessieCatalog, icebergRestCatalog, loadIcebergTable, loadIcebergTableFromObjectStore, loadIcebergTableFromRest, loadTable, planFiles2 as planFiles, scanBatches, scanPlannedIcebergRows, scanRows };