@gscdump/engine 0.33.10 → 0.35.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,5 @@
1
- import { asyncBufferFromUrl, cachedAsyncBuffer, parseDecimal } from "./hyparquet.mjs";
2
- import { ByteWriter, parquetWrite } from "./hyparquet-writer.mjs";
1
+ import { asyncBufferFromUrl, parseDecimal } from "./hyparquet.mjs";
2
+ import { ByteWriter } from "./hyparquet-writer.mjs";
3
3
  import { gunzip } from "./hyparquet-compressors.mjs";
4
4
  function readZigZag(reader) {
5
5
  let result = 0;
@@ -167,24 +167,6 @@ function bytesToUuid(bytes) {
167
167
  }
168
168
  return hex;
169
169
  }
170
- function sanitize(name) {
171
- let result = "";
172
- for (let i = 0; i < name.length; i++) {
173
- const ch = name.charAt(i);
174
- const isLetter = /^[A-Za-z]$/.test(ch);
175
- const isDigit = /^[0-9]$/.test(ch);
176
- if (i === 0) if (isLetter || ch === "_") result += ch;
177
- else result += isDigit ? "_" + ch : "_x" + ch.charCodeAt(0).toString(16).toUpperCase();
178
- else if (isLetter || isDigit || ch === "_") result += ch;
179
- else result += "_x" + ch.charCodeAt(0).toString(16).toUpperCase();
180
- }
181
- return result;
182
- }
183
- function bytesToHex$1(bytes) {
184
- let hex = "";
185
- for (let i = 0; i < bytes.length; i++) hex += bytes[i].toString(16).padStart(2, "0");
186
- return hex;
187
- }
188
170
  function uuid4() {
189
171
  if (globalThis.crypto?.randomUUID) return globalThis.crypto.randomUUID();
190
172
  return "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, (c) => {
@@ -243,92 +225,6 @@ function urlResolver({ requestInit } = {}) {
243
225
  }
244
226
  };
245
227
  }
246
- function cachingResolver(base) {
247
- const cache = /* @__PURE__ */ new Map();
248
- const out = { reader(path, byteLength) {
249
- let buf = cache.get(path);
250
- if (!buf) {
251
- buf = (async () => cachedAsyncBuffer(await base.reader(path, byteLength)))();
252
- cache.set(path, buf);
253
- buf.catch(() => {
254
- if (cache.get(path) === buf) cache.delete(path);
255
- });
256
- }
257
- return buf;
258
- } };
259
- if (base.writer) {
260
- const baseWriter = base.writer;
261
- out.writer = (path, options) => {
262
- const w = baseWriter(path, options);
263
- const origFinish = w.finish.bind(w);
264
- w.finish = async function() {
265
- await origFinish();
266
- cache.delete(path);
267
- };
268
- return w;
269
- };
270
- }
271
- if (base.deleter) {
272
- const baseDeleter = base.deleter;
273
- out.deleter = async (path) => {
274
- await baseDeleter(path);
275
- cache.delete(path);
276
- };
277
- }
278
- return out;
279
- }
280
- function s3Lister({ requestInit } = {}) {
281
- return async function list(url) {
282
- const s3parts = s3ParseUrl(url);
283
- if (!s3parts) throw new Error(`not an S3 URL: ${url}`);
284
- const { bucket, prefix } = s3parts;
285
- const listUrl = `https://${bucket}.s3.amazonaws.com/?list-type=2&prefix=${prefix.replace(/\/$/, "")}/&delimiter=/`;
286
- const res = await fetch(listUrl, requestInit);
287
- if (!res.ok) throw new Error(`${res.status} ${res.statusText}`);
288
- return ((await res.text()).match(/<Contents>(.*?)<\/Contents>/gs) || []).map((match) => {
289
- const keyMatch = match.match(/<Key>(.*?)<\/Key>/);
290
- if (!keyMatch) throw new Error("failed to parse S3 list response");
291
- return keyMatch[1].split("/").pop() ?? "";
292
- }).filter(Boolean);
293
- };
294
- }
295
- function s3ParseUrl(url) {
296
- if (url.startsWith("s3://") || url.startsWith("s3a://")) {
297
- const parts = url.split("/");
298
- return {
299
- bucket: parts[2],
300
- prefix: parts.slice(3).join("/")
301
- };
302
- }
303
- if (url.startsWith("https://s3.amazonaws.com/")) {
304
- const parts = url.split("/");
305
- return {
306
- bucket: parts[3],
307
- prefix: parts.slice(4).join("/")
308
- };
309
- }
310
- const m = url.match(/^https:\/\/([a-z0-9][a-z0-9-]*)\.s3(?:[.-][a-z0-9-]+)?\.amazonaws\.com\/(.*)$/);
311
- if (m) return {
312
- bucket: m[1],
313
- prefix: m[2]
314
- };
315
- }
316
- async function resolveText(resolver, path) {
317
- const ab = await resolver.reader(path);
318
- let buf = await ab.slice(0, ab.byteLength);
319
- if (isGzip(buf)) buf = await decompressGzip(buf);
320
- return new TextDecoder().decode(buf);
321
- }
322
- function isGzip(buf) {
323
- if (buf.byteLength < 2) return false;
324
- const view = new Uint8Array(buf, 0, 2);
325
- return view[0] === 31 && view[1] === 139;
326
- }
327
- async function decompressGzip(buf) {
328
- if (!globalThis.DecompressionStream) throw new Error("gzip decompression is not supported in this environment");
329
- const stream = new Blob([buf]).stream().pipeThrough(new DecompressionStream("gzip"));
330
- return await new Response(stream).arrayBuffer();
331
- }
332
228
  async function fetchAvroRecords(url, resolver, byteLength) {
333
229
  const lengthHint = byteLength !== void 0 && Number.isFinite(byteLength) ? byteLength : void 0;
334
230
  const ab = await resolver.reader(url, lengthHint);
@@ -499,182 +395,6 @@ function parseIcebergJson(text) {
499
395
  if (i !== text.length) throw new Error(`unexpected trailing input at ${i}`);
500
396
  return value;
501
397
  }
502
- function metadataFileVersionNumber(file) {
503
- const match = file.match(/^(?:v(\d+)|(\d+)-.+)(?:\.metadata\.json|\.gz\.metadata\.json|\.metadata\.json\.gz)$/);
504
- if (!match) return void 0;
505
- return Number(match[1] ?? match[2]);
506
- }
507
- function metadataFileVersionName(file) {
508
- if (metadataFileVersionNumber(file) === void 0) return void 0;
509
- return file.replace(/(?:\.metadata\.json\.gz|\.gz\.metadata\.json|\.metadata\.json)$/, "");
510
- }
511
- function metadataVersions(files) {
512
- const versions = /* @__PURE__ */ new Map();
513
- for (const file of files) {
514
- const version = metadataFileVersionNumber(file);
515
- const name = metadataFileVersionName(file);
516
- if (version === void 0 || name === void 0) continue;
517
- const current = versions.get(version);
518
- const paddedVersion = String(version).padStart(5, "0");
519
- if (current === void 0 || metadataFilePreference(file, paddedVersion) < metadataFilePreference(`${current}.metadata.json`, paddedVersion)) versions.set(version, name);
520
- }
521
- return [...versions.entries()].sort(([a], [b]) => a - b).map(([, name]) => name);
522
- }
523
- function icebergLatestVersion({ tableUrl, resolver, lister }) {
524
- resolver ??= urlResolver();
525
- lister ??= s3Lister();
526
- const url = `${tableUrl}/metadata/version-hint.text`;
527
- return resolveText(resolver, url).then((text) => {
528
- const version = parseInt(text);
529
- if (isNaN(version)) throw new Error(`invalid version: ${text}`);
530
- return `v${version}`;
531
- }).catch(() => {
532
- const metadataDir = `${tableUrl}/metadata`;
533
- return lister(metadataDir).then((files) => {
534
- const versions = metadataVersions(files);
535
- if (versions.length === 0) throw new Error("no metadata files found");
536
- return versions[versions.length - 1];
537
- });
538
- }).catch((err) => {
539
- throw new Error(`failed to determine latest iceberg version: ${err.message}`);
540
- });
541
- }
542
- async function resolveMetadata({ tableUrl, metadataFileName, resolver, lister }) {
543
- resolver ??= urlResolver();
544
- lister ??= s3Lister();
545
- if (!metadataFileName) metadataFileName = `${await icebergLatestVersion({
546
- tableUrl,
547
- resolver,
548
- lister
549
- })}.metadata.json`;
550
- const url = `${tableUrl}/metadata/${metadataFileName}`;
551
- try {
552
- return {
553
- metadata: parseIcebergJson(await resolveText(resolver, url)),
554
- metadataFileName
555
- };
556
- } catch (err) {
557
- try {
558
- const metadataDir = `${tableUrl}/metadata`;
559
- const match = findMetadataFile(await lister(metadataDir), metadataFileName);
560
- if (match) return {
561
- metadata: parseIcebergJson(await resolveText(resolver, `${metadataDir}/${match}`)),
562
- metadataFileName: match
563
- };
564
- } catch {}
565
- throw new Error(`failed to get iceberg metadata: ${err.message}`);
566
- }
567
- }
568
- function findMetadataFile(files, metadataFileName) {
569
- if (files.includes(metadataFileName)) return metadataFileName;
570
- const version = metadataFileVersionNumber(metadataFileName);
571
- if (version === void 0) return void 0;
572
- const versionNum = String(version).padStart(5, "0");
573
- return files.filter((f) => metadataFileVersionNumber(f) === version).sort((a, b) => metadataFilePreference(a, versionNum) - metadataFilePreference(b, versionNum))[0];
574
- }
575
- async function loadLatestFileCatalogMetadata({ tableUrl, resolver, lister, maxProbe = 64 }) {
576
- resolver ??= urlResolver();
577
- lister ??= s3Lister();
578
- let files;
579
- try {
580
- files = await lister(`${tableUrl}/metadata`);
581
- } catch (err) {
582
- const fallback = await hintProbeFallback(resolver, tableUrl, maxProbe);
583
- if (fallback) return fallback;
584
- throw err;
585
- }
586
- let highest = -1;
587
- let highestFile;
588
- for (const file of files) {
589
- const v = metadataFileVersionNumber(file);
590
- if (v === void 0) continue;
591
- if (v > highest) {
592
- highest = v;
593
- highestFile = file;
594
- }
595
- }
596
- if (highest < 0 || !highestFile) throw new Error(`no metadata files found at ${tableUrl}/metadata`);
597
- const metadataLocation = `${tableUrl}/metadata/${highestFile}`;
598
- const text = await resolveText(resolver, metadataLocation);
599
- return {
600
- version: highest,
601
- metadata: parseIcebergJson(text),
602
- metadataFileName: highestFile,
603
- metadataLocation
604
- };
605
- }
606
- async function hintProbeFallback(resolver, tableUrl, maxProbe) {
607
- let hintVersion;
608
- try {
609
- const text = await resolveText(resolver, `${tableUrl}/metadata/version-hint.text`);
610
- const parsed = parseInt(text);
611
- if (!isNaN(parsed)) hintVersion = parsed;
612
- } catch {}
613
- if (hintVersion === void 0 || hintVersion < 0) return void 0;
614
- let lastFound = await tryReadVersion(resolver, tableUrl, hintVersion);
615
- if (!lastFound) return void 0;
616
- let probe = hintVersion + 1;
617
- const limit = hintVersion + maxProbe;
618
- while (probe <= limit) {
619
- const next = await tryReadVersion(resolver, tableUrl, probe);
620
- if (!next) break;
621
- lastFound = next;
622
- probe++;
623
- }
624
- if (probe > limit) return void 0;
625
- return lastFound;
626
- }
627
- async function tryReadVersion(resolver, tableUrl, version) {
628
- const fileName = `v${version}.metadata.json`;
629
- const metadataLocation = `${tableUrl}/metadata/${fileName}`;
630
- try {
631
- return {
632
- version,
633
- metadata: parseIcebergJson(await resolveText(resolver, metadataLocation)),
634
- metadataFileName: fileName,
635
- metadataLocation
636
- };
637
- } catch {
638
- return;
639
- }
640
- }
641
- function metadataFilePreference(file, paddedVersion) {
642
- if (file === `v${Number(paddedVersion)}.metadata.json`) return 0;
643
- if (file === `v${Number(paddedVersion)}.gz.metadata.json`) return 1;
644
- if (file === `v${Number(paddedVersion)}.metadata.json.gz`) return 2;
645
- if (file.startsWith(`${paddedVersion}-`) && file.endsWith(".metadata.json")) return 3;
646
- if (file.startsWith(`${paddedVersion}-`) && file.endsWith(".gz.metadata.json")) return 4;
647
- if (file.startsWith(`${paddedVersion}-`) && file.endsWith(".metadata.json.gz")) return 5;
648
- return 6;
649
- }
650
- async function restCatalogConnect({ url, warehouse, requestInit }) {
651
- const base = url.replace(/\/$/, "");
652
- const configUrl = warehouse ? `${base}/v1/config?warehouse=${encodeURIComponent(warehouse)}` : `${base}/v1/config`;
653
- const res = await fetch(configUrl, requestInit);
654
- if (!res.ok) await throwRestError(res);
655
- const body = parseIcebergJson(await res.text());
656
- const defaults = body.defaults ?? {};
657
- const overrides = body.overrides ?? {};
658
- const prefix = overrides.prefix ?? defaults.prefix ?? "";
659
- return Object.freeze({
660
- type: "rest",
661
- url: base,
662
- prefix: typeof prefix === "string" ? prefix : "",
663
- defaults,
664
- overrides,
665
- requestInit
666
- });
667
- }
668
- function restCatalogListTables(ctx, { namespace }) {
669
- const ns = encodeNamespace(namespace);
670
- return paginate({}, async (query) => {
671
- const body = parseIcebergJson(await (await restFetch(ctx, `namespaces/${ns}/tables${query}`)).text());
672
- return {
673
- items: body.identifiers ?? [],
674
- nextPageToken: body["next-page-token"]
675
- };
676
- });
677
- }
678
398
  async function restCatalogLoadTable(ctx, { namespace, table }) {
679
399
  const body = parseIcebergJson(await (await restFetch(ctx, `namespaces/${encodeNamespace(namespace)}/tables/${encodeURIComponent(table)}`)).text());
680
400
  return {
@@ -705,39 +425,6 @@ async function restCatalogCreateTable(ctx, { namespace, table, schema, location,
705
425
  config: responseBody.config ?? {}
706
426
  };
707
427
  }
708
- async function restCatalogUpdateTable(ctx, { namespace, table, requirements, updates }) {
709
- const responseBody = parseIcebergJson(await (await restFetch(ctx, `namespaces/${encodeNamespace(namespace)}/tables/${encodeURIComponent(table)}`, {
710
- method: "POST",
711
- headers: { "content-type": "application/json" },
712
- body: stringifyIcebergJson({
713
- requirements,
714
- updates
715
- })
716
- })).text());
717
- return {
718
- metadataLocation: responseBody["metadata-location"],
719
- metadata: responseBody.metadata,
720
- config: responseBody.config ?? {}
721
- };
722
- }
723
- async function restCatalogDropTable(ctx, { namespace, table, purgeRequested }) {
724
- await restFetch(ctx, `namespaces/${encodeNamespace(namespace)}/tables/${encodeURIComponent(table)}${purgeRequested ? "?purgeRequested=true" : ""}`, { method: "DELETE" });
725
- }
726
- async function restCatalogCreateNamespace(ctx, { namespace, properties }) {
727
- const ns = Array.isArray(namespace) ? namespace : namespace.split(".");
728
- const body = parseIcebergJson(await (await restFetch(ctx, "namespaces", {
729
- method: "POST",
730
- headers: { "content-type": "application/json" },
731
- body: stringifyIcebergJson({
732
- namespace: ns,
733
- properties: properties ?? {}
734
- })
735
- })).text());
736
- return {
737
- namespace: body.namespace ?? ns,
738
- properties: body.properties ?? {}
739
- };
740
- }
741
428
  function encodeNamespace(namespace) {
742
429
  return (Array.isArray(namespace) ? namespace : namespace.split(".")).map((p) => encodeURIComponent(p)).join("%1F");
743
430
  }
@@ -786,64 +473,6 @@ async function throwRestError(res) {
786
473
  err.status = res.status;
787
474
  throw err;
788
475
  }
789
- async function paginate(baseParams, fetchPage) {
790
- const out = [];
791
- let pageToken;
792
- while (true) {
793
- const params = { ...baseParams };
794
- if (pageToken) params.pageToken = pageToken;
795
- const keys = Object.keys(params);
796
- const { items, nextPageToken } = await fetchPage(keys.length ? "?" + keys.map((k) => `${k}=${params[k]}`).join("&") : "");
797
- out.push(...items);
798
- if (!nextPageToken) return out;
799
- pageToken = encodeURIComponent(nextPageToken);
800
- }
801
- }
802
- async function loadTable({ catalog, namespace, table, tableUrl, resolver }) {
803
- if (catalog.type === "rest") {
804
- if (!namespace || !table) throw new Error("namespace and table are required for rest catalogs");
805
- const { metadata } = await restCatalogLoadTable(catalog, {
806
- namespace,
807
- table
808
- });
809
- return {
810
- metadata,
811
- metadataFileName: void 0,
812
- tableUrl: metadata.location,
813
- resolver
814
- };
815
- }
816
- if (catalog.type === "file") {
817
- if (!tableUrl) throw new Error("tableUrl is required for file catalogs");
818
- const eff = resolver ?? catalog.resolver;
819
- if (catalog.conditionalCommits) {
820
- const { metadata, metadataFileName, version } = await loadLatestFileCatalogMetadata({
821
- tableUrl,
822
- resolver: eff,
823
- lister: catalog.lister
824
- });
825
- return {
826
- metadata,
827
- metadataFileName,
828
- version,
829
- tableUrl,
830
- resolver: eff
831
- };
832
- }
833
- const { metadata, metadataFileName } = await resolveMetadata({
834
- tableUrl,
835
- resolver: eff,
836
- lister: catalog.lister
837
- });
838
- return {
839
- metadata,
840
- metadataFileName,
841
- tableUrl,
842
- resolver: eff
843
- };
844
- }
845
- throw new Error(`unknown catalog type: ${catalog?.type}`);
846
- }
847
476
  function validateSchemaForVersion(schema, formatVersion) {
848
477
  for (const field of schema.fields) validateFieldForVersion(field, formatVersion, field.name);
849
478
  }
@@ -937,46 +566,6 @@ function decimalRequiredBytes(precision) {
937
566
  }
938
567
  return n;
939
568
  }
940
- function decimalToFixedBytes(value, precision, scale, label) {
941
- const size = decimalRequiredBytes(precision);
942
- if (value instanceof Uint8Array) {
943
- if (value.length !== size) throw new Error(`expected ${label}`);
944
- return value;
945
- }
946
- if (typeof value !== "number" && typeof value !== "bigint") throw new Error(`expected ${label}`);
947
- const factor = 10n ** BigInt(scale);
948
- const unscaled = typeof value === "bigint" ? value * factor : BigInt(Math.round(value * Number(factor)));
949
- const limit = 10n ** BigInt(precision);
950
- if (unscaled >= limit || unscaled <= -limit) throw new Error(`${label} exceeds precision ${precision}`);
951
- return bigintToFixedBytes(unscaled, size, label);
952
- }
953
- function toUint8Array(value) {
954
- return value instanceof Uint8Array ? value : new Uint8Array(value);
955
- }
956
- function uuidToBytes(value, label) {
957
- if (value instanceof Uint8Array) {
958
- if (value.length !== 16) throw new Error(`expected ${label}`);
959
- return value;
960
- }
961
- if (typeof value !== "string") throw new Error(`expected ${label}`);
962
- const hex = value.toLowerCase().replace(/-/g, "");
963
- if (!/^[0-9a-f]{32}$/.test(hex)) throw new Error(`expected ${label}`);
964
- const bytes = /* @__PURE__ */ new Uint8Array(16);
965
- for (let i = 0; i < bytes.length; i++) bytes[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16);
966
- return bytes;
967
- }
968
- function bigintToFixedBytes(value, size, label) {
969
- const bytes = new Uint8Array(size);
970
- let v = value;
971
- for (let i = size - 1; i >= 0; i--) {
972
- bytes[i] = Number(v & 255n);
973
- v >>= 8n;
974
- }
975
- const negative = value < 0n;
976
- const signBitSet = (bytes[0] & 128) !== 0;
977
- if (!negative && (v !== 0n || signBitSet) || negative && (v !== -1n || !signBitSet)) throw new Error(`${label} does not fit in ${size} bytes`);
978
- return bytes;
979
- }
980
569
  function parseTransform(transform) {
981
570
  if (transform === "identity" || transform === "void" || transform === "year" || transform === "month" || transform === "day" || transform === "hour") return { kind: transform };
982
571
  let m = /^bucket\[(\d+)\]$/.exec(transform);
@@ -1011,130 +600,6 @@ function transformResultType(transform, sourceType) {
1011
600
  case "bucket": return "int";
1012
601
  }
1013
602
  }
1014
- function applyTransform(transform, value, sourceType) {
1015
- const parsed = parseTransform(transform);
1016
- validateTransformSource(parsed, sourceType);
1017
- if (value == null) return null;
1018
- switch (parsed.kind) {
1019
- case "identity": return value;
1020
- case "void": return null;
1021
- case "year": return yearTransform(value, sourceType);
1022
- case "month": return monthTransform(value, sourceType);
1023
- case "day": return dayTransform(value, sourceType);
1024
- case "hour": return hourTransform(value, sourceType);
1025
- case "bucket": return bucketTransform(value, sourceType, parsed.n);
1026
- case "truncate": return truncateTransform(value, sourceType, parsed.w);
1027
- }
1028
- }
1029
- function dateAsMillis(value, sourceType, transform) {
1030
- const t = typeName(sourceType);
1031
- validateTransformSource({ kind: transform }, sourceType);
1032
- if (value instanceof Date) return value.getTime();
1033
- const n = typeof value === "bigint" ? value : BigInt(value);
1034
- switch (t) {
1035
- case "date": return Number(n) * 864e5;
1036
- case "timestamp":
1037
- case "timestamptz": return Number(n / 1000n);
1038
- default: return Number(n / 1000000n);
1039
- }
1040
- }
1041
- function yearTransform(v, t) {
1042
- return new Date(dateAsMillis(v, t, "year")).getUTCFullYear() - 1970;
1043
- }
1044
- function monthTransform(v, t) {
1045
- const d = new Date(dateAsMillis(v, t, "month"));
1046
- return (d.getUTCFullYear() - 1970) * 12 + d.getUTCMonth();
1047
- }
1048
- function dayTransform(v, t) {
1049
- return Math.floor(dateAsMillis(v, t, "day") / 864e5);
1050
- }
1051
- function hourTransform(v, t) {
1052
- return Math.floor(dateAsMillis(v, t, "hour") / 36e5);
1053
- }
1054
- function bucketTransform(value, sourceType, n) {
1055
- return (murmur3_32(bucketBytes(value, sourceType), 0) & 2147483647) % n;
1056
- }
1057
- function bucketBytes(value, sourceType) {
1058
- const t = typeName(sourceType);
1059
- if (t.startsWith("decimal(")) return decimalToUnscaledBytes(value, t);
1060
- if (t === "uuid") return uuidToBytes(value, "uuid partition value");
1061
- if (t.startsWith("fixed[") || t === "binary" || t === "fixed") return value instanceof Uint8Array ? value : new Uint8Array(value);
1062
- switch (t) {
1063
- case "int":
1064
- case "long":
1065
- case "date":
1066
- case "time":
1067
- case "timestamp":
1068
- case "timestamptz":
1069
- case "timestamp_ns":
1070
- case "timestamptz_ns": {
1071
- let v;
1072
- if (t === "date") v = value instanceof Date ? BigInt(Math.floor(value.getTime() / 864e5)) : BigInt(value);
1073
- else if (t === "timestamp" || t === "timestamptz") v = value instanceof Date ? BigInt(value.getTime()) * 1000n : BigInt(value);
1074
- else if (t === "timestamp_ns" || t === "timestamptz_ns") v = value instanceof Date ? BigInt(value.getTime()) * 1000n : BigInt(value) / 1000n;
1075
- else v = typeof value === "bigint" ? value : BigInt(value);
1076
- const out = /* @__PURE__ */ new Uint8Array(8);
1077
- new DataView(out.buffer).setBigInt64(0, v, true);
1078
- return out;
1079
- }
1080
- case "string": return new TextEncoder().encode(String(value));
1081
- default: throw new Error(`bucket transform: unsupported source type ${t}`);
1082
- }
1083
- }
1084
- function decimalToUnscaledBytes(value, decimalType) {
1085
- const m = /^decimal\((\d+),\s*(\d+)\)$/.exec(decimalType);
1086
- if (!m) throw new Error(`bucket transform: invalid decimal type ${decimalType}`);
1087
- const scale = parseInt(m[2], 10);
1088
- const factor = 10n ** BigInt(scale);
1089
- const unscaled = typeof value === "bigint" ? value * factor : BigInt(Math.round(Number(value) * Number(factor)));
1090
- const bytes = [];
1091
- let v = unscaled;
1092
- while (true) {
1093
- const byte = Number(v & 255n);
1094
- bytes.unshift(byte);
1095
- v >>= 8n;
1096
- const sign = byte & 128;
1097
- if (!sign && v === 0n || sign && v === -1n) break;
1098
- }
1099
- return new Uint8Array(bytes);
1100
- }
1101
- function truncateTransform(value, sourceType, w) {
1102
- const t = typeName(sourceType);
1103
- if (t.startsWith("decimal(")) {
1104
- const m = /^decimal\((\d+),\s*(\d+)\)$/.exec(t);
1105
- if (!m) throw new Error(`truncate transform: invalid decimal type ${t}`);
1106
- const scale = parseInt(m[2], 10);
1107
- const factor = 10n ** BigInt(scale);
1108
- const unscaled = typeof value === "bigint" ? value * factor : BigInt(Math.round(Number(value) * Number(factor)));
1109
- const W = BigInt(w);
1110
- const truncated = unscaled - (unscaled % W + W) % W;
1111
- return Number(truncated) / Number(factor);
1112
- }
1113
- if (t === "binary") return (value instanceof Uint8Array ? value : new Uint8Array(value)).slice(0, w);
1114
- switch (t) {
1115
- case "int": {
1116
- const v = Number(value);
1117
- return v - (v % w + w) % w;
1118
- }
1119
- case "long": {
1120
- const W = BigInt(w);
1121
- const v = typeof value === "bigint" ? value : BigInt(value);
1122
- return v - (v % W + W) % W;
1123
- }
1124
- case "string": {
1125
- const s = String(value);
1126
- let count = 0;
1127
- let i = 0;
1128
- while (i < s.length && count < w) {
1129
- const code = s.codePointAt(i);
1130
- i += code > 65535 ? 2 : 1;
1131
- count++;
1132
- }
1133
- return s.slice(0, i);
1134
- }
1135
- default: throw new Error(`truncate transform: unsupported source type ${t}`);
1136
- }
1137
- }
1138
603
  function validateTransformSource(parsed, sourceType) {
1139
604
  const t = typeName(sourceType);
1140
605
  switch (parsed.kind) {
@@ -1158,80 +623,6 @@ function validateTransformSource(parsed, sourceType) {
1158
623
  throw new Error("hour transform: unsupported source type " + t);
1159
624
  }
1160
625
  }
1161
- function murmur3_32(data, seed) {
1162
- const c1 = 3432918353;
1163
- const c2 = 461845907;
1164
- const len = data.length;
1165
- const nBlocks = len >>> 2;
1166
- let h1 = seed >>> 0;
1167
- for (let i = 0; i < nBlocks; i++) {
1168
- const off = i * 4;
1169
- let k1 = data[off] | data[off + 1] << 8 | data[off + 2] << 16 | data[off + 3] << 24;
1170
- k1 = Math.imul(k1, c1);
1171
- k1 = k1 << 15 | k1 >>> 17;
1172
- k1 = Math.imul(k1, c2);
1173
- h1 ^= k1;
1174
- h1 = h1 << 13 | h1 >>> 19;
1175
- h1 = Math.imul(h1, 5) + 3864292196 | 0;
1176
- }
1177
- let k1 = 0;
1178
- const tail = nBlocks * 4;
1179
- switch (len & 3) {
1180
- case 3: k1 ^= data[tail + 2] << 16;
1181
- case 2: k1 ^= data[tail + 1] << 8;
1182
- case 1:
1183
- k1 ^= data[tail];
1184
- k1 = Math.imul(k1, c1);
1185
- k1 = k1 << 15 | k1 >>> 17;
1186
- k1 = Math.imul(k1, c2);
1187
- h1 ^= k1;
1188
- }
1189
- h1 ^= len;
1190
- h1 ^= h1 >>> 16;
1191
- h1 = Math.imul(h1, 2246822507);
1192
- h1 ^= h1 >>> 13;
1193
- h1 = Math.imul(h1, 3266489909);
1194
- h1 ^= h1 >>> 16;
1195
- return h1 >>> 0;
1196
- }
1197
- function groupByPartition(records, schema, partitionSpec) {
1198
- const sourceFields = partitionSpec.fields.map((pf) => {
1199
- const sourceId = pf["source-id"];
1200
- if (sourceId === void 0) throw new Error(`partition field ${pf.name} is missing source-id`);
1201
- const sourceField = schema.fields.find((f) => f.id === sourceId);
1202
- if (!sourceField) throw new Error(`partition source field id ${sourceId} not found in schema`);
1203
- return {
1204
- partitionName: pf.name,
1205
- sourceName: sourceField.name,
1206
- sourceType: sourceField.type,
1207
- sourceWriteDefault: sourceField["write-default"],
1208
- transform: pf.transform,
1209
- resultType: transformResultType(pf.transform, sourceField.type)
1210
- };
1211
- });
1212
- const groups = /* @__PURE__ */ new Map();
1213
- for (const record of records) {
1214
- const partition = {};
1215
- const keyParts = [];
1216
- for (const { partitionName, sourceName, sourceType, sourceWriteDefault, transform, resultType } of sourceFields) {
1217
- let v = record[sourceName];
1218
- if (v === void 0 && sourceWriteDefault !== void 0) v = sourceWriteDefault;
1219
- partition[partitionName] = applyTransform(transform, v === void 0 ? null : v, sourceType);
1220
- keyParts.push(partitionKeyPart(partition[partitionName], resultType));
1221
- }
1222
- const key = JSON.stringify(keyParts);
1223
- let group = groups.get(key);
1224
- if (!group) {
1225
- group = {
1226
- partition,
1227
- records: []
1228
- };
1229
- groups.set(key, group);
1230
- }
1231
- group.records.push(record);
1232
- }
1233
- return [...groups.values()];
1234
- }
1235
626
  function validatePartitionSpecForWrite(schema, partitionSpec, label = "partition spec") {
1236
627
  for (const pf of partitionSpec.fields) {
1237
628
  const sourceId = pf["source-id"];
@@ -1241,56 +632,6 @@ function validatePartitionSpecForWrite(schema, partitionSpec, label = "partition
1241
632
  icebergTypeToAvro(transformResultType(pf.transform, sourceField.type), pf["field-id"]);
1242
633
  }
1243
634
  }
1244
- function partitionAvroSchema(schema, partitionSpec) {
1245
- return {
1246
- type: "record",
1247
- name: "r102",
1248
- fields: partitionSpec.fields.map((pf) => {
1249
- const sourceField = schema.fields.find((f) => f.id === pf["source-id"]);
1250
- if (!sourceField) throw new Error(`partition source field id ${pf["source-id"]} not found`);
1251
- const resultType = transformResultType(pf.transform, sourceField.type);
1252
- return {
1253
- name: pf.name,
1254
- "field-id": pf["field-id"],
1255
- default: null,
1256
- type: ["null", icebergTypeToAvro(resultType, pf["field-id"])]
1257
- };
1258
- })
1259
- };
1260
- }
1261
- function partitionSpecJson(partitionSpec) {
1262
- return JSON.stringify(partitionSpec.fields);
1263
- }
1264
- function partitionToAvroRecord(partition, schema, partitionSpec) {
1265
- const out = {};
1266
- for (const pf of partitionSpec.fields) {
1267
- const sourceField = schema.fields.find((f) => f.id === pf["source-id"]);
1268
- if (!sourceField) throw new Error(`partition source field id ${pf["source-id"]} not found`);
1269
- const resultType = transformResultType(pf.transform, sourceField.type);
1270
- const value = partition[pf.name];
1271
- out[pf.name] = value == null ? null : coerceForAvro(value, resultType);
1272
- }
1273
- return out;
1274
- }
1275
- function partitionKeyPart(value, type) {
1276
- if (value === null || value === void 0) return "__null__";
1277
- const name = typeof type === "string" ? type : type.type;
1278
- if (name === "uuid") return `uuid:${bytesToHex$1(uuidToBytes(value, "uuid partition value"))}`;
1279
- if (typeof value === "number" && (name === "float" || name === "double")) return `${name}:${floatPartitionKey(value, name)}`;
1280
- if (name === "long") return `long:${BigInt(value)}`;
1281
- if (typeof value === "bigint") return `b:${value.toString()}`;
1282
- if (value instanceof Date) return `d:${value.getTime()}`;
1283
- if (value instanceof Uint8Array) return `x:${bytesToHex$1(value)}`;
1284
- return `${typeof value}:${String(value)}`;
1285
- }
1286
- function floatPartitionKey(value, type) {
1287
- if (Number.isNaN(value)) return "nan";
1288
- const bytes = new Uint8Array(type === "float" ? 4 : 8);
1289
- const view = new DataView(bytes.buffer);
1290
- if (type === "float") view.setFloat32(0, value, false);
1291
- else view.setFloat64(0, value, false);
1292
- return bytesToHex$1(bytes);
1293
- }
1294
635
  function icebergTypeToAvro(type, fieldId) {
1295
636
  const name = typeof type === "string" ? type : type.type;
1296
637
  const decimal = parseDecimalType(name);
@@ -1353,21 +694,6 @@ function icebergTypeToAvro(type, fieldId) {
1353
694
  default: throw new Error(`unsupported partition source type: ${name}`);
1354
695
  }
1355
696
  }
1356
- function coerceForAvro(value, type) {
1357
- const name = typeof type === "string" ? type : type.type;
1358
- if (name === "long") return typeof value === "bigint" ? value : BigInt(value);
1359
- if (name === "uuid") return uuidToBytes(value, "uuid partition value");
1360
- const decimal = parseDecimalType(name);
1361
- if (decimal) return decimalToFixedBytes(value, decimal.precision, decimal.scale, `decimal(${decimal.precision},${decimal.scale}) partition value`);
1362
- const fixed = /^fixed\[(\d+)\]$/.exec(name);
1363
- if (fixed) {
1364
- const bytes = toUint8Array(value);
1365
- const expected = parseInt(fixed[1], 10);
1366
- if (bytes.length !== expected) throw new Error(`expected fixed[${expected}] partition value`);
1367
- return bytes;
1368
- }
1369
- return value;
1370
- }
1371
697
  async function icebergCreate({ tableUrl, resolver, schema, formatVersion, partitionSpec, sortOrder, properties, conditionalCommits }) {
1372
698
  if (!tableUrl) throw new Error("tableUrl is required");
1373
699
  if (formatVersion === void 0) {
@@ -1430,1921 +756,56 @@ function maxPartitionFieldId(partitionFields = []) {
1430
756
  for (const pf of partitionFields) if (max < pf["field-id"]) max = pf["field-id"];
1431
757
  return max;
1432
758
  }
1433
- async function fileCatalogCommit({ tableUrl, metadata, metadataFileName, currentVersion, staged, resolver, conditionalCommits }) {
1434
- if (!tableUrl) throw new Error("tableUrl is required");
1435
- if (!resolver?.writer) throw new Error("resolver.writer is required");
1436
- checkRequirements(metadata, staged.requirements);
1437
- const updated = applyUpdates(staged.updates.some((up) => up.action === "add-snapshot") ? metadata : {
1438
- ...metadata,
1439
- "last-updated-ms": Date.now()
1440
- }, staged.updates);
1441
- const priorMetadataLog = metadata["metadata-log"] ?? [];
1442
- const derivedVersion = currentVersion ?? deriveCurrentVersion(priorMetadataLog);
1443
- const newVersion = derivedVersion + 1;
1444
- const currentMetadataPath = metadataFileName ? `${tableUrl}/metadata/${metadataFileName}` : `${tableUrl}/metadata/v${derivedVersion}.metadata.json`;
1445
- const newMetadataPath = `${tableUrl}/metadata/v${newVersion}.metadata.json`;
1446
- const appendedLog = [...priorMetadataLog, {
1447
- "timestamp-ms": metadata["last-updated-ms"],
1448
- "metadata-file": currentMetadataPath
1449
- }];
1450
- const max = Number(updated.properties?.["write.metadata.previous-versions-max"] ?? 100);
1451
- const droppedLog = max > 0 && appendedLog.length > max ? appendedLog.slice(0, appendedLog.length - max) : [];
1452
- const trimmedLog = droppedLog.length > 0 ? appendedLog.slice(-max) : appendedLog;
1453
- const newMetadata = {
1454
- ...updated,
1455
- "metadata-log": trimmedLog
1456
- };
1457
- const metaWriter = conditionalCommits ? resolver.writer(newMetadataPath, { ifNoneMatch: "*" }) : resolver.writer(newMetadataPath);
1458
- metaWriter.appendBytes(new TextEncoder().encode(stringifyIcebergJson(newMetadata, 2)));
1459
- await metaWriter.finish();
1460
- try {
1461
- const hintWriter = resolver.writer(`${tableUrl}/metadata/version-hint.text`);
1462
- hintWriter.appendBytes(new TextEncoder().encode(String(newVersion)));
1463
- await hintWriter.finish();
1464
- } catch {}
1465
- if (updated.properties?.["write.metadata.delete-after-commit.enabled"] === "true" && droppedLog.length > 0 && resolver.deleter) {
1466
- const { deleter } = resolver;
1467
- await Promise.allSettled(droppedLog.map((entry) => deleter(entry["metadata-file"])));
1468
- }
1469
- return newMetadata;
1470
- }
1471
- function deriveCurrentVersion(priorMetadataLog) {
1472
- if (priorMetadataLog.length === 0) return 1;
1473
- const match = (priorMetadataLog[priorMetadataLog.length - 1]["metadata-file"].split("/").pop() ?? "").match(/^(?:v(\d+)|0*(\d+)-[0-9a-f-]+)\.metadata\.json$/);
1474
- if (match) return Number(match[1] ?? match[2]) + 1;
1475
- return priorMetadataLog.length + 1;
1476
- }
1477
- function checkRequirements(metadata, requirements) {
1478
- for (const req of requirements) if (req.type === "assert-create") throw new Error("requirement failed: assert-create against an existing table");
1479
- else if (req.type === "assert-table-uuid") {
1480
- if (metadata["table-uuid"] !== req.uuid) throw new Error(`requirement failed: table-uuid expected ${req.uuid}, got ${metadata["table-uuid"]}`);
1481
- } else if (req.type === "assert-ref-snapshot-id") {
1482
- let current = (metadata.refs ?? {})[req.ref]?.["snapshot-id"] ?? null;
1483
- if (current === null && req.ref === "main") current = metadata["current-snapshot-id"] ?? null;
1484
- const expected = req["snapshot-id"];
1485
- if (!(current === expected || current != null && expected != null && BigInt(current) === BigInt(expected))) throw new Error(`requirement failed: ref ${req.ref} expected snapshot ${expected}, got ${current}`);
1486
- } else if (req.type === "assert-next-row-id") {
1487
- const current = Number(metadata["next-row-id"] ?? 0);
1488
- if (current !== req["next-row-id"]) throw new Error(`requirement failed: next-row-id expected ${req["next-row-id"]}, got ${current}`);
1489
- } else if (req.type === "assert-current-schema-id") {
1490
- const current = metadata["current-schema-id"];
1491
- if (current !== req["current-schema-id"]) throw new Error(`requirement failed: current-schema-id expected ${req["current-schema-id"]}, got ${current}`);
1492
- } else if (req.type === "assert-last-assigned-field-id") {
1493
- const current = metadata["last-column-id"];
1494
- if (current !== req["last-assigned-field-id"]) throw new Error(`requirement failed: last-assigned-field-id expected ${req["last-assigned-field-id"]}, got ${current}`);
1495
- } else if (req.type === "assert-last-assigned-partition-id") {
1496
- const current = metadata["last-partition-id"];
1497
- if (current !== req["last-assigned-partition-id"]) throw new Error(`requirement failed: last-assigned-partition-id expected ${req["last-assigned-partition-id"]}, got ${current}`);
1498
- } else if (req.type === "assert-default-spec-id") {
1499
- const current = metadata["default-spec-id"];
1500
- if (current !== req["default-spec-id"]) throw new Error(`requirement failed: default-spec-id expected ${req["default-spec-id"]}, got ${current}`);
1501
- } else if (req.type === "assert-default-sort-order-id") {
1502
- const current = metadata["default-sort-order-id"];
1503
- if (current !== req["default-sort-order-id"]) throw new Error(`requirement failed: default-sort-order-id expected ${req["default-sort-order-id"]}, got ${current}`);
1504
- } else throw new Error(`unknown requirement: ${JSON.stringify(req)}`);
1505
- }
1506
- function applyUpdates(metadata, updates) {
1507
- let next = { ...metadata };
1508
- for (const up of updates) if (up.action === "add-snapshot") {
1509
- const snap = up.snapshot;
1510
- const priorSnapshots = next.snapshots ?? [];
1511
- if (priorSnapshots.some((s) => s["snapshot-id"] === snap["snapshot-id"])) throw new Error(`add-snapshot: snapshot-id ${snap["snapshot-id"]} already exists`);
1512
- next = {
1513
- ...next,
1514
- snapshots: [...priorSnapshots, snap],
1515
- "last-sequence-number": Math.max(next["last-sequence-number"] ?? 0, snap["sequence-number"]),
1516
- "last-updated-ms": snap["timestamp-ms"]
1517
- };
1518
- if (next["format-version"] >= 3 && snap["first-row-id"] !== void 0 && snap["added-rows"] !== void 0) {
1519
- const nextRowId = snap["first-row-id"] + snap["added-rows"];
1520
- next["next-row-id"] = Math.max(Number(next["next-row-id"] ?? 0), nextRowId);
1521
- }
1522
- } else if (up.action === "set-properties") next = {
1523
- ...next,
1524
- properties: {
1525
- ...next.properties,
1526
- ...up.updates
1527
- }
1528
- };
1529
- else if (up.action === "remove-properties") {
1530
- const properties = { ...next.properties };
1531
- for (const key of up.removals) delete properties[key];
1532
- next = {
1533
- ...next,
1534
- properties
1535
- };
1536
- } else if (up.action === "add-schema") {
1537
- const schemas = next.schemas ?? [];
1538
- let schemaId = up.schema["schema-id"];
1539
- if (schemaId === -1) schemaId = schemas.reduce((m, s) => Math.max(m, s["schema-id"]), -1) + 1;
1540
- else if (schemas.some((s) => s["schema-id"] === schemaId)) throw new Error(`add-schema: schema-id ${schemaId} already exists`);
1541
- const newSchema = {
1542
- ...up.schema,
1543
- "schema-id": schemaId
1544
- };
1545
- validateSchemaForVersion(newSchema, next["format-version"]);
1546
- const priorLastColumnId = next["last-column-id"] ?? 0;
1547
- validateAssignedFieldIds(newSchema, currentAssignedIdIndex(schemas, next["current-schema-id"]), priorLastColumnId);
1548
- validateSchemaEvolution(schemas, newSchema, priorLastColumnId, next["format-version"]);
1549
- validateNewRequiredFields(newSchema, priorLastColumnId);
1550
- next = {
1551
- ...next,
1552
- schemas: [...schemas, newSchema],
1553
- "last-column-id": Math.max(priorLastColumnId, maxFieldId(newSchema.fields))
1554
- };
1555
- } else if (up.action === "set-current-schema") {
1556
- let id = up["schema-id"];
1557
- const schemas = next.schemas ?? [];
1558
- if (id === -1) {
1559
- if (schemas.length === 0) throw new Error("set-current-schema: table has no schemas");
1560
- id = schemas[schemas.length - 1]["schema-id"];
1561
- } else if (!schemas.some((s) => s["schema-id"] === id)) throw new Error(`set-current-schema: schema-id ${id} not found`);
1562
- next = {
1563
- ...next,
1564
- "current-schema-id": id
1565
- };
1566
- } else if (up.action === "add-sort-order") {
1567
- const orders = next["sort-orders"] ?? [];
1568
- let orderId = up["sort-order"]["order-id"];
1569
- if (orderId === -1) orderId = orders.reduce((m, o) => Math.max(m, o["order-id"]), -1) + 1;
1570
- else if (orders.some((o) => o["order-id"] === orderId)) throw new Error(`add-sort-order: order-id ${orderId} already exists`);
1571
- const newOrder = {
1572
- ...up["sort-order"],
1573
- "order-id": orderId
1574
- };
1575
- next = {
1576
- ...next,
1577
- "sort-orders": [...orders, newOrder]
1578
- };
1579
- } else if (up.action === "set-default-sort-order") {
1580
- let id = up["sort-order-id"];
1581
- const orders = next["sort-orders"] ?? [];
1582
- if (id === -1) {
1583
- if (orders.length === 0) throw new Error("set-default-sort-order: table has no sort orders");
1584
- id = orders[orders.length - 1]["order-id"];
1585
- } else if (!orders.some((o) => o["order-id"] === id)) throw new Error(`set-default-sort-order: sort-order-id ${id} not found`);
1586
- next = {
1587
- ...next,
1588
- "default-sort-order-id": id
1589
- };
1590
- } else if (up.action === "add-spec") {
1591
- const specs = next["partition-specs"] ?? [];
1592
- let specId = up.spec["spec-id"];
1593
- if (specId === -1) specId = specs.reduce((m, s) => Math.max(m, s["spec-id"]), -1) + 1;
1594
- else if (specs.some((s) => s["spec-id"] === specId)) throw new Error(`add-spec: spec-id ${specId} already exists`);
1595
- const newSpec = {
1596
- ...up.spec,
1597
- "spec-id": specId
1598
- };
1599
- validatePartitionSpecEvolution(specs, newSpec, currentSchemaForMetadata(next));
1600
- let nextLastPartitionId = next["last-partition-id"] ?? 0;
1601
- for (const f of newSpec.fields) if (f["field-id"] > nextLastPartitionId) nextLastPartitionId = f["field-id"];
1602
- next = {
1603
- ...next,
1604
- "partition-specs": [...specs, newSpec],
1605
- "last-partition-id": nextLastPartitionId
1606
- };
1607
- } else if (up.action === "set-default-spec") {
1608
- let id = up["spec-id"];
1609
- const specs = next["partition-specs"] ?? [];
1610
- if (id === -1) {
1611
- if (specs.length === 0) throw new Error("set-default-spec: table has no partition specs");
1612
- id = specs[specs.length - 1]["spec-id"];
1613
- } else if (!specs.some((s) => s["spec-id"] === id)) throw new Error(`set-default-spec: spec-id ${id} not found`);
1614
- next = {
1615
- ...next,
1616
- "default-spec-id": id
1617
- };
1618
- } else if (up.action === "remove-snapshots") {
1619
- const removeIds = new Set(up["snapshot-ids"]);
1620
- const snapshots = (next.snapshots ?? []).filter((s) => !removeIds.has(s["snapshot-id"]));
1621
- const log = (next["snapshot-log"] ?? []).filter((e) => !removeIds.has(e["snapshot-id"]));
1622
- next = {
1623
- ...next,
1624
- snapshots,
1625
- "snapshot-log": log
1626
- };
1627
- } else if (up.action === "set-snapshot-ref") {
1628
- const ref = {
1629
- "snapshot-id": up["snapshot-id"],
1630
- type: up.type
1631
- };
1632
- if (up["min-snapshots-to-keep"] !== void 0) ref["min-snapshots-to-keep"] = up["min-snapshots-to-keep"];
1633
- if (up["max-snapshot-age-ms"] !== void 0) ref["max-snapshot-age-ms"] = up["max-snapshot-age-ms"];
1634
- if (up["max-ref-age-ms"] !== void 0) ref["max-ref-age-ms"] = up["max-ref-age-ms"];
1635
- next = {
1636
- ...next,
1637
- refs: {
1638
- ...next.refs,
1639
- [up["ref-name"]]: ref
1640
- }
1641
- };
1642
- if (up["ref-name"] === "main" && up.type === "branch") {
1643
- next["current-snapshot-id"] = up["snapshot-id"];
1644
- next["snapshot-log"] = [...next["snapshot-log"] ?? [], {
1645
- "timestamp-ms": next["last-updated-ms"],
1646
- "snapshot-id": up["snapshot-id"]
1647
- }];
759
+ async function icebergManifests({ metadata, resolver, snapshotId, partitionFilter }) {
760
+ resolver ??= urlResolver();
761
+ const rawTarget = snapshotId ?? metadata["current-snapshot-id"];
762
+ if (rawTarget == null || rawTarget < 0) throw new Error("No current snapshot id found in table metadata");
763
+ const targetId = BigInt(rawTarget);
764
+ const snapshot = metadata.snapshots?.find((s) => BigInt(s["snapshot-id"]) === targetId);
765
+ if (!snapshot) throw new Error(`Snapshot ${rawTarget} not found in metadata`);
766
+ let manifests = [];
767
+ if (snapshot["manifest-list"]) {
768
+ const manifestListUrl = snapshot["manifest-list"];
769
+ manifests = await fetchAvroRecords(manifestListUrl, resolver);
770
+ } else if (snapshot.manifests) manifests = snapshot.manifests;
771
+ else throw new Error("No manifest information found in snapshot");
772
+ if (partitionFilter) manifests = manifests.filter((manifest) => {
773
+ let keep = true;
774
+ try {
775
+ keep = partitionFilter(manifest.partitions, manifest.partition_spec_id ?? 0, manifest) !== false;
776
+ } catch {
777
+ keep = true;
1648
778
  }
1649
- } else throw new Error(`unknown update: ${JSON.stringify(up)}`);
1650
- return next;
1651
- }
1652
- function currentAssignedIdIndex(schemas, currentSchemaId) {
1653
- const currentSchema = schemas.find((s) => s["schema-id"] === currentSchemaId) ?? schemas[schemas.length - 1];
1654
- const assignedIds = /* @__PURE__ */ new Map();
1655
- if (currentSchema) indexAssignedFieldIds(currentSchema.fields, "", assignedIds);
1656
- return assignedIds;
779
+ return keep;
780
+ });
781
+ return await fetchManifests(manifests, resolver);
1657
782
  }
1658
- function indexAssignedFieldIds(fields, prefix, assignedIds) {
1659
- for (const field of fields) {
1660
- const path = prefix ? `${prefix}.${field.name}` : field.name;
1661
- assignedIds.set(field.id, {
1662
- kind: "field",
1663
- path
1664
- });
1665
- indexAssignedTypeIds(field.type, path, assignedIds);
783
+ const MANIFEST_FETCH_CONCURRENCY = 8;
784
+ async function fetchOneManifest(manifest, resolver) {
785
+ const url = manifest.manifest_path;
786
+ const entries = await fetchAvroRecords(url, resolver, Number(manifest.manifest_length));
787
+ for (const entry of entries) {
788
+ entry.partition_spec_id = manifest.partition_spec_id ?? 0;
789
+ if (entry.sequence_number === void 0) entry.sequence_number = manifest.sequence_number ?? 0n;
790
+ if (entry.status === 1) {
791
+ if (entry.sequence_number === void 0) entry.sequence_number = manifest.sequence_number;
792
+ if (entry.file_sequence_number === void 0) entry.file_sequence_number = manifest.sequence_number;
793
+ } else if (entry.sequence_number === void 0 || entry.file_sequence_number === void 0) throw new Error("iceberg manifest entry missing sequence number");
1666
794
  }
795
+ assignFirstRowIds(manifest, entries);
796
+ return {
797
+ url,
798
+ entries
799
+ };
1667
800
  }
1668
- function indexAssignedTypeIds(type, path, assignedIds) {
1669
- if (typeof type === "string") return;
1670
- if (type.type === "struct") indexAssignedFieldIds(type.fields, path, assignedIds);
1671
- else if (type.type === "list") {
1672
- assignedIds.set(type["element-id"], {
1673
- kind: "list element",
1674
- path: `${path}.element`
1675
- });
1676
- indexAssignedTypeIds(type.element, `${path}.element`, assignedIds);
1677
- } else if (type.type === "map") {
1678
- assignedIds.set(type["key-id"], {
1679
- kind: "map key",
1680
- path: `${path}.key`
1681
- });
1682
- assignedIds.set(type["value-id"], {
1683
- kind: "map value",
1684
- path: `${path}.value`
1685
- });
1686
- indexAssignedTypeIds(type.key, `${path}.key`, assignedIds);
1687
- indexAssignedTypeIds(type.value, `${path}.value`, assignedIds);
1688
- }
1689
- }
1690
- function validateAssignedFieldIds(schema, priorAssignedIds, priorLastColumnId) {
1691
- validateAssignedFields(schema.fields, "", priorAssignedIds, priorLastColumnId);
1692
- }
1693
- function validateAssignedFields(fields, prefix, priorAssignedIds, priorLastColumnId) {
1694
- for (const field of fields) {
1695
- const path = prefix ? `${prefix}.${field.name}` : field.name;
1696
- validateAssignedId(field.id, "field", path, priorAssignedIds, priorLastColumnId);
1697
- validateAssignedTypeIds(field.type, path, priorAssignedIds, priorLastColumnId);
1698
- }
1699
- }
1700
- function validateAssignedTypeIds(type, path, priorAssignedIds, priorLastColumnId) {
1701
- if (typeof type === "string") return;
1702
- if (type.type === "struct") validateAssignedFields(type.fields, path, priorAssignedIds, priorLastColumnId);
1703
- else if (type.type === "list") {
1704
- validateAssignedId(type["element-id"], "list element", `${path}.element`, priorAssignedIds, priorLastColumnId);
1705
- validateAssignedTypeIds(type.element, `${path}.element`, priorAssignedIds, priorLastColumnId);
1706
- } else if (type.type === "map") {
1707
- validateAssignedId(type["key-id"], "map key", `${path}.key`, priorAssignedIds, priorLastColumnId);
1708
- validateAssignedId(type["value-id"], "map value", `${path}.value`, priorAssignedIds, priorLastColumnId);
1709
- validateAssignedTypeIds(type.key, `${path}.key`, priorAssignedIds, priorLastColumnId);
1710
- validateAssignedTypeIds(type.value, `${path}.value`, priorAssignedIds, priorLastColumnId);
1711
- }
1712
- }
1713
- function validateAssignedId(id, kind, path, priorAssignedIds, priorLastColumnId) {
1714
- if (id > priorLastColumnId) return;
1715
- const prior = priorAssignedIds.get(id);
1716
- if (!prior) throw new Error(`add-schema: ${kind} ${path} uses unassigned id ${id} (last-column-id ${priorLastColumnId})`);
1717
- if (prior.kind !== kind) throw new Error(`add-schema: ${kind} ${path} uses id ${id} previously assigned to ${prior.kind} ${prior.path}`);
1718
- }
1719
- function validateNewRequiredFields(schema, priorLastColumnId) {
1720
- for (const field of schema.fields) if (field.id > priorLastColumnId && field.required) {
1721
- if (field["initial-default"] == null) throw new Error(`add-schema: required field ${field.name} (id ${field.id}) needs a non-null initial-default`);
1722
- if (field["write-default"] == null) throw new Error(`add-schema: required field ${field.name} (id ${field.id}) needs a non-null write-default`);
1723
- }
1724
- }
1725
- function validateSchemaEvolution(schemas, newSchema, priorLastColumnId, formatVersion) {
1726
- for (const field of newSchema.fields) {
1727
- if (field.id > priorLastColumnId) continue;
1728
- const prior = latestFieldById(schemas, field.id);
1729
- if (!prior) continue;
1730
- if (!canPromoteType(prior.type, field.type, formatVersion)) throw new Error(`add-schema: cannot promote field ${field.name} from ${typeToString(prior.type)} to ${typeToString(field.type)}`);
1731
- if (!defaultsEqual(prior["initial-default"], field["initial-default"])) throw new Error(`add-schema: initial-default for field ${field.name} cannot change`);
1732
- }
1733
- }
1734
- function latestFieldById(schemas, id) {
1735
- for (let i = schemas.length - 1; i >= 0; i--) {
1736
- const field = schemas[i].fields.find((f) => f.id === id);
1737
- if (field) return field;
1738
- }
1739
- }
1740
- function canPromoteType(from, to, formatVersion) {
1741
- if (typesEqual(from, to)) return true;
1742
- if (typeof from !== "string" || typeof to !== "string") return false;
1743
- if (formatVersion >= 3 && from === "unknown") return true;
1744
- if (from === "int" && to === "long") return true;
1745
- if (from === "float" && to === "double") return true;
1746
- if (formatVersion >= 3 && from === "date" && (to === "timestamp" || to === "timestamp_ns")) return true;
1747
- return decimalPromotionAllowed(from, to);
1748
- }
1749
- function typesEqual(a, b) {
1750
- if (typeof a === "string" || typeof b === "string") return a === b;
1751
- return JSON.stringify(a) === JSON.stringify(b);
1752
- }
1753
- function decimalPromotionAllowed(from, to) {
1754
- const a = parseDecimalType(from);
1755
- const b = parseDecimalType(to);
1756
- return Boolean(a && b && b.precision > a.precision && b.scale === a.scale);
1757
- }
1758
- function typeToString(type) {
1759
- return typeof type === "string" ? type : JSON.stringify(type);
1760
- }
1761
- function defaultsEqual(a, b) {
1762
- if (Object.is(a, b)) return true;
1763
- if (!a || !b || typeof a !== "object" || typeof b !== "object") return false;
1764
- if (Array.isArray(a) || Array.isArray(b)) {
1765
- if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length) return false;
1766
- for (let i = 0; i < a.length; i++) if (!defaultsEqual(a[i], b[i])) return false;
1767
- return true;
1768
- }
1769
- const aKeys = Object.keys(a);
1770
- const bKeys = Object.keys(b);
1771
- if (aKeys.length !== bKeys.length) return false;
1772
- for (const key of aKeys) {
1773
- if (!Object.hasOwn(b, key)) return false;
1774
- if (!defaultsEqual(a[key], b[key])) return false;
1775
- }
1776
- return true;
1777
- }
1778
- function validatePartitionSpecEvolution(specs, newSpec, schema) {
1779
- validateWritablePartitionSpec(newSpec, schema);
1780
- if (specs.some((spec) => partitionSpecsEquivalent(spec, newSpec))) throw new Error("add-spec: equivalent partition spec already exists");
1781
- for (const field of newSpec.fields) {
1782
- const equivalent = equivalentPartitionField(specs, field);
1783
- if (equivalent && equivalent["field-id"] !== field["field-id"]) throw new Error(`add-spec: partition field ${field.name} must reuse field-id ${equivalent["field-id"]}`);
1784
- }
1785
- }
1786
- function validateWritablePartitionSpec(spec, schema) {
1787
- try {
1788
- validatePartitionSpecForWrite(schema, spec, "add-spec");
1789
- } catch (err) {
1790
- const message = err instanceof Error ? err.message : String(err);
1791
- if (message.startsWith("unsupported partition transform: ")) throw new Error(`add-spec: ${message}`);
1792
- throw err;
1793
- }
1794
- }
1795
- function currentSchemaForMetadata(metadata) {
1796
- const schema = metadata.schemas?.find((s) => s["schema-id"] === metadata["current-schema-id"]);
1797
- if (!schema) throw new Error("add-spec: current schema not found in metadata");
1798
- return schema;
1799
- }
1800
- function equivalentPartitionField(specs, field) {
1801
- for (const spec of specs) {
1802
- const found = spec.fields.find((existing) => partitionFieldsEquivalent(existing, field));
1803
- if (found) return found;
1804
- }
1805
- }
1806
- function partitionSpecsEquivalent(a, b) {
1807
- if (a.fields.length !== b.fields.length) return false;
1808
- for (let i = 0; i < a.fields.length; i++) if (!partitionFieldsEquivalent(a.fields[i], b.fields[i])) return false;
1809
- return true;
1810
- }
1811
- function partitionFieldsEquivalent(a, b) {
1812
- return a["source-id"] === b["source-id"] && idsListEquivalent(a["source-ids"], b["source-ids"]) && a.transform === b.transform && a.name === b.name;
1813
- }
1814
- function idsListEquivalent(a, b) {
1815
- if (a === void 0 || b === void 0) return a === b;
1816
- if (a.length !== b.length) return false;
1817
- for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
1818
- return true;
1819
- }
1820
- function avroWrite({ writer, schema, records, blockSize = 512, metadata }) {
1821
- writer.appendUint32(23749199);
1822
- const meta = {
1823
- ...metadata,
1824
- "avro.schema": typeof schema === "string" ? schema : JSON.stringify(schema),
1825
- "avro.codec": "null"
1826
- };
1827
- appendZigZag(writer, Object.keys(meta).length);
1828
- for (const [key, value] of Object.entries(meta)) {
1829
- const kb = new TextEncoder().encode(key);
1830
- appendZigZag(writer, kb.length);
1831
- writer.appendBytes(kb);
1832
- const vb = new TextEncoder().encode(value);
1833
- appendZigZag(writer, vb.length);
1834
- writer.appendBytes(vb);
1835
- }
1836
- writer.appendVarInt(0);
1837
- const sync = /* @__PURE__ */ new Uint8Array(16);
1838
- for (let i = 0; i < 16; i++) sync[i] = Math.random() * 256 | 0;
1839
- writer.appendBytes(sync);
1840
- for (let i = 0; i < records.length; i += blockSize) {
1841
- const block = records.slice(i, i + blockSize);
1842
- appendZigZag(writer, block.length);
1843
- const blockWriter = new ByteWriter();
1844
- for (const record of block) for (const { name, type } of schema.fields) writeType(blockWriter, type, record[name]);
1845
- appendZigZag(writer, blockWriter.offset);
1846
- writer.appendBytes(blockWriter.getBytes());
1847
- writer.appendBytes(sync);
1848
- }
1849
- return writer.finish();
1850
- }
1851
- function writeType(writer, schema, value) {
1852
- if (Array.isArray(schema)) {
1853
- const unionIndex = schema.findIndex((s) => {
1854
- if (Array.isArray(s)) throw new Error("nested unions not supported");
1855
- const tag = typeof s === "string" ? s : s.type === "record" || s.type === "array" || s.type === "fixed" ? s.type : s.logicalType;
1856
- if (value == null) return tag === "null";
1857
- if (tag === "boolean") return typeof value === "boolean";
1858
- if (tag === "int") return typeof value === "number" && Number.isInteger(value);
1859
- if (tag === "long") return typeof value === "bigint" || typeof value === "number";
1860
- if (tag === "float" || tag === "double") return typeof value === "number";
1861
- if (tag === "string") return typeof value === "string";
1862
- if (tag === "bytes") return value instanceof Uint8Array;
1863
- if (tag === "date") return value instanceof Date || typeof value === "number";
1864
- if (tag === "time-millis") return typeof value === "number";
1865
- if (tag === "time-micros") return typeof value === "bigint" || typeof value === "number";
1866
- if (tag === "timestamp-millis" || tag === "timestamp-micros" || tag === "timestamp-nanos") return value instanceof Date || typeof value === "bigint" || typeof value === "number";
1867
- if (tag === "decimal") return typeof value === "number" || typeof value === "bigint";
1868
- if (tag === "record") return typeof value === "object" && value !== null;
1869
- if (tag === "array") return Array.isArray(value);
1870
- if (tag === "fixed") {
1871
- if (value instanceof Uint8Array) return true;
1872
- return typeof s === "object" && "logicalType" in s && s.logicalType === "uuid" && typeof value === "string";
1873
- }
1874
- return false;
1875
- });
1876
- if (unionIndex === -1) throw new Error("union branch not found");
1877
- appendZigZag(writer, unionIndex);
1878
- writeType(writer, schema[unionIndex], value);
1879
- } else if (typeof schema === "string") {
1880
- if (schema === "null") {} else if (schema === "boolean") writer.appendUint8(value ? 1 : 0);
1881
- else if (schema === "int") {
1882
- if (typeof value !== "number" || !Number.isInteger(value)) throw new Error("expected integer value");
1883
- appendZigZag(writer, value);
1884
- } else if (schema === "long") {
1885
- if (typeof value !== "bigint") throw new Error("expected bigint value");
1886
- appendZigZag64(writer, value);
1887
- } else if (schema === "float") {
1888
- if (typeof value !== "number") throw new Error("expected number value");
1889
- writer.appendFloat32(value);
1890
- } else if (schema === "double") {
1891
- if (typeof value !== "number") throw new Error("expected number value");
1892
- writer.appendFloat64(value);
1893
- } else if (schema === "bytes") {
1894
- if (!(value instanceof Uint8Array)) throw new Error("expected Uint8Array value");
1895
- appendZigZag(writer, value.length);
1896
- writer.appendBytes(value);
1897
- } else if (schema === "string") {
1898
- if (typeof value !== "string") throw new Error("expected string value");
1899
- const b = new TextEncoder().encode(value);
1900
- appendZigZag(writer, b.length);
1901
- writer.appendBytes(b);
1902
- }
1903
- } else if (schema.type === "record") for (const f of schema.fields) writeType(writer, f.type, value[f.name]);
1904
- else if (schema.type === "array") {
1905
- if (value.length) {
1906
- appendZigZag(writer, value.length);
1907
- for (const it of value) writeType(writer, schema.items, it);
1908
- }
1909
- writer.appendVarInt(0);
1910
- } else if (schema.type === "fixed") {
1911
- const bytes = schema.logicalType === "uuid" && typeof value === "string" ? uuidStringToBytes$1(value) : value;
1912
- if (!(bytes instanceof Uint8Array)) throw new Error("expected Uint8Array value");
1913
- if (bytes.length !== schema.size) throw new Error(`expected fixed[${schema.size}] value`);
1914
- writer.appendBytes(bytes);
1915
- } else if ("logicalType" in schema) if (schema.logicalType === "date") appendZigZag(writer, value instanceof Date ? Math.floor(value.getTime() / 864e5) : value);
1916
- else if (schema.logicalType === "time-millis") appendZigZag(writer, value);
1917
- else if (schema.logicalType === "time-micros") appendZigZag64(writer, BigInt(value));
1918
- else if (schema.logicalType === "timestamp-millis") appendZigZag64(writer, value instanceof Date ? BigInt(value.getTime()) : BigInt(value));
1919
- else if (schema.logicalType === "timestamp-micros") appendZigZag64(writer, value instanceof Date ? BigInt(value.getTime()) * 1000n : BigInt(value));
1920
- else if (schema.logicalType === "timestamp-nanos") appendZigZag64(writer, value instanceof Date ? BigInt(value.getTime()) * 1000000n : BigInt(value));
1921
- else if (schema.logicalType === "decimal") {
1922
- const scale = "scale" in schema ? schema.scale ?? 0 : 0;
1923
- let u;
1924
- if (typeof value === "bigint") u = value;
1925
- else if (typeof value === "number") u = BigInt(Math.round(value * 10 ** scale));
1926
- else throw new Error("decimal value must be bigint or number");
1927
- const b = bigIntToBytes(u);
1928
- appendZigZag(writer, b.length);
1929
- writer.appendBytes(b);
1930
- } else throw new Error(`unknown logical type ${schema.logicalType}`);
1931
- else throw new Error(`unknown schema type ${JSON.stringify(schema)}`);
1932
- }
1933
- function appendZigZag(writer, v) {
1934
- writer.appendVarInt(v << 1 ^ v >> 31);
1935
- }
1936
- function appendZigZag64(writer, v) {
1937
- writer.appendVarBigInt(v << 1n ^ v >> 63n);
1938
- }
1939
- function uuidStringToBytes$1(value) {
1940
- const hex = value.toLowerCase().replace(/-/g, "");
1941
- if (!/^[0-9a-f]{32}$/.test(hex)) throw new Error("expected uuid string");
1942
- const bytes = /* @__PURE__ */ new Uint8Array(16);
1943
- for (let i = 0; i < 16; i++) bytes[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16);
1944
- return bytes;
1945
- }
1946
- function bigIntToBytes(value) {
1947
- const neg = value < 0n;
1948
- let abs = neg ? -value : value;
1949
- const out = [];
1950
- while (abs > 0n) {
1951
- out.unshift(Number(abs & 255n));
1952
- abs >>= 8n;
1953
- }
1954
- if (out.length === 0) out.push(0);
1955
- if (neg) {
1956
- for (let i = 0; i < out.length; i++) out[i] ^= 255;
1957
- for (let i = out.length - 1; i >= 0; i--) {
1958
- out[i] = out[i] + 1 & 255;
1959
- if (out[i]) break;
1960
- }
1961
- if ((out[0] & 128) === 0) out.unshift(255);
1962
- } else if ((out[0] & 128) !== 0) out.unshift(0);
1963
- return Uint8Array.from(out);
1964
- }
1965
- function manifestEntrySchema(schema, partitionSpec, formatVersion, manifestContent = 0) {
1966
- const dataFileFields = [
1967
- {
1968
- name: "content",
1969
- type: "int",
1970
- "field-id": 134
1971
- },
1972
- {
1973
- name: "file_path",
1974
- type: "string",
1975
- "field-id": 100
1976
- },
1977
- {
1978
- name: "file_format",
1979
- type: "string",
1980
- "field-id": 101
1981
- },
1982
- {
1983
- name: "partition",
1984
- "field-id": 102,
1985
- type: partitionAvroSchema(schema, partitionSpec)
1986
- },
1987
- {
1988
- name: "record_count",
1989
- type: "long",
1990
- "field-id": 103
1991
- },
1992
- {
1993
- name: "file_size_in_bytes",
1994
- type: "long",
1995
- "field-id": 104
1996
- },
1997
- mapField("column_sizes", 108, "k117_v118", 117, 118, "long"),
1998
- mapField("value_counts", 109, "k119_v120", 119, 120, "long"),
1999
- mapField("null_value_counts", 110, "k121_v122", 121, 122, "long"),
2000
- mapField("nan_value_counts", 137, "k138_v139", 138, 139, "long"),
2001
- mapField("lower_bounds", 125, "k126_v127", 126, 127, "bytes"),
2002
- mapField("upper_bounds", 128, "k129_v130", 129, 130, "bytes"),
2003
- {
2004
- name: "sort_order_id",
2005
- type: ["null", "int"],
2006
- default: null,
2007
- "field-id": 140
2008
- }
2009
- ];
2010
- if (manifestContent === 1) {
2011
- dataFileFields.push({
2012
- name: "equality_ids",
2013
- "field-id": 135,
2014
- default: null,
2015
- type: ["null", {
2016
- type: "array",
2017
- items: "int",
2018
- "element-id": 136
2019
- }]
2020
- });
2021
- dataFileFields.push({
2022
- name: "referenced_data_file",
2023
- type: ["null", "string"],
2024
- default: null,
2025
- "field-id": 143
2026
- });
2027
- if (formatVersion >= 3) {
2028
- dataFileFields.push({
2029
- name: "content_offset",
2030
- type: ["null", "long"],
2031
- default: null,
2032
- "field-id": 144
2033
- });
2034
- dataFileFields.push({
2035
- name: "content_size_in_bytes",
2036
- type: ["null", "long"],
2037
- default: null,
2038
- "field-id": 145
2039
- });
2040
- }
2041
- }
2042
- if (formatVersion >= 3) dataFileFields.push({
2043
- name: "first_row_id",
2044
- type: ["null", "long"],
2045
- default: null,
2046
- "field-id": 142
2047
- });
2048
- return {
2049
- type: "record",
2050
- name: "manifest_entry",
2051
- fields: [
2052
- {
2053
- name: "status",
2054
- type: "int",
2055
- "field-id": 0
2056
- },
2057
- {
2058
- name: "snapshot_id",
2059
- type: ["null", "long"],
2060
- default: null,
2061
- "field-id": 1
2062
- },
2063
- {
2064
- name: "sequence_number",
2065
- type: ["null", "long"],
2066
- default: null,
2067
- "field-id": 3
2068
- },
2069
- {
2070
- name: "file_sequence_number",
2071
- type: ["null", "long"],
2072
- default: null,
2073
- "field-id": 4
2074
- },
2075
- {
2076
- name: "data_file",
2077
- "field-id": 2,
2078
- type: {
2079
- type: "record",
2080
- name: "r2",
2081
- fields: dataFileFields
2082
- }
2083
- }
2084
- ]
2085
- };
2086
- }
2087
- function mapField(name, fieldId, recName, keyId, valueId, valueType) {
2088
- return {
2089
- name,
2090
- "field-id": fieldId,
2091
- default: null,
2092
- type: ["null", {
2093
- type: "array",
2094
- logicalType: "map",
2095
- items: {
2096
- type: "record",
2097
- name: recName,
2098
- fields: [{
2099
- name: "key",
2100
- type: "int",
2101
- "field-id": keyId
2102
- }, {
2103
- name: "value",
2104
- type: valueType,
2105
- "field-id": valueId
2106
- }]
2107
- }
2108
- }]
2109
- };
2110
- }
2111
- function icebergSchemaJson(schema) {
2112
- return JSON.stringify(schema);
2113
- }
2114
- function encodeMap(m) {
2115
- if (!m) return null;
2116
- const entries = Object.entries(m);
2117
- if (!entries.length) return null;
2118
- return entries.map(([k, value]) => ({
2119
- key: Number(k),
2120
- value
2121
- }));
2122
- }
2123
- function writeDataManifest({ writer, schema, partitionSpec, snapshotId, dataFiles, formatVersion = 2 }) {
2124
- const records = dataFiles.map((dataFile) => {
2125
- if (dataFile.content !== 0) throw new Error(`writeDataManifest expects data files (content=0), got content=${dataFile.content}`);
2126
- return manifestEntryRecord(dataFile, schema, partitionSpec, snapshotId, formatVersion, 0);
2127
- });
2128
- return avroWrite({
2129
- writer,
2130
- schema: manifestEntrySchema(schema, partitionSpec, formatVersion, 0),
2131
- records,
2132
- metadata: {
2133
- "format-version": String(formatVersion),
2134
- content: "data",
2135
- schema: icebergSchemaJson(schema),
2136
- "partition-spec": partitionSpecJson(partitionSpec),
2137
- "partition-spec-id": String(partitionSpec["spec-id"])
2138
- }
2139
- });
2140
- }
2141
- function manifestEntryRecord(dataFile, schema, partitionSpec, snapshotId, formatVersion, manifestContent) {
2142
- const dataFileRecord = {
2143
- content: dataFile.content,
2144
- file_path: dataFile.file_path,
2145
- file_format: dataFile.file_format.toUpperCase(),
2146
- partition: partitionToAvroRecord(dataFile.partition ?? {}, schema, partitionSpec),
2147
- record_count: dataFile.record_count,
2148
- file_size_in_bytes: dataFile.file_size_in_bytes,
2149
- column_sizes: encodeMap(dataFile.column_sizes),
2150
- value_counts: encodeMap(dataFile.value_counts),
2151
- null_value_counts: encodeMap(dataFile.null_value_counts),
2152
- nan_value_counts: encodeMap(dataFile.nan_value_counts),
2153
- lower_bounds: encodeMap(dataFile.lower_bounds),
2154
- upper_bounds: encodeMap(dataFile.upper_bounds),
2155
- sort_order_id: dataFile.content === 1 ? null : dataFile.sort_order_id ?? 0
2156
- };
2157
- if (manifestContent === 1) {
2158
- dataFileRecord.equality_ids = dataFile.equality_ids?.length ? dataFile.equality_ids : null;
2159
- dataFileRecord.referenced_data_file = dataFile.referenced_data_file ?? null;
2160
- if (formatVersion >= 3) {
2161
- dataFileRecord.content_offset = dataFile.content_offset ?? null;
2162
- dataFileRecord.content_size_in_bytes = dataFile.content_size_in_bytes ?? null;
2163
- }
2164
- }
2165
- if (formatVersion >= 3) dataFileRecord.first_row_id = dataFile.first_row_id ?? null;
2166
- return {
2167
- status: 1,
2168
- snapshot_id: snapshotId,
2169
- sequence_number: null,
2170
- file_sequence_number: null,
2171
- data_file: dataFileRecord
2172
- };
2173
- }
2174
- function manifestFileSchema(formatVersion) {
2175
- const fields = [
2176
- {
2177
- name: "manifest_path",
2178
- type: "string",
2179
- "field-id": 500
2180
- },
2181
- {
2182
- name: "manifest_length",
2183
- type: "long",
2184
- "field-id": 501
2185
- },
2186
- {
2187
- name: "partition_spec_id",
2188
- type: "int",
2189
- "field-id": 502
2190
- },
2191
- {
2192
- name: "content",
2193
- type: "int",
2194
- "field-id": 517
2195
- },
2196
- {
2197
- name: "sequence_number",
2198
- type: "long",
2199
- "field-id": 515
2200
- },
2201
- {
2202
- name: "min_sequence_number",
2203
- type: "long",
2204
- "field-id": 516
2205
- },
2206
- {
2207
- name: "added_snapshot_id",
2208
- type: "long",
2209
- "field-id": 503
2210
- },
2211
- {
2212
- name: "added_files_count",
2213
- type: "int",
2214
- "field-id": 504
2215
- },
2216
- {
2217
- name: "existing_files_count",
2218
- type: "int",
2219
- "field-id": 505
2220
- },
2221
- {
2222
- name: "deleted_files_count",
2223
- type: "int",
2224
- "field-id": 506
2225
- },
2226
- {
2227
- name: "added_rows_count",
2228
- type: "long",
2229
- "field-id": 512
2230
- },
2231
- {
2232
- name: "existing_rows_count",
2233
- type: "long",
2234
- "field-id": 513
2235
- },
2236
- {
2237
- name: "deleted_rows_count",
2238
- type: "long",
2239
- "field-id": 514
2240
- },
2241
- {
2242
- name: "partitions",
2243
- type: ["null", {
2244
- type: "array",
2245
- "element-id": 508,
2246
- items: {
2247
- type: "record",
2248
- name: "r508",
2249
- fields: [
2250
- {
2251
- name: "contains_null",
2252
- type: "boolean",
2253
- "field-id": 509
2254
- },
2255
- {
2256
- name: "contains_nan",
2257
- type: ["null", "boolean"],
2258
- default: null,
2259
- "field-id": 518
2260
- },
2261
- {
2262
- name: "lower_bound",
2263
- type: ["null", "bytes"],
2264
- default: null,
2265
- "field-id": 510
2266
- },
2267
- {
2268
- name: "upper_bound",
2269
- type: ["null", "bytes"],
2270
- default: null,
2271
- "field-id": 511
2272
- }
2273
- ]
2274
- }
2275
- }],
2276
- default: null,
2277
- "field-id": 507
2278
- }
2279
- ];
2280
- if (formatVersion >= 3) fields.push({
2281
- name: "first_row_id",
2282
- type: ["null", "long"],
2283
- default: null,
2284
- "field-id": 520
2285
- });
2286
- return {
2287
- type: "record",
2288
- name: "manifest_file",
2289
- fields
2290
- };
2291
- }
2292
- function writeManifestList({ writer, snapshotId, sequenceNumber, manifests, formatVersion = 2 }) {
2293
- const records = manifests.map((m) => {
2294
- const record = {
2295
- manifest_path: m.manifest_path,
2296
- manifest_length: m.manifest_length,
2297
- partition_spec_id: m.partition_spec_id,
2298
- content: m.content,
2299
- sequence_number: m.sequence_number ?? sequenceNumber,
2300
- min_sequence_number: m.min_sequence_number ?? sequenceNumber,
2301
- added_snapshot_id: m.added_snapshot_id,
2302
- added_files_count: m.added_files_count,
2303
- existing_files_count: m.existing_files_count,
2304
- deleted_files_count: m.deleted_files_count,
2305
- added_rows_count: m.added_rows_count,
2306
- existing_rows_count: m.existing_rows_count,
2307
- deleted_rows_count: m.deleted_rows_count,
2308
- partitions: m.partitions ?? null
2309
- };
2310
- if (formatVersion >= 3) record.first_row_id = m.content === 0 ? m.first_row_id ?? null : null;
2311
- return record;
2312
- });
2313
- return avroWrite({
2314
- writer,
2315
- schema: manifestFileSchema(formatVersion),
2316
- records,
2317
- metadata: {
2318
- "format-version": String(formatVersion),
2319
- "snapshot-id": String(snapshotId),
2320
- "sequence-number": String(sequenceNumber)
2321
- }
2322
- });
2323
- }
2324
- function isGeoType(name) {
2325
- return name.startsWith("geometry") || name.startsWith("geography");
2326
- }
2327
- function computeGeoBounds(records, field) {
2328
- let partial;
2329
- let nulls = 0n;
2330
- const writeDefault = field["write-default"];
2331
- for (const record of records) {
2332
- let v = record[field.name];
2333
- if (v === void 0 && writeDefault !== void 0) v = writeDefault;
2334
- if (v === null || v === void 0) {
2335
- nulls++;
2336
- continue;
2337
- }
2338
- if (typeof v !== "object") throw new Error("geospatial column expects GeoJSON geometries");
2339
- partial = extendBoundsFromGeometry(partial, v);
2340
- }
2341
- const result = {
2342
- value_count: BigInt(records.length),
2343
- null_count: nulls
2344
- };
2345
- const { xmin, ymin, xmax, ymax, zmin, zmax, mmin, mmax } = partial ?? {};
2346
- if (xmin === void 0 || ymin === void 0 || xmax === void 0 || ymax === void 0) return result;
2347
- const hasZ = zmin !== void 0;
2348
- const hasM = mmin !== void 0;
2349
- return {
2350
- ...result,
2351
- lower: encodeGeoPoint(xmin, ymin, zmin, mmin, hasZ, hasM),
2352
- upper: encodeGeoPoint(xmax, ymax, zmax, mmax, hasZ, hasM)
2353
- };
2354
- }
2355
- function extendBoundsFromGeometry(bbox, geometry) {
2356
- if (geometry.type === "GeometryCollection") {
2357
- for (const child of geometry.geometries || []) bbox = extendBoundsFromGeometry(bbox, child);
2358
- return bbox;
2359
- }
2360
- return extendBoundsFromCoordinates(bbox, geometry.coordinates);
2361
- }
2362
- function extendBoundsFromCoordinates(bbox, coordinates) {
2363
- if (typeof coordinates[0] === "number") {
2364
- bbox = updateAxis(bbox, "xmin", "xmax", coordinates[0]);
2365
- bbox = updateAxis(bbox, "ymin", "ymax", coordinates[1]);
2366
- if (coordinates.length > 2) bbox = updateAxis(bbox, "zmin", "zmax", coordinates[2]);
2367
- if (coordinates.length > 3) bbox = updateAxis(bbox, "mmin", "mmax", coordinates[3]);
2368
- return bbox;
2369
- }
2370
- for (const child of coordinates) bbox = extendBoundsFromCoordinates(bbox, child);
2371
- return bbox;
2372
- }
2373
- function updateAxis(bbox, minKey, maxKey, value) {
2374
- if (value === void 0 || !Number.isFinite(value)) return bbox;
2375
- if (!bbox) bbox = {};
2376
- const min = bbox[minKey];
2377
- const max = bbox[maxKey];
2378
- if (min === void 0 || value < min) bbox[minKey] = value;
2379
- if (max === void 0 || value > max) bbox[maxKey] = value;
2380
- return bbox;
2381
- }
2382
- function encodeGeoPoint(x, y, z, m, hasZ, hasM) {
2383
- const len = !hasZ && !hasM ? 16 : hasZ && !hasM ? 24 : 32;
2384
- const buf = new ArrayBuffer(len);
2385
- const view = new DataView(buf);
2386
- view.setFloat64(0, x, true);
2387
- view.setFloat64(8, y, true);
2388
- if (len === 24) view.setFloat64(16, z, true);
2389
- else if (len === 32) {
2390
- view.setFloat64(16, hasZ ? z : NaN, true);
2391
- view.setFloat64(24, m, true);
2392
- }
2393
- return new Uint8Array(buf);
2394
- }
2395
- function serializeValue(value, type) {
2396
- const name = typeName(type);
2397
- if (name.startsWith("decimal(")) {
2398
- const m = /^decimal\((\d+),\s*(\d+)\)$/.exec(name);
2399
- if (!m) return void 0;
2400
- const scale = parseInt(m[2], 10);
2401
- if (typeof value !== "number" && typeof value !== "bigint") return void 0;
2402
- const factor = 10n ** BigInt(scale);
2403
- return twosComplementMinBigEndian(typeof value === "bigint" ? value * factor : BigInt(Math.round(value * Number(factor))));
2404
- }
2405
- if (name.startsWith("fixed[")) return value instanceof Uint8Array ? value : void 0;
2406
- switch (name) {
2407
- case "boolean": return new Uint8Array([value ? 1 : 0]);
2408
- case "int": {
2409
- const buf = /* @__PURE__ */ new ArrayBuffer(4);
2410
- new DataView(buf).setInt32(0, value, true);
2411
- return new Uint8Array(buf);
2412
- }
2413
- case "long": {
2414
- const buf = /* @__PURE__ */ new ArrayBuffer(8);
2415
- new DataView(buf).setBigInt64(0, typeof value === "bigint" ? value : BigInt(value), true);
2416
- return new Uint8Array(buf);
2417
- }
2418
- case "float": {
2419
- const buf = /* @__PURE__ */ new ArrayBuffer(4);
2420
- new DataView(buf).setFloat32(0, value, true);
2421
- return new Uint8Array(buf);
2422
- }
2423
- case "double": {
2424
- const buf = /* @__PURE__ */ new ArrayBuffer(8);
2425
- new DataView(buf).setFloat64(0, value, true);
2426
- return new Uint8Array(buf);
2427
- }
2428
- case "date": {
2429
- const days = value instanceof Date ? Math.floor(value.getTime() / 864e5) : Number(value);
2430
- const buf = /* @__PURE__ */ new ArrayBuffer(4);
2431
- new DataView(buf).setInt32(0, days, true);
2432
- return new Uint8Array(buf);
2433
- }
2434
- case "time": {
2435
- const buf = /* @__PURE__ */ new ArrayBuffer(8);
2436
- new DataView(buf).setBigInt64(0, typeof value === "bigint" ? value : BigInt(value), true);
2437
- return new Uint8Array(buf);
2438
- }
2439
- case "timestamp":
2440
- case "timestamptz": {
2441
- const buf = /* @__PURE__ */ new ArrayBuffer(8);
2442
- new DataView(buf).setBigInt64(0, timestampToMicros(value), true);
2443
- return new Uint8Array(buf);
2444
- }
2445
- case "timestamp_ns":
2446
- case "timestamptz_ns": {
2447
- const buf = /* @__PURE__ */ new ArrayBuffer(8);
2448
- new DataView(buf).setBigInt64(0, timestampToNanos(value), true);
2449
- return new Uint8Array(buf);
2450
- }
2451
- case "string": return new TextEncoder().encode(value);
2452
- case "binary": return value instanceof Uint8Array ? value : void 0;
2453
- case "uuid":
2454
- if (value instanceof Uint8Array && value.length === 16) return value;
2455
- if (typeof value === "string") return uuidStringToBytes(value);
2456
- return;
2457
- default: return;
2458
- }
2459
- }
2460
- function compare(a, b, type) {
2461
- switch (typeName(type)) {
2462
- case "boolean": return (a ? 1 : 0) - (b ? 1 : 0);
2463
- case "int": return a < b ? -1 : a > b ? 1 : 0;
2464
- case "float":
2465
- case "double": return compareFloating(a, b);
2466
- case "long": {
2467
- const ai = typeof a === "bigint" ? a : BigInt(a);
2468
- const bi = typeof b === "bigint" ? b : BigInt(b);
2469
- return ai < bi ? -1 : ai > bi ? 1 : 0;
2470
- }
2471
- case "date": {
2472
- const ad = dateToDays(a);
2473
- const bd = dateToDays(b);
2474
- if (Number.isNaN(ad) || Number.isNaN(bd)) return NaN;
2475
- return ad < bd ? -1 : ad > bd ? 1 : 0;
2476
- }
2477
- case "timestamp":
2478
- case "timestamptz": return compareBigInt(timestampToMicros(a), timestampToMicros(b));
2479
- case "timestamp_ns":
2480
- case "timestamptz_ns": return compareBigInt(timestampToNanos(a), timestampToNanos(b));
2481
- case "string": return a < b ? -1 : a > b ? 1 : 0;
2482
- case "binary":
2483
- case "uuid": return compareBytes(a, b);
2484
- default:
2485
- if (typeName(type).startsWith("fixed[")) return compareBytes(a, b);
2486
- return a < b ? -1 : a > b ? 1 : 0;
2487
- }
2488
- }
2489
- function compareFloating(a, b) {
2490
- if (Object.is(a, b)) return 0;
2491
- if (a === 0 && b === 0) return Object.is(a, -0) ? -1 : 1;
2492
- return a < b ? -1 : a > b ? 1 : 0;
2493
- }
2494
- function compareBigInt(a, b) {
2495
- return a < b ? -1 : a > b ? 1 : 0;
2496
- }
2497
- function dateToDays(value) {
2498
- if (value instanceof Date) return Math.floor(value.getTime() / 864e5);
2499
- if (typeof value === "bigint") return Number(value);
2500
- if (typeof value === "number") return value;
2501
- if (typeof value === "string") {
2502
- const ms = Date.parse(value);
2503
- return Number.isNaN(ms) ? NaN : Math.floor(ms / 864e5);
2504
- }
2505
- return NaN;
2506
- }
2507
- function timestampToMicros(value) {
2508
- return typeof value === "bigint" ? value : value instanceof Date ? BigInt(value.getTime()) * 1000n : BigInt(value);
2509
- }
2510
- function timestampToNanos(value) {
2511
- return typeof value === "bigint" ? value : value instanceof Date ? BigInt(value.getTime()) * 1000000n : BigInt(value);
2512
- }
2513
- function compareBytes(a, b) {
2514
- const len = Math.min(a.length, b.length);
2515
- for (let i = 0; i < len; i++) if (a[i] !== b[i]) return a[i] - b[i];
2516
- return a.length - b.length;
2517
- }
2518
- function twosComplementMinBigEndian(value) {
2519
- const bytes = [];
2520
- let v = value;
2521
- while (true) {
2522
- const byte = Number(v & 255n);
2523
- bytes.unshift(byte);
2524
- v >>= 8n;
2525
- const sign = byte & 128;
2526
- if (!sign && v === 0n || sign && v === -1n) break;
2527
- }
2528
- return new Uint8Array(bytes);
2529
- }
2530
- function uuidStringToBytes(s) {
2531
- const hex = s.replace(/-/g, "");
2532
- if (hex.length !== 32) return void 0;
2533
- const out = /* @__PURE__ */ new Uint8Array(16);
2534
- for (let i = 0; i < 16; i++) {
2535
- const byte = parseInt(hex.slice(i * 2, i * 2 + 2), 16);
2536
- if (Number.isNaN(byte)) return void 0;
2537
- out[i] = byte;
2538
- }
2539
- return out;
2540
- }
2541
- const TRUNCATE_LIMIT = 16;
2542
- function computeColumnStats(records, schema) {
2543
- const value_counts = {};
2544
- const null_value_counts = {};
2545
- const nan_value_counts = {};
2546
- const lower_bounds = {};
2547
- const upper_bounds = {};
2548
- for (const field of schema.fields) {
2549
- const type = typeName(field.type);
2550
- if (type === "unknown") continue;
2551
- if (type === "list" || type === "map" || type === "struct") continue;
2552
- if (isGeoType(type)) {
2553
- const { value_count, null_count, lower, upper } = computeGeoBounds(records, field);
2554
- value_counts[field.id] = value_count;
2555
- null_value_counts[field.id] = null_count;
2556
- if (lower) lower_bounds[field.id] = lower;
2557
- if (upper) upper_bounds[field.id] = upper;
2558
- continue;
2559
- }
2560
- let nulls = 0n;
2561
- let nans = 0n;
2562
- let min;
2563
- let max;
2564
- const isFloat = type === "float" || type === "double";
2565
- const trackBounds = hasComparableBounds(field.type);
2566
- const writeDefault = field["write-default"];
2567
- for (const record of records) {
2568
- let v = record[field.name];
2569
- if (v === void 0 && writeDefault !== void 0) v = writeDefault;
2570
- if (v === null || v === void 0) {
2571
- nulls++;
2572
- continue;
2573
- }
2574
- if (isFloat && Number.isNaN(v)) {
2575
- nans++;
2576
- continue;
2577
- }
2578
- if (trackBounds) {
2579
- if (min === void 0 || compare(v, min, field.type) < 0) min = v;
2580
- if (max === void 0 || compare(v, max, field.type) > 0) max = v;
2581
- }
2582
- }
2583
- value_counts[field.id] = BigInt(records.length);
2584
- null_value_counts[field.id] = nulls;
2585
- if (isFloat) nan_value_counts[field.id] = nans;
2586
- if (min !== void 0) {
2587
- const lo = serializeValue(truncateLower(min, field.type), field.type);
2588
- if (lo) lower_bounds[field.id] = lo;
2589
- }
2590
- if (max !== void 0) {
2591
- const truncated = truncateUpper(max, field.type);
2592
- if (truncated !== void 0) {
2593
- const hi = serializeValue(truncated, field.type);
2594
- if (hi) upper_bounds[field.id] = hi;
2595
- }
2596
- }
2597
- }
2598
- return {
2599
- value_counts,
2600
- null_value_counts,
2601
- nan_value_counts,
2602
- lower_bounds,
2603
- upper_bounds
2604
- };
2605
- }
2606
- function hasComparableBounds(type) {
2607
- const name = typeName(type);
2608
- if (isGeoType(name)) return false;
2609
- return name !== "unknown" && name !== "variant";
2610
- }
2611
- function computeFieldSummary(values, type) {
2612
- const name = typeName(type);
2613
- const isFloat = name === "float" || name === "double";
2614
- const trackBounds = hasComparableBounds(type);
2615
- let containsNull = false;
2616
- let containsNan = false;
2617
- let min;
2618
- let max;
2619
- for (const v of values) {
2620
- if (v === null || v === void 0) {
2621
- containsNull = true;
2622
- continue;
2623
- }
2624
- if (isFloat && Number.isNaN(v)) {
2625
- containsNan = true;
2626
- continue;
2627
- }
2628
- if (trackBounds) {
2629
- if (min === void 0 || compare(v, min, type) < 0) min = v;
2630
- if (max === void 0 || compare(v, max, type) > 0) max = v;
2631
- }
2632
- }
2633
- const summary = { contains_null: containsNull };
2634
- if (isFloat) summary.contains_nan = containsNan;
2635
- if (min !== void 0) {
2636
- const lo = serializeValue(truncateLower(min, type), type);
2637
- if (lo) summary.lower_bound = lo;
2638
- }
2639
- if (max !== void 0) {
2640
- const truncated = truncateUpper(max, type);
2641
- if (truncated !== void 0) {
2642
- const hi = serializeValue(truncated, type);
2643
- if (hi) summary.upper_bound = hi;
2644
- }
2645
- }
2646
- return summary;
2647
- }
2648
- function truncateLower(value, type) {
2649
- const name = typeName(type);
2650
- if (name === "string" && typeof value === "string") {
2651
- const cps = Array.from(value);
2652
- if (cps.length <= TRUNCATE_LIMIT) return value;
2653
- return cps.slice(0, TRUNCATE_LIMIT).join("");
2654
- }
2655
- if ((name === "binary" || name.startsWith("fixed[")) && value instanceof Uint8Array) {
2656
- if (value.length <= TRUNCATE_LIMIT) return value;
2657
- return value.slice(0, TRUNCATE_LIMIT);
2658
- }
2659
- return value;
2660
- }
2661
- function truncateUpper(value, type) {
2662
- const name = typeName(type);
2663
- if (name === "string" && typeof value === "string") {
2664
- const cps = Array.from(value);
2665
- if (cps.length <= TRUNCATE_LIMIT) return value;
2666
- const prefix = cps.slice(0, TRUNCATE_LIMIT);
2667
- while (prefix.length > 0) {
2668
- const cp = prefix[prefix.length - 1].codePointAt(0);
2669
- const next = cp + 1 === 55296 ? 57344 : cp + 1;
2670
- if (next <= 1114111) {
2671
- prefix[prefix.length - 1] = String.fromCodePoint(next);
2672
- return prefix.join("");
2673
- }
2674
- prefix.pop();
2675
- }
2676
- return;
2677
- }
2678
- if ((name === "binary" || name.startsWith("fixed[")) && value instanceof Uint8Array) {
2679
- if (value.length <= TRUNCATE_LIMIT) return value;
2680
- const prefix = value.slice(0, TRUNCATE_LIMIT);
2681
- for (let i = prefix.length - 1; i >= 0; i--) if (prefix[i] < 255) {
2682
- const out = prefix.slice(0, i + 1);
2683
- out[i]++;
2684
- return out;
2685
- }
2686
- return;
2687
- }
2688
- return value;
2689
- }
2690
- function currentSnapshot(metadata) {
2691
- const id = metadata["current-snapshot-id"];
2692
- if (id === void 0) return void 0;
2693
- return metadata.snapshots?.find((s) => s["snapshot-id"] === id);
2694
- }
2695
- async function loadPriorManifests(metadata, resolver) {
2696
- const snap = currentSnapshot(metadata);
2697
- if (!snap?.["manifest-list"]) return [];
2698
- return await fetchAvroRecords(snap["manifest-list"], resolver);
2699
- }
2700
- async function buildSnapshotUpdate({ tableUrl, metadata, resolver, snapshotId, sequenceNumber, manifestUuid, timestampMs, formatVersion, newManifests, summary, writtenFiles, priorManifests, skipPriorManifestPaths }) {
2701
- const writerFn = resolver.writer;
2702
- if (!writerFn) throw new Error("resolver.writer is required");
2703
- const rowLineage = formatVersion >= 3;
2704
- const firstRowId = rowLineage ? BigInt(metadata["next-row-id"] ?? 0) : 0n;
2705
- priorManifests ??= await loadPriorManifests(metadata, resolver);
2706
- if (skipPriorManifestPaths?.size) priorManifests = priorManifests.filter((manifest) => !skipPriorManifestPaths.has(manifest.manifest_path));
2707
- const allManifests = [...priorManifests, ...newManifests];
2708
- const addedRows = rowLineage ? assignFirstRowIds$1(allManifests, firstRowId) : 0n;
2709
- const manifestListPath = `${tableUrl}/metadata/snap-${snapshotId}-1-${manifestUuid}.avro`;
2710
- await writeManifestList({
2711
- writer: writerFn(manifestListPath),
2712
- snapshotId,
2713
- sequenceNumber,
2714
- manifests: allManifests,
2715
- formatVersion
2716
- });
2717
- const snapshot = {
2718
- "snapshot-id": Number(snapshotId),
2719
- "sequence-number": Number(sequenceNumber),
2720
- "timestamp-ms": timestampMs,
2721
- "manifest-list": manifestListPath,
2722
- summary,
2723
- "schema-id": metadata["current-schema-id"]
2724
- };
2725
- if (rowLineage) {
2726
- snapshot["first-row-id"] = toMetadataLong(firstRowId);
2727
- snapshot["added-rows"] = toMetadataLong(addedRows);
2728
- }
2729
- const rawCurrentSnapshotId = metadata["current-snapshot-id"];
2730
- const currentSnapshotId = rawCurrentSnapshotId === void 0 || rawCurrentSnapshotId === null || rawCurrentSnapshotId === -1 ? null : rawCurrentSnapshotId;
2731
- if (currentSnapshotId !== null) snapshot["parent-snapshot-id"] = currentSnapshotId;
2732
- const requirements = [{
2733
- type: "assert-table-uuid",
2734
- uuid: metadata["table-uuid"]
2735
- }, {
2736
- type: "assert-ref-snapshot-id",
2737
- ref: "main",
2738
- "snapshot-id": currentSnapshotId
2739
- }];
2740
- if (rowLineage) requirements.push({
2741
- type: "assert-next-row-id",
2742
- "next-row-id": toMetadataLong(metadata["next-row-id"] ?? 0)
2743
- });
2744
- return {
2745
- snapshot,
2746
- requirements,
2747
- updates: [{
2748
- action: "add-snapshot",
2749
- snapshot
2750
- }, {
2751
- action: "set-snapshot-ref",
2752
- "ref-name": "main",
2753
- type: "branch",
2754
- "snapshot-id": snapshot["snapshot-id"]
2755
- }],
2756
- writtenFiles: [...writtenFiles, manifestListPath]
2757
- };
2758
- }
2759
- function buildPartitionSummaries(partitions, schema, partitionSpec) {
2760
- return partitionSpec.fields.map((pf) => {
2761
- const sourceField = schema.fields.find((f) => f.id === pf["source-id"]);
2762
- if (!sourceField) throw new Error(`partition source field id ${pf["source-id"]} not found`);
2763
- const resultType = transformResultType(pf.transform, sourceField.type);
2764
- return computeFieldSummary(partitions.map((p) => p[pf.name]), resultType);
2765
- });
2766
- }
2767
- function assignFirstRowIds$1(manifests, firstRowId) {
2768
- let nextFirstRowId = firstRowId;
2769
- let assignedRows = 0n;
2770
- for (const manifest of manifests) {
2771
- if (manifest.content !== 0) {
2772
- manifest.first_row_id = void 0;
2773
- continue;
2774
- }
2775
- const rowIdRange = BigInt(manifest.added_rows_count ?? 0) + BigInt(manifest.existing_rows_count ?? 0);
2776
- if (manifest.first_row_id == null) {
2777
- manifest.first_row_id = nextFirstRowId;
2778
- nextFirstRowId += rowIdRange;
2779
- assignedRows += rowIdRange;
2780
- } else {
2781
- const manifestEnd = BigInt(manifest.first_row_id) + rowIdRange;
2782
- if (manifestEnd > nextFirstRowId) nextFirstRowId = manifestEnd;
2783
- }
2784
- }
2785
- return assignedRows;
2786
- }
2787
- function toMetadataLong(value) {
2788
- const out = Number(value);
2789
- if (!Number.isSafeInteger(out)) throw new Error(`metadata long exceeds JavaScript safe integer range: ${value}`);
2790
- return out;
2791
- }
2792
- function writeParquet({ writer, schema, records, codec }) {
2793
- const columnData = [];
2794
- const parquetFields = [];
2795
- let rootChildren = 0;
2796
- for (const field of schema.fields) {
2797
- const name = sanitize(field.name);
2798
- const fieldElements = icebergTypeToParquetFields(name, field.type, field.required, field.id);
2799
- if (!fieldElements.length) continue;
2800
- columnData.push({
2801
- name,
2802
- data: extractColumn(records, field)
2803
- });
2804
- parquetFields.push(...fieldElements);
2805
- rootChildren++;
2806
- }
2807
- return parquetWrite({
2808
- writer,
2809
- columnData,
2810
- schema: [{
2811
- name: "root",
2812
- num_children: rootChildren
2813
- }, ...parquetFields],
2814
- kvMetadata: [{
2815
- key: "iceberg.schema",
2816
- value: JSON.stringify(schema)
2817
- }],
2818
- codec
2819
- });
2820
- }
2821
- function extractColumn(records, field) {
2822
- const out = new Array(records.length);
2823
- for (let i = 0; i < records.length; i++) out[i] = materializeFieldValue(records[i][field.name], field);
2824
- return out;
2825
- }
2826
- function materializeFieldValue(value, field) {
2827
- const writeDefault = field["write-default"];
2828
- return materializeNestedDefaults(value !== void 0 ? value : writeDefault !== void 0 ? writeDefault : null, field.type);
2829
- }
2830
- function materializeNestedDefaults(value, type) {
2831
- if (value === null || value === void 0 || typeof type !== "object") return value;
2832
- if (type.type === "struct") {
2833
- if (typeof value !== "object" || Array.isArray(value)) return value;
2834
- const out = { ...value };
2835
- for (const child of type.fields) out[child.name] = materializeFieldValue(value[child.name], child);
2836
- return out;
2837
- }
2838
- if (type.type === "list") {
2839
- if (!Array.isArray(value)) return value;
2840
- return value.map((v) => materializeNestedDefaults(v, type.element));
2841
- }
2842
- if (type.type === "map") return materializeMapDefaults(value, type);
2843
- return value;
2844
- }
2845
- function materializeMapDefaults(value, type) {
2846
- if (typeof type.key !== "object" && typeof type.value !== "object") return value;
2847
- if (value instanceof Map) return Array.from(value.entries(), ([key, entryValue]) => ({
2848
- key: materializeNestedDefaults(key, type.key),
2849
- value: materializeNestedDefaults(entryValue, type.value)
2850
- }));
2851
- if (Array.isArray(value)) return value.map((entry) => {
2852
- if (entry && typeof entry === "object" && "key" in entry && "value" in entry) return {
2853
- key: materializeNestedDefaults(entry.key, type.key),
2854
- value: materializeNestedDefaults(entry.value, type.value)
2855
- };
2856
- if (Array.isArray(entry) && entry.length === 2) return {
2857
- key: materializeNestedDefaults(entry[0], type.key),
2858
- value: materializeNestedDefaults(entry[1], type.value)
2859
- };
2860
- return entry;
2861
- });
2862
- if (typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, entryValue]) => [key, materializeNestedDefaults(entryValue, type.value)]));
2863
- return value;
2864
- }
2865
- function icebergTypeToParquetFields(name, type, required, fieldId) {
2866
- const repetition_type = required ? "REQUIRED" : "OPTIONAL";
2867
- if (typeof type === "object") {
2868
- if (type.type === "list") {
2869
- const elementFields = icebergTypeToParquetFields("element", type.element, type["element-required"], type["element-id"]);
2870
- if (!elementFields.length) throw new Error(`unsupported iceberg list element type: ${typeName(type.element)}`);
2871
- return [
2872
- {
2873
- name,
2874
- converted_type: "LIST",
2875
- logical_type: { type: "LIST" },
2876
- repetition_type,
2877
- num_children: 1,
2878
- field_id: fieldId
2879
- },
2880
- {
2881
- name: "list",
2882
- repetition_type: "REPEATED",
2883
- num_children: 1
2884
- },
2885
- ...elementFields
2886
- ];
2887
- }
2888
- if (type.type === "struct") {
2889
- const allChildren = [];
2890
- let directChildren = 0;
2891
- for (const child of type.fields) {
2892
- const sub = icebergTypeToParquetFields(child.name, child.type, child.required, child.id);
2893
- if (!sub.length) continue;
2894
- allChildren.push(...sub);
2895
- directChildren++;
2896
- }
2897
- if (!directChildren) throw new Error(`struct ${name} has no writable children`);
2898
- return [{
2899
- name,
2900
- repetition_type,
2901
- num_children: directChildren,
2902
- field_id: fieldId
2903
- }, ...allChildren];
2904
- }
2905
- if (type.type === "map") {
2906
- if (type.key !== "string" && type.key !== "int") throw new Error(`unsupported iceberg map key type: ${typeName(type.key)}`);
2907
- const keyFields = icebergTypeToParquetFields("key", type.key, true, type["key-id"]);
2908
- const valueFields = icebergTypeToParquetFields("value", type.value, type["value-required"], type["value-id"]);
2909
- if (!keyFields.length) throw new Error(`unsupported iceberg map key type: ${typeName(type.key)}`);
2910
- if (!valueFields.length) throw new Error(`unsupported iceberg map value type: ${typeName(type.value)}`);
2911
- return [
2912
- {
2913
- name,
2914
- converted_type: "MAP",
2915
- logical_type: { type: "MAP" },
2916
- repetition_type,
2917
- num_children: 1,
2918
- field_id: fieldId
2919
- },
2920
- {
2921
- name: "key_value",
2922
- repetition_type: "REPEATED",
2923
- num_children: 2
2924
- },
2925
- ...keyFields,
2926
- ...valueFields
2927
- ];
2928
- }
2929
- throw new Error(`unsupported iceberg nested type: ${JSON.stringify(type)}`);
2930
- }
2931
- if (type.startsWith("geometry")) return [{
2932
- name,
2933
- type: "BYTE_ARRAY",
2934
- logical_type: { type: "GEOMETRY" },
2935
- repetition_type,
2936
- field_id: fieldId
2937
- }];
2938
- if (type.startsWith("geography")) return [{
2939
- name,
2940
- type: "BYTE_ARRAY",
2941
- logical_type: { type: "GEOGRAPHY" },
2942
- repetition_type,
2943
- field_id: fieldId
2944
- }];
2945
- const decimal = parseDecimalType(type);
2946
- if (decimal) {
2947
- const { precision, scale } = decimal;
2948
- return [{
2949
- name,
2950
- type: "FIXED_LEN_BYTE_ARRAY",
2951
- type_length: decimalRequiredBytes(precision),
2952
- converted_type: "DECIMAL",
2953
- logical_type: {
2954
- type: "DECIMAL",
2955
- precision,
2956
- scale
2957
- },
2958
- precision,
2959
- scale,
2960
- repetition_type,
2961
- field_id: fieldId
2962
- }];
2963
- }
2964
- const fixedLen = parseFixedType(type);
2965
- if (fixedLen !== void 0) return [{
2966
- name,
2967
- type: "FIXED_LEN_BYTE_ARRAY",
2968
- type_length: fixedLen,
2969
- repetition_type,
2970
- field_id: fieldId
2971
- }];
2972
- switch (type) {
2973
- case "unknown":
2974
- if (required) throw new Error("unsupported required iceberg type: unknown");
2975
- return [];
2976
- case "variant": return [
2977
- {
2978
- name,
2979
- repetition_type,
2980
- num_children: 2,
2981
- logical_type: { type: "VARIANT" },
2982
- field_id: fieldId
2983
- },
2984
- {
2985
- name: "metadata",
2986
- type: "BYTE_ARRAY",
2987
- repetition_type: "REQUIRED"
2988
- },
2989
- {
2990
- name: "value",
2991
- type: "BYTE_ARRAY",
2992
- repetition_type: "OPTIONAL"
2993
- }
2994
- ];
2995
- case "boolean": return [{
2996
- name,
2997
- type: "BOOLEAN",
2998
- repetition_type,
2999
- field_id: fieldId
3000
- }];
3001
- case "int": return [{
3002
- name,
3003
- type: "INT32",
3004
- repetition_type,
3005
- field_id: fieldId
3006
- }];
3007
- case "long": return [{
3008
- name,
3009
- type: "INT64",
3010
- repetition_type,
3011
- field_id: fieldId
3012
- }];
3013
- case "float": return [{
3014
- name,
3015
- type: "FLOAT",
3016
- repetition_type,
3017
- field_id: fieldId
3018
- }];
3019
- case "double": return [{
3020
- name,
3021
- type: "DOUBLE",
3022
- repetition_type,
3023
- field_id: fieldId
3024
- }];
3025
- case "string": return [{
3026
- name,
3027
- type: "BYTE_ARRAY",
3028
- converted_type: "UTF8",
3029
- repetition_type,
3030
- field_id: fieldId
3031
- }];
3032
- case "binary": return [{
3033
- name,
3034
- type: "BYTE_ARRAY",
3035
- repetition_type,
3036
- field_id: fieldId
3037
- }];
3038
- case "uuid": return [{
3039
- name,
3040
- type: "FIXED_LEN_BYTE_ARRAY",
3041
- type_length: 16,
3042
- logical_type: { type: "UUID" },
3043
- repetition_type,
3044
- field_id: fieldId
3045
- }];
3046
- case "date": return [{
3047
- name,
3048
- type: "INT32",
3049
- converted_type: "DATE",
3050
- logical_type: { type: "DATE" },
3051
- repetition_type,
3052
- field_id: fieldId
3053
- }];
3054
- case "time": return [{
3055
- name,
3056
- type: "INT64",
3057
- converted_type: "TIME_MICROS",
3058
- logical_type: {
3059
- type: "TIME",
3060
- isAdjustedToUTC: false,
3061
- unit: "MICROS"
3062
- },
3063
- repetition_type,
3064
- field_id: fieldId
3065
- }];
3066
- case "timestamp": return [timestampField(name, repetition_type, false, "MICROS", fieldId)];
3067
- case "timestamptz": return [timestampField(name, repetition_type, true, "MICROS", fieldId)];
3068
- case "timestamp_ns": return [timestampField(name, repetition_type, false, "NANOS", fieldId)];
3069
- case "timestamptz_ns": return [timestampField(name, repetition_type, true, "NANOS", fieldId)];
3070
- default: throw new Error(`unsupported iceberg type: ${type}`);
3071
- }
3072
- }
3073
- function parseFixedType(type) {
3074
- const m = /^fixed\[(\d+)\]$/.exec(type);
3075
- if (!m) return void 0;
3076
- return parseInt(m[1], 10);
3077
- }
3078
- function timestampField(name, repetition_type, isAdjustedToUTC, unit, field_id) {
3079
- return {
3080
- name,
3081
- type: "INT64",
3082
- logical_type: {
3083
- type: "TIMESTAMP",
3084
- isAdjustedToUTC,
3085
- unit
3086
- },
3087
- repetition_type,
3088
- field_id
3089
- };
3090
- }
3091
- function buildSortComparator(sortOrder, schema) {
3092
- if (!sortOrder?.fields?.length) return void 0;
3093
- const fields = sortOrder.fields.map((sf) => {
3094
- const sourceId = sf["source-id"] ?? sf["source-ids"]?.[0];
3095
- const sourceField = schema.fields.find((f) => f.id === sourceId);
3096
- if (!sourceField) throw new Error(`sort source field id ${sourceId} not found in schema`);
3097
- return {
3098
- name: sourceField.name,
3099
- transform: sf.transform,
3100
- sourceType: sourceField.type,
3101
- resultType: transformResultType(sf.transform, sourceField.type),
3102
- desc: sf.direction === "desc",
3103
- nullsFirst: sf["null-order"] === "nulls-first"
3104
- };
3105
- });
3106
- return (a, b) => {
3107
- for (const f of fields) {
3108
- const c = compareKeys(sortKey(a[f.name], f.transform, f.sourceType), sortKey(b[f.name], f.transform, f.sourceType), f.resultType, f.desc, f.nullsFirst);
3109
- if (c !== 0) return c;
3110
- }
3111
- return 0;
3112
- };
3113
- }
3114
- function sortKey(value, transform, sourceType) {
3115
- if (value === null || value === void 0) return null;
3116
- if (transform === "identity") return value;
3117
- return applyTransform(transform, value, sourceType);
3118
- }
3119
- function compareKeys(ka, kb, resultType, desc, nullsFirst) {
3120
- const aNull = ka === null || ka === void 0;
3121
- const bNull = kb === null || kb === void 0;
3122
- if (aNull && bNull) return 0;
3123
- if (aNull) return nullsFirst ? -1 : 1;
3124
- if (bNull) return nullsFirst ? 1 : -1;
3125
- const aNaN = typeof ka === "number" && Number.isNaN(ka);
3126
- const bNaN = typeof kb === "number" && Number.isNaN(kb);
3127
- if (aNaN || bNaN) {
3128
- if (aNaN && bNaN) return 0;
3129
- const c = aNaN ? 1 : -1;
3130
- return desc ? -c : c;
3131
- }
3132
- const c = compare(ka, kb, resultType);
3133
- return desc ? -c : c;
3134
- }
3135
- async function prepareAppend({ tableUrl, metadata, records, resolver, sortOrderId }) {
3136
- if (!tableUrl) throw new Error("tableUrl is required");
3137
- if (!resolver?.writer) throw new Error("resolver.writer is required");
3138
- const writerFn = resolver.writer;
3139
- if (metadata["format-version"] !== 2 && metadata["format-version"] !== 3) throw new Error(`unsupported format-version: ${metadata["format-version"]}`);
3140
- const formatVersion = metadata["format-version"];
3141
- const partitionSpec = metadata["partition-specs"].find((s) => s["spec-id"] === metadata["default-spec-id"]);
3142
- if (!partitionSpec) throw new Error("default partition spec not found in metadata");
3143
- const schema = metadata.schemas.find((s) => s["schema-id"] === metadata["current-schema-id"]);
3144
- if (!schema) throw new Error("current schema not found in metadata");
3145
- validateSchemaForVersion(schema, formatVersion);
3146
- const snapshotId = newSnapshotId(metadata);
3147
- const manifestUuid = uuid4();
3148
- checkWriteFormat(metadata.properties?.["write.format.default"]);
3149
- const codec = resolveParquetCodec(metadata.properties?.["write.parquet.compression-codec"]);
3150
- const orderId = sortOrderId ?? metadata["default-sort-order-id"] ?? 0;
3151
- const sortOrder = (metadata["sort-orders"] ?? []).find((o) => o["order-id"] === orderId);
3152
- if (sortOrderId !== void 0 && !sortOrder) throw new Error(`sort order ${sortOrderId} not found in metadata`);
3153
- const comparator = buildSortComparator(sortOrder, schema);
3154
- const appliedSortOrderId = comparator ? orderId : 0;
3155
- const groups = partitionSpec.fields.length ? groupByPartition(records, schema, partitionSpec) : [{
3156
- partition: {},
3157
- records
3158
- }];
3159
- const writtenDataFiles = await Promise.all(groups.map(async (group) => {
3160
- const sortedRecords = comparator ? [...group.records].sort(comparator) : group.records;
3161
- const dataPath = `${tableUrl}/data/${uuid4()}.parquet`;
3162
- const dataWriter = writerFn(dataPath);
3163
- await writeParquet({
3164
- writer: dataWriter,
3165
- schema,
3166
- records: sortedRecords,
3167
- codec
3168
- });
3169
- const stats = computeColumnStats(sortedRecords, schema);
3170
- return {
3171
- partition: group.partition,
3172
- records: sortedRecords,
3173
- dataFile: {
3174
- content: 0,
3175
- file_path: dataPath,
3176
- file_format: "parquet",
3177
- partition: group.partition,
3178
- record_count: BigInt(sortedRecords.length),
3179
- file_size_in_bytes: BigInt(dataWriter.offset),
3180
- value_counts: stats.value_counts,
3181
- null_value_counts: stats.null_value_counts,
3182
- nan_value_counts: stats.nan_value_counts,
3183
- lower_bounds: stats.lower_bounds,
3184
- upper_bounds: stats.upper_bounds,
3185
- sort_order_id: appliedSortOrderId
3186
- },
3187
- path: dataPath
3188
- };
3189
- }));
3190
- const manifestPath = `${tableUrl}/metadata/${manifestUuid}-m0.avro`;
3191
- const manifestWriter = writerFn(manifestPath);
3192
- await writeDataManifest({
3193
- writer: manifestWriter,
3194
- schema,
3195
- partitionSpec,
3196
- snapshotId,
3197
- dataFiles: writtenDataFiles.map((f) => f.dataFile),
3198
- formatVersion
3199
- });
3200
- const manifestLength = BigInt(manifestWriter.offset);
3201
- const addedRowCount = writtenDataFiles.reduce((sum, f) => sum + BigInt(f.records.length), 0n);
3202
- const addedFilesSize = writtenDataFiles.reduce((sum, f) => sum + f.dataFile.file_size_in_bytes, 0n);
3203
- const partitions = buildPartitionSummaries(writtenDataFiles.map((f) => f.dataFile.partition), schema, partitionSpec);
3204
- return {
3205
- snapshotId,
3206
- manifestUuid,
3207
- formatVersion,
3208
- manifestPath,
3209
- manifestLength,
3210
- partitionSpecId: partitionSpec["spec-id"],
3211
- partitions,
3212
- addedDataFilesCount: writtenDataFiles.length,
3213
- addedRowCount,
3214
- addedFilesSize,
3215
- recordsCount: records.length,
3216
- writtenFiles: [...writtenDataFiles.map((f) => f.path), manifestPath]
3217
- };
3218
- }
3219
- async function stageSnapshotForAppend({ tableUrl, metadata, prepared, resolver, snapshotProperties }) {
3220
- if (!tableUrl) throw new Error("tableUrl is required");
3221
- if (!resolver?.writer) throw new Error("resolver.writer is required");
3222
- const sequenceNumber = BigInt(metadata["last-sequence-number"] ?? 0) + 1n;
3223
- const timestampMs = Date.now();
3224
- const newManifest = {
3225
- manifest_path: prepared.manifestPath,
3226
- manifest_length: prepared.manifestLength,
3227
- partition_spec_id: prepared.partitionSpecId,
3228
- content: 0,
3229
- sequence_number: sequenceNumber,
3230
- min_sequence_number: sequenceNumber,
3231
- added_snapshot_id: prepared.snapshotId,
3232
- added_files_count: prepared.addedDataFilesCount,
3233
- existing_files_count: 0,
3234
- deleted_files_count: 0,
3235
- added_rows_count: prepared.addedRowCount,
3236
- existing_rows_count: 0n,
3237
- deleted_rows_count: 0n,
3238
- partitions: prepared.partitions
3239
- };
3240
- const prevSummary = currentSnapshot(metadata)?.summary;
3241
- const prevTotals = {
3242
- records: BigInt(prevSummary?.["total-records"] ?? "0"),
3243
- size: BigInt(prevSummary?.["total-files-size"] ?? "0"),
3244
- files: BigInt(prevSummary?.["total-data-files"] ?? "0")
3245
- };
3246
- const summary = {
3247
- operation: "append",
3248
- "added-data-files": String(prepared.addedDataFilesCount),
3249
- "added-records": String(prepared.recordsCount),
3250
- "added-files-size": String(prepared.addedFilesSize),
3251
- "changed-partition-count": String(prepared.addedDataFilesCount),
3252
- "total-records": String(prevTotals.records + BigInt(prepared.recordsCount)),
3253
- "total-files-size": String(prevTotals.size + prepared.addedFilesSize),
3254
- "total-data-files": String(prevTotals.files + BigInt(prepared.addedDataFilesCount)),
3255
- "total-delete-files": "0",
3256
- "total-position-deletes": "0",
3257
- "total-equality-deletes": "0",
3258
- ...snapshotProperties ?? {}
3259
- };
3260
- return await buildSnapshotUpdate({
3261
- tableUrl,
3262
- metadata,
3263
- resolver,
3264
- snapshotId: prepared.snapshotId,
3265
- sequenceNumber,
3266
- manifestUuid: prepared.manifestUuid,
3267
- timestampMs,
3268
- formatVersion: prepared.formatVersion,
3269
- newManifests: [newManifest],
3270
- summary,
3271
- writtenFiles: []
3272
- });
3273
- }
3274
- function checkWriteFormat(value) {
3275
- if (value === void 0) return;
3276
- if (value.toLowerCase() !== "parquet") throw new Error(`unsupported write.format.default: ${value}`);
3277
- }
3278
- function resolveParquetCodec(value) {
3279
- if (value === void 0) return void 0;
3280
- switch (value.toLowerCase()) {
3281
- case "snappy": return "SNAPPY";
3282
- case "none":
3283
- case "uncompressed": return "UNCOMPRESSED";
3284
- default: throw new Error(`unsupported write.parquet.compression-codec: ${value}`);
3285
- }
3286
- }
3287
- function newSnapshotId(metadata) {
3288
- const used = new Set((metadata?.snapshots ?? []).map((s) => BigInt(s["snapshot-id"])));
3289
- const arr = /* @__PURE__ */ new BigInt64Array(1);
3290
- for (let attempt = 0; attempt < 32; attempt++) {
3291
- globalThis.crypto.getRandomValues(arr);
3292
- const masked = arr[0] & 9007199254740991n;
3293
- const id = masked === 0n ? 1n : masked;
3294
- if (!used.has(id)) return id;
3295
- }
3296
- throw new Error("newSnapshotId: failed to find an unused id after 32 attempts");
3297
- }
3298
- async function icebergManifests({ metadata, resolver, snapshotId, partitionFilter }) {
3299
- resolver ??= urlResolver();
3300
- const rawTarget = snapshotId ?? metadata["current-snapshot-id"];
3301
- if (rawTarget == null || rawTarget < 0) throw new Error("No current snapshot id found in table metadata");
3302
- const targetId = BigInt(rawTarget);
3303
- const snapshot = metadata.snapshots?.find((s) => BigInt(s["snapshot-id"]) === targetId);
3304
- if (!snapshot) throw new Error(`Snapshot ${rawTarget} not found in metadata`);
3305
- let manifests = [];
3306
- if (snapshot["manifest-list"]) {
3307
- const manifestListUrl = snapshot["manifest-list"];
3308
- manifests = await fetchAvroRecords(manifestListUrl, resolver);
3309
- } else if (snapshot.manifests) manifests = snapshot.manifests;
3310
- else throw new Error("No manifest information found in snapshot");
3311
- if (partitionFilter) manifests = manifests.filter((manifest) => {
3312
- let keep = true;
3313
- try {
3314
- keep = partitionFilter(manifest.partitions, manifest.partition_spec_id ?? 0, manifest) !== false;
3315
- } catch {
3316
- keep = true;
3317
- }
3318
- return keep;
3319
- });
3320
- return await fetchManifests(manifests, resolver);
3321
- }
3322
- const MANIFEST_FETCH_CONCURRENCY = 8;
3323
- async function fetchOneManifest(manifest, resolver) {
3324
- const url = manifest.manifest_path;
3325
- const entries = await fetchAvroRecords(url, resolver, Number(manifest.manifest_length));
3326
- for (const entry of entries) {
3327
- entry.partition_spec_id = manifest.partition_spec_id ?? 0;
3328
- if (entry.sequence_number === void 0) entry.sequence_number = manifest.sequence_number ?? 0n;
3329
- if (entry.status === 1) {
3330
- if (entry.sequence_number === void 0) entry.sequence_number = manifest.sequence_number;
3331
- if (entry.file_sequence_number === void 0) entry.file_sequence_number = manifest.sequence_number;
3332
- } else if (entry.sequence_number === void 0 || entry.file_sequence_number === void 0) throw new Error("iceberg manifest entry missing sequence number");
3333
- }
3334
- assignFirstRowIds(manifest, entries);
3335
- return {
3336
- url,
3337
- entries
3338
- };
3339
- }
3340
- async function fetchManifests(manifests, resolver) {
3341
- const results = new Array(manifests.length);
3342
- let next = 0;
3343
- async function worker() {
3344
- while (next < manifests.length) {
3345
- const i = next++;
3346
- results[i] = await fetchOneManifest(manifests[i], resolver);
3347
- }
801
+ async function fetchManifests(manifests, resolver) {
802
+ const results = new Array(manifests.length);
803
+ let next = 0;
804
+ async function worker() {
805
+ while (next < manifests.length) {
806
+ const i = next++;
807
+ results[i] = await fetchOneManifest(manifests[i], resolver);
808
+ }
3348
809
  }
3349
810
  const poolSize = Math.min(MANIFEST_FETCH_CONCURRENCY, manifests.length);
3350
811
  const workers = [];
@@ -3364,44 +825,13 @@ function assignFirstRowIds(manifest, entries) {
3364
825
  }
3365
826
  }
3366
827
  }
3367
- const DEFAULT_RETRY = Object.freeze({
828
+ Object.freeze({
3368
829
  maxAttempts: 50,
3369
830
  initialMs: 50,
3370
831
  maxMs: 3e3,
3371
832
  factor: 2,
3372
833
  totalTimeoutMs: 1800 * 1e3
3373
834
  });
3374
- async function icebergAppend({ catalog, namespace, table, tableUrl, resolver, records, sortOrderId, snapshotProperties }) {
3375
- const ctx = await loadTable({
3376
- catalog,
3377
- namespace,
3378
- table,
3379
- tableUrl,
3380
- resolver
3381
- });
3382
- const prepared = await prepareAppend({
3383
- tableUrl: ctx.tableUrl,
3384
- metadata: ctx.metadata,
3385
- records,
3386
- resolver: requireResolver(ctx.resolver, "icebergAppend"),
3387
- sortOrderId
3388
- });
3389
- return await commitWithRetry({
3390
- catalog,
3391
- target: {
3392
- namespace,
3393
- table
3394
- },
3395
- ctx,
3396
- stage: (workingCtx) => stageSnapshotForAppend({
3397
- tableUrl: workingCtx.tableUrl,
3398
- metadata: workingCtx.metadata,
3399
- prepared,
3400
- resolver: requireResolver(workingCtx.resolver, "icebergAppend"),
3401
- snapshotProperties
3402
- })
3403
- });
3404
- }
3405
835
  async function icebergCreateTable({ catalog, namespace, table, tableUrl, schema, partitionSpec, sortOrder, properties, formatVersion, stageCreate }) {
3406
836
  if (catalog.type === "rest") {
3407
837
  if (!namespace || !table) throw new Error("namespace and table are required for rest catalogs");
@@ -3430,280 +860,7 @@ async function icebergCreateTable({ catalog, namespace, table, tableUrl, schema,
3430
860
  conditionalCommits: catalog.conditionalCommits
3431
861
  });
3432
862
  }
3433
- async function icebergDropTable({ catalog, namespace, table, tableUrl, lister, purgeRequested }) {
3434
- if (catalog.type === "rest") {
3435
- if (!namespace || !table) throw new Error("namespace and table are required for rest catalogs");
3436
- await restCatalogDropTable(catalog, {
3437
- namespace,
3438
- table,
3439
- purgeRequested
3440
- });
3441
- return;
3442
- }
3443
- if (!tableUrl) throw new Error("tableUrl is required for file catalogs");
3444
- if (!lister) throw new Error("lister is required to drop a file catalog table");
3445
- const { deleter } = catalog.resolver;
3446
- if (!deleter) throw new Error("resolver.deleter is required to drop a file catalog table");
3447
- const dirs = purgeRequested ? ["metadata", "data"] : ["metadata"];
3448
- for (const dir of dirs) {
3449
- const names = await lister(`${tableUrl}/${dir}`).catch(() => []);
3450
- await Promise.allSettled(names.map((n) => deleter(`${tableUrl}/${dir}/${n}`)));
3451
- }
3452
- }
3453
- function requireResolver(resolver, caller) {
3454
- if (!resolver) throw new Error(`${caller}: resolver is required`);
3455
- return resolver;
3456
- }
3457
- async function commitStaged(catalog, target, ctx, staged) {
3458
- if (catalog.type === "rest") {
3459
- const { metadata } = await restCatalogUpdateTable(catalog, {
3460
- namespace: target.namespace,
3461
- table: target.table,
3462
- requirements: staged.requirements,
3463
- updates: staged.updates
3464
- });
3465
- return metadata;
3466
- }
3467
- if (!ctx.resolver) throw new Error("resolver is required to commit to a file catalog");
3468
- return await fileCatalogCommit({
3469
- tableUrl: ctx.tableUrl,
3470
- metadata: ctx.metadata,
3471
- metadataFileName: ctx.metadataFileName,
3472
- currentVersion: ctx.version,
3473
- staged,
3474
- resolver: ctx.resolver,
3475
- conditionalCommits: catalog.type === "file" && catalog.conditionalCommits
3476
- });
3477
- }
3478
- async function commitWithRetry({ catalog, target, ctx, stage }) {
3479
- const retryEnabled = catalog.type === "rest" || catalog.type === "file" && catalog.conditionalCommits === true;
3480
- const policy = resolveRetryPolicy(ctx.metadata);
3481
- const startedAt = Date.now();
3482
- let workingCtx = ctx;
3483
- for (let attempt = 1; attempt <= policy.maxAttempts; attempt++) {
3484
- const staged = await stage(workingCtx);
3485
- try {
3486
- return await commitStaged(catalog, target, workingCtx, staged);
3487
- } catch (err) {
3488
- if (!retryEnabled || !isCommitConflict(err)) throw err;
3489
- if (attempt === policy.maxAttempts) throw new Error(`${catalog.type} catalog commit failed after ${policy.maxAttempts} attempts due to concurrent commits`);
3490
- const elapsed = Date.now() - startedAt;
3491
- if (elapsed >= policy.totalTimeoutMs) throw new Error(`${catalog.type} catalog commit retry budget exhausted after ${attempt} attempts and ${elapsed}ms (limit ${policy.totalTimeoutMs}ms)`);
3492
- const remaining = policy.totalTimeoutMs - elapsed;
3493
- await sleep(Math.min(jitteredBackoff(attempt, policy), remaining));
3494
- workingCtx = await reloadCtx(catalog, target, workingCtx, err);
3495
- }
3496
- }
3497
- throw new Error("unreachable");
3498
- }
3499
- async function reloadCtx(catalog, target, workingCtx, lastErr) {
3500
- if (catalog.type === "rest") {
3501
- if (!target.namespace || !target.table) throw lastErr;
3502
- const { metadata } = await restCatalogLoadTable(catalog, {
3503
- namespace: target.namespace,
3504
- table: target.table
3505
- });
3506
- return {
3507
- metadata,
3508
- metadataFileName: workingCtx.metadataFileName,
3509
- version: workingCtx.version,
3510
- tableUrl: workingCtx.tableUrl,
3511
- resolver: workingCtx.resolver
3512
- };
3513
- }
3514
- if (!workingCtx.resolver) throw lastErr;
3515
- const fresh = await loadLatestFileCatalogMetadata({
3516
- tableUrl: workingCtx.tableUrl,
3517
- resolver: workingCtx.resolver,
3518
- lister: catalog.lister
3519
- });
3520
- return {
3521
- metadata: fresh.metadata,
3522
- metadataFileName: fresh.metadataFileName,
3523
- version: fresh.version,
3524
- tableUrl: workingCtx.tableUrl,
3525
- resolver: workingCtx.resolver
3526
- };
3527
- }
3528
- function resolveRetryPolicy(metadata) {
3529
- const props = metadata.properties ?? {};
3530
- const numRetries = parseTableProp(props["commit.retry.num-retries"]);
3531
- const maxAttempts = numRetries === void 0 ? DEFAULT_RETRY.maxAttempts : numRetries + 1;
3532
- const initialMs = parseTableProp(props["commit.retry.min-wait-ms"]) ?? DEFAULT_RETRY.initialMs;
3533
- const maxMs = parseTableProp(props["commit.retry.max-wait-ms"]) ?? DEFAULT_RETRY.maxMs;
3534
- const totalTimeoutMs = parseTableProp(props["commit.retry.total-timeout-ms"]) ?? DEFAULT_RETRY.totalTimeoutMs;
3535
- return {
3536
- maxAttempts,
3537
- initialMs,
3538
- maxMs,
3539
- factor: DEFAULT_RETRY.factor,
3540
- totalTimeoutMs
3541
- };
3542
- }
3543
- function parseTableProp(value) {
3544
- if (value === void 0 || value === null || value === "") return void 0;
3545
- const n = Number(value);
3546
- if (!Number.isFinite(n) || n < 0) return void 0;
3547
- return n;
3548
- }
3549
- function jitteredBackoff(attempt, policy) {
3550
- if (policy.initialMs === 0 || policy.maxMs === 0) return 0;
3551
- const base = Math.min(policy.maxMs, policy.initialMs * policy.factor ** (attempt - 1));
3552
- return Math.floor(Math.random() * base);
3553
- }
3554
- function sleep(ms) {
3555
- if (ms <= 0) return Promise.resolve();
3556
- return new Promise((resolve) => setTimeout(resolve, ms));
3557
- }
3558
- function isCommitConflict(err) {
3559
- if (!err || typeof err !== "object") return false;
3560
- const { status } = err;
3561
- return status === 412 || status === 409;
3562
- }
3563
- const enc = new TextEncoder();
3564
- function s3SignedResolver({ accessKeyId, secretAccessKey, sessionToken, region, endpoint, pathStyle = false }) {
3565
- const ep = endpoint ? new URL(endpoint.replace(/\/$/, "") + "/") : void 0;
3566
- function toHttps(url) {
3567
- if (!url.startsWith("s3://") && !url.startsWith("s3a://")) return url;
3568
- const rest = url.slice(url.indexOf("://") + 3);
3569
- const slash = rest.indexOf("/");
3570
- if (slash === -1) throw new Error(`invalid S3 URL: ${url}`);
3571
- const bucket = rest.slice(0, slash);
3572
- const key = rest.slice(slash + 1);
3573
- if (ep) {
3574
- if (pathStyle) return `${ep.origin}${ep.pathname}${bucket}/${key}`;
3575
- return `${ep.protocol}//${bucket}.${ep.host}/${key}`;
3576
- }
3577
- return `https://${bucket}.s3.amazonaws.com/${key}`;
3578
- }
3579
- async function signRequest(method, url, body, extra = {}) {
3580
- const u = new URL(url);
3581
- const xAmzDate = (/* @__PURE__ */ new Date()).toISOString().replace(/[-:]|\.\d{3}/g, "");
3582
- const dStamp = xAmzDate.slice(0, 8);
3583
- const payloadHash = body !== void 0 ? await sha256hex(body) : await sha256hex("");
3584
- const lc = {};
3585
- for (const [k, v] of Object.entries(extra)) lc[k.toLowerCase()] = String(v);
3586
- lc["host"] = u.host;
3587
- lc["x-amz-date"] = xAmzDate;
3588
- lc["x-amz-content-sha256"] = payloadHash;
3589
- if (sessionToken) lc["x-amz-security-token"] = sessionToken;
3590
- const sortedKeys = Object.keys(lc).sort();
3591
- const canonicalHeaders = sortedKeys.map((k) => `${k}:${lc[k].trim().replace(/\s+/g, " ")}\n`).join("");
3592
- const signedHeaders = sortedKeys.join(";");
3593
- const canonicalRequest = [
3594
- method,
3595
- u.pathname.split("/").map((seg) => encodeRfc3986(decodeURIComponent(seg))).join("/"),
3596
- [...u.searchParams.entries()].sort((a, b) => {
3597
- if (a[0] !== b[0]) return a[0] < b[0] ? -1 : 1;
3598
- return a[1] < b[1] ? -1 : a[1] > b[1] ? 1 : 0;
3599
- }).map(([k, v]) => `${encodeRfc3986(k)}=${encodeRfc3986(v)}`).join("&"),
3600
- canonicalHeaders,
3601
- signedHeaders,
3602
- payloadHash
3603
- ].join("\n");
3604
- const credentialScope = `${dStamp}/${region}/s3/aws4_request`;
3605
- const stringToSign = [
3606
- "AWS4-HMAC-SHA256",
3607
- xAmzDate,
3608
- credentialScope,
3609
- await sha256hex(canonicalRequest)
3610
- ].join("\n");
3611
- const signature = bytesToHex(await hmac(await deriveSigningKey(secretAccessKey, dStamp, region, "s3"), stringToSign));
3612
- const out = {};
3613
- for (const [k, v] of Object.entries(lc)) {
3614
- if (k === "host") continue;
3615
- out[k] = v;
3616
- }
3617
- out["Authorization"] = `AWS4-HMAC-SHA256 Credential=${accessKeyId}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}`;
3618
- return out;
3619
- }
3620
- return {
3621
- async reader(path, byteLength) {
3622
- const url = toHttps(path);
3623
- let len = byteLength;
3624
- if (len === void 0) {
3625
- const headers = await signRequest("HEAD", url);
3626
- const res = await fetch(url, {
3627
- method: "HEAD",
3628
- headers
3629
- });
3630
- if (!res.ok) throw new Error(`HEAD ${path}: ${res.status} ${res.statusText}`);
3631
- len = Number(res.headers.get("content-length"));
3632
- if (!Number.isFinite(len)) throw new Error(`HEAD ${path}: missing Content-Length`);
3633
- }
3634
- const fileLength = len;
3635
- return {
3636
- byteLength: fileLength,
3637
- async slice(start, end) {
3638
- const range = `bytes=${start}-${(end ?? fileLength) - 1}`;
3639
- const headers = await signRequest("GET", url, void 0, { range });
3640
- const res = await fetch(url, {
3641
- method: "GET",
3642
- headers
3643
- });
3644
- if (!res.ok) throw new Error(`GET ${path} ${range}: ${res.status} ${res.statusText}`);
3645
- return await res.arrayBuffer();
3646
- }
3647
- };
3648
- },
3649
- writer(path, options) {
3650
- const w = new ByteWriter();
3651
- w.finish = async function() {
3652
- const url = toHttps(path);
3653
- const body = w.getBytes().slice();
3654
- const extra = {};
3655
- if (options?.ifNoneMatch) extra["if-none-match"] = options.ifNoneMatch;
3656
- const headers = await signRequest("PUT", url, body, extra);
3657
- const res = await fetch(url, {
3658
- method: "PUT",
3659
- headers,
3660
- body
3661
- });
3662
- if (!res.ok) {
3663
- const err = /* @__PURE__ */ new Error(`PUT ${path}: ${res.status} ${res.statusText}`);
3664
- err.status = res.status;
3665
- throw err;
3666
- }
3667
- };
3668
- return w;
3669
- },
3670
- async deleter(path) {
3671
- const url = toHttps(path);
3672
- const headers = await signRequest("DELETE", url);
3673
- const res = await fetch(url, {
3674
- method: "DELETE",
3675
- headers
3676
- });
3677
- if (!res.ok && res.status !== 404) throw new Error(`DELETE ${path}: ${res.status} ${res.statusText}`);
3678
- }
3679
- };
3680
- }
3681
- async function sha256hex(data) {
3682
- const bytes = typeof data === "string" ? enc.encode(data) : data;
3683
- const hash = await crypto.subtle.digest("SHA-256", bytes);
3684
- return bytesToHex(new Uint8Array(hash));
3685
- }
3686
- async function hmac(key, data) {
3687
- const keyBytes = typeof key === "string" ? enc.encode(key) : key;
3688
- const dataBytes = typeof data === "string" ? enc.encode(data) : data;
3689
- const cryptoKey = await crypto.subtle.importKey("raw", keyBytes, {
3690
- name: "HMAC",
3691
- hash: "SHA-256"
3692
- }, false, ["sign"]);
3693
- const sig = await crypto.subtle.sign("HMAC", cryptoKey, dataBytes);
3694
- return new Uint8Array(sig);
3695
- }
3696
- async function deriveSigningKey(secret, dateStamp, region, service) {
3697
- return await hmac(await hmac(await hmac(await hmac(`AWS4${secret}`, dateStamp), region), service), "aws4_request");
3698
- }
3699
- function encodeRfc3986(str) {
3700
- return encodeURIComponent(str).replace(/[!*'()]/g, (c) => "%" + c.charCodeAt(0).toString(16).toUpperCase());
3701
- }
3702
- function bytesToHex(bytes) {
3703
- let s = "";
3704
- for (const b of bytes) s += b.toString(16).padStart(2, "0");
3705
- return s;
3706
- }
863
+ new TextEncoder();
3707
864
  (() => {
3708
865
  if (typeof setImmediate === "function") return () => new Promise((resolve) => setImmediate(resolve));
3709
866
  if (typeof MessageChannel !== "undefined") {
@@ -3720,4 +877,4 @@ function bytesToHex(bytes) {
3720
877
  }
3721
878
  return () => new Promise((resolve) => setTimeout(resolve, 0));
3722
879
  })();
3723
- export { cachingResolver, icebergAppend, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver };
880
+ export { icebergCreateTable, icebergManifests, restCatalogLoadTable };