@gscdump/lakehouse 7.6.0 → 7.8.0

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f → icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471}/node_modules/icebird/src/avro/avro.read.mjs

@@ -1,36 +1,54 @@
+import { boundedBytes } from "./avro.budget.mjs";
+import { checkBlockEnd, readBlockEnd, readZigZag, readZigZagBigInt } from "./avro.metadata.mjs";
+import { gunzip } from "../../../../../hyparquet-compressors@1.1.2_patch_hash=3f8f44805c034b2563763dcd5d37050ea6b1d5d45b46ea37adc4a721e748a1c5/node_modules/hyparquet-compressors/src/gzip.mjs";
+import "../../../../../hyparquet-compressors@1.1.2_patch_hash=3f8f44805c034b2563763dcd5d37050ea6b1d5d45b46ea37adc4a721e748a1c5/node_modules/hyparquet-compressors/src/index.mjs";
 import { parseDecimal } from "../../../../../hyparquet@1.31.1/node_modules/hyparquet/src/convert.mjs";
-import { gunzip } from "../../../../../hyparquet-compressors@1.1.2/node_modules/hyparquet-compressors/src/gzip.mjs";
-import "../../../../../hyparquet-compressors@1.1.2/node_modules/hyparquet-compressors/src/index.mjs";
-import { readZigZag, readZigZagBigInt } from "./avro.metadata.mjs";
 const textDecoder = new TextDecoder();
-function avroRead({ reader, metadata, syncMarker }) {
-	const decode = compileType(metadata["avro.schema"]);
+function avroRead({ reader, metadata, syncMarker, recordKind = "entries" }) {
+	const budget = reader.budget;
+	const decode = compileType(metadata["avro.schema"], budget);
 	const blocks = [];
 	while (reader.offset < reader.view.byteLength) {
 		let recordCount = readZigZag(reader);
+		if (budget && recordCount <= 0) budget.fail(/* @__PURE__ */ new RangeError("Invalid Avro record count"));
 		if (recordCount === 0) break;
 		if (recordCount < 0) recordCount = -recordCount;
+		budget?.records(recordCount, recordKind);
+		budget?.availableValues(recordCount);
 		const blockSize = readZigZag(reader);
-		let data = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, blockSize);
+		let data = boundedBytes(reader, blockSize, budget?.limits.maxObjectBytes);
 		reader.offset += blockSize;
-		const blockSync = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, 16);
+		const blockSync = boundedBytes(reader, 16, 16);
 		reader.offset += 16;
 		for (let i = 0; i < 16; i++) if (blockSync[i] !== syncMarker[i]) throw new Error("sync marker does not match");
 		const codec = metadata["avro.codec"];
-		if (codec === "deflate") data = gunzip(data);
+		if (codec === "deflate") data = budget ? gunzip(data, void 0, 0, 0, {
+			raw: true,
+			maxOutputBytes: budget.inflateLimit()
+		}) : gunzip(data);
 		else if (codec !== "null") throw new Error(`unsupported codec: ${codec}`);
+		budget?.inflated(data.byteLength);
 		const dataReader = {
 			view: new DataView(data.buffer, data.byteOffset, data.byteLength),
-			offset: 0
+			offset: 0,
+			...budget ? { budget } : {}
 		};
 		for (let i = 0; i < recordCount; i++) blocks.push(decode(dataReader));
+		if (budget && dataReader.offset !== data.byteLength) budget.fail(/* @__PURE__ */ new RangeError("Avro block length mismatch"));
 	}
 	return blocks;
 }
 function readType(reader, type) {
+	reader.budget?.values(1);
+	return readTypeValue(reader, type);
+}
+function readTypeValue(reader, type) {
 	if (type === "null") return;
-	else if (Array.isArray(type)) return readType(reader, type[readZigZag(reader)]);
-	else if (typeof type === "object" && type.type === "record") {
+	else if (Array.isArray(type)) {
+		const unionIndex = readZigZag(reader);
+		reader.budget?.bound(unionIndex, type.length - 1, "unionIndex");
+		return readType(reader, type[unionIndex]);
+	} else if (typeof type === "object" && type.type === "record") {
 		const obj = {};
 		for (const subField of type.fields) obj[subField.name] = readType(reader, subField.type);
 		return obj;
@@ -39,11 +57,14 @@ function readType(reader, type) {
 		while (true) {
 			let count = readZigZag(reader);
 			if (count === 0) break;
+			let end;
 			if (count < 0) {
 				count = -count;
-				readZigZag(reader);
+				end = readBlockEnd(reader);
 			}
+			reader.budget?.availableValues(count);
 			for (let i = 0; i < count; i++) arr.push(readType(reader, type.items));
+			checkBlockEnd(reader, end);
 		}
 		return arr;
 	} else if (typeof type === "object" && type.type === "map") {
@@ -51,18 +72,24 @@ function readType(reader, type) {
 		while (true) {
 			let count = readZigZag(reader);
 			if (count === 0) break;
+			let end;
 			if (count < 0) {
 				count = -count;
-				readZigZag(reader);
+				end = readBlockEnd(reader);
 			}
+			reader.budget?.availableValues(count);
 			for (let i = 0; i < count; i++) {
 				const key = readType(reader, "string");
 				map[key] = readType(reader, type.values);
 			}
+			checkBlockEnd(reader, end);
 		}
 		return map;
-	} else if (typeof type === "object" && type.type === "enum") return type.symbols[readZigZag(reader)];
-	else if (typeof type === "object" && "logicalType" in type && type.logicalType) {
+	} else if (typeof type === "object" && type.type === "enum") {
+		const index = readZigZag(reader);
+		reader.budget?.bound(index, type.symbols.length - 1, "enumIndex");
+		return type.symbols[index];
+	} else if (typeof type === "object" && "logicalType" in type && type.logicalType) {
 		if (type.logicalType === "date" && type.type === "int") {
 			const value = readZigZag(reader);
 			return /* @__PURE__ */ new Date(value * 864e5);
@@ -103,12 +130,12 @@ function readType(reader, type) {
 		return value;
 	} else if (type === "bytes") {
 		const length = readZigZag(reader);
-		const bytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, length);
+		const bytes = boundedBytes(reader, length);
 		reader.offset += length;
 		return bytes;
 	} else if (type === "string") {
 		const length = readZigZag(reader);
-		const bytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, length);
+		const bytes = boundedBytes(reader, length);
 		const text = textDecoder.decode(bytes);
 		reader.offset += length;
 		return text;
@@ -116,7 +143,7 @@ function readType(reader, type) {
 	else throw new Error(`unsupported type: ${JSON.stringify(type)}`);
 }
 function readFixed(reader, size) {
-	const bytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, size);
+	const bytes = boundedBytes(reader, size);
 	reader.offset += size;
 	return bytes;
 }
@@ -128,39 +155,53 @@ function bytesToUuid(bytes) {
 	}
 	return hex;
 }
-function compileType(type) {
+function compileType(type, budget) {
+	const decode = compileTypeValue(type, budget);
+	return budget ? (reader) => {
+		budget.values(1);
+		return decode(reader);
+	} : decode;
+}
+function compileTypeValue(type, budget) {
 	if (type === "null") return () => void 0;
 	if (type === "int") return readZigZag;
 	if (type === "long") return readZigZagBigInt;
 	if (type === "string" || type === "bytes") return (reader) => {
 		const length = readZigZag(reader);
-		const bytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, length);
+		const bytes = boundedBytes(reader, length);
 		reader.offset += length;
 		return type === "string" ? textDecoder.decode(bytes) : bytes;
 	};
 	if (Array.isArray(type)) {
-		const readers = type.map(compileType);
-		return (reader) => readers[readZigZag(reader)](reader);
+		const readers = type.map((item) => compileType(item, budget));
+		return (reader) => {
+			const index = readZigZag(reader);
+			budget?.bound(index, readers.length - 1, "unionIndex");
+			return readers[index](reader);
+		};
 	}
 	if (typeof type === "object" && type.type === "array") {
-		const item = compileType(type.items);
+		const item = compileType(type.items, budget);
 		return (reader) => {
 			const arr = [];
 			while (true) {
 				let count = readZigZag(reader);
 				if (count === 0) return arr;
+				let end;
 				if (count < 0) {
 					count = -count;
-					readZigZag(reader);
+					end = readBlockEnd(reader);
 				}
+				budget?.availableValues(count);
 				for (let i = 0; i < count; i++) arr.push(item(reader));
+				checkBlockEnd(reader, end);
 			}
 		};
 	}
 	if (typeof type === "object" && type.type === "record") {
 		const fields = type.fields.map((field) => ({
 			name: field.name,
-			read: compileType(field.type)
+			read: compileType(field.type, budget)
 		}));
 		if (fields.length === 2) {
 			const [{ name: firstName, read: firstRead }, { name: secondName, read: secondRead }] = fields;
@@ -177,6 +218,6 @@ function compileType(type) {
 			return obj;
 		};
 	}
-	return (reader) => readType(reader, type);
+	return (reader) => readTypeValue(reader, type);
 }
 export { avroRead };

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f → icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471}/node_modules/icebird/src/fetch.mjs

@@ -1,13 +1,13 @@
+import { avroMetadata } from "./avro/avro.metadata.mjs";
+import { compressors } from "../../../../hyparquet-compressors@1.1.2_patch_hash=3f8f44805c034b2563763dcd5d37050ea6b1d5d45b46ea37adc4a721e748a1c5/node_modules/hyparquet-compressors/src/compressors.mjs";
+import "../../../../hyparquet-compressors@1.1.2_patch_hash=3f8f44805c034b2563763dcd5d37050ea6b1d5d45b46ea37adc4a721e748a1c5/node_modules/hyparquet-compressors/src/index.mjs";
+import { avroRead } from "./avro/avro.read.mjs";
 import { parquetMetadataAsync } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/metadata.mjs";
 import { asyncBufferFromUrl, cachedAsyncBuffer } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/utils.mjs";
 import { parquetReadObjects } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/read.mjs";
 import "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/node.mjs";
-import { compressors } from "../../../../hyparquet-compressors@1.1.2/node_modules/hyparquet-compressors/src/compressors.mjs";
-import "../../../../hyparquet-compressors@1.1.2/node_modules/hyparquet-compressors/src/index.mjs";
 import { ByteWriter } from "../../../../hyparquet-writer@0.16.10_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/bytewriter.mjs";
 import "../../../../hyparquet-writer@0.16.10_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/node.mjs";
-import { avroMetadata } from "./avro/avro.metadata.mjs";
-import { avroRead } from "./avro/avro.read.mjs";
 import { sanitize } from "./utils.mjs";
 import { isDeletionVector } from "./delete.mjs";
 import { puffinReadDeletionVector } from "./puffin/puffin.mjs";
@@ -68,40 +68,6 @@ function urlResolver({ requestInit } = {}) {
 		}
 	};
 }
-function cachingResolver(base) {
-	const cache = /* @__PURE__ */ new Map();
-	const out = { reader(path, byteLength) {
-		let buf = cache.get(path);
-		if (!buf) {
-			buf = (async () => cachedAsyncBuffer(await base.reader(path, byteLength)))();
-			cache.set(path, buf);
-			buf.catch(() => {
-				if (cache.get(path) === buf) cache.delete(path);
-			});
-		}
-		return buf;
-	} };
-	if (base.writer) {
-		const baseWriter = base.writer;
-		out.writer = (path, options) => {
-			const w = baseWriter(path, options);
-			const origFinish = w.finish.bind(w);
-			w.finish = async function() {
-				await origFinish();
-				cache.delete(path);
-			};
-			return w;
-		};
-	}
-	if (base.deleter) {
-		const baseDeleter = base.deleter;
-		out.deleter = async (path) => {
-			await baseDeleter(path);
-			cache.delete(path);
-		};
-	}
-	return out;
-}
 function s3Lister({ requestInit } = {}) {
 	return async function list(url) {
 		const s3parts = s3ParseUrl(url);
@@ -251,7 +217,8 @@ async function decompressGzip(buf) {
 	const stream = new Blob([buf]).stream().pipeThrough(new DecompressionStream("gzip"));
 	return await new Response(stream).arrayBuffer();
 }
-async function fetchAvroRecords(url, resolver, byteLength) {
+async function fetchAvroRecords(url, resolver, byteLength, budget, recordKind) {
+	if (budget) return (await fetchBoundedAvroObject(url, resolver, byteLength, budget, recordKind)).records;
 	const lengthHint = byteLength !== void 0 && Number.isFinite(byteLength) ? byteLength : void 0;
 	const ab = await resolver.reader(url, lengthHint);
 	const buffer = await ab.slice(0, ab.byteLength);
@@ -266,4 +233,36 @@ async function fetchAvroRecords(url, resolver, byteLength) {
 		syncMarker
 	});
 }
-export { cachingResolver, fetchAvroRecords, fetchDeleteMaps, readParquetMetadata, resolveText, s3Lister, s3ParseUrl, translateS3Url, urlResolver };
+async function fetchBoundedAvroObject(url, resolver, byteLength, budget, recordKind = "entries") {
+	try {
+		budget.check();
+		if (byteLength !== void 0) budget.bound(byteLength, budget.limits.maxObjectBytes, "lengthHint");
+		const ab = await resolver.reader(url, byteLength);
+		budget.check();
+		const length = ab.byteLength;
+		budget.object(length);
+		if (byteLength !== void 0 && byteLength !== length) throw new Error("Manifest object length changed");
+		const buffer = await ab.slice(0, length);
+		budget.check();
+		if (buffer.byteLength !== length) throw new Error("Manifest object range length changed");
+		const reader = {
+			view: new DataView(buffer),
+			offset: 0,
+			budget
+		};
+		const { metadata, syncMarker } = avroMetadata(reader);
+		return {
+			metadata,
+			records: avroRead({
+				reader,
+				metadata,
+				syncMarker,
+				recordKind
+			}),
+			byteLength: length
+		};
+	} catch (error) {
+		return budget.fail(error);
+	}
+}
+export { fetchAvroRecords, fetchBoundedAvroObject, fetchDeleteMaps, readParquetMetadata, resolveText, s3Lister, s3ParseUrl, translateS3Url, urlResolver };

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f → icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471}/node_modules/icebird/src/manifest.mjs

@@ -1,6 +1,7 @@
+import { createAvroBudget } from "./avro/avro.budget.mjs";
 import { avroMetadata } from "./avro/avro.metadata.mjs";
 import { avroRead } from "./avro/avro.read.mjs";
-import { fetchAvroRecords, urlResolver } from "./fetch.mjs";
+import { fetchAvroRecords, fetchBoundedAvroObject, urlResolver } from "./fetch.mjs";
 const MANIFEST_FETCH_CONCURRENCY = 8;
 function memoRead(cache, key, read, retain = true) {
 	if (!cache) return read();
@@ -15,7 +16,10 @@ function memoRead(cache, key, read, retain = true) {
 	});
 	return pending;
 }
-async function icebergManifests({ metadata, resolver, snapshotId, partitionFilter, manifestCache }) {
+async function icebergManifests({ metadata, resolver, snapshotId, partitionFilter, manifestCache, readLimits }) {
+	const budget = createAvroBudget(readLimits);
+	if (budget && manifestCache !== void 0) throw new TypeError("Bounded manifest reads require an uncached walk");
+	const inlineEntries = budget ? /* @__PURE__ */ new Map() : void 0;
 	resolver ??= urlResolver();
 	const rawTarget = snapshotId ?? metadata["current-snapshot-id"];
 	if (rawTarget == null || rawTarget < 0) throw new Error("No current snapshot id found in table metadata");
@@ -25,8 +29,8 @@ async function icebergManifests({ metadata, resolver, snapshotId, partitionFilte
 	let manifests;
 	if (snapshot["manifest-list"]) {
 		const manifestListUrl = snapshot["manifest-list"];
-		manifests = await memoRead(manifestCache?.lists, manifestListUrl, () => fetchAvroRecords(manifestListUrl, resolver));
-	} else if (snapshot.manifests) manifests = await memoRead(manifestCache?.lists, `inline:${targetId}`, () => resolveInlineManifests(snapshot, resolver));
+		manifests = await memoRead(manifestCache?.lists, manifestListUrl, () => fetchAvroRecords(manifestListUrl, resolver, void 0, budget, "manifests"));
+	} else if (snapshot.manifests) manifests = await memoRead(manifestCache?.lists, `inline:${targetId}`, () => resolveInlineManifests(snapshot, resolver, budget, inlineEntries));
 	else throw new Error("No manifest information found in snapshot");
 	if (partitionFilter) manifests = manifests.filter((manifest) => {
 		try {
@@ -35,22 +39,35 @@ async function icebergManifests({ metadata, resolver, snapshotId, partitionFilte
 			return true;
 		}
 	});
-	return await fetchManifests(manifests, resolver, manifestCache);
+	return await fetchManifests(manifests, resolver, manifestCache, budget, inlineEntries);
 }
-async function resolveInlineManifests(snapshot, resolver) {
-	return await mapWithConcurrency(snapshot.manifests ?? [], async (manifestPath) => {
-		const ab = await resolver.reader(manifestPath);
-		const buffer = await ab.slice(0, ab.byteLength);
-		const reader = {
-			view: new DataView(buffer),
-			offset: 0
-		};
-		const { metadata, syncMarker } = await avroMetadata(reader);
-		const entries = await avroRead({
-			reader,
-			metadata,
-			syncMarker
-		});
+async function resolveInlineManifests(snapshot, resolver, budget, inlineEntries) {
+	const inline = snapshot.manifests ?? [];
+	budget?.records(inline.length, "manifests");
+	return await mapWithConcurrency(inline, async (manifestPath) => {
+		budget?.check();
+		let metadata, entries, byteLength;
+		if (budget) {
+			const object = await fetchBoundedAvroObject(manifestPath, resolver, void 0, budget);
+			metadata = object.metadata;
+			entries = object.records;
+			byteLength = object.byteLength;
+			inlineEntries.set(manifestPath, entries);
+		} else {
+			const ab = await resolver.reader(manifestPath);
+			const buffer = await ab.slice(0, ab.byteLength);
+			const reader = {
+				view: new DataView(buffer),
+				offset: 0
+			};
+			const header = await avroMetadata(reader);
+			metadata = header.metadata;
+			byteLength = ab.byteLength;
+			entries = await avroRead({
+				reader,
+				...header
+			});
+		}
 		const counts = [
 			0,
 			0,
@@ -67,7 +84,7 @@ async function resolveInlineManifests(snapshot, resolver) {
 		}
 		return {
 			manifest_path: manifestPath,
-			manifest_length: BigInt(ab.byteLength),
+			manifest_length: BigInt(byteLength),
 			partition_spec_id: Number(metadata["partition-spec-id"] ?? 0),
 			content: 0,
 			added_snapshot_id: BigInt(snapshot["snapshot-id"]),
@@ -78,11 +95,12 @@ async function resolveInlineManifests(snapshot, resolver) {
 			existing_rows_count: rows[0],
 			deleted_rows_count: rows[2]
 		};
-	});
+	}, budget);
 }
-async function fetchOneManifest(manifest, resolver) {
+async function fetchOneManifest(manifest, resolver, budget, inlineEntries) {
+	budget?.check();
 	const url = manifest.manifest_path;
-	const entries = await fetchAvroRecords(url, resolver, Number(manifest.manifest_length));
+	const entries = inlineEntries?.get(url) ?? await fetchAvroRecords(url, resolver, Number(manifest.manifest_length), budget, "entries");
 	for (const entry of entries) {
 		entry.partition_spec_id = manifest.partition_spec_id ?? 0;
 		if (entry.snapshot_id == null) entry.snapshot_id = manifest.added_snapshot_id;
@@ -98,16 +116,20 @@ async function fetchOneManifest(manifest, resolver) {
 		entries
 	};
 }
-async function fetchManifests(manifests, resolver, manifestCache) {
-	return await mapWithConcurrency(manifests, (manifest) => memoRead(manifestCache?.entries, manifest.manifest_path, () => fetchOneManifest(manifest, resolver), false));
+async function fetchManifests(manifests, resolver, manifestCache, budget, inlineEntries) {
+	return await mapWithConcurrency(manifests, (manifest) => memoRead(manifestCache?.entries, manifest.manifest_path, () => fetchOneManifest(manifest, resolver, budget, inlineEntries), false), budget);
 }
-async function mapWithConcurrency(items, fn) {
+async function mapWithConcurrency(items, fn, budget) {
 	const results = new Array(items.length);
 	let next = 0;
 	async function worker() {
-		while (next < items.length) {
+		while (next < items.length) try {
+			budget?.check();
 			const index = next++;
 			results[index] = await fn(items[index]);
+		} catch (error) {
+			if (budget) return budget.fail(error);
+			throw error;
 		}
 	}
 	const poolSize = Math.min(MANIFEST_FETCH_CONCURRENCY, items.length);

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f → icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471}/node_modules/icebird/src/read.mjs

@@ -1,3 +1,5 @@
+import { compressors } from "../../../../hyparquet-compressors@1.1.2_patch_hash=3f8f44805c034b2563763dcd5d37050ea6b1d5d45b46ea37adc4a721e748a1c5/node_modules/hyparquet-compressors/src/compressors.mjs";
+import "../../../../hyparquet-compressors@1.1.2_patch_hash=3f8f44805c034b2563763dcd5d37050ea6b1d5d45b46ea37adc4a721e748a1c5/node_modules/hyparquet-compressors/src/index.mjs";
 import { isListLike, isMapLike } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/schema.mjs";
 import { parquetSchema } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/metadata.mjs";
 import { cachedAsyncBuffer } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/utils.mjs";
@@ -5,8 +7,6 @@ import { columnsNeededForFilter, matchFilter } from "../../../../hyparquet@1.31.
 import { parquetScan } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/scan.mjs";
 import { parquetReadObjects } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/read.mjs";
 import "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/node.mjs";
-import { compressors } from "../../../../hyparquet-compressors@1.1.2/node_modules/hyparquet-compressors/src/compressors.mjs";
-import "../../../../hyparquet-compressors@1.1.2/node_modules/hyparquet-compressors/src/index.mjs";
 import { equalityMatch, sanitize } from "./utils.mjs";
 import { applicablePositionDeletes, deleteFileAppliesToDataEntry } from "./delete.mjs";
 import { readParquetMetadata } from "./fetch.mjs";

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/fetch.d.mts

@@ -0,0 +1,2 @@
+import "../../../../hyparquet@1.31.1/node_modules/hyparquet/types/node.mjs";
+import "../src/types.mjs";

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f → icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471}/node_modules/icebird/types/index.d.mts

@@ -1,9 +1,9 @@
 import { icebergManifests } from "./manifest.mjs";
 import { icebergAppend, icebergCreateTable, icebergDropTable } from "./write/write.mjs";
 import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./catalog/rest.mjs";
-import { cachingResolver } from "./fetch.mjs";
 import "./create.mjs";
 import "./catalog/file.mjs";
+import "./fetch.mjs";
 import { s3SignedResolver } from "./s3.mjs";
 import "../../../../squirreling@0.16.6/node_modules/squirreling/src/index.mjs";
 import "./read.mjs";

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f → icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471}/node_modules/icebird/types/manifest.d.mts

@@ -28,10 +28,26 @@ export type ManifestPartitionFilter = (partitions: Manifest['partitions'], parti
  * @param {number | bigint} [options.snapshotId] - Optional snapshot id; defaults to `current-snapshot-id`.
  * @returns {Promise<ManifestList>}
  */
-export declare function icebergManifests({ metadata, resolver, snapshotId, partitionFilter, manifestCache }: {
+export declare function icebergManifests({ metadata, resolver, snapshotId, partitionFilter, manifestCache, readLimits }: {
   metadata: TableMetadata;
   resolver?: Resolver;
   snapshotId?: number | bigint;
   partitionFilter?: ManifestPartitionFilter;
   manifestCache?: ManifestReadCache;
-}): Promise<ManifestList>;
+  /** Strict uncached Avro read admission. Omitted preserves existing behavior. */
+  readLimits?: ManifestReadLimits;
+}): Promise<ManifestList>;
+/** Per-walk source, expansion and materialization-work bounds; not total JS heap. */
+export interface ManifestReadLimits {
+  maxObjectBytes: number;
+  maxTotalObjectBytes: number;
+  maxManifests: number;
+  maxEntries: number;
+  maxInflatedBlockBytes: number;
+  maxTotalInflatedBytes: number;
+  maxValueBytes: number;
+  maxDecodedValues: number;
+  maxHeaderBytes: number;
+  /** Positive integer no greater than 128. */
+  maxSchemaDepth: number;
+}

package/dist/catalog.d.mts

@@ -1,10 +1,11 @@
-import { ManifestReadCache } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/types/manifest.mjs";
-import { icebergAppend } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/types/write/write.mjs";
+import { ManifestReadCache, ManifestReadLimits } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/manifest.mjs";
+import { icebergAppend } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/write/write.mjs";
 import { CatalogCache } from "./catalog-cache.mjs";
+import "./manifest-read-limits.mjs";
 import { IcebergPrimitiveType, IcebergS3Config } from "./schema.mjs";
 import { PartitionValueMatch } from "./partition-prune.mjs";
-import { restCatalogConnect, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/types/catalog/rest.mjs";
-import { cachingResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/types/fetch.mjs";
+import { restCatalogConnect, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/catalog/rest.mjs";
+import { Resolver } from "icebird/src/types.js";
 /** A field in an icebird table `Schema`. */
 export interface IcebergSchemaField {
   id: number;
@@ -61,7 +62,7 @@ export interface IcebergConnection {
   /** icebird REST catalog context, passed as `{ catalog }` to icebird write fns. */
   catalog: Awaited<ReturnType<typeof restCatalogConnect>>;
   /** icebird S3 resolver (caching-wrapped), passed as `{ resolver }` to icebird fns. */
-  resolver: ReturnType<typeof cachingResolver>;
+  resolver: Resolver;
   /** The namespace the dataset's table lives under. */
   namespace: string;
   /**
@@ -259,6 +260,13 @@ export interface ResolveIcebergDataFilesOptions {
   cache?: CatalogCache;
   clock?: () => number;
   profiler?: QueryProfiler;
+  /**
+   * Opt-in Avro source/decode-work limits for one uncached manifest walk.
+   * Bypasses decoded manifest, month and resolved-file caches; catalog metadata
+   * and snapshot caching keep their existing authority and are not byte-bounded
+   * by this option. No default limits or fallback to unbounded decoding.
+   */
+  readLimits?: ManifestReadLimits;
 }
 /**
  * Drop the cached snapshot pointer for `(namespace, table)` so the next read

package/dist/catalog.mjs

@@ -1,12 +1,13 @@
 import { stringifyBigintSafe } from "./bigint.mjs";
-import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/catalog/rest.mjs";
-import { cachingResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/fetch.mjs";
-import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/manifest.mjs";
-import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/s3.mjs";
+import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/catalog/rest.mjs";
+import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/manifest.mjs";
+import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/s3.mjs";
 import { deriveAppendIdentity } from "./append-identity.mjs";
 import { cacheGet, cachePut, reportCatalogCacheError } from "./catalog-cache.mjs";
 import { toIcebergDayCount } from "./date.mjs";
+import { captureManifestReadLimits } from "./manifest-read-limits.mjs";
 import { MULTI_MONTH_MANIFEST, buildManifestPartitionFilter, manifestMonthBucket } from "./partition-prune.mjs";
+import { byteCappedCachingResolver } from "./resolver-byte-cache.mjs";
 const connectionReadBatch = Symbol("iceberg-connection-read-batch");
 function beginConnectionRead(conn) {
 	const batch = conn[connectionReadBatch];
@@ -52,7 +53,7 @@ function startConnectSpan(profiler, name) {
 }
 let icebirdWriteModule;
 function useIcebirdWriteModule() {
-	icebirdWriteModule ??= import("./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/write/write.mjs");
+	icebirdWriteModule ??= import("./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/write/write.mjs");
 	return icebirdWriteModule;
 }
 const CATALOG_CONFIG_TTL_MS = 36e5;
@@ -115,7 +116,7 @@ async function connectIcebergCatalog(config, opts = {}) {
 		endpoint: config.s3.endpoint,
 		pathStyle: true
 	});
-	const resolver = withVerifiedWriterByteLengths(cachingResolver(s3Resolver));
+	const resolver = withUncachedReader(withVerifiedWriterByteLengths(byteCappedCachingResolver(s3Resolver)), s3Resolver);
 	return {
 		catalog,
 		resolver,
@@ -128,6 +129,17 @@ async function connectIcebergCatalog(config, opts = {}) {
 		}
 	};
 }
+const uncachedReader = Symbol("iceberg-uncached-reader");
+function withUncachedReader(resolver, base) {
+	return {
+		...resolver,
+		[uncachedReader]: (path, byteLength) => base.reader(path, byteLength)
+	};
+}
+function boundedReadResolver(resolver) {
+	const read = resolver[uncachedReader];
+	return read ? { reader: read } : resolver;
+}
 function withVerifiedWriterByteLengths(resolver) {
 	if (!resolver.writer) return resolver;
 	const baseWriter = resolver.writer;
@@ -643,12 +655,14 @@ function toListedFile(entry, matches, monthFieldName, wantedMonths, dateFieldId)
 		...dateBounds ? { dateBounds } : {}
 	};
 }
-async function resolveViaFullWalk(conn, metadata, partitionFilter, matches, monthFieldName, wantedMonths, dateFieldId, manifestCache) {
+async function resolveViaFullWalk(conn, metadata, partitionFilter, matches, monthFieldName, wantedMonths, dateFieldId, manifestCache, readLimits) {
+	const resolver = readLimits ? boundedReadResolver(conn.resolver) : conn.resolver;
 	const manifests = await icebergManifests({
 		metadata,
-		resolver: conn.resolver,
+		resolver,
 		partitionFilter,
-		manifestCache
+		manifestCache,
+		readLimits
 	});
 	const entries = [];
 	for (const m of manifests) for (const entry of m.entries) {
@@ -745,6 +759,7 @@ function resolveIcebergDataFiles(conn, opts) {
 	return resolveIcebergDataFilesInBatch(conn, opts).finally(release);
 }
 async function resolveIcebergDataFilesInBatch(conn, opts) {
+	const readLimits = captureManifestReadLimits(opts.readLimits);
 	const { namespace, table } = opts;
 	const profiler = opts.profiler;
 	const now = (opts.clock ?? Date.now)();
@@ -756,7 +771,7 @@ async function resolveIcebergDataFilesInBatch(conn, opts) {
 	if (snapshotId == null) return [];
 	const scope = conn.cacheScope ?? "";
 	const filesKey = resolvedFilesKey(scope, namespace, table, snapshotId, opts.matches, opts.range);
-	if (opts.cache) {
+	if (opts.cache && !readLimits) {
 		const endCache = profiler?.start("iceberg.cache");
 		const cached = await cacheGet(opts.cache, filesKey, now);
 		endCache?.({ hit: cached !== void 0 });
@@ -778,8 +793,8 @@ async function resolveIcebergDataFilesInBatch(conn, opts) {
 	const monthFieldName = monthField?.name;
 	const dateFieldId = dateColumnFieldId(metadata, monthField?.sourceColumn);
 	const dayWindow = rangeDayWindow(opts.range);
-	const manifestCache = manifestReadCacheFor(conn, namespace, table, metadata);
-	const walk = opts.cache && monthFieldName ? await resolveViaMonthCache(conn, opts.cache, metadata, partitionFilter, opts.partitionSpec, opts.matches, monthFieldName, wantedMonths, dateFieldId, scope, namespace, table, now, manifestCache) : await resolveViaFullWalk(conn, metadata, partitionFilter, opts.matches, monthFieldName, wantedMonths, dateFieldId, manifestCache);
+	const manifestCache = readLimits ? void 0 : manifestReadCacheFor(conn, namespace, table, metadata);
+	const walk = opts.cache && monthFieldName && !readLimits ? await resolveViaMonthCache(conn, opts.cache, metadata, partitionFilter, opts.partitionSpec, opts.matches, monthFieldName, wantedMonths, dateFieldId, scope, namespace, table, now, manifestCache) : await resolveViaFullWalk(conn, metadata, partitionFilter, opts.matches, monthFieldName, wantedMonths, dateFieldId, manifestCache, readLimits);
 	let boundsPruned = 0;
 	const out = [];
 	for (const file of walk.entries) {
@@ -798,7 +813,7 @@ async function resolveIcebergDataFilesInBatch(conn, opts) {
 		monthsHit: walk.monthsHit,
 		manifestsWalked: walk.manifestsWalked
 	});
-	if (opts.cache) {
+	if (opts.cache && !readLimits) {
 		const freshKey = resolvedFilesKey(scope, namespace, table, snapshotId, opts.matches, opts.range);
 		await cachePut(opts.cache, freshKey, out, RESOLVED_FILES_TTL_MS, now);
 	}