@gscdump/lakehouse 7.5.1 → 7.7.0

package/README.md

@@ -45,6 +45,74 @@ For retry decisions, import `isCommitRateLimited`, `isCommitServerError`, or
 Use the dataset registry to define datasets.
 Raw Icebird table creation and append functions live under `unsafe-raw` for package adapters and diagnostics.
 
+## Opt-in manifest read limits
+
+`icebergManifests` on `@gscdump/lakehouse/unsafe-raw` accepts an optional
+`readLimits: ManifestReadLimits`. Omission preserves the existing reader and
+caller-owned manifest cache behavior. No higher-level file resolver or host
+automatically enables these limits.
+
+```ts
+import { icebergManifests } from '@gscdump/lakehouse/unsafe-raw'
+
+const manifests = await icebergManifests({
+  metadata,
+  resolver,
+  readLimits: {
+    maxObjectBytes: 2 * 1024 * 1024,
+    maxTotalObjectBytes: 8 * 1024 * 1024,
+    maxManifests: 8,
+    maxEntries: 128,
+    maxInflatedBlockBytes: 2 * 1024 * 1024,
+    maxTotalInflatedBytes: 8 * 1024 * 1024,
+    maxValueBytes: 1024 * 1024,
+    maxDecodedValues: 100_000,
+    maxHeaderBytes: 64 * 1024,
+    maxSchemaDepth: 32,
+  },
+})
+```
+
+These are example admission values, not defaults or a total JavaScript heap
+guarantee. Every field is required and must be a positive safe integer;
+`maxSchemaDepth` must also be at most 128. The reader copies and validates the
+policy before its first asynchronous read. Unknown fields are rejected.
+
+The per-object and cumulative source-byte checks happen after `resolver.reader`
+reports `byteLength`, **before** the reader requests `slice(0, byteLength)`.
+Declared manifest lengths and returned slice lengths must match. An injected
+resolver can allocate or fetch inside `reader` itself; those operations are
+outside this boundary and need their own transport limits and immutable-object
+identity checks. No HEAD, catalog JSON, network-request, or freshness bound is
+implied by this option.
+
+Manifest-list counts are admitted before record materialization and pruning;
+entry counts accumulate across all fetched manifests and blocks. V1 inline
+manifest counts are admitted before reads, with each file decoded once in a
+private per-call cache. Supplying `manifestCache` with `readLimits` is rejected:
+previously decoded cache entries carry no verifiable budget provenance. Defaults
+retain their existing cache behavior.
+
+For Avro `null` and raw `deflate` blocks, decoded block bytes and their cumulative
+total are bounded. The existing pure-JavaScript inflater checks output growth
+before allocating or writing past that limit. Bounded DEFLATE returns an exact-size
+backing buffer so byte views do not retain spare inflater capacity; growth and
+final-copy buffers can temporarily overlap. Header framing is byte-bounded
+before JSON parsing; schema depth/work is checked before recursive compilation.
+Variable and fixed values are checked before views or UTF-8 decoding, and
+`maxDecodedValues` bounds decoding work across header/schema nodes and decoded
+value visits, including arrays of zero-byte nulls. Strict reads also reject
+invalid counts, varints, indexes, truncated values and inconsistent block sizes.
+
+These limits do not measure JavaScript object/string overhead, the JSON parser's
+internal allocations, inflater tables, caller-retained output, injected resolver
+caches or total process memory. Output remains an eager array within the chosen
+count/byte/work policy. A refusal rejects the complete call; there is no eager
+fallback or partial success. Already-dispatched resolver reads cannot be
+physically cancelled by the AsyncBuffer interface, but a first failure stops new
+dispatches and prevents late responses from being decoded. Choose compatible
+limits and validate the complete consumer path before enabling them.
+
 ## Append retries
 
 Supply a durable `commitRetry.appendId` when a dataset append belongs to a job or receipt.

package/dist/_virtual/node_modules/.pnpm/{hyparquet-compressors@1.1.2 → hyparquet-compressors@1.1.2_patch_hash=821c89dbb6181f0c34d9abebd844451458a4f16c04e7aa7a19abca0c09c2238d}/node_modules/hyparquet-compressors/src/gzip.mjs

@@ -39,12 +39,15 @@ function gzipStart(input, i) {
 	for (let zs = (flag >> 3 & 1) + (flag >> 4 & 1); zs > 0; zs -= Number(!input[i++]));
 	return i + (flag & 2);
 }
-function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
-	let out = output ?? /* @__PURE__ */ new Uint8Array(1024);
+function gunzip(input, output, inputIndex = 0, outputIndex = 0, limits) {
+	const maximum = limits?.maxOutputBytes;
+	if (limits && (limits.raw !== true || !Number.isSafeInteger(maximum) || maximum < 0 || output || inputIndex !== 0 || outputIndex !== 0)) throw new RangeError("Invalid bounded raw DEFLATE options");
+	let out = output ?? new Uint8Array(limits ? Math.min(1024, maximum) : 1024);
+	if (limits && !input.length) throw new Error("unexpected EOF");
 	if (!(input.length - inputIndex)) return out;
-	const payloadStart = gzipStart(input, inputIndex);
-	if (payloadStart === input.length - 8) return out;
-	if (payloadStart > input.length - 8) throw new Error("unexpected EOF");
+	const payloadStart = limits ? 0 : gzipStart(input, inputIndex);
+	if (!limits && payloadStart === input.length - 8) return out;
+	if (!limits && payloadStart > input.length - 8) throw new Error("unexpected EOF");
 	let pos = payloadStart * 8;
 	let final = 0;
 	let lengthBits = 0;
@@ -53,9 +56,10 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 	let distMap;
 	const totalBits = input.length * 8;
 	function ensureSize(length) {
+		if (limits && (!Number.isSafeInteger(length) || length > maximum)) throw new RangeError("DEFLATE output byte limit exceeded");
 		if (!output && length > out.length) {
 			const old = out;
-			out = new Uint8Array(Math.max(old.length * 2, length));
+			out = new Uint8Array(limits ? Math.min(maximum, Math.max(old.length * 2, length)) : Math.max(old.length * 2, length));
 			out.set(old);
 		}
 	}
@@ -67,6 +71,7 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 			if (!type) {
 				const s = shft(pos) + 4;
 				const l = input[s - 4] | input[s - 3] << 8;
+				if (limits && (s > input.length || ((input[s - 2] | input[s - 1] << 8) ^ l) !== 65535)) throw new Error("invalid stored block length");
 				const t = s + l;
 				if (t > input.length) throw new Error("unexpected EOF");
 				ensureSize(outputIndex + l);
@@ -93,6 +98,7 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 				const codeLengthMap = huffMap(codeLengthTree, codeLengthBits, 1);
 				for (let i = 0; i < tl;) {
 					const r = codeLengthMap[bits(input, pos, clbMask)];
+					if (limits && !r) throw new Error("invalid code length");
 					pos += r & 15;
 					const symbol = r >> 4;
 					if (symbol < 16) lengthDistanceTree[i++] = symbol;
@@ -100,6 +106,7 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 						let copy = 0;
 						let n = 0;
 						if (symbol === 16) {
+							if (limits && i === 0) throw new Error("invalid code length repeat");
 							n = 3 + bits(input, pos, 3);
 							pos += 2;
 							copy = lengthDistanceTree[i - 1];
@@ -110,6 +117,7 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 							n = 11 + bits(input, pos, 127);
 							pos += 7;
 						}
+						if (limits && i + n > tl) throw new Error("invalid code length repeat");
 						while (n--) lengthDistanceTree[i++] = copy;
 					}
 				}
@@ -122,7 +130,7 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 			} else throw new Error("invalid block type");
 			if (pos > totalBits) throw new Error("unexpected EOF");
 		}
-		ensureSize(outputIndex + 131072);
+		ensureSize(limits ? Math.min(maximum, outputIndex + 131072) : outputIndex + 131072);
 		const lms = (1 << lengthBits) - 1;
 		const dms = (1 << distBits) - 1;
 		for (;;) {
@@ -131,8 +139,10 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 			pos += code & 15;
 			if (pos > totalBits) throw new Error("unexpected EOF");
 			if (!code) throw new Error("invalid length/literal");
-			if (sym < 256) out[outputIndex++] = sym;
-			else if (sym === 256) {
+			if (sym < 256) {
+				if (limits) ensureSize(outputIndex + 1);
+				out[outputIndex++] = sym;
+			} else if (sym === 256) {
 				lengthMap = void 0;
 				break;
 			} else {
@@ -147,6 +157,7 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 				const d = distMap[bits16(input, pos) & dms];
 				const dsym = d >> 4;
 				if (!d) throw new Error("invalid distance");
+				if (limits && dsym > 29) throw new Error("invalid distance symbol");
 				pos += d & 15;
 				let dt = fixedDistance[dsym];
 				if (dsym > 3) {
@@ -163,10 +174,12 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 		}
 		if (lengthMap) final = 1;
 	} while (!final);
-	if (outputIndex < out.length) {
+	if (limits && Math.ceil(pos / 8) !== input.length) throw new Error("trailing raw DEFLATE bytes");
+	if (!limits && outputIndex < out.length) {
 		const nextBlock = Math.ceil(pos / 8) + 8;
 		gunzip(input, out, nextBlock, outputIndex);
 	}
+	if (limits && outputIndex !== out.length) return out.slice(0, outputIndex);
 	if (!output) return out.subarray(0, outputIndex);
 	return out;
 }

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/avro/avro.budget.mjs

@@ -0,0 +1,115 @@
+const fields = [
+	"maxObjectBytes",
+	"maxTotalObjectBytes",
+	"maxManifests",
+	"maxEntries",
+	"maxInflatedBlockBytes",
+	"maxTotalInflatedBytes",
+	"maxValueBytes",
+	"maxDecodedValues",
+	"maxHeaderBytes",
+	"maxSchemaDepth"
+];
+function createAvroBudget(options) {
+	if (options === void 0) return void 0;
+	if (!options || typeof options !== "object" || Array.isArray(options)) throw new TypeError("Invalid manifest read limits");
+	const limits = {};
+	for (const key of Object.keys(options)) if (!fields.includes(key)) throw new TypeError(`Unknown manifest read limit: ${key}`);
+	for (const key of fields) {
+		const value = options[key];
+		if (!Number.isSafeInteger(value) || value <= 0) throw new RangeError(`Invalid manifest read limit: ${key}`);
+		limits[key] = value;
+	}
+	if (limits.maxSchemaDepth > 128) throw new RangeError("Manifest schema depth must not exceed 128");
+	Object.freeze(limits);
+	const used = {
+		objectBytes: 0,
+		inflatedBytes: 0,
+		manifests: 0,
+		entries: 0,
+		values: 0
+	};
+	let failed = false, failure;
+	const check = () => {
+		if (failed) throw failure;
+	};
+	const fail = (error) => {
+		if (!failed) {
+			failed = true;
+			failure = error;
+		}
+		throw failure;
+	};
+	const bound = (value, max, label) => {
+		check();
+		if (!Number.isSafeInteger(value) || value < 0 || value > max) fail(/* @__PURE__ */ new RangeError(`Manifest read limit exceeded: ${label}`));
+	};
+	const charge = (key, value, max) => {
+		bound(value, max - used[key], key);
+		used[key] += value;
+	};
+	return Object.freeze({
+		limits,
+		check,
+		fail,
+		bound,
+		object(bytes) {
+			bound(bytes, limits.maxObjectBytes, "objectBytes");
+			charge("objectBytes", bytes, limits.maxTotalObjectBytes);
+		},
+		records(count, kind) {
+			charge(kind, count, kind === "manifests" ? limits.maxManifests : limits.maxEntries);
+		},
+		values(count) {
+			charge("values", count, limits.maxDecodedValues);
+		},
+		availableValues(count) {
+			bound(count, limits.maxDecodedValues - used.values, "values");
+		},
+		inflateLimit() {
+			check();
+			return Math.min(limits.maxInflatedBlockBytes, limits.maxTotalInflatedBytes - used.inflatedBytes);
+		},
+		inflated(bytes) {
+			bound(bytes, limits.maxInflatedBlockBytes, "inflatedBlockBytes");
+			charge("inflatedBytes", bytes, limits.maxTotalInflatedBytes);
+		},
+		schema(value) {
+			const stack = [{
+				value,
+				depth: 0
+			}];
+			while (stack.length) {
+				const next = stack.pop();
+				bound(next.depth, limits.maxSchemaDepth, "schemaDepth");
+				charge("values", 1, limits.maxDecodedValues);
+				if (next.value && typeof next.value === "object") {
+					const keys = Object.keys(next.value);
+					bound(stack.length + keys.length, limits.maxDecodedValues - used.values, "schemaValues");
+					for (const key of keys) stack.push({
+						value: next.value[key],
+						depth: next.depth + 1
+					});
+				}
+			}
+		}
+	});
+}
+function boundedBytes(reader, length, max) {
+	const budget = reader.budget;
+	budget?.bound(length, max ?? budget.limits.maxValueBytes, "valueBytes");
+	budget?.bound(reader.offset + length, reader.headerEnd ?? reader.view.byteLength, "encodedBytes");
+	return new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, length);
+}
+function boundedLong(reader) {
+	let value = 0n;
+	for (let shift = 0; shift < 70; shift += 7) {
+		reader.budget.bound(reader.offset + 1, reader.headerEnd ?? reader.view.byteLength, "varintBytes");
+		const byte = reader.view.getUint8(reader.offset++);
+		if (shift === 63 && byte > 1) reader.budget.fail(/* @__PURE__ */ new RangeError("Invalid Avro long"));
+		value |= BigInt(byte & 127) << BigInt(shift);
+		if (!(byte & 128)) return value >> 1n ^ -(value & 1n);
+	}
+	return reader.budget.fail(/* @__PURE__ */ new RangeError("Invalid Avro long"));
+}
+export { boundedBytes, boundedLong, createAvroBudget };

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/avro/avro.metadata.mjs

@@ -0,0 +1,97 @@
+import { boundedBytes, boundedLong } from "./avro.budget.mjs";
+const smallLongs = Array.from({ length: 128 }, (_, i) => BigInt(i >>> 1 ^ -(i & 1)));
+function readZigZag(reader) {
+	if (reader.budget) {
+		const value = Number(boundedLong(reader));
+		if (!Number.isSafeInteger(value)) reader.budget.fail(/* @__PURE__ */ new RangeError("Avro integer exceeds safe range"));
+		return value;
+	}
+	let result = 0;
+	let shift = 0;
+	while (true) {
+		const byte = reader.view.getUint8(reader.offset++);
+		result |= (byte & 127) << shift;
+		if (!(byte & 128)) return result >>> 1 ^ -(result & 1);
+		shift += 7;
+	}
+}
+function readZigZagBigInt(reader) {
+	if (reader.budget) return boundedLong(reader);
+	const first = reader.view.getUint8(reader.offset++);
+	if (first < 128) return smallLongs[first];
+	let prefix = first & 127;
+	for (let shift = 7; shift < 28; shift += 7) {
+		const byte = reader.view.getUint8(reader.offset++);
+		prefix |= (byte & 127) << shift;
+		if (!(byte & 128)) return BigInt(prefix >>> 1 ^ -(prefix & 1));
+	}
+	let high = 0;
+	let factor = 1;
+	while (true) {
+		const byte = reader.view.getUint8(reader.offset++);
+		high += (byte & 127) * factor;
+		if (!(byte & 128)) {
+			const result = BigInt(high) << 27n | BigInt(prefix >>> 1);
+			return prefix & 1 ? ~result : result;
+		}
+		factor *= 128;
+	}
+}
+function readBlockEnd(reader) {
+	const size = readZigZag(reader);
+	if (!reader.budget) return void 0;
+	reader.budget.bound(size, reader.view.byteLength, "containerBlockBytes");
+	const end = reader.offset + size;
+	reader.budget.bound(end, reader.headerEnd ?? reader.view.byteLength, "containerBlockEnd");
+	return end;
+}
+function checkBlockEnd(reader, end) {
+	if (end !== void 0 && reader.offset !== end) reader.budget.fail(/* @__PURE__ */ new RangeError("Avro container block length mismatch"));
+}
+function readAvroString(reader) {
+	const length = readZigZag(reader);
+	const bytes = boundedBytes(reader, length);
+	reader.budget?.values(1);
+	reader.offset += length;
+	return new TextDecoder().decode(bytes);
+}
+function avroMetadata(reader) {
+	if (reader.budget) reader.headerEnd = Math.min(reader.view.byteLength, reader.offset + reader.budget.limits.maxHeaderBytes);
+	reader.budget?.bound(reader.offset + 4, reader.headerEnd, "headerBytes");
+	if (reader.view.getUint32(reader.offset) !== 1331849729) throw new Error("avro invalid magic bytes");
+	reader.offset += 4;
+	const metadata = {};
+	let mapCount = readZigZag(reader);
+	while (mapCount !== 0) {
+		let end;
+		if (mapCount < 0) {
+			mapCount = -mapCount;
+			end = readBlockEnd(reader);
+		}
+		reader.budget?.availableValues(mapCount * 2);
+		for (let i = 0; i < mapCount; i++) {
+			const key = readAvroString(reader);
+			metadata[key] = readAvroString(reader);
+		}
+		checkBlockEnd(reader, end);
+		mapCount = readZigZag(reader);
+	}
+	metadata["avro.schema"] = JSON.parse(metadata["avro.schema"]);
+	reader.budget?.schema(metadata["avro.schema"]);
+	if (metadata["schema"]) {
+		metadata["schema"] = JSON.parse(metadata["schema"]);
+		reader.budget?.schema(metadata["schema"]);
+	}
+	if (metadata["iceberg.schema"]) {
+		metadata["iceberg.schema"] = JSON.parse(metadata["iceberg.schema"]);
+		reader.budget?.schema(metadata["iceberg.schema"]);
+	}
+	const syncMarker = boundedBytes(reader, 16, reader.budget?.limits.maxHeaderBytes);
+	reader.offset += 16;
+	delete reader.headerEnd;
+	return {
+		metadata,
+		syncMarker
+	};
+}
+export { avroMetadata, checkBlockEnd, readBlockEnd, readZigZag, readZigZagBigInt };

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f → icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471}/node_modules/icebird/src/avro/avro.read.mjs

@@ -1,36 +1,54 @@
 import { parseDecimal } from "../../../../../hyparquet@1.31.1/node_modules/hyparquet/src/convert.mjs";
-import { gunzip } from "../../../../../hyparquet-compressors@1.1.2/node_modules/hyparquet-compressors/src/gzip.mjs";
-import "../../../../../hyparquet-compressors@1.1.2/node_modules/hyparquet-compressors/src/index.mjs";
-import { readZigZag, readZigZagBigInt } from "./avro.metadata.mjs";
+import { gunzip } from "../../../../../hyparquet-compressors@1.1.2_patch_hash=821c89dbb6181f0c34d9abebd844451458a4f16c04e7aa7a19abca0c09c2238d/node_modules/hyparquet-compressors/src/gzip.mjs";
+import "../../../../../hyparquet-compressors@1.1.2_patch_hash=821c89dbb6181f0c34d9abebd844451458a4f16c04e7aa7a19abca0c09c2238d/node_modules/hyparquet-compressors/src/index.mjs";
+import { boundedBytes } from "./avro.budget.mjs";
+import { checkBlockEnd, readBlockEnd, readZigZag, readZigZagBigInt } from "./avro.metadata.mjs";
 const textDecoder = new TextDecoder();
-function avroRead({ reader, metadata, syncMarker }) {
-	const decode = compileType(metadata["avro.schema"]);
+function avroRead({ reader, metadata, syncMarker, recordKind = "entries" }) {
+	const budget = reader.budget;
+	const decode = compileType(metadata["avro.schema"], budget);
 	const blocks = [];
 	while (reader.offset < reader.view.byteLength) {
 		let recordCount = readZigZag(reader);
+		if (budget && recordCount <= 0) budget.fail(/* @__PURE__ */ new RangeError("Invalid Avro record count"));
 		if (recordCount === 0) break;
 		if (recordCount < 0) recordCount = -recordCount;
+		budget?.records(recordCount, recordKind);
+		budget?.availableValues(recordCount);
 		const blockSize = readZigZag(reader);
-		let data = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, blockSize);
+		let data = boundedBytes(reader, blockSize, budget?.limits.maxObjectBytes);
 		reader.offset += blockSize;
-		const blockSync = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, 16);
+		const blockSync = boundedBytes(reader, 16, 16);
 		reader.offset += 16;
 		for (let i = 0; i < 16; i++) if (blockSync[i] !== syncMarker[i]) throw new Error("sync marker does not match");
 		const codec = metadata["avro.codec"];
-		if (codec === "deflate") data = gunzip(data);
+		if (codec === "deflate") data = budget ? gunzip(data, void 0, 0, 0, {
+			raw: true,
+			maxOutputBytes: budget.inflateLimit()
+		}) : gunzip(data);
 		else if (codec !== "null") throw new Error(`unsupported codec: ${codec}`);
+		budget?.inflated(data.byteLength);
 		const dataReader = {
 			view: new DataView(data.buffer, data.byteOffset, data.byteLength),
-			offset: 0
+			offset: 0,
+			...budget ? { budget } : {}
 		};
 		for (let i = 0; i < recordCount; i++) blocks.push(decode(dataReader));
+		if (budget && dataReader.offset !== data.byteLength) budget.fail(/* @__PURE__ */ new RangeError("Avro block length mismatch"));
 	}
 	return blocks;
 }
 function readType(reader, type) {
+	reader.budget?.values(1);
+	return readTypeValue(reader, type);
+}
+function readTypeValue(reader, type) {
 	if (type === "null") return;
-	else if (Array.isArray(type)) return readType(reader, type[readZigZag(reader)]);
-	else if (typeof type === "object" && type.type === "record") {
+	else if (Array.isArray(type)) {
+		const unionIndex = readZigZag(reader);
+		reader.budget?.bound(unionIndex, type.length - 1, "unionIndex");
+		return readType(reader, type[unionIndex]);
+	} else if (typeof type === "object" && type.type === "record") {
 		const obj = {};
 		for (const subField of type.fields) obj[subField.name] = readType(reader, subField.type);
 		return obj;
@@ -39,11 +57,14 @@ function readType(reader, type) {
 		while (true) {
 			let count = readZigZag(reader);
 			if (count === 0) break;
+			let end;
 			if (count < 0) {
 				count = -count;
-				readZigZag(reader);
+				end = readBlockEnd(reader);
 			}
+			reader.budget?.availableValues(count);
 			for (let i = 0; i < count; i++) arr.push(readType(reader, type.items));
+			checkBlockEnd(reader, end);
 		}
 		return arr;
 	} else if (typeof type === "object" && type.type === "map") {
@@ -51,18 +72,24 @@ function readType(reader, type) {
 		while (true) {
 			let count = readZigZag(reader);
 			if (count === 0) break;
+			let end;
 			if (count < 0) {
 				count = -count;
-				readZigZag(reader);
+				end = readBlockEnd(reader);
 			}
+			reader.budget?.availableValues(count);
 			for (let i = 0; i < count; i++) {
 				const key = readType(reader, "string");
 				map[key] = readType(reader, type.values);
 			}
+			checkBlockEnd(reader, end);
 		}
 		return map;
-	} else if (typeof type === "object" && type.type === "enum") return type.symbols[readZigZag(reader)];
-	else if (typeof type === "object" && "logicalType" in type && type.logicalType) {
+	} else if (typeof type === "object" && type.type === "enum") {
+		const index = readZigZag(reader);
+		reader.budget?.bound(index, type.symbols.length - 1, "enumIndex");
+		return type.symbols[index];
+	} else if (typeof type === "object" && "logicalType" in type && type.logicalType) {
 		if (type.logicalType === "date" && type.type === "int") {
 			const value = readZigZag(reader);
 			return /* @__PURE__ */ new Date(value * 864e5);
@@ -103,12 +130,12 @@ function readType(reader, type) {
 		return value;
 	} else if (type === "bytes") {
 		const length = readZigZag(reader);
-		const bytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, length);
+		const bytes = boundedBytes(reader, length);
 		reader.offset += length;
 		return bytes;
 	} else if (type === "string") {
 		const length = readZigZag(reader);
-		const bytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, length);
+		const bytes = boundedBytes(reader, length);
 		const text = textDecoder.decode(bytes);
 		reader.offset += length;
 		return text;
@@ -116,7 +143,7 @@ function readType(reader, type) {
 	else throw new Error(`unsupported type: ${JSON.stringify(type)}`);
 }
 function readFixed(reader, size) {
-	const bytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, size);
+	const bytes = boundedBytes(reader, size);
 	reader.offset += size;
 	return bytes;
 }
@@ -128,39 +155,53 @@ function bytesToUuid(bytes) {
 	}
 	return hex;
 }
-function compileType(type) {
+function compileType(type, budget) {
+	const decode = compileTypeValue(type, budget);
+	return budget ? (reader) => {
+		budget.values(1);
+		return decode(reader);
+	} : decode;
+}
+function compileTypeValue(type, budget) {
 	if (type === "null") return () => void 0;
 	if (type === "int") return readZigZag;
 	if (type === "long") return readZigZagBigInt;
 	if (type === "string" || type === "bytes") return (reader) => {
 		const length = readZigZag(reader);
-		const bytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, length);
+		const bytes = boundedBytes(reader, length);
 		reader.offset += length;
 		return type === "string" ? textDecoder.decode(bytes) : bytes;
 	};
 	if (Array.isArray(type)) {
-		const readers = type.map(compileType);
-		return (reader) => readers[readZigZag(reader)](reader);
+		const readers = type.map((item) => compileType(item, budget));
+		return (reader) => {
+			const index = readZigZag(reader);
+			budget?.bound(index, readers.length - 1, "unionIndex");
+			return readers[index](reader);
+		};
 	}
 	if (typeof type === "object" && type.type === "array") {
-		const item = compileType(type.items);
+		const item = compileType(type.items, budget);
 		return (reader) => {
 			const arr = [];
 			while (true) {
 				let count = readZigZag(reader);
 				if (count === 0) return arr;
+				let end;
 				if (count < 0) {
 					count = -count;
-					readZigZag(reader);
+					end = readBlockEnd(reader);
 				}
+				budget?.availableValues(count);
 				for (let i = 0; i < count; i++) arr.push(item(reader));
+				checkBlockEnd(reader, end);
 			}
 		};
 	}
 	if (typeof type === "object" && type.type === "record") {
 		const fields = type.fields.map((field) => ({
 			name: field.name,
-			read: compileType(field.type)
+			read: compileType(field.type, budget)
 		}));
 		if (fields.length === 2) {
 			const [{ name: firstName, read: firstRead }, { name: secondName, read: secondRead }] = fields;
@@ -177,6 +218,6 @@ function compileType(type) {
 			return obj;
 		};
 	}
-	return (reader) => readType(reader, type);
+	return (reader) => readTypeValue(reader, type);
 }
 export { avroRead };

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f → icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471}/node_modules/icebird/src/fetch.mjs

@@ -2,8 +2,8 @@ import { parquetMetadataAsync } from "../../../../hyparquet@1.31.1/node_modules/
 import { asyncBufferFromUrl, cachedAsyncBuffer } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/utils.mjs";
 import { parquetReadObjects } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/read.mjs";
 import "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/node.mjs";
-import { compressors } from "../../../../hyparquet-compressors@1.1.2/node_modules/hyparquet-compressors/src/compressors.mjs";
-import "../../../../hyparquet-compressors@1.1.2/node_modules/hyparquet-compressors/src/index.mjs";
+import { compressors } from "../../../../hyparquet-compressors@1.1.2_patch_hash=821c89dbb6181f0c34d9abebd844451458a4f16c04e7aa7a19abca0c09c2238d/node_modules/hyparquet-compressors/src/compressors.mjs";
+import "../../../../hyparquet-compressors@1.1.2_patch_hash=821c89dbb6181f0c34d9abebd844451458a4f16c04e7aa7a19abca0c09c2238d/node_modules/hyparquet-compressors/src/index.mjs";
 import { ByteWriter } from "../../../../hyparquet-writer@0.16.10_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/bytewriter.mjs";
 import "../../../../hyparquet-writer@0.16.10_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/node.mjs";
 import { avroMetadata } from "./avro/avro.metadata.mjs";
@@ -251,7 +251,8 @@ async function decompressGzip(buf) {
 	const stream = new Blob([buf]).stream().pipeThrough(new DecompressionStream("gzip"));
 	return await new Response(stream).arrayBuffer();
 }
-async function fetchAvroRecords(url, resolver, byteLength) {
+async function fetchAvroRecords(url, resolver, byteLength, budget, recordKind) {
+	if (budget) return (await fetchBoundedAvroObject(url, resolver, byteLength, budget, recordKind)).records;
 	const lengthHint = byteLength !== void 0 && Number.isFinite(byteLength) ? byteLength : void 0;
 	const ab = await resolver.reader(url, lengthHint);
 	const buffer = await ab.slice(0, ab.byteLength);
@@ -266,4 +267,36 @@ async function fetchAvroRecords(url, resolver, byteLength) {
 		syncMarker
 	});
 }
-export { cachingResolver, fetchAvroRecords, fetchDeleteMaps, readParquetMetadata, resolveText, s3Lister, s3ParseUrl, translateS3Url, urlResolver };
+async function fetchBoundedAvroObject(url, resolver, byteLength, budget, recordKind = "entries") {
+	try {
+		budget.check();
+		if (byteLength !== void 0) budget.bound(byteLength, budget.limits.maxObjectBytes, "lengthHint");
+		const ab = await resolver.reader(url, byteLength);
+		budget.check();
+		const length = ab.byteLength;
+		budget.object(length);
+		if (byteLength !== void 0 && byteLength !== length) throw new Error("Manifest object length changed");
+		const buffer = await ab.slice(0, length);
+		budget.check();
+		if (buffer.byteLength !== length) throw new Error("Manifest object range length changed");
+		const reader = {
+			view: new DataView(buffer),
+			offset: 0,
+			budget
+		};
+		const { metadata, syncMarker } = avroMetadata(reader);
+		return {
+			metadata,
+			records: avroRead({
+				reader,
+				metadata,
+				syncMarker,
+				recordKind
+			}),
+			byteLength: length
+		};
+	} catch (error) {
+		return budget.fail(error);
+	}
+}
+export { cachingResolver, fetchAvroRecords, fetchBoundedAvroObject, fetchDeleteMaps, readParquetMetadata, resolveText, s3Lister, s3ParseUrl, translateS3Url, urlResolver };