@gscdump/lakehouse 7.7.0 → 7.8.0

package/README.md

@@ -49,8 +49,8 @@ Raw Icebird table creation and append functions live under `unsafe-raw` for pack
 
 `icebergManifests` on `@gscdump/lakehouse/unsafe-raw` accepts an optional
 `readLimits: ManifestReadLimits`. Omission preserves the existing reader and
-caller-owned manifest cache behavior. No higher-level file resolver or host
-automatically enables these limits.
+caller-owned manifest cache behavior. No file resolver or host automatically
+enables these limits.
 
 ```ts
 import { icebergManifests } from '@gscdump/lakehouse/unsafe-raw'
@@ -113,6 +113,41 @@ physically cancelled by the AsyncBuffer interface, but a first failure stops new
 dispatches and prevents late responses from being decoded. Choose compatible
 limits and validate the complete consumer path before enabling them.
 
+The same optional `readLimits` can be passed to `resolveIcebergDataFiles`,
+`dataset.resolveDataFiles(..., { readLimits })`, and the engine's
+`listIcebergDataFiles`. These wrappers capture and validate all ten values before
+catalog reads or lazy imports, then perform one bounded manifest walk. Per-object
+and aggregate limits remain independent upper bounds; a smaller aggregate limit
+does not expand the per-object allowance. Invalid, missing or unknown values are
+rejected even when the table has no current snapshot.
+
+In this opt-in mode, decoded manifest, resolved-file and month caches are neither
+read nor populated: old cache entries cannot establish compliance with a new
+policy. The existing catalog configuration, snapshot pointer and table metadata
+caches remain in use, with their existing identity/freshness rules. Catalog JSON
+and those metadata caches are outside the Avro allocation bound. Existing
+partition, Site, search-type, date pruning and output ordering are unchanged.
+
+This has an explicit warm-read cost. A default exact-range cache hit can avoid
+all Avro reads; a default month hit can need only the manifest list. A bounded
+call reads the list and every selected manifest again (subject to any injected
+resolver's own byte cache).
+
+A connection from `connectIcebergCatalog` keeps the objects it reads in a
+per-connection byte cache. That cache holds at most 16 MiB of fetched bytes.
+If a read takes it over 16 MiB, it evicts the least recently read object
+first. It does not keep an object larger than 16 MiB, and it does not keep a
+failed read. Concurrent reads of one path share one S3 request. A successful
+write or delete of a path evicts that path. A bounded call reads through the uncached
+S3 resolver instead, so the connection keeps none of its manifest bytes after
+the call. It also does not reuse bytes that earlier unbounded calls cached. If
+you replace `connection.resolver`, the bounded call uses your resolver, and your
+resolver owns its retention. Each concurrent bounded call owns its own limits;
+the option does not impose a connection-wide or process-wide budget. There is
+no automatic fallback to cached or unbounded results after refusal. Validate
+retained files and request costs before enabling a policy in a host; this does
+not bound a remote SQL provider's internal manifest planner.
+
 ## Append retries
 
 Supply a durable `commitRetry.appendId` when a dataset append belongs to a job or receipt.

package/dist/_virtual/node_modules/.pnpm/hyparquet-compressors@1.1.2_patch_hash=3f8f44805c034b2563763dcd5d37050ea6b1d5d45b46ea37adc4a721e748a1c5/node_modules/hyparquet-compressors/src/gzip.mjs

@@ -0,0 +1,196 @@
+import { fixedDistance, fixedDistanceExtraBits, fixedDistanceMap, fixedLength, fixedLengthExtraBits, fixedLengthMap, huffMap } from "./gzip.huffman.mjs";
+const codeLengthIndexMap = new Uint8Array([
+	16,
+	17,
+	18,
+	0,
+	8,
+	7,
+	9,
+	6,
+	10,
+	5,
+	11,
+	4,
+	12,
+	3,
+	13,
+	2,
+	14,
+	1,
+	15
+]);
+function bits(input, pos, mask) {
+	const o = pos / 8 | 0;
+	return (input[o] | input[o + 1] << 8) >> (pos & 7) & mask;
+}
+function bits16(d, p) {
+	const o = p / 8 | 0;
+	return (d[o] | d[o + 1] << 8 | d[o + 2] << 16) >> (p & 7);
+}
+function shft(p) {
+	return (p + 7) / 8 | 0;
+}
+function gzipStart(input, i) {
+	if (input[i++] !== 31 || input[i++] !== 139 || input[i++] !== 8) return 0;
+	const flag = input[i++];
+	if (flag & 224) throw new Error("invalid gzip header flags");
+	i += 6;
+	if (flag & 4) i += (input[i] | input[i + 1] << 8) + 2;
+	for (let zs = (flag >> 3 & 1) + (flag >> 4 & 1); zs > 0; zs -= Number(!input[i++]));
+	return i + (flag & 2);
+}
+function gunzip(input, output, inputIndex = 0, outputIndex = 0, limits) {
+	const maximum = limits?.maxOutputBytes;
+	if (limits && (limits.raw !== true || !Number.isSafeInteger(maximum) || maximum < 0 || output || inputIndex !== 0 || outputIndex !== 0)) throw new RangeError("Invalid bounded raw DEFLATE options");
+	let out = output ?? new Uint8Array(limits ? Math.min(1024, maximum) : 1024);
+	if (limits && !input.length) throw new Error("unexpected EOF");
+	if (!(input.length - inputIndex)) return out;
+	const totalBits = input.length * 8;
+	function ensureSize(length) {
+		if (limits && (!Number.isSafeInteger(length) || length > maximum)) throw new RangeError("DEFLATE output byte limit exceeded");
+		if (!output && length > out.length) {
+			const old = out;
+			out = new Uint8Array(limits ? Math.min(maximum, Math.max(old.length * 2, length)) : Math.max(old.length * 2, length));
+			out.set(old);
+		}
+	}
+	let memberStart = inputIndex;
+	for (;;) {
+		const payloadStart = limits ? 0 : gzipStart(input, memberStart);
+		const isGzip = !limits && payloadStart > 0;
+		if (isGzip && payloadStart === input.length - 8) break;
+		if (isGzip && payloadStart > input.length - 8) throw new Error("unexpected EOF");
+		const pos = inflate(payloadStart * 8);
+		if (limits && Math.ceil(pos / 8) !== input.length) throw new Error("trailing raw DEFLATE bytes");
+		if (!isGzip || output && outputIndex >= out.length) break;
+		const nextBlock = Math.ceil(pos / 8) + 8;
+		if (nextBlock > input.length) throw new Error("unexpected EOF");
+		if (nextBlock === input.length) break;
+		if (input[nextBlock] !== 31 || input[nextBlock + 1] !== 139 || input[nextBlock + 2] !== 8) throw new Error("invalid gzip member");
+		memberStart = nextBlock;
+	}
+	function inflate(pos) {
+		let final = 0;
+		let lengthBits = 0;
+		let distBits = 0;
+		let lengthMap;
+		let distMap;
+		do {
+			if (!lengthMap) {
+				final = bits(input, pos, 1);
+				const type = bits(input, pos + 1, 3);
+				pos += 3;
+				if (!type) {
+					const s = shft(pos) + 4;
+					const l = input[s - 4] | input[s - 3] << 8;
+					if (limits && (s > input.length || ((input[s - 2] | input[s - 1] << 8) ^ l) !== 65535)) throw new Error("invalid stored block length");
+					const t = s + l;
+					if (t > input.length) throw new Error("unexpected EOF");
+					ensureSize(outputIndex + l);
+					out.set(input.subarray(s, t), outputIndex);
+					outputIndex += l;
+					pos = t * 8;
+					continue;
+				} else if (type === 1) {
+					lengthMap = fixedLengthMap;
+					distMap = fixedDistanceMap;
+					lengthBits = 9;
+					distBits = 5;
+				} else if (type === 2) {
+					const hLiteral = bits(input, pos, 31) + 257;
+					const hcLengths = bits(input, pos + 10, 15) + 4;
+					const tl = hLiteral + bits(input, pos + 5, 31) + 1;
+					pos += 14;
+					const lengthDistanceTree = new Uint8Array(tl);
+					const codeLengthTree = /* @__PURE__ */ new Uint8Array(19);
+					for (let i = 0; i < hcLengths; ++i) codeLengthTree[codeLengthIndexMap[i]] = bits(input, pos + i * 3, 7);
+					pos += hcLengths * 3;
+					const codeLengthBits = Math.max(...codeLengthTree);
+					const clbMask = (1 << codeLengthBits) - 1;
+					const codeLengthMap = huffMap(codeLengthTree, codeLengthBits, 1);
+					for (let i = 0; i < tl;) {
+						const r = codeLengthMap[bits(input, pos, clbMask)];
+						if (limits && !r) throw new Error("invalid code length");
+						pos += r & 15;
+						const symbol = r >> 4;
+						if (symbol < 16) lengthDistanceTree[i++] = symbol;
+						else {
+							let copy = 0;
+							let n = 0;
+							if (symbol === 16) {
+								if (limits && i === 0) throw new Error("invalid code length repeat");
+								n = 3 + bits(input, pos, 3);
+								pos += 2;
+								copy = lengthDistanceTree[i - 1];
+							} else if (symbol === 17) {
+								n = 3 + bits(input, pos, 7);
+								pos += 3;
+							} else if (symbol === 18) {
+								n = 11 + bits(input, pos, 127);
+								pos += 7;
+							}
+							if (limits && i + n > tl) throw new Error("invalid code length repeat");
+							while (n--) lengthDistanceTree[i++] = copy;
+						}
+					}
+					const lengthTree = lengthDistanceTree.subarray(0, hLiteral);
+					const distanceTree = lengthDistanceTree.subarray(hLiteral);
+					lengthBits = Math.max(...lengthTree);
+					distBits = Math.max(...distanceTree);
+					lengthMap = huffMap(lengthTree, lengthBits, 1);
+					distMap = huffMap(distanceTree, distBits, 1);
+				} else throw new Error("invalid block type");
+				if (pos > totalBits) throw new Error("unexpected EOF");
+			}
+			ensureSize(limits ? Math.min(maximum, outputIndex + 131072) : outputIndex + 131072);
+			const lms = (1 << lengthBits) - 1;
+			const dms = (1 << distBits) - 1;
+			for (;;) {
+				const code = lengthMap[bits16(input, pos) & lms];
+				const sym = code >> 4;
+				pos += code & 15;
+				if (pos > totalBits) throw new Error("unexpected EOF");
+				if (!code) throw new Error("invalid length/literal");
+				if (sym < 256) {
+					if (limits) ensureSize(outputIndex + 1);
+					out[outputIndex++] = sym;
+				} else if (sym === 256) {
+					lengthMap = void 0;
+					break;
+				} else {
+					let add = sym - 254;
+					if (sym > 264) {
+						const index = sym - 257;
+						const b = fixedLengthExtraBits[index];
+						add = bits(input, pos, (1 << b) - 1) + fixedLength[index];
+						pos += b;
+					}
+					if (!distMap) throw new Error("invalid distance map");
+					const d = distMap[bits16(input, pos) & dms];
+					const dsym = d >> 4;
+					if (!d) throw new Error("invalid distance");
+					if (limits && dsym > 29) throw new Error("invalid distance symbol");
+					pos += d & 15;
+					let dt = fixedDistance[dsym];
+					if (dsym > 3) {
+						const b = fixedDistanceExtraBits[dsym];
+						dt += bits16(input, pos) & (1 << b) - 1;
+						pos += b;
+					}
+					if (pos > totalBits) throw new Error("unexpected EOF");
+					const end = outputIndex + add;
+					if (outputIndex < dt) throw new Error("unexpected dictionary case");
+					ensureSize(end);
+					for (; outputIndex < end; outputIndex++) out[outputIndex] = out[outputIndex - dt];
+				}
+			}
+			if (lengthMap) final = 1;
+		} while (!final);
+		return pos;
+	}
+	if (limits && outputIndex !== out.length) return out.slice(0, outputIndex);
+	if (!output) return out.subarray(0, outputIndex);
+	return out;
+}
+export { gunzip };

package/dist/_virtual/node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/src/column.mjs

@@ -1,5 +1,5 @@
-import { Encodings, PageTypes } from "./constants.mjs";
 import { convert, convertWithDictionary } from "./convert.mjs";
+import { Encodings, PageTypes } from "./constants.mjs";
 import { isFlatColumn } from "./schema.mjs";
 import { deserializeTCompactProtocol } from "./thrift.mjs";
 import { assembleLists } from "./assemble.mjs";

package/dist/_virtual/node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/src/datapage.mjs

@@ -1,8 +1,8 @@
+import { snappyUncompress } from "./snappy.mjs";
 import { getMaxDefinitionLevel, getMaxRepetitionLevel } from "./schema.mjs";
 import { deltaBinaryUnpack, deltaByteArray, deltaLengthByteArray } from "./delta.mjs";
 import { byteStreamSplit, readRleBitPackedHybrid } from "./encoding.mjs";
 import { readPlain } from "./plain.mjs";
-import { snappyUncompress } from "./snappy.mjs";
 function readDataPage(bytes, daph, { type, element, schemaPath }) {
 	const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
 	const reader = {

package/dist/_virtual/node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/src/index.mjs

@@ -1,7 +1,7 @@
+import "./snappy.mjs";
 import { parquetMetadataAsync, parquetSchema } from "./metadata.mjs";
 import "./indexes.mjs";
 import { asyncBufferFromUrl, cachedAsyncBuffer, toJson } from "./utils.mjs";
-import "./snappy.mjs";
 import { parquetScan } from "./scan.mjs";
 import { parquetReadObjects } from "./read.mjs";
 import "./query.mjs";

package/dist/_virtual/node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/src/indexes.mjs

@@ -1,5 +1,5 @@
-import { BoundaryOrders } from "./constants.mjs";
 import { DEFAULT_PARSERS } from "./convert.mjs";
+import { BoundaryOrders } from "./constants.mjs";
 import { deserializeTCompactProtocol } from "./thrift.mjs";
 import { convertMetadata } from "./metadata.mjs";
 function readColumnIndex(reader, schema, parsers = void 0) {

package/dist/_virtual/node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/src/metadata.mjs

@@ -1,5 +1,5 @@
-import { CompressionCodecs, ConvertedTypes, EdgeInterpolationAlgorithms, Encodings, FieldRepetitionTypes, PageTypes, ParquetTypes } from "./constants.mjs";
 import { DEFAULT_PARSERS, parseDecimal, parseFloat16 } from "./convert.mjs";
+import { CompressionCodecs, ConvertedTypes, EdgeInterpolationAlgorithms, Encodings, FieldRepetitionTypes, PageTypes, ParquetTypes } from "./constants.mjs";
 import { getSchemaPath } from "./schema.mjs";
 import { deserializeTCompactProtocol } from "./thrift.mjs";
 import { markGeoColumns } from "./geoparquet.mjs";

package/dist/_virtual/node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/src/node.mjs

@@ -1,7 +1,7 @@
+import "./snappy.mjs";
 import { parquetMetadataAsync, parquetSchema } from "./metadata.mjs";
 import "./indexes.mjs";
 import { asyncBufferFromUrl, cachedAsyncBuffer, toJson } from "./utils.mjs";
-import "./snappy.mjs";
 import { parquetScan } from "./scan.mjs";
 import { parquetReadObjects } from "./read.mjs";
 import "./index.mjs";

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/avro/avro.read.mjs

@@ -1,8 +1,8 @@
-import { parseDecimal } from "../../../../../hyparquet@1.31.1/node_modules/hyparquet/src/convert.mjs";
-import { gunzip } from "../../../../../hyparquet-compressors@1.1.2_patch_hash=821c89dbb6181f0c34d9abebd844451458a4f16c04e7aa7a19abca0c09c2238d/node_modules/hyparquet-compressors/src/gzip.mjs";
-import "../../../../../hyparquet-compressors@1.1.2_patch_hash=821c89dbb6181f0c34d9abebd844451458a4f16c04e7aa7a19abca0c09c2238d/node_modules/hyparquet-compressors/src/index.mjs";
 import { boundedBytes } from "./avro.budget.mjs";
 import { checkBlockEnd, readBlockEnd, readZigZag, readZigZagBigInt } from "./avro.metadata.mjs";
+import { gunzip } from "../../../../../hyparquet-compressors@1.1.2_patch_hash=3f8f44805c034b2563763dcd5d37050ea6b1d5d45b46ea37adc4a721e748a1c5/node_modules/hyparquet-compressors/src/gzip.mjs";
+import "../../../../../hyparquet-compressors@1.1.2_patch_hash=3f8f44805c034b2563763dcd5d37050ea6b1d5d45b46ea37adc4a721e748a1c5/node_modules/hyparquet-compressors/src/index.mjs";
+import { parseDecimal } from "../../../../../hyparquet@1.31.1/node_modules/hyparquet/src/convert.mjs";
 const textDecoder = new TextDecoder();
 function avroRead({ reader, metadata, syncMarker, recordKind = "entries" }) {
 	const budget = reader.budget;

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/fetch.mjs

@@ -1,13 +1,13 @@
+import { avroMetadata } from "./avro/avro.metadata.mjs";
+import { compressors } from "../../../../hyparquet-compressors@1.1.2_patch_hash=3f8f44805c034b2563763dcd5d37050ea6b1d5d45b46ea37adc4a721e748a1c5/node_modules/hyparquet-compressors/src/compressors.mjs";
+import "../../../../hyparquet-compressors@1.1.2_patch_hash=3f8f44805c034b2563763dcd5d37050ea6b1d5d45b46ea37adc4a721e748a1c5/node_modules/hyparquet-compressors/src/index.mjs";
+import { avroRead } from "./avro/avro.read.mjs";
 import { parquetMetadataAsync } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/metadata.mjs";
 import { asyncBufferFromUrl, cachedAsyncBuffer } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/utils.mjs";
 import { parquetReadObjects } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/read.mjs";
 import "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/node.mjs";
-import { compressors } from "../../../../hyparquet-compressors@1.1.2_patch_hash=821c89dbb6181f0c34d9abebd844451458a4f16c04e7aa7a19abca0c09c2238d/node_modules/hyparquet-compressors/src/compressors.mjs";
-import "../../../../hyparquet-compressors@1.1.2_patch_hash=821c89dbb6181f0c34d9abebd844451458a4f16c04e7aa7a19abca0c09c2238d/node_modules/hyparquet-compressors/src/index.mjs";
 import { ByteWriter } from "../../../../hyparquet-writer@0.16.10_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/bytewriter.mjs";
 import "../../../../hyparquet-writer@0.16.10_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/node.mjs";
-import { avroMetadata } from "./avro/avro.metadata.mjs";
-import { avroRead } from "./avro/avro.read.mjs";
 import { sanitize } from "./utils.mjs";
 import { isDeletionVector } from "./delete.mjs";
 import { puffinReadDeletionVector } from "./puffin/puffin.mjs";
@@ -68,40 +68,6 @@ function urlResolver({ requestInit } = {}) {
 		}
 	};
 }
-function cachingResolver(base) {
-	const cache = /* @__PURE__ */ new Map();
-	const out = { reader(path, byteLength) {
-		let buf = cache.get(path);
-		if (!buf) {
-			buf = (async () => cachedAsyncBuffer(await base.reader(path, byteLength)))();
-			cache.set(path, buf);
-			buf.catch(() => {
-				if (cache.get(path) === buf) cache.delete(path);
-			});
-		}
-		return buf;
-	} };
-	if (base.writer) {
-		const baseWriter = base.writer;
-		out.writer = (path, options) => {
-			const w = baseWriter(path, options);
-			const origFinish = w.finish.bind(w);
-			w.finish = async function() {
-				await origFinish();
-				cache.delete(path);
-			};
-			return w;
-		};
-	}
-	if (base.deleter) {
-		const baseDeleter = base.deleter;
-		out.deleter = async (path) => {
-			await baseDeleter(path);
-			cache.delete(path);
-		};
-	}
-	return out;
-}
 function s3Lister({ requestInit } = {}) {
 	return async function list(url) {
 		const s3parts = s3ParseUrl(url);
@@ -299,4 +265,4 @@ async function fetchBoundedAvroObject(url, resolver, byteLength, budget, recordK
 		return budget.fail(error);
 	}
 }
-export { cachingResolver, fetchAvroRecords, fetchBoundedAvroObject, fetchDeleteMaps, readParquetMetadata, resolveText, s3Lister, s3ParseUrl, translateS3Url, urlResolver };
+export { fetchAvroRecords, fetchBoundedAvroObject, fetchDeleteMaps, readParquetMetadata, resolveText, s3Lister, s3ParseUrl, translateS3Url, urlResolver };

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/read.mjs

@@ -1,3 +1,5 @@
+import { compressors } from "../../../../hyparquet-compressors@1.1.2_patch_hash=3f8f44805c034b2563763dcd5d37050ea6b1d5d45b46ea37adc4a721e748a1c5/node_modules/hyparquet-compressors/src/compressors.mjs";
+import "../../../../hyparquet-compressors@1.1.2_patch_hash=3f8f44805c034b2563763dcd5d37050ea6b1d5d45b46ea37adc4a721e748a1c5/node_modules/hyparquet-compressors/src/index.mjs";
 import { isListLike, isMapLike } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/schema.mjs";
 import { parquetSchema } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/metadata.mjs";
 import { cachedAsyncBuffer } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/utils.mjs";
@@ -5,8 +7,6 @@ import { columnsNeededForFilter, matchFilter } from "../../../../hyparquet@1.31.
 import { parquetScan } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/scan.mjs";
 import { parquetReadObjects } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/read.mjs";
 import "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/node.mjs";
-import { compressors } from "../../../../hyparquet-compressors@1.1.2_patch_hash=821c89dbb6181f0c34d9abebd844451458a4f16c04e7aa7a19abca0c09c2238d/node_modules/hyparquet-compressors/src/compressors.mjs";
-import "../../../../hyparquet-compressors@1.1.2_patch_hash=821c89dbb6181f0c34d9abebd844451458a4f16c04e7aa7a19abca0c09c2238d/node_modules/hyparquet-compressors/src/index.mjs";
 import { equalityMatch, sanitize } from "./utils.mjs";
 import { applicablePositionDeletes, deleteFileAppliesToDataEntry } from "./delete.mjs";
 import { readParquetMetadata } from "./fetch.mjs";

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/fetch.d.mts

@@ -1,36 +1,2 @@
 import "../../../../hyparquet@1.31.1/node_modules/hyparquet/types/node.mjs";
-import { Resolver } from "../src/types.mjs";
-/**
- * Wrap a `Resolver` so reads of the same path share one HTTP fetch and
- * subsequent range reads share one in-memory buffer. Writes and deletes
- * through the same resolver invalidate the cache entry for their path on
- * success, so a commit pipeline (`icebergAppend` → reload metadata) sees the
- * new bytes without manual invalidation.
- *
- * - `reader(path)` is memoized by path. The returned buffer is passed through
- *   `cachedAsyncBuffer` so range reads within a single file are also deduped.
- *   `byteLength` is a fetch hint and not part of the cache key — the bytes
- *   are the same either way.
- * - `writer(path).finish()` invalidates the cache entry only when the
- *   underlying finish resolves; a rejected finish (e.g. an `If-None-Match: *`
- *   collision returning 412/409) leaves the cached bytes intact, since the
- *   server-side object hasn't changed.
- * - `deleter(path)` invalidates the cache entry on success.
- * - `writer` and `deleter` are omitted when the base resolver omits them, so
- *   wrapping a read-only resolver still yields a read-only resolver.
- *
- * Iceberg writes are almost entirely create-new-path (data parquets, manifest
- * avros, snapshot avros, vN.metadata.json all live at fresh paths per
- * commit). Only `version-hint.text` and equivalent catalog-pointer files are
- * truly mutated in place, so path-level invalidation is sufficient to keep
- * single-process write-then-read pipelines consistent.
- *
- * Cross-process freshness is out of scope: a reader in process B does not
- * see writes committed by process A through a different cache. Use a
- * short-lived resolver, an external coordination layer, or wrap with TTL /
- * conditional-GET revalidation if you need that.
- *
- * @param {Resolver} base
- * @returns {Resolver}
- */
-export declare function cachingResolver(base: Resolver): Resolver;
+import "../src/types.mjs";

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/index.d.mts

@@ -1,9 +1,9 @@
 import { icebergManifests } from "./manifest.mjs";
 import { icebergAppend, icebergCreateTable, icebergDropTable } from "./write/write.mjs";
 import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./catalog/rest.mjs";
-import { cachingResolver } from "./fetch.mjs";
 import "./create.mjs";
 import "./catalog/file.mjs";
+import "./fetch.mjs";
 import { s3SignedResolver } from "./s3.mjs";
 import "../../../../squirreling@0.16.6/node_modules/squirreling/src/index.mjs";
 import "./read.mjs";

package/dist/catalog.d.mts

@@ -1,10 +1,11 @@
-import { ManifestReadCache } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/manifest.mjs";
+import { ManifestReadCache, ManifestReadLimits } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/manifest.mjs";
 import { icebergAppend } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/write/write.mjs";
 import { CatalogCache } from "./catalog-cache.mjs";
+import "./manifest-read-limits.mjs";
 import { IcebergPrimitiveType, IcebergS3Config } from "./schema.mjs";
 import { PartitionValueMatch } from "./partition-prune.mjs";
 import { restCatalogConnect, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/catalog/rest.mjs";
-import { cachingResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/fetch.mjs";
+import { Resolver } from "icebird/src/types.js";
 /** A field in an icebird table `Schema`. */
 export interface IcebergSchemaField {
   id: number;
@@ -61,7 +62,7 @@ export interface IcebergConnection {
   /** icebird REST catalog context, passed as `{ catalog }` to icebird write fns. */
   catalog: Awaited<ReturnType<typeof restCatalogConnect>>;
   /** icebird S3 resolver (caching-wrapped), passed as `{ resolver }` to icebird fns. */
-  resolver: ReturnType<typeof cachingResolver>;
+  resolver: Resolver;
   /** The namespace the dataset's table lives under. */
   namespace: string;
   /**
@@ -259,6 +260,13 @@ export interface ResolveIcebergDataFilesOptions {
   cache?: CatalogCache;
   clock?: () => number;
   profiler?: QueryProfiler;
+  /**
+   * Opt-in Avro source/decode-work limits for one uncached manifest walk.
+   * Bypasses decoded manifest, month and resolved-file caches; catalog metadata
+   * and snapshot caching keep their existing authority and are not byte-bounded
+   * by this option. No default limits or fallback to unbounded decoding.
+   */
+  readLimits?: ManifestReadLimits;
 }
 /**
  * Drop the cached snapshot pointer for `(namespace, table)` so the next read

package/dist/catalog.mjs

@@ -1,12 +1,13 @@
 import { stringifyBigintSafe } from "./bigint.mjs";
 import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/catalog/rest.mjs";
-import { cachingResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/fetch.mjs";
 import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/manifest.mjs";
 import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/s3.mjs";
 import { deriveAppendIdentity } from "./append-identity.mjs";
 import { cacheGet, cachePut, reportCatalogCacheError } from "./catalog-cache.mjs";
 import { toIcebergDayCount } from "./date.mjs";
+import { captureManifestReadLimits } from "./manifest-read-limits.mjs";
 import { MULTI_MONTH_MANIFEST, buildManifestPartitionFilter, manifestMonthBucket } from "./partition-prune.mjs";
+import { byteCappedCachingResolver } from "./resolver-byte-cache.mjs";
 const connectionReadBatch = Symbol("iceberg-connection-read-batch");
 function beginConnectionRead(conn) {
 	const batch = conn[connectionReadBatch];
@@ -115,7 +116,7 @@ async function connectIcebergCatalog(config, opts = {}) {
 		endpoint: config.s3.endpoint,
 		pathStyle: true
 	});
-	const resolver = withVerifiedWriterByteLengths(cachingResolver(s3Resolver));
+	const resolver = withUncachedReader(withVerifiedWriterByteLengths(byteCappedCachingResolver(s3Resolver)), s3Resolver);
 	return {
 		catalog,
 		resolver,
@@ -128,6 +129,17 @@ async function connectIcebergCatalog(config, opts = {}) {
 		}
 	};
 }
+const uncachedReader = Symbol("iceberg-uncached-reader");
+function withUncachedReader(resolver, base) {
+	return {
+		...resolver,
+		[uncachedReader]: (path, byteLength) => base.reader(path, byteLength)
+	};
+}
+function boundedReadResolver(resolver) {
+	const read = resolver[uncachedReader];
+	return read ? { reader: read } : resolver;
+}
 function withVerifiedWriterByteLengths(resolver) {
 	if (!resolver.writer) return resolver;
 	const baseWriter = resolver.writer;
@@ -643,12 +655,14 @@ function toListedFile(entry, matches, monthFieldName, wantedMonths, dateFieldId)
 		...dateBounds ? { dateBounds } : {}
 	};
 }
-async function resolveViaFullWalk(conn, metadata, partitionFilter, matches, monthFieldName, wantedMonths, dateFieldId, manifestCache) {
+async function resolveViaFullWalk(conn, metadata, partitionFilter, matches, monthFieldName, wantedMonths, dateFieldId, manifestCache, readLimits) {
+	const resolver = readLimits ? boundedReadResolver(conn.resolver) : conn.resolver;
 	const manifests = await icebergManifests({
 		metadata,
-		resolver: conn.resolver,
+		resolver,
 		partitionFilter,
-		manifestCache
+		manifestCache,
+		readLimits
 	});
 	const entries = [];
 	for (const m of manifests) for (const entry of m.entries) {
@@ -745,6 +759,7 @@ function resolveIcebergDataFiles(conn, opts) {
 	return resolveIcebergDataFilesInBatch(conn, opts).finally(release);
 }
 async function resolveIcebergDataFilesInBatch(conn, opts) {
+	const readLimits = captureManifestReadLimits(opts.readLimits);
 	const { namespace, table } = opts;
 	const profiler = opts.profiler;
 	const now = (opts.clock ?? Date.now)();
@@ -756,7 +771,7 @@ async function resolveIcebergDataFilesInBatch(conn, opts) {
 	if (snapshotId == null) return [];
 	const scope = conn.cacheScope ?? "";
 	const filesKey = resolvedFilesKey(scope, namespace, table, snapshotId, opts.matches, opts.range);
-	if (opts.cache) {
+	if (opts.cache && !readLimits) {
 		const endCache = profiler?.start("iceberg.cache");
 		const cached = await cacheGet(opts.cache, filesKey, now);
 		endCache?.({ hit: cached !== void 0 });
@@ -778,8 +793,8 @@ async function resolveIcebergDataFilesInBatch(conn, opts) {
 	const monthFieldName = monthField?.name;
 	const dateFieldId = dateColumnFieldId(metadata, monthField?.sourceColumn);
 	const dayWindow = rangeDayWindow(opts.range);
-	const manifestCache = manifestReadCacheFor(conn, namespace, table, metadata);
-	const walk = opts.cache && monthFieldName ? await resolveViaMonthCache(conn, opts.cache, metadata, partitionFilter, opts.partitionSpec, opts.matches, monthFieldName, wantedMonths, dateFieldId, scope, namespace, table, now, manifestCache) : await resolveViaFullWalk(conn, metadata, partitionFilter, opts.matches, monthFieldName, wantedMonths, dateFieldId, manifestCache);
+	const manifestCache = readLimits ? void 0 : manifestReadCacheFor(conn, namespace, table, metadata);
+	const walk = opts.cache && monthFieldName && !readLimits ? await resolveViaMonthCache(conn, opts.cache, metadata, partitionFilter, opts.partitionSpec, opts.matches, monthFieldName, wantedMonths, dateFieldId, scope, namespace, table, now, manifestCache) : await resolveViaFullWalk(conn, metadata, partitionFilter, opts.matches, monthFieldName, wantedMonths, dateFieldId, manifestCache, readLimits);
 	let boundsPruned = 0;
 	const out = [];
 	for (const file of walk.entries) {
@@ -798,7 +813,7 @@ async function resolveIcebergDataFilesInBatch(conn, opts) {
 		monthsHit: walk.monthsHit,
 		manifestsWalked: walk.manifestsWalked
 	});
-	if (opts.cache) {
+	if (opts.cache && !readLimits) {
 		const freshKey = resolvedFilesKey(scope, namespace, table, snapshotId, opts.matches, opts.range);
 		await cachePut(opts.cache, freshKey, out, RESOLVED_FILES_TTL_MS, now);
 	}

package/dist/dataset.d.mts

@@ -1,4 +1,6 @@
+import { ManifestReadLimits } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/manifest.mjs";
 import { CatalogCache } from "./catalog-cache.mjs";
+import "./manifest-read-limits.mjs";
 import { IcebergColumnType, IcebergPartitionField, IcebergTableSpec } from "./schema.mjs";
 import { ManifestPartitionFilter, PartitionValueMatch } from "./partition-prune.mjs";
 import { CommitRetryOptions, ConnectIcebergOptions, IcebergCatalogConfig, IcebergConnection, IcebergListedDataFile, IcebergPartitionSpec, IcebergSchema, IcebergSortOrder, IcebergTableOpResult, QueryProfiler } from "./catalog.mjs";
@@ -97,6 +99,8 @@ export interface ResolveDataFilesOptions {
   cache?: CatalogCache;
   clock?: () => number;
   profiler?: QueryProfiler;
+  /** Uncached Avro read limits; catalog metadata/snapshot caching is unchanged. */
+  readLimits?: ManifestReadLimits;
 }
 export interface IcebergDataset {
   readonly def: IcebergDatasetDef;

package/dist/dataset.mjs

@@ -1,4 +1,5 @@
 import { coerceBigIntToNumber } from "./bigint.mjs";
+import { captureManifestReadLimits } from "./manifest-read-limits.mjs";
 import { buildManifestPartitionFilter } from "./partition-prune.mjs";
 import { ICEBERG_TYPE_MAP } from "./schema.mjs";
 const INT32_MIN = -2147483648;
@@ -299,6 +300,7 @@ function defineIcebergDataset(def) {
 		});
 	}
 	async function resolveDataFiles(conn, identity, range, dims, opts = {}) {
+		const readLimits = captureManifestReadLimits(opts.readLimits);
 		const { resolveDatasetDataFiles } = await import("./dataset-runtime.mjs");
 		return resolveDatasetDataFiles(conn, {
 			namespace: def.namespace,
@@ -308,7 +310,8 @@ function defineIcebergDataset(def) {
 			range,
 			cache: opts.cache,
 			clock: opts.clock,
-			profiler: opts.profiler
+			profiler: opts.profiler,
+			readLimits
 		});
 	}
 	return {

package/dist/index.d.mts

@@ -1,8 +1,10 @@
 import { bigintJsonReplacer, coerceBigIntToNumber, encodeJsonBigintSafe, stringifyBigintSafe } from "./bigint.mjs";
+import { ManifestReadLimits } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/manifest.mjs";
 import { CatalogCache, CatalogCacheGetOutcome, cacheGet, cachePut } from "./catalog-cache.mjs";
+import "./manifest-read-limits.mjs";
 import { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_TYPE_MAP, IcebergColumn, IcebergColumnType, IcebergPartitionField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding } from "./schema.mjs";
 import { IcebergFieldSummary, ManifestPartitionFilter, PartitionValueMatch, buildManifestPartitionFilter } from "./partition-prune.mjs";
 import { CommitRetryOptions, ConnectIcebergOptions, IcebergCatalogConfig, IcebergConnection, IcebergDataFileDateBounds, IcebergListedDataFile, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableOpResult, QueryProfiler, ResolveIcebergDataFilesOptions, catalogCacheScope, connectIcebergCatalog, createIcebergNamespace, dropIcebergTables, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles } from "./catalog.mjs";
 import { AppendBatchSource, AppendBatchesOptions, AppendBatchesResult, AppendCommitOptions, AppendResult, AppendSink, AppendSinkCloseResult, AppendSinkOptions, DatasetDim, DatasetIdentity, IcebergDataset, IcebergDatasetColumnDef, IcebergDatasetDef, IcebergDatasetLedger, ResolveDataFilesOptions, defineIcebergDataset, deriveTableSpec } from "./dataset.mjs";
 import { toIcebergDayCount } from "./date.mjs";
-export { type AppendBatchSource, type AppendBatchesOptions, type AppendBatchesResult, type AppendCommitOptions, type AppendResult, type AppendSink, type AppendSinkCloseResult, type AppendSinkOptions, type CatalogCache, type CatalogCacheGetOutcome, type CommitRetryOptions, type ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, type DatasetDim, type DatasetIdentity, ICEBERG_TYPE_MAP, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergDataFileDateBounds, type IcebergDataset, type IcebergDatasetColumnDef, type IcebergDatasetDef, type IcebergDatasetLedger, type IcebergFieldSummary, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, type IcebergTableOpResult, type IcebergTableSpec, type ManifestPartitionFilter, type PartitionKeyEncoding, type PartitionValueMatch, type QueryProfiler, type ResolveDataFilesOptions, type ResolveIcebergDataFilesOptions, bigintJsonReplacer, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, coerceBigIntToNumber, connectIcebergCatalog, createIcebergNamespace, defineIcebergDataset, deriveTableSpec, dropIcebergTables, encodeJsonBigintSafe, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles, stringifyBigintSafe, toIcebergDayCount };
+export { type AppendBatchSource, type AppendBatchesOptions, type AppendBatchesResult, type AppendCommitOptions, type AppendResult, type AppendSink, type AppendSinkCloseResult, type AppendSinkOptions, type CatalogCache, type CatalogCacheGetOutcome, type CommitRetryOptions, type ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, type DatasetDim, type DatasetIdentity, ICEBERG_TYPE_MAP, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergDataFileDateBounds, type IcebergDataset, type IcebergDatasetColumnDef, type IcebergDatasetDef, type IcebergDatasetLedger, type IcebergFieldSummary, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, type IcebergTableOpResult, type IcebergTableSpec, type ManifestPartitionFilter, type ManifestReadLimits, type PartitionKeyEncoding, type PartitionValueMatch, type QueryProfiler, type ResolveDataFilesOptions, type ResolveIcebergDataFilesOptions, bigintJsonReplacer, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, coerceBigIntToNumber, connectIcebergCatalog, createIcebergNamespace, defineIcebergDataset, deriveTableSpec, dropIcebergTables, encodeJsonBigintSafe, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles, stringifyBigintSafe, toIcebergDayCount };

package/dist/manifest-read-limits.d.mts

@@ -0,0 +1,2 @@
+import { ManifestReadLimits } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/manifest.mjs";
+export type { ManifestReadLimits };

package/dist/manifest-read-limits.mjs

@@ -0,0 +1,26 @@
+const fields = [
+	"maxObjectBytes",
+	"maxTotalObjectBytes",
+	"maxManifests",
+	"maxEntries",
+	"maxInflatedBlockBytes",
+	"maxTotalInflatedBytes",
+	"maxValueBytes",
+	"maxDecodedValues",
+	"maxHeaderBytes",
+	"maxSchemaDepth"
+];
+function captureManifestReadLimits(options) {
+	if (options === void 0) return void 0;
+	if (!options || typeof options !== "object" || Array.isArray(options)) throw new TypeError("Invalid manifest read limits");
+	for (const key of Object.keys(options)) if (!fields.includes(key)) throw new TypeError(`Unknown manifest read limit: ${key}`);
+	const limits = {};
+	for (const key of fields) {
+		const value = options[key];
+		if (!Number.isSafeInteger(value) || value <= 0) throw new RangeError(`Invalid manifest read limit: ${key}`);
+		limits[key] = value;
+	}
+	if (limits.maxSchemaDepth > 128) throw new RangeError("Manifest schema depth must not exceed 128");
+	return Object.freeze(limits);
+}
+export { captureManifestReadLimits };

package/dist/resolver-byte-cache.mjs

@@ -0,0 +1,85 @@
+import { cachedAsyncBuffer } from "./_virtual/node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/src/utils.mjs";
+import "./_virtual/node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/src/node.mjs";
+const CONNECTION_CACHE_MAX_BYTES = 16777216;
+function byteCappedCachingResolver(base, maxBytes = CONNECTION_CACHE_MAX_BYTES) {
+	const entries = /* @__PURE__ */ new Map();
+	let totalBytes = 0;
+	function touch(path, entry) {
+		entries.delete(path);
+		entries.set(path, entry);
+	}
+	function evict(path, account) {
+		if (entries.get(path)?.account !== account) return;
+		entries.delete(path);
+		totalBytes -= account.bytes;
+	}
+	function retain(path, account, bytes) {
+		const entry = entries.get(path);
+		if (entry?.account !== account) return;
+		account.bytes += bytes;
+		totalBytes += bytes;
+		touch(path, entry);
+		if (account.bytes > maxBytes) {
+			evict(path, account);
+			return;
+		}
+		for (const [oldest, old] of entries) {
+			if (totalBytes <= maxBytes) break;
+			if (old.account.bytes > 0) evict(oldest, old.account);
+		}
+	}
+	function counted(path, account, file) {
+		return {
+			byteLength: file.byteLength,
+			slice(start, end) {
+				return Promise.resolve(file.slice(start, end)).then((bytes) => {
+					retain(path, account, bytes.byteLength);
+					return bytes;
+				}, (error) => {
+					evict(path, account);
+					throw error;
+				});
+			}
+		};
+	}
+	function invalidate(path) {
+		const entry = entries.get(path);
+		if (entry) evict(path, entry.account);
+	}
+	const out = { reader(path, byteLength) {
+		const hit = entries.get(path);
+		if (hit) {
+			touch(path, hit);
+			return hit.buffer;
+		}
+		const account = { bytes: 0 };
+		const buffer = (async () => cachedAsyncBuffer(counted(path, account, await base.reader(path, byteLength))))();
+		entries.set(path, {
+			buffer,
+			account
+		});
+		buffer.catch(() => evict(path, account));
+		return buffer;
+	} };
+	if (base.writer) {
+		const baseWriter = base.writer;
+		out.writer = (path, options) => {
+			const writer = baseWriter(path, options);
+			const finish = writer.finish.bind(writer);
+			writer.finish = async function() {
+				await finish();
+				invalidate(path);
+			};
+			return writer;
+		};
+	}
+	if (base.deleter) {
+		const baseDeleter = base.deleter;
+		out.deleter = async (path) => {
+			await baseDeleter(path);
+			invalidate(path);
+		};
+	}
+	return out;
+}
+export { CONNECTION_CACHE_MAX_BYTES, byteCappedCachingResolver };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/lakehouse",
   "type": "module",
-  "version": "7.7.0",
+  "version": "7.8.0",
   "description": "Dataset-agnostic Iceberg lakehouse layer + dataset registry for R2 Data Catalog producers (ADR-0021).",
   "author": {
     "name": "Harlan Wilton",

package/dist/_virtual/node_modules/.pnpm/hyparquet-compressors@1.1.2_patch_hash=821c89dbb6181f0c34d9abebd844451458a4f16c04e7aa7a19abca0c09c2238d/node_modules/hyparquet-compressors/src/gzip.mjs

@@ -1,186 +0,0 @@
-import { fixedDistance, fixedDistanceExtraBits, fixedDistanceMap, fixedLength, fixedLengthExtraBits, fixedLengthMap, huffMap } from "./gzip.huffman.mjs";
-const codeLengthIndexMap = new Uint8Array([
-	16,
-	17,
-	18,
-	0,
-	8,
-	7,
-	9,
-	6,
-	10,
-	5,
-	11,
-	4,
-	12,
-	3,
-	13,
-	2,
-	14,
-	1,
-	15
-]);
-function bits(input, pos, mask) {
-	const o = pos / 8 | 0;
-	return (input[o] | input[o + 1] << 8) >> (pos & 7) & mask;
-}
-function bits16(d, p) {
-	const o = p / 8 | 0;
-	return (d[o] | d[o + 1] << 8 | d[o + 2] << 16) >> (p & 7);
-}
-function shft(p) {
-	return (p + 7) / 8 | 0;
-}
-function gzipStart(input, i) {
-	if (input[i++] !== 31 || input[i++] !== 139 || input[i++] !== 8) return 0;
-	const flag = input[i++];
-	i += 6;
-	if (flag & 4) i += (input[i + 10] | input[i + 11] << 8) + 2;
-	for (let zs = (flag >> 3 & 1) + (flag >> 4 & 1); zs > 0; zs -= Number(!input[i++]));
-	return i + (flag & 2);
-}
-function gunzip(input, output, inputIndex = 0, outputIndex = 0, limits) {
-	const maximum = limits?.maxOutputBytes;
-	if (limits && (limits.raw !== true || !Number.isSafeInteger(maximum) || maximum < 0 || output || inputIndex !== 0 || outputIndex !== 0)) throw new RangeError("Invalid bounded raw DEFLATE options");
-	let out = output ?? new Uint8Array(limits ? Math.min(1024, maximum) : 1024);
-	if (limits && !input.length) throw new Error("unexpected EOF");
-	if (!(input.length - inputIndex)) return out;
-	const payloadStart = limits ? 0 : gzipStart(input, inputIndex);
-	if (!limits && payloadStart === input.length - 8) return out;
-	if (!limits && payloadStart > input.length - 8) throw new Error("unexpected EOF");
-	let pos = payloadStart * 8;
-	let final = 0;
-	let lengthBits = 0;
-	let distBits = 0;
-	let lengthMap;
-	let distMap;
-	const totalBits = input.length * 8;
-	function ensureSize(length) {
-		if (limits && (!Number.isSafeInteger(length) || length > maximum)) throw new RangeError("DEFLATE output byte limit exceeded");
-		if (!output && length > out.length) {
-			const old = out;
-			out = new Uint8Array(limits ? Math.min(maximum, Math.max(old.length * 2, length)) : Math.max(old.length * 2, length));
-			out.set(old);
-		}
-	}
-	do {
-		if (!lengthMap) {
-			final = bits(input, pos, 1);
-			const type = bits(input, pos + 1, 3);
-			pos += 3;
-			if (!type) {
-				const s = shft(pos) + 4;
-				const l = input[s - 4] | input[s - 3] << 8;
-				if (limits && (s > input.length || ((input[s - 2] | input[s - 1] << 8) ^ l) !== 65535)) throw new Error("invalid stored block length");
-				const t = s + l;
-				if (t > input.length) throw new Error("unexpected EOF");
-				ensureSize(outputIndex + l);
-				out.set(input.subarray(s, t), outputIndex);
-				outputIndex += l;
-				pos = t * 8;
-				continue;
-			} else if (type === 1) {
-				lengthMap = fixedLengthMap;
-				distMap = fixedDistanceMap;
-				lengthBits = 9;
-				distBits = 5;
-			} else if (type === 2) {
-				const hLiteral = bits(input, pos, 31) + 257;
-				const hcLengths = bits(input, pos + 10, 15) + 4;
-				const tl = hLiteral + bits(input, pos + 5, 31) + 1;
-				pos += 14;
-				const lengthDistanceTree = new Uint8Array(tl);
-				const codeLengthTree = /* @__PURE__ */ new Uint8Array(19);
-				for (let i = 0; i < hcLengths; ++i) codeLengthTree[codeLengthIndexMap[i]] = bits(input, pos + i * 3, 7);
-				pos += hcLengths * 3;
-				const codeLengthBits = Math.max(...codeLengthTree);
-				const clbMask = (1 << codeLengthBits) - 1;
-				const codeLengthMap = huffMap(codeLengthTree, codeLengthBits, 1);
-				for (let i = 0; i < tl;) {
-					const r = codeLengthMap[bits(input, pos, clbMask)];
-					if (limits && !r) throw new Error("invalid code length");
-					pos += r & 15;
-					const symbol = r >> 4;
-					if (symbol < 16) lengthDistanceTree[i++] = symbol;
-					else {
-						let copy = 0;
-						let n = 0;
-						if (symbol === 16) {
-							if (limits && i === 0) throw new Error("invalid code length repeat");
-							n = 3 + bits(input, pos, 3);
-							pos += 2;
-							copy = lengthDistanceTree[i - 1];
-						} else if (symbol === 17) {
-							n = 3 + bits(input, pos, 7);
-							pos += 3;
-						} else if (symbol === 18) {
-							n = 11 + bits(input, pos, 127);
-							pos += 7;
-						}
-						if (limits && i + n > tl) throw new Error("invalid code length repeat");
-						while (n--) lengthDistanceTree[i++] = copy;
-					}
-				}
-				const lengthTree = lengthDistanceTree.subarray(0, hLiteral);
-				const distanceTree = lengthDistanceTree.subarray(hLiteral);
-				lengthBits = Math.max(...lengthTree);
-				distBits = Math.max(...distanceTree);
-				lengthMap = huffMap(lengthTree, lengthBits, 1);
-				distMap = huffMap(distanceTree, distBits, 1);
-			} else throw new Error("invalid block type");
-			if (pos > totalBits) throw new Error("unexpected EOF");
-		}
-		ensureSize(limits ? Math.min(maximum, outputIndex + 131072) : outputIndex + 131072);
-		const lms = (1 << lengthBits) - 1;
-		const dms = (1 << distBits) - 1;
-		for (;;) {
-			const code = lengthMap[bits16(input, pos) & lms];
-			const sym = code >> 4;
-			pos += code & 15;
-			if (pos > totalBits) throw new Error("unexpected EOF");
-			if (!code) throw new Error("invalid length/literal");
-			if (sym < 256) {
-				if (limits) ensureSize(outputIndex + 1);
-				out[outputIndex++] = sym;
-			} else if (sym === 256) {
-				lengthMap = void 0;
-				break;
-			} else {
-				let add = sym - 254;
-				if (sym > 264) {
-					const index = sym - 257;
-					const b = fixedLengthExtraBits[index];
-					add = bits(input, pos, (1 << b) - 1) + fixedLength[index];
-					pos += b;
-				}
-				if (!distMap) throw new Error("invalid distance map");
-				const d = distMap[bits16(input, pos) & dms];
-				const dsym = d >> 4;
-				if (!d) throw new Error("invalid distance");
-				if (limits && dsym > 29) throw new Error("invalid distance symbol");
-				pos += d & 15;
-				let dt = fixedDistance[dsym];
-				if (dsym > 3) {
-					const b = fixedDistanceExtraBits[dsym];
-					dt += bits16(input, pos) & (1 << b) - 1;
-					pos += b;
-				}
-				if (pos > totalBits) throw new Error("unexpected EOF");
-				const end = outputIndex + add;
-				if (outputIndex < dt) throw new Error("unexpected dictionary case");
-				ensureSize(end);
-				for (; outputIndex < end; outputIndex++) out[outputIndex] = out[outputIndex - dt];
-			}
-		}
-		if (lengthMap) final = 1;
-	} while (!final);
-	if (limits && Math.ceil(pos / 8) !== input.length) throw new Error("trailing raw DEFLATE bytes");
-	if (!limits && outputIndex < out.length) {
-		const nextBlock = Math.ceil(pos / 8) + 8;
-		gunzip(input, out, nextBlock, outputIndex);
-	}
-	if (limits && outputIndex !== out.length) return out.slice(0, outputIndex);
-	if (!output) return out.subarray(0, outputIndex);
-	return out;
-}
-export { gunzip };