@gscdump/lakehouse 1.4.1 → 1.4.3

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/puffin/deletion-vector.mjs

@@ -0,0 +1,122 @@
+import { readRoaringBitmap32, writeRoaringBitmap32 } from "./roaring.mjs";
+const DV_MAGIC = 3520280932;
+function readDeletionVector(bytes) {
+	const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+	const payloadLength = view.getUint32(0, false);
+	if (payloadLength !== bytes.byteLength - 8) throw new Error(`invalid deletion vector length: ${payloadLength}`);
+	if (view.getUint32(4, false) !== DV_MAGIC) throw new Error("invalid deletion vector magic");
+	const expectedCrc = view.getUint32(bytes.byteLength - 4, false);
+	if (crc32(bytes.subarray(4, bytes.byteLength - 4)) !== expectedCrc) throw new Error("deletion vector crc mismatch");
+	return readPositionVector(bytes.subarray(8, bytes.byteLength - 4));
+}
+function readPositionVector(bytes) {
+	const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+	let offset = 0;
+	const bitmapCount = view.getBigUint64(offset, true);
+	offset += 8;
+	if (bitmapCount > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error(`too many deletion vector bitmaps: ${bitmapCount}`);
+	const positions = /* @__PURE__ */ new Set();
+	for (let i = 0; i < Number(bitmapCount); i++) {
+		const key = view.getUint32(offset, true);
+		offset += 4;
+		const start = offset;
+		const values = readRoaringBitmap32(bytes.subarray(start));
+		offset = start + roaringBitmap32Length(bytes.subarray(start));
+		const keyBase = BigInt(key) << 32n;
+		for (const value of values) positions.add(keyBase + BigInt(value));
+	}
+	return positions;
+}
+function roaringBitmap32Length(bytes) {
+	const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+	let offset = 0;
+	const cookie = view.getUint32(offset, true);
+	offset += 4;
+	let size;
+	let runContainers;
+	if (cookie === 12346) {
+		size = view.getUint32(offset, true);
+		offset += 4;
+	} else if ((cookie & 65535) === 12347) {
+		size = (cookie >>> 16) + 1;
+		const runBytes = Math.ceil(size / 8);
+		runContainers = bytes.subarray(offset, offset + runBytes);
+		offset += runBytes;
+	} else throw new Error(`invalid roaring cookie: ${cookie}`);
+	const cardinalities = new Array(size);
+	const types = new Array(size);
+	for (let i = 0; i < size; i++) {
+		offset += 2;
+		cardinalities[i] = view.getUint16(offset, true) + 1;
+		offset += 2;
+		types[i] = isRunContainer(runContainers, i) ? "run" : cardinalities[i] <= 4096 ? "array" : "bitmap";
+	}
+	if (cookie === 12346 || size >= 4) offset += 4 * size;
+	for (let i = 0; i < size; i++) if (types[i] === "array") offset += cardinalities[i] * 2;
+	else if (types[i] === "bitmap") offset += 8192;
+	else {
+		const runCount = view.getUint16(offset, true);
+		offset += 2 + runCount * 4;
+	}
+	return offset;
+}
+function isRunContainer(runContainers, index) {
+	if (!runContainers) return false;
+	return Boolean(runContainers[Math.floor(index / 8)] & 1 << index % 8);
+}
+function writeDeletionVector(positions) {
+	const byKey = /* @__PURE__ */ new Map();
+	for (const pos of positions) {
+		if (typeof pos !== "bigint") throw new Error(`deletion vector position must be bigint: ${pos}`);
+		if (pos < 0n) throw new Error(`deletion vector position must be non-negative: ${pos}`);
+		const key = Number(pos >> 32n);
+		const value = Number(pos & 4294967295n);
+		let bucket = byKey.get(key);
+		if (!bucket) {
+			bucket = [];
+			byKey.set(key, bucket);
+		}
+		bucket.push(value);
+	}
+	const bitmaps = [...byKey.entries()].sort(([a], [b]) => a - b).map(([key, values]) => {
+		values.sort((a, b) => a - b);
+		const dedup = [];
+		for (const v of values) if (!dedup.length || dedup[dedup.length - 1] !== v) dedup.push(v);
+		return {
+			key,
+			bitmap: writeRoaringBitmap32(dedup)
+		};
+	});
+	const payloadLength = 4 + (8 + bitmaps.reduce((sum, b) => sum + 4 + b.bitmap.byteLength, 0));
+	const out = new Uint8Array(4 + payloadLength + 4);
+	const view = new DataView(out.buffer);
+	view.setUint32(0, payloadLength, false);
+	view.setUint32(4, DV_MAGIC, false);
+	view.setBigUint64(8, BigInt(bitmaps.length), true);
+	let offset = 16;
+	for (const { key, bitmap } of bitmaps) {
+		view.setUint32(offset, key, true);
+		offset += 4;
+		out.set(bitmap, offset);
+		offset += bitmap.byteLength;
+	}
+	view.setUint32(out.byteLength - 4, crc32(out.subarray(4, out.byteLength - 4)), false);
+	return out;
+}
+let crcTable;
+function crc32(bytes) {
+	crcTable ??= makeCrcTable();
+	let crc = 4294967295;
+	for (const byte of bytes) crc = crcTable[(crc ^ byte) & 255] ^ crc >>> 8;
+	return ~crc >>> 0;
+}
+function makeCrcTable() {
+	const table = /* @__PURE__ */ new Uint32Array(256);
+	for (let i = 0; i < table.length; i++) {
+		let c = i;
+		for (let j = 0; j < 8; j++) c = c & 1 ? 3988292384 ^ c >>> 1 : c >>> 1;
+		table[i] = c >>> 0;
+	}
+	return table;
+}
+export { readDeletionVector, writeDeletionVector };

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/puffin/puffin.mjs

@@ -0,0 +1,78 @@
+import { readDeletionVector } from "./deletion-vector.mjs";
+const PUFFIN_MAGIC = 1346781489;
+async function puffinReadDeletionVector(file, { offset, length, referencedDataFile }) {
+	const buffer = await file.slice(0, file.byteLength);
+	const bytes = new Uint8Array(buffer);
+	const blob = readPuffinMetadata(bytes).blobs.find((blob) => {
+		return blob.type === "deletion-vector-v1" && BigInt(blob.offset) === BigInt(offset) && BigInt(blob.length) === BigInt(length);
+	});
+	if (!blob) throw new Error("puffin deletion-vector-v1 blob not found");
+	if (blob["compression-codec"]) throw new Error(`unsupported puffin blob compression: ${blob["compression-codec"]}`);
+	if (referencedDataFile && blob.properties?.["referenced-data-file"] !== referencedDataFile) throw new Error("puffin deletion vector referenced-data-file mismatch");
+	const start = toSafeNumber(blob.offset);
+	const end = start + toSafeNumber(blob.length);
+	return readDeletionVector(bytes.subarray(start, end));
+}
+function readPuffinMetadata(bytes) {
+	const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+	if (view.getUint32(0, false) !== PUFFIN_MAGIC) throw new Error("invalid puffin file magic");
+	if (view.getUint32(bytes.byteLength - 4, false) !== PUFFIN_MAGIC) throw new Error("invalid puffin footer magic");
+	const payloadSize = view.getInt32(bytes.byteLength - 12, true);
+	if (payloadSize < 0) throw new Error(`invalid puffin footer payload size: ${payloadSize}`);
+	const flags = view.getUint32(bytes.byteLength - 8, true);
+	const compressed = Boolean(flags & 1);
+	if (flags & -2) throw new Error(`unsupported puffin footer flags: ${flags}`);
+	if (compressed) throw new Error("compressed puffin footers are not supported");
+	const payloadStart = bytes.byteLength - 12 - payloadSize;
+	const footerMagicStart = payloadStart - 4;
+	if (footerMagicStart < 4 || view.getUint32(footerMagicStart, false) !== PUFFIN_MAGIC) throw new Error("invalid puffin footer start magic");
+	const payload = bytes.subarray(payloadStart, payloadStart + payloadSize);
+	return JSON.parse(new TextDecoder().decode(payload));
+}
+function toSafeNumber(value) {
+	const out = Number(value);
+	if (!Number.isSafeInteger(out)) throw new Error(`puffin offset exceeds safe integer range: ${value}`);
+	return out;
+}
+function writePuffinFile({ blobs, properties }) {
+	if (!Array.isArray(blobs) || !blobs.length) throw new Error("writePuffinFile requires at least one blob");
+	let cursor = 4;
+	const footer = { blobs: blobs.map((blob) => {
+		if (!blob.type) throw new Error("puffin blob type is required");
+		if (!(blob.data instanceof Uint8Array)) throw new Error("puffin blob data must be a Uint8Array");
+		const meta = {
+			type: blob.type,
+			fields: blob.fields ?? [],
+			"snapshot-id": toSafeNumber(blob.snapshotId ?? -1),
+			"sequence-number": toSafeNumber(blob.sequenceNumber ?? -1),
+			offset: cursor,
+			length: blob.data.byteLength
+		};
+		if (blob.properties) meta.properties = blob.properties;
+		cursor += blob.data.byteLength;
+		return meta;
+	}) };
+	if (properties) footer.properties = properties;
+	const footerPayload = new TextEncoder().encode(JSON.stringify(footer));
+	const totalSize = 4 + blobs.reduce((sum, b) => sum + b.data.byteLength, 0) + 4 + footerPayload.byteLength + 4 + 4 + 4;
+	const out = new Uint8Array(totalSize);
+	const view = new DataView(out.buffer);
+	let offset = 0;
+	view.setUint32(offset, PUFFIN_MAGIC, false);
+	offset += 4;
+	for (const blob of blobs) {
+		out.set(blob.data, offset);
+		offset += blob.data.byteLength;
+	}
+	view.setUint32(offset, PUFFIN_MAGIC, false);
+	offset += 4;
+	out.set(footerPayload, offset);
+	offset += footerPayload.byteLength;
+	view.setInt32(offset, footerPayload.byteLength, true);
+	offset += 4;
+	view.setUint32(offset, 0, true);
+	offset += 4;
+	view.setUint32(offset, PUFFIN_MAGIC, false);
+	return out;
+}
+export { puffinReadDeletionVector, readPuffinMetadata, writePuffinFile };

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/puffin/roaring.mjs

@@ -0,0 +1,132 @@
+function readRoaringBitmap32(bytes) {
+	const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+	let offset = 0;
+	const cookie = view.getUint32(offset, true);
+	offset += 4;
+	let size;
+	let runContainers;
+	if (cookie === 12346) {
+		size = view.getUint32(offset, true);
+		offset += 4;
+	} else if ((cookie & 65535) === 12347) {
+		size = (cookie >>> 16) + 1;
+		const runBytes = Math.ceil(size / 8);
+		runContainers = bytes.subarray(offset, offset + runBytes);
+		offset += runBytes;
+	} else throw new Error(`invalid roaring cookie: ${cookie}`);
+	const keys = new Array(size);
+	const cardinalities = new Array(size);
+	const types = new Array(size);
+	for (let i = 0; i < size; i++) {
+		keys[i] = view.getUint16(offset, true);
+		cardinalities[i] = view.getUint16(offset + 2, true) + 1;
+		offset += 4;
+		types[i] = isRunContainer(runContainers, i) ? "run" : cardinalities[i] <= 4096 ? "array" : "bitmap";
+	}
+	if (cookie === 12346 || size >= 4) offset += 4 * size;
+	const values = [];
+	for (let i = 0; i < size; i++) {
+		const keyBase = keys[i] * 65536;
+		const cardinality = cardinalities[i];
+		const type = types[i];
+		if (type === "array") for (let j = 0; j < cardinality; j++) {
+			values.push(keyBase + view.getUint16(offset, true));
+			offset += 2;
+		}
+		else if (type === "bitmap") for (let word = 0; word < 1024; word++) {
+			let bits = view.getBigUint64(offset, true);
+			offset += 8;
+			while (bits) {
+				const bit = trailingZeroes64(bits);
+				values.push(keyBase + word * 64 + bit);
+				bits &= bits - 1n;
+			}
+		}
+		else {
+			const runCount = view.getUint16(offset, true);
+			offset += 2;
+			for (let run = 0; run < runCount; run++) {
+				const start = view.getUint16(offset, true);
+				const length = view.getUint16(offset + 2, true) + 1;
+				offset += 4;
+				for (let value = start; value < start + length; value++) values.push(keyBase + value);
+			}
+		}
+	}
+	return values;
+}
+function isRunContainer(runContainers, index) {
+	if (!runContainers) return false;
+	return Boolean(runContainers[Math.floor(index / 8)] & 1 << index % 8);
+}
+function writeRoaringBitmap32(values) {
+	const containers = /* @__PURE__ */ new Map();
+	for (const value of values) {
+		if (!Number.isInteger(value) || value < 0 || value > 4294967295) throw new Error(`roaring value out of range: ${value}`);
+		const key = Math.floor(value / 65536);
+		const lo = value % 65536;
+		let bucket = containers.get(key);
+		if (!bucket) {
+			bucket = [];
+			containers.set(key, bucket);
+		}
+		const last = bucket.length ? bucket[bucket.length - 1] : -1;
+		if (lo < last) throw new Error(`roaring values must be sorted: ${value}`);
+		if (lo > last) bucket.push(lo);
+	}
+	const bodies = [...containers.keys()].sort((a, b) => a - b).map((key) => {
+		const items = containers.get(key);
+		if (items.length <= 4096) {
+			const buf = new Uint8Array(items.length * 2);
+			const view = new DataView(buf.buffer);
+			for (let i = 0; i < items.length; i++) view.setUint16(i * 2, items[i], true);
+			return {
+				key,
+				cardinality: items.length,
+				bytes: buf
+			};
+		}
+		const buf = /* @__PURE__ */ new Uint8Array(8192);
+		for (const v of items) buf[v >>> 3] |= 1 << (v & 7);
+		return {
+			key,
+			cardinality: items.length,
+			bytes: buf
+		};
+	});
+	const size = bodies.length;
+	const headerSize = 8 + 4 * size + 4 * size;
+	const totalSize = headerSize + bodies.reduce((sum, b) => sum + b.bytes.byteLength, 0);
+	const out = new Uint8Array(totalSize);
+	const view = new DataView(out.buffer);
+	let offset = 0;
+	view.setUint32(offset, 12346, true);
+	offset += 4;
+	view.setUint32(offset, size, true);
+	offset += 4;
+	for (const body of bodies) {
+		view.setUint16(offset, body.key, true);
+		view.setUint16(offset + 2, body.cardinality - 1, true);
+		offset += 4;
+	}
+	let bodyOffset = headerSize;
+	for (const body of bodies) {
+		view.setUint32(offset, bodyOffset, true);
+		offset += 4;
+		bodyOffset += body.bytes.byteLength;
+	}
+	for (const body of bodies) {
+		out.set(body.bytes, offset);
+		offset += body.bytes.byteLength;
+	}
+	return out;
+}
+function trailingZeroes64(value) {
+	let count = 0;
+	while ((value & 1n) === 0n) {
+		value >>= 1n;
+		count++;
+	}
+	return count;
+}
+export { readRoaringBitmap32, writeRoaringBitmap32 };

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/read.mjs

@@ -0,0 +1,324 @@
+import { parquetMetadataAsync } from "../../../../hyparquet@1.26.2/node_modules/hyparquet/src/metadata.mjs";
+import { cachedAsyncBuffer, concat } from "../../../../hyparquet@1.26.2/node_modules/hyparquet/src/utils.mjs";
+import { parquetReadObjects } from "../../../../hyparquet@1.26.2/node_modules/hyparquet/src/read.mjs";
+import "../../../../hyparquet@1.26.2/node_modules/hyparquet/src/node.mjs";
+import { compressors } from "../../../../hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/compressors.mjs";
+import "../../../../hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/index.mjs";
+import { equalityMatch, sanitize } from "./utils.mjs";
+import { fetchDeleteMaps, urlResolver } from "./fetch.mjs";
+import { icebergManifests, splitManifestEntries } from "./manifest.mjs";
+import { icebergMetadata } from "./metadata.mjs";
+import { deleteFileAppliesToDataEntry } from "./delete.mjs";
+const DEFAULT_ROW_GROUP_CONCURRENCY = 4;
+async function icebergRead({ tableUrl, rowStart = 0, rowEnd = Infinity, metadataFileName, metadata, snapshotId, resolver, lister, rowGroupConcurrency = DEFAULT_ROW_GROUP_CONCURRENCY }) {
+	if (!tableUrl) throw new Error("tableUrl is required");
+	if (rowStart > rowEnd) throw new Error("rowStart must be less than rowEnd");
+	if (rowStart < 0) throw new Error("rowStart must be positive");
+	rowGroupConcurrency = normalizeRowGroupConcurrency(rowGroupConcurrency);
+	resolver ??= urlResolver();
+	metadata ??= await icebergMetadata({
+		tableUrl,
+		metadataFileName,
+		resolver,
+		lister
+	});
+	const manifestList = await icebergManifests({
+		metadata,
+		resolver,
+		snapshotId
+	});
+	const currentSchemaId = metadata["current-schema-id"];
+	const schema = metadata.schemas.find((s) => s["schema-id"] === currentSchemaId);
+	if (!schema) throw new Error("current schema not found in metadata");
+	const rowLineage = metadata["format-version"] >= 3;
+	const { dataEntries, deleteEntries } = splitManifestEntries(manifestList);
+	if (dataEntries.length === 0) throw new Error("No data manifest files found for current snapshot");
+	const hasDeletes = deleteEntries.length > 0;
+	const deleteMaps = fetchDeleteMaps(deleteEntries, resolver);
+	const totalRowsToRead = rowEnd === Infinity ? Infinity : rowEnd - rowStart;
+	let fileIndex = 0;
+	let skipRows = rowStart;
+	if (hasDeletes) skipRows = 0;
+	else while (fileIndex < dataEntries.length && skipRows >= dataEntries[fileIndex].data_file.record_count) {
+		skipRows -= Number(dataEntries[fileIndex].data_file.record_count);
+		fileIndex++;
+	}
+	const fileReads = [];
+	let rowsRemaining = hasDeletes ? Infinity : totalRowsToRead;
+	for (let i = fileIndex; i < dataEntries.length && rowsRemaining > 0; i++) {
+		const recordCount = Number(dataEntries[i].data_file.record_count);
+		const fileRowStart = i === fileIndex ? skipRows : 0;
+		const availableRows = recordCount - fileRowStart;
+		if (availableRows <= 0) continue;
+		const rowsToRead = rowsRemaining === Infinity ? availableRows : Math.min(rowsRemaining, availableRows);
+		fileReads.push({
+			entry: dataEntries[i],
+			fileRowStart,
+			fileRowEnd: fileRowStart + rowsToRead
+		});
+		if (rowsRemaining !== Infinity) rowsRemaining -= rowsToRead;
+	}
+	const { positionDeletesMap, equalityDeleteGroups } = await deleteMaps;
+	const rows = (await Promise.all(fileReads.map(async ({ entry, fileRowStart, fileRowEnd }) => {
+		const rows = [];
+		for await (const batch of readDataFile({
+			dataEntry: entry,
+			fileRowStart,
+			fileRowEnd,
+			schema,
+			metadata,
+			resolver,
+			rowLineage,
+			positionDeletesMap,
+			equalityDeleteGroups,
+			rowGroupConcurrency
+		})) concat(rows, batch);
+		return rows;
+	}))).flat();
+	if (!hasDeletes) return rows;
+	return rows.slice(rowStart, rowEnd === Infinity ? void 0 : rowEnd);
+}
+async function* readDataFile({ dataEntry, fileRowStart, fileRowEnd, schema, metadata, resolver, rowLineage, positionDeletesMap, equalityDeleteGroups, wantedColumns, rowGroupConcurrency = 1, filter, signal }) {
+	const { data_file, sequence_number, partition_spec_id } = dataEntry;
+	if (sequence_number === void 0) throw new Error("sequence number not found, check v2 inheritance logic");
+	const sequenceNumber = sequence_number;
+	const partitionSpec = metadata["partition-specs"].find((s) => s["spec-id"] === partition_spec_id);
+	const asyncBuffer = cachedAsyncBuffer(await resolver.reader(data_file.file_path, Number(data_file.file_size_in_bytes)));
+	const parquetMetadata = await parquetMetadataAsync(asyncBuffer);
+	const kv = parquetMetadata.key_value_metadata?.find((k) => k.key === "iceberg.schema");
+	let parquetIcebergSchema;
+	if (kv?.value) parquetIcebergSchema = JSON.parse(kv.value);
+	else if (parquetMetadata.schema.some((s) => s.field_id !== void 0)) parquetIcebergSchema = parquetSchemaToIceberg(parquetMetadata.schema);
+	else parquetIcebergSchema = schema;
+	const parquetColumnNames = [];
+	for (const field of schema.fields) {
+		const parquetField = parquetIcebergSchema.fields.find((f) => f.id === field.id);
+		if (parquetField && field.type !== "unknown") parquetColumnNames.push(sanitize(parquetField.name));
+		else parquetColumnNames.push(void 0);
+	}
+	const lineageColumns = rowLineage ? rowLineageColumnNames(parquetIcebergSchema) : {};
+	const dataColumnNamesById = columnNamesById(parquetIcebergSchema);
+	const positionDeleteGroups = positionDeletesMap.get(data_file.file_path);
+	const positionDeletes = /* @__PURE__ */ new Set();
+	if (positionDeleteGroups) for (const group of positionDeleteGroups) {
+		if (!deleteFileAppliesToDataEntry(dataEntry, group.deleteEntry, metadata, "position")) continue;
+		for (const pos of group.positions) positionDeletes.add(pos);
+	}
+	const applicableEqualityGroups = equalityDeleteGroups.filter((group) => deleteFileAppliesToDataEntry(dataEntry, group.deleteEntry, metadata, "equality"));
+	const wantedSet = wantedColumns ? new Set(wantedColumns) : null;
+	const wantsLineageOutput = !wantedSet || wantedSet.has("_row_id") || wantedSet.has("_last_updated_sequence_number");
+	const columns = [];
+	for (let i = 0; i < schema.fields.length; i++) {
+		const parquetName = parquetColumnNames[i];
+		if (!parquetName) continue;
+		if (wantedSet && !wantedSet.has(schema.fields[i].name)) continue;
+		columns.push(parquetName);
+	}
+	for (const group of applicableEqualityGroups) for (const predicate of group.rows) for (const key of Object.keys(predicate)) {
+		const colName = dataColumnNamesById[Number(key)] ?? key;
+		if (colName === "file_path" || colName === "pos") continue;
+		if (!columns.includes(colName)) columns.push(colName);
+	}
+	if (rowLineage && wantsLineageOutput) {
+		for (const c of [lineageColumns.rowId, lineageColumns.lastUpdatedSequenceNumber]) if (c && !columns.includes(c)) columns.push(c);
+	}
+	const parquetColumns = columns.length > 0 ? columns : parquetColumnNames.filter((n) => n !== void 0);
+	const icebergToParquet = {};
+	for (let i = 0; i < schema.fields.length; i++) {
+		const parquetName = parquetColumnNames[i];
+		if (parquetName) icebergToParquet[schema.fields[i].name] = parquetName;
+	}
+	const parquetFilter = filter ? remapFilterColumns(filter, icebergToParquet) : void 0;
+	async function readRowGroupRange({ readStart, readEnd }) {
+		const rows = await parquetReadObjects({
+			file: asyncBuffer,
+			metadata: parquetMetadata,
+			columns: parquetColumns,
+			rowStart: readStart,
+			rowEnd: readEnd,
+			compressors,
+			filter: parquetFilter,
+			filterStrict: false,
+			useBloomFilters: true,
+			utf8: false
+		});
+		const batch = [];
+		for (let idx = 0; idx < rows.length; idx++) {
+			const row = rows[idx];
+			const pos = BigInt(readStart + idx);
+			if (positionDeletes.has(pos)) continue;
+			if (applicableEqualityGroups.some((group) => group.rows.some((predicate) => equalityMatch(row, predicate, dataColumnNamesById)))) continue;
+			const mapped = {};
+			for (let i = 0; i < schema.fields.length; i++) {
+				const field = schema.fields[i];
+				if (wantedSet && !wantedSet.has(field.name)) continue;
+				const parquetColumnName = parquetColumnNames[i];
+				if (parquetColumnName) mapped[field.name] = row[parquetColumnName];
+				else {
+					const partitionField = partitionSpec?.fields.find((pf) => pf["source-id"] === field.id && pf.transform === "identity");
+					let nameMapping;
+					if (metadata.properties?.["schema.name-mapping.default"]) nameMapping = nameMappingById(JSON.parse(metadata.properties["schema.name-mapping.default"]), field.id);
+					if (partitionField && Object.hasOwn(data_file.partition, partitionField.name)) mapped[field.name] = data_file.partition[partitionField.name] ?? null;
+					else if (nameMapping) {
+						for (const name of nameMapping.names) if (parquetColumnNames.indexOf(name) !== -1) {
+							mapped[field.name] = row[name];
+							break;
+						}
+					} else if (field["initial-default"] !== void 0) mapped[field.name] = field["initial-default"];
+					else mapped[field.name] = null;
+				}
+			}
+			if (rowLineage && wantsLineageOutput) {
+				applyRowLineage(mapped, {
+					row,
+					pos,
+					firstRowId: data_file.first_row_id,
+					sequenceNumber,
+					rowIdColumn: lineageColumns.rowId,
+					lastUpdatedSequenceNumberColumn: lineageColumns.lastUpdatedSequenceNumber
+				});
+				if (wantedSet && !wantedSet.has("_row_id")) delete mapped._row_id;
+				if (wantedSet && !wantedSet.has("_last_updated_sequence_number")) delete mapped._last_updated_sequence_number;
+			}
+			batch.push(mapped);
+		}
+		return batch;
+	}
+	const rowGroupRanges = [];
+	let groupStart = 0;
+	for (const rowGroup of parquetMetadata.row_groups) {
+		if (signal?.aborted) throw new DOMException("Aborted", "AbortError");
+		const rowCount = Number(rowGroup.num_rows);
+		const groupEnd = groupStart + rowCount;
+		if (groupEnd <= fileRowStart) {
+			groupStart = groupEnd;
+			continue;
+		}
+		if (groupStart >= fileRowEnd) break;
+		const readStart = Math.max(groupStart, fileRowStart);
+		const readEnd = Math.min(groupEnd, fileRowEnd);
+		rowGroupRanges.push({
+			readStart,
+			readEnd
+		});
+		groupStart = groupEnd;
+	}
+	const concurrency = normalizeRowGroupConcurrency(rowGroupConcurrency);
+	if (concurrency === 1 || rowGroupRanges.length <= 1) {
+		for (const range of rowGroupRanges) {
+			if (signal?.aborted) throw new DOMException("Aborted", "AbortError");
+			const batch = await readRowGroupRange(range);
+			if (batch.length > 0) yield batch;
+		}
+		return;
+	}
+	const tasks = /* @__PURE__ */ new Map();
+	let nextStart = 0;
+	let nextYield = 0;
+	function startTask(index) {
+		const task = readRowGroupRange(rowGroupRanges[index]).then((batch) => ({ batch }), (error) => ({ error }));
+		tasks.set(index, task);
+	}
+	while (nextStart < rowGroupRanges.length && tasks.size < concurrency) {
+		startTask(nextStart);
+		nextStart++;
+	}
+	while (nextYield < rowGroupRanges.length) {
+		if (signal?.aborted) throw new DOMException("Aborted", "AbortError");
+		const task = tasks.get(nextYield);
+		if (!task) throw new Error("internal row group scheduling error");
+		const result = await task;
+		tasks.delete(nextYield);
+		if (result.error) throw result.error;
+		while (nextStart < rowGroupRanges.length && tasks.size < concurrency) {
+			startTask(nextStart);
+			nextStart++;
+		}
+		if (result.batch && result.batch.length > 0) yield result.batch;
+		nextYield++;
+	}
+}
+function remapFilterColumns(filter, mapping) {
+	const anyFilter = filter;
+	for (const op of [
+		"$and",
+		"$or",
+		"$nor"
+	]) {
+		const subs = anyFilter[op];
+		if (Array.isArray(subs)) {
+			const out = [];
+			for (const sub of subs) {
+				const m = remapFilterColumns(sub, mapping);
+				if (!m) return void 0;
+				out.push(m);
+			}
+			return { [op]: out };
+		}
+	}
+	const out = {};
+	for (const [key, cond] of Object.entries(anyFilter)) {
+		const mapped = mapping[key];
+		if (!mapped) return void 0;
+		out[mapped] = cond;
+	}
+	return out;
+}
+function normalizeRowGroupConcurrency(value) {
+	if (value === Infinity) return Number.MAX_SAFE_INTEGER;
+	const normalized = Math.floor(value);
+	if (!Number.isFinite(normalized) || normalized < 1) throw new Error("rowGroupConcurrency must be at least 1");
+	return normalized;
+}
+function nameMappingById(mappings, fieldId) {
+	for (const m of mappings) {
+		if (m["field-id"] === fieldId) return m;
+		if (m.fields) {
+			const hit = nameMappingById(m.fields, fieldId);
+			if (hit) return hit;
+		}
+	}
+}
+function rowLineageColumnNames(parquetIcebergSchema) {
+	return {
+		rowId: columnNameByFieldId(parquetIcebergSchema, 2147483540),
+		lastUpdatedSequenceNumber: columnNameByFieldId(parquetIcebergSchema, 2147483539)
+	};
+}
+function columnNameByFieldId(schema, fieldId) {
+	const field = schema.fields.find((f) => f.id === fieldId);
+	return field ? sanitize(field.name) : void 0;
+}
+function parquetSchemaToIceberg(parquetSchema) {
+	const fields = [];
+	for (const elem of parquetSchema) {
+		if (elem.field_id === void 0) continue;
+		fields.push({
+			id: elem.field_id,
+			name: elem.name,
+			required: false,
+			type: "unknown"
+		});
+	}
+	return {
+		type: "struct",
+		"schema-id": 0,
+		fields
+	};
+}
+function columnNamesById(schema) {
+	const out = {};
+	for (const field of schema.fields) out[field.id] = sanitize(field.name);
+	return out;
+}
+function applyRowLineage(out, { row, pos, firstRowId, sequenceNumber, rowIdColumn, lastUpdatedSequenceNumberColumn }) {
+	const storedRowId = rowIdColumn ? row[rowIdColumn] : void 0;
+	const storedLastUpdatedSequenceNumber = lastUpdatedSequenceNumberColumn ? row[lastUpdatedSequenceNumberColumn] : void 0;
+	if (storedRowId != null) out._row_id = storedRowId;
+	else if (firstRowId != null) out._row_id = BigInt(firstRowId) + pos;
+	else out._row_id = null;
+	if (storedLastUpdatedSequenceNumber != null) out._last_updated_sequence_number = storedLastUpdatedSequenceNumber;
+	else if (firstRowId != null) out._last_updated_sequence_number = sequenceNumber;
+	else out._last_updated_sequence_number = null;
+}
+export { icebergRead, readDataFile };

package/dist/{node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057 → _virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057}/node_modules/icebird/src/s3.mjs

@@ -1,5 +1,5 @@
-import { ByteWriter } from "../../../../hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/bytewriter.mjs";
-import "../../../../hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/node.mjs";
+import { ByteWriter } from "../../../../hyparquet-writer@0.16.1_patch_hash=7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/bytewriter.mjs";
+import "../../../../hyparquet-writer@0.16.1_patch_hash=7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/node.mjs";
 import { signRequest } from "./sigv4.mjs";
 function s3SignedResolver({ accessKeyId, secretAccessKey, sessionToken, region, endpoint, pathStyle = false }) {
 	const ep = endpoint ? new URL(endpoint.replace(/\/$/, "") + "/") : void 0;

package/dist/{node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057 → _virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057}/node_modules/icebird/src/types.d.mts

@@ -1,7 +1,7 @@
 import { AsyncBuffer } from "../../../../hyparquet@1.26.2/node_modules/hyparquet/types/index.mjs";
 import "../../../../hyparquet@1.26.2/node_modules/hyparquet/types/node.mjs";
-import { Writer } from "../../../../hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/types/index.mjs";
-import "../../../../hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/types/node.mjs";
+import { Writer } from "../../../../hyparquet-writer@0.16.1_patch_hash=7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/types/index.mjs";
+import "../../../../hyparquet-writer@0.16.1_patch_hash=7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/types/node.mjs";
 interface WriterOptions {
   /**
    * If `'*'`, the writer must create the object only if it does not already