@gscdump/lakehouse 1.4.1 → 1.4.2

package/dist/_virtual/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/variant.mjs

@@ -0,0 +1,225 @@
+import { DEFAULT_PARSERS } from "./convert.mjs";
+const decoder = new TextDecoder();
+const metadataCache = /* @__PURE__ */ new WeakMap();
+function decodeVariantColumn(value, parsers = DEFAULT_PARSERS) {
+	if (Array.isArray(value)) return value.map((entry) => decodeVariantColumn(entry, parsers));
+	if (typeof value !== "object") return value;
+	if ("metadata" in value) {
+		const metadata = parseVariantMetadata(value.metadata);
+		const shreddedFields = value.typed_value && decodeTypedValue(value.typed_value, metadata, parsers);
+		const binaryValue = value.value && readVariant(makeReader(value.value), metadata, parsers);
+		if (shreddedFields && binaryValue) return {
+			...binaryValue,
+			...shreddedFields
+		};
+		return shreddedFields ?? binaryValue;
+	}
+	return value;
+}
+function decodeTypedValue(typedValue, metadata, parsers) {
+	if (typedValue instanceof Date) return typedValue;
+	if (typedValue && typeof typedValue === "object" && !Array.isArray(typedValue) && !(typedValue instanceof Uint8Array)) {
+		if ("typed_value" in typedValue && typedValue.typed_value !== null && typedValue.typed_value !== void 0) return decodeTypedValue(typedValue.typed_value, metadata, parsers);
+		if ("value" in typedValue && typedValue.value instanceof Uint8Array) return readVariant(makeReader(typedValue.value), metadata, parsers);
+		if ("typed_value" in typedValue || "value" in typedValue) return null;
+		const result = {};
+		for (const [key, field] of Object.entries(typedValue)) {
+			if (!metadata.dictionary.includes(key)) continue;
+			result[key] = decodeTypedValue(field, metadata, parsers);
+		}
+		return result;
+	}
+	if (typedValue instanceof Uint8Array) return readVariant(makeReader(typedValue), metadata, parsers);
+	if (Array.isArray(typedValue)) return typedValue.map((element) => decodeTypedValue(element, metadata, parsers));
+	return typedValue;
+}
+function makeReader(bytes) {
+	return {
+		view: new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength),
+		offset: 0
+	};
+}
+function parseVariantMetadata(bytes) {
+	let bufferCache = metadataCache.get(bytes.buffer);
+	if (!bufferCache) {
+		bufferCache = /* @__PURE__ */ new Map();
+		metadataCache.set(bytes.buffer, bufferCache);
+	}
+	const key = `${bytes.byteOffset}:${bytes.byteLength}`;
+	const cached = bufferCache.get(key);
+	if (cached) return cached;
+	const reader = makeReader(bytes);
+	const header = reader.view.getUint8(reader.offset++);
+	const version = header & 15;
+	if (version !== 1) throw new Error(`parquet unsupported variant metadata version: ${version}`);
+	const sorted = (header >> 4 & 1) === 1;
+	const offsetSize = (header >> 6 & 3) + 1;
+	const dictionarySize = readUnsigned(reader, offsetSize);
+	const offsets = new Array(dictionarySize + 1);
+	for (let i = 0; i < offsets.length; i++) offsets[i] = readUnsigned(reader, offsetSize);
+	const base = reader.offset;
+	const dictionary = new Array(dictionarySize);
+	for (let i = 0; i < dictionarySize; i++) {
+		const start = offsets[i];
+		const end = offsets[i + 1];
+		const strBytes = new Uint8Array(bytes.buffer, bytes.byteOffset + base + start, end - start);
+		dictionary[i] = decoder.decode(strBytes);
+	}
+	const metadata = {
+		dictionary,
+		sorted
+	};
+	bufferCache.set(key, metadata);
+	return metadata;
+}
+function readUnsigned(reader, byteWidth) {
+	let value = 0;
+	for (let i = 0; i < byteWidth; i++) value |= reader.view.getUint8(reader.offset + i) << i * 8;
+	reader.offset += byteWidth;
+	return value;
+}
+function readVariant(reader, metadata, parsers) {
+	const typeByte = reader.view.getUint8(reader.offset++);
+	const basicType = typeByte & 3;
+	const header = typeByte >> 2;
+	if (basicType === 0) return readVariantPrimitive(reader, header, parsers);
+	if (basicType === 2) return readVariantObject(reader, header, metadata, parsers);
+	if (basicType === 3) return readVariantArray(reader, header, metadata, parsers);
+	const bytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, header);
+	reader.offset += header;
+	return decoder.decode(bytes);
+}
+function readVariantPrimitive(reader, typeId, parsers) {
+	switch (typeId) {
+		case 0: return null;
+		case 1: return true;
+		case 2: return false;
+		case 3: {
+			const value = reader.view.getInt8(reader.offset);
+			reader.offset += 1;
+			return value;
+		}
+		case 4: {
+			const value = reader.view.getInt16(reader.offset, true);
+			reader.offset += 2;
+			return value;
+		}
+		case 5: {
+			const value = reader.view.getInt32(reader.offset, true);
+			reader.offset += 4;
+			return value;
+		}
+		case 6: {
+			const value = reader.view.getBigInt64(reader.offset, true);
+			reader.offset += 8;
+			return value;
+		}
+		case 7: {
+			const value = reader.view.getFloat64(reader.offset, true);
+			reader.offset += 8;
+			return value;
+		}
+		case 8: return readVariantDecimal(reader, 4);
+		case 9: return readVariantDecimal(reader, 8);
+		case 10: return readVariantDecimal(reader, 16);
+		case 11: {
+			const value = reader.view.getInt32(reader.offset, true);
+			reader.offset += 4;
+			return parsers.dateFromDays(value);
+		}
+		case 12:
+		case 13: {
+			const value = reader.view.getBigInt64(reader.offset, true);
+			reader.offset += 8;
+			return parsers.timestampFromMicroseconds(value);
+		}
+		case 14: {
+			const value = reader.view.getFloat32(reader.offset, true);
+			reader.offset += 4;
+			return value;
+		}
+		case 15: return readVariantBinary(reader);
+		case 16: {
+			const bytes = readVariantBinary(reader);
+			return decoder.decode(bytes);
+		}
+		case 17: {
+			const value = reader.view.getBigInt64(reader.offset, true);
+			reader.offset += 8;
+			return value;
+		}
+		case 18:
+		case 19: {
+			const value = reader.view.getBigInt64(reader.offset, true);
+			reader.offset += 8;
+			return parsers.timestampFromNanoseconds(value);
+		}
+		case 20: {
+			const bytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, 16);
+			reader.offset += 16;
+			const hex = Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("");
+			return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
+		}
+		default: throw new Error(`parquet unsupported variant primitive type: ${typeId}`);
+	}
+}
+function readVariantObject(reader, header, metadata, parsers) {
+	const offsetWidth = (header & 3) + 1;
+	const idWidth = (header >> 2 & 3) + 1;
+	const numElements = header >> 4 & 1 ? readUnsigned(reader, 4) : reader.view.getUint8(reader.offset++);
+	const fieldIds = new Array(numElements);
+	for (let i = 0; i < numElements; i++) fieldIds[i] = readUnsigned(reader, idWidth);
+	const offsets = new Array(numElements + 1);
+	for (let i = 0; i < offsets.length; i++) offsets[i] = readUnsigned(reader, offsetWidth);
+	const out = {};
+	for (let i = 0; i < numElements; i++) {
+		const key = metadata.dictionary[fieldIds[i]];
+		out[key] = readVariant({
+			view: reader.view,
+			offset: reader.offset + offsets[i]
+		}, metadata, parsers);
+	}
+	reader.offset += offsets[offsets.length - 1];
+	return out;
+}
+function readVariantArray(reader, header, metadata, parsers) {
+	const fieldOffsetSize = header & 3;
+	const isLarge = header >> 2 & 1;
+	const offsetWidth = fieldOffsetSize + 1;
+	const numElements = readUnsigned(reader, isLarge ? 4 : 1);
+	const offsets = new Array(numElements + 1);
+	for (let i = 0; i < offsets.length; i++) offsets[i] = readUnsigned(reader, offsetWidth);
+	const valuesStart = reader.offset;
+	const result = new Array(numElements);
+	for (let i = 0; i < numElements; i++) result[i] = readVariant({
+		view: reader.view,
+		offset: valuesStart + offsets[i]
+	}, metadata, parsers);
+	reader.offset = valuesStart + offsets[offsets.length - 1];
+	return result;
+}
+function readVariantDecimal(reader, width) {
+	const scale = reader.view.getUint8(reader.offset);
+	reader.offset += 1;
+	let unscaled;
+	if (width === 4) {
+		unscaled = BigInt(reader.view.getInt32(reader.offset, true));
+		reader.offset += 4;
+	} else if (width === 8) {
+		unscaled = reader.view.getBigInt64(reader.offset, true);
+		reader.offset += 8;
+	} else {
+		const low = reader.view.getBigUint64(reader.offset, true);
+		unscaled = reader.view.getBigInt64(reader.offset + 8, true) << 64n | low;
+		reader.offset += 16;
+	}
+	return Number(unscaled) * 10 ** -scale;
+}
+function readVariantBinary(reader) {
+	const length = reader.view.getUint32(reader.offset, true);
+	reader.offset += 4;
+	const bytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, length);
+	reader.offset += length;
+	return bytes;
+}
+export { decodeVariantColumn };

package/dist/_virtual/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/wkb.mjs

@@ -0,0 +1,94 @@
+function wkbToGeojson(reader) {
+	const flags = getFlags(reader);
+	if (flags.type === 1) return {
+		type: "Point",
+		coordinates: readPosition(reader, flags)
+	};
+	else if (flags.type === 2) return {
+		type: "LineString",
+		coordinates: readLine(reader, flags)
+	};
+	else if (flags.type === 3) return {
+		type: "Polygon",
+		coordinates: readPolygon(reader, flags)
+	};
+	else if (flags.type === 4) {
+		const points = [];
+		for (let i = 0; i < flags.count; i++) points.push(readPosition(reader, getFlags(reader)));
+		return {
+			type: "MultiPoint",
+			coordinates: points
+		};
+	} else if (flags.type === 5) {
+		const lines = [];
+		for (let i = 0; i < flags.count; i++) lines.push(readLine(reader, getFlags(reader)));
+		return {
+			type: "MultiLineString",
+			coordinates: lines
+		};
+	} else if (flags.type === 6) {
+		const polygons = [];
+		for (let i = 0; i < flags.count; i++) polygons.push(readPolygon(reader, getFlags(reader)));
+		return {
+			type: "MultiPolygon",
+			coordinates: polygons
+		};
+	} else if (flags.type === 7) {
+		const geometries = [];
+		for (let i = 0; i < flags.count; i++) geometries.push(wkbToGeojson(reader));
+		return {
+			type: "GeometryCollection",
+			geometries
+		};
+	} else throw new Error(`Unsupported geometry type: ${flags.type}`);
+}
+function getFlags(reader) {
+	const { view } = reader;
+	const littleEndian = view.getUint8(reader.offset++) === 1;
+	const rawType = view.getUint32(reader.offset, littleEndian);
+	reader.offset += 4;
+	const type = rawType % 1e3;
+	const flags = Math.floor(rawType / 1e3);
+	let count = 0;
+	if (type > 1 && type <= 7) {
+		count = view.getUint32(reader.offset, littleEndian);
+		reader.offset += 4;
+	}
+	let dim = 2;
+	if (flags) dim++;
+	if (flags === 3) dim++;
+	return {
+		littleEndian,
+		type,
+		dim,
+		count
+	};
+}
+function readPosition(reader, flags) {
+	const points = [];
+	for (let i = 0; i < flags.dim; i++) {
+		const coord = reader.view.getFloat64(reader.offset, flags.littleEndian);
+		reader.offset += 8;
+		points.push(coord);
+	}
+	return points;
+}
+function readLine(reader, flags) {
+	const points = [];
+	for (let i = 0; i < flags.count; i++) points.push(readPosition(reader, flags));
+	return points;
+}
+function readPolygon(reader, flags) {
+	const { view } = reader;
+	const rings = [];
+	for (let r = 0; r < flags.count; r++) {
+		const count = view.getUint32(reader.offset, flags.littleEndian);
+		reader.offset += 4;
+		rings.push(readLine(reader, {
+			...flags,
+			count
+		}));
+	}
+	return rings;
+}
+export { wkbToGeojson };

package/dist/_virtual/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/xxhash.mjs

@@ -0,0 +1,68 @@
+const MASK = 18446744073709551615n;
+const PRIME1 = 11400714785074694791n;
+const PRIME2 = 14029467366897019727n;
+const PRIME3 = 1609587929392839161n;
+const PRIME4 = 9650029242287828579n;
+const PRIME5 = 2870177450012600261n;
+function rotl64(x, r) {
+	return (x << r | x >> 64n - r) & MASK;
+}
+function round(acc, val) {
+	acc = acc + val * PRIME2 & MASK;
+	acc = rotl64(acc, 31n);
+	return acc * PRIME1 & MASK;
+}
+function mergeRound(acc, val) {
+	acc ^= round(0n, val);
+	return acc * PRIME1 + PRIME4 & MASK;
+}
+function xxhash64(input, seed = 0n) {
+	const view = new DataView(input.buffer, input.byteOffset, input.byteLength);
+	const len = input.byteLength;
+	let offset = 0;
+	let h64;
+	if (len >= 32) {
+		let v1 = seed + PRIME1 + PRIME2 & MASK;
+		let v2 = seed + PRIME2 & MASK;
+		let v3 = seed;
+		let v4 = seed - PRIME1 & MASK;
+		while (offset + 32 <= len) {
+			v1 = round(v1, view.getBigUint64(offset, true));
+			offset += 8;
+			v2 = round(v2, view.getBigUint64(offset, true));
+			offset += 8;
+			v3 = round(v3, view.getBigUint64(offset, true));
+			offset += 8;
+			v4 = round(v4, view.getBigUint64(offset, true));
+			offset += 8;
+		}
+		h64 = rotl64(v1, 1n) + rotl64(v2, 7n) + rotl64(v3, 12n) + rotl64(v4, 18n) & MASK;
+		h64 = mergeRound(h64, v1);
+		h64 = mergeRound(h64, v2);
+		h64 = mergeRound(h64, v3);
+		h64 = mergeRound(h64, v4);
+	} else h64 = seed + PRIME5 & MASK;
+	h64 = h64 + BigInt(len) & MASK;
+	while (offset + 8 <= len) {
+		h64 ^= round(0n, view.getBigUint64(offset, true));
+		h64 = rotl64(h64, 27n) * PRIME1 + PRIME4 & MASK;
+		offset += 8;
+	}
+	if (offset + 4 <= len) {
+		h64 ^= BigInt(view.getUint32(offset, true)) * PRIME1 & MASK;
+		h64 = rotl64(h64, 23n) * PRIME2 + PRIME3 & MASK;
+		offset += 4;
+	}
+	while (offset < len) {
+		h64 ^= BigInt(view.getUint8(offset)) * PRIME5 & MASK;
+		h64 = rotl64(h64, 11n) * PRIME1 & MASK;
+		offset += 1;
+	}
+	h64 ^= h64 >> 33n;
+	h64 = h64 * PRIME2 & MASK;
+	h64 ^= h64 >> 29n;
+	h64 = h64 * PRIME3 & MASK;
+	h64 ^= h64 >> 32n;
+	return h64;
+}
+export { xxhash64 };

package/dist/{node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057 → _virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057}/node_modules/icebird/src/avro/avro.write.mjs

@@ -1,5 +1,5 @@
-import { ByteWriter } from "../../../../../hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/bytewriter.mjs";
-import "../../../../../hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/node.mjs";
+import { ByteWriter } from "../../../../../hyparquet-writer@0.16.1_patch_hash=7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/bytewriter.mjs";
+import "../../../../../hyparquet-writer@0.16.1_patch_hash=7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/node.mjs";
 function avroWrite({ writer, schema, records, blockSize = 512, metadata }) {
 	writer.appendUint32(23749199);
 	const meta = {

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/delete.mjs

@@ -0,0 +1,34 @@
+import { valuesEqual } from "./utils.mjs";
+function deleteFileAppliesToDataEntry(dataEntry, deleteEntry, metadata, deleteType) {
+	const dataSequenceNumber = dataEntry.sequence_number;
+	const deleteSequenceNumber = deleteEntry.sequence_number;
+	if (dataSequenceNumber === void 0) throw new Error("data file missing sequence number");
+	if (deleteSequenceNumber === void 0) throw new Error("delete file missing sequence number");
+	if (deleteType === "equality") {
+		if (deleteSequenceNumber <= dataSequenceNumber) return false;
+		if (isUnpartitioned(metadata, deleteEntry.partition_spec_id)) return true;
+	} else if (deleteSequenceNumber < dataSequenceNumber) return false;
+	return samePartition(dataEntry, deleteEntry);
+}
+function isUnpartitioned(metadata, specId) {
+	return metadata["partition-specs"].find((s) => s["spec-id"] === specId)?.fields.length === 0;
+}
+function samePartition(dataEntry, deleteEntry) {
+	if (dataEntry.partition_spec_id !== deleteEntry.partition_spec_id) return false;
+	return partitionsEqual(dataEntry.data_file.partition, deleteEntry.data_file.partition);
+}
+function partitionsEqual(a, b) {
+	const aKeys = Object.keys(a);
+	const bKeys = Object.keys(b);
+	if (aKeys.length !== bKeys.length) return false;
+	for (const key of aKeys) {
+		if (!Object.hasOwn(b, key)) return false;
+		if (!partitionValuesEqual(a[key], b[key])) return false;
+	}
+	return true;
+}
+function partitionValuesEqual(a, b) {
+	if (typeof a === "number" && typeof b === "number") return Object.is(a, b);
+	return valuesEqual(a, b);
+}
+export { deleteFileAppliesToDataEntry };

package/dist/{node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057 → _virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057}/node_modules/icebird/src/fetch.mjs

@@ -1,12 +1,15 @@
+import { parquetMetadataAsync } from "../../../../hyparquet@1.26.2/node_modules/hyparquet/src/metadata.mjs";
 import { asyncBufferFromUrl, cachedAsyncBuffer } from "../../../../hyparquet@1.26.2/node_modules/hyparquet/src/utils.mjs";
+import { parquetReadObjects } from "../../../../hyparquet@1.26.2/node_modules/hyparquet/src/read.mjs";
 import "../../../../hyparquet@1.26.2/node_modules/hyparquet/src/node.mjs";
+import { compressors } from "../../../../hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/compressors.mjs";
 import "../../../../hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/index.mjs";
-import { ByteWriter } from "../../../../hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/bytewriter.mjs";
-import "../../../../hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/node.mjs";
+import { ByteWriter } from "../../../../hyparquet-writer@0.16.1_patch_hash=7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/bytewriter.mjs";
+import "../../../../hyparquet-writer@0.16.1_patch_hash=7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/node.mjs";
 import { avroMetadata } from "./avro/avro.metadata.mjs";
 import { avroRead } from "./avro/avro.read.mjs";
-import "./puffin/puffin.mjs";
-import "./utils.mjs";
+import { puffinReadDeletionVector } from "./puffin/puffin.mjs";
+import { sanitize } from "./utils.mjs";
 function translateS3Url(url) {
 	if (url.startsWith("s3a://") || url.startsWith("s3://")) {
 		const rest = url.slice(url.indexOf("://") + 3);
@@ -128,6 +131,102 @@ function s3ParseUrl(url) {
 		prefix: m[2]
 	};
 }
+async function fetchDeleteMaps(deleteEntries, resolver) {
+	const positionDeletesMap = /* @__PURE__ */ new Map();
+	const equalityDeleteGroups = [];
+	await Promise.all(deleteEntries.map(async (deleteEntry) => {
+		const { content, file_path, file_size_in_bytes } = deleteEntry.data_file;
+		const file = cachedAsyncBuffer(await resolver.reader(file_path, Number(file_size_in_bytes)));
+		if (content === 1) if (isDeletionVector(deleteEntry)) {
+			const { referenced_data_file, content_offset, content_size_in_bytes } = deleteEntry.data_file;
+			if (!referenced_data_file) throw new Error("deletion vector missing referenced_data_file");
+			if (content_offset == null) throw new Error("deletion vector missing content_offset");
+			if (content_size_in_bytes == null) throw new Error("deletion vector missing content_size_in_bytes");
+			const positions = await puffinReadDeletionVector(file, {
+				offset: content_offset,
+				length: content_size_in_bytes,
+				referencedDataFile: referenced_data_file
+			});
+			const set = /* @__PURE__ */ new Set();
+			for (const pos of positions) set.add(pos);
+			addPositionDeleteGroup(positionDeletesMap, referenced_data_file, deleteEntry, set);
+		} else {
+			const deleteRows = await parquetReadObjects({
+				file,
+				compressors
+			});
+			const positionsByPath = /* @__PURE__ */ new Map();
+			for (const deleteRow of deleteRows) {
+				const { file_path, pos } = deleteRow;
+				if (!file_path) throw new Error("position delete missing target file path");
+				if (pos === void 0) throw new Error("position delete missing pos");
+				let set = positionsByPath.get(file_path);
+				if (!set) {
+					set = /* @__PURE__ */ new Set();
+					positionsByPath.set(file_path, set);
+				}
+				set.add(pos);
+			}
+			for (const [targetPath, positions] of positionsByPath) addPositionDeleteGroup(positionDeletesMap, targetPath, deleteEntry, positions);
+		}
+		else if (content === 2) {
+			const { sequence_number } = deleteEntry;
+			const equalityIds = deleteEntry.data_file.equality_ids;
+			if (sequence_number === void 0) throw new Error("equality delete missing sequence number");
+			if (!equalityIds?.length) throw new Error("equality delete missing equality_ids");
+			const metadata = await parquetMetadataAsync(file);
+			const columnNamesById = equalityColumnNamesById(metadata, equalityIds);
+			const deleteRows = await parquetReadObjects({
+				file,
+				metadata,
+				columns: equalityIds.map((id) => columnNamesById[id]),
+				compressors
+			});
+			const rows = [];
+			for (const deleteRow of deleteRows) rows.push(equalityPredicate(deleteRow, equalityIds, columnNamesById));
+			equalityDeleteGroups.push({
+				deleteEntry,
+				rows
+			});
+		}
+	}));
+	return {
+		positionDeletesMap,
+		equalityDeleteGroups
+	};
+}
+function addPositionDeleteGroup(positionDeletesMap, targetPath, deleteEntry, positions) {
+	let groups = positionDeletesMap.get(targetPath);
+	if (!groups) {
+		groups = [];
+		positionDeletesMap.set(targetPath, groups);
+	}
+	groups.push({
+		deleteEntry,
+		positions
+	});
+}
+function isDeletionVector(deleteEntry) {
+	const dataFile = deleteEntry.data_file;
+	return dataFile.file_format.toLowerCase() === "puffin" || dataFile.content_offset != null || dataFile.content_size_in_bytes != null;
+}
+function equalityColumnNamesById(parquetMetadata, equalityIds) {
+	const kv = parquetMetadata.key_value_metadata?.find((k) => k.key === "iceberg.schema");
+	if (!kv?.value) throw new Error("equality delete missing iceberg.schema parquet metadata");
+	const schema = JSON.parse(kv.value);
+	const out = {};
+	for (const id of equalityIds) {
+		const field = schema.fields.find((f) => f.id === id);
+		if (!field) throw new Error(`equality delete missing field id ${id} in parquet schema`);
+		out[id] = sanitize(field.name);
+	}
+	return out;
+}
+function equalityPredicate(deleteRow, equalityIds, columnNamesById) {
+	const out = {};
+	for (const id of equalityIds) out[id] = deleteRow[columnNamesById[id]];
+	return out;
+}
 async function resolveText(resolver, path) {
 	const ab = await resolver.reader(path);
 	let buf = await ab.slice(0, ab.byteLength);
@@ -159,4 +258,4 @@ async function fetchAvroRecords(url, resolver, byteLength) {
 		syncMarker
 	});
 }
-export { cachingResolver, fetchAvroRecords, resolveText, s3Lister, s3ParseUrl, translateS3Url, urlResolver };
+export { cachingResolver, fetchAvroRecords, fetchDeleteMaps, resolveText, s3Lister, s3ParseUrl, translateS3Url, urlResolver };

package/dist/{node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057 → _virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057}/node_modules/icebird/src/index.mjs

@@ -1,13 +1,13 @@
-import { cachingResolver } from "./fetch.mjs";
-import "./metadata.mjs";
 import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./catalog/rest.mjs";
-import "./create.mjs";
+import "./fetch.mjs";
 import { icebergManifests } from "./manifest.mjs";
+import { s3SignedResolver } from "./s3.mjs";
+import "./metadata.mjs";
+import "./create.mjs";
 import "./read.mjs";
-import { icebergAppend, icebergAppendBatches, icebergCreateTable, icebergDropTable } from "./write/write.mjs";
+import { icebergAppend, icebergCreateTable, icebergDropTable } from "./write/write.mjs";
 import "./catalog/file.mjs";
-import { s3SignedResolver } from "./s3.mjs";
 import "../../../../squirreling@0.15.0/node_modules/squirreling/src/index.mjs";
 import "./sql/icebergDataSource.mjs";
 import "./sql/icebergQuery.mjs";
-export { cachingResolver, icebergAppend, icebergAppendBatches, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver };
+export { icebergAppend, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver };

package/dist/{node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057 → _virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057}/node_modules/icebird/src/manifest.mjs

@@ -68,4 +68,17 @@ function assignFirstRowIds(manifest, entries) {
 		}
 	}
 }
-export { icebergManifests };
+function splitManifestEntries(manifests) {
+	const dataEntries = [];
+	const deleteEntries = [];
+	for (const { entries } of manifests) for (const entry of entries) {
+		if (entry.status === 2) continue;
+		if (entry.data_file.content) deleteEntries.push(entry);
+		else dataEntries.push(entry);
+	}
+	return {
+		dataEntries,
+		deleteEntries
+	};
+}
+export { icebergManifests, splitManifestEntries };

package/dist/{node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057 → _virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057}/node_modules/icebird/src/metadata.mjs

@@ -1,5 +1,5 @@
-import { resolveText, s3Lister, urlResolver } from "./fetch.mjs";
 import { parseIcebergJson } from "./json.mjs";
+import { resolveText, s3Lister, urlResolver } from "./fetch.mjs";
 function metadataFileVersionNumber(file) {
 	const match = file.match(/^(?:v(\d+)|(\d+)-.+)(?:\.metadata\.json|\.gz\.metadata\.json|\.metadata\.json\.gz)$/);
 	if (!match) return void 0;
@@ -40,6 +40,14 @@ function icebergLatestVersion({ tableUrl, resolver, lister }) {
 		throw new Error(`failed to determine latest iceberg version of ${tableUrl}: ${err.message}`);
 	});
 }
+async function icebergMetadata({ tableUrl, metadataFileName, resolver, lister }) {
+	return (await resolveMetadata({
+		tableUrl,
+		metadataFileName,
+		resolver,
+		lister
+	})).metadata;
+}
 async function resolveMetadata({ tableUrl, metadataFileName, resolver, lister }) {
 	resolver ??= urlResolver();
 	lister ??= s3Lister();
@@ -148,4 +156,4 @@ function metadataFilePreference(file, paddedVersion) {
 	if (file.startsWith(`${paddedVersion}-`) && file.endsWith(".metadata.json.gz")) return 5;
 	return 6;
 }
-export { icebergLatestVersion, loadLatestFileCatalogMetadata, resolveMetadata };
+export { icebergLatestVersion, icebergMetadata, loadLatestFileCatalogMetadata, resolveMetadata };