npm - @gscdump/lakehouse - Versions diffs - 3.3.0 → 3.4.1 - Mend

@gscdump/lakehouse 3.3.0 → 3.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (232) hide show

package/dist/THIRD-PARTY-LICENSES.md CHANGED Viewed

@@ -304,11 +304,11 @@ Repository: https://github.com/101arrowz/fzstd
 ---------------------------------------
-## hyparquet, hyparquet, hyparquet, hyparquet-compressors, hyparquet-writer, icebird, squirreling
+## hyparquet, hyparquet-compressors, hyparquet-writer, icebird, squirreling
 License: MIT
 By: Hyperparam
-Repositories: https://github.com/hyparam/hyparquet, https://github.com/hyparam/hyparquet, https://github.com/hyparam/hyparquet, https://github.com/hyparam/hyparquet-compressors, https://github.com/hyparam/hyparquet-writer, https://github.com/hyparam/icebird, https://github.com/hyparam/squirreling
+Repositories: https://github.com/hyparam/hyparquet, https://github.com/hyparam/hyparquet-compressors, https://github.com/hyparam/hyparquet-writer, https://github.com/hyparam/icebird, https://github.com/hyparam/squirreling
 > The MIT License (MIT)
 >

package/dist/_virtual/node_modules/.pnpm/{hyparquet-writer@0.16.6_patch_hash=549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a → hyparquet-writer@0.16.9_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f}/node_modules/hyparquet-writer/src/bloom.mjs RENAMED Viewed

@@ -1,4 +1,4 @@
-import { hashParquetValue } from "../../../../hyparquet@1.28.1/node_modules/hyparquet/src/bloom.mjs";
+import { hashParquetValue } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/bloom.mjs";
 import { serializeTCompactProtocol } from "./thrift.mjs";
 const SALT = new Uint32Array([
 	1203114875,

@@ -22,37 +22,37 @@ var ByteWriter = class {
 		return new Uint8Array(this.buffer, 0, this.index);
 	}
 	appendUint8(value) {
-		this.ensure(this.index + 1);
+		this.ensure(1);
 		this.view.setUint8(this.index, value);
 		this.offset++;
 		this.index++;
 	}
 	appendUint32(value) {
-		this.ensure(this.index + 4);
+		this.ensure(4);
 		this.view.setUint32(this.index, value, true);
 		this.offset += 4;
 		this.index += 4;
 	}
 	appendInt32(value) {
-		this.ensure(this.index + 4);
+		this.ensure(4);
 		this.view.setInt32(this.index, value, true);
 		this.offset += 4;
 		this.index += 4;
 	}
 	appendInt64(value) {
-		this.ensure(this.index + 8);
+		this.ensure(8);
 		this.view.setBigInt64(this.index, BigInt(value), true);
 		this.offset += 8;
 		this.index += 8;
 	}
 	appendFloat32(value) {
-		this.ensure(this.index + 8);
+		this.ensure(4);
 		this.view.setFloat32(this.index, value, true);
 		this.offset += 4;
 		this.index += 4;
 	}
 	appendFloat64(value) {
-		this.ensure(this.index + 8);
+		this.ensure(8);
 		this.view.setFloat64(this.index, value, true);
 		this.offset += 8;
 		this.index += 8;
@@ -61,7 +61,7 @@ var ByteWriter = class {
 		this.appendBytes(new Uint8Array(value));
 	}
 	appendBytes(value) {
-		this.ensure(this.index + value.length);
+		this.ensure(value.length);
 		new Uint8Array(this.buffer, this.index, value.length).set(value);
 		this.offset += value.length;
 		this.index += value.length;

@@ -11,7 +11,7 @@ function writeColumn({ writer, column, pageData }) {
 	const offsetStart = writer.offset;
 	const encodings = [];
 	const isGeospatial = element?.logical_type?.type === "GEOMETRY" || element?.logical_type?.type === "GEOGRAPHY";
-	const statistics = stats ? getStatistics(values) : void 0;
+	const statistics = stats ? getStatistics(values, element) : void 0;
 	const geospatial_statistics = stats && isGeospatial ? geospatialStatistics(values) : void 0;
 	let bloomFilter;
 	if (column.bloomFilter) {
@@ -70,7 +70,7 @@ function writeColumn({ writer, column, pageData }) {
 			pageData: pageChunk
 		});
 		if (columnIndex) {
-			const { min_value, max_value, null_count = 0n } = getStatistics(values.slice(start, end));
+			const { min_value, max_value, null_count = 0n } = getStatistics(values.slice(start, end), element);
 			columnIndex.null_pages.push(null_count === BigInt(end - start));
 			columnIndex.min_values.push(unconvertMinMax(min_value, element, false) ?? /* @__PURE__ */ new Uint8Array());
 			columnIndex.max_values.push(unconvertMinMax(max_value, element, true) ?? /* @__PURE__ */ new Uint8Array());
@@ -167,7 +167,7 @@ function getPageBoundaries(values, type, type_length, pageSize) {
 	});
 	return boundaries;
 }
-function getStatistics(values) {
+function getStatistics(values, element) {
 	let min_value = void 0;
 	let max_value = void 0;
 	let null_count = 0n;
@@ -178,8 +178,9 @@ function getStatistics(values) {
 		}
 		if (typeof value === "object" && !(value instanceof Uint8Array)) continue;
 		if (typeof value === "number" && Number.isNaN(value)) continue;
-		if (min_value === void 0 || compareValues(value, min_value) < 0) min_value = value;
-		if (max_value === void 0 || compareValues(value, max_value) > 0) max_value = value;
+		const statisticValue = element.converted_type === "DECIMAL" && typeof value === "number" ? BigInt(Math.round(value * 10 ** (element.scale || 0))) : value;
+		if (min_value === void 0 || compareValues(statisticValue, min_value) < 0) min_value = statisticValue;
+		if (max_value === void 0 || compareValues(statisticValue, max_value) > 0) max_value = statisticValue;
 	}
 	if (min_value === 0) min_value = -0;
 	if (max_value === 0) max_value = 0;

@@ -1,6 +1,6 @@
+import { Encodings, PageTypes } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/constants.mjs";
 import { ByteWriter } from "./bytewriter.mjs";
 import { serializeTCompactProtocol } from "./thrift.mjs";
-import { Encodings, PageTypes } from "../../../../hyparquet@1.28.1/node_modules/hyparquet/src/constants.mjs";
 import { deltaBinaryPack, deltaByteArray, deltaLengthByteArray } from "./delta.mjs";
 import { writeRleBitPackedHybrid } from "./encoding.mjs";
 import { writePlain } from "./plain.mjs";

@@ -1,4 +1,4 @@
-import { getMaxDefinitionLevel, isListLike, isMapLike } from "../../../../hyparquet@1.28.1/node_modules/hyparquet/src/schema.mjs";
+import { getMaxDefinitionLevel, isListLike, isMapLike } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/schema.mjs";
 function encodeNestedValues(treePath, rows) {
 	const schemaPath = treePath.map((n) => n.element);
 	if (treePath.length < 2) throw new Error("parquet schema path must include column");

@@ -1,5 +1,5 @@
+import { BoundaryOrders } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/constants.mjs";
 import { serializeTCompactProtocol } from "./thrift.mjs";
-import { BoundaryOrders } from "../../../../hyparquet@1.28.1/node_modules/hyparquet/src/constants.mjs";
 function writeIndexes(writer, pageIndexes) {
 	for (const { chunk, columnIndex } of pageIndexes) writeColumnIndex(writer, chunk, columnIndex);
 	for (const { chunk, offsetIndex } of pageIndexes) writeOffsetIndex(writer, chunk, offsetIndex);

@@ -1,6 +1,6 @@
-import { getSchemaPath } from "../../../../hyparquet@1.28.1/node_modules/hyparquet/src/schema.mjs";
+import { CompressionCodecs, ConvertedTypes, EdgeInterpolationAlgorithms, Encodings, FieldRepetitionTypes, PageTypes, ParquetTypes } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/constants.mjs";
+import { getSchemaPath } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/schema.mjs";
 import { ThriftStruct, serializeTCompactProtocol } from "./thrift.mjs";
-import { CompressionCodecs, ConvertedTypes, EdgeInterpolationAlgorithms, Encodings, FieldRepetitionTypes, PageTypes, ParquetTypes } from "../../../../hyparquet@1.28.1/node_modules/hyparquet/src/constants.mjs";
 import { unconvertStatistics } from "./unconvert.mjs";
 function writeMetadata(writer, metadata) {
 	const compact = {

@@ -1,4 +1,4 @@
-import { getSchemaPath } from "../../../../hyparquet@1.28.1/node_modules/hyparquet/src/schema.mjs";
+import { getSchemaPath } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/schema.mjs";
 import { writeBlooms } from "./bloom.mjs";
 import { encodeVariantColumn } from "./variant.mjs";
 import { writeColumn } from "./column.mjs";

@@ -1,20 +1,15 @@
 import { ByteWriter } from "./bytewriter.mjs";
-const BLOCK_SIZE = 65536;
 const MAX_HASH_TABLE_BITS = 14;
+const MAX_OFFSET = 65535;
 const globalHashTables = new Array(15);
 function snappyCompress(input) {
 	const writer = new ByteWriter();
 	writer.appendVarInt(input.length);
-	let pos = 0;
-	while (pos < input.length) {
-		const fragmentSize = Math.min(input.length - pos, BLOCK_SIZE);
-		compressFragment(writer, input, pos, fragmentSize);
-		pos += fragmentSize;
-	}
+	compressInput(writer, input);
 	return writer.getBytes();
 }
 function hashFunc(key, hashFuncShift) {
-	return key * 506832829 >>> hashFuncShift;
+	return Math.imul(key, 506832829) >>> hashFuncShift;
 }
 function load32(array, pos) {
 	return array[pos] + (array[pos + 1] << 8) + (array[pos + 2] << 16) + (array[pos + 3] << 24);
@@ -24,13 +19,11 @@ function equals32(array, pos1, pos2) {
 }
 function emitLiteral(writer, input, ip, len) {
 	if (len <= 60) writer.appendUint8(len - 1 << 2);
-	else if (len < 256) {
-		writer.appendUint8(240);
-		writer.appendUint8(len - 1);
-	} else {
-		writer.appendUint8(244);
-		writer.appendUint8(len - 1 & 255);
-		writer.appendUint8(len - 1 >>> 8);
+	else {
+		const value = len - 1;
+		const bytes = value < 256 ? 1 : value < 65536 ? 2 : value < 16777216 ? 3 : 4;
+		writer.appendUint8(59 + bytes << 2);
+		for (let i = 0; i < bytes; i++) writer.appendUint8(value >>> i * 8);
 	}
 	writer.appendBytes(input.subarray(ip, ip + len));
 }
@@ -55,18 +48,19 @@ function emitCopy(writer, offset, len) {
 	}
 	emitCopyLessThan64(writer, offset, len);
 }
-function compressFragment(writer, input, ip, inputSize) {
+function compressInput(writer, input) {
+	const inputSize = input.length;
 	let hashTableBits = 1;
 	while (1 << hashTableBits <= inputSize && hashTableBits <= MAX_HASH_TABLE_BITS) hashTableBits++;
 	hashTableBits--;
 	const hashFuncShift = 32 - hashTableBits;
-	globalHashTables[hashTableBits] ??= new Uint16Array(1 << hashTableBits);
+	globalHashTables[hashTableBits] ??= new Uint32Array(1 << hashTableBits);
 	const hashTable = globalHashTables[hashTableBits];
 	hashTable.fill(0);
-	const ipEnd = ip + inputSize;
+	const ipEnd = inputSize;
 	let ipLimit;
-	const baseIp = ip;
-	let nextEmit = ip;
+	let ip = 0;
+	let nextEmit = 0;
 	let hash, nextHash;
 	let nextIp, candidate, skip;
 	let bytesBetweenHashLookups;
@@ -92,9 +86,9 @@ function compressFragment(writer, input, ip, inputSize) {
 					break;
 				}
 				nextHash = hashFunc(load32(input, nextIp), hashFuncShift);
-				candidate = baseIp + hashTable[hash];
-				hashTable[hash] = ip - baseIp;
-			} while (!equals32(input, ip, candidate));
+				candidate = hashTable[hash];
+				hashTable[hash] = ip;
+			} while (ip - candidate > MAX_OFFSET || !equals32(input, ip, candidate));
 			if (!flag) break;
 			emitLiteral(writer, input, nextEmit, ip - nextEmit);
 			do {
@@ -110,11 +104,11 @@ function compressFragment(writer, input, ip, inputSize) {
 					break;
 				}
 				prevHash = hashFunc(load32(input, ip - 1), hashFuncShift);
-				hashTable[prevHash] = ip - 1 - baseIp;
+				hashTable[prevHash] = ip - 1;
 				curHash = hashFunc(load32(input, ip), hashFuncShift);
-				candidate = baseIp + hashTable[curHash];
-				hashTable[curHash] = ip - baseIp;
-			} while (equals32(input, ip, candidate));
+				candidate = hashTable[curHash];
+				hashTable[curHash] = ip;
+			} while (ip - candidate <= MAX_OFFSET && equals32(input, ip, candidate));
 			if (!flag) break;
 			ip++;
 			nextHash = hashFunc(load32(input, ip), hashFuncShift);

@@ -1,4 +1,4 @@
-import "../../../../hyparquet@1.28.1/node_modules/hyparquet/types/node.mjs";
+import "../../../../hyparquet@1.29.2/node_modules/hyparquet/types/node.mjs";
 interface Writer {
   buffer: ArrayBuffer;
   view: DataView;

@@ -1,5 +1,5 @@
-import { toJson } from "../../../../hyparquet@1.28.1/node_modules/hyparquet/src/utils.mjs";
-import "../../../../hyparquet@1.28.1/node_modules/hyparquet/src/node.mjs";
+import { toJson } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/utils.mjs";
+import "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/node.mjs";
 import { geojsonToWkb } from "./wkb.mjs";
 const dayMillis = 864e5;
 function mapConverted(values, convert) {
@@ -15,9 +15,10 @@ function unconvert(element, values) {
 	const { type, converted_type: ctype, logical_type: ltype } = element;
 	if (ctype === "DECIMAL") {
 		const factor = 10 ** (element.scale || 0);
-		return values.map((v) => {
+		return Array.from(values, (v) => {
 			if (v === null || v === void 0) return v;
-			if (typeof v !== "number") throw new Error("DECIMAL must be a number");
+			if (typeof v === "bigint") return unconvertDecimal(element, v);
+			if (typeof v !== "number") throw new Error("DECIMAL must be a number or bigint");
 			return unconvertDecimal(element, BigInt(Math.round(v * factor)));
 		});
 	}
@@ -120,6 +121,7 @@ function minMaxIsExact(value, element) {
 	const { type } = element;
 	if (type !== "BYTE_ARRAY" && type !== "FIXED_LEN_BYTE_ARRAY") return void 0;
 	if (element.logical_type?.type === "UUID") return void 0;
+	if (element.converted_type === "DECIMAL") return void 0;
 	return (value instanceof Uint8Array ? value : new TextEncoder().encode(value.toString())).length > STATS_TRUNCATE_LENGTH ? false : void 0;
 }
 function unconvertMinMax(value, element, isMax) {
@@ -128,13 +130,13 @@ function unconvertMinMax(value, element, isMax) {
 	if (type === "BOOLEAN") return new Uint8Array([value ? 1 : 0]);
 	if (element.logical_type?.type === "UUID" && (typeof value === "string" || value instanceof Uint8Array)) return unconvertUuid(value);
 	if (converted_type === "DECIMAL") {
-		if (typeof value !== "number") throw new Error("DECIMAL must be a number");
+		if (typeof value !== "number" && typeof value !== "bigint") throw new Error("DECIMAL must be a number or bigint");
 		const factor = 10 ** (element.scale || 0);
-		const out = unconvertDecimal(element, BigInt(Math.round(value * factor)));
+		const out = unconvertDecimal(element, typeof value === "bigint" ? value : BigInt(Math.round(value * factor)));
 		if (out instanceof Uint8Array) return out;
 		if (typeof out === "number") {
 			const buffer = /* @__PURE__ */ new ArrayBuffer(4);
-			new DataView(buffer).setFloat32(0, out, true);
+			new DataView(buffer).setInt32(0, out, true);
 			return new Uint8Array(buffer);
 		}
 		if (typeof out === "bigint") {

@@ -10,6 +10,7 @@ function encodeVariantColumn(values, shredding, column) {
 	if (column?.required) {
 		for (let i = 0; i < values.length; i++) if (values[i] === void 0) throw new Error(`required variant column ${column.name} has undefined value at index ${i}`);
 	}
+	const scratchWriters = [];
 	const shreddingConfig = shredding && normalizeShreddingConfig(shredding);
 	if (shreddingConfig) {
 		const metadataCache = /* @__PURE__ */ new Map();
@@ -20,7 +21,7 @@ function encodeVariantColumn(values, shredding, column) {
 			const { metadata, keyIndex } = getVariantRowMetadata(keys, metadataCache);
 			return {
 				metadata,
-				...encodeShredded(value, shreddingConfig, keyIndex, true)
+				...encodeShredded(value, shreddingConfig, keyIndex, true, scratchWriters)
 			};
 		});
 	}
@@ -32,29 +33,29 @@ function encodeVariantColumn(values, shredding, column) {
 		if (value === void 0) return null;
 		return {
 			metadata,
-			value: writeVariantValue(value, keyIndex)
+			value: writeVariantValue(value, keyIndex, scratchWriters)
 		};
 	});
 }
-function encodeShredded(value, shredType, keyIndex, allowPartialObjects) {
+function encodeShredded(value, shredType, keyIndex, allowPartialObjects, scratchWriters) {
 	if (value === null || value === void 0) return {
 		value: VARIANT_NULL,
 		typed_value: null
 	};
 	if (Array.isArray(shredType)) {
 		if (!Array.isArray(value)) return {
-			value: writeVariantValue(value, keyIndex),
+			value: writeVariantValue(value, keyIndex, scratchWriters),
 			typed_value: null
 		};
 		const elemShred = shredType[0];
 		return {
 			value: null,
-			typed_value: value.map((el) => encodeShredded(el, elemShred, keyIndex, false))
+			typed_value: value.map((el) => encodeShredded(el, elemShred, keyIndex, false, scratchWriters))
 		};
 	}
 	if (typeof shredType === "object") {
 		if (typeof value !== "object" || Array.isArray(value) || value instanceof Date || value instanceof Uint8Array) return {
-			value: writeVariantValue(value, keyIndex),
+			value: writeVariantValue(value, keyIndex, scratchWriters),
 			typed_value: null
 		};
 		const remaining = {};
@@ -65,21 +66,21 @@ function encodeShredded(value, shredType, keyIndex, allowPartialObjects) {
 			hasRemaining = true;
 		}
 		if (hasRemaining && !allowPartialObjects) return {
-			value: writeVariantValue(value, keyIndex),
+			value: writeVariantValue(value, keyIndex, scratchWriters),
 			typed_value: null
 		};
 		const fieldNames = Object.keys(shredType);
 		if (fieldNames.some((fieldName) => (!Object.prototype.hasOwnProperty.call(value, fieldName) || value[fieldName] === void 0) && keyIndex.has(fieldName))) return {
-			value: writeVariantValue(value, keyIndex),
+			value: writeVariantValue(value, keyIndex, scratchWriters),
 			typed_value: null
 		};
 		const typedValue = {};
 		for (const fieldName of fieldNames) {
 			if (!Object.prototype.hasOwnProperty.call(value, fieldName) || value[fieldName] === void 0) continue;
-			typedValue[fieldName] = encodeShredded(value[fieldName], shredType[fieldName], keyIndex, false);
+			typedValue[fieldName] = encodeShredded(value[fieldName], shredType[fieldName], keyIndex, false, scratchWriters);
 		}
 		return {
-			value: hasRemaining ? writeVariantValue(remaining, keyIndex) : null,
+			value: hasRemaining ? writeVariantValue(remaining, keyIndex, scratchWriters) : null,
 			typed_value: typedValue
 		};
 	}
@@ -88,7 +89,7 @@ function encodeShredded(value, shredType, keyIndex, allowPartialObjects) {
 		typed_value: value
 	};
 	return {
-		value: writeVariantValue(value, keyIndex),
+		value: writeVariantValue(value, keyIndex, scratchWriters),
 		typed_value: null
 	};
 }
@@ -255,12 +256,12 @@ function writeVariantMetadata(dictionary) {
 	}
 	return bytes;
 }
-function writeVariantValue(value, keyIndex) {
+function writeVariantValue(value, keyIndex, scratchWriters) {
 	const writer = new ByteWriter(8);
-	writeValue(value, writer, keyIndex);
+	writeValue(value, writer, keyIndex, scratchWriters);
 	return writer.getBytes();
 }
-function writeValue(val, writer, keyIndex) {
+function writeValue(val, writer, keyIndex, scratchWriters) {
 	if (val === null || val === void 0) {
 		writer.appendUint8(0);
 		return;
@@ -325,16 +326,16 @@ function writeValue(val, writer, keyIndex) {
 		return;
 	}
 	if (Array.isArray(val)) {
-		writeVariantArray(val, writer, keyIndex);
+		writeVariantArray(val, writer, keyIndex, scratchWriters);
 		return;
 	}
 	if (typeof val === "object") {
-		writeVariantObject(val, writer, keyIndex);
+		writeVariantObject(val, writer, keyIndex, scratchWriters);
 		return;
 	}
 	throw new Error(`variant cannot encode value: ${val}`);
 }
-function writeVariantObject(obj, writer, keyIndex) {
+function writeVariantObject(obj, writer, keyIndex, scratchWriters) {
 	const entries = Object.keys(obj).filter((key) => obj[key] !== void 0).map((key) => {
 		const id = keyIndex.get(key);
 		if (id === void 0) throw new Error(`variant key not in dictionary: ${key}`);
@@ -346,11 +347,11 @@ function writeVariantObject(obj, writer, keyIndex) {
 	entries.sort((a, b) => a.id - b.id);
 	const numElements = entries.length;
 	const idWidth = byteWidth(numElements > 0 ? entries[numElements - 1].id : 0);
-	const scratch = new ByteWriter(8);
+	const scratch = scratchWriters.pop() ?? new ByteWriter(8);
 	const offsets = new Array(numElements + 1);
 	offsets[0] = 0;
 	for (let i = 0; i < numElements; i++) {
-		writeValue(obj[entries[i].key], scratch, keyIndex);
+		writeValue(obj[entries[i].key], scratch, keyIndex, scratchWriters);
 		offsets[i + 1] = scratch.index;
 	}
 	const offsetWidth = byteWidth(offsets[numElements]);
@@ -361,14 +362,17 @@ function writeVariantObject(obj, writer, keyIndex) {
 	for (const { id } of entries) appendUnsignedLE(writer, id, idWidth);
 	for (const off of offsets) appendUnsignedLE(writer, off, offsetWidth);
 	writer.appendBytes(scratch.getBytes());
+	scratch.index = 0;
+	scratch.offset = 0;
+	scratchWriters.push(scratch);
 }
-function writeVariantArray(arr, writer, keyIndex) {
+function writeVariantArray(arr, writer, keyIndex, scratchWriters) {
 	const numElements = arr.length;
-	const scratch = new ByteWriter(8);
+	const scratch = scratchWriters.pop() ?? new ByteWriter(8);
 	const offsets = new Array(numElements + 1);
 	offsets[0] = 0;
 	for (let i = 0; i < numElements; i++) {
-		writeValue(arr[i], scratch, keyIndex);
+		writeValue(arr[i], scratch, keyIndex, scratchWriters);
 		offsets[i + 1] = scratch.index;
 	}
 	const offsetWidth = byteWidth(offsets[numElements]);
@@ -378,6 +382,9 @@ function writeVariantArray(arr, writer, keyIndex) {
 	else writer.appendUint8(numElements);
 	for (const off of offsets) appendUnsignedLE(writer, off, offsetWidth);
 	writer.appendBytes(scratch.getBytes());
+	scratch.index = 0;
+	scratch.offset = 0;
+	scratchWriters.push(scratch);
 }
 function byteWidth(maxValue) {
 	if (maxValue <= 255) return 1;

@@ -1,4 +1,4 @@
-import "../../../../hyparquet@1.28.1/node_modules/hyparquet/types/node.mjs";
+import "../../../../hyparquet@1.29.2/node_modules/hyparquet/types/node.mjs";
 import { Writer as Writer$1 } from "../src/types.mjs";
 import "./write.mjs";
 import "./write-rows.mjs";

package/dist/_virtual/node_modules/.pnpm/hyparquet-writer@0.16.9_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/types/parquet-writer.d.mts ADDED Viewed

	@@ -0,0 +1,2 @@
1	+ import "../../../../hyparquet@1.29.2/node_modules/hyparquet/types/node.mjs";
2	+ import "../src/types.mjs";

	@@ -0,0 +1,2 @@
1	+ import "../../../../hyparquet@1.29.2/node_modules/hyparquet/types/node.mjs";
2	+ import "../src/types.mjs";

package/dist/_virtual/node_modules/.pnpm/{hyparquet@1.29.1 → hyparquet@1.29.2}/node_modules/hyparquet/src/index.mjs RENAMED Viewed

@@ -1,8 +1,8 @@
-import { parquetMetadataAsync } from "./metadata.mjs";
+import { parquetMetadataAsync, parquetSchema } from "./metadata.mjs";
 import "./indexes.mjs";
-import { asyncBufferFromUrl, cachedAsyncBuffer } from "./utils.mjs";
+import { asyncBufferFromUrl, cachedAsyncBuffer, toJson } from "./utils.mjs";
 import "./snappy.mjs";
 import { parquetScan } from "./scan.mjs";
 import { parquetReadObjects } from "./read.mjs";
 import "./query.mjs";
-export { asyncBufferFromUrl, cachedAsyncBuffer, parquetMetadataAsync, parquetReadObjects, parquetScan };
+export { asyncBufferFromUrl, cachedAsyncBuffer, parquetMetadataAsync, parquetReadObjects, parquetScan, parquetSchema, toJson };

package/dist/_virtual/node_modules/.pnpm/{hyparquet@1.29.1 → hyparquet@1.29.2}/node_modules/hyparquet/src/node.mjs RENAMED Viewed

@@ -1,8 +1,8 @@
-import { parquetMetadataAsync } from "./metadata.mjs";
+import { parquetMetadataAsync, parquetSchema } from "./metadata.mjs";
 import "./indexes.mjs";
-import { asyncBufferFromUrl, cachedAsyncBuffer } from "./utils.mjs";
+import { asyncBufferFromUrl, cachedAsyncBuffer, toJson } from "./utils.mjs";
 import "./snappy.mjs";
 import { parquetScan } from "./scan.mjs";
 import { parquetReadObjects } from "./read.mjs";
 import "./index.mjs";
-export { asyncBufferFromUrl, cachedAsyncBuffer, parquetMetadataAsync, parquetReadObjects, parquetScan };
+export { asyncBufferFromUrl, cachedAsyncBuffer, parquetMetadataAsync, parquetReadObjects, parquetScan, parquetSchema, toJson };

package/dist/_virtual/node_modules/.pnpm/{hyparquet@1.29.1 → hyparquet@1.29.2}/node_modules/hyparquet/src/utils.mjs RENAMED Viewed

@@ -1,4 +1,21 @@
 import { defaultInitialFetchSize } from "./metadata.mjs";
+function toJson(obj) {
+	if (obj === void 0) return null;
+	if (typeof obj === "bigint") return Number(obj);
+	if (Object.is(obj, -0)) return 0;
+	if (Array.isArray(obj)) return obj.map(toJson);
+	if (obj instanceof Uint8Array) return Array.from(obj);
+	if (obj instanceof Date) return obj.toISOString();
+	if (obj instanceof Object) {
+		const newObj = {};
+		for (const key of Object.keys(obj)) {
+			if (obj[key] === void 0) continue;
+			newObj[key] = toJson(obj[key]);
+		}
+		return newObj;
+	}
+	return obj;
+}
 function concat(aaa, bbb) {
 	const chunk = 1e4;
 	for (let i = 0; i < bbb.length; i += chunk) aaa.push(...bbb.slice(i, i + chunk));
@@ -125,4 +142,4 @@ function flatten(chunks) {
 	for (const chunk of chunks) concat(output, chunk);
 	return output;
 }
-export { asyncBufferFromUrl, byteLengthFromUrl, cachedAsyncBuffer, concat, equals, flatten };
+export { asyncBufferFromUrl, byteLengthFromUrl, cachedAsyncBuffer, concat, equals, flatten, toJson };

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.26_patch_hash=75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244 → icebird@0.8.27_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/avro/avro.read.mjs RENAMED Viewed

@@ -1,4 +1,4 @@
-import { parseDecimal } from "../../../../../hyparquet@1.29.1/node_modules/hyparquet/src/convert.mjs";
+import { parseDecimal } from "../../../../../hyparquet@1.29.2/node_modules/hyparquet/src/convert.mjs";
 import { gunzip } from "../../../../../hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/gzip.mjs";
 import "../../../../../hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/index.mjs";
 import { readZigZag, readZigZagBigInt } from "./avro.metadata.mjs";

@@ -1,5 +1,5 @@
-import { ByteWriter } from "../../../../../hyparquet-writer@0.16.6_patch_hash=549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/src/bytewriter.mjs";
-import "../../../../../hyparquet-writer@0.16.6_patch_hash=549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/src/node.mjs";
+import { ByteWriter } from "../../../../../hyparquet-writer@0.16.9_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/bytewriter.mjs";
+import "../../../../../hyparquet-writer@0.16.9_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/node.mjs";
 function avroWrite({ writer, schema, records, blockSize = 512, metadata }) {
 	writer.appendUint32(23749199);
 	const meta = {

@@ -10,6 +10,18 @@ function deleteFileAppliesToDataEntry(dataEntry, deleteEntry, metadata, deleteTy
 	} else if (deleteSequenceNumber < dataSequenceNumber) return false;
 	return samePartition(dataEntry, deleteEntry);
 }
+function isDeletionVector(deleteEntry) {
+	const dataFile = deleteEntry.data_file;
+	return dataFile.file_format.toLowerCase() === "puffin" || dataFile.content_offset != null || dataFile.content_size_in_bytes != null;
+}
+function applicablePositionDeletes(dataEntry, positionDeleteGroups, metadata) {
+	const positions = /* @__PURE__ */ new Set();
+	if (!positionDeleteGroups) return positions;
+	const applicable = positionDeleteGroups.filter((group) => deleteFileAppliesToDataEntry(dataEntry, group.deleteEntry, metadata, "position"));
+	const vectors = applicable.filter((group) => isDeletionVector(group.deleteEntry));
+	for (const group of vectors.length ? vectors : applicable) for (const pos of group.positions) positions.add(pos);
+	return positions;
+}
 function isUnpartitioned(metadata, specId) {
 	return metadata["partition-specs"].find((s) => s["spec-id"] === specId)?.fields.length === 0;
 }
@@ -31,4 +43,4 @@ function partitionValuesEqual(a, b) {
 	if (typeof a === "number" && typeof b === "number") return Object.is(a, b);
 	return valuesEqual(a, b);
 }
-export { deleteFileAppliesToDataEntry };
+export { applicablePositionDeletes, deleteFileAppliesToDataEntry, isDeletionVector };

@@ -1,15 +1,16 @@
-import { parquetMetadataAsync } from "../../../../hyparquet@1.29.1/node_modules/hyparquet/src/metadata.mjs";
-import { asyncBufferFromUrl, cachedAsyncBuffer } from "../../../../hyparquet@1.29.1/node_modules/hyparquet/src/utils.mjs";
-import { parquetReadObjects } from "../../../../hyparquet@1.29.1/node_modules/hyparquet/src/read.mjs";
-import "../../../../hyparquet@1.29.1/node_modules/hyparquet/src/node.mjs";
+import { parquetMetadataAsync } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/metadata.mjs";
+import { asyncBufferFromUrl, cachedAsyncBuffer } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/utils.mjs";
+import { parquetReadObjects } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/read.mjs";
+import "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/node.mjs";
 import { compressors } from "../../../../hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/compressors.mjs";
 import "../../../../hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/index.mjs";
-import { ByteWriter } from "../../../../hyparquet-writer@0.16.6_patch_hash=549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/src/bytewriter.mjs";
-import "../../../../hyparquet-writer@0.16.6_patch_hash=549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/src/node.mjs";
+import { ByteWriter } from "../../../../hyparquet-writer@0.16.9_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/bytewriter.mjs";
+import "../../../../hyparquet-writer@0.16.9_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/node.mjs";
 import { avroMetadata } from "./avro/avro.metadata.mjs";
 import { avroRead } from "./avro/avro.read.mjs";
-import { puffinReadDeletionVector } from "./puffin/puffin.mjs";
 import { sanitize } from "./utils.mjs";
+import { isDeletionVector } from "./delete.mjs";
+import { puffinReadDeletionVector } from "./puffin/puffin.mjs";
 const SMALL_PARQUET_FILE_SIZE = 8388608;
 const SMALL_PARQUET_FOOTER_SIZE = 131072;
 function readParquetMetadata(file) {
@@ -217,10 +218,6 @@ function addPositionDeleteGroup(positionDeletesMap, targetPath, deleteEntry, pos
 		positions
 	});
 }
-function isDeletionVector(deleteEntry) {
-	const dataFile = deleteEntry.data_file;
-	return dataFile.file_format.toLowerCase() === "puffin" || dataFile.content_offset != null || dataFile.content_size_in_bytes != null;
-}
 function equalityColumnNamesById(parquetMetadata, equalityIds) {
 	const kv = parquetMetadata.key_value_metadata?.find((k) => k.key === "iceberg.schema");
 	if (!kv?.value) throw new Error("equality delete missing iceberg.schema parquet metadata");