@gscdump/engine 2.1.1 → 2.3.0

package/dist/adapters/hyparquet.d.mts

@@ -1,5 +1,5 @@
-import { ParquetQueryFilter } from "../node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/types/index.mjs";
-import "../node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/types/node.mjs";
+import { ParquetQueryFilter } from "../node_modules/.pnpm/hyparquet@1.28.2/node_modules/hyparquet/types/index.mjs";
+import "../node_modules/.pnpm/hyparquet@1.28.2/node_modules/hyparquet/types/node.mjs";
 import { CodecCtx, DataSource, ParquetCodec, Row, TableName } from "../storage.mjs";
 import { ColumnDef } from "../schema.mjs";
 declare function encodeRowsToParquet(table: TableName, rows: readonly Row[]): Uint8Array;

package/dist/adapters/hyparquet.mjs

@@ -1,10 +1,10 @@
 import { TABLE_METADATA } from "../drizzle-schema.mjs";
 import { SCHEMAS } from "../schema.mjs";
-import { parquetReadObjects } from "../node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/read.mjs";
-import "../node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/node.mjs";
-import { ByteWriter } from "../node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/bytewriter.mjs";
-import { parquetWriteRows } from "../node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/write-rows.mjs";
-import "../node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/node.mjs";
+import { parquetReadObjects } from "../node_modules/.pnpm/hyparquet@1.28.2/node_modules/hyparquet/src/read.mjs";
+import "../node_modules/.pnpm/hyparquet@1.28.2/node_modules/hyparquet/src/node.mjs";
+import { ByteWriter } from "../node_modules/.pnpm/hyparquet-writer@0.16.6_patch_hash_549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/src/bytewriter.mjs";
+import { parquetWriteRows } from "../node_modules/.pnpm/hyparquet-writer@0.16.6_patch_hash_549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/src/write-rows.mjs";
+import "../node_modules/.pnpm/hyparquet-writer@0.16.6_patch_hash_549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/src/node.mjs";
 const ROW_GROUP_SIZE = 25e3;
 const DEFAULT_COMPACTION_READ_CONCURRENCY = 8;
 const MAX_COMPACTION_READ_CONCURRENCY = 32;
@@ -197,7 +197,8 @@ async function decodeParquetToRows(bytes, opts = {}) {
 		...opts.columns ? { columns: [...opts.columns] } : {},
 		...opts.filter ? {
 			filter: opts.filter,
-			useBloomFilters: true
+			useBloomFilters: true,
+			usePageIndex: true
 		} : {}
 	}));
 }

package/dist/iceberg/catalog.d.mts

@@ -1,7 +1,7 @@
 import { EngineError } from "../errors.mjs";
 import { IcebergTableName, PartitionKeyEncoding } from "./schema.mjs";
-import { icebergCreateTable } from "../node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/write/write.mjs";
-import "../node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/index.mjs";
+import { icebergCreateTable } from "../node_modules/.pnpm/icebird@0.8.22_patch_hash_4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6/node_modules/icebird/types/write/write.mjs";
+import "../node_modules/.pnpm/icebird@0.8.22_patch_hash_4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6/node_modules/icebird/types/index.mjs";
 import { Result } from "gscdump/result";
 import { CommitRetryOptions, ConnectIcebergOptions, IcebergCatalogConfig, IcebergConnection, IcebergListedDataFile, QueryProfiler } from "@gscdump/lakehouse";
 type IcebirdCreateTableOptions = Parameters<typeof icebergCreateTable>[0];

package/dist/iceberg/catalog.mjs

@@ -2,8 +2,8 @@ import { engineErrors } from "../errors.mjs";
 import { TABLE_METADATA } from "../drizzle-schema.mjs";
 import "../schema.mjs";
 import { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_PARTITION_SPEC, ICEBERG_TABLES, icebergSchemasFor } from "./schema.mjs";
-import { icebergCreateTable } from "../node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/write.mjs";
-import "../node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/index.mjs";
+import { icebergCreateTable } from "../node_modules/.pnpm/icebird@0.8.22_patch_hash_4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6/node_modules/icebird/src/write/write.mjs";
+import "../node_modules/.pnpm/icebird@0.8.22_patch_hash_4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6/node_modules/icebird/src/index.mjs";
 import { err, ok } from "gscdump/result";
 import { ICEBERG_TYPE_MAP, connectIcebergCatalog, dropIcebergTables, resolveIcebergDataFiles } from "@gscdump/lakehouse";
 import { icebergAppendRetrying } from "@gscdump/lakehouse/unsafe-raw";

package/dist/node_modules/.pnpm/{hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec → hyparquet-writer@0.16.6_patch_hash_549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a}/node_modules/hyparquet-writer/src/bloom.mjs

@@ -1,4 +1,4 @@
-import { hashParquetValue } from "../../../../hyparquet@1.26.1/node_modules/hyparquet/src/bloom.mjs";
+import { hashParquetValue } from "../../../../hyparquet@1.28.1/node_modules/hyparquet/src/bloom.mjs";
 import { serializeTCompactProtocol } from "./thrift.mjs";
 const SALT = new Uint32Array([
 	1203114875,

package/dist/node_modules/.pnpm/hyparquet-writer@0.16.6_patch_hash_549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/src/bytewriter.mjs

@@ -0,0 +1,92 @@
+var ByteWriter = class {
+	constructor(initalSize = 1024) {
+		this.buffer = new ArrayBuffer(initalSize);
+		this.view = new DataView(this.buffer);
+		this.offset = 0;
+		this.index = 0;
+	}
+	ensure(size) {
+		if (this.index + size > this.buffer.byteLength) {
+			const newSize = Math.max(this.buffer.byteLength * 2, this.index + size);
+			const newBuffer = new ArrayBuffer(newSize);
+			new Uint8Array(newBuffer).set(new Uint8Array(this.buffer));
+			this.buffer = newBuffer;
+			this.view = new DataView(this.buffer);
+		}
+	}
+	finish() {}
+	getBuffer() {
+		return this.buffer.slice(0, this.index);
+	}
+	getBytes() {
+		return new Uint8Array(this.buffer, 0, this.index);
+	}
+	appendUint8(value) {
+		this.ensure(this.index + 1);
+		this.view.setUint8(this.index, value);
+		this.offset++;
+		this.index++;
+	}
+	appendUint32(value) {
+		this.ensure(this.index + 4);
+		this.view.setUint32(this.index, value, true);
+		this.offset += 4;
+		this.index += 4;
+	}
+	appendInt32(value) {
+		this.ensure(this.index + 4);
+		this.view.setInt32(this.index, value, true);
+		this.offset += 4;
+		this.index += 4;
+	}
+	appendInt64(value) {
+		this.ensure(this.index + 8);
+		this.view.setBigInt64(this.index, BigInt(value), true);
+		this.offset += 8;
+		this.index += 8;
+	}
+	appendFloat32(value) {
+		this.ensure(this.index + 8);
+		this.view.setFloat32(this.index, value, true);
+		this.offset += 4;
+		this.index += 4;
+	}
+	appendFloat64(value) {
+		this.ensure(this.index + 8);
+		this.view.setFloat64(this.index, value, true);
+		this.offset += 8;
+		this.index += 8;
+	}
+	appendBuffer(value) {
+		this.appendBytes(new Uint8Array(value));
+	}
+	appendBytes(value) {
+		this.ensure(this.index + value.length);
+		new Uint8Array(this.buffer, this.index, value.length).set(value);
+		this.offset += value.length;
+		this.index += value.length;
+	}
+	appendVarInt(value) {
+		while (true) if ((value & -128) === 0) {
+			this.appendUint8(value);
+			return;
+		} else {
+			this.appendUint8(value & 127 | 128);
+			value >>>= 7;
+		}
+	}
+	appendVarBigInt(value) {
+		while (true) if ((value & -128n) === 0n) {
+			this.appendUint8(Number(value));
+			return;
+		} else {
+			this.appendUint8(Number(value & 127n | 128n));
+			value >>= 7n;
+		}
+	}
+	appendZigZag(value) {
+		if (typeof value === "number") this.appendVarInt(value << 1 ^ value >> 31);
+		else this.appendVarBigInt(value << 1n ^ value >> 63n);
+	}
+};
+export { ByteWriter };

package/dist/node_modules/.pnpm/{hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec → hyparquet-writer@0.16.6_patch_hash_549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a}/node_modules/hyparquet-writer/src/column.mjs

@@ -4,7 +4,7 @@ import { estimateValueSize, useDictionary, writeDictionaryPage } from "./diction
 import { geospatialStatistics } from "./geospatial.mjs";
 import { unconvert, unconvertMinMax } from "./unconvert.mjs";
 function writeColumn({ writer, column, pageData }) {
-	const { columnName, element, schemaPath, stats, pageSize, encoding: userEncoding } = column;
+	const { columnName, element, schemaPath, stats, pageSize, dictionarySize, encoding: userEncoding } = column;
 	const { type, type_length } = element;
 	if (!type) throw new Error(`column ${columnName} cannot determine type`);
 	const { values, definitionLevels, repetitionLevels, maxDefinitionLevel } = pageData;
@@ -21,7 +21,9 @@ function writeColumn({ writer, column, pageData }) {
 		bloomFilter = builder.finalize();
 	}
 	let dictionary_page_offset;
-	const { dictionary, indexes } = useDictionary(values, type, type_length, userEncoding, pageSize);
+	const selectPhysical = element.converted_type !== "UTF8";
+	const dictionaryValues = selectPhysical ? unconvert(element, values) : values;
+	const { dictionary, indexes } = useDictionary(dictionaryValues, type, type_length, userEncoding, dictionarySize, maxDefinitionLevel === 0);
 	let encoding;
 	let writeValues;
 	let writeType = type;
@@ -30,11 +32,11 @@ function writeColumn({ writer, column, pageData }) {
 		writeType = "INT32";
 		encoding = "RLE_DICTIONARY";
 		dictionary_page_offset = BigInt(writer.offset);
-		const unconverted = unconvert(element, dictionary);
+		const unconverted = selectPhysical ? dictionary : unconvert(element, dictionary);
 		writeDictionaryPage(writer, column, unconverted);
 	} else {
-		writeValues = unconvert(element, values);
-		encoding = userEncoding ?? (type === "BOOLEAN" && values.length > 16 ? "RLE" : "PLAIN");
+		writeValues = selectPhysical ? dictionaryValues : unconvert(element, values);
+		encoding = userEncoding && userEncoding !== "RLE_DICTIONARY" ? userEncoding : type === "BOOLEAN" && values.length > 16 ? "RLE" : "PLAIN";
 	}
 	encodings.push(encoding);
 	const pageBoundaries = getPageBoundaries(writeValues, writeType, type_length, pageSize);
@@ -74,12 +76,14 @@ function writeColumn({ writer, column, pageData }) {
 			columnIndex.max_values.push(unconvertMinMax(max_value, element, true) ?? /* @__PURE__ */ new Uint8Array());
 			columnIndex.null_counts?.push(null_count);
 			if (prevMinValue !== void 0 && min_value !== void 0) {
-				if (prevMinValue > min_value) ascending = false;
-				if (prevMinValue < min_value) descending = false;
+				const order = compareValues(prevMinValue, min_value);
+				if (order > 0) ascending = false;
+				if (order < 0) descending = false;
 			}
 			if (prevMaxValue !== void 0 && max_value !== void 0) {
-				if (prevMaxValue > max_value) ascending = false;
-				if (prevMaxValue < max_value) descending = false;
+				const order = compareValues(prevMaxValue, max_value);
+				if (order > 0) ascending = false;
+				if (order < 0) descending = false;
 			}
 			prevMinValue = min_value;
 			prevMaxValue = max_value;
@@ -172,10 +176,10 @@ function getStatistics(values) {
 			null_count++;
 			continue;
 		}
-		if (typeof value === "object") continue;
+		if (typeof value === "object" && !(value instanceof Uint8Array)) continue;
 		if (typeof value === "number" && Number.isNaN(value)) continue;
-		if (min_value === void 0 || value < min_value) min_value = value;
-		if (max_value === void 0 || value > max_value) max_value = value;
+		if (min_value === void 0 || compareValues(value, min_value) < 0) min_value = value;
+		if (max_value === void 0 || compareValues(value, max_value) > 0) max_value = value;
 	}
 	if (min_value === 0) min_value = -0;
 	if (max_value === 0) max_value = 0;
@@ -185,4 +189,14 @@ function getStatistics(values) {
 		null_count
 	};
 }
+function compareValues(left, right) {
+	if (left instanceof Uint8Array && right instanceof Uint8Array) {
+		const length = Math.min(left.length, right.length);
+		for (let i = 0; i < length; i += 1) if (left[i] !== right[i]) return left[i] - right[i];
+		return left.length - right.length;
+	}
+	if (left < right) return -1;
+	if (left > right) return 1;
+	return 0;
+}
 export { writeColumn };

package/dist/node_modules/.pnpm/{hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec → hyparquet-writer@0.16.6_patch_hash_549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a}/node_modules/hyparquet-writer/src/datapage.mjs

@@ -1,6 +1,6 @@
 import { ByteWriter } from "./bytewriter.mjs";
 import { serializeTCompactProtocol } from "./thrift.mjs";
-import { Encodings, PageTypes } from "../../../../hyparquet@1.26.1/node_modules/hyparquet/src/constants.mjs";
+import { Encodings, PageTypes } from "../../../../hyparquet@1.28.1/node_modules/hyparquet/src/constants.mjs";
 import { deltaBinaryPack, deltaByteArray, deltaLengthByteArray } from "./delta.mjs";
 import { writeRleBitPackedHybrid } from "./encoding.mjs";
 import { writePlain } from "./plain.mjs";

package/dist/node_modules/.pnpm/hyparquet-writer@0.16.6_patch_hash_549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/src/delta.mjs

@@ -0,0 +1,203 @@
+const BLOCK_SIZE = 128;
+const MINIBLOCKS_PER_BLOCK = 4;
+const VALUES_PER_MINIBLOCK = BLOCK_SIZE / MINIBLOCKS_PER_BLOCK;
+function deltaBinaryPack(writer, values) {
+	if (values.length === 0) {
+		writer.appendVarInt(BLOCK_SIZE);
+		writer.appendVarInt(MINIBLOCKS_PER_BLOCK);
+		writer.appendVarInt(0);
+		writer.appendVarInt(0);
+		return;
+	}
+	if (typeof values[0] !== "number" && typeof values[0] !== "bigint") throw new Error("deltaBinaryPack only supports number or bigint arrays");
+	if (usesInt32NumberPath(values)) deltaBinaryPackInt32(writer, values);
+	else deltaBinaryPackBigInt(writer, values);
+}
+function usesInt32NumberPath(values) {
+	if (values instanceof Int32Array) return true;
+	for (let i = 0; i < values.length; i++) {
+		const value = values[i];
+		if (typeof value !== "number" || !Number.isInteger(value) || value < -2147483648 || value > 2147483647) return false;
+	}
+	return true;
+}
+function deltaBinaryPackInt32(writer, values) {
+	const count = values.length;
+	writer.appendVarInt(BLOCK_SIZE);
+	writer.appendVarInt(MINIBLOCKS_PER_BLOCK);
+	writer.appendVarInt(count);
+	appendZigZagNumber(writer, Number(values[0]));
+	const packedByBitWidth = [];
+	const blockDeltas = new Float64Array(BLOCK_SIZE);
+	const bitWidths = new Uint8Array(MINIBLOCKS_PER_BLOCK);
+	let index = 1;
+	while (index < count) {
+		const blockEnd = Math.min(index + BLOCK_SIZE, count);
+		const blockSize = blockEnd - index;
+		let minDelta = Number(values[index]) - Number(values[index - 1]);
+		blockDeltas[0] = minDelta;
+		for (let i = 1; i < blockSize; i++) {
+			const delta = Number(values[index + i]) - Number(values[index + i - 1]);
+			blockDeltas[i] = delta;
+			if (delta < minDelta) minDelta = delta;
+		}
+		appendZigZagNumber(writer, minDelta);
+		for (let mb = 0; mb < MINIBLOCKS_PER_BLOCK; mb++) {
+			const mbStart = mb * VALUES_PER_MINIBLOCK;
+			const mbEnd = Math.min(mbStart + VALUES_PER_MINIBLOCK, blockSize);
+			let maxAdjusted = 0;
+			for (let i = mbStart; i < mbEnd; i++) {
+				const adjusted = blockDeltas[i] - minDelta;
+				if (adjusted > maxAdjusted) maxAdjusted = adjusted;
+			}
+			bitWidths[mb] = bitWidthNumber(maxAdjusted);
+		}
+		writer.appendBytes(bitWidths);
+		for (let mb = 0; mb < MINIBLOCKS_PER_BLOCK; mb++) {
+			const bitWidth = bitWidths[mb];
+			if (bitWidth === 0) continue;
+			const mbStart = mb * VALUES_PER_MINIBLOCK;
+			const mbEnd = Math.min(mbStart + VALUES_PER_MINIBLOCK, blockSize);
+			const packed = packedByBitWidth[bitWidth] ??= new Uint8Array(bitWidth * 4);
+			let byteIndex = 0;
+			let buffer = 0;
+			let bitsUsed = 0;
+			if (bitWidth <= 25) for (let i = 0; i < VALUES_PER_MINIBLOCK; i++) {
+				const adjusted = mbStart + i < mbEnd ? blockDeltas[mbStart + i] - minDelta : 0;
+				buffer |= adjusted << bitsUsed;
+				bitsUsed += bitWidth;
+				while (bitsUsed >= 8) {
+					packed[byteIndex++] = buffer & 255;
+					buffer >>>= 8;
+					bitsUsed -= 8;
+				}
+			}
+			else for (let i = 0; i < VALUES_PER_MINIBLOCK; i++) {
+				const adjusted = mbStart + i < mbEnd ? blockDeltas[mbStart + i] - minDelta : 0;
+				buffer += adjusted * 2 ** bitsUsed;
+				bitsUsed += bitWidth;
+				while (bitsUsed >= 8) {
+					packed[byteIndex++] = buffer % 256;
+					buffer = Math.floor(buffer / 256);
+					bitsUsed -= 8;
+				}
+			}
+			writer.appendBytes(packed);
+		}
+		index = blockEnd;
+	}
+}
+function deltaBinaryPackBigInt(writer, values) {
+	const count = values.length;
+	writer.appendVarInt(BLOCK_SIZE);
+	writer.appendVarInt(MINIBLOCKS_PER_BLOCK);
+	writer.appendVarInt(count);
+	writer.appendZigZag(values[0]);
+	let index = 1;
+	while (index < count) {
+		const blockEnd = Math.min(index + BLOCK_SIZE, count);
+		const blockSize = blockEnd - index;
+		const blockDeltas = new BigInt64Array(blockSize);
+		let minDelta = BigInt(values[index]) - BigInt(values[index - 1]);
+		blockDeltas[0] = minDelta;
+		for (let i = 1; i < blockSize; i++) {
+			const delta = BigInt(values[index + i]) - BigInt(values[index + i - 1]);
+			blockDeltas[i] = delta;
+			if (delta < minDelta) minDelta = delta;
+		}
+		writer.appendZigZag(minDelta);
+		const bitWidths = new Uint8Array(MINIBLOCKS_PER_BLOCK);
+		for (let mb = 0; mb < MINIBLOCKS_PER_BLOCK; mb++) {
+			const mbStart = mb * VALUES_PER_MINIBLOCK;
+			const mbEnd = Math.min(mbStart + VALUES_PER_MINIBLOCK, blockSize);
+			let maxAdjusted = 0n;
+			for (let i = mbStart; i < mbEnd; i++) {
+				const adjusted = blockDeltas[i] - minDelta;
+				if (adjusted > maxAdjusted) maxAdjusted = adjusted;
+			}
+			bitWidths[mb] = bitWidth(maxAdjusted);
+		}
+		writer.appendBytes(bitWidths);
+		for (let mb = 0; mb < MINIBLOCKS_PER_BLOCK; mb++) {
+			const bitWidth = bitWidths[mb];
+			if (bitWidth === 0) continue;
+			const mbStart = mb * VALUES_PER_MINIBLOCK;
+			const mbEnd = Math.min(mbStart + VALUES_PER_MINIBLOCK, blockSize);
+			let buffer = 0n;
+			let bitsUsed = 0;
+			for (let i = 0; i < VALUES_PER_MINIBLOCK; i++) {
+				const adjusted = mbStart + i < mbEnd ? blockDeltas[mbStart + i] - minDelta : 0n;
+				buffer |= adjusted << BigInt(bitsUsed);
+				bitsUsed += bitWidth;
+				while (bitsUsed >= 8) {
+					writer.appendUint8(Number(buffer & 255n));
+					buffer >>= 8n;
+					bitsUsed -= 8;
+				}
+			}
+		}
+		index = blockEnd;
+	}
+}
+function appendZigZagNumber(writer, value) {
+	let encoded = value < 0 ? -value * 2 - 1 : value * 2;
+	while (encoded >= 128) {
+		writer.appendUint8(encoded % 128 + 128);
+		encoded = Math.floor(encoded / 128);
+	}
+	writer.appendUint8(encoded);
+}
+function bitWidthNumber(value) {
+	if (value === 0) return 0;
+	return value > 4294967295 ? 33 : 32 - Math.clz32(value);
+}
+function deltaLengthByteArray(writer, values) {
+	const lengths = new Int32Array(values.length);
+	for (let i = 0; i < values.length; i++) {
+		const value = values[i];
+		if (!(value instanceof Uint8Array)) throw new Error("deltaLengthByteArray expects Uint8Array values");
+		lengths[i] = value.length;
+	}
+	deltaBinaryPack(writer, lengths);
+	for (const value of values) writer.appendBytes(value);
+}
+function deltaByteArray(writer, values) {
+	if (values.length === 0) {
+		deltaBinaryPack(writer, []);
+		deltaBinaryPack(writer, []);
+		return;
+	}
+	const prefixLengths = new Int32Array(values.length);
+	const suffixLengths = new Int32Array(values.length);
+	const value = values[0];
+	if (!(value instanceof Uint8Array)) throw new Error("deltaByteArray expects Uint8Array values");
+	prefixLengths[0] = 0;
+	suffixLengths[0] = value.length;
+	for (let i = 1; i < values.length; i++) {
+		const prev = values[i - 1];
+		const curr = values[i];
+		let prefixLen;
+		if (curr === prev) prefixLen = curr.length;
+		else {
+			if (!(curr instanceof Uint8Array)) throw new Error("deltaByteArray expects Uint8Array values");
+			prefixLen = 0;
+			const maxPrefix = Math.min(prev.length, curr.length);
+			while (prefixLen < maxPrefix && prev[prefixLen] === curr[prefixLen]) prefixLen++;
+		}
+		prefixLengths[i] = prefixLen;
+		suffixLengths[i] = curr.length - prefixLen;
+	}
+	deltaBinaryPack(writer, prefixLengths);
+	deltaBinaryPack(writer, suffixLengths);
+	for (let i = 0; i < values.length; i++) if (suffixLengths[i] > 0) writer.appendBytes(values[i].subarray(prefixLengths[i]));
+}
+function bitWidth(value) {
+	if (value === 0n) return 0;
+	let bits = 0;
+	while (value > 0n) {
+		bits++;
+		value >>= 1n;
+	}
+	return bits;
+}
+export { deltaBinaryPack, deltaByteArray, deltaLengthByteArray };

package/dist/node_modules/.pnpm/{hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec → hyparquet-writer@0.16.6_patch_hash_549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a}/node_modules/hyparquet-writer/src/dictionary.mjs

@@ -1,6 +1,7 @@
 import { ByteWriter } from "./bytewriter.mjs";
 import { writePlain } from "./plain.mjs";
 import { writePageHeader } from "./datapage.mjs";
+const textEncoder = new TextEncoder();
 function estimateValueSize(value, type, type_length) {
 	if (value === null || value === void 0) return 0;
 	if (type === "BOOLEAN") return .125;
@@ -27,23 +28,62 @@ function bytesEqual(a, b) {
 	for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
 	return true;
 }
-function useDictionary(values, type, type_length, encoding, pageSize) {
+const sampleRejectionRatio = .9;
+function useDictionary(values, type, type_length, encoding, dictionarySize, required) {
 	if (encoding && encoding !== "RLE_DICTIONARY") return {};
 	if (type === "BOOLEAN") return {};
-	const sample = values.slice(0, 1e3);
-	const sampleKeys = /* @__PURE__ */ new Set();
-	for (const value of sample) sampleKeys.add(value instanceof Uint8Array ? hashBytes(value) : value);
-	if (sampleKeys.size === 0 || sampleKeys.size / sample.length > .5) return {};
+	const forced = encoding === "RLE_DICTIONARY";
+	const byteArrayPrefixSize = type === "BYTE_ARRAY" ? 4 : 0;
+	if (!forced) {
+		const sampleSize = Math.min(values.length, 1e3);
+		const sampleSizes = /* @__PURE__ */ new Map();
+		const sampleHashes = /* @__PURE__ */ new Map();
+		let sampleDictionarySize = 0;
+		let sampleTotalSize = 0;
+		for (let i = 0; i < sampleSize; i++) {
+			const value = values[sampleSize === 1 ? 0 : Math.floor(i * (values.length - 1) / (sampleSize - 1))];
+			let key = value;
+			if (value instanceof Uint8Array) {
+				key = sampleHashes.get(value);
+				if (key === void 0) {
+					key = hashBytes(value);
+					sampleHashes.set(value, key);
+				}
+			}
+			let valueSize = sampleSizes.get(key);
+			if (valueSize === void 0) {
+				valueSize = estimateValueSize(value, type, type_length) + (value === null || value === void 0 ? 0 : byteArrayPrefixSize);
+				sampleSizes.set(key, valueSize);
+				sampleDictionarySize += valueSize;
+			}
+			sampleTotalSize += valueSize;
+		}
+		if (!sampleTotalSize || sampleDictionarySize / sampleTotalSize > sampleRejectionRatio) return {};
+	}
 	const dictionary = [];
 	const indexes = new Array(values.length);
 	const valueIndex = /* @__PURE__ */ new Map();
+	const valueSizes = [];
 	const hashBuckets = /* @__PURE__ */ new Map();
 	let dictSize = 0;
+	let physicalDictSize = 0;
+	let totalSize = 0;
+	let nonNullCount = 0;
 	for (let i = 0; i < values.length; i++) {
 		const value = values[i];
-		if (value === null || value === void 0) continue;
+		if (value === null || value === void 0) {
+			if (required) throw new Error("parquet required value is undefined");
+			continue;
+		}
+		nonNullCount++;
 		let index;
 		if (value instanceof Uint8Array) {
+			totalSize += value.byteLength;
+			index = valueIndex.get(value);
+			if (index !== void 0) {
+				indexes[i] = index;
+				continue;
+			}
 			const hash = hashBytes(value);
 			const bucket = hashBuckets.get(hash);
 			if (bucket) {
@@ -54,24 +94,39 @@ function useDictionary(values, type, type_length, encoding, pageSize) {
 			}
 			if (index === void 0) {
 				dictSize += value.byteLength;
-				if (pageSize && dictSize > pageSize) return {};
+				if (!forced && dictionarySize) {
+					physicalDictSize += value.byteLength;
+					if (physicalDictSize > dictionarySize) return {};
+				}
 				index = dictionary.length;
 				dictionary.push(value);
 				if (bucket) bucket.push(index);
 				else hashBuckets.set(hash, [index]);
 			}
+			valueIndex.set(value, index);
 		} else {
 			index = valueIndex.get(value);
+			const valueSize = index === void 0 ? estimateValueSize(value, type, type_length) : valueSizes[index];
+			totalSize += valueSize;
 			if (index === void 0) {
-				dictSize += estimateValueSize(value, type, type_length);
-				if (pageSize && dictSize > pageSize) return {};
+				dictSize += valueSize;
+				if (!forced && dictionarySize) {
+					physicalDictSize += typeof value === "string" ? textEncoder.encode(value).byteLength : valueSize;
+					if (physicalDictSize > dictionarySize) return {};
+				}
 				index = dictionary.length;
 				dictionary.push(value);
+				valueSizes[index] = valueSize;
 				valueIndex.set(value, index);
 			}
 		}
 		indexes[i] = index;
 	}
+	const bitWidth = Math.ceil(Math.log2(dictionary.length));
+	const indexSize = Math.ceil(nonNullCount * bitWidth / 8);
+	const plainSize = totalSize + nonNullCount * byteArrayPrefixSize;
+	const dictionaryValueSize = dictSize + dictionary.length * byteArrayPrefixSize;
+	if (!forced && 2 * (dictionaryValueSize + indexSize) > plainSize) return {};
 	return {
 		dictionary,
 		indexes

package/dist/node_modules/.pnpm/{hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec → hyparquet-writer@0.16.6_patch_hash_549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a}/node_modules/hyparquet-writer/src/dremel.mjs

@@ -1,4 +1,4 @@
-import { getMaxDefinitionLevel, isListLike, isMapLike } from "../../../../hyparquet@1.26.1/node_modules/hyparquet/src/schema.mjs";
+import { getMaxDefinitionLevel, isListLike, isMapLike } from "../../../../hyparquet@1.28.1/node_modules/hyparquet/src/schema.mjs";
 function encodeNestedValues(treePath, rows) {
 	const schemaPath = treePath.map((n) => n.element);
 	if (treePath.length < 2) throw new Error("parquet schema path must include column");

package/dist/node_modules/.pnpm/{hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec → hyparquet-writer@0.16.6_patch_hash_549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a}/node_modules/hyparquet-writer/src/indexes.mjs

@@ -1,5 +1,5 @@
 import { serializeTCompactProtocol } from "./thrift.mjs";
-import { BoundaryOrders } from "../../../../hyparquet@1.26.1/node_modules/hyparquet/src/constants.mjs";
+import { BoundaryOrders } from "../../../../hyparquet@1.28.1/node_modules/hyparquet/src/constants.mjs";
 function writeIndexes(writer, pageIndexes) {
 	for (const { chunk, columnIndex } of pageIndexes) writeColumnIndex(writer, chunk, columnIndex);
 	for (const { chunk, offsetIndex } of pageIndexes) writeOffsetIndex(writer, chunk, offsetIndex);

package/dist/node_modules/.pnpm/{hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec → hyparquet-writer@0.16.6_patch_hash_549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a}/node_modules/hyparquet-writer/src/metadata.mjs

@@ -1,6 +1,6 @@
-import { getSchemaPath } from "../../../../hyparquet@1.26.1/node_modules/hyparquet/src/schema.mjs";
-import { serializeTCompactProtocol } from "./thrift.mjs";
-import { CompressionCodecs, ConvertedTypes, EdgeInterpolationAlgorithms, Encodings, FieldRepetitionTypes, PageTypes, ParquetTypes } from "../../../../hyparquet@1.26.1/node_modules/hyparquet/src/constants.mjs";
+import { getSchemaPath } from "../../../../hyparquet@1.28.1/node_modules/hyparquet/src/schema.mjs";
+import { ThriftStruct, serializeTCompactProtocol } from "./thrift.mjs";
+import { CompressionCodecs, ConvertedTypes, EdgeInterpolationAlgorithms, Encodings, FieldRepetitionTypes, PageTypes, ParquetTypes } from "../../../../hyparquet@1.28.1/node_modules/hyparquet/src/constants.mjs";
 import { unconvertStatistics } from "./unconvert.mjs";
 function writeMetadata(writer, metadata) {
 	const compact = {
@@ -51,16 +51,7 @@ function writeMetadata(writer, metadata) {
 						field_3: c.meta_data.size_statistics.definition_level_histogram
 					},
 					field_17: c.meta_data.geospatial_statistics && {
-						field_1: c.meta_data.geospatial_statistics.bbox && {
-							field_1: c.meta_data.geospatial_statistics.bbox.xmin,
-							field_2: c.meta_data.geospatial_statistics.bbox.xmax,
-							field_3: c.meta_data.geospatial_statistics.bbox.ymin,
-							field_4: c.meta_data.geospatial_statistics.bbox.ymax,
-							field_5: c.meta_data.geospatial_statistics.bbox.zmin,
-							field_6: c.meta_data.geospatial_statistics.bbox.zmax,
-							field_7: c.meta_data.geospatial_statistics.bbox.mmin,
-							field_8: c.meta_data.geospatial_statistics.bbox.mmax
-						},
+						field_1: c.meta_data.geospatial_statistics.bbox && thriftBoundingBox(c.meta_data.geospatial_statistics.bbox),
 						field_2: c.meta_data.geospatial_statistics.geospatial_types
 					}
 				},
@@ -91,6 +82,50 @@ function writeMetadata(writer, metadata) {
 	const metadataLength = writer.offset - metadataStart;
 	writer.appendUint32(metadataLength);
 }
+function thriftBoundingBox(bbox) {
+	return new ThriftStruct([
+		[
+			1,
+			7,
+			bbox.xmin
+		],
+		[
+			2,
+			7,
+			bbox.xmax
+		],
+		[
+			3,
+			7,
+			bbox.ymin
+		],
+		[
+			4,
+			7,
+			bbox.ymax
+		],
+		[
+			5,
+			7,
+			bbox.zmin
+		],
+		[
+			6,
+			7,
+			bbox.zmax
+		],
+		[
+			7,
+			7,
+			bbox.mmin
+		],
+		[
+			8,
+			7,
+			bbox.mmax
+		]
+	]);
+}
 function schemaElement(schema, path) {
 	const tree = getSchemaPath(schema, path);
 	return tree[tree.length - 1].element;