@gscdump/engine 3.5.0 → 3.6.0

package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/types/index.d.mts

@@ -1,4 +1,4 @@
-import { AsyncBuffer as AsyncBuffer$1, ParquetQueryFilter as ParquetQueryFilter$1 } from "../src/types.mjs";
+import { AsyncBuffer as AsyncBuffer$1 } from "../src/types.mjs";
 import "./indexes.mjs";
 import "./metadata.mjs";
 import "./read.mjs";
@@ -6,54 +6,4 @@ import "./scan.mjs";
 import "./query.mjs";
 import "./snappy.mjs";
 import "./utils.mjs";
-type AsyncBuffer = AsyncBuffer$1;
-type ParquetQueryFilter = ParquetQueryFilter$1;
-/**
- * Explicitly export types for use in downstream typescript projects through
- * `import { ParquetReadOptions } from 'hyparquet'` for example.
- *
- * @template {any} T
- * @typedef {import('../src/types.d.ts').Awaitable<T>} Awaitable<T>
- */
-/**
- * @typedef {import('../src/types.d.ts').AsyncBuffer} AsyncBuffer
- * @typedef {import('../src/types.d.ts').AsyncRowGroup} AsyncRowGroup
- * @typedef {import('../src/types.d.ts').DataReader} DataReader
- * @typedef {import('../src/types.d.ts').FileMetaData} FileMetaData
- * @typedef {import('../src/types.d.ts').SchemaTree} SchemaTree
- * @typedef {import('../src/types.d.ts').SchemaElement} SchemaElement
- * @typedef {import('../src/types.d.ts').ParquetType} ParquetType
- * @typedef {import('../src/types.d.ts').FieldRepetitionType} FieldRepetitionType
- * @typedef {import('../src/types.d.ts').ConvertedType} ConvertedType
- * @typedef {import('../src/types.d.ts').TimeUnit} TimeUnit
- * @typedef {import('../src/types.d.ts').LogicalType} LogicalType
- * @typedef {import('../src/types.d.ts').RowGroup} RowGroup
- * @typedef {import('../src/types.d.ts').ColumnChunk} ColumnChunk
- * @typedef {import('../src/types.d.ts').ColumnMetaData} ColumnMetaData
- * @typedef {import('../src/types.d.ts').Encoding} Encoding
- * @typedef {import('../src/types.d.ts').CompressionCodec} CompressionCodec
- * @typedef {import('../src/types.d.ts').Compressors} Compressors
- * @typedef {import('../src/types.d.ts').KeyValue} KeyValue
- * @typedef {import('../src/types.d.ts').Statistics} Statistics
- * @typedef {import('../src/types.d.ts').GeospatialStatistics} GeospatialStatistics
- * @typedef {import('../src/types.d.ts').BoundingBox} BoundingBox
- * @typedef {import('../src/types.d.ts').PageType} PageType
- * @typedef {import('../src/types.d.ts').PageHeader} PageHeader
- * @typedef {import('../src/types.d.ts').DataPageHeader} DataPageHeader
- * @typedef {import('../src/types.d.ts').DictionaryPageHeader} DictionaryPageHeader
- * @typedef {import('../src/types.d.ts').DecodedArray} DecodedArray
- * @typedef {import('../src/types.d.ts').OffsetIndex} OffsetIndex
- * @typedef {import('../src/types.d.ts').ColumnIndex} ColumnIndex
- * @typedef {import('../src/types.d.ts').BoundaryOrder} BoundaryOrder
- * @typedef {import('../src/types.d.ts').ColumnData} ColumnData
- * @typedef {import('../src/types.d.ts').SubColumnData} SubColumnData
- * @typedef {import('../src/types.d.ts').ParquetReadOptions} ParquetReadOptions
- * @typedef {import('../src/types.d.ts').ParquetRowRange} ParquetRowRange
- * @typedef {import('../src/types.d.ts').ParquetScan} ParquetScan
- * @typedef {import('../src/types.d.ts').ParquetScanColumnOptions} ParquetScanColumnOptions
- * @typedef {import('../src/types.d.ts').ParquetScanOptions} ParquetScanOptions
- * @typedef {import('../src/types.d.ts').MetadataOptions} MetadataOptions
- * @typedef {import('../src/types.d.ts').ParquetParsers} ParquetParsers
- * @typedef {import('../src/types.d.ts').ParquetQueryFilter} ParquetQueryFilter
- */
-export { AsyncBuffer, ParquetQueryFilter };
+export type AsyncBuffer = AsyncBuffer$1;

package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/types/node.d.mts

@@ -1,3 +1,3 @@
 import "../src/types.mjs";
-import { AsyncBuffer, ParquetQueryFilter } from "./index.mjs";
-export { AsyncBuffer, ParquetQueryFilter };
+import { AsyncBuffer } from "./index.mjs";
+export { AsyncBuffer };

package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/assemble.mjs

@@ -0,0 +1,144 @@
+import { getMaxDefinitionLevel, isListLike, isMapLike } from "./schema.mjs";
+import { decodeVariantColumn } from "./variant.mjs";
+function assembleLists(output, definitionLevels, repetitionLevels, values, schemaPath) {
+	const maxDefinitionLevel = getMaxDefinitionLevel(schemaPath);
+	if (!definitionLevels?.length && !repetitionLevels.length) {
+		if (!maxDefinitionLevel || !values.length) return values;
+		definitionLevels = new Array(values.length).fill(maxDefinitionLevel);
+	}
+	const n = definitionLevels?.length || repetitionLevels.length;
+	const repetitionPath = schemaPath.map(({ element }) => element.repetition_type);
+	let valueIndex = 0;
+	const containerStack = [output];
+	let currentContainer = output;
+	let currentDepth = 0;
+	let currentDefLevel = 0;
+	let currentRepLevel = 0;
+	if (repetitionLevels[0]) while (currentDepth < repetitionPath.length - 2 && currentRepLevel < repetitionLevels[0]) {
+		currentDepth++;
+		if (repetitionPath[currentDepth] !== "REQUIRED") {
+			currentContainer = currentContainer.at(-1);
+			containerStack.push(currentContainer);
+			currentDefLevel++;
+		}
+		if (repetitionPath[currentDepth] === "REPEATED") currentRepLevel++;
+	}
+	for (let i = 0; i < n; i++) {
+		const def = definitionLevels?.length ? definitionLevels[i] : maxDefinitionLevel;
+		const rep = repetitionLevels[i];
+		while (currentDepth && (rep < currentRepLevel || repetitionPath[currentDepth] !== "REPEATED")) {
+			if (repetitionPath[currentDepth] !== "REQUIRED") {
+				containerStack.pop();
+				currentDefLevel--;
+			}
+			if (repetitionPath[currentDepth] === "REPEATED") currentRepLevel--;
+			currentDepth--;
+		}
+		currentContainer = containerStack.at(-1);
+		while ((currentDepth < repetitionPath.length - 2 || repetitionPath[currentDepth + 1] === "REPEATED") && (currentDefLevel < def || repetitionPath[currentDepth + 1] === "REQUIRED")) {
+			currentDepth++;
+			if (repetitionPath[currentDepth] !== "REQUIRED") {
+				const newList = [];
+				currentContainer.push(newList);
+				currentContainer = newList;
+				containerStack.push(newList);
+				currentDefLevel++;
+			}
+			if (repetitionPath[currentDepth] === "REPEATED") currentRepLevel++;
+		}
+		if (def === maxDefinitionLevel) currentContainer.push(values[valueIndex++]);
+		else if (currentDepth === repetitionPath.length - 2) currentContainer.push(null);
+		else currentContainer.push([]);
+	}
+	if (!output.length) for (let i = 0; i < maxDefinitionLevel; i++) {
+		const newList = [];
+		currentContainer.push(newList);
+		currentContainer = newList;
+	}
+	return output;
+}
+function assembleNested(subcolumnData, schema, parsers, depth = 0) {
+	const path = schema.path.join(".");
+	const optional = schema.element.repetition_type === "OPTIONAL";
+	const nextDepth = optional ? depth + 1 : depth;
+	if (isListLike(schema)) {
+		let sublist = schema.children[0];
+		let subDepth = nextDepth;
+		if (sublist.children.length === 1) {
+			sublist = sublist.children[0];
+			subDepth++;
+		}
+		assembleNested(subcolumnData, sublist, parsers, subDepth);
+		const subcolumn = sublist.path.join(".");
+		const values = subcolumnData.get(subcolumn);
+		if (!values) throw new Error("parquet list column missing values");
+		if (optional) flattenAtDepth(values, depth);
+		subcolumnData.set(path, values);
+		subcolumnData.delete(subcolumn);
+		return;
+	}
+	if (isMapLike(schema)) {
+		const mapName = schema.children[0].element.name;
+		assembleNested(subcolumnData, schema.children[0].children[0], parsers, nextDepth + 1);
+		assembleNested(subcolumnData, schema.children[0].children[1], parsers, nextDepth + 1);
+		const keys = subcolumnData.get(`${path}.${mapName}.key`);
+		const values = subcolumnData.get(`${path}.${mapName}.value`);
+		if (!keys) throw new Error("parquet map column missing keys");
+		if (!values) throw new Error("parquet map column missing values");
+		if (keys.length !== values.length) throw new Error("parquet map column key/value length mismatch");
+		const out = assembleMaps(keys, values, nextDepth);
+		if (optional) flattenAtDepth(out, depth);
+		subcolumnData.delete(`${path}.${mapName}.key`);
+		subcolumnData.delete(`${path}.${mapName}.value`);
+		subcolumnData.set(path, out);
+		return;
+	}
+	if (schema.children.length) {
+		const invertDepth = schema.element.repetition_type === "REQUIRED" ? depth : depth + 1;
+		const struct = {};
+		for (const child of schema.children) {
+			assembleNested(subcolumnData, child, parsers, invertDepth);
+			const childData = subcolumnData.get(child.path.join("."));
+			if (!childData) throw new Error("parquet struct missing child data");
+			struct[child.element.name] = childData;
+		}
+		for (const child of schema.children) subcolumnData.delete(child.path.join("."));
+		let inverted = invertStruct(struct, invertDepth);
+		if (schema.element.logical_type?.type === "VARIANT") inverted = decodeVariantColumn(inverted, parsers);
+		if (optional) flattenAtDepth(inverted, depth);
+		subcolumnData.set(path, inverted);
+	}
+}
+function flattenAtDepth(arr, depth) {
+	for (let i = 0; i < arr.length; i++) if (depth) flattenAtDepth(arr[i], depth - 1);
+	else arr[i] = arr[i][0];
+}
+function assembleMaps(keys, values, depth) {
+	const out = [];
+	for (let i = 0; i < keys.length; i++) if (depth) out.push(assembleMaps(keys[i], values[i], depth - 1));
+	else if (keys[i]) {
+		const obj = {};
+		for (let j = 0; j < keys[i].length; j++) {
+			const value = values[i][j];
+			obj[keys[i][j]] = value === void 0 ? null : value;
+		}
+		out.push(obj);
+	} else out.push(void 0);
+	return out;
+}
+function invertStruct(struct, depth) {
+	const keys = Object.keys(struct);
+	const length = struct[keys[0]]?.length;
+	const out = [];
+	for (let i = 0; i < length; i++) {
+		const obj = {};
+		for (const key of keys) {
+			if (struct[key].length !== length) throw new Error("parquet struct parsing error");
+			obj[key] = struct[key][i];
+		}
+		if (depth) out.push(invertStruct(obj, depth - 1));
+		else out.push(obj);
+	}
+	return out;
+}
+export { assembleLists, assembleNested };

package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/bloom.mjs

@@ -0,0 +1,124 @@
+import { deserializeTCompactProtocol } from "./thrift.mjs";
+import { xxhash64 } from "./xxhash.mjs";
+const textEncoder = new TextEncoder();
+const SALT = new Uint32Array([
+	1203114875,
+	1150766481,
+	2284105051,
+	2729912477,
+	1884591559,
+	770785867,
+	2667333959,
+	1550580529
+]);
+function blockIndex(hash, numBlocks) {
+	return Number((hash >> 32n) * BigInt(numBlocks) >> 32n);
+}
+function blockMask(hash) {
+	const m = /* @__PURE__ */ new Uint32Array(8);
+	const low = Number(hash & 4294967295n) | 0;
+	for (let i = 0; i < 8; i++) m[i] = 1 << (Math.imul(low, SALT[i]) >>> 27);
+	return m;
+}
+function sbbfContains(blocks, hash) {
+	const offset = blockIndex(hash, blocks.length >> 3) << 3;
+	const m = blockMask(hash);
+	for (let i = 0; i < 8; i++) if ((blocks[offset + i] & m[i]) === 0) return false;
+	return true;
+}
+function readBloomFilter(reader) {
+	const header = deserializeTCompactProtocol(reader);
+	const numBytes = header.field_1;
+	if (typeof numBytes !== "number" || numBytes <= 0 || numBytes % 32 !== 0) return void 0;
+	if (!header.field_2?.field_1) return void 0;
+	if (!header.field_3?.field_1) return void 0;
+	if (!header.field_4?.field_1) return void 0;
+	const { view, offset } = reader;
+	if (offset + numBytes > view.byteLength) throw new Error(`parquet bloom filter truncated: need ${numBytes} bytes, have ${view.byteLength - offset}`);
+	const blocks = new Uint32Array(numBytes >> 2);
+	for (let i = 0; i < blocks.length; i++) blocks[i] = view.getUint32(offset + i * 4, true);
+	reader.offset = offset + numBytes;
+	return {
+		numBytes,
+		blocks
+	};
+}
+function hashParquetValue(value, element) {
+	if (value === null || value === void 0) return void 0;
+	const { type, converted_type, logical_type } = element;
+	if (type === "BOOLEAN") {
+		if (typeof value !== "boolean") return void 0;
+		return xxhash64(new Uint8Array([value ? 1 : 0]));
+	}
+	if (type === "FLOAT") {
+		if (typeof value !== "number") return void 0;
+		const buf = /* @__PURE__ */ new ArrayBuffer(4);
+		new DataView(buf).setFloat32(0, value, true);
+		return xxhash64(new Uint8Array(buf));
+	}
+	if (type === "DOUBLE") {
+		if (typeof value !== "number") return void 0;
+		const buf = /* @__PURE__ */ new ArrayBuffer(8);
+		new DataView(buf).setFloat64(0, value, true);
+		return xxhash64(new Uint8Array(buf));
+	}
+	if (type === "INT32") {
+		if (converted_type === "DATE" || converted_type === "DECIMAL" || converted_type === "TIME_MILLIS") return void 0;
+		if (logical_type?.type === "DATE" || logical_type?.type === "TIME" || logical_type?.type === "DECIMAL") return void 0;
+		if (typeof value !== "number" || !Number.isInteger(value)) return void 0;
+		const buf = /* @__PURE__ */ new ArrayBuffer(4);
+		new DataView(buf).setInt32(0, value | 0, true);
+		return xxhash64(new Uint8Array(buf));
+	}
+	if (type === "INT64") {
+		if (converted_type === "TIMESTAMP_MILLIS" || converted_type === "TIMESTAMP_MICROS") return void 0;
+		if (converted_type === "TIME_MICROS" || converted_type === "DECIMAL") return void 0;
+		if (logical_type?.type === "TIMESTAMP" || logical_type?.type === "TIME" || logical_type?.type === "DECIMAL") return void 0;
+		let bigValue;
+		if (typeof value === "bigint") bigValue = value;
+		else if (typeof value === "number" && Number.isSafeInteger(value)) bigValue = BigInt(value);
+		else return void 0;
+		const buf = /* @__PURE__ */ new ArrayBuffer(8);
+		new DataView(buf).setBigUint64(0, BigInt.asUintN(64, bigValue), true);
+		return xxhash64(new Uint8Array(buf));
+	}
+	if (type === "BYTE_ARRAY") {
+		if (converted_type === "JSON" || converted_type === "BSON" || converted_type === "DECIMAL") return void 0;
+		if (logical_type?.type === "JSON" || logical_type?.type === "BSON" || logical_type?.type === "VARIANT") return void 0;
+		if (logical_type?.type === "GEOMETRY" || logical_type?.type === "GEOGRAPHY") return void 0;
+		if (typeof value === "string") return xxhash64(textEncoder.encode(value));
+		if (value instanceof Uint8Array) return xxhash64(value);
+		return;
+	}
+	if (type === "FIXED_LEN_BYTE_ARRAY") {
+		if (converted_type === "DECIMAL" || converted_type === "INTERVAL") return void 0;
+		if (logical_type?.type === "DECIMAL" || logical_type?.type === "UUID" || logical_type?.type === "FLOAT16") return void 0;
+		if (logical_type?.type === "GEOMETRY" || logical_type?.type === "GEOGRAPHY") return void 0;
+		if (value instanceof Uint8Array) return xxhash64(value);
+		return;
+	}
+}
+function bloomEligibleColumns(filter) {
+	const out = /* @__PURE__ */ new Set();
+	walkBloomEligible(filter, out);
+	return out;
+}
+function walkBloomEligible(filter, out) {
+	if (!filter) return;
+	if ("$and" in filter && Array.isArray(filter.$and)) {
+		for (const sub of filter.$and) walkBloomEligible(sub, out);
+		return;
+	}
+	if ("$or" in filter && Array.isArray(filter.$or)) {
+		for (const sub of filter.$or) walkBloomEligible(sub, out);
+		return;
+	}
+	if ("$nor" in filter) return;
+	for (const [field, condition] of Object.entries(filter)) {
+		if (field.startsWith("$")) continue;
+		if (typeof condition === "object" && condition !== null && !Array.isArray(condition)) {
+			if ("$eq" in condition || "$in" in condition) out.add(field);
+		} else out.add(field);
+	}
+}
+export { bloomEligibleColumns, hashParquetValue, readBloomFilter, sbbfContains };

package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/column.mjs

@@ -0,0 +1,129 @@
+import { Encodings, PageTypes } from "./constants.mjs";
+import { convert, convertWithDictionary } from "./convert.mjs";
+import { isFlatColumn } from "./schema.mjs";
+import { deserializeTCompactProtocol } from "./thrift.mjs";
+import { assembleLists } from "./assemble.mjs";
+import { readPlain } from "./plain.mjs";
+import { decompressPage, readDataPage, readDataPageV2 } from "./datapage.mjs";
+function readColumn(reader, { groupStart, selectStart, selectEnd }, columnDecoder, onPage) {
+	const { pathInSchema, schemaPath } = columnDecoder;
+	const isFlat = isFlatColumn(schemaPath);
+	const chunks = [];
+	let dictionary = void 0;
+	let lastChunk = void 0;
+	let rowCount = 0;
+	let skipped = 0;
+	const emitLastChunk = onPage && (() => {
+		lastChunk && onPage({
+			pathInSchema,
+			columnData: lastChunk,
+			rowStart: groupStart + rowCount - lastChunk.length,
+			rowEnd: groupStart + rowCount
+		});
+	});
+	while (isFlat ? rowCount < selectEnd : reader.offset < reader.view.byteLength - 1) {
+		if (reader.offset >= reader.view.byteLength - 1) break;
+		const header = parquetHeader(reader);
+		if (header.type === "DICTIONARY_PAGE") {
+			const { data } = readPage(reader, header, columnDecoder, dictionary, void 0, 0);
+			if (data) dictionary = convert(data, columnDecoder);
+		} else {
+			const lastChunkLength = lastChunk?.length || 0;
+			const result = readPage(reader, header, columnDecoder, dictionary, lastChunk, selectStart - rowCount);
+			if (result.skipped) {
+				if (!chunks.length) skipped += result.skipped;
+				rowCount += result.skipped;
+			} else if (result.data && lastChunk === result.data) rowCount += result.data.length - lastChunkLength;
+			else if (result.data && result.data.length) {
+				emitLastChunk?.();
+				chunks.push(result.data);
+				rowCount += result.data.length;
+				lastChunk = result.data;
+			}
+		}
+	}
+	emitLastChunk?.();
+	return {
+		data: chunks,
+		skipped
+	};
+}
+function readPage(reader, header, columnDecoder, dictionary, previousChunk, pageStart) {
+	const { type, element, schemaPath, codec, compressors } = columnDecoder;
+	const compressedBytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, header.compressed_page_size);
+	reader.offset += header.compressed_page_size;
+	if (header.type === "DATA_PAGE") {
+		const daph = header.data_page_header;
+		if (!daph) throw new Error("parquet data page header is undefined");
+		if (pageStart > daph.num_values && isFlatColumn(schemaPath)) return { skipped: daph.num_values };
+		const page = decompressPage(compressedBytes, Number(header.uncompressed_page_size), codec, compressors);
+		const { definitionLevels, repetitionLevels, dataPage } = readDataPage(page, daph, columnDecoder);
+		const values = convertWithDictionary(dataPage, dictionary, daph.encoding, columnDecoder);
+		return {
+			skipped: 0,
+			data: assembleLists(Array.isArray(previousChunk) ? previousChunk : [], definitionLevels, repetitionLevels, values, schemaPath)
+		};
+	} else if (header.type === "DATA_PAGE_V2") {
+		const daph2 = header.data_page_header_v2;
+		if (!daph2) throw new Error("parquet data page header v2 is undefined");
+		if (pageStart > daph2.num_rows) return { skipped: daph2.num_values };
+		const { definitionLevels, repetitionLevels, dataPage } = readDataPageV2(compressedBytes, header, columnDecoder);
+		const values = convertWithDictionary(dataPage, dictionary, daph2.encoding, columnDecoder);
+		return {
+			skipped: 0,
+			data: assembleLists(Array.isArray(previousChunk) ? previousChunk : [], definitionLevels, repetitionLevels, values, schemaPath)
+		};
+	} else if (header.type === "DICTIONARY_PAGE") {
+		const diph = header.dictionary_page_header;
+		if (!diph) throw new Error("parquet dictionary page header is undefined");
+		const page = decompressPage(compressedBytes, Number(header.uncompressed_page_size), codec, compressors);
+		const reader = {
+			view: new DataView(page.buffer, page.byteOffset, page.byteLength),
+			offset: 0
+		};
+		return {
+			skipped: 0,
+			data: readPlain(reader, type, diph.num_values, element.type_length)
+		};
+	} else throw new Error(`parquet unsupported page type: ${header.type}`);
+}
+function parquetHeader(reader) {
+	const header = deserializeTCompactProtocol(reader);
+	return {
+		type: PageTypes[header.field_1],
+		uncompressed_page_size: header.field_2,
+		compressed_page_size: header.field_3,
+		crc: header.field_4,
+		data_page_header: header.field_5 && {
+			num_values: header.field_5.field_1,
+			encoding: Encodings[header.field_5.field_2],
+			definition_level_encoding: Encodings[header.field_5.field_3],
+			repetition_level_encoding: Encodings[header.field_5.field_4],
+			statistics: header.field_5.field_5 && {
+				max: header.field_5.field_5.field_1,
+				min: header.field_5.field_5.field_2,
+				null_count: header.field_5.field_5.field_3,
+				distinct_count: header.field_5.field_5.field_4,
+				max_value: header.field_5.field_5.field_5,
+				min_value: header.field_5.field_5.field_6
+			}
+		},
+		index_page_header: header.field_6,
+		dictionary_page_header: header.field_7 && {
+			num_values: header.field_7.field_1,
+			encoding: Encodings[header.field_7.field_2],
+			is_sorted: header.field_7.field_3
+		},
+		data_page_header_v2: header.field_8 && {
+			num_values: header.field_8.field_1,
+			num_nulls: header.field_8.field_2,
+			num_rows: header.field_8.field_3,
+			encoding: Encodings[header.field_8.field_4],
+			definition_levels_byte_length: header.field_8.field_5,
+			repetition_levels_byte_length: header.field_8.field_6,
+			is_compressed: header.field_8.field_7 === void 0 ? true : header.field_8.field_7,
+			statistics: header.field_8.field_8
+		}
+	};
+}
+export { readColumn, readPage };

package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/constants.mjs

@@ -0,0 +1,80 @@
+const ParquetTypes = [
+	"BOOLEAN",
+	"INT32",
+	"INT64",
+	"INT96",
+	"FLOAT",
+	"DOUBLE",
+	"BYTE_ARRAY",
+	"FIXED_LEN_BYTE_ARRAY"
+];
+const Encodings = [
+	"PLAIN",
+	"GROUP_VAR_INT",
+	"PLAIN_DICTIONARY",
+	"RLE",
+	"BIT_PACKED",
+	"DELTA_BINARY_PACKED",
+	"DELTA_LENGTH_BYTE_ARRAY",
+	"DELTA_BYTE_ARRAY",
+	"RLE_DICTIONARY",
+	"BYTE_STREAM_SPLIT"
+];
+const FieldRepetitionTypes = [
+	"REQUIRED",
+	"OPTIONAL",
+	"REPEATED"
+];
+const ConvertedTypes = [
+	"UTF8",
+	"MAP",
+	"MAP_KEY_VALUE",
+	"LIST",
+	"ENUM",
+	"DECIMAL",
+	"DATE",
+	"TIME_MILLIS",
+	"TIME_MICROS",
+	"TIMESTAMP_MILLIS",
+	"TIMESTAMP_MICROS",
+	"UINT_8",
+	"UINT_16",
+	"UINT_32",
+	"UINT_64",
+	"INT_8",
+	"INT_16",
+	"INT_32",
+	"INT_64",
+	"JSON",
+	"BSON",
+	"INTERVAL"
+];
+const CompressionCodecs = [
+	"UNCOMPRESSED",
+	"SNAPPY",
+	"GZIP",
+	"LZO",
+	"BROTLI",
+	"LZ4",
+	"ZSTD",
+	"LZ4_RAW"
+];
+const PageTypes = [
+	"DATA_PAGE",
+	"INDEX_PAGE",
+	"DICTIONARY_PAGE",
+	"DATA_PAGE_V2"
+];
+const BoundaryOrders = [
+	"UNORDERED",
+	"ASCENDING",
+	"DESCENDING"
+];
+const EdgeInterpolationAlgorithms = [
+	"SPHERICAL",
+	"VINCENTY",
+	"THOMAS",
+	"ANDOYER",
+	"KARNEY"
+];
+export { BoundaryOrders, CompressionCodecs, ConvertedTypes, EdgeInterpolationAlgorithms, Encodings, FieldRepetitionTypes, PageTypes, ParquetTypes };

package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/convert.mjs

@@ -0,0 +1,118 @@
+import { wkbToGeojson } from "./wkb.mjs";
+const decoder = new TextDecoder();
+const DEFAULT_PARSERS = {
+	timestampFromMilliseconds(millis) {
+		return new Date(Number(millis));
+	},
+	timestampFromMicroseconds(micros) {
+		return new Date(Number(micros / 1000n));
+	},
+	timestampFromNanoseconds(nanos) {
+		return new Date(Number(nanos / 1000000n));
+	},
+	dateFromDays(days) {
+		return /* @__PURE__ */ new Date(days * 864e5);
+	},
+	stringFromBytes(bytes) {
+		return bytes && decoder.decode(bytes);
+	},
+	jsonFromBytes(bytes) {
+		return bytes && JSON.parse(decoder.decode(bytes));
+	},
+	geometryFromBytes(bytes) {
+		return bytes && wkbToGeojson({
+			view: new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength),
+			offset: 0
+		});
+	},
+	geographyFromBytes(bytes) {
+		return bytes && wkbToGeojson({
+			view: new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength),
+			offset: 0
+		});
+	},
+	uuidFromBytes(bytes) {
+		if (!bytes) return void 0;
+		const hex = Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("");
+		return hex.slice(0, 8) + "-" + hex.slice(8, 12) + "-" + hex.slice(12, 16) + "-" + hex.slice(16, 20) + "-" + hex.slice(20, 32);
+	}
+};
+function convertWithDictionary(data, dictionary, encoding, columnDecoder) {
+	if (dictionary && encoding.endsWith("_DICTIONARY")) {
+		let output = data;
+		if (data instanceof Uint8Array && !(dictionary instanceof Uint8Array)) output = new dictionary.constructor(data.length);
+		for (let i = 0; i < data.length; i++) output[i] = dictionary[data[i]];
+		return output;
+	} else return convert(data, columnDecoder);
+}
+function convert(data, columnDecoder) {
+	const { element, parsers, utf8 = true, schemaPath } = columnDecoder;
+	const { type, converted_type: ctype, logical_type: ltype } = element;
+	const nullable = element.repetition_type !== "REQUIRED";
+	if (schemaPath?.some((s) => s.element.logical_type?.type === "VARIANT") && type === "BYTE_ARRAY" && ctype !== "UTF8" && ltype?.type !== "STRING") return data;
+	if (ctype === "DECIMAL") {
+		const factor = 10 ** -(element.scale || 0);
+		const arr = new Array(data.length);
+		for (let i = 0; i < arr.length; i++) if (data[i] instanceof Uint8Array) arr[i] = parseDecimal(data[i]) * factor;
+		else arr[i] = Number(data[i]) * factor;
+		return arr;
+	}
+	if (!ctype && type === "INT96") return Array.from(data).map((v) => parsers.timestampFromNanoseconds(parseInt96Nanos(v)));
+	if (ctype === "DATE") return Array.from(data).map((v) => parsers.dateFromDays(v));
+	if (ctype === "TIMESTAMP_MILLIS") return Array.from(data).map((v) => parsers.timestampFromMilliseconds(v));
+	if (ctype === "TIMESTAMP_MICROS") return Array.from(data).map((v) => parsers.timestampFromMicroseconds(v));
+	if (ctype === "JSON") return data.map((v) => parsers.jsonFromBytes(v));
+	if (ctype === "BSON") throw new Error("parquet bson not supported");
+	if (ctype === "INTERVAL") throw new Error("parquet interval not supported");
+	if (ltype?.type === "GEOMETRY") return data.map((v) => parsers.geometryFromBytes(v));
+	if (ltype?.type === "GEOGRAPHY") return data.map((v) => parsers.geographyFromBytes(v));
+	if (ltype?.type === "UUID") return data.map((v) => parsers.uuidFromBytes(v));
+	if (ctype === "UTF8" || ltype?.type === "STRING" || utf8 && type === "BYTE_ARRAY") return data.map((v) => parsers.stringFromBytes(v));
+	if (ctype === "UINT_64" || ltype?.type === "INTEGER" && ltype.bitWidth === 64 && !ltype.isSigned) {
+		if (data instanceof BigInt64Array) return new BigUint64Array(data.buffer, data.byteOffset, data.length);
+		const arr = nullable ? new Array(data.length) : new BigUint64Array(data.length);
+		for (let i = 0; i < arr.length; i++) arr[i] = data[i];
+		return arr;
+	}
+	if (ctype === "UINT_32" || ltype?.type === "INTEGER" && ltype.bitWidth === 32 && !ltype.isSigned) {
+		if (data instanceof Int32Array) return new Uint32Array(data.buffer, data.byteOffset, data.length);
+		const arr = nullable ? new Array(data.length) : new Uint32Array(data.length);
+		for (let i = 0; i < arr.length; i++) arr[i] = data[i] < 0 ? 4294967296 + data[i] : data[i];
+		return arr;
+	}
+	if (ltype?.type === "FLOAT16") return Array.from(data).map(parseFloat16);
+	if (ltype?.type === "TIMESTAMP") {
+		const { unit } = ltype;
+		let parser = parsers.timestampFromMilliseconds;
+		if (unit === "MICROS") parser = parsers.timestampFromMicroseconds;
+		if (unit === "NANOS") parser = parsers.timestampFromNanoseconds;
+		const arr = new Array(data.length);
+		for (let i = 0; i < arr.length; i++) arr[i] = parser(data[i]);
+		return arr;
+	}
+	return data;
+}
+function parseDecimal(bytes) {
+	if (!bytes.length) return 0;
+	let value = 0n;
+	for (const byte of bytes) value = value * 256n + BigInt(byte);
+	const bits = bytes.length * 8;
+	if (value >= 2n ** BigInt(bits - 1)) value -= 2n ** BigInt(bits);
+	return Number(value);
+}
+function parseInt96Nanos(value) {
+	const days = (value >> 64n) - 2440588n;
+	const nano = value & 18446744073709551615n;
+	return days * 86400000000000n + nano;
+}
+function parseFloat16(bytes) {
+	if (!bytes) return void 0;
+	const int16 = bytes[1] << 8 | bytes[0];
+	const sign = int16 >> 15 ? -1 : 1;
+	const exp = int16 >> 10 & 31;
+	const frac = int16 & 1023;
+	if (exp === 0) return sign * 2 ** -14 * (frac / 1024);
+	if (exp === 31) return frac ? NaN : sign * Infinity;
+	return sign * 2 ** (exp - 15) * (1 + frac / 1024);
+}
+export { DEFAULT_PARSERS, convert, convertWithDictionary, parseDecimal, parseFloat16 };