@gscdump/lakehouse 1.4.1 → 1.4.3

package/dist/_virtual/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/plain.mjs

@@ -0,0 +1,78 @@
+function readPlain(reader, type, count, fixedLength) {
+	if (count === 0) return [];
+	if (type === "BOOLEAN") return readPlainBoolean(reader, count);
+	else if (type === "INT32") return readPlainInt32(reader, count);
+	else if (type === "INT64") return readPlainInt64(reader, count);
+	else if (type === "INT96") return readPlainInt96(reader, count);
+	else if (type === "FLOAT") return readPlainFloat(reader, count);
+	else if (type === "DOUBLE") return readPlainDouble(reader, count);
+	else if (type === "BYTE_ARRAY") return readPlainByteArray(reader, count);
+	else if (type === "FIXED_LEN_BYTE_ARRAY") {
+		if (!fixedLength) throw new Error("parquet missing fixed length");
+		return readPlainByteArrayFixed(reader, count, fixedLength);
+	} else throw new Error(`parquet unhandled type: ${type}`);
+}
+function readPlainBoolean(reader, count) {
+	const values = new Array(count);
+	for (let i = 0; i < count; i++) {
+		const byteOffset = reader.offset + (i / 8 | 0);
+		const bitOffset = i % 8;
+		values[i] = (reader.view.getUint8(byteOffset) & 1 << bitOffset) !== 0;
+	}
+	reader.offset += Math.ceil(count / 8);
+	return values;
+}
+function readPlainInt32(reader, count) {
+	const values = (reader.view.byteOffset + reader.offset) % 4 ? new Int32Array(align(reader.view.buffer, reader.view.byteOffset + reader.offset, count * 4)) : new Int32Array(reader.view.buffer, reader.view.byteOffset + reader.offset, count);
+	reader.offset += count * 4;
+	return values;
+}
+function readPlainInt64(reader, count) {
+	const values = (reader.view.byteOffset + reader.offset) % 8 ? new BigInt64Array(align(reader.view.buffer, reader.view.byteOffset + reader.offset, count * 8)) : new BigInt64Array(reader.view.buffer, reader.view.byteOffset + reader.offset, count);
+	reader.offset += count * 8;
+	return values;
+}
+function readPlainInt96(reader, count) {
+	const values = new Array(count);
+	for (let i = 0; i < count; i++) {
+		const low = reader.view.getBigInt64(reader.offset + i * 12, true);
+		const high = reader.view.getInt32(reader.offset + i * 12 + 8, true);
+		values[i] = BigInt(high) << 64n | low;
+	}
+	reader.offset += count * 12;
+	return values;
+}
+function readPlainFloat(reader, count) {
+	const values = (reader.view.byteOffset + reader.offset) % 4 ? new Float32Array(align(reader.view.buffer, reader.view.byteOffset + reader.offset, count * 4)) : new Float32Array(reader.view.buffer, reader.view.byteOffset + reader.offset, count);
+	reader.offset += count * 4;
+	return values;
+}
+function readPlainDouble(reader, count) {
+	const values = (reader.view.byteOffset + reader.offset) % 8 ? new Float64Array(align(reader.view.buffer, reader.view.byteOffset + reader.offset, count * 8)) : new Float64Array(reader.view.buffer, reader.view.byteOffset + reader.offset, count);
+	reader.offset += count * 8;
+	return values;
+}
+function readPlainByteArray(reader, count) {
+	const values = new Array(count);
+	for (let i = 0; i < count; i++) {
+		const length = reader.view.getUint32(reader.offset, true);
+		reader.offset += 4;
+		values[i] = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, length);
+		reader.offset += length;
+	}
+	return values;
+}
+function readPlainByteArrayFixed(reader, count, fixedLength) {
+	const values = new Array(count);
+	for (let i = 0; i < count; i++) {
+		values[i] = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, fixedLength);
+		reader.offset += fixedLength;
+	}
+	return values;
+}
+function align(buffer, offset, size) {
+	const aligned = new ArrayBuffer(size);
+	new Uint8Array(aligned).set(new Uint8Array(buffer, offset, size));
+	return aligned;
+}
+export { readPlain };

package/dist/_virtual/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/plan.mjs

@@ -0,0 +1,147 @@
+import { getPhysicalColumns } from "./schema.mjs";
+import { parquetSchema } from "./metadata.mjs";
+import { bloomEligibleColumns, readBloomFilter } from "./bloom.mjs";
+import { canSkipRowGroup } from "./filter.mjs";
+const runLimit = 1 << 21;
+function parquetPlan({ metadata, rowStart = 0, rowEnd = Infinity, columns, filter, filterStrict = true, useOffsetIndex = false, bloomFiltersByGroup, schemaElements }) {
+	if (!metadata) throw new Error("parquetPlan requires metadata");
+	const groups = [];
+	const fetches = [];
+	const indexes = [];
+	const physicalColumns = getPhysicalColumns(parquetSchema(metadata));
+	let groupStart = 0;
+	let rgIdx = 0;
+	for (const rowGroup of metadata.row_groups) {
+		const groupRows = Number(rowGroup.num_rows);
+		const groupEnd = groupStart + groupRows;
+		const bloomFilters = bloomFiltersByGroup?.[rgIdx];
+		if (groupRows > 0 && groupEnd > rowStart && groupStart < rowEnd && !canSkipRowGroup({
+			rowGroup,
+			physicalColumns,
+			filter,
+			strict: filterStrict,
+			bloomFilters,
+			schemaElements
+		})) {
+			const chunks = [];
+			let groupStartByte = Infinity;
+			let groupEndByte = -Infinity;
+			for (const chunk of rowGroup.columns) {
+				const meta = chunk.meta_data;
+				if (chunk.file_path) throw new Error("parquet file_path not supported");
+				if (!meta) throw new Error("parquet column metadata is undefined");
+				if (!columns || columns.includes(meta.path_in_schema[0])) {
+					const columnOffset = meta.dictionary_page_offset || meta.data_page_offset;
+					const startByte = Number(columnOffset);
+					const endByte = Number(columnOffset + meta.total_compressed_size);
+					if (startByte < groupStartByte) groupStartByte = startByte;
+					if (endByte > groupEndByte) groupEndByte = endByte;
+					if (useOffsetIndex && chunk.offset_index_offset && chunk.offset_index_length && (rowStart > groupStart || rowEnd < groupEnd)) {
+						const offsetIndexStart = Number(chunk.offset_index_offset);
+						chunks.push({
+							columnMetadata: meta,
+							offsetIndex: {
+								startByte: offsetIndexStart,
+								endByte: offsetIndexStart + chunk.offset_index_length
+							},
+							range: {
+								startByte,
+								endByte
+							}
+						});
+					} else chunks.push({
+						columnMetadata: meta,
+						range: {
+							startByte,
+							endByte
+						}
+					});
+				}
+			}
+			const selectStart = Math.max(rowStart - groupStart, 0);
+			const selectEnd = Math.min(rowEnd - groupStart, groupRows);
+			groups.push({
+				chunks,
+				rowGroup,
+				groupStart,
+				groupRows,
+				selectStart,
+				selectEnd
+			});
+			let run;
+			for (const chunk of chunks) if ("offsetIndex" in chunk) indexes.push(chunk.offsetIndex);
+			else {
+				const { range } = chunk;
+				if (columns) fetches.push(range);
+				else if (run && range.endByte - run.startByte <= runLimit) run.endByte = range.endByte;
+				else {
+					if (run) fetches.push(run);
+					run = { ...range };
+				}
+			}
+			if (run) fetches.push(run);
+		}
+		groupStart = groupEnd;
+		rgIdx++;
+	}
+	if (!isFinite(rowEnd)) rowEnd = groupStart;
+	fetches.push(...indexes);
+	return {
+		metadata,
+		rowStart,
+		rowEnd,
+		columns,
+		fetches,
+		groups
+	};
+}
+async function prefetchBloomFilters({ file, metadata, filter, filterStrict = true }) {
+	const result = metadata.row_groups.map(() => ({}));
+	const eligibleCols = bloomEligibleColumns(filter);
+	if (eligibleCols.size === 0) return result;
+	const physicalColumns = getPhysicalColumns(parquetSchema(metadata));
+	const tasks = [];
+	metadata.row_groups.forEach((rowGroup, rgIdx) => {
+		if (canSkipRowGroup({
+			rowGroup,
+			physicalColumns,
+			filter,
+			strict: filterStrict
+		})) return;
+		for (const colName of eligibleCols) {
+			const columnIdx = physicalColumns.indexOf(colName);
+			if (columnIdx === -1) continue;
+			const meta = rowGroup.columns[columnIdx]?.meta_data;
+			if (!meta?.bloom_filter_offset || !meta.bloom_filter_length) continue;
+			const start = Number(meta.bloom_filter_offset);
+			const end = start + meta.bloom_filter_length;
+			tasks.push((async () => {
+				const buffer = await file.slice(start, end);
+				const bloom = readBloomFilter({
+					view: new DataView(buffer),
+					offset: 0
+				});
+				if (bloom) result[rgIdx][colName] = bloom;
+			})());
+		}
+	});
+	if (tasks.length) await Promise.all(tasks);
+	return result;
+}
+function prefetchAsyncBuffer(file, { fetches }) {
+	const promises = fetches.map(({ startByte, endByte }) => file.slice(startByte, endByte));
+	return {
+		byteLength: file.byteLength,
+		slice(start, end = file.byteLength) {
+			const index = fetches.findIndex(({ startByte, endByte }) => startByte <= start && end <= endByte);
+			if (index < 0) return file.slice(start, end);
+			if (fetches[index].startByte !== start || fetches[index].endByte !== end) {
+				const startOffset = start - fetches[index].startByte;
+				const endOffset = end - fetches[index].startByte;
+				if (promises[index] instanceof Promise) return promises[index].then((buffer) => buffer.slice(startOffset, endOffset));
+				else return promises[index].slice(startOffset, endOffset);
+			} else return promises[index];
+		}
+	};
+}
+export { parquetPlan, prefetchAsyncBuffer, prefetchBloomFilters };

package/dist/_virtual/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/read.mjs

@@ -0,0 +1,106 @@
+import { parquetMetadataAsync, parquetSchema } from "./metadata.mjs";
+import { concat } from "./utils.mjs";
+import { columnsNeededForFilter, matchFilter } from "./filter.mjs";
+import { parquetPlan, prefetchAsyncBuffer, prefetchBloomFilters } from "./plan.mjs";
+import { assembleAsync, asyncGroupToRows, readRowGroup } from "./rowgroup.mjs";
+async function parquetRead(options) {
+	options.metadata ??= await parquetMetadataAsync(options.file, options);
+	const { rowStart = 0, rowEnd, columns, onChunk, onComplete, rowFormat, filter, filterStrict = true } = options;
+	if (filter && rowFormat !== "object") throw new Error("parquet filter requires rowFormat: \"object\"");
+	const filterColumns = columnsNeededForFilter(filter);
+	if (filterColumns.length) {
+		const schemaColumns = parquetSchema(options.metadata).children.map((c) => c.element.name);
+		const missingColumns = filterColumns.filter((c) => !schemaColumns.includes(c));
+		if (missingColumns.length) throw new Error(`parquet filter columns not found: ${missingColumns.join(", ")}`);
+	}
+	let readColumns = columns;
+	let requiresProjection = false;
+	if (columns && filter) {
+		const missingFilterColumns = filterColumns.filter((c) => !columns.includes(c));
+		if (missingFilterColumns.length) {
+			readColumns = [...columns, ...missingFilterColumns];
+			requiresProjection = true;
+		}
+	}
+	let readOptions = readColumns !== columns ? {
+		...options,
+		columns: readColumns
+	} : options;
+	readOptions = await withBloomFilters(readOptions);
+	const asyncGroups = parquetReadAsync(readOptions);
+	if (!onComplete && !onChunk) {
+		await awaitAllColumns(asyncGroups);
+		return;
+	}
+	const schemaTree = parquetSchema(options.metadata);
+	const assembled = asyncGroups.map((arg) => assembleAsync(arg, schemaTree, options.parsers));
+	if (onChunk) for (const asyncGroup of assembled) for (const asyncColumn of asyncGroup.asyncColumns) asyncColumn.data.then(({ data, skipped }) => {
+		let rowStart = asyncGroup.groupStart + skipped;
+		for (const columnData of data) {
+			onChunk({
+				columnName: asyncColumn.pathInSchema[0],
+				columnData,
+				rowStart,
+				rowEnd: rowStart + columnData.length
+			});
+			rowStart += columnData.length;
+		}
+	}, () => {});
+	if (onComplete) {
+		await awaitAllColumns(assembled);
+		const rows = [];
+		for (const asyncGroup of assembled) {
+			const selectStart = Math.max(rowStart - asyncGroup.groupStart, 0);
+			const selectEnd = Math.min((rowEnd ?? Infinity) - asyncGroup.groupStart, asyncGroup.groupRows);
+			const groupData = rowFormat === "object" ? await asyncGroupToRows(asyncGroup, selectStart, selectEnd, readColumns, "object") : await asyncGroupToRows(asyncGroup, selectStart, selectEnd, columns, "array");
+			if (filter) {
+				for (const row of groupData) if (matchFilter(row, filter, filterStrict)) {
+					if (requiresProjection && columns) {
+						for (const col of filterColumns) if (!columns.includes(col)) delete row[col];
+					}
+					rows.push(row);
+				}
+			} else concat(rows, groupData);
+		}
+		onComplete(rows);
+	} else await awaitAllColumns(assembled);
+}
+async function awaitAllColumns(asyncGroups) {
+	const all = asyncGroups.flatMap((g) => g.asyncColumns.map((c) => c.data));
+	const failed = (await Promise.allSettled(all)).find((r) => r.status === "rejected");
+	if (failed) throw failed.reason;
+}
+function parquetReadAsync(options) {
+	if (!options.metadata) throw new Error("parquet requires metadata");
+	const plan = parquetPlan(options);
+	options.file = prefetchAsyncBuffer(options.file, plan);
+	return plan.groups.map((groupPlan) => readRowGroup(options, plan, groupPlan));
+}
+async function withBloomFilters(options) {
+	if (!options.useBloomFilters) return options;
+	if (!options.filter || !options.metadata) return options;
+	const schemaTree = parquetSchema(options.metadata);
+	const schemaElements = {};
+	for (const child of schemaTree.children) schemaElements[child.element.name] = child.element;
+	const bloomFiltersByGroup = await prefetchBloomFilters({
+		file: options.file,
+		metadata: options.metadata,
+		filter: options.filter,
+		filterStrict: options.filterStrict
+	});
+	return {
+		...options,
+		bloomFiltersByGroup,
+		schemaElements
+	};
+}
+function parquetReadObjects(options) {
+	return new Promise((onComplete, reject) => {
+		parquetRead({
+			...options,
+			rowFormat: "object",
+			onComplete
+		}).catch(reject);
+	});
+}
+export { parquetRead, parquetReadAsync, parquetReadObjects };

package/dist/_virtual/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/rowgroup.mjs

@@ -0,0 +1,154 @@
+import { DEFAULT_PARSERS } from "./convert.mjs";
+import { getSchemaPath } from "./schema.mjs";
+import { readOffsetIndex } from "./indexes.mjs";
+import { flatten } from "./utils.mjs";
+import { assembleNested } from "./assemble.mjs";
+import { readColumn } from "./column.mjs";
+function readRowGroup(options, { metadata }, groupPlan) {
+	const asyncColumns = [];
+	for (const chunk of groupPlan.chunks) {
+		const { data_page_offset, dictionary_page_offset, path_in_schema: pathInSchema } = chunk.columnMetadata;
+		const schemaPath = getSchemaPath(metadata.schema, pathInSchema);
+		const columnDecoder = {
+			pathInSchema,
+			element: schemaPath[schemaPath.length - 1].element,
+			schemaPath,
+			parsers: {
+				...DEFAULT_PARSERS,
+				...options.parsers
+			},
+			...options,
+			...chunk.columnMetadata
+		};
+		let { startByte, endByte } = chunk.range;
+		if (!("offsetIndex" in chunk)) {
+			asyncColumns.push({
+				pathInSchema,
+				data: Promise.resolve(options.file.slice(startByte, endByte)).then((buffer) => {
+					return readColumn({
+						view: new DataView(buffer),
+						offset: 0
+					}, groupPlan, columnDecoder, options.onPage);
+				})
+			});
+			continue;
+		}
+		asyncColumns.push({
+			pathInSchema,
+			data: Promise.resolve(options.file.slice(chunk.offsetIndex.startByte, chunk.offsetIndex.endByte)).then(async (arrayBuffer) => {
+				const { selectStart, selectEnd } = groupPlan;
+				const pages = readOffsetIndex({
+					view: new DataView(arrayBuffer),
+					offset: 0
+				}).page_locations;
+				let skipped = -1;
+				const hasDict = dictionary_page_offset || data_page_offset < pages[0].offset;
+				for (let i = 0; i < pages.length; i++) {
+					const page = pages[i];
+					const pageStart = Number(page.first_row_index);
+					const pageEnd = i + 1 < pages.length ? Number(pages[i + 1].first_row_index) : groupPlan.groupRows;
+					if (skipped < 0 && !hasDict && pageEnd > selectStart) {
+						startByte = Number(page.offset);
+						skipped = pageStart;
+					}
+					if (pageStart < selectEnd) endByte = Number(page.offset) + page.compressed_page_size;
+				}
+				if (skipped < 0) skipped = 0;
+				const buffer = await options.file.slice(startByte, endByte);
+				const { data, skipped: columnSkipped } = readColumn({
+					view: new DataView(buffer),
+					offset: 0
+				}, skipped ? {
+					...groupPlan,
+					groupStart: groupPlan.groupStart + skipped,
+					selectStart: groupPlan.selectStart - skipped,
+					selectEnd: groupPlan.selectEnd - skipped
+				} : groupPlan, columnDecoder, options.onPage);
+				return {
+					data,
+					skipped: skipped + columnSkipped
+				};
+			})
+		});
+	}
+	return {
+		groupStart: groupPlan.groupStart,
+		groupRows: groupPlan.groupRows,
+		asyncColumns
+	};
+}
+async function asyncGroupToRows({ asyncColumns }, selectStart, selectEnd, columns, rowFormat) {
+	const asyncPages = await Promise.all(asyncColumns.map((column) => column.data.then(({ skipped, data }) => ({
+		skipped,
+		data: flatten(data)
+	}))));
+	const selectCount = selectEnd - selectStart;
+	if (rowFormat === "object") {
+		const groupData = Array(selectCount);
+		for (let selectRow = 0; selectRow < selectCount; selectRow++) {
+			const rowData = {};
+			for (let i = 0; i < asyncColumns.length; i++) {
+				const { data, skipped } = asyncPages[i];
+				rowData[asyncColumns[i].pathInSchema[0]] = data[selectStart + selectRow - skipped];
+			}
+			groupData[selectRow] = rowData;
+		}
+		return groupData;
+	}
+	const includedColumnNames = asyncColumns.map((child) => child.pathInSchema[0]).filter((name) => !columns || columns.includes(name));
+	const columnOrder = columns ?? includedColumnNames;
+	const columnIndexes = columnOrder.map((name) => asyncColumns.findIndex((column) => column.pathInSchema[0] === name));
+	const groupData = Array(selectCount);
+	for (let selectRow = 0; selectRow < selectCount; selectRow++) {
+		const rowData = Array(asyncColumns.length);
+		for (let i = 0; i < columnOrder.length; i++) {
+			const colIdx = columnIndexes[i];
+			if (colIdx < 0) throw new Error(`parquet column not found: ${columnOrder[i]}`);
+			const { data, skipped } = asyncPages[colIdx];
+			rowData[i] = data[selectStart + selectRow - skipped];
+		}
+		groupData[selectRow] = rowData;
+	}
+	return groupData;
+}
+function assembleAsync(asyncRowGroup, schemaTree, parsers) {
+	const { asyncColumns } = asyncRowGroup;
+	parsers = {
+		...DEFAULT_PARSERS,
+		...parsers
+	};
+	const assembled = [];
+	for (const child of schemaTree.children) if (child.children.length) {
+		const childColumns = asyncColumns.filter((column) => column.pathInSchema[0] === child.element.name);
+		if (!childColumns.length) continue;
+		assembled.push({
+			pathInSchema: child.path,
+			data: (async () => {
+				const resolved = await Promise.all(childColumns.map((c) => c.data));
+				const subcolumnData = /* @__PURE__ */ new Map();
+				let minLength = Infinity;
+				for (let i = 0; i < childColumns.length; i++) {
+					const flat = flatten(resolved[i].data);
+					subcolumnData.set(childColumns[i].pathInSchema.join("."), flat);
+					minLength = Math.min(minLength, flat.length);
+				}
+				for (const [key, value] of subcolumnData) if (value.length > minLength) subcolumnData.set(key, value.slice(0, minLength));
+				assembleNested(subcolumnData, child, parsers);
+				const assembled = subcolumnData.get(child.element.name);
+				if (!assembled) throw new Error("parquet column data not assembled");
+				return {
+					data: [assembled],
+					skipped: 0
+				};
+			})()
+		});
+	} else {
+		const asyncColumn = asyncColumns.find((column) => column.pathInSchema[0] === child.element.name);
+		if (asyncColumn) assembled.push(asyncColumn);
+	}
+	return {
+		...asyncRowGroup,
+		asyncColumns: assembled
+	};
+}
+export { assembleAsync, asyncGroupToRows, readRowGroup };

package/dist/_virtual/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/schema.mjs

@@ -0,0 +1,75 @@
+function schemaTree(schema, rootIndex, path) {
+	const element = schema[rootIndex];
+	const children = [];
+	let count = 1;
+	if (element.num_children) while (children.length < element.num_children) {
+		const childElement = schema[rootIndex + count];
+		const child = schemaTree(schema, rootIndex + count, [...path, childElement.name]);
+		count += child.count;
+		children.push(child);
+	}
+	return {
+		count,
+		element,
+		children,
+		path
+	};
+}
+function getSchemaPath(schema, name) {
+	let tree = schemaTree(schema, 0, []);
+	const path = [tree];
+	for (const part of name) {
+		const child = tree.children.find((child) => child.element.name === part);
+		if (!child) throw new Error(`parquet schema element not found: ${name}`);
+		path.push(child);
+		tree = child;
+	}
+	return path;
+}
+function getPhysicalColumns(schemaTree) {
+	const columns = [];
+	function traverse(node) {
+		if (node.children.length) for (const child of node.children) traverse(child);
+		else columns.push(node.path.join("."));
+	}
+	traverse(schemaTree);
+	return columns;
+}
+function getMaxRepetitionLevel(schemaPath) {
+	let maxLevel = 0;
+	for (const { element } of schemaPath) if (element.repetition_type === "REPEATED") maxLevel++;
+	return maxLevel;
+}
+function getMaxDefinitionLevel(schemaPath) {
+	let maxLevel = 0;
+	for (const { element } of schemaPath.slice(1)) if (element.repetition_type !== "REQUIRED") maxLevel++;
+	return maxLevel;
+}
+function isListLike(schema) {
+	if (!schema) return false;
+	if (schema.element.converted_type !== "LIST") return false;
+	if (schema.children.length > 1) return false;
+	const firstChild = schema.children[0];
+	if (firstChild.children.length > 1) return false;
+	if (firstChild.element.repetition_type !== "REPEATED") return false;
+	return true;
+}
+function isMapLike(schema) {
+	if (!schema) return false;
+	if (schema.element.converted_type !== "MAP") return false;
+	if (schema.children.length > 1) return false;
+	const firstChild = schema.children[0];
+	if (firstChild.children.length !== 2) return false;
+	if (firstChild.element.repetition_type !== "REPEATED") return false;
+	if (firstChild.children.find((child) => child.element.name === "key")?.element.repetition_type === "REPEATED") return false;
+	if (firstChild.children.find((child) => child.element.name === "value")?.element.repetition_type === "REPEATED") return false;
+	return true;
+}
+function isFlatColumn(schemaPath) {
+	if (schemaPath.length !== 2) return false;
+	const [, column] = schemaPath;
+	if (column.element.repetition_type === "REPEATED") return false;
+	if (column.children.length) return false;
+	return true;
+}
+export { getMaxDefinitionLevel, getMaxRepetitionLevel, getPhysicalColumns, getSchemaPath, isFlatColumn, isListLike, isMapLike };

package/dist/_virtual/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/thrift.mjs

@@ -0,0 +1,86 @@
+const STOP = 0;
+const TRUE = 1;
+const FALSE = 2;
+const BYTE = 3;
+const I16 = 4;
+const I32 = 5;
+const I64 = 6;
+const DOUBLE = 7;
+const BINARY = 8;
+const LIST = 9;
+const STRUCT = 12;
+function deserializeTCompactProtocol(reader) {
+	const value = {};
+	let fid = 0;
+	while (reader.offset < reader.view.byteLength) {
+		const byte = reader.view.getUint8(reader.offset++);
+		const type = byte & 15;
+		if (type === STOP) break;
+		const delta = byte >> 4;
+		fid = delta ? fid + delta : readZigZag(reader);
+		value[`field_${fid}`] = readElement(reader, type);
+	}
+	return value;
+}
+function readElement(reader, type) {
+	switch (type) {
+		case TRUE: return true;
+		case FALSE: return false;
+		case BYTE: return reader.view.getInt8(reader.offset++);
+		case I16:
+		case I32: return readZigZag(reader);
+		case I64: return readZigZagBigInt(reader);
+		case DOUBLE: {
+			const value = reader.view.getFloat64(reader.offset, true);
+			reader.offset += 8;
+			return value;
+		}
+		case BINARY: {
+			const stringLength = readVarInt(reader);
+			const strBytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, stringLength);
+			reader.offset += stringLength;
+			return strBytes;
+		}
+		case LIST: {
+			const byte = reader.view.getUint8(reader.offset++);
+			const elemType = byte & 15;
+			let listSize = byte >> 4;
+			if (listSize === 15) listSize = readVarInt(reader);
+			const boolType = elemType === TRUE || elemType === FALSE;
+			const values = new Array(listSize);
+			for (let i = 0; i < listSize; i++) values[i] = boolType ? readElement(reader, BYTE) === 1 : readElement(reader, elemType);
+			return values;
+		}
+		case STRUCT: return deserializeTCompactProtocol(reader);
+		default: throw new Error(`thrift unhandled type: ${type}`);
+	}
+}
+function readVarInt(reader) {
+	let result = 0;
+	let shift = 0;
+	while (true) {
+		const byte = reader.view.getUint8(reader.offset++);
+		result |= (byte & 127) << shift;
+		if (!(byte & 128)) return result;
+		shift += 7;
+	}
+}
+function readVarBigInt(reader) {
+	let result = 0n;
+	let shift = 0n;
+	while (true) {
+		const byte = reader.view.getUint8(reader.offset++);
+		result |= BigInt(byte & 127) << shift;
+		if (!(byte & 128)) return result;
+		shift += 7n;
+	}
+}
+function readZigZag(reader) {
+	const zigzag = readVarInt(reader);
+	return zigzag >>> 1 ^ -(zigzag & 1);
+}
+function readZigZagBigInt(reader) {
+	const zigzag = readVarBigInt(reader);
+	return zigzag >> 1n ^ -(zigzag & 1n);
+}
+export { deserializeTCompactProtocol, readVarInt, readZigZag, readZigZagBigInt };

package/dist/{node_modules → _virtual/node_modules}/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/utils.mjs

@@ -1,4 +1,26 @@
 import { defaultInitialFetchSize } from "./metadata.mjs";
+function concat(aaa, bbb) {
+	const chunk = 1e4;
+	for (let i = 0; i < bbb.length; i += chunk) aaa.push(...bbb.slice(i, i + chunk));
+}
+function equals(a, b, strict = true) {
+	if (strict ? a === b : a == b) return true;
+	if (!a || !b || typeof a !== "object" || typeof b !== "object") return false;
+	if (a instanceof Uint8Array && b instanceof Uint8Array) {
+		if (a.length !== b.length) return false;
+		for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
+		return true;
+	}
+	if (Array.isArray(a) && Array.isArray(b)) {
+		if (a.length !== b.length) return false;
+		for (let i = 0; i < a.length; i++) if (!equals(a[i], b[i], strict)) return false;
+		return true;
+	}
+	const aKeys = Object.keys(a);
+	if (aKeys.length !== Object.keys(b).length) return false;
+	for (const k of aKeys) if (!equals(a[k], b[k], strict)) return false;
+	return true;
+}
 async function byteLengthFromUrlUsingGet(url, requestInit = {}, fetchFn = globalThis.fetch) {
 	const controller = new AbortController();
 	const headers = new Headers(requestInit.headers);
@@ -95,4 +117,11 @@ function cacheKey(start, end, size) {
 	} else if (size === void 0) return `${start},`;
 	else return `${start},${size}`;
 }
-export { asyncBufferFromUrl, byteLengthFromUrl, cachedAsyncBuffer };
+function flatten(chunks) {
+	if (!chunks) return [];
+	if (chunks.length === 1) return chunks[0];
+	const output = [];
+	for (const chunk of chunks) concat(output, chunk);
+	return output;
+}
+export { asyncBufferFromUrl, byteLengthFromUrl, cachedAsyncBuffer, concat, equals, flatten };