@gscdump/engine 3.4.4 → 3.6.0

package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/metadata.mjs

@@ -0,0 +1,241 @@
+import { CompressionCodecs, ConvertedTypes, EdgeInterpolationAlgorithms, Encodings, FieldRepetitionTypes, PageTypes, ParquetTypes } from "./constants.mjs";
+import { DEFAULT_PARSERS, parseDecimal, parseFloat16 } from "./convert.mjs";
+import { getSchemaPath } from "./schema.mjs";
+import { deserializeTCompactProtocol } from "./thrift.mjs";
+import { markGeoColumns } from "./geoparquet.mjs";
+const defaultInitialFetchSize = 1 << 19;
+const decoder = new TextDecoder();
+function decode(value) {
+	return value && decoder.decode(value);
+}
+async function parquetMetadataAsync(asyncBuffer, { parsers, initialFetchSize = defaultInitialFetchSize, geoparquet = true } = {}) {
+	if (!asyncBuffer || !(asyncBuffer.byteLength >= 0)) throw new Error("parquet expected AsyncBuffer");
+	const footerOffset = Math.max(0, asyncBuffer.byteLength - initialFetchSize);
+	const footerBuffer = await asyncBuffer.slice(footerOffset, asyncBuffer.byteLength);
+	const footerView = new DataView(footerBuffer);
+	if (footerView.getUint32(footerBuffer.byteLength - 4, true) !== 827474256) throw new Error("parquet file invalid (footer != PAR1)");
+	const metadataLength = footerView.getUint32(footerBuffer.byteLength - 8, true);
+	if (metadataLength > asyncBuffer.byteLength - 8) throw new Error(`parquet metadata length ${metadataLength} exceeds available buffer ${asyncBuffer.byteLength - 8}`);
+	if (metadataLength + 8 > initialFetchSize) {
+		const metadataOffset = asyncBuffer.byteLength - metadataLength - 8;
+		const metadataBuffer = await asyncBuffer.slice(metadataOffset, footerOffset);
+		const combinedBuffer = new ArrayBuffer(metadataLength + 8);
+		const combinedView = new Uint8Array(combinedBuffer);
+		combinedView.set(new Uint8Array(metadataBuffer));
+		combinedView.set(new Uint8Array(footerBuffer), footerOffset - metadataOffset);
+		return parquetMetadata(combinedBuffer, {
+			parsers,
+			geoparquet
+		});
+	} else return parquetMetadata(footerBuffer, {
+		parsers,
+		geoparquet
+	});
+}
+function parquetMetadata(arrayBuffer, { parsers, geoparquet = true } = {}) {
+	if (!(arrayBuffer instanceof ArrayBuffer)) throw new Error("parquet expected ArrayBuffer");
+	const view = new DataView(arrayBuffer);
+	parsers = {
+		...DEFAULT_PARSERS,
+		...parsers
+	};
+	if (view.byteLength < 8) throw new Error("parquet file is too short");
+	if (view.getUint32(view.byteLength - 4, true) !== 827474256) throw new Error("parquet file invalid (footer != PAR1)");
+	const metadataLengthOffset = view.byteLength - 8;
+	const metadataLength = view.getUint32(metadataLengthOffset, true);
+	if (metadataLength > view.byteLength - 8) throw new Error(`parquet metadata length ${metadataLength} exceeds available buffer ${view.byteLength - 8}`);
+	const reader = {
+		view,
+		offset: metadataLengthOffset - metadataLength
+	};
+	const metadata = deserializeTCompactProtocol(reader);
+	const version = metadata.field_1;
+	const schema = metadata.field_2.map((field) => ({
+		type: ParquetTypes[field.field_1],
+		type_length: field.field_2,
+		repetition_type: FieldRepetitionTypes[field.field_3],
+		name: decode(field.field_4),
+		num_children: field.field_5,
+		converted_type: ConvertedTypes[field.field_6],
+		scale: field.field_7,
+		precision: field.field_8,
+		field_id: field.field_9,
+		logical_type: logicalType(field.field_10)
+	}));
+	const columnSchema = schema.filter((e) => e.type);
+	const num_rows = metadata.field_3;
+	const row_groups = metadata.field_4.map((rowGroup) => ({
+		columns: rowGroup.field_1.map((column, columnIndex) => ({
+			file_path: decode(column.field_1),
+			file_offset: column.field_2,
+			meta_data: column.field_3 && {
+				type: ParquetTypes[column.field_3.field_1],
+				encodings: column.field_3.field_2?.map((e) => Encodings[e]),
+				path_in_schema: column.field_3.field_3.map(decode),
+				codec: CompressionCodecs[column.field_3.field_4],
+				num_values: column.field_3.field_5,
+				total_uncompressed_size: column.field_3.field_6,
+				total_compressed_size: column.field_3.field_7,
+				key_value_metadata: column.field_3.field_8?.map((kv) => ({
+					key: decode(kv.field_1),
+					value: decode(kv.field_2)
+				})),
+				data_page_offset: column.field_3.field_9,
+				index_page_offset: column.field_3.field_10,
+				dictionary_page_offset: column.field_3.field_11,
+				statistics: convertStats(column.field_3.field_12, columnSchema[columnIndex], parsers),
+				encoding_stats: column.field_3.field_13?.map((encodingStat) => ({
+					page_type: PageTypes[encodingStat.field_1],
+					encoding: Encodings[encodingStat.field_2],
+					count: encodingStat.field_3
+				})),
+				bloom_filter_offset: column.field_3.field_14,
+				bloom_filter_length: column.field_3.field_15,
+				size_statistics: column.field_3.field_16 && {
+					unencoded_byte_array_data_bytes: column.field_3.field_16.field_1,
+					repetition_level_histogram: column.field_3.field_16.field_2,
+					definition_level_histogram: column.field_3.field_16.field_3
+				},
+				geospatial_statistics: column.field_3.field_17 && {
+					bbox: column.field_3.field_17.field_1 && {
+						xmin: column.field_3.field_17.field_1.field_1,
+						xmax: column.field_3.field_17.field_1.field_2,
+						ymin: column.field_3.field_17.field_1.field_3,
+						ymax: column.field_3.field_17.field_1.field_4,
+						zmin: column.field_3.field_17.field_1.field_5,
+						zmax: column.field_3.field_17.field_1.field_6,
+						mmin: column.field_3.field_17.field_1.field_7,
+						mmax: column.field_3.field_17.field_1.field_8
+					},
+					geospatial_types: column.field_3.field_17.field_2
+				}
+			},
+			offset_index_offset: column.field_4,
+			offset_index_length: column.field_5,
+			column_index_offset: column.field_6,
+			column_index_length: column.field_7,
+			crypto_metadata: column.field_8,
+			encrypted_column_metadata: column.field_9
+		})),
+		total_byte_size: rowGroup.field_2,
+		num_rows: rowGroup.field_3,
+		sorting_columns: rowGroup.field_4?.map((sortingColumn) => ({
+			column_idx: sortingColumn.field_1,
+			descending: sortingColumn.field_2,
+			nulls_first: sortingColumn.field_3
+		})),
+		file_offset: rowGroup.field_5,
+		total_compressed_size: rowGroup.field_6,
+		ordinal: rowGroup.field_7
+	}));
+	const key_value_metadata = metadata.field_5?.map((kv) => ({
+		key: decode(kv.field_1),
+		value: decode(kv.field_2)
+	}));
+	const created_by = decode(metadata.field_6);
+	if (geoparquet) markGeoColumns(schema, key_value_metadata);
+	return {
+		version,
+		schema,
+		num_rows,
+		row_groups,
+		key_value_metadata,
+		created_by,
+		metadata_length: metadataLength
+	};
+}
+function parquetSchema({ schema }) {
+	return getSchemaPath(schema, [])[0];
+}
+function logicalType(logicalType) {
+	if (logicalType?.field_1) return { type: "STRING" };
+	if (logicalType?.field_2) return { type: "MAP" };
+	if (logicalType?.field_3) return { type: "LIST" };
+	if (logicalType?.field_4) return { type: "ENUM" };
+	if (logicalType?.field_5) return {
+		type: "DECIMAL",
+		scale: logicalType.field_5.field_1,
+		precision: logicalType.field_5.field_2
+	};
+	if (logicalType?.field_6) return { type: "DATE" };
+	if (logicalType?.field_7) return {
+		type: "TIME",
+		isAdjustedToUTC: logicalType.field_7.field_1,
+		unit: timeUnit(logicalType.field_7.field_2)
+	};
+	if (logicalType?.field_8) return {
+		type: "TIMESTAMP",
+		isAdjustedToUTC: logicalType.field_8.field_1,
+		unit: timeUnit(logicalType.field_8.field_2)
+	};
+	if (logicalType?.field_10) return {
+		type: "INTEGER",
+		bitWidth: logicalType.field_10.field_1,
+		isSigned: logicalType.field_10.field_2
+	};
+	if (logicalType?.field_11) return { type: "NULL" };
+	if (logicalType?.field_12) return { type: "JSON" };
+	if (logicalType?.field_13) return { type: "BSON" };
+	if (logicalType?.field_14) return { type: "UUID" };
+	if (logicalType?.field_15) return { type: "FLOAT16" };
+	if (logicalType?.field_16) return {
+		type: "VARIANT",
+		specification_version: logicalType.field_16.field_1
+	};
+	if (logicalType?.field_17) return {
+		type: "GEOMETRY",
+		crs: decode(logicalType.field_17.field_1)
+	};
+	if (logicalType?.field_18) return {
+		type: "GEOGRAPHY",
+		crs: decode(logicalType.field_18.field_1),
+		algorithm: EdgeInterpolationAlgorithms[logicalType.field_18.field_2]
+	};
+	return logicalType;
+}
+function timeUnit(unit) {
+	if (unit.field_1) return "MILLIS";
+	if (unit.field_2) return "MICROS";
+	if (unit.field_3) return "NANOS";
+	throw new Error("parquet time unit required");
+}
+function convertStats(stats, schema, parsers) {
+	return stats && {
+		max: convertMetadata(stats.field_1, schema, parsers),
+		min: convertMetadata(stats.field_2, schema, parsers),
+		null_count: stats.field_3,
+		distinct_count: stats.field_4,
+		max_value: convertMetadata(stats.field_5, schema, parsers),
+		min_value: convertMetadata(stats.field_6, schema, parsers),
+		is_max_value_exact: stats.field_7,
+		is_min_value_exact: stats.field_8
+	};
+}
+function convertMetadata(value, schema, parsers) {
+	const { type, converted_type, logical_type } = schema;
+	if (value === void 0) return value;
+	if (type === "BOOLEAN") return value[0] === 1;
+	if (type === "BYTE_ARRAY") return parsers.stringFromBytes(value);
+	const view = new DataView(value.buffer, value.byteOffset, value.byteLength);
+	if (type === "FLOAT" && view.byteLength === 4) return view.getFloat32(0, true);
+	if (type === "DOUBLE" && view.byteLength === 8) return view.getFloat64(0, true);
+	if (type === "INT32" && converted_type === "DECIMAL" && view.byteLength === 4) return view.getInt32(0, true) * 10 ** -(schema.scale || 0);
+	if (type === "INT64" && converted_type === "DECIMAL" && view.byteLength === 8) return Number(view.getBigInt64(0, true)) * 10 ** -(schema.scale || 0);
+	if (type === "INT32" && converted_type === "DATE") return parsers.dateFromDays(view.getInt32(0, true));
+	if (type === "INT64" && converted_type === "TIMESTAMP_MILLIS") return parsers.timestampFromMilliseconds(view.getBigInt64(0, true));
+	if (type === "INT64" && converted_type === "TIMESTAMP_MICROS") return parsers.timestampFromMicroseconds(view.getBigInt64(0, true));
+	if (type === "INT64" && logical_type?.type === "TIMESTAMP" && logical_type?.unit === "NANOS") return parsers.timestampFromNanoseconds(view.getBigInt64(0, true));
+	if (type === "INT64" && logical_type?.type === "TIMESTAMP" && logical_type?.unit === "MICROS") return parsers.timestampFromMicroseconds(view.getBigInt64(0, true));
+	if (type === "INT64" && logical_type?.type === "TIMESTAMP") return parsers.timestampFromMilliseconds(view.getBigInt64(0, true));
+	const unsigned = converted_type?.startsWith("UINT_") || logical_type?.type === "INTEGER" && !logical_type.isSigned;
+	if (type === "INT32" && unsigned && view.byteLength === 4) return view.getUint32(0, true);
+	if (type === "INT64" && unsigned && view.byteLength === 8) return view.getBigUint64(0, true);
+	if (type === "INT32" && view.byteLength === 4) return view.getInt32(0, true);
+	if (type === "INT64" && view.byteLength === 8) return view.getBigInt64(0, true);
+	if (converted_type === "DECIMAL") return parseDecimal(value) * 10 ** -(schema.scale || 0);
+	if (logical_type?.type === "FLOAT16") return parseFloat16(value);
+	if (logical_type?.type === "UUID") return parsers.uuidFromBytes(value);
+	if (type === "FIXED_LEN_BYTE_ARRAY") return value;
+	return value;
+}
+export { convertMetadata, defaultInitialFetchSize, parquetMetadata, parquetMetadataAsync, parquetSchema };

package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/node.mjs

@@ -0,0 +1,7 @@
+import { parquetMetadataAsync, parquetSchema } from "./metadata.mjs";
+import "./indexes.mjs";
+import "./utils.mjs";
+import "./snappy.mjs";
+import { parquetReadObjects } from "./read.mjs";
+import "./index.mjs";
+export { parquetMetadataAsync, parquetReadObjects, parquetSchema };

package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/plain.mjs

@@ -0,0 +1,79 @@
+function readPlain(reader, type, count, fixedLength) {
+	if (count === 0) return [];
+	if (type === "BOOLEAN") return readPlainBoolean(reader, count);
+	else if (type === "INT32") return readPlainInt32(reader, count);
+	else if (type === "INT64") return readPlainInt64(reader, count);
+	else if (type === "INT96") return readPlainInt96(reader, count);
+	else if (type === "FLOAT") return readPlainFloat(reader, count);
+	else if (type === "DOUBLE") return readPlainDouble(reader, count);
+	else if (type === "BYTE_ARRAY") return readPlainByteArray(reader, count);
+	else if (type === "FIXED_LEN_BYTE_ARRAY") {
+		if (!fixedLength) throw new Error("parquet missing fixed length");
+		return readPlainByteArrayFixed(reader, count, fixedLength);
+	} else throw new Error(`parquet unhandled type: ${type}`);
+}
+function readPlainBoolean(reader, count) {
+	const values = new Array(count);
+	for (let i = 0; i < count; i++) {
+		const byteOffset = reader.offset + (i / 8 | 0);
+		const bitOffset = i % 8;
+		const byte = reader.view.getUint8(byteOffset);
+		values[i] = (byte & 1 << bitOffset) !== 0;
+	}
+	reader.offset += Math.ceil(count / 8);
+	return values;
+}
+function readPlainInt32(reader, count) {
+	const values = (reader.view.byteOffset + reader.offset) % 4 ? new Int32Array(align(reader.view.buffer, reader.view.byteOffset + reader.offset, count * 4)) : new Int32Array(reader.view.buffer, reader.view.byteOffset + reader.offset, count);
+	reader.offset += count * 4;
+	return values;
+}
+function readPlainInt64(reader, count) {
+	const values = (reader.view.byteOffset + reader.offset) % 8 ? new BigInt64Array(align(reader.view.buffer, reader.view.byteOffset + reader.offset, count * 8)) : new BigInt64Array(reader.view.buffer, reader.view.byteOffset + reader.offset, count);
+	reader.offset += count * 8;
+	return values;
+}
+function readPlainInt96(reader, count) {
+	const values = new Array(count);
+	for (let i = 0; i < count; i++) {
+		const low = reader.view.getBigInt64(reader.offset + i * 12, true);
+		const high = reader.view.getInt32(reader.offset + i * 12 + 8, true);
+		values[i] = BigInt(high) << 64n | low;
+	}
+	reader.offset += count * 12;
+	return values;
+}
+function readPlainFloat(reader, count) {
+	const values = (reader.view.byteOffset + reader.offset) % 4 ? new Float32Array(align(reader.view.buffer, reader.view.byteOffset + reader.offset, count * 4)) : new Float32Array(reader.view.buffer, reader.view.byteOffset + reader.offset, count);
+	reader.offset += count * 4;
+	return values;
+}
+function readPlainDouble(reader, count) {
+	const values = (reader.view.byteOffset + reader.offset) % 8 ? new Float64Array(align(reader.view.buffer, reader.view.byteOffset + reader.offset, count * 8)) : new Float64Array(reader.view.buffer, reader.view.byteOffset + reader.offset, count);
+	reader.offset += count * 8;
+	return values;
+}
+function readPlainByteArray(reader, count) {
+	const values = new Array(count);
+	for (let i = 0; i < count; i++) {
+		const length = reader.view.getUint32(reader.offset, true);
+		reader.offset += 4;
+		values[i] = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, length);
+		reader.offset += length;
+	}
+	return values;
+}
+function readPlainByteArrayFixed(reader, count, fixedLength) {
+	const values = new Array(count);
+	for (let i = 0; i < count; i++) {
+		values[i] = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, fixedLength);
+		reader.offset += fixedLength;
+	}
+	return values;
+}
+function align(buffer, offset, size) {
+	const aligned = new ArrayBuffer(size);
+	new Uint8Array(aligned).set(new Uint8Array(buffer, offset, size));
+	return aligned;
+}
+export { readPlain };