@gscdump/engine 1.4.0 → 1.4.1

package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/schema.mjs

@@ -0,0 +1,254 @@
+import { normalizeShreddingConfig } from "./variant.mjs";
+function schemaFromColumnData({ columnData, schemaOverrides }) {
+	const schema = [{
+		name: "root",
+		num_children: columnData.length
+	}];
+	for (const { name, data, type, nullable, shredding } of columnData) if (schemaOverrides?.[name]) {
+		const override = schemaOverrides[name];
+		if (type || nullable !== void 0) throw new Error(`cannot provide both type and schema override for column ${name}`);
+		if (override.name !== name) throw new Error(`schema override for column ${name} must have matching name, got ${override.name}`);
+		if (override.type === "FIXED_LEN_BYTE_ARRAY" && !override.type_length) throw new Error("schema override for FIXED_LEN_BYTE_ARRAY must include type_length");
+		if (override.num_children) throw new Error("schema override does not support nested types");
+		schema.push(override);
+	} else if (type === "VARIANT") {
+		const repetition_type = nullable === false ? "REQUIRED" : "OPTIONAL";
+		const shreddingConfig = shredding && shredding !== true ? normalizeShreddingConfig(shredding) : void 0;
+		if (shreddingConfig) schema.push({
+			name,
+			repetition_type,
+			num_children: 3,
+			logical_type: { type: "VARIANT" }
+		}, {
+			name: "metadata",
+			type: "BYTE_ARRAY",
+			repetition_type: "REQUIRED"
+		}, {
+			name: "value",
+			type: "BYTE_ARRAY",
+			repetition_type: "OPTIONAL"
+		}, ...buildVariantTypedValue(shreddingConfig));
+		else schema.push({
+			name,
+			repetition_type,
+			num_children: 2,
+			logical_type: { type: "VARIANT" }
+		}, {
+			name: "metadata",
+			type: "BYTE_ARRAY",
+			repetition_type: "REQUIRED"
+		}, {
+			name: "value",
+			type: "BYTE_ARRAY",
+			repetition_type: "OPTIONAL"
+		});
+	} else if (type) schema.push(basicTypeToSchemaElement(name, type, nullable));
+	else schema.push(autoSchemaElement(name, data.slice(0, 1e3)));
+	return schema;
+}
+function buildVariantTypedValue(shredType) {
+	if (Array.isArray(shredType)) return [
+		{
+			name: "typed_value",
+			repetition_type: "OPTIONAL",
+			converted_type: "LIST",
+			num_children: 1
+		},
+		{
+			name: "list",
+			repetition_type: "REPEATED",
+			num_children: 1
+		},
+		{
+			name: "element",
+			repetition_type: "REQUIRED",
+			num_children: 2
+		},
+		{
+			name: "value",
+			type: "BYTE_ARRAY",
+			repetition_type: "OPTIONAL"
+		},
+		...buildVariantTypedValue(shredType[0])
+	];
+	if (typeof shredType === "object") {
+		const fieldNames = Object.keys(shredType);
+		const elements = [{
+			name: "typed_value",
+			repetition_type: "OPTIONAL",
+			num_children: fieldNames.length
+		}];
+		for (const fieldName of fieldNames) elements.push({
+			name: fieldName,
+			repetition_type: "OPTIONAL",
+			num_children: 2
+		}, {
+			name: "value",
+			type: "BYTE_ARRAY",
+			repetition_type: "OPTIONAL"
+		}, ...buildVariantTypedValue(shredType[fieldName]));
+		return elements;
+	}
+	return [shreddedLeafElement(shredType)];
+}
+function shreddedLeafElement(type) {
+	switch (type) {
+		case "STRING": return {
+			name: "typed_value",
+			type: "BYTE_ARRAY",
+			converted_type: "UTF8",
+			repetition_type: "OPTIONAL"
+		};
+		case "INT32": return {
+			name: "typed_value",
+			type: "INT32",
+			repetition_type: "OPTIONAL"
+		};
+		case "INT64": return {
+			name: "typed_value",
+			type: "INT64",
+			repetition_type: "OPTIONAL"
+		};
+		case "DOUBLE": return {
+			name: "typed_value",
+			type: "DOUBLE",
+			repetition_type: "OPTIONAL"
+		};
+		case "FLOAT": return {
+			name: "typed_value",
+			type: "FLOAT",
+			repetition_type: "OPTIONAL"
+		};
+		case "BOOLEAN": return {
+			name: "typed_value",
+			type: "BOOLEAN",
+			repetition_type: "OPTIONAL"
+		};
+		case "TIMESTAMP": return {
+			name: "typed_value",
+			type: "INT64",
+			converted_type: "TIMESTAMP_MICROS",
+			repetition_type: "OPTIONAL"
+		};
+		default: throw new Error(`unsupported shredded field type: ${type}`);
+	}
+}
+function basicTypeToSchemaElement(name, type, nullable) {
+	const repetition_type = nullable === false ? "REQUIRED" : "OPTIONAL";
+	if (type === "STRING") return {
+		name,
+		type: "BYTE_ARRAY",
+		converted_type: "UTF8",
+		repetition_type
+	};
+	if (type === "JSON") return {
+		name,
+		type: "BYTE_ARRAY",
+		converted_type: "JSON",
+		repetition_type
+	};
+	if (type === "TIMESTAMP") return {
+		name,
+		type: "INT64",
+		converted_type: "TIMESTAMP_MILLIS",
+		repetition_type
+	};
+	if (type === "UUID") return {
+		name,
+		type: "FIXED_LEN_BYTE_ARRAY",
+		type_length: 16,
+		logical_type: { type: "UUID" },
+		repetition_type
+	};
+	if (type === "FLOAT16") return {
+		name,
+		type: "FIXED_LEN_BYTE_ARRAY",
+		type_length: 2,
+		logical_type: { type: "FLOAT16" },
+		repetition_type
+	};
+	if (type === "GEOMETRY") return {
+		name,
+		type: "BYTE_ARRAY",
+		logical_type: { type: "GEOMETRY" },
+		repetition_type
+	};
+	if (type === "GEOGRAPHY") return {
+		name,
+		type: "BYTE_ARRAY",
+		logical_type: { type: "GEOGRAPHY" },
+		repetition_type
+	};
+	return {
+		name,
+		type,
+		repetition_type
+	};
+}
+function autoSchemaElement(name, values) {
+	let type;
+	let repetition_type = "REQUIRED";
+	let converted_type;
+	if (values instanceof Int32Array) return {
+		name,
+		type: "INT32",
+		repetition_type
+	};
+	if (values instanceof BigInt64Array) return {
+		name,
+		type: "INT64",
+		repetition_type
+	};
+	if (values instanceof Float32Array) return {
+		name,
+		type: "FLOAT",
+		repetition_type
+	};
+	if (values instanceof Float64Array) return {
+		name,
+		type: "DOUBLE",
+		repetition_type
+	};
+	for (const value of values) if (value === null || value === void 0) repetition_type = "OPTIONAL";
+	else {
+		let valueType;
+		let valueConvertedType;
+		if (typeof value === "boolean") valueType = "BOOLEAN";
+		else if (typeof value === "bigint") valueType = "INT64";
+		else if (Number.isInteger(value)) valueType = "INT32";
+		else if (typeof value === "number") valueType = "DOUBLE";
+		else if (value instanceof Uint8Array) valueType = "BYTE_ARRAY";
+		else if (typeof value === "string") {
+			valueType = "BYTE_ARRAY";
+			valueConvertedType = "UTF8";
+		} else if (value instanceof Date) {
+			valueType = "INT64";
+			valueConvertedType = "TIMESTAMP_MILLIS";
+		} else if (typeof value === "object") {
+			valueType = "BYTE_ARRAY";
+			valueConvertedType = "JSON";
+		} else throw new Error(`cannot determine parquet type for: ${value}`);
+		if (type === void 0) {
+			type = valueType;
+			converted_type = valueConvertedType;
+		} else if (type === "INT32" && valueType === "DOUBLE") type = "DOUBLE";
+		else if (type === "DOUBLE" && valueType === "INT32") valueType = "DOUBLE";
+		else if (type !== valueType || converted_type !== valueConvertedType) throw new Error(`parquet cannot write mixed types: ${converted_type ?? type} and ${valueConvertedType ?? valueType}`);
+	}
+	if (!type) {
+		type = "BYTE_ARRAY";
+		repetition_type = "OPTIONAL";
+	}
+	return {
+		name,
+		type,
+		repetition_type,
+		converted_type
+	};
+}
+function getMaxRepetitionLevel(schemaPath) {
+	let maxLevel = 0;
+	for (const element of schemaPath) if (element.repetition_type === "REPEATED") maxLevel++;
+	return maxLevel;
+}
+export { autoSchemaElement, getMaxRepetitionLevel, schemaFromColumnData };

package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/snappy.mjs

@@ -0,0 +1,125 @@
+import { ByteWriter } from "./bytewriter.mjs";
+const BLOCK_SIZE = 65536;
+const MAX_HASH_TABLE_BITS = 14;
+const globalHashTables = new Array(15);
+function snappyCompress(input) {
+	const writer = new ByteWriter();
+	writer.appendVarInt(input.length);
+	let pos = 0;
+	while (pos < input.length) {
+		const fragmentSize = Math.min(input.length - pos, BLOCK_SIZE);
+		compressFragment(writer, input, pos, fragmentSize);
+		pos += fragmentSize;
+	}
+	return writer.getBytes();
+}
+function hashFunc(key, hashFuncShift) {
+	return key * 506832829 >>> hashFuncShift;
+}
+function load32(array, pos) {
+	return array[pos] + (array[pos + 1] << 8) + (array[pos + 2] << 16) + (array[pos + 3] << 24);
+}
+function equals32(array, pos1, pos2) {
+	return array[pos1] === array[pos2] && array[pos1 + 1] === array[pos2 + 1] && array[pos1 + 2] === array[pos2 + 2] && array[pos1 + 3] === array[pos2 + 3];
+}
+function emitLiteral(writer, input, ip, len) {
+	if (len <= 60) writer.appendUint8(len - 1 << 2);
+	else if (len < 256) {
+		writer.appendUint8(240);
+		writer.appendUint8(len - 1);
+	} else {
+		writer.appendUint8(244);
+		writer.appendUint8(len - 1 & 255);
+		writer.appendUint8(len - 1 >>> 8);
+	}
+	writer.appendBytes(input.subarray(ip, ip + len));
+}
+function emitCopyLessThan64(writer, offset, len) {
+	if (len < 12 && offset < 2048) {
+		writer.appendUint8(1 + (len - 4 << 2) + (offset >>> 8 << 5));
+		writer.appendUint8(offset & 255);
+	} else {
+		writer.appendUint8(2 + (len - 1 << 2));
+		writer.appendUint8(offset & 255);
+		writer.appendUint8(offset >>> 8);
+	}
+}
+function emitCopy(writer, offset, len) {
+	while (len >= 68) {
+		emitCopyLessThan64(writer, offset, 64);
+		len -= 64;
+	}
+	if (len > 64) {
+		emitCopyLessThan64(writer, offset, 60);
+		len -= 60;
+	}
+	emitCopyLessThan64(writer, offset, len);
+}
+function compressFragment(writer, input, ip, inputSize) {
+	let hashTableBits = 1;
+	while (1 << hashTableBits <= inputSize && hashTableBits <= MAX_HASH_TABLE_BITS) hashTableBits++;
+	hashTableBits--;
+	const hashFuncShift = 32 - hashTableBits;
+	globalHashTables[hashTableBits] ??= new Uint16Array(1 << hashTableBits);
+	const hashTable = globalHashTables[hashTableBits];
+	hashTable.fill(0);
+	const ipEnd = ip + inputSize;
+	let ipLimit;
+	const baseIp = ip;
+	let nextEmit = ip;
+	let hash, nextHash;
+	let nextIp, candidate, skip;
+	let bytesBetweenHashLookups;
+	let base, matched, offset;
+	let prevHash, curHash;
+	let flag = true;
+	const INPUT_MARGIN = 15;
+	if (inputSize >= INPUT_MARGIN) {
+		ipLimit = ipEnd - INPUT_MARGIN;
+		ip++;
+		nextHash = hashFunc(load32(input, ip), hashFuncShift);
+		while (flag) {
+			skip = 32;
+			nextIp = ip;
+			do {
+				ip = nextIp;
+				hash = nextHash;
+				bytesBetweenHashLookups = skip >>> 5;
+				skip++;
+				nextIp = ip + bytesBetweenHashLookups;
+				if (ip > ipLimit) {
+					flag = false;
+					break;
+				}
+				nextHash = hashFunc(load32(input, nextIp), hashFuncShift);
+				candidate = baseIp + hashTable[hash];
+				hashTable[hash] = ip - baseIp;
+			} while (!equals32(input, ip, candidate));
+			if (!flag) break;
+			emitLiteral(writer, input, nextEmit, ip - nextEmit);
+			do {
+				base = ip;
+				matched = 4;
+				while (ip + matched < ipEnd && input[ip + matched] === input[candidate + matched]) matched++;
+				ip += matched;
+				offset = base - candidate;
+				emitCopy(writer, offset, matched);
+				nextEmit = ip;
+				if (ip >= ipLimit) {
+					flag = false;
+					break;
+				}
+				prevHash = hashFunc(load32(input, ip - 1), hashFuncShift);
+				hashTable[prevHash] = ip - 1 - baseIp;
+				curHash = hashFunc(load32(input, ip), hashFuncShift);
+				candidate = baseIp + hashTable[curHash];
+				hashTable[curHash] = ip - baseIp;
+			} while (equals32(input, ip, candidate));
+			if (!flag) break;
+			ip++;
+			nextHash = hashFunc(load32(input, ip), hashFuncShift);
+		}
+	}
+	if (nextEmit < ipEnd) emitLiteral(writer, input, nextEmit, ipEnd - nextEmit);
+}
+export { snappyCompress };

package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/splitstream.mjs

@@ -0,0 +1,38 @@
+function writeByteStreamSplit(writer, values, type, typeLength) {
+	const count = values.length;
+	let bytes;
+	let width;
+	if (type === "FLOAT") {
+		const typed = values instanceof Float32Array ? values : new Float32Array(numberArray(values));
+		bytes = new Uint8Array(typed.buffer, typed.byteOffset, typed.byteLength);
+		width = 4;
+	} else if (type === "DOUBLE") {
+		const typed = values instanceof Float64Array ? values : new Float64Array(numberArray(values));
+		bytes = new Uint8Array(typed.buffer, typed.byteOffset, typed.byteLength);
+		width = 8;
+	} else if (type === "INT32") {
+		const typed = values instanceof Int32Array ? values : new Int32Array(numberArray(values));
+		bytes = new Uint8Array(typed.buffer, typed.byteOffset, typed.byteLength);
+		width = 4;
+	} else if (type === "INT64") {
+		const typed = bigIntArray(values);
+		bytes = new Uint8Array(typed.buffer, typed.byteOffset, typed.byteLength);
+		width = 8;
+	} else if (type === "FIXED_LEN_BYTE_ARRAY") {
+		if (!typeLength) throw new Error("parquet byte_stream_split missing type_length");
+		width = typeLength;
+		bytes = new Uint8Array(count * width);
+		for (let i = 0; i < count; i++) bytes.set(values[i], i * width);
+	} else throw new Error(`parquet byte_stream_split unsupported type: ${type}`);
+	for (let b = 0; b < width; b++) for (let i = 0; i < count; i++) writer.appendUint8(bytes[i * width + b]);
+}
+function numberArray(values) {
+	if (Array.isArray(values) && values.every((v) => typeof v === "number")) return values;
+	throw new Error("Expected number array for BYTE_STREAM_SPLIT encoding");
+}
+function bigIntArray(values) {
+	if (values instanceof BigInt64Array) return values;
+	if (Array.isArray(values) && values.every((v) => typeof v === "bigint")) return new BigInt64Array(values);
+	throw new Error("Expected bigint array for BYTE_STREAM_SPLIT encoding");
+}
+export { writeByteStreamSplit };

package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/thrift.mjs

@@ -0,0 +1,79 @@
+const STOP = 0;
+const TRUE = 1;
+const FALSE = 2;
+const BYTE = 3;
+const I32 = 5;
+const I64 = 6;
+const DOUBLE = 7;
+const BINARY = 8;
+const LIST = 9;
+const STRUCT = 12;
+function serializeTCompactProtocol(writer, data) {
+	writeElement(writer, STRUCT, data);
+}
+function writeElement(writer, type, value) {
+	if (type === TRUE) return;
+	if (type === FALSE) return;
+	if (type === BYTE && typeof value === "number") writer.appendUint8(value);
+	else if (type === I32 && typeof value === "number") writer.appendZigZag(value);
+	else if (type === I64 && typeof value === "bigint") writer.appendZigZag(value);
+	else if (type === DOUBLE && typeof value === "number") writer.appendFloat64(value);
+	else if (type === BINARY && typeof value === "string") {
+		const bytes = new TextEncoder().encode(value);
+		writer.appendVarInt(bytes.length);
+		writer.appendBytes(bytes);
+	} else if (type === BINARY && value instanceof Uint8Array) {
+		writer.appendVarInt(value.byteLength);
+		writer.appendBytes(value);
+	} else if (type === LIST && Array.isArray(value)) {
+		const elemType = getCompactTypeForList(value);
+		if (value.length > 14) {
+			writer.appendUint8(240 | elemType);
+			writer.appendVarInt(value.length);
+		} else writer.appendUint8(value.length << 4 | elemType);
+		if (elemType === FALSE) for (const v of value) writer.appendUint8(v ? 1 : 0);
+		else for (const v of value) writeElement(writer, elemType, v);
+	} else if (type === STRUCT && typeof value === "object") {
+		let lastFid = 0;
+		for (const [k, v] of Object.entries(value)) {
+			if (v === void 0) continue;
+			const fid = parseInt(k.replace(/^field_/, ""), 10);
+			if (Number.isNaN(fid)) throw new Error(`thrift invalid field name: ${k}. Expected "field_###"`);
+			const t = getCompactTypeForValue(v);
+			const delta = fid - lastFid;
+			if (delta <= 0) throw new Error(`thrift non-monotonic field id: fid=${fid}, lastFid=${lastFid}`);
+			if (delta > 15) {
+				writer.appendUint8(t);
+				writer.appendZigZag(fid);
+			} else writer.appendUint8(delta << 4 | t);
+			writeElement(writer, t, v);
+			lastFid = fid;
+		}
+		writer.appendUint8(STOP);
+	} else throw new Error(`thrift invalid type ${type} for value ${value}`);
+}
+function getCompactTypeForValue(value) {
+	if (value === true) return TRUE;
+	if (value === false) return FALSE;
+	if (Number.isInteger(value)) return I32;
+	if (typeof value === "number") return DOUBLE;
+	if (typeof value === "bigint") return I64;
+	if (typeof value === "string") return BINARY;
+	if (value instanceof Uint8Array) return BINARY;
+	if (Array.isArray(value)) return LIST;
+	if (value && typeof value === "object") return STRUCT;
+	throw new Error(`Cannot determine thrift compact type for: ${value}`);
+}
+function getCompactTypeForList(value) {
+	let elemType = 0;
+	for (const v of value) {
+		let t = getCompactTypeForValue(v);
+		if (t === TRUE) t = FALSE;
+		if (!elemType) elemType = t;
+		if (elemType === DOUBLE && t === I32) t = DOUBLE;
+		if (elemType === I32 && t === DOUBLE) elemType = DOUBLE;
+		if (t !== elemType) throw new Error(`thrift invalid type for list element: ${v} (expected type ${elemType})`);
+	}
+	return elemType ?? BYTE;
+}
+export { serializeTCompactProtocol };

package/dist/{_chunks/libs/hyparquet-writer.d.mts → node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/types.d.mts}

@@ -1,4 +1,5 @@
-interface Writer$1 {
+import "../../../../hyparquet@1.26.1/node_modules/hyparquet/types/node.mjs";
+interface Writer {
   buffer: ArrayBuffer;
   view: DataView;
   offset: number; // total bytes written
@@ -19,5 +20,4 @@ interface Writer$1 {
   appendVarBigInt(value: bigint): void;
   appendZigZag(value: number | bigint): void;
 }
-type Writer = Writer$1;
 export { Writer };