@gscdump/lakehouse 1.4.0 → 1.4.2

package/dist/_chunks/libs/hyparquet.mjs

@@ -1,474 +0,0 @@
-new TextDecoder();
-function parseDecimal(bytes) {
-	if (!bytes.length) return 0;
-	let value = 0n;
-	for (const byte of bytes) value = value * 256n + BigInt(byte);
-	const bits = bytes.length * 8;
-	if (value >= 2n ** BigInt(bits - 1)) value -= 2n ** BigInt(bits);
-	return Number(value);
-}
-const defaultInitialFetchSize = 1 << 19;
-new TextDecoder();
-new TextEncoder();
-new Uint32Array([
-	1203114875,
-	1150766481,
-	2284105051,
-	2729912477,
-	1884591559,
-	770785867,
-	2667333959,
-	1550580529
-]);
-async function byteLengthFromUrlUsingGet(url, requestInit = {}, fetchFn = globalThis.fetch) {
-	const controller = new AbortController();
-	const headers = new Headers(requestInit.headers);
-	headers.set("Range", "bytes=0-0");
-	const res = await fetchFn(url, {
-		...requestInit,
-		headers,
-		signal: controller.signal
-	});
-	if (!res.ok) throw new Error(`fetch with range failed ${res.status}`);
-	if (res.status === 206) {
-		const contentRange = res.headers.get("Content-Range");
-		if (!contentRange) throw new Error("missing content-range header");
-		const match = contentRange.match(/bytes \d+-\d+\/(\d+)/);
-		if (!match) throw new Error(`invalid content-range header: ${contentRange}`);
-		return parseInt(match[1]);
-	}
-	if (res.status === 200) {
-		const contentLength = res.headers.get("Content-Length");
-		controller.abort();
-		if (contentLength) return parseInt(contentLength);
-	}
-	throw new Error("server does not support range requests and missing content-length");
-}
-async function byteLengthFromUrl(url, requestInit, customFetch) {
-	const fetch = customFetch ?? globalThis.fetch;
-	const res = await fetch(url, {
-		...requestInit,
-		method: "HEAD"
-	});
-	if (res.status === 403) return byteLengthFromUrlUsingGet(url, requestInit, fetch);
-	if (!res.ok) throw new Error(`fetch head failed ${res.status}`);
-	const length = res.headers.get("Content-Length");
-	if (!length) return byteLengthFromUrlUsingGet(url, requestInit, fetch);
-	return parseInt(length);
-}
-async function asyncBufferFromUrl({ url, byteLength, requestInit, fetch: customFetch }) {
-	if (!url) throw new Error("missing url");
-	const fetch = customFetch ?? globalThis.fetch;
-	byteLength ??= await byteLengthFromUrl(url, requestInit, fetch);
-	let buffer = void 0;
-	const init = requestInit || {};
-	return {
-		byteLength,
-		async slice(start, end) {
-			if (buffer) return buffer.then((buffer) => buffer.slice(start, end));
-			const headers = new Headers(init.headers);
-			const endStr = end === void 0 ? "" : end - 1;
-			headers.set("Range", `bytes=${start}-${endStr}`);
-			const res = await fetch(url, {
-				...init,
-				headers
-			});
-			if (!res.ok || !res.body) throw new Error(`fetch failed ${res.status}`);
-			if (res.status === 200) {
-				buffer = res.arrayBuffer();
-				return buffer.then((buffer) => buffer.slice(start, end));
-			} else if (res.status === 206) return res.arrayBuffer();
-			else throw new Error(`fetch received unexpected status code ${res.status}`);
-		}
-	};
-}
-function cachedAsyncBuffer({ byteLength, slice }, { minSize = defaultInitialFetchSize } = {}) {
-	if (byteLength < minSize) {
-		const buffer = slice(0, byteLength);
-		return {
-			byteLength,
-			async slice(start, end) {
-				return (await buffer).slice(start, end);
-			}
-		};
-	}
-	const cache = /* @__PURE__ */ new Map();
-	return {
-		byteLength,
-		slice(start, end) {
-			const key = cacheKey(start, end, byteLength);
-			const cached = cache.get(key);
-			if (cached) return cached;
-			const promise = slice(start, end);
-			cache.set(key, promise);
-			return promise;
-		}
-	};
-}
-function cacheKey(start, end, size) {
-	if (start < 0) {
-		if (end !== void 0) throw new Error(`invalid suffix range [${start}, ${end}]`);
-		if (size === void 0) return `${start},`;
-		return `${size + start},${size}`;
-	} else if (end !== void 0) {
-		if (start > end) throw new Error(`invalid empty range [${start}, ${end}]`);
-		return `${start},${end}`;
-	} else if (size === void 0) return `${start},`;
-	else return `${start},${size}`;
-}
-new TextDecoder();
-const WORD_MASK = [
-	0,
-	255,
-	65535,
-	16777215,
-	4294967295
-];
-function copyBytes(fromArray, fromPos, toArray, toPos, length) {
-	for (let i = 0; i < length; i++) toArray[toPos + i] = fromArray[fromPos + i];
-}
-function snappyUncompress(input, output) {
-	const inputLength = input.byteLength;
-	const outputLength = output.byteLength;
-	let pos = 0;
-	let outPos = 0;
-	while (pos < inputLength) {
-		const c = input[pos];
-		pos++;
-		if (c < 128) break;
-	}
-	if (outputLength && pos >= inputLength) throw new Error("invalid snappy length header");
-	while (pos < inputLength) {
-		const c = input[pos];
-		let len = 0;
-		pos++;
-		if (pos >= inputLength) throw new Error("missing eof marker");
-		if ((c & 3) === 0) {
-			let len = (c >>> 2) + 1;
-			if (len > 60) {
-				if (pos + 3 >= inputLength) throw new Error("snappy error literal pos + 3 >= inputLength");
-				const lengthSize = len - 60;
-				len = input[pos] + (input[pos + 1] << 8) + (input[pos + 2] << 16) + (input[pos + 3] << 24);
-				len = (len & WORD_MASK[lengthSize]) + 1;
-				pos += lengthSize;
-			}
-			if (pos + len > inputLength) throw new Error("snappy error literal exceeds input length");
-			copyBytes(input, pos, output, outPos, len);
-			pos += len;
-			outPos += len;
-		} else {
-			let offset = 0;
-			switch (c & 3) {
-				case 1:
-					len = (c >>> 2 & 7) + 4;
-					offset = input[pos] + (c >>> 5 << 8);
-					pos++;
-					break;
-				case 2:
-					if (inputLength <= pos + 1) throw new Error("snappy error end of input");
-					len = (c >>> 2) + 1;
-					offset = input[pos] + (input[pos + 1] << 8);
-					pos += 2;
-					break;
-				case 3:
-					if (inputLength <= pos + 3) throw new Error("snappy error end of input");
-					len = (c >>> 2) + 1;
-					offset = input[pos] + (input[pos + 1] << 8) + (input[pos + 2] << 16) + (input[pos + 3] << 24);
-					pos += 4;
-					break;
-				default: break;
-			}
-			if (offset === 0 || isNaN(offset)) throw new Error(`invalid offset ${offset} pos ${pos} inputLength ${inputLength}`);
-			if (offset > outPos) throw new Error("cannot copy from before start of buffer");
-			copyBytes(output, outPos - offset, output, outPos, len);
-			outPos += len;
-		}
-	}
-	if (outPos !== outputLength) throw new Error("premature end of input");
-}
-function schemaTree(schema, rootIndex, path) {
-	const element = schema[rootIndex];
-	const children = [];
-	let count = 1;
-	if (element.num_children) while (children.length < element.num_children) {
-		const childElement = schema[rootIndex + count];
-		const child = schemaTree(schema, rootIndex + count, [...path, childElement.name]);
-		count += child.count;
-		children.push(child);
-	}
-	return {
-		count,
-		element,
-		children,
-		path
-	};
-}
-function getSchemaPath(schema, name) {
-	let tree = schemaTree(schema, 0, []);
-	const path = [tree];
-	for (const part of name) {
-		const child = tree.children.find((child) => child.element.name === part);
-		if (!child) throw new Error(`parquet schema element not found: ${name}`);
-		path.push(child);
-		tree = child;
-	}
-	return path;
-}
-function getMaxDefinitionLevel(schemaPath) {
-	let maxLevel = 0;
-	for (const { element } of schemaPath.slice(1)) if (element.repetition_type !== "REQUIRED") maxLevel++;
-	return maxLevel;
-}
-function isListLike(schema) {
-	if (!schema) return false;
-	if (schema.element.converted_type !== "LIST") return false;
-	if (schema.children.length > 1) return false;
-	const firstChild = schema.children[0];
-	if (firstChild.children.length > 1) return false;
-	if (firstChild.element.repetition_type !== "REPEATED") return false;
-	return true;
-}
-function isMapLike(schema) {
-	if (!schema) return false;
-	if (schema.element.converted_type !== "MAP") return false;
-	if (schema.children.length > 1) return false;
-	const firstChild = schema.children[0];
-	if (firstChild.children.length !== 2) return false;
-	if (firstChild.element.repetition_type !== "REPEATED") return false;
-	if (firstChild.children.find((child) => child.element.name === "key")?.element.repetition_type === "REPEATED") return false;
-	if (firstChild.children.find((child) => child.element.name === "value")?.element.repetition_type === "REPEATED") return false;
-	return true;
-}
-const MASK = 18446744073709551615n;
-const PRIME1 = 11400714785074694791n;
-const PRIME2 = 14029467366897019727n;
-const PRIME3 = 1609587929392839161n;
-const PRIME4 = 9650029242287828579n;
-const PRIME5 = 2870177450012600261n;
-function rotl64(x, r) {
-	return (x << r | x >> 64n - r) & MASK;
-}
-function round(acc, val) {
-	acc = acc + val * PRIME2 & MASK;
-	acc = rotl64(acc, 31n);
-	return acc * PRIME1 & MASK;
-}
-function mergeRound(acc, val) {
-	acc ^= round(0n, val);
-	return acc * PRIME1 + PRIME4 & MASK;
-}
-function xxhash64(input, seed = 0n) {
-	const view = new DataView(input.buffer, input.byteOffset, input.byteLength);
-	const len = input.byteLength;
-	let offset = 0;
-	let h64;
-	if (len >= 32) {
-		let v1 = seed + PRIME1 + PRIME2 & MASK;
-		let v2 = seed + PRIME2 & MASK;
-		let v3 = seed;
-		let v4 = seed - PRIME1 & MASK;
-		while (offset + 32 <= len) {
-			v1 = round(v1, view.getBigUint64(offset, true));
-			offset += 8;
-			v2 = round(v2, view.getBigUint64(offset, true));
-			offset += 8;
-			v3 = round(v3, view.getBigUint64(offset, true));
-			offset += 8;
-			v4 = round(v4, view.getBigUint64(offset, true));
-			offset += 8;
-		}
-		h64 = rotl64(v1, 1n) + rotl64(v2, 7n) + rotl64(v3, 12n) + rotl64(v4, 18n) & MASK;
-		h64 = mergeRound(h64, v1);
-		h64 = mergeRound(h64, v2);
-		h64 = mergeRound(h64, v3);
-		h64 = mergeRound(h64, v4);
-	} else h64 = seed + PRIME5 & MASK;
-	h64 = h64 + BigInt(len) & MASK;
-	while (offset + 8 <= len) {
-		h64 ^= round(0n, view.getBigUint64(offset, true));
-		h64 = rotl64(h64, 27n) * PRIME1 + PRIME4 & MASK;
-		offset += 8;
-	}
-	if (offset + 4 <= len) {
-		h64 ^= BigInt(view.getUint32(offset, true)) * PRIME1 & MASK;
-		h64 = rotl64(h64, 23n) * PRIME2 + PRIME3 & MASK;
-		offset += 4;
-	}
-	while (offset < len) {
-		h64 ^= BigInt(view.getUint8(offset)) * PRIME5 & MASK;
-		h64 = rotl64(h64, 11n) * PRIME1 & MASK;
-		offset += 1;
-	}
-	h64 ^= h64 >> 33n;
-	h64 = h64 * PRIME2 & MASK;
-	h64 ^= h64 >> 29n;
-	h64 = h64 * PRIME3 & MASK;
-	h64 ^= h64 >> 32n;
-	return h64;
-}
-const textEncoder = new TextEncoder();
-new Uint32Array([
-	1203114875,
-	1150766481,
-	2284105051,
-	2729912477,
-	1884591559,
-	770785867,
-	2667333959,
-	1550580529
-]);
-function hashParquetValue(value, element) {
-	if (value === null || value === void 0) return void 0;
-	const { type, converted_type, logical_type } = element;
-	if (type === "BOOLEAN") {
-		if (typeof value !== "boolean") return void 0;
-		return xxhash64(new Uint8Array([value ? 1 : 0]));
-	}
-	if (type === "FLOAT") {
-		if (typeof value !== "number") return void 0;
-		const buf = /* @__PURE__ */ new ArrayBuffer(4);
-		new DataView(buf).setFloat32(0, value, true);
-		return xxhash64(new Uint8Array(buf));
-	}
-	if (type === "DOUBLE") {
-		if (typeof value !== "number") return void 0;
-		const buf = /* @__PURE__ */ new ArrayBuffer(8);
-		new DataView(buf).setFloat64(0, value, true);
-		return xxhash64(new Uint8Array(buf));
-	}
-	if (type === "INT32") {
-		if (converted_type === "DATE" || converted_type === "DECIMAL" || converted_type === "TIME_MILLIS") return void 0;
-		if (logical_type?.type === "DATE" || logical_type?.type === "TIME" || logical_type?.type === "DECIMAL") return void 0;
-		if (typeof value !== "number" || !Number.isInteger(value)) return void 0;
-		const buf = /* @__PURE__ */ new ArrayBuffer(4);
-		new DataView(buf).setInt32(0, value | 0, true);
-		return xxhash64(new Uint8Array(buf));
-	}
-	if (type === "INT64") {
-		if (converted_type === "TIMESTAMP_MILLIS" || converted_type === "TIMESTAMP_MICROS") return void 0;
-		if (converted_type === "TIME_MICROS" || converted_type === "DECIMAL") return void 0;
-		if (logical_type?.type === "TIMESTAMP" || logical_type?.type === "TIME" || logical_type?.type === "DECIMAL") return void 0;
-		let bigValue;
-		if (typeof value === "bigint") bigValue = value;
-		else if (typeof value === "number" && Number.isSafeInteger(value)) bigValue = BigInt(value);
-		else return void 0;
-		const buf = /* @__PURE__ */ new ArrayBuffer(8);
-		new DataView(buf).setBigUint64(0, BigInt.asUintN(64, bigValue), true);
-		return xxhash64(new Uint8Array(buf));
-	}
-	if (type === "BYTE_ARRAY") {
-		if (converted_type === "JSON" || converted_type === "BSON" || converted_type === "DECIMAL") return void 0;
-		if (logical_type?.type === "JSON" || logical_type?.type === "BSON" || logical_type?.type === "VARIANT") return void 0;
-		if (logical_type?.type === "GEOMETRY" || logical_type?.type === "GEOGRAPHY") return void 0;
-		if (typeof value === "string") return xxhash64(textEncoder.encode(value));
-		if (value instanceof Uint8Array) return xxhash64(value);
-		return;
-	}
-	if (type === "FIXED_LEN_BYTE_ARRAY") {
-		if (converted_type === "DECIMAL" || converted_type === "INTERVAL") return void 0;
-		if (logical_type?.type === "DECIMAL" || logical_type?.type === "UUID" || logical_type?.type === "FLOAT16") return void 0;
-		if (logical_type?.type === "GEOMETRY" || logical_type?.type === "GEOGRAPHY") return void 0;
-		if (value instanceof Uint8Array) return xxhash64(value);
-		return;
-	}
-}
-const ParquetTypes = [
-	"BOOLEAN",
-	"INT32",
-	"INT64",
-	"INT96",
-	"FLOAT",
-	"DOUBLE",
-	"BYTE_ARRAY",
-	"FIXED_LEN_BYTE_ARRAY"
-];
-const Encodings = [
-	"PLAIN",
-	"GROUP_VAR_INT",
-	"PLAIN_DICTIONARY",
-	"RLE",
-	"BIT_PACKED",
-	"DELTA_BINARY_PACKED",
-	"DELTA_LENGTH_BYTE_ARRAY",
-	"DELTA_BYTE_ARRAY",
-	"RLE_DICTIONARY",
-	"BYTE_STREAM_SPLIT"
-];
-const FieldRepetitionTypes = [
-	"REQUIRED",
-	"OPTIONAL",
-	"REPEATED"
-];
-const ConvertedTypes = [
-	"UTF8",
-	"MAP",
-	"MAP_KEY_VALUE",
-	"LIST",
-	"ENUM",
-	"DECIMAL",
-	"DATE",
-	"TIME_MILLIS",
-	"TIME_MICROS",
-	"TIMESTAMP_MILLIS",
-	"TIMESTAMP_MICROS",
-	"UINT_8",
-	"UINT_16",
-	"UINT_32",
-	"UINT_64",
-	"INT_8",
-	"INT_16",
-	"INT_32",
-	"INT_64",
-	"JSON",
-	"BSON",
-	"INTERVAL"
-];
-const CompressionCodecs = [
-	"UNCOMPRESSED",
-	"SNAPPY",
-	"GZIP",
-	"LZO",
-	"BROTLI",
-	"LZ4",
-	"ZSTD",
-	"LZ4_RAW"
-];
-const PageTypes = [
-	"DATA_PAGE",
-	"INDEX_PAGE",
-	"DICTIONARY_PAGE",
-	"DATA_PAGE_V2"
-];
-const BoundaryOrders = [
-	"UNORDERED",
-	"ASCENDING",
-	"DESCENDING"
-];
-const EdgeInterpolationAlgorithms = [
-	"SPHERICAL",
-	"VINCENTY",
-	"THOMAS",
-	"ANDOYER",
-	"KARNEY"
-];
-new TextDecoder();
-new TextDecoder();
-function toJson(obj) {
-	if (obj === void 0) return null;
-	if (typeof obj === "bigint") return Number(obj);
-	if (Object.is(obj, -0)) return 0;
-	if (Array.isArray(obj)) return obj.map(toJson);
-	if (obj instanceof Uint8Array) return Array.from(obj);
-	if (obj instanceof Date) return obj.toISOString();
-	if (obj instanceof Object) {
-		const newObj = {};
-		for (const key of Object.keys(obj)) {
-			if (obj[key] === void 0) continue;
-			newObj[key] = toJson(obj[key]);
-		}
-		return newObj;
-	}
-	return obj;
-}
-new TextDecoder();
-export { BoundaryOrders, CompressionCodecs, ConvertedTypes, EdgeInterpolationAlgorithms, Encodings, FieldRepetitionTypes, PageTypes, ParquetTypes, asyncBufferFromUrl, cachedAsyncBuffer, getMaxDefinitionLevel, getSchemaPath, hashParquetValue, isListLike, isMapLike, parseDecimal, snappyUncompress, toJson };