@gscdump/lakehouse 0.1.0

package/dist/_chunks/libs/hyparquet.d.mts

@@ -0,0 +1,10 @@
+/**
+ * File-like object that can read slices of a file asynchronously.
+ */
+interface AsyncBuffer$1 {
+  byteLength: number;
+  slice(start: number, end?: number): Awaitable<ArrayBuffer>;
+}
+type Awaitable<T> = T | Promise<T>;
+type AsyncBuffer = AsyncBuffer$1;
+export { AsyncBuffer };

package/dist/_chunks/libs/hyparquet.mjs

@@ -0,0 +1,460 @@
+const ParquetTypes = [
+	"BOOLEAN",
+	"INT32",
+	"INT64",
+	"INT96",
+	"FLOAT",
+	"DOUBLE",
+	"BYTE_ARRAY",
+	"FIXED_LEN_BYTE_ARRAY"
+];
+const Encodings = [
+	"PLAIN",
+	"GROUP_VAR_INT",
+	"PLAIN_DICTIONARY",
+	"RLE",
+	"BIT_PACKED",
+	"DELTA_BINARY_PACKED",
+	"DELTA_LENGTH_BYTE_ARRAY",
+	"DELTA_BYTE_ARRAY",
+	"RLE_DICTIONARY",
+	"BYTE_STREAM_SPLIT"
+];
+const FieldRepetitionTypes = [
+	"REQUIRED",
+	"OPTIONAL",
+	"REPEATED"
+];
+const ConvertedTypes = [
+	"UTF8",
+	"MAP",
+	"MAP_KEY_VALUE",
+	"LIST",
+	"ENUM",
+	"DECIMAL",
+	"DATE",
+	"TIME_MILLIS",
+	"TIME_MICROS",
+	"TIMESTAMP_MILLIS",
+	"TIMESTAMP_MICROS",
+	"UINT_8",
+	"UINT_16",
+	"UINT_32",
+	"UINT_64",
+	"INT_8",
+	"INT_16",
+	"INT_32",
+	"INT_64",
+	"JSON",
+	"BSON",
+	"INTERVAL"
+];
+const CompressionCodecs = [
+	"UNCOMPRESSED",
+	"SNAPPY",
+	"GZIP",
+	"LZO",
+	"BROTLI",
+	"LZ4",
+	"ZSTD",
+	"LZ4_RAW"
+];
+const PageTypes = [
+	"DATA_PAGE",
+	"INDEX_PAGE",
+	"DICTIONARY_PAGE",
+	"DATA_PAGE_V2"
+];
+const BoundaryOrders = [
+	"UNORDERED",
+	"ASCENDING",
+	"DESCENDING"
+];
+const EdgeInterpolationAlgorithms = [
+	"SPHERICAL",
+	"VINCENTY",
+	"THOMAS",
+	"ANDOYER",
+	"KARNEY"
+];
+new TextDecoder();
+function parseDecimal(bytes) {
+	if (!bytes.length) return 0;
+	let value = 0n;
+	for (const byte of bytes) value = value * 256n + BigInt(byte);
+	const bits = bytes.length * 8;
+	if (value >= 2n ** BigInt(bits - 1)) value -= 2n ** BigInt(bits);
+	return Number(value);
+}
+function schemaTree(schema, rootIndex, path) {
+	const element = schema[rootIndex];
+	const children = [];
+	let count = 1;
+	if (element.num_children) while (children.length < element.num_children) {
+		const childElement = schema[rootIndex + count];
+		const child = schemaTree(schema, rootIndex + count, [...path, childElement.name]);
+		count += child.count;
+		children.push(child);
+	}
+	return {
+		count,
+		element,
+		children,
+		path
+	};
+}
+function getSchemaPath(schema, name) {
+	let tree = schemaTree(schema, 0, []);
+	const path = [tree];
+	for (const part of name) {
+		const child = tree.children.find((child) => child.element.name === part);
+		if (!child) throw new Error(`parquet schema element not found: ${name}`);
+		path.push(child);
+		tree = child;
+	}
+	return path;
+}
+function getMaxDefinitionLevel(schemaPath) {
+	let maxLevel = 0;
+	for (const { element } of schemaPath.slice(1)) if (element.repetition_type !== "REQUIRED") maxLevel++;
+	return maxLevel;
+}
+function isListLike(schema) {
+	if (!schema) return false;
+	if (schema.element.converted_type !== "LIST") return false;
+	if (schema.children.length > 1) return false;
+	const firstChild = schema.children[0];
+	if (firstChild.children.length > 1) return false;
+	if (firstChild.element.repetition_type !== "REPEATED") return false;
+	return true;
+}
+function isMapLike(schema) {
+	if (!schema) return false;
+	if (schema.element.converted_type !== "MAP") return false;
+	if (schema.children.length > 1) return false;
+	const firstChild = schema.children[0];
+	if (firstChild.children.length !== 2) return false;
+	if (firstChild.element.repetition_type !== "REPEATED") return false;
+	if (firstChild.children.find((child) => child.element.name === "key")?.element.repetition_type === "REPEATED") return false;
+	if (firstChild.children.find((child) => child.element.name === "value")?.element.repetition_type === "REPEATED") return false;
+	return true;
+}
+const defaultInitialFetchSize = 1 << 19;
+new TextDecoder();
+const MASK = 18446744073709551615n;
+const PRIME1 = 11400714785074694791n;
+const PRIME2 = 14029467366897019727n;
+const PRIME3 = 1609587929392839161n;
+const PRIME4 = 9650029242287828579n;
+const PRIME5 = 2870177450012600261n;
+function rotl64(x, r) {
+	return (x << r | x >> 64n - r) & MASK;
+}
+function round(acc, val) {
+	acc = acc + val * PRIME2 & MASK;
+	acc = rotl64(acc, 31n);
+	return acc * PRIME1 & MASK;
+}
+function mergeRound(acc, val) {
+	acc ^= round(0n, val);
+	return acc * PRIME1 + PRIME4 & MASK;
+}
+function xxhash64(input, seed = 0n) {
+	const view = new DataView(input.buffer, input.byteOffset, input.byteLength);
+	const len = input.byteLength;
+	let offset = 0;
+	let h64;
+	if (len >= 32) {
+		let v1 = seed + PRIME1 + PRIME2 & MASK;
+		let v2 = seed + PRIME2 & MASK;
+		let v3 = seed;
+		let v4 = seed - PRIME1 & MASK;
+		while (offset + 32 <= len) {
+			v1 = round(v1, view.getBigUint64(offset, true));
+			offset += 8;
+			v2 = round(v2, view.getBigUint64(offset, true));
+			offset += 8;
+			v3 = round(v3, view.getBigUint64(offset, true));
+			offset += 8;
+			v4 = round(v4, view.getBigUint64(offset, true));
+			offset += 8;
+		}
+		h64 = rotl64(v1, 1n) + rotl64(v2, 7n) + rotl64(v3, 12n) + rotl64(v4, 18n) & MASK;
+		h64 = mergeRound(h64, v1);
+		h64 = mergeRound(h64, v2);
+		h64 = mergeRound(h64, v3);
+		h64 = mergeRound(h64, v4);
+	} else h64 = seed + PRIME5 & MASK;
+	h64 = h64 + BigInt(len) & MASK;
+	while (offset + 8 <= len) {
+		h64 ^= round(0n, view.getBigUint64(offset, true));
+		h64 = rotl64(h64, 27n) * PRIME1 + PRIME4 & MASK;
+		offset += 8;
+	}
+	if (offset + 4 <= len) {
+		h64 ^= BigInt(view.getUint32(offset, true)) * PRIME1 & MASK;
+		h64 = rotl64(h64, 23n) * PRIME2 + PRIME3 & MASK;
+		offset += 4;
+	}
+	while (offset < len) {
+		h64 ^= BigInt(view.getUint8(offset)) * PRIME5 & MASK;
+		h64 = rotl64(h64, 11n) * PRIME1 & MASK;
+		offset += 1;
+	}
+	h64 ^= h64 >> 33n;
+	h64 = h64 * PRIME2 & MASK;
+	h64 ^= h64 >> 29n;
+	h64 = h64 * PRIME3 & MASK;
+	h64 ^= h64 >> 32n;
+	return h64;
+}
+const textEncoder = new TextEncoder();
+new Uint32Array([
+	1203114875,
+	1150766481,
+	2284105051,
+	2729912477,
+	1884591559,
+	770785867,
+	2667333959,
+	1550580529
+]);
+function hashParquetValue(value, element) {
+	if (value === null || value === void 0) return void 0;
+	const { type, converted_type, logical_type } = element;
+	if (type === "BOOLEAN") {
+		if (typeof value !== "boolean") return void 0;
+		return xxhash64(new Uint8Array([value ? 1 : 0]));
+	}
+	if (type === "FLOAT") {
+		if (typeof value !== "number") return void 0;
+		const buf = /* @__PURE__ */ new ArrayBuffer(4);
+		new DataView(buf).setFloat32(0, value, true);
+		return xxhash64(new Uint8Array(buf));
+	}
+	if (type === "DOUBLE") {
+		if (typeof value !== "number") return void 0;
+		const buf = /* @__PURE__ */ new ArrayBuffer(8);
+		new DataView(buf).setFloat64(0, value, true);
+		return xxhash64(new Uint8Array(buf));
+	}
+	if (type === "INT32") {
+		if (converted_type === "DATE" || converted_type === "DECIMAL" || converted_type === "TIME_MILLIS") return void 0;
+		if (logical_type?.type === "DATE" || logical_type?.type === "TIME" || logical_type?.type === "DECIMAL") return void 0;
+		if (typeof value !== "number" || !Number.isInteger(value)) return void 0;
+		const buf = /* @__PURE__ */ new ArrayBuffer(4);
+		new DataView(buf).setInt32(0, value | 0, true);
+		return xxhash64(new Uint8Array(buf));
+	}
+	if (type === "INT64") {
+		if (converted_type === "TIMESTAMP_MILLIS" || converted_type === "TIMESTAMP_MICROS") return void 0;
+		if (converted_type === "TIME_MICROS" || converted_type === "DECIMAL") return void 0;
+		if (logical_type?.type === "TIMESTAMP" || logical_type?.type === "TIME" || logical_type?.type === "DECIMAL") return void 0;
+		let bigValue;
+		if (typeof value === "bigint") bigValue = value;
+		else if (typeof value === "number" && Number.isSafeInteger(value)) bigValue = BigInt(value);
+		else return void 0;
+		const buf = /* @__PURE__ */ new ArrayBuffer(8);
+		new DataView(buf).setBigUint64(0, BigInt.asUintN(64, bigValue), true);
+		return xxhash64(new Uint8Array(buf));
+	}
+	if (type === "BYTE_ARRAY") {
+		if (converted_type === "JSON" || converted_type === "BSON" || converted_type === "DECIMAL") return void 0;
+		if (logical_type?.type === "JSON" || logical_type?.type === "BSON" || logical_type?.type === "VARIANT") return void 0;
+		if (logical_type?.type === "GEOMETRY" || logical_type?.type === "GEOGRAPHY") return void 0;
+		if (typeof value === "string") return xxhash64(textEncoder.encode(value));
+		if (value instanceof Uint8Array) return xxhash64(value);
+		return;
+	}
+	if (type === "FIXED_LEN_BYTE_ARRAY") {
+		if (converted_type === "DECIMAL" || converted_type === "INTERVAL") return void 0;
+		if (logical_type?.type === "DECIMAL" || logical_type?.type === "UUID" || logical_type?.type === "FLOAT16") return void 0;
+		if (logical_type?.type === "GEOMETRY" || logical_type?.type === "GEOGRAPHY") return void 0;
+		if (value instanceof Uint8Array) return xxhash64(value);
+		return;
+	}
+}
+function toJson(obj) {
+	if (obj === void 0) return null;
+	if (typeof obj === "bigint") return Number(obj);
+	if (Object.is(obj, -0)) return 0;
+	if (Array.isArray(obj)) return obj.map(toJson);
+	if (obj instanceof Uint8Array) return Array.from(obj);
+	if (obj instanceof Date) return obj.toISOString();
+	if (obj instanceof Object) {
+		const newObj = {};
+		for (const key of Object.keys(obj)) {
+			if (obj[key] === void 0) continue;
+			newObj[key] = toJson(obj[key]);
+		}
+		return newObj;
+	}
+	return obj;
+}
+async function byteLengthFromUrlUsingGet(url, requestInit = {}, fetchFn = globalThis.fetch) {
+	const controller = new AbortController();
+	const headers = new Headers(requestInit.headers);
+	headers.set("Range", "bytes=0-0");
+	const res = await fetchFn(url, {
+		...requestInit,
+		headers,
+		signal: controller.signal
+	});
+	if (!res.ok) throw new Error(`fetch with range failed ${res.status}`);
+	if (res.status === 206) {
+		const contentRange = res.headers.get("Content-Range");
+		if (!contentRange) throw new Error("missing content-range header");
+		const match = contentRange.match(/bytes \d+-\d+\/(\d+)/);
+		if (!match) throw new Error(`invalid content-range header: ${contentRange}`);
+		return parseInt(match[1]);
+	}
+	if (res.status === 200) {
+		const contentLength = res.headers.get("Content-Length");
+		controller.abort();
+		if (contentLength) return parseInt(contentLength);
+	}
+	throw new Error("server does not support range requests and missing content-length");
+}
+async function byteLengthFromUrl(url, requestInit, customFetch) {
+	const fetch = customFetch ?? globalThis.fetch;
+	const res = await fetch(url, {
+		...requestInit,
+		method: "HEAD"
+	});
+	if (res.status === 403) return byteLengthFromUrlUsingGet(url, requestInit, fetch);
+	if (!res.ok) throw new Error(`fetch head failed ${res.status}`);
+	const length = res.headers.get("Content-Length");
+	if (!length) return byteLengthFromUrlUsingGet(url, requestInit, fetch);
+	return parseInt(length);
+}
+async function asyncBufferFromUrl({ url, byteLength, requestInit, fetch: customFetch }) {
+	if (!url) throw new Error("missing url");
+	const fetch = customFetch ?? globalThis.fetch;
+	byteLength ??= await byteLengthFromUrl(url, requestInit, fetch);
+	let buffer = void 0;
+	const init = requestInit || {};
+	return {
+		byteLength,
+		async slice(start, end) {
+			if (buffer) return buffer.then((buffer) => buffer.slice(start, end));
+			const headers = new Headers(init.headers);
+			const endStr = end === void 0 ? "" : end - 1;
+			headers.set("Range", `bytes=${start}-${endStr}`);
+			const res = await fetch(url, {
+				...init,
+				headers
+			});
+			if (!res.ok || !res.body) throw new Error(`fetch failed ${res.status}`);
+			if (res.status === 200) {
+				buffer = res.arrayBuffer();
+				return buffer.then((buffer) => buffer.slice(start, end));
+			} else if (res.status === 206) return res.arrayBuffer();
+			else throw new Error(`fetch received unexpected status code ${res.status}`);
+		}
+	};
+}
+function cachedAsyncBuffer({ byteLength, slice }, { minSize = defaultInitialFetchSize } = {}) {
+	if (byteLength < minSize) {
+		const buffer = slice(0, byteLength);
+		return {
+			byteLength,
+			async slice(start, end) {
+				return (await buffer).slice(start, end);
+			}
+		};
+	}
+	const cache = /* @__PURE__ */ new Map();
+	return {
+		byteLength,
+		slice(start, end) {
+			const key = cacheKey(start, end, byteLength);
+			const cached = cache.get(key);
+			if (cached) return cached;
+			const promise = slice(start, end);
+			cache.set(key, promise);
+			return promise;
+		}
+	};
+}
+function cacheKey(start, end, size) {
+	if (start < 0) {
+		if (end !== void 0) throw new Error(`invalid suffix range [${start}, ${end}]`);
+		if (size === void 0) return `${start},`;
+		return `${size + start},${size}`;
+	} else if (end !== void 0) {
+		if (start > end) throw new Error(`invalid empty range [${start}, ${end}]`);
+		return `${start},${end}`;
+	} else if (size === void 0) return `${start},`;
+	else return `${start},${size}`;
+}
+new TextDecoder();
+const WORD_MASK = [
+	0,
+	255,
+	65535,
+	16777215,
+	4294967295
+];
+function copyBytes(fromArray, fromPos, toArray, toPos, length) {
+	for (let i = 0; i < length; i++) toArray[toPos + i] = fromArray[fromPos + i];
+}
+function snappyUncompress(input, output) {
+	const inputLength = input.byteLength;
+	const outputLength = output.byteLength;
+	let pos = 0;
+	let outPos = 0;
+	while (pos < inputLength) {
+		const c = input[pos];
+		pos++;
+		if (c < 128) break;
+	}
+	if (outputLength && pos >= inputLength) throw new Error("invalid snappy length header");
+	while (pos < inputLength) {
+		const c = input[pos];
+		let len = 0;
+		pos++;
+		if (pos >= inputLength) throw new Error("missing eof marker");
+		if ((c & 3) === 0) {
+			let len = (c >>> 2) + 1;
+			if (len > 60) {
+				if (pos + 3 >= inputLength) throw new Error("snappy error literal pos + 3 >= inputLength");
+				const lengthSize = len - 60;
+				len = input[pos] + (input[pos + 1] << 8) + (input[pos + 2] << 16) + (input[pos + 3] << 24);
+				len = (len & WORD_MASK[lengthSize]) + 1;
+				pos += lengthSize;
+			}
+			if (pos + len > inputLength) throw new Error("snappy error literal exceeds input length");
+			copyBytes(input, pos, output, outPos, len);
+			pos += len;
+			outPos += len;
+		} else {
+			let offset = 0;
+			switch (c & 3) {
+				case 1:
+					len = (c >>> 2 & 7) + 4;
+					offset = input[pos] + (c >>> 5 << 8);
+					pos++;
+					break;
+				case 2:
+					if (inputLength <= pos + 1) throw new Error("snappy error end of input");
+					len = (c >>> 2) + 1;
+					offset = input[pos] + (input[pos + 1] << 8);
+					pos += 2;
+					break;
+				case 3:
+					if (inputLength <= pos + 3) throw new Error("snappy error end of input");
+					len = (c >>> 2) + 1;
+					offset = input[pos] + (input[pos + 1] << 8) + (input[pos + 2] << 16) + (input[pos + 3] << 24);
+					pos += 4;
+					break;
+				default: break;
+			}
+			if (offset === 0 || isNaN(offset)) throw new Error(`invalid offset ${offset} pos ${pos} inputLength ${inputLength}`);
+			if (offset > outPos) throw new Error("cannot copy from before start of buffer");
+			copyBytes(output, outPos - offset, output, outPos, len);
+			outPos += len;
+		}
+	}
+	if (outPos !== outputLength) throw new Error("premature end of input");
+}
+export { BoundaryOrders, CompressionCodecs, ConvertedTypes, EdgeInterpolationAlgorithms, Encodings, FieldRepetitionTypes, PageTypes, ParquetTypes, asyncBufferFromUrl, cachedAsyncBuffer, getMaxDefinitionLevel, getSchemaPath, hashParquetValue, isListLike, isMapLike, parseDecimal, snappyUncompress, toJson };