@gscdump/lakehouse 1.4.0 → 1.4.1

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/convert.mjs

@@ -0,0 +1,11 @@
+import "./wkb.mjs";
+new TextDecoder();
+function parseDecimal(bytes) {
+	if (!bytes.length) return 0;
+	let value = 0n;
+	for (const byte of bytes) value = value * 256n + BigInt(byte);
+	const bits = bytes.length * 8;
+	if (value >= 2n ** BigInt(bits - 1)) value -= 2n ** BigInt(bits);
+	return Number(value);
+}
+export { parseDecimal };

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/datapage.mjs

@@ -0,0 +1,6 @@
+import "./schema.mjs";
+import "./delta.mjs";
+import "./encoding.mjs";
+import "./plain.mjs";
+import "./snappy.mjs";
+export {};

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/delta.mjs

@@ -0,0 +1,2 @@
+import "./thrift.mjs";
+export {};

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/encoding.mjs

@@ -0,0 +1,2 @@
+import "./thrift.mjs";
+export {};

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/filter.mjs

@@ -0,0 +1,3 @@
+import "./bloom.mjs";
+import "./utils.mjs";
+export {};

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/geoparquet.mjs

@@ -0,0 +1 @@
+export {};

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/index.mjs

@@ -0,0 +1,7 @@
+import "./metadata.mjs";
+import "./indexes.mjs";
+import { asyncBufferFromUrl, cachedAsyncBuffer } from "./utils.mjs";
+import "./snappy.mjs";
+import "./read.mjs";
+import "./query.mjs";
+export { asyncBufferFromUrl, cachedAsyncBuffer };

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/indexes.mjs

@@ -0,0 +1,5 @@
+import "./constants.mjs";
+import "./convert.mjs";
+import "./thrift.mjs";
+import "./metadata.mjs";
+export {};

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/metadata.mjs

@@ -0,0 +1,8 @@
+import "./constants.mjs";
+import "./convert.mjs";
+import "./schema.mjs";
+import "./thrift.mjs";
+import "./geoparquet.mjs";
+const defaultInitialFetchSize = 1 << 19;
+new TextDecoder();
+export { defaultInitialFetchSize };

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/node.mjs

@@ -0,0 +1,4 @@
+import { asyncBufferFromUrl, cachedAsyncBuffer } from "./utils.mjs";
+import "./snappy.mjs";
+import "./index.mjs";
+export { asyncBufferFromUrl, cachedAsyncBuffer };

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/plain.mjs

@@ -0,0 +1 @@
+export {};

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/plan.mjs

@@ -0,0 +1,5 @@
+import "./schema.mjs";
+import "./metadata.mjs";
+import "./bloom.mjs";
+import "./filter.mjs";
+export {};

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/query.mjs

@@ -0,0 +1,3 @@
+import "./metadata.mjs";
+import "./read.mjs";
+export {};

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/read.mjs

@@ -0,0 +1,6 @@
+import "./metadata.mjs";
+import "./utils.mjs";
+import "./filter.mjs";
+import "./plan.mjs";
+import "./rowgroup.mjs";
+export {};

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/rowgroup.mjs

@@ -0,0 +1,7 @@
+import "./convert.mjs";
+import "./schema.mjs";
+import "./indexes.mjs";
+import "./utils.mjs";
+import "./assemble.mjs";
+import "./column.mjs";
+export {};

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/schema.mjs

@@ -0,0 +1 @@
+export {};

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/snappy.mjs

@@ -0,0 +1,70 @@
+const WORD_MASK = [
+	0,
+	255,
+	65535,
+	16777215,
+	4294967295
+];
+function copyBytes(fromArray, fromPos, toArray, toPos, length) {
+	for (let i = 0; i < length; i++) toArray[toPos + i] = fromArray[fromPos + i];
+}
+function snappyUncompress(input, output) {
+	const inputLength = input.byteLength;
+	const outputLength = output.byteLength;
+	let pos = 0;
+	let outPos = 0;
+	while (pos < inputLength) {
+		const c = input[pos];
+		pos++;
+		if (c < 128) break;
+	}
+	if (outputLength && pos >= inputLength) throw new Error("invalid snappy length header");
+	while (pos < inputLength) {
+		const c = input[pos];
+		let len = 0;
+		pos++;
+		if (pos >= inputLength) throw new Error("missing eof marker");
+		if ((c & 3) === 0) {
+			let len = (c >>> 2) + 1;
+			if (len > 60) {
+				if (pos + 3 >= inputLength) throw new Error("snappy error literal pos + 3 >= inputLength");
+				const lengthSize = len - 60;
+				len = input[pos] + (input[pos + 1] << 8) + (input[pos + 2] << 16) + (input[pos + 3] << 24);
+				len = (len & WORD_MASK[lengthSize]) + 1;
+				pos += lengthSize;
+			}
+			if (pos + len > inputLength) throw new Error("snappy error literal exceeds input length");
+			copyBytes(input, pos, output, outPos, len);
+			pos += len;
+			outPos += len;
+		} else {
+			let offset = 0;
+			switch (c & 3) {
+				case 1:
+					len = (c >>> 2 & 7) + 4;
+					offset = input[pos] + (c >>> 5 << 8);
+					pos++;
+					break;
+				case 2:
+					if (inputLength <= pos + 1) throw new Error("snappy error end of input");
+					len = (c >>> 2) + 1;
+					offset = input[pos] + (input[pos + 1] << 8);
+					pos += 2;
+					break;
+				case 3:
+					if (inputLength <= pos + 3) throw new Error("snappy error end of input");
+					len = (c >>> 2) + 1;
+					offset = input[pos] + (input[pos + 1] << 8) + (input[pos + 2] << 16) + (input[pos + 3] << 24);
+					pos += 4;
+					break;
+				default: break;
+			}
+			if (offset === 0 || isNaN(offset)) throw new Error(`invalid offset ${offset} pos ${pos} inputLength ${inputLength}`);
+			if (offset > outPos) throw new Error("cannot copy from before start of buffer");
+			copyBytes(output, outPos - offset, output, outPos, len);
+			outPos += len;
+		}
+	}
+	if (outPos !== outputLength) throw new Error("premature end of input");
+}
+export { snappyUncompress };

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/thrift.mjs

@@ -0,0 +1 @@
+export {};

package/dist/{_chunks/libs/hyparquet.d.mts → node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/types.d.mts}

@@ -1,10 +1,9 @@
 /**
  * File-like object that can read slices of a file asynchronously.
  */
-interface AsyncBuffer$1 {
+interface AsyncBuffer {
   byteLength: number;
   slice(start: number, end?: number): Awaitable<ArrayBuffer>;
 }
 type Awaitable<T> = T | Promise<T>;
-type AsyncBuffer = AsyncBuffer$1;
-export { AsyncBuffer };
+export { AsyncBuffer, Awaitable };

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/utils.mjs

@@ -0,0 +1,98 @@
+import { defaultInitialFetchSize } from "./metadata.mjs";
+async function byteLengthFromUrlUsingGet(url, requestInit = {}, fetchFn = globalThis.fetch) {
+	const controller = new AbortController();
+	const headers = new Headers(requestInit.headers);
+	headers.set("Range", "bytes=0-0");
+	const res = await fetchFn(url, {
+		...requestInit,
+		headers,
+		signal: controller.signal
+	});
+	if (!res.ok) throw new Error(`fetch with range failed ${res.status}`);
+	if (res.status === 206) {
+		const contentRange = res.headers.get("Content-Range");
+		if (!contentRange) throw new Error("missing content-range header");
+		const match = contentRange.match(/bytes \d+-\d+\/(\d+)/);
+		if (!match) throw new Error(`invalid content-range header: ${contentRange}`);
+		return parseInt(match[1]);
+	}
+	if (res.status === 200) {
+		const contentLength = res.headers.get("Content-Length");
+		controller.abort();
+		if (contentLength) return parseInt(contentLength);
+	}
+	throw new Error("server does not support range requests and missing content-length");
+}
+async function byteLengthFromUrl(url, requestInit, customFetch) {
+	const fetch = customFetch ?? globalThis.fetch;
+	const res = await fetch(url, {
+		...requestInit,
+		method: "HEAD"
+	});
+	if (res.status === 403) return byteLengthFromUrlUsingGet(url, requestInit, fetch);
+	if (!res.ok) throw new Error(`fetch head failed ${res.status}`);
+	const length = res.headers.get("Content-Length");
+	if (!length) return byteLengthFromUrlUsingGet(url, requestInit, fetch);
+	return parseInt(length);
+}
+async function asyncBufferFromUrl({ url, byteLength, requestInit, fetch: customFetch }) {
+	if (!url) throw new Error("missing url");
+	const fetch = customFetch ?? globalThis.fetch;
+	byteLength ??= await byteLengthFromUrl(url, requestInit, fetch);
+	let buffer = void 0;
+	const init = requestInit || {};
+	return {
+		byteLength,
+		async slice(start, end) {
+			if (buffer) return buffer.then((buffer) => buffer.slice(start, end));
+			const headers = new Headers(init.headers);
+			const endStr = end === void 0 ? "" : end - 1;
+			headers.set("Range", `bytes=${start}-${endStr}`);
+			const res = await fetch(url, {
+				...init,
+				headers
+			});
+			if (!res.ok || !res.body) throw new Error(`fetch failed ${res.status}`);
+			if (res.status === 200) {
+				buffer = res.arrayBuffer();
+				return buffer.then((buffer) => buffer.slice(start, end));
+			} else if (res.status === 206) return res.arrayBuffer();
+			else throw new Error(`fetch received unexpected status code ${res.status}`);
+		}
+	};
+}
+function cachedAsyncBuffer({ byteLength, slice }, { minSize = defaultInitialFetchSize } = {}) {
+	if (byteLength < minSize) {
+		const buffer = slice(0, byteLength);
+		return {
+			byteLength,
+			async slice(start, end) {
+				return (await buffer).slice(start, end);
+			}
+		};
+	}
+	const cache = /* @__PURE__ */ new Map();
+	return {
+		byteLength,
+		slice(start, end) {
+			const key = cacheKey(start, end, byteLength);
+			const cached = cache.get(key);
+			if (cached) return cached;
+			const promise = slice(start, end);
+			cache.set(key, promise);
+			return promise;
+		}
+	};
+}
+function cacheKey(start, end, size) {
+	if (start < 0) {
+		if (end !== void 0) throw new Error(`invalid suffix range [${start}, ${end}]`);
+		if (size === void 0) return `${start},`;
+		return `${size + start},${size}`;
+	} else if (end !== void 0) {
+		if (start > end) throw new Error(`invalid empty range [${start}, ${end}]`);
+		return `${start},${end}`;
+	} else if (size === void 0) return `${start},`;
+	else return `${start},${size}`;
+}
+export { asyncBufferFromUrl, byteLengthFromUrl, cachedAsyncBuffer };

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/variant.mjs

@@ -0,0 +1,3 @@
+import "./convert.mjs";
+new TextDecoder();
+export {};

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/wkb.mjs

@@ -0,0 +1 @@
+export {};

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/xxhash.mjs

@@ -0,0 +1 @@
+export {};

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/types/index.d.mts

@@ -0,0 +1,9 @@
+import { AsyncBuffer as AsyncBuffer$1 } from "../src/types.mjs";
+import "./query.mjs";
+import "./snappy.mjs";
+import "./indexes.mjs";
+import "./metadata.mjs";
+import "./read.mjs";
+import "./utils.mjs";
+type AsyncBuffer = AsyncBuffer$1;
+export { AsyncBuffer };

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/types/indexes.d.mts

@@ -0,0 +1 @@
+import "../src/types.mjs";

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/types/metadata.d.mts

@@ -0,0 +1 @@
+import "../src/types.mjs";

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/types/node.d.mts

@@ -0,0 +1,3 @@
+import "../src/types.mjs";
+import { AsyncBuffer } from "./index.mjs";
+export { AsyncBuffer };

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/types/query.d.mts

@@ -0,0 +1 @@
+import "../src/types.mjs";

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/types/read.d.mts

@@ -0,0 +1 @@
+import "../src/types.mjs";

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/types/snappy.d.mts

@@ -0,0 +1 @@
+export { };

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/types/utils.d.mts

@@ -0,0 +1 @@
+import "../src/types.mjs";

package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/avro/avro.metadata.mjs

@@ -0,0 +1,53 @@
+function readZigZag(reader) {
+	let result = 0;
+	let shift = 0;
+	while (true) {
+		const byte = reader.view.getUint8(reader.offset++);
+		result |= (byte & 127) << shift;
+		if (!(byte & 128)) return result >>> 1 ^ -(result & 1);
+		shift += 7;
+	}
+}
+function readZigZagBigInt(reader) {
+	let result = 0n;
+	let shift = 0n;
+	while (true) {
+		const byte = reader.view.getUint8(reader.offset++);
+		result |= BigInt(byte & 127) << shift;
+		if (!(byte & 128)) return result >> 1n ^ -(result & 1n);
+		shift += 7n;
+	}
+}
+function readAvroString(reader) {
+	const length = readZigZag(reader);
+	const bytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, length);
+	reader.offset += length;
+	return new TextDecoder().decode(bytes);
+}
+function avroMetadata(reader) {
+	if (reader.view.getUint32(reader.offset) !== 1331849729) throw new Error("avro invalid magic bytes");
+	reader.offset += 4;
+	const metadata = {};
+	let mapCount = readZigZag(reader);
+	while (mapCount !== 0) {
+		if (mapCount < 0) {
+			mapCount = -mapCount;
+			readZigZag(reader);
+		}
+		for (let i = 0; i < mapCount; i++) {
+			const key = readAvroString(reader);
+			metadata[key] = readAvroString(reader);
+		}
+		mapCount = readZigZag(reader);
+	}
+	metadata["avro.schema"] = JSON.parse(metadata["avro.schema"]);
+	if (metadata["schema"]) metadata["schema"] = JSON.parse(metadata["schema"]);
+	if (metadata["iceberg.schema"]) metadata["iceberg.schema"] = JSON.parse(metadata["iceberg.schema"]);
+	const syncMarker = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, 16);
+	reader.offset += 16;
+	return {
+		metadata,
+		syncMarker
+	};
+}
+export { avroMetadata, readZigZag, readZigZagBigInt };

package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/avro/avro.read.mjs

@@ -0,0 +1,136 @@
+import { parseDecimal } from "../../../../../hyparquet@1.26.2/node_modules/hyparquet/src/convert.mjs";
+import { gunzip } from "../../../../../hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/gzip.mjs";
+import "../../../../../hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/index.mjs";
+import { readZigZag, readZigZagBigInt } from "./avro.metadata.mjs";
+function avroRead({ reader, metadata, syncMarker }) {
+	const blocks = [];
+	while (reader.offset < reader.view.byteLength) {
+		let recordCount = readZigZag(reader);
+		if (recordCount === 0) break;
+		if (recordCount < 0) recordCount = -recordCount;
+		const blockSize = readZigZag(reader);
+		let data = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, blockSize);
+		reader.offset += blockSize;
+		const blockSync = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, 16);
+		reader.offset += 16;
+		for (let i = 0; i < 16; i++) if (blockSync[i] !== syncMarker[i]) throw new Error("sync marker does not match");
+		const codec = metadata["avro.codec"];
+		if (codec === "deflate") data = gunzip(data);
+		else if (codec !== "null") throw new Error(`unsupported codec: ${codec}`);
+		const { fields } = metadata["avro.schema"];
+		const dataReader = {
+			view: new DataView(data.buffer, data.byteOffset, data.byteLength),
+			offset: 0
+		};
+		for (let i = 0; i < recordCount; i++) {
+			const obj = {};
+			for (const field of fields) {
+				const value = readType(dataReader, field.type);
+				obj[field.name] = value;
+			}
+			blocks.push(obj);
+		}
+	}
+	return blocks;
+}
+function readType(reader, type) {
+	if (type === "null") return;
+	else if (Array.isArray(type)) return readType(reader, type[readZigZag(reader)]);
+	else if (typeof type === "object" && type.type === "record") {
+		const obj = {};
+		for (const subField of type.fields) obj[subField.name] = readType(reader, subField.type);
+		return obj;
+	} else if (typeof type === "object" && type.type === "array") {
+		const arr = [];
+		while (true) {
+			let count = readZigZag(reader);
+			if (count === 0) break;
+			if (count < 0) {
+				count = -count;
+				readZigZag(reader);
+			}
+			for (let i = 0; i < count; i++) arr.push(readType(reader, type.items));
+		}
+		return arr;
+	} else if (typeof type === "object" && type.type === "map") {
+		const map = {};
+		while (true) {
+			let count = readZigZag(reader);
+			if (count === 0) break;
+			if (count < 0) {
+				count = -count;
+				readZigZag(reader);
+			}
+			for (let i = 0; i < count; i++) {
+				const key = readType(reader, "string");
+				map[key] = readType(reader, type.values);
+			}
+		}
+		return map;
+	} else if (typeof type === "object" && type.type === "enum") return type.symbols[readZigZag(reader)];
+	else if (typeof type === "object" && "logicalType" in type && type.logicalType) if (type.logicalType === "date" && type.type === "int") {
+		const value = readZigZag(reader);
+		return /* @__PURE__ */ new Date(value * 864e5);
+	} else if (type.logicalType === "time-millis" && type.type === "int") return readZigZag(reader);
+	else if (type.logicalType === "time-micros" && type.type === "long") return readZigZagBigInt(reader);
+	else if (type.logicalType === "timestamp-millis" && type.type === "long") {
+		const value = readZigZagBigInt(reader);
+		return new Date(Number(value));
+	} else if (type.logicalType === "timestamp-micros" && type.type === "long") {
+		const value = readZigZagBigInt(reader);
+		return new Date(Number(value / 1000n));
+	} else if (type.logicalType === "timestamp-nanos" && type.type === "long") {
+		const value = readZigZagBigInt(reader);
+		return new Date(Number(value / 1000000n));
+	} else if (type.logicalType === "decimal" && "precision" in type) {
+		const bytes = type.type === "fixed" ? readFixed(reader, type.size) : readType(reader, type.type);
+		const factor = 10 ** -(type.scale || 0);
+		return parseDecimal(bytes) * factor;
+	} else if (type.logicalType === "uuid" && type.type === "fixed" && type.size === 16) return bytesToUuid(readFixed(reader, 16));
+	else {
+		console.warn(`unknown logical type: ${type.logicalType}`);
+		return type.type === "fixed" ? readFixed(reader, type.size) : readType(reader, type.type);
+	}
+	else if (typeof type === "object" && type.type === "fixed") return readFixed(reader, type.size);
+	else if (type === "boolean") {
+		const value = reader.view.getUint8(reader.offset) === 1;
+		reader.offset++;
+		return value;
+	} else if (type === "int") return readZigZag(reader);
+	else if (type === "long") return readZigZagBigInt(reader);
+	else if (type === "float") {
+		const value = reader.view.getFloat32(reader.offset, true);
+		reader.offset += 4;
+		return value;
+	} else if (type === "double") {
+		const value = reader.view.getFloat64(reader.offset, true);
+		reader.offset += 8;
+		return value;
+	} else if (type === "bytes") {
+		const length = readZigZag(reader);
+		const bytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, length);
+		reader.offset += length;
+		return bytes;
+	} else if (type === "string") {
+		const length = readZigZag(reader);
+		const bytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, length);
+		const text = new TextDecoder().decode(bytes);
+		reader.offset += length;
+		return text;
+	} else if (typeof type === "object" && typeof type.type === "string") return readType(reader, type.type);
+	else throw new Error(`unsupported type: ${JSON.stringify(type)}`);
+}
+function readFixed(reader, size) {
+	const bytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, size);
+	reader.offset += size;
+	return bytes;
+}
+function bytesToUuid(bytes) {
+	let hex = "";
+	for (let i = 0; i < 16; i++) {
+		hex += bytes[i].toString(16).padStart(2, "0");
+		if (i === 3 || i === 5 || i === 7 || i === 9) hex += "-";
+	}
+	return hex;
+}
+export { avroRead };