@gscdump/engine 1.4.0 → 1.4.2

package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/brotli.streams.mjs

@@ -0,0 +1,21 @@
+function BrotliInput(buffer) {
+	this.buffer = buffer;
+	this.pos = 0;
+}
+BrotliInput.prototype.read = function(buf, i, count) {
+	if (this.pos + count > this.buffer.length) count = this.buffer.length - this.pos;
+	for (let p = 0; p < count; p++) buf[i + p] = this.buffer[this.pos + p];
+	this.pos += count;
+	return count;
+};
+function BrotliOutput(buf) {
+	this.buffer = buf;
+	this.pos = 0;
+}
+BrotliOutput.prototype.write = function(buf, count) {
+	if (this.pos + count > this.buffer.length) throw new Error("brotli output buffer is not large enough");
+	this.buffer.set(buf.subarray(0, count), this.pos);
+	this.pos += count;
+	return count;
+};
+export { BrotliInput, BrotliOutput };

package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/brotli.transform.mjs

@@ -0,0 +1,190 @@
+import { getDictionary } from "./brotli.dictionary.mjs";
+const kIdentity = 0;
+const kOmitLast1 = 1;
+const kOmitLast2 = 2;
+const kOmitLast3 = 3;
+const kOmitLast4 = 4;
+const kOmitLast5 = 5;
+const kOmitLast6 = 6;
+const kOmitLast7 = 7;
+const kOmitLast8 = 8;
+const kOmitLast9 = 9;
+const kUppercaseFirst = 10;
+const kUppercaseAll = 11;
+const kOmitFirst1 = 12;
+const kOmitFirst2 = 13;
+const kOmitFirst3 = 14;
+const kOmitFirst4 = 15;
+const kOmitFirst5 = 16;
+const kOmitFirst6 = 17;
+const kOmitFirst7 = 18;
+const kOmitFirst9 = 20;
+function Transform(prefix, transform, suffix) {
+	this.prefix = new Uint8Array(prefix.length);
+	this.transform = transform;
+	this.suffix = new Uint8Array(suffix.length);
+	for (let i = 0; i < prefix.length; i++) this.prefix[i] = prefix.charCodeAt(i);
+	for (let i = 0; i < suffix.length; i++) this.suffix[i] = suffix.charCodeAt(i);
+}
+const kTransforms = [
+	new Transform("", kIdentity, ""),
+	new Transform("", kIdentity, " "),
+	new Transform(" ", kIdentity, " "),
+	new Transform("", kOmitFirst1, ""),
+	new Transform("", kUppercaseFirst, " "),
+	new Transform("", kIdentity, " the "),
+	new Transform(" ", kIdentity, ""),
+	new Transform("s ", kIdentity, " "),
+	new Transform("", kIdentity, " of "),
+	new Transform("", kUppercaseFirst, ""),
+	new Transform("", kIdentity, " and "),
+	new Transform("", kOmitFirst2, ""),
+	new Transform("", kOmitLast1, ""),
+	new Transform(", ", kIdentity, " "),
+	new Transform("", kIdentity, ", "),
+	new Transform(" ", kUppercaseFirst, " "),
+	new Transform("", kIdentity, " in "),
+	new Transform("", kIdentity, " to "),
+	new Transform("e ", kIdentity, " "),
+	new Transform("", kIdentity, "\""),
+	new Transform("", kIdentity, "."),
+	new Transform("", kIdentity, "\">"),
+	new Transform("", kIdentity, "\n"),
+	new Transform("", kOmitLast3, ""),
+	new Transform("", kIdentity, "]"),
+	new Transform("", kIdentity, " for "),
+	new Transform("", kOmitFirst3, ""),
+	new Transform("", kOmitLast2, ""),
+	new Transform("", kIdentity, " a "),
+	new Transform("", kIdentity, " that "),
+	new Transform(" ", kUppercaseFirst, ""),
+	new Transform("", kIdentity, ". "),
+	new Transform(".", kIdentity, ""),
+	new Transform(" ", kIdentity, ", "),
+	new Transform("", kOmitFirst4, ""),
+	new Transform("", kIdentity, " with "),
+	new Transform("", kIdentity, "'"),
+	new Transform("", kIdentity, " from "),
+	new Transform("", kIdentity, " by "),
+	new Transform("", kOmitFirst5, ""),
+	new Transform("", kOmitFirst6, ""),
+	new Transform(" the ", kIdentity, ""),
+	new Transform("", kOmitLast4, ""),
+	new Transform("", kIdentity, ". The "),
+	new Transform("", kUppercaseAll, ""),
+	new Transform("", kIdentity, " on "),
+	new Transform("", kIdentity, " as "),
+	new Transform("", kIdentity, " is "),
+	new Transform("", kOmitLast7, ""),
+	new Transform("", kOmitLast1, "ing "),
+	new Transform("", kIdentity, "\n	"),
+	new Transform("", kIdentity, ":"),
+	new Transform(" ", kIdentity, ". "),
+	new Transform("", kIdentity, "ed "),
+	new Transform("", kOmitFirst9, ""),
+	new Transform("", kOmitFirst7, ""),
+	new Transform("", kOmitLast6, ""),
+	new Transform("", kIdentity, "("),
+	new Transform("", kUppercaseFirst, ", "),
+	new Transform("", kOmitLast8, ""),
+	new Transform("", kIdentity, " at "),
+	new Transform("", kIdentity, "ly "),
+	new Transform(" the ", kIdentity, " of "),
+	new Transform("", kOmitLast5, ""),
+	new Transform("", kOmitLast9, ""),
+	new Transform(" ", kUppercaseFirst, ", "),
+	new Transform("", kUppercaseFirst, "\""),
+	new Transform(".", kIdentity, "("),
+	new Transform("", kUppercaseAll, " "),
+	new Transform("", kUppercaseFirst, "\">"),
+	new Transform("", kIdentity, "=\""),
+	new Transform(" ", kIdentity, "."),
+	new Transform(".com/", kIdentity, ""),
+	new Transform(" the ", kIdentity, " of the "),
+	new Transform("", kUppercaseFirst, "'"),
+	new Transform("", kIdentity, ". This "),
+	new Transform("", kIdentity, ","),
+	new Transform(".", kIdentity, " "),
+	new Transform("", kUppercaseFirst, "("),
+	new Transform("", kUppercaseFirst, "."),
+	new Transform("", kIdentity, " not "),
+	new Transform(" ", kIdentity, "=\""),
+	new Transform("", kIdentity, "er "),
+	new Transform(" ", kUppercaseAll, " "),
+	new Transform("", kIdentity, "al "),
+	new Transform(" ", kUppercaseAll, ""),
+	new Transform("", kIdentity, "='"),
+	new Transform("", kUppercaseAll, "\""),
+	new Transform("", kUppercaseFirst, ". "),
+	new Transform(" ", kIdentity, "("),
+	new Transform("", kIdentity, "ful "),
+	new Transform(" ", kUppercaseFirst, ". "),
+	new Transform("", kIdentity, "ive "),
+	new Transform("", kIdentity, "less "),
+	new Transform("", kUppercaseAll, "'"),
+	new Transform("", kIdentity, "est "),
+	new Transform(" ", kUppercaseFirst, "."),
+	new Transform("", kUppercaseAll, "\">"),
+	new Transform(" ", kIdentity, "='"),
+	new Transform("", kUppercaseFirst, ","),
+	new Transform("", kIdentity, "ize "),
+	new Transform("", kUppercaseAll, "."),
+	new Transform("Â\xA0", kIdentity, ""),
+	new Transform(" ", kIdentity, ","),
+	new Transform("", kUppercaseFirst, "=\""),
+	new Transform("", kUppercaseAll, "=\""),
+	new Transform("", kIdentity, "ous "),
+	new Transform("", kUppercaseAll, ", "),
+	new Transform("", kUppercaseFirst, "='"),
+	new Transform(" ", kUppercaseFirst, ","),
+	new Transform(" ", kUppercaseAll, "=\""),
+	new Transform(" ", kUppercaseAll, ", "),
+	new Transform("", kUppercaseAll, ","),
+	new Transform("", kUppercaseAll, "("),
+	new Transform("", kUppercaseAll, ". "),
+	new Transform(" ", kUppercaseAll, "."),
+	new Transform("", kUppercaseAll, "='"),
+	new Transform(" ", kUppercaseAll, ". "),
+	new Transform(" ", kUppercaseFirst, "=\""),
+	new Transform(" ", kUppercaseAll, "='"),
+	new Transform(" ", kUppercaseFirst, "='")
+];
+const kNumTransforms = kTransforms.length;
+function ToUpperCase(p, i) {
+	if (p[i] < 192) {
+		if (p[i] >= 97 && p[i] <= 122) p[i] ^= 32;
+		return 1;
+	}
+	if (p[i] < 224) {
+		p[i + 1] ^= 32;
+		return 2;
+	}
+	p[i + 2] ^= 5;
+	return 3;
+}
+function transformDictionaryWord(dst, idx, word, len, transform) {
+	const dictionary = getDictionary();
+	const { prefix } = kTransforms[transform];
+	const { suffix } = kTransforms[transform];
+	const t = kTransforms[transform].transform;
+	let skip = t < kOmitFirst1 ? 0 : t - (kOmitFirst1 - 1);
+	const start_idx = idx;
+	if (skip > len) skip = len;
+	let prefix_pos = 0;
+	while (prefix_pos < prefix.length) dst[idx++] = prefix[prefix_pos++];
+	word += skip;
+	len -= skip;
+	if (t <= kOmitLast9) len -= t;
+	for (let i = 0; i < len; i++) dst[idx++] = dictionary[word + i];
+	let uppercase = idx - len;
+	if (t === kUppercaseFirst) ToUpperCase(dst, uppercase);
+	else if (t === kUppercaseAll) while (len > 0) {
+		const step = ToUpperCase(dst, uppercase);
+		uppercase += step;
+		len -= step;
+	}
+	let suffix_pos = 0;
+	while (suffix_pos < suffix.length) dst[idx++] = suffix[suffix_pos++];
+	return idx - start_idx;
+}
+export { kNumTransforms, kTransforms, transformDictionaryWord };

package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/compressors.mjs

@@ -0,0 +1,7 @@
+import "../../../../fzstd@0.1.1/node_modules/fzstd/esm/index.mjs";
+import { snappyUncompressor } from "../../../../../../vendor/hysnappy-purejs.mjs";
+import "./gzip.mjs";
+import "./brotli.mjs";
+import "./lz4.mjs";
+snappyUncompressor();
+export {};

package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/gzip.huffman.mjs

@@ -0,0 +1,121 @@
+const HUFFMAN_MAX_TABLE_SIZE = 1080;
+const fixedLengthExtraBits = new Uint8Array([
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	1,
+	1,
+	1,
+	1,
+	2,
+	2,
+	2,
+	2,
+	3,
+	3,
+	3,
+	3,
+	4,
+	4,
+	4,
+	4,
+	5,
+	5,
+	5,
+	5,
+	0,
+	0,
+	0,
+	0
+]);
+const fixedDistanceExtraBits = new Uint8Array([
+	0,
+	0,
+	0,
+	0,
+	1,
+	1,
+	2,
+	2,
+	3,
+	3,
+	4,
+	4,
+	5,
+	5,
+	6,
+	6,
+	7,
+	7,
+	8,
+	8,
+	9,
+	9,
+	10,
+	10,
+	11,
+	11,
+	12,
+	12,
+	13,
+	13,
+	0,
+	0
+]);
+function freb(eb, start) {
+	const base = /* @__PURE__ */ new Uint16Array(31);
+	for (let i = 0; i < 31; i++) base[i] = start += 1 << eb[i - 1];
+	const rev = new Int32Array(base[30]);
+	for (let i = 1; i < 30; i++) for (let j = base[i]; j < base[i + 1]; ++j) rev[j] = j - base[i] << 5 | i;
+	return {
+		base,
+		rev
+	};
+}
+const { base: fixedLength, rev: revfl } = freb(fixedLengthExtraBits, 2);
+fixedLength[28] = 258;
+revfl[258] = 28;
+const { base: fixedDistance } = freb(fixedDistanceExtraBits, 0);
+const rev = /* @__PURE__ */ new Uint16Array(32768);
+for (let i = 0; i < 32768; i++) {
+	let x = (i & 43690) >> 1 | (i & 21845) << 1;
+	x = (x & 52428) >> 2 | (x & 13107) << 2;
+	x = (x & 61680) >> 4 | (x & 3855) << 4;
+	rev[i] = ((x & 65280) >> 8 | (x & 255) << 8) >> 1;
+}
+function huffMap(cd, maxBits, r) {
+	const l = new Uint16Array(maxBits);
+	for (let i = 0; i < cd.length; i++) if (cd[i]) ++l[cd[i] - 1];
+	const le = new Uint16Array(maxBits);
+	for (let i = 1; i < maxBits; i++) le[i] = le[i - 1] + l[i - 1] << 1;
+	let co;
+	if (r) {
+		co = new Uint16Array(1 << maxBits);
+		const rvb = 15 - maxBits;
+		for (let i = 0; i < cd.length; i++) if (cd[i]) {
+			const sv = i << 4 | cd[i];
+			const freeBits = maxBits - cd[i];
+			let startValue = le[cd[i] - 1]++ << freeBits;
+			for (const endValue = startValue | (1 << freeBits) - 1; startValue <= endValue; startValue++) co[rev[startValue] >> rvb] = sv;
+		}
+	} else {
+		co = new Uint16Array(cd.length);
+		for (let i = 0; i < cd.length; i++) if (cd[i]) co[i] = rev[le[cd[i] - 1]++] >> 15 - cd[i];
+	}
+	return co;
+}
+const fixedLengthTree = /* @__PURE__ */ new Uint8Array(288);
+for (let i = 0; i < 144; i++) fixedLengthTree[i] = 8;
+for (let i = 144; i < 256; i++) fixedLengthTree[i] = 9;
+for (let i = 256; i < 280; i++) fixedLengthTree[i] = 7;
+for (let i = 280; i < 288; i++) fixedLengthTree[i] = 8;
+const fixedDistanceTree = /* @__PURE__ */ new Uint8Array(32);
+for (let i = 0; i < 32; i++) fixedDistanceTree[i] = 5;
+const fixedLengthMap = /*#__PURE__*/ huffMap(fixedLengthTree, 9, 1);
+const fixedDistanceMap = /*#__PURE__*/ huffMap(fixedDistanceTree, 5, 1);
+export { HUFFMAN_MAX_TABLE_SIZE, fixedDistance, fixedDistanceExtraBits, fixedDistanceMap, fixedLength, fixedLengthExtraBits, fixedLengthMap, huffMap };

package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/gzip.mjs

@@ -0,0 +1,176 @@
+import { fixedDistance, fixedDistanceExtraBits, fixedDistanceMap, fixedLength, fixedLengthExtraBits, fixedLengthMap, huffMap } from "./gzip.huffman.mjs";
+const codeLengthIndexMap = new Uint8Array([
+	16,
+	17,
+	18,
+	0,
+	8,
+	7,
+	9,
+	6,
+	10,
+	5,
+	11,
+	4,
+	12,
+	3,
+	13,
+	2,
+	14,
+	1,
+	15
+]);
+function bits(input, pos, mask) {
+	const o = pos / 8 | 0;
+	return (input[o] | input[o + 1] << 8) >> (pos & 7) & mask;
+}
+function bits16(d, p) {
+	const o = p / 8 | 0;
+	return (d[o] | d[o + 1] << 8 | d[o + 2] << 16) >> (p & 7);
+}
+function shft(p) {
+	return (p + 7) / 8 | 0;
+}
+function gzipStart(input, i) {
+	if (input[i++] !== 31 || input[i++] !== 139 || input[i++] !== 8) return 0;
+	const flag = input[i++];
+	i += 6;
+	if (flag & 4) i += (input[i + 10] | input[i + 11] << 8) + 2;
+	for (let zs = (flag >> 3 & 1) + (flag >> 4 & 1); zs > 0; zs -= Number(!input[i++]));
+	return i + (flag & 2);
+}
+function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
+	let out = output ?? /* @__PURE__ */ new Uint8Array(1024);
+	if (!(input.length - inputIndex)) return out;
+	const payloadStart = gzipStart(input, inputIndex);
+	if (payloadStart === input.length - 8) return out;
+	if (payloadStart > input.length - 8) throw new Error("unexpected EOF");
+	let pos = payloadStart * 8;
+	let final = 0;
+	let lengthBits = 0;
+	let distBits = 0;
+	let lengthMap;
+	let distMap;
+	const totalBits = input.length * 8;
+	function ensureSize(length) {
+		if (!output && length > out.length) {
+			const old = out;
+			out = new Uint8Array(Math.max(old.length * 2, length));
+			out.set(old);
+		}
+	}
+	do {
+		if (!lengthMap) {
+			final = bits(input, pos, 1);
+			const type = bits(input, pos + 1, 3);
+			pos += 3;
+			if (!type) {
+				const s = shft(pos) + 4;
+				const l = input[s - 4] | input[s - 3] << 8;
+				const t = s + l;
+				if (t > input.length) throw new Error("unexpected EOF");
+				ensureSize(outputIndex + l);
+				out.set(input.subarray(s, t), outputIndex);
+				outputIndex += l;
+				pos = t * 8;
+				continue;
+			} else if (type === 1) {
+				lengthMap = fixedLengthMap;
+				distMap = fixedDistanceMap;
+				lengthBits = 9;
+				distBits = 5;
+			} else if (type === 2) {
+				const hLiteral = bits(input, pos, 31) + 257;
+				const hcLengths = bits(input, pos + 10, 15) + 4;
+				const tl = hLiteral + bits(input, pos + 5, 31) + 1;
+				pos += 14;
+				const lengthDistanceTree = new Uint8Array(tl);
+				const codeLengthTree = /* @__PURE__ */ new Uint8Array(19);
+				for (let i = 0; i < hcLengths; ++i) codeLengthTree[codeLengthIndexMap[i]] = bits(input, pos + i * 3, 7);
+				pos += hcLengths * 3;
+				const codeLengthBits = Math.max(...codeLengthTree);
+				const clbMask = (1 << codeLengthBits) - 1;
+				const codeLengthMap = huffMap(codeLengthTree, codeLengthBits, 1);
+				for (let i = 0; i < tl;) {
+					const r = codeLengthMap[bits(input, pos, clbMask)];
+					pos += r & 15;
+					const symbol = r >> 4;
+					if (symbol < 16) lengthDistanceTree[i++] = symbol;
+					else {
+						let copy = 0;
+						let n = 0;
+						if (symbol === 16) {
+							n = 3 + bits(input, pos, 3);
+							pos += 2;
+							copy = lengthDistanceTree[i - 1];
+						} else if (symbol === 17) {
+							n = 3 + bits(input, pos, 7);
+							pos += 3;
+						} else if (symbol === 18) {
+							n = 11 + bits(input, pos, 127);
+							pos += 7;
+						}
+						while (n--) lengthDistanceTree[i++] = copy;
+					}
+				}
+				const lengthTree = lengthDistanceTree.subarray(0, hLiteral);
+				const distanceTree = lengthDistanceTree.subarray(hLiteral);
+				lengthBits = Math.max(...lengthTree);
+				distBits = Math.max(...distanceTree);
+				lengthMap = huffMap(lengthTree, lengthBits, 1);
+				distMap = huffMap(distanceTree, distBits, 1);
+			} else throw new Error("invalid block type");
+			if (pos > totalBits) throw new Error("unexpected EOF");
+		}
+		ensureSize(outputIndex + 131072);
+		const lms = (1 << lengthBits) - 1;
+		const dms = (1 << distBits) - 1;
+		let lpos = pos;
+		for (;; lpos = pos) {
+			const code = lengthMap[bits16(input, pos) & lms];
+			const sym = code >> 4;
+			pos += code & 15;
+			if (pos > totalBits) throw new Error("unexpected EOF");
+			if (!code) throw new Error("invalid length/literal");
+			if (sym < 256) out[outputIndex++] = sym;
+			else if (sym === 256) {
+				lpos = pos;
+				lengthMap = void 0;
+				break;
+			} else {
+				let add = sym - 254;
+				if (sym > 264) {
+					const index = sym - 257;
+					const b = fixedLengthExtraBits[index];
+					add = bits(input, pos, (1 << b) - 1) + fixedLength[index];
+					pos += b;
+				}
+				if (!distMap) throw new Error("invalid distance map");
+				const d = distMap[bits16(input, pos) & dms];
+				const dsym = d >> 4;
+				if (!d) throw new Error("invalid distance");
+				pos += d & 15;
+				let dt = fixedDistance[dsym];
+				if (dsym > 3) {
+					const b = fixedDistanceExtraBits[dsym];
+					dt += bits16(input, pos) & (1 << b) - 1;
+					pos += b;
+				}
+				if (pos > totalBits) throw new Error("unexpected EOF");
+				const end = outputIndex + add;
+				if (outputIndex < dt) throw new Error("unexpected dictionary case");
+				ensureSize(end);
+				for (; outputIndex < end; outputIndex++) out[outputIndex] = out[outputIndex - dt];
+			}
+		}
+		pos = lpos;
+		if (lengthMap) final = 1;
+	} while (!final);
+	if (outputIndex < out.length) {
+		const nextBlock = Math.ceil(pos / 8) + 8;
+		gunzip(input, out, nextBlock, outputIndex);
+	}
+	if (!output) return out.subarray(0, outputIndex);
+	return out;
+}
+export { gunzip };

package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/index.mjs

@@ -0,0 +1,7 @@
+import "../../../../fzstd@0.1.1/node_modules/fzstd/esm/index.mjs";
+import "../../../../../../vendor/hysnappy-purejs.mjs";
+import "./gzip.mjs";
+import "./brotli.mjs";
+import "./lz4.mjs";
+import "./compressors.mjs";
+export {};

package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/lz4.mjs

@@ -0,0 +1,52 @@
+function decompressLz4(input, outputLength) {
+	const output = new Uint8Array(outputLength);
+	try {
+		let i = 0;
+		let o = 0;
+		while (i < input.length - 8) {
+			const expectedOutputLength = input[i++] << 24 | input[i++] << 16 | input[i++] << 8 | input[i++];
+			const expectedInputLength = input[i++] << 24 | input[i++] << 16 | input[i++] << 8 | input[i++];
+			if (input.length - i < expectedInputLength) throw new Error("lz4 not hadoop");
+			if (output.length < expectedOutputLength) throw new Error("lz4 not hadoop");
+			if (lz4basic(input.subarray(i, i + expectedInputLength), output, o) !== expectedOutputLength) throw new Error("lz4 not hadoop");
+			i += expectedInputLength;
+			o += expectedOutputLength;
+			if (i === input.length) return output;
+		}
+		if (i < input.length) throw new Error("lz4 not hadoop");
+	} catch (error) {
+		if (error instanceof Error && error.message !== "lz4 not hadoop") throw error;
+		lz4basic(input, output, 0);
+	}
+	return output;
+}
+function decompressLz4Raw(input, outputLength) {
+	const output = new Uint8Array(outputLength);
+	lz4basic(input, output, 0);
+	return output;
+}
+function lz4basic(input, output, outputIndex) {
+	let len = outputIndex;
+	for (let i = 0; i < input.length;) {
+		const token = input[i++];
+		let literals = token >> 4;
+		if (literals) {
+			let byte = literals + 240;
+			while (byte === 255) literals += byte = input[i++];
+			output.set(input.subarray(i, i + literals), len);
+			len += literals;
+			i += literals;
+			if (i >= input.length) return len - outputIndex;
+		}
+		const offset = input[i++] | input[i++] << 8;
+		if (!offset || offset > len) throw new Error(`lz4 offset out of range ${offset}`);
+		let matchLength = (token & 15) + 4;
+		let byte = matchLength + 240;
+		while (byte === 255) matchLength += byte = input[i++];
+		let pos = len - offset;
+		const end = len + matchLength;
+		while (len < end) output[len++] = output[pos++];
+	}
+	return len - outputIndex;
+}
+export { decompressLz4, decompressLz4Raw };

package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/bloom.mjs

@@ -0,0 +1,93 @@
+import { hashParquetValue } from "../../../../hyparquet@1.26.1/node_modules/hyparquet/src/bloom.mjs";
+import { serializeTCompactProtocol } from "./thrift.mjs";
+const SALT = new Uint32Array([
+	1203114875,
+	1150766481,
+	2284105051,
+	2729912477,
+	1884591559,
+	770785867,
+	2667333959,
+	1550580529
+]);
+const BYTES_PER_BLOCK = 32;
+const MIN_BYTES = 32;
+const MAX_BYTES = 128 * 1024 * 1024;
+function blockIndex(hash, numBlocks) {
+	return Number((hash >> 32n) * BigInt(numBlocks) >> 32n);
+}
+function blockMask(hash) {
+	const m = /* @__PURE__ */ new Uint32Array(8);
+	const low = Number(hash & 4294967295n) | 0;
+	for (let i = 0; i < 8; i++) m[i] = 1 << (Math.imul(low, SALT[i]) >>> 27);
+	return m;
+}
+function sbbfInsert(blocks, hash) {
+	const offset = blockIndex(hash, blocks.length >> 3) << 3;
+	const m = blockMask(hash);
+	for (let i = 0; i < 8; i++) blocks[offset + i] |= m[i];
+}
+function nextPowerOfTwo(n) {
+	let p = 1;
+	while (p < n) p <<= 1;
+	return p;
+}
+function optimalNumBytes(ndv, fpp) {
+	if (!(fpp > 0 && fpp < 1)) throw new Error(`bloom filter fpp must be in (0, 1), got ${fpp}`);
+	if (!(ndv >= 0)) throw new Error(`bloom filter ndv must be >= 0, got ${ndv}`);
+	const m = -8 * ndv / Math.log(1 - fpp ** (1 / 8));
+	let numBits = Math.ceil(m);
+	if (!isFinite(numBits) || numBits > MAX_BYTES << 3) numBits = MAX_BYTES << 3;
+	const blockBits = BYTES_PER_BLOCK << 3;
+	numBits = Math.ceil(numBits / blockBits) * blockBits;
+	let numBytes = numBits >> 3;
+	if (numBytes < MIN_BYTES) numBytes = MIN_BYTES;
+	if (numBytes < 1024) numBytes = nextPowerOfTwo(numBytes);
+	return numBytes;
+}
+var BloomBuilder = class {
+	constructor(element, { fpp = .01, maxBytes = 1024 * 1024 } = {}) {
+		this.element = element;
+		this.fpp = fpp;
+		this.maxBytes = maxBytes;
+		this.hashes = /* @__PURE__ */ new Set();
+		this.skipped = 0;
+	}
+	insert(value) {
+		if (value === null || value === void 0) return;
+		const h = hashParquetValue(value, this.element);
+		if (h === void 0) {
+			this.skipped++;
+			return;
+		}
+		this.hashes.add(h);
+	}
+	finalize() {
+		if (this.skipped > 0 || this.hashes.size === 0) return void 0;
+		const numBytes = optimalNumBytes(this.hashes.size, this.fpp);
+		if (numBytes > this.maxBytes) return void 0;
+		const blocks = new Uint32Array(numBytes >> 2);
+		for (const h of this.hashes) sbbfInsert(blocks, h);
+		return blocks;
+	}
+};
+function writeBloomFilter(writer, blocks) {
+	if (blocks.length % 8 !== 0) throw new Error(`bloom filter block count must be a multiple of 8 uint32 words, got ${blocks.length}`);
+	serializeTCompactProtocol(writer, {
+		field_1: blocks.byteLength,
+		field_2: { field_1: {} },
+		field_3: { field_1: {} },
+		field_4: { field_1: {} }
+	});
+	for (let i = 0; i < blocks.length; i++) writer.appendUint32(blocks[i]);
+}
+function writeBlooms(writer, pageIndexes) {
+	for (const { chunk, bloomFilter } of pageIndexes) {
+		if (!bloomFilter || !chunk.meta_data) continue;
+		const offset = writer.offset;
+		writeBloomFilter(writer, bloomFilter);
+		chunk.meta_data.bloom_filter_offset = BigInt(offset);
+		chunk.meta_data.bloom_filter_length = writer.offset - offset;
+	}
+}
+export { BloomBuilder, optimalNumBytes, sbbfInsert, writeBloomFilter, writeBlooms };