@gscdump/lakehouse 3.8.0 → 4.0.0

package/dist/THIRD-PARTY-LICENSES.md

@@ -304,11 +304,11 @@ Repository: https://github.com/101arrowz/fzstd
 
 ---------------------------------------
 
-## hyparquet, hyparquet, hyparquet-compressors, hyparquet-writer, icebird, squirreling
+## hyparquet, hyparquet-compressors, hyparquet-writer, icebird, squirreling
 
 License: MIT
 By: Hyperparam
-Repositories: https://github.com/hyparam/hyparquet, https://github.com/hyparam/hyparquet, https://github.com/hyparam/hyparquet-compressors, https://github.com/hyparam/hyparquet-writer, https://github.com/hyparam/icebird, https://github.com/hyparam/squirreling
+Repositories: https://github.com/hyparam/hyparquet, https://github.com/hyparam/hyparquet-compressors, https://github.com/hyparam/hyparquet-writer, https://github.com/hyparam/icebird, https://github.com/hyparam/squirreling
 
 > The MIT License (MIT)
 > 

package/dist/_virtual/node_modules/.pnpm/hyparquet-compressors@1.1.2/node_modules/hyparquet-compressors/src/brotli.bitreader.mjs

@@ -0,0 +1,93 @@
+const BROTLI_READ_SIZE = 4096;
+const BROTLI_IBUF_SIZE = 8224;
+const BROTLI_IBUF_MASK = 2 * BROTLI_READ_SIZE - 1;
+const kBitMask = new Uint32Array([
+	0,
+	1,
+	3,
+	7,
+	15,
+	31,
+	63,
+	127,
+	255,
+	511,
+	1023,
+	2047,
+	4095,
+	8191,
+	16383,
+	32767,
+	65535,
+	131071,
+	262143,
+	524287,
+	1048575,
+	2097151,
+	4194303,
+	8388607,
+	16777215
+]);
+var BrotliBitReader = class {
+	constructor(input) {
+		this.buf_ = new Uint8Array(BROTLI_IBUF_SIZE);
+		this.input_ = input;
+		this.buf_ptr_ = 0;
+		this.val_ = 0;
+		this.pos_ = 0;
+		this.bit_pos_ = 0;
+		this.bit_end_pos_ = 0;
+		this.eos_ = 0;
+		this.reset();
+	}
+	static READ_SIZE = BROTLI_READ_SIZE;
+	static IBUF_MASK = BROTLI_IBUF_MASK;
+	reset() {
+		this.buf_ptr_ = 0;
+		this.val_ = 0;
+		this.pos_ = 0;
+		this.bit_pos_ = 0;
+		this.bit_end_pos_ = 0;
+		this.eos_ = 0;
+		this.readMoreInput();
+		for (let i = 0; i < 4; i++) {
+			this.val_ |= this.buf_[this.pos_] << 8 * i;
+			this.pos_++;
+		}
+		return this.bit_end_pos_ > 0;
+	}
+	readMoreInput() {
+		if (this.bit_end_pos_ > 256) {} else if (this.eos_) {
+			if (this.bit_pos_ > this.bit_end_pos_) throw new Error("Unexpected end of input " + this.bit_pos_ + " " + this.bit_end_pos_);
+		} else {
+			const dst = this.buf_ptr_;
+			const bytes_read = this.input_.read(this.buf_, dst, BROTLI_READ_SIZE);
+			if (bytes_read < 0) throw new Error("Unexpected end of input");
+			if (bytes_read < BROTLI_READ_SIZE) {
+				this.eos_ = 1;
+				for (let p = 0; p < 32; p++) this.buf_[dst + bytes_read + p] = 0;
+			}
+			if (dst === 0) {
+				for (let p = 0; p < 32; p++) this.buf_[8192 + p] = this.buf_[p];
+				this.buf_ptr_ = BROTLI_READ_SIZE;
+			} else this.buf_ptr_ = 0;
+			this.bit_end_pos_ += bytes_read << 3;
+		}
+	}
+	fillBitWindow() {
+		while (this.bit_pos_ >= 8) {
+			this.val_ >>>= 8;
+			this.val_ |= this.buf_[this.pos_ & BROTLI_IBUF_MASK] << 24;
+			this.pos_++;
+			this.bit_pos_ = this.bit_pos_ - 8 >>> 0;
+			this.bit_end_pos_ = this.bit_end_pos_ - 8 >>> 0;
+		}
+	}
+	readBits(n_bits) {
+		if (32 - this.bit_pos_ < n_bits) this.fillBitWindow();
+		const val = this.val_ >>> this.bit_pos_ & kBitMask[n_bits];
+		this.bit_pos_ += n_bits;
+		return val;
+	}
+};
+export { BrotliBitReader };

package/dist/_virtual/node_modules/.pnpm/{hyparquet-compressors@1.1.1 → hyparquet-compressors@1.1.2}/node_modules/hyparquet-compressors/src/brotli.huffman.mjs

@@ -47,10 +47,12 @@ const kMaxHuffmanTableSize = new Uint16Array([
 	1048,
 	1080
 ]);
-function HuffmanCode(bits, value) {
-	this.bits = bits;
-	this.value = value;
-}
+var HuffmanCode = class {
+	constructor(bits, value) {
+		this.bits = bits;
+		this.value = value;
+	}
+};
 const kCodeLengthRepeatCode = 16;
 const MAX_LENGTH = 15;
 function getNextKey(key, len) {
@@ -249,17 +251,19 @@ function readHuffmanCodeLengths(code_length_code_lengths, num_symbols, code_leng
 	if (space !== 0) throw new Error("[ReadHuffmanCodeLengths] space = " + space);
 	for (; symbol < num_symbols; symbol++) code_lengths[symbol] = 0;
 }
-function HuffmanTreeGroup(alphabet_size, num_htrees) {
-	this.alphabet_size = alphabet_size;
-	this.num_htrees = num_htrees;
-	this.codes = new Array(num_htrees + num_htrees * kMaxHuffmanTableSize[alphabet_size + 31 >>> 5]);
-	this.htrees = new Uint32Array(num_htrees);
-}
-HuffmanTreeGroup.prototype.decode = function(br) {
-	let next = 0;
-	for (let i = 0; i < this.num_htrees; i++) {
-		this.htrees[i] = next;
-		next += readHuffmanCode(this.alphabet_size, this.codes, next, br);
+var HuffmanTreeGroup = class {
+	constructor(alphabet_size, num_htrees) {
+		this.alphabet_size = alphabet_size;
+		this.num_htrees = num_htrees;
+		this.codes = new Array(num_htrees + num_htrees * kMaxHuffmanTableSize[alphabet_size + 31 >>> 5]);
+		this.htrees = new Uint32Array(num_htrees);
+	}
+	decode(br) {
+		let next = 0;
+		for (let i = 0; i < this.num_htrees; i++) {
+			this.htrees[i] = next;
+			next += readHuffmanCode(this.alphabet_size, this.codes, next, br);
+		}
 	}
 };
 export { HuffmanCode, HuffmanTreeGroup, readHuffmanCode, readSymbol };

package/dist/_virtual/node_modules/.pnpm/{hyparquet-compressors@1.1.1 → hyparquet-compressors@1.1.2}/node_modules/hyparquet-compressors/src/brotli.mjs

@@ -113,7 +113,6 @@ function decompressBrotli(input, outputLength) {
 function brotli(input, output) {
 	let pos = 0;
 	let input_end = 0;
-	let window_bits = 0;
 	let max_distance = 0;
 	const dist_rb = [
 		16,
@@ -122,8 +121,6 @@ function brotli(input, output) {
 		4
 	];
 	let dist_rb_idx = 0;
-	let prev_byte1 = 0;
-	let prev_byte2 = 0;
 	const hgroup = [
 		new HuffmanTreeGroup(0, 0),
 		new HuffmanTreeGroup(0, 0),
@@ -131,7 +128,7 @@ function brotli(input, output) {
 	];
 	const kRingBufferWriteAheadSlack = 128 + BrotliBitReader.READ_SIZE;
 	const br = new BrotliBitReader(input);
-	window_bits = decodeWindowBits(br);
+	const window_bits = decodeWindowBits(br);
 	const max_backward_distance = (1 << window_bits) - 16;
 	const ringbuffer_size = 1 << window_bits;
 	const ringbuffer_mask = ringbuffer_size - 1;
@@ -144,7 +141,6 @@ function brotli(input, output) {
 		block_len_trees[x] = new HuffmanCode(0, 0);
 	}
 	while (!input_end) {
-		let meta_block_remaining_len = 0;
 		const block_length = [
 			1 << 28,
 			1 << 28,
@@ -165,14 +161,13 @@ function brotli(input, output) {
 			1
 		];
 		const block_type_rb_index = [0];
-		let context_offset = 0;
 		for (let i = 0; i < 3; i++) {
 			hgroup[i].codes = [];
 			hgroup[i].htrees = /* @__PURE__ */ new Uint32Array();
 		}
 		br.readMoreInput();
 		const _out = decodeMetaBlockLength(br);
-		meta_block_remaining_len = _out.meta_block_length;
+		let meta_block_remaining_len = _out.meta_block_length;
 		if (pos + meta_block_remaining_len > output.buffer.length) {
 			const tmp = new Uint8Array(pos + meta_block_remaining_len);
 			tmp.set(output.buffer);
@@ -246,15 +241,14 @@ function brotli(input, output) {
 			const copyIndex = kCopyRangeLut[range_idx] + (cmd_code & 7);
 			const copyCode = kCopyLengthPrefixCode[copyIndex];
 			const copyLength = copyCode.offset + br.readBits(copyCode.nbits);
-			prev_byte1 = ringbuffer[pos - 1 & ringbuffer_mask];
-			prev_byte2 = ringbuffer[pos - 2 & ringbuffer_mask];
+			let prev_byte1 = ringbuffer[pos - 1 & ringbuffer_mask];
+			let prev_byte2 = ringbuffer[pos - 2 & ringbuffer_mask];
 			for (let j = 0; j < insertLength; j++) {
 				br.readMoreInput();
 				if (block_length[0] === 0) {
 					decodeBlockType(num_block_types[0], block_type_trees, 0, block_type, block_type_rb, block_type_rb_index, br);
 					block_length[0] = readBlockLength(block_len_trees, 0, br);
-					context_offset = block_type[0] << kLiteralContextBits;
-					context_map_slice = context_offset;
+					context_map_slice = block_type[0] << kLiteralContextBits;
 					context_mode = context_modes[block_type[0]];
 					context_lookup_offset1 = lookupOffsets[context_mode];
 					context_lookup_offset2 = lookupOffsets[context_mode + 1];
@@ -326,8 +320,6 @@ function brotli(input, output) {
 					meta_block_remaining_len--;
 				}
 			}
-			prev_byte1 = ringbuffer[pos - 1 & ringbuffer_mask];
-			prev_byte2 = ringbuffer[pos - 2 & ringbuffer_mask];
 		}
 		pos &= 1073741823;
 	}

package/dist/_virtual/node_modules/.pnpm/hyparquet-compressors@1.1.2/node_modules/hyparquet-compressors/src/brotli.streams.mjs

@@ -0,0 +1,25 @@
+var BrotliInput = class {
+	constructor(buffer) {
+		this.buffer = buffer;
+		this.pos = 0;
+	}
+	read(buf, i, count) {
+		if (this.pos + count > this.buffer.length) count = this.buffer.length - this.pos;
+		for (let p = 0; p < count; p++) buf[i + p] = this.buffer[this.pos + p];
+		this.pos += count;
+		return count;
+	}
+};
+var BrotliOutput = class {
+	constructor(buf) {
+		this.buffer = buf;
+		this.pos = 0;
+	}
+	write(buf, count) {
+		if (this.pos + count > this.buffer.length) throw new Error("brotli output buffer is not large enough");
+		this.buffer.set(buf.subarray(0, count), this.pos);
+		this.pos += count;
+		return count;
+	}
+};
+export { BrotliInput, BrotliOutput };

package/dist/_virtual/node_modules/.pnpm/{hyparquet-compressors@1.1.1 → hyparquet-compressors@1.1.2}/node_modules/hyparquet-compressors/src/brotli.transform.mjs

@@ -19,13 +19,15 @@ const kOmitFirst5 = 16;
 const kOmitFirst6 = 17;
 const kOmitFirst7 = 18;
 const kOmitFirst9 = 20;
-function Transform(prefix, transform, suffix) {
-	this.prefix = new Uint8Array(prefix.length);
-	this.transform = transform;
-	this.suffix = new Uint8Array(suffix.length);
-	for (let i = 0; i < prefix.length; i++) this.prefix[i] = prefix.charCodeAt(i);
-	for (let i = 0; i < suffix.length; i++) this.suffix[i] = suffix.charCodeAt(i);
-}
+var Transform = class {
+	constructor(prefix, transform, suffix) {
+		this.prefix = new Uint8Array(prefix.length);
+		this.transform = transform;
+		this.suffix = new Uint8Array(suffix.length);
+		for (let i = 0; i < prefix.length; i++) this.prefix[i] = prefix.charCodeAt(i);
+		for (let i = 0; i < suffix.length; i++) this.suffix[i] = suffix.charCodeAt(i);
+	}
+};
 const kTransforms = [
 	new Transform("", kIdentity, ""),
 	new Transform("", kIdentity, " "),

package/dist/_virtual/node_modules/.pnpm/{hyparquet-compressors@1.1.1 → hyparquet-compressors@1.1.2}/node_modules/hyparquet-compressors/src/gzip.mjs

@@ -125,8 +125,7 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 		ensureSize(outputIndex + 131072);
 		const lms = (1 << lengthBits) - 1;
 		const dms = (1 << distBits) - 1;
-		let lpos = pos;
-		for (;; lpos = pos) {
+		for (;;) {
 			const code = lengthMap[bits16(input, pos) & lms];
 			const sym = code >> 4;
 			pos += code & 15;
@@ -134,7 +133,6 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 			if (!code) throw new Error("invalid length/literal");
 			if (sym < 256) out[outputIndex++] = sym;
 			else if (sym === 256) {
-				lpos = pos;
 				lengthMap = void 0;
 				break;
 			} else {
@@ -163,7 +161,6 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 				for (; outputIndex < end; outputIndex++) out[outputIndex] = out[outputIndex - dt];
 			}
 		}
-		pos = lpos;
 		if (lengthMap) final = 1;
 	} while (!final);
 	if (outputIndex < out.length) {

package/dist/_virtual/node_modules/.pnpm/{hyparquet-compressors@1.1.1 → hyparquet-compressors@1.1.2}/node_modules/hyparquet-compressors/src/lz4.mjs

@@ -40,9 +40,10 @@ function lz4basic(input, output, outputIndex) {
 		}
 		const offset = input[i++] | input[i++] << 8;
 		if (!offset || offset > len) throw new Error(`lz4 offset out of range ${offset}`);
-		let matchLength = (token & 15) + 4;
+		let matchLength = token & 15;
 		let byte = matchLength + 240;
 		while (byte === 255) matchLength += byte = input[i++];
+		matchLength += 4;
 		let pos = len - offset;
 		const end = len + matchLength;
 		while (len < end) output[len++] = output[pos++];

package/dist/_virtual/node_modules/.pnpm/{hyparquet-writer@0.16.9_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f → hyparquet-writer@0.16.10_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f}/node_modules/hyparquet-writer/src/bloom.mjs

@@ -1,4 +1,4 @@
-import { hashParquetValue } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/bloom.mjs";
+import { hashParquetValue } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/bloom.mjs";
 import { serializeTCompactProtocol } from "./thrift.mjs";
 const SALT = new Uint32Array([
 	1203114875,

package/dist/_virtual/node_modules/.pnpm/{hyparquet-writer@0.16.9_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f → hyparquet-writer@0.16.10_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f}/node_modules/hyparquet-writer/src/column.mjs

@@ -39,7 +39,7 @@ function writeColumn({ writer, column, pageData }) {
 		encoding = userEncoding && userEncoding !== "RLE_DICTIONARY" ? userEncoding : type === "BOOLEAN" && values.length > 16 ? "RLE" : "PLAIN";
 	}
 	encodings.push(encoding);
-	const pageBoundaries = getPageBoundaries(writeValues, writeType, type_length, pageSize);
+	const pageBoundaries = getPageBoundaries(writeValues, writeType, type_length, pageSize, repetitionLevels);
 	const columnIndex = column.columnIndex && pageBoundaries.length > 1 ? {
 		null_pages: [],
 		min_values: [],
@@ -141,7 +141,7 @@ function writeColumn({ writer, column, pageData }) {
 		bloomFilter
 	};
 }
-function getPageBoundaries(values, type, type_length, pageSize) {
+function getPageBoundaries(values, type, type_length, pageSize, repetitionLevels = []) {
 	if (!pageSize) return [{
 		start: 0,
 		end: values.length
@@ -152,7 +152,8 @@ function getPageBoundaries(values, type, type_length, pageSize) {
 	for (let i = 0; i < values.length; i++) {
 		const valueSize = estimateValueSize(values[i], type, type_length);
 		accumulatedBytes += valueSize;
-		if (accumulatedBytes >= pageSize && i > start) {
+		const rowStart = !repetitionLevels.length || repetitionLevels[i] === 0;
+		if (accumulatedBytes >= pageSize && i > start && rowStart) {
 			boundaries.push({
 				start,
 				end: i
@@ -200,4 +201,4 @@ function compareValues(left, right) {
 	if (left > right) return 1;
 	return 0;
 }
-export { writeColumn };
+export { getPageBoundaries, writeColumn };

package/dist/_virtual/node_modules/.pnpm/{hyparquet-writer@0.16.9_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f → hyparquet-writer@0.16.10_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f}/node_modules/hyparquet-writer/src/datapage.mjs

@@ -1,4 +1,4 @@
-import { Encodings, PageTypes } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/constants.mjs";
+import { Encodings, PageTypes } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/constants.mjs";
 import { ByteWriter } from "./bytewriter.mjs";
 import { serializeTCompactProtocol } from "./thrift.mjs";
 import { deltaBinaryPack, deltaByteArray, deltaLengthByteArray } from "./delta.mjs";

package/dist/_virtual/node_modules/.pnpm/{hyparquet-writer@0.16.9_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f → hyparquet-writer@0.16.10_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f}/node_modules/hyparquet-writer/src/delta.mjs

@@ -17,7 +17,7 @@ function usesInt32NumberPath(values) {
 	if (values instanceof Int32Array) return true;
 	for (let i = 0; i < values.length; i++) {
 		const value = values[i];
-		if (typeof value !== "number" || !Number.isInteger(value) || value < -2147483648 || value > 2147483647) return false;
+		if (typeof value !== "number" || value !== (value | 0)) return false;
 	}
 	return true;
 }
@@ -26,7 +26,7 @@ function deltaBinaryPackInt32(writer, values) {
 	writer.appendVarInt(BLOCK_SIZE);
 	writer.appendVarInt(MINIBLOCKS_PER_BLOCK);
 	writer.appendVarInt(count);
-	appendZigZagNumber(writer, Number(values[0]));
+	appendZigZagNumber(writer, values[0]);
 	const packedByBitWidth = [];
 	const blockDeltas = new Float64Array(BLOCK_SIZE);
 	const bitWidths = new Uint8Array(MINIBLOCKS_PER_BLOCK);
@@ -34,10 +34,25 @@ function deltaBinaryPackInt32(writer, values) {
 	while (index < count) {
 		const blockEnd = Math.min(index + BLOCK_SIZE, count);
 		const blockSize = blockEnd - index;
-		let minDelta = Number(values[index]) - Number(values[index - 1]);
+		const firstDelta = values[index] - values[index - 1];
+		let previous = values[index];
+		let constantEnd = index + 1;
+		while (constantEnd < blockEnd) {
+			const value = values[constantEnd];
+			if (value - previous !== firstDelta) break;
+			previous = value;
+			constantEnd++;
+		}
+		if (constantEnd === blockEnd) {
+			appendZigZagNumber(writer, firstDelta);
+			writer.appendUint32(0);
+			index = blockEnd;
+			continue;
+		}
+		let minDelta = values[index] - values[index - 1];
 		blockDeltas[0] = minDelta;
 		for (let i = 1; i < blockSize; i++) {
-			const delta = Number(values[index + i]) - Number(values[index + i - 1]);
+			const delta = values[index + i] - values[index + i - 1];
 			blockDeltas[i] = delta;
 			if (delta < minDelta) minDelta = delta;
 		}
@@ -93,11 +108,12 @@ function deltaBinaryPackBigInt(writer, values) {
 	writer.appendVarInt(MINIBLOCKS_PER_BLOCK);
 	writer.appendVarInt(count);
 	writer.appendZigZag(values[0]);
+	const blockDeltas = new BigInt64Array(BLOCK_SIZE);
+	const bitWidths = new Uint8Array(MINIBLOCKS_PER_BLOCK);
 	let index = 1;
 	while (index < count) {
 		const blockEnd = Math.min(index + BLOCK_SIZE, count);
 		const blockSize = blockEnd - index;
-		const blockDeltas = new BigInt64Array(blockSize);
 		let minDelta = BigInt(values[index]) - BigInt(values[index - 1]);
 		blockDeltas[0] = minDelta;
 		for (let i = 1; i < blockSize; i++) {
@@ -106,7 +122,6 @@ function deltaBinaryPackBigInt(writer, values) {
 			if (delta < minDelta) minDelta = delta;
 		}
 		writer.appendZigZag(minDelta);
-		const bitWidths = new Uint8Array(MINIBLOCKS_PER_BLOCK);
 		for (let mb = 0; mb < MINIBLOCKS_PER_BLOCK; mb++) {
 			const mbStart = mb * VALUES_PER_MINIBLOCK;
 			const mbEnd = Math.min(mbStart + VALUES_PER_MINIBLOCK, blockSize);

package/dist/_virtual/node_modules/.pnpm/{hyparquet-writer@0.16.9_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f → hyparquet-writer@0.16.10_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f}/node_modules/hyparquet-writer/src/dremel.mjs

@@ -1,4 +1,4 @@
-import { getMaxDefinitionLevel, isListLike, isMapLike } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/schema.mjs";
+import { getMaxDefinitionLevel, isListLike, isMapLike } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/schema.mjs";
 function encodeNestedValues(treePath, rows) {
 	const schemaPath = treePath.map((n) => n.element);
 	if (treePath.length < 2) throw new Error("parquet schema path must include column");

package/dist/_virtual/node_modules/.pnpm/{hyparquet-writer@0.16.9_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f → hyparquet-writer@0.16.10_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f}/node_modules/hyparquet-writer/src/indexes.mjs

@@ -1,4 +1,4 @@
-import { BoundaryOrders } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/constants.mjs";
+import { BoundaryOrders } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/constants.mjs";
 import { serializeTCompactProtocol } from "./thrift.mjs";
 function writeIndexes(writer, pageIndexes) {
 	for (const { chunk, columnIndex } of pageIndexes) writeColumnIndex(writer, chunk, columnIndex);

package/dist/_virtual/node_modules/.pnpm/{hyparquet-writer@0.16.9_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f → hyparquet-writer@0.16.10_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f}/node_modules/hyparquet-writer/src/metadata.mjs

@@ -1,5 +1,5 @@
-import { CompressionCodecs, ConvertedTypes, EdgeInterpolationAlgorithms, Encodings, FieldRepetitionTypes, PageTypes, ParquetTypes } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/constants.mjs";
-import { getSchemaPath } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/schema.mjs";
+import { CompressionCodecs, ConvertedTypes, EdgeInterpolationAlgorithms, Encodings, FieldRepetitionTypes, PageTypes, ParquetTypes } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/constants.mjs";
+import { getSchemaPath } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/schema.mjs";
 import { ThriftStruct, serializeTCompactProtocol } from "./thrift.mjs";
 import { unconvertStatistics } from "./unconvert.mjs";
 function writeMetadata(writer, metadata) {

package/dist/_virtual/node_modules/.pnpm/{hyparquet-writer@0.16.9_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f → hyparquet-writer@0.16.10_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f}/node_modules/hyparquet-writer/src/parquet-writer.mjs

@@ -1,4 +1,4 @@
-import { getSchemaPath } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/schema.mjs";
+import { getSchemaPath } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/schema.mjs";
 import { writeBlooms } from "./bloom.mjs";
 import { encodeVariantColumn } from "./variant.mjs";
 import { writeColumn } from "./column.mjs";

package/dist/_virtual/node_modules/.pnpm/{hyparquet-writer@0.16.9_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f → hyparquet-writer@0.16.10_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f}/node_modules/hyparquet-writer/src/types.d.mts

@@ -1,4 +1,4 @@
-import "../../../../hyparquet@1.29.2/node_modules/hyparquet/types/node.mjs";
+import "../../../../hyparquet@1.31.1/node_modules/hyparquet/types/node.mjs";
 export interface Writer {
   buffer: ArrayBuffer;
   view: DataView;

package/dist/_virtual/node_modules/.pnpm/{hyparquet-writer@0.16.9_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f → hyparquet-writer@0.16.10_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f}/node_modules/hyparquet-writer/src/unconvert.mjs

@@ -1,5 +1,5 @@
-import { toJson } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/utils.mjs";
-import "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/node.mjs";
+import { toJson } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/utils.mjs";
+import "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/node.mjs";
 import { geojsonToWkb } from "./wkb.mjs";
 const dayMillis = 864e5;
 function mapConverted(values, convert) {

package/dist/_virtual/node_modules/.pnpm/{hyparquet-writer@0.16.9_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f → hyparquet-writer@0.16.10_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f}/node_modules/hyparquet-writer/types/index.d.mts

@@ -1,4 +1,4 @@
-import "../../../../hyparquet@1.29.2/node_modules/hyparquet/types/node.mjs";
+import "../../../../hyparquet@1.31.1/node_modules/hyparquet/types/node.mjs";
 import { Writer as Writer$1 } from "../src/types.mjs";
 import "./write.mjs";
 import "./write-rows.mjs";

package/dist/_virtual/node_modules/.pnpm/hyparquet-writer@0.16.10_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/types/parquet-writer.d.mts

@@ -0,0 +1,2 @@
+import "../../../../hyparquet@1.31.1/node_modules/hyparquet/types/node.mjs";
+import "../src/types.mjs";

package/dist/_virtual/node_modules/.pnpm/hyparquet-writer@0.16.10_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/types/schema.d.mts

@@ -0,0 +1,2 @@
+import "../../../../hyparquet@1.31.1/node_modules/hyparquet/types/node.mjs";
+import "../src/types.mjs";

package/dist/_virtual/node_modules/.pnpm/{hyparquet@1.29.2 → hyparquet@1.31.1}/node_modules/hyparquet/src/filter.mjs

@@ -29,7 +29,7 @@ function matchFilter(record, filter, strict = true) {
 			if (operator === "$ne") return !equals(value, target, strict);
 			if (operator === "$in") return Array.isArray(target) && matchesIn(value, target, strict);
 			if (operator === "$nin") return Array.isArray(target) && !matchesIn(value, target, strict);
-			if (operator === "$not") return !matchFilter({ [field]: value }, { [field]: target }, strict);
+			if (operator === "$not") return !matchFilter({ value }, { value: target }, strict);
 			return true;
 		});
 	});

package/dist/_virtual/node_modules/.pnpm/{hyparquet@1.29.2 → hyparquet@1.31.1}/node_modules/hyparquet/src/indexes.mjs

@@ -3,15 +3,15 @@ import { DEFAULT_PARSERS } from "./convert.mjs";
 import { deserializeTCompactProtocol } from "./thrift.mjs";
 import { convertMetadata } from "./metadata.mjs";
 function readColumnIndex(reader, schema, parsers = void 0) {
-	parsers = {
+	const allParsers = {
 		...DEFAULT_PARSERS,
 		...parsers
 	};
 	const thrift = deserializeTCompactProtocol(reader);
 	return {
 		null_pages: thrift.field_1,
-		min_values: thrift.field_2.map((m) => convertMetadata(m, schema, parsers)),
-		max_values: thrift.field_3.map((m) => convertMetadata(m, schema, parsers)),
+		min_values: thrift.field_2.map((m) => convertMetadata(m, schema, allParsers)),
+		max_values: thrift.field_3.map((m) => convertMetadata(m, schema, allParsers)),
 		boundary_order: BoundaryOrders[thrift.field_4],
 		null_counts: thrift.field_5,
 		repetition_level_histograms: thrift.field_6,

package/dist/_virtual/node_modules/.pnpm/{hyparquet@1.29.2 → hyparquet@1.31.1}/node_modules/hyparquet/src/metadata.mjs

@@ -35,7 +35,7 @@ async function parquetMetadataAsync(asyncBuffer, { parsers, initialFetchSize = d
 function parquetMetadata(arrayBuffer, { parsers, geoparquet = true } = {}) {
 	if (!(arrayBuffer instanceof ArrayBuffer)) throw new Error("parquet expected ArrayBuffer");
 	const view = new DataView(arrayBuffer);
-	parsers = {
+	const allParsers = {
 		...DEFAULT_PARSERS,
 		...parsers
 	};
@@ -83,7 +83,7 @@ function parquetMetadata(arrayBuffer, { parsers, geoparquet = true } = {}) {
 				data_page_offset: column.field_3.field_9,
 				index_page_offset: column.field_3.field_10,
 				dictionary_page_offset: column.field_3.field_11,
-				statistics: convertStats(column.field_3.field_12, columnSchema[columnIndex], parsers),
+				statistics: convertStats(column.field_3.field_12, columnSchema[columnIndex], allParsers),
 				encoding_stats: column.field_3.field_13?.map((encodingStat) => ({
 					page_type: PageTypes[encodingStat.field_1],
 					encoding: Encodings[encodingStat.field_2],

package/dist/_virtual/node_modules/.pnpm/{hyparquet@1.29.2 → hyparquet@1.31.1}/node_modules/hyparquet/src/read.mjs

@@ -4,10 +4,12 @@ import { columnsNeededForFilter, matchFilter } from "./filter.mjs";
 import "./plan.mjs";
 import { assembleAsync, asyncGroupToRows } from "./rowgroup.mjs";
 import { prepareParquetRead, readParquetPlan } from "./scan.mjs";
+const rowIndex = Symbol("rowIndex");
 async function parquetRead(options) {
 	options.metadata ??= await parquetMetadataAsync(options.file, options);
 	const { rowStart = 0, rowEnd, columns, onChunk, onComplete, rowFormat, filter, filterStrict = true } = options;
 	if (filter && rowFormat !== "object") throw new Error("parquet filter requires rowFormat: \"object\"");
+	if (options.includeRowIndex && rowFormat !== "object") throw new Error("parquet includeRowIndex requires rowFormat: \"object\"");
 	const filterColumns = columnsNeededForFilter(filter);
 	let readColumns = columns;
 	if (columns && filterColumns.length) {
@@ -49,6 +51,7 @@ async function parquetRead(options) {
 			const selectStart = asyncGroup.selectStart ?? Math.max(rowStart - asyncGroup.groupStart, 0);
 			const selectEnd = asyncGroup.selectEnd ?? Math.min((rowEnd ?? Infinity) - asyncGroup.groupStart, asyncGroup.groupRows);
 			const groupData = rowFormat === "object" ? await asyncGroupToRows(asyncGroup, selectStart, selectEnd, readColumns, "object") : await asyncGroupToRows(asyncGroup, selectStart, selectEnd, columns, "array");
+			if (options.includeRowIndex) for (let i = 0; i < groupData.length; i++) Object.defineProperty(groupData[i], rowIndex, { value: asyncGroup.groupStart + selectStart + i });
 			if (filter) {
 				for (const row of groupData) if (matchFilter(row, filter, filterStrict)) {
 					if (requiresProjection && columns) {
@@ -75,4 +78,4 @@ function parquetReadObjects(options) {
 		}).catch(reject);
 	});
 }
-export { parquetRead, parquetReadObjects };
+export { parquetRead, parquetReadObjects, rowIndex };

package/dist/_virtual/node_modules/.pnpm/{hyparquet@1.29.2 → hyparquet@1.31.1}/node_modules/hyparquet/src/rowgroup.mjs

@@ -13,12 +13,12 @@ function readRowGroup(options, { metadata }, groupPlan) {
 			pathInSchema,
 			element: schemaPath[schemaPath.length - 1].element,
 			schemaPath,
+			...options,
+			...chunk.columnMetadata,
 			parsers: {
 				...DEFAULT_PARSERS,
 				...options.parsers
-			},
-			...options,
-			...chunk.columnMetadata
+			}
 		};
 		const { startByte, endByte } = chunk.range;
 		if ("pageLocations" in chunk) asyncColumns.push({
@@ -133,7 +133,7 @@ async function asyncGroupToRows({ asyncColumns }, selectStart, selectEnd, column
 }
 function assembleAsync(asyncRowGroup, schemaTree, parsers) {
 	const { asyncColumns } = asyncRowGroup;
-	parsers = {
+	const allParsers = {
 		...DEFAULT_PARSERS,
 		...parsers
 	};
@@ -147,14 +147,14 @@ function assembleAsync(asyncRowGroup, schemaTree, parsers) {
 				const resolved = await Promise.all(childColumns.map((c) => c.data));
 				const subcolumnData = /* @__PURE__ */ new Map();
 				const flattened = resolved.map(({ data }) => flatten(data));
-				const skipped = Math.max(...resolved.map((result) => result.skipped));
-				const end = Math.min(...resolved.map((result, i) => result.skipped + flattened[i].length));
+				const skipped = Math.max(asyncRowGroup.selectStart ?? 0, ...resolved.map((result) => result.skipped));
+				const end = Math.min(asyncRowGroup.selectEnd ?? Infinity, ...resolved.map((result, i) => result.skipped + flattened[i].length));
 				for (let i = 0; i < childColumns.length; i++) {
 					const start = skipped - resolved[i].skipped;
 					const length = Math.max(0, end - skipped);
 					subcolumnData.set(childColumns[i].pathInSchema.join("."), flattened[i].slice(start, start + length));
 				}
-				assembleNested(subcolumnData, child, parsers);
+				assembleNested(subcolumnData, child, allParsers);
 				const assembled = subcolumnData.get(child.element.name);
 				if (!assembled) throw new Error("parquet column data not assembled");
 				return {