@gscdump/engine 1.4.0 → 1.4.1

package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/bytewriter.mjs

@@ -0,0 +1,91 @@
+function ByteWriter(initalSize = 1024) {
+	this.buffer = new ArrayBuffer(initalSize);
+	this.view = new DataView(this.buffer);
+	this.offset = 0;
+	this.index = 0;
+	return this;
+}
+ByteWriter.prototype.ensure = function(size) {
+	if (this.index + size > this.buffer.byteLength) {
+		const newSize = Math.max(this.buffer.byteLength * 2, this.index + size);
+		const newBuffer = new ArrayBuffer(newSize);
+		new Uint8Array(newBuffer).set(new Uint8Array(this.buffer));
+		this.buffer = newBuffer;
+		this.view = new DataView(this.buffer);
+	}
+};
+ByteWriter.prototype.finish = function() {};
+ByteWriter.prototype.getBuffer = function() {
+	return this.buffer.slice(0, this.index);
+};
+ByteWriter.prototype.getBytes = function() {
+	return new Uint8Array(this.buffer, 0, this.index);
+};
+ByteWriter.prototype.appendUint8 = function(value) {
+	this.ensure(this.index + 1);
+	this.view.setUint8(this.index, value);
+	this.offset++;
+	this.index++;
+};
+ByteWriter.prototype.appendUint32 = function(value) {
+	this.ensure(this.index + 4);
+	this.view.setUint32(this.index, value, true);
+	this.offset += 4;
+	this.index += 4;
+};
+ByteWriter.prototype.appendInt32 = function(value) {
+	this.ensure(this.index + 4);
+	this.view.setInt32(this.index, value, true);
+	this.offset += 4;
+	this.index += 4;
+};
+ByteWriter.prototype.appendInt64 = function(value) {
+	this.ensure(this.index + 8);
+	this.view.setBigInt64(this.index, BigInt(value), true);
+	this.offset += 8;
+	this.index += 8;
+};
+ByteWriter.prototype.appendFloat32 = function(value) {
+	this.ensure(this.index + 8);
+	this.view.setFloat32(this.index, value, true);
+	this.offset += 4;
+	this.index += 4;
+};
+ByteWriter.prototype.appendFloat64 = function(value) {
+	this.ensure(this.index + 8);
+	this.view.setFloat64(this.index, value, true);
+	this.offset += 8;
+	this.index += 8;
+};
+ByteWriter.prototype.appendBuffer = function(value) {
+	this.appendBytes(new Uint8Array(value));
+};
+ByteWriter.prototype.appendBytes = function(value) {
+	this.ensure(this.index + value.length);
+	new Uint8Array(this.buffer, this.index, value.length).set(value);
+	this.offset += value.length;
+	this.index += value.length;
+};
+ByteWriter.prototype.appendVarInt = function(value) {
+	while (true) if ((value & -128) === 0) {
+		this.appendUint8(value);
+		return;
+	} else {
+		this.appendUint8(value & 127 | 128);
+		value >>>= 7;
+	}
+};
+ByteWriter.prototype.appendVarBigInt = function(value) {
+	while (true) if ((value & -128n) === 0n) {
+		this.appendUint8(Number(value));
+		return;
+	} else {
+		this.appendUint8(Number(value & 127n | 128n));
+		value >>= 7n;
+	}
+};
+ByteWriter.prototype.appendZigZag = function(value) {
+	if (typeof value === "number") this.appendVarInt(value << 1 ^ value >> 31);
+	else this.appendVarBigInt(value << 1n ^ value >> 63n);
+};
+export { ByteWriter };

package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/column.mjs

@@ -0,0 +1,186 @@
+import { BloomBuilder } from "./bloom.mjs";
+import { writeDataPageV2 } from "./datapage.mjs";
+import { estimateValueSize, useDictionary, writeDictionaryPage } from "./dictionary.mjs";
+import { geospatialStatistics } from "./geospatial.mjs";
+import { unconvert, unconvertMinMax } from "./unconvert.mjs";
+function writeColumn({ writer, column, pageData }) {
+	const { columnName, element, schemaPath, stats, pageSize, encoding: userEncoding } = column;
+	const { type, type_length } = element;
+	if (!type) throw new Error(`column ${columnName} cannot determine type`);
+	const { values, definitionLevels, repetitionLevels, maxDefinitionLevel } = pageData;
+	const offsetStart = writer.offset;
+	const encodings = [];
+	const isGeospatial = element?.logical_type?.type === "GEOMETRY" || element?.logical_type?.type === "GEOGRAPHY";
+	const statistics = stats ? getStatistics(values) : void 0;
+	const geospatial_statistics = stats && isGeospatial ? geospatialStatistics(values) : void 0;
+	let bloomFilter;
+	if (column.bloomFilter) {
+		const builder = new BloomBuilder(element, typeof column.bloomFilter === "object" ? column.bloomFilter : void 0);
+		for (const v of values) builder.insert(v);
+		bloomFilter = builder.finalize();
+	}
+	let dictionary_page_offset;
+	const { dictionary, indexes } = useDictionary(values, type, type_length, userEncoding, pageSize);
+	let encoding;
+	let writeValues;
+	let writeType = type;
+	if (dictionary && indexes) {
+		writeValues = indexes;
+		writeType = "INT32";
+		encoding = "RLE_DICTIONARY";
+		dictionary_page_offset = BigInt(writer.offset);
+		writeDictionaryPage(writer, column, unconvert(element, dictionary));
+	} else {
+		writeValues = unconvert(element, values);
+		encoding = userEncoding ?? (type === "BOOLEAN" && values.length > 16 ? "RLE" : "PLAIN");
+	}
+	encodings.push(encoding);
+	const pageBoundaries = getPageBoundaries(writeValues, writeType, type_length, pageSize);
+	const columnIndex = column.columnIndex && pageBoundaries.length > 1 ? {
+		null_pages: [],
+		min_values: [],
+		max_values: [],
+		boundary_order: "UNORDERED",
+		null_counts: []
+	} : void 0;
+	const offsetIndex = column.offsetIndex && pageBoundaries.length > 1 ? { page_locations: [] } : void 0;
+	const data_page_offset = BigInt(writer.offset);
+	let first_row_index = 0n;
+	let prevStart = 0;
+	let prevMinValue;
+	let prevMaxValue;
+	let ascending = true;
+	let descending = true;
+	for (const { start, end } of pageBoundaries) {
+		const pageOffset = writer.offset;
+		const pageChunk = {
+			values: writeValues.slice(start, end),
+			definitionLevels: definitionLevels.slice(start, end),
+			repetitionLevels: repetitionLevels.slice(start, end),
+			maxDefinitionLevel
+		};
+		writeDataPageV2({
+			writer,
+			column,
+			encoding,
+			pageData: pageChunk
+		});
+		if (columnIndex) {
+			const { min_value, max_value, null_count = 0n } = getStatistics(values.slice(start, end));
+			columnIndex.null_pages.push(null_count === BigInt(end - start));
+			columnIndex.min_values.push(unconvertMinMax(min_value, element, false) ?? /* @__PURE__ */ new Uint8Array());
+			columnIndex.max_values.push(unconvertMinMax(max_value, element, true) ?? /* @__PURE__ */ new Uint8Array());
+			columnIndex.null_counts?.push(null_count);
+			if (prevMinValue !== void 0 && min_value !== void 0) {
+				if (prevMinValue > min_value) ascending = false;
+				if (prevMinValue < min_value) descending = false;
+			}
+			if (prevMaxValue !== void 0 && max_value !== void 0) {
+				if (prevMaxValue > max_value) ascending = false;
+				if (prevMaxValue < max_value) descending = false;
+			}
+			prevMinValue = min_value;
+			prevMaxValue = max_value;
+		}
+		if (offsetIndex) {
+			if (repetitionLevels.length) {
+				for (let i = prevStart + 1; i <= start; i++) if (repetitionLevels[i] === 0) first_row_index++;
+			} else first_row_index = BigInt(start);
+			offsetIndex.page_locations.push({
+				offset: BigInt(pageOffset),
+				compressed_page_size: writer.offset - pageOffset,
+				first_row_index
+			});
+		}
+		prevStart = start;
+	}
+	if (columnIndex) {
+		if (ascending) columnIndex.boundary_order = "ASCENDING";
+		else if (descending) columnIndex.boundary_order = "DESCENDING";
+	}
+	let encoding_stats;
+	if (stats) {
+		encoding_stats = [];
+		if (dictionary_page_offset !== void 0) encoding_stats.push({
+			page_type: "DICTIONARY_PAGE",
+			encoding: "PLAIN",
+			count: 1
+		});
+		encoding_stats.push({
+			page_type: "DATA_PAGE_V2",
+			encoding,
+			count: pageBoundaries.length
+		});
+	}
+	return {
+		chunk: {
+			meta_data: {
+				type,
+				encodings,
+				path_in_schema: schemaPath.slice(1).map((s) => s.name),
+				codec: column.codec ?? "UNCOMPRESSED",
+				num_values: BigInt(values.length),
+				total_compressed_size: BigInt(writer.offset - offsetStart),
+				total_uncompressed_size: BigInt(writer.offset - offsetStart),
+				data_page_offset,
+				dictionary_page_offset,
+				statistics,
+				encoding_stats,
+				geospatial_statistics
+			},
+			file_offset: BigInt(offsetStart)
+		},
+		columnIndex,
+		offsetIndex,
+		bloomFilter
+	};
+}
+function getPageBoundaries(values, type, type_length, pageSize) {
+	if (!pageSize) return [{
+		start: 0,
+		end: values.length
+	}];
+	const boundaries = [];
+	let start = 0;
+	let accumulatedBytes = 0;
+	for (let i = 0; i < values.length; i++) {
+		const valueSize = estimateValueSize(values[i], type, type_length);
+		accumulatedBytes += valueSize;
+		if (accumulatedBytes >= pageSize && i > start) {
+			boundaries.push({
+				start,
+				end: i
+			});
+			start = i;
+			accumulatedBytes = valueSize;
+		}
+	}
+	if (start < values.length) boundaries.push({
+		start,
+		end: values.length
+	});
+	return boundaries;
+}
+function getStatistics(values) {
+	let min_value = void 0;
+	let max_value = void 0;
+	let null_count = 0n;
+	for (const value of values) {
+		if (value === null || value === void 0) {
+			null_count++;
+			continue;
+		}
+		if (typeof value === "object") continue;
+		if (typeof value === "number" && Number.isNaN(value)) continue;
+		if (min_value === void 0 || value < min_value) min_value = value;
+		if (max_value === void 0 || value > max_value) max_value = value;
+	}
+	if (min_value === 0) min_value = -0;
+	if (max_value === 0) max_value = 0;
+	return {
+		min_value,
+		max_value,
+		null_count
+	};
+}
+export { writeColumn };

package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/datapage.mjs

@@ -0,0 +1,113 @@
+import { ByteWriter } from "./bytewriter.mjs";
+import { serializeTCompactProtocol } from "./thrift.mjs";
+import { Encodings, PageTypes } from "../../../../hyparquet@1.26.1/node_modules/hyparquet/src/constants.mjs";
+import { deltaBinaryPack, deltaByteArray, deltaLengthByteArray } from "./delta.mjs";
+import { writeRleBitPackedHybrid } from "./encoding.mjs";
+import { writePlain } from "./plain.mjs";
+import { getMaxRepetitionLevel } from "./schema.mjs";
+import { writeByteStreamSplit } from "./splitstream.mjs";
+function writeDataPageV2({ writer, column, encoding, pageData }) {
+	const { columnName, element, codec, compressors } = column;
+	const { type, type_length, repetition_type } = element;
+	if (!type) throw new Error(`column ${columnName} cannot determine type`);
+	if (repetition_type === "REPEATED") throw new Error(`column ${columnName} repeated types not supported`);
+	const levelWriter = new ByteWriter();
+	const { definition_levels_byte_length, repetition_levels_byte_length, num_nulls, num_values, num_rows } = writeLevels(levelWriter, column, pageData);
+	const nonnull = num_nulls ? pageData.values.filter((v) => v !== null && v !== void 0) : pageData.values;
+	const page = new ByteWriter();
+	if (encoding === "PLAIN") writePlain(page, nonnull, type, type_length);
+	else if (encoding === "RLE") {
+		if (type !== "BOOLEAN") throw new Error("RLE encoding only supported for BOOLEAN type");
+		const rleData = new ByteWriter();
+		writeRleBitPackedHybrid(rleData, nonnull, 1);
+		page.appendUint32(rleData.offset);
+		page.appendBytes(rleData.getBytes());
+	} else if (encoding === "PLAIN_DICTIONARY" || encoding === "RLE_DICTIONARY") {
+		let maxValue = 0;
+		for (const v of nonnull) if (v > maxValue) maxValue = v;
+		const bitWidth = Math.ceil(Math.log2(maxValue + 1));
+		page.appendUint8(bitWidth);
+		writeRleBitPackedHybrid(page, nonnull, bitWidth);
+	} else if (encoding === "DELTA_BINARY_PACKED") {
+		if (type !== "INT32" && type !== "INT64") throw new Error("DELTA_BINARY_PACKED encoding only supported for INT32 and INT64 types");
+		deltaBinaryPack(page, nonnull);
+	} else if (encoding === "DELTA_LENGTH_BYTE_ARRAY") {
+		if (type !== "BYTE_ARRAY") throw new Error("DELTA_LENGTH_BYTE_ARRAY encoding only supported for BYTE_ARRAY type");
+		deltaLengthByteArray(page, nonnull);
+	} else if (encoding === "DELTA_BYTE_ARRAY") {
+		if (type !== "BYTE_ARRAY") throw new Error("DELTA_BYTE_ARRAY encoding only supported for BYTE_ARRAY type");
+		deltaByteArray(page, nonnull);
+	} else if (encoding === "BYTE_STREAM_SPLIT") writeByteStreamSplit(page, nonnull, type, type_length);
+	else throw new Error(`parquet unsupported encoding: ${encoding}`);
+	const pageBytes = page.getBytes();
+	const compressedBytes = compressors[codec]?.(pageBytes) ?? pageBytes;
+	writePageHeader(writer, {
+		type: "DATA_PAGE_V2",
+		uncompressed_page_size: levelWriter.offset + page.offset,
+		compressed_page_size: levelWriter.offset + compressedBytes.length,
+		data_page_header_v2: {
+			num_values,
+			num_nulls,
+			num_rows,
+			encoding,
+			definition_levels_byte_length,
+			repetition_levels_byte_length,
+			is_compressed: !!codec
+		}
+	});
+	writer.appendBytes(levelWriter.getBytes());
+	writer.appendBytes(compressedBytes);
+}
+function writePageHeader(writer, header) {
+	serializeTCompactProtocol(writer, {
+		field_1: PageTypes.indexOf(header.type),
+		field_2: header.uncompressed_page_size,
+		field_3: header.compressed_page_size,
+		field_4: header.crc,
+		field_5: header.data_page_header && {
+			field_1: header.data_page_header.num_values,
+			field_2: Encodings.indexOf(header.data_page_header.encoding),
+			field_3: Encodings.indexOf(header.data_page_header.definition_level_encoding),
+			field_4: Encodings.indexOf(header.data_page_header.repetition_level_encoding)
+		},
+		field_7: header.dictionary_page_header && {
+			field_1: header.dictionary_page_header.num_values,
+			field_2: Encodings.indexOf(header.dictionary_page_header.encoding)
+		},
+		field_8: header.data_page_header_v2 && {
+			field_1: header.data_page_header_v2.num_values,
+			field_2: header.data_page_header_v2.num_nulls,
+			field_3: header.data_page_header_v2.num_rows,
+			field_4: Encodings.indexOf(header.data_page_header_v2.encoding),
+			field_5: header.data_page_header_v2.definition_levels_byte_length,
+			field_6: header.data_page_header_v2.repetition_levels_byte_length,
+			field_7: header.data_page_header_v2.is_compressed ? void 0 : false
+		}
+	});
+}
+function writeLevels(writer, column, dataPage) {
+	const { schemaPath } = column;
+	const { values, definitionLevels, repetitionLevels, maxDefinitionLevel } = dataPage;
+	const num_values = definitionLevels.length || values.length;
+	let num_nulls = 0;
+	let num_rows = 0;
+	if (repetitionLevels.length) {
+		for (let i = 0; i < repetitionLevels.length; i++) if (repetitionLevels[i] === 0) num_rows++;
+	} else num_rows = values.length;
+	if (definitionLevels.length) {
+		for (let i = 0; i < definitionLevels.length; i++) if (definitionLevels[i] < maxDefinitionLevel) num_nulls++;
+	}
+	const maxRepetitionLevel = getMaxRepetitionLevel(schemaPath);
+	let repetition_levels_byte_length = 0;
+	if (maxRepetitionLevel) repetition_levels_byte_length = writeRleBitPackedHybrid(writer, repetitionLevels, Math.ceil(Math.log2(maxRepetitionLevel + 1)));
+	let definition_levels_byte_length = 0;
+	if (maxDefinitionLevel) definition_levels_byte_length = writeRleBitPackedHybrid(writer, definitionLevels, Math.ceil(Math.log2(maxDefinitionLevel + 1)));
+	return {
+		definition_levels_byte_length,
+		repetition_levels_byte_length,
+		num_values,
+		num_nulls,
+		num_rows
+	};
+}
+export { writeDataPageV2, writePageHeader };

package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/delta.mjs

@@ -0,0 +1,111 @@
+const BLOCK_SIZE = 128;
+const MINIBLOCKS_PER_BLOCK = 4;
+const VALUES_PER_MINIBLOCK = BLOCK_SIZE / MINIBLOCKS_PER_BLOCK;
+function deltaBinaryPack(writer, values) {
+	const count = values.length;
+	if (count === 0) {
+		writer.appendVarInt(BLOCK_SIZE);
+		writer.appendVarInt(MINIBLOCKS_PER_BLOCK);
+		writer.appendVarInt(0);
+		writer.appendVarInt(0);
+		return;
+	}
+	if (typeof values[0] !== "number" && typeof values[0] !== "bigint") throw new Error("deltaBinaryPack only supports number or bigint arrays");
+	writer.appendVarInt(BLOCK_SIZE);
+	writer.appendVarInt(MINIBLOCKS_PER_BLOCK);
+	writer.appendVarInt(count);
+	writer.appendZigZag(values[0]);
+	let index = 1;
+	while (index < count) {
+		const blockEnd = Math.min(index + BLOCK_SIZE, count);
+		const blockSize = blockEnd - index;
+		const blockDeltas = new BigInt64Array(blockSize);
+		let minDelta = BigInt(values[index]) - BigInt(values[index - 1]);
+		blockDeltas[0] = minDelta;
+		for (let i = 1; i < blockSize; i++) {
+			const delta = BigInt(values[index + i]) - BigInt(values[index + i - 1]);
+			blockDeltas[i] = delta;
+			if (delta < minDelta) minDelta = delta;
+		}
+		writer.appendZigZag(minDelta);
+		const bitWidths = new Uint8Array(MINIBLOCKS_PER_BLOCK);
+		for (let mb = 0; mb < MINIBLOCKS_PER_BLOCK; mb++) {
+			const mbStart = mb * VALUES_PER_MINIBLOCK;
+			const mbEnd = Math.min(mbStart + VALUES_PER_MINIBLOCK, blockSize);
+			let maxAdjusted = 0n;
+			for (let i = mbStart; i < mbEnd; i++) {
+				const adjusted = blockDeltas[i] - minDelta;
+				if (adjusted > maxAdjusted) maxAdjusted = adjusted;
+			}
+			bitWidths[mb] = bitWidth(maxAdjusted);
+		}
+		writer.appendBytes(bitWidths);
+		for (let mb = 0; mb < MINIBLOCKS_PER_BLOCK; mb++) {
+			const bitWidth = bitWidths[mb];
+			if (bitWidth === 0) continue;
+			const mbStart = mb * VALUES_PER_MINIBLOCK;
+			const mbEnd = Math.min(mbStart + VALUES_PER_MINIBLOCK, blockSize);
+			let buffer = 0n;
+			let bitsUsed = 0;
+			for (let i = 0; i < VALUES_PER_MINIBLOCK; i++) {
+				const adjusted = mbStart + i < mbEnd ? blockDeltas[mbStart + i] - minDelta : 0n;
+				buffer |= adjusted << BigInt(bitsUsed);
+				bitsUsed += bitWidth;
+				while (bitsUsed >= 8) {
+					writer.appendUint8(Number(buffer & 255n));
+					buffer >>= 8n;
+					bitsUsed -= 8;
+				}
+			}
+		}
+		index = blockEnd;
+	}
+}
+function deltaLengthByteArray(writer, values) {
+	const lengths = new Int32Array(values.length);
+	for (let i = 0; i < values.length; i++) {
+		const value = values[i];
+		if (!(value instanceof Uint8Array)) throw new Error("deltaLengthByteArray expects Uint8Array values");
+		lengths[i] = value.length;
+	}
+	deltaBinaryPack(writer, lengths);
+	for (const value of values) writer.appendBytes(value);
+}
+function deltaByteArray(writer, values) {
+	if (values.length === 0) {
+		deltaBinaryPack(writer, []);
+		deltaBinaryPack(writer, []);
+		return;
+	}
+	const prefixLengths = new Int32Array(values.length);
+	const suffixLengths = new Int32Array(values.length);
+	const suffixes = new Array(values.length);
+	if (!(values[0] instanceof Uint8Array)) throw new Error("deltaByteArray expects Uint8Array values");
+	prefixLengths[0] = 0;
+	suffixLengths[0] = values[0].length;
+	suffixes[0] = values[0];
+	for (let i = 1; i < values.length; i++) {
+		const prev = values[i - 1];
+		const curr = values[i];
+		if (!(curr instanceof Uint8Array)) throw new Error("deltaByteArray expects Uint8Array values");
+		let prefixLen = 0;
+		const maxPrefix = Math.min(prev.length, curr.length);
+		while (prefixLen < maxPrefix && prev[prefixLen] === curr[prefixLen]) prefixLen++;
+		prefixLengths[i] = prefixLen;
+		suffixLengths[i] = curr.length - prefixLen;
+		suffixes[i] = curr.subarray(prefixLen);
+	}
+	deltaBinaryPack(writer, prefixLengths);
+	deltaBinaryPack(writer, suffixLengths);
+	for (const suffix of suffixes) writer.appendBytes(suffix);
+}
+function bitWidth(value) {
+	if (value === 0n) return 0;
+	let bits = 0;
+	while (value > 0n) {
+		bits++;
+		value >>= 1n;
+	}
+	return bits;
+}
+export { deltaBinaryPack, deltaByteArray, deltaLengthByteArray };

package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/dictionary.mjs

@@ -0,0 +1,99 @@
+import { ByteWriter } from "./bytewriter.mjs";
+import { writePlain } from "./plain.mjs";
+import { writePageHeader } from "./datapage.mjs";
+function estimateValueSize(value, type, type_length) {
+	if (value === null || value === void 0) return 0;
+	if (type === "BOOLEAN") return .125;
+	if (type === "INT32" || type === "FLOAT") return 4;
+	if (type === "INT64" || type === "DOUBLE") return 8;
+	if (type === "INT96") return 12;
+	if (type === "FIXED_LEN_BYTE_ARRAY") return type_length ?? 0;
+	if (type === "BYTE_ARRAY") {
+		if (value instanceof Uint8Array) return value.byteLength;
+		if (typeof value === "string") return value.length;
+	}
+	return 0;
+}
+function hashBytes(bytes) {
+	let h = 2166136261;
+	for (let i = 0; i < bytes.length; i++) {
+		h ^= bytes[i];
+		h = Math.imul(h, 16777619);
+	}
+	return h >>> 0;
+}
+function bytesEqual(a, b) {
+	if (a.length !== b.length) return false;
+	for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
+	return true;
+}
+function useDictionary(values, type, type_length, encoding, pageSize) {
+	if (encoding && encoding !== "RLE_DICTIONARY") return {};
+	if (type === "BOOLEAN") return {};
+	const sample = values.slice(0, 1e3);
+	const sampleKeys = /* @__PURE__ */ new Set();
+	for (const value of sample) sampleKeys.add(value instanceof Uint8Array ? hashBytes(value) : value);
+	if (sampleKeys.size === 0 || sampleKeys.size / sample.length > .5) return {};
+	const dictionary = [];
+	const indexes = new Array(values.length);
+	const valueIndex = /* @__PURE__ */ new Map();
+	const hashBuckets = /* @__PURE__ */ new Map();
+	let dictSize = 0;
+	for (let i = 0; i < values.length; i++) {
+		const value = values[i];
+		if (value === null || value === void 0) continue;
+		let index;
+		if (value instanceof Uint8Array) {
+			const hash = hashBytes(value);
+			const bucket = hashBuckets.get(hash);
+			if (bucket) {
+				for (const j of bucket) if (bytesEqual(dictionary[j], value)) {
+					index = j;
+					break;
+				}
+			}
+			if (index === void 0) {
+				dictSize += value.byteLength;
+				if (pageSize && dictSize > pageSize) return {};
+				index = dictionary.length;
+				dictionary.push(value);
+				if (bucket) bucket.push(index);
+				else hashBuckets.set(hash, [index]);
+			}
+		} else {
+			index = valueIndex.get(value);
+			if (index === void 0) {
+				dictSize += estimateValueSize(value, type, type_length);
+				if (pageSize && dictSize > pageSize) return {};
+				index = dictionary.length;
+				dictionary.push(value);
+				valueIndex.set(value, index);
+			}
+		}
+		indexes[i] = index;
+	}
+	return {
+		dictionary,
+		indexes
+	};
+}
+function writeDictionaryPage(writer, column, dictionary) {
+	const { element, codec, compressors } = column;
+	const { type, type_length } = element;
+	if (!type) throw new Error(`column ${column.columnName} cannot determine type`);
+	const dictionaryPage = new ByteWriter();
+	writePlain(dictionaryPage, dictionary, type, type_length);
+	const dictionaryBytes = dictionaryPage.getBytes();
+	const compressedBytes = compressors[codec]?.(dictionaryBytes) ?? dictionaryBytes;
+	writePageHeader(writer, {
+		type: "DICTIONARY_PAGE",
+		uncompressed_page_size: dictionaryBytes.byteLength,
+		compressed_page_size: compressedBytes.byteLength,
+		dictionary_page_header: {
+			num_values: dictionary.length,
+			encoding: "PLAIN"
+		}
+	});
+	writer.appendBytes(compressedBytes);
+}
+export { estimateValueSize, useDictionary, writeDictionaryPage };