npm - @gscdump/engine - Versions diffs - 3.4.0 → 3.4.2 - Mend

@gscdump/engine 3.4.0 → 3.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (240) hide show

package/dist/THIRD-PARTY-LICENSES.md CHANGED Viewed

@@ -35,11 +35,11 @@ Repository: https://github.com/101arrowz/fzstd
 ---------------------------------------
-## hyparquet, hyparquet, hyparquet, hyparquet-compressors, hyparquet-writer, icebird, squirreling
+## hyparquet, hyparquet-compressors, hyparquet-writer, icebird, squirreling
 License: MIT
 By: Hyperparam
-Repositories: https://github.com/hyparam/hyparquet, https://github.com/hyparam/hyparquet, https://github.com/hyparam/hyparquet, https://github.com/hyparam/hyparquet-compressors, https://github.com/hyparam/hyparquet-writer, https://github.com/hyparam/icebird, https://github.com/hyparam/squirreling
+Repositories: https://github.com/hyparam/hyparquet, https://github.com/hyparam/hyparquet-compressors, https://github.com/hyparam/hyparquet-writer, https://github.com/hyparam/icebird, https://github.com/hyparam/squirreling
 > The MIT License (MIT)
 >

package/dist/adapters/hyparquet.d.mts CHANGED Viewed

@@ -37,8 +37,10 @@ interface DecodeParquetOptions {
   /**
    * Project a subset of columns. hyparquet only fetches + decodes the named
    * column chunks, so a read that needs 2 of 14 columns skips the other 12's
-   * pages entirely. Omit to read every column. Names not present in the file
-   * are ignored by the reader.
+   * pages entirely. Omit to read every column. A name the file lacks is
+   * dropped from the projection before the read, so the decoded rows simply
+   * lack that key. That is what lets one projection span files written before
+   * a column existed (union-by-name); hyparquet itself throws on such a name.
    */
   columns?: readonly string[];
 }

package/dist/adapters/hyparquet.mjs CHANGED Viewed

@@ -1,10 +1,11 @@
 import { TABLE_METADATA } from "../drizzle-schema.mjs";
 import { SCHEMAS } from "../schema.mjs";
+import { parquetMetadataAsync, parquetSchema } from "../node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/metadata.mjs";
 import { parquetReadObjects } from "../node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/read.mjs";
 import "../node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/node.mjs";
-import { ByteWriter } from "../node_modules/.pnpm/hyparquet-writer@0.16.6_patch_hash_549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/src/bytewriter.mjs";
-import { parquetWriteRows } from "../node_modules/.pnpm/hyparquet-writer@0.16.6_patch_hash_549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/src/write-rows.mjs";
-import "../node_modules/.pnpm/hyparquet-writer@0.16.6_patch_hash_549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/src/node.mjs";
+import { ByteWriter } from "../node_modules/.pnpm/hyparquet-writer@0.16.9_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/bytewriter.mjs";
+import { parquetWriteRows } from "../node_modules/.pnpm/hyparquet-writer@0.16.9_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/write-rows.mjs";
+import "../node_modules/.pnpm/hyparquet-writer@0.16.9_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/node.mjs";
 const ROW_GROUP_SIZE = 25e3;
 const DEFAULT_COMPACTION_READ_CONCURRENCY = 8;
 const MAX_COMPACTION_READ_CONCURRENCY = 32;
@@ -189,12 +190,17 @@ function asyncBufferFromBytes(bytes) {
 }
 async function decodeParquetToRows(bytes, opts = {}) {
 	if (bytes.byteLength === 0) return [];
+	const file = asyncBufferFromBytes(bytes);
+	const projection = opts.columns ? await presentColumns(file, opts.columns) : null;
 	return normalizeDecodedDates(await parquetReadObjects({
-		file: asyncBufferFromBytes(bytes),
+		file,
 		...opts.rowStart === void 0 ? {} : { rowStart: opts.rowStart },
 		...opts.rowEnd === void 0 ? {} : { rowEnd: opts.rowEnd },
 		...opts.rowStart !== void 0 || opts.rowEnd !== void 0 ? { useOffsetIndex: true } : {},
-		...opts.columns ? { columns: [...opts.columns] } : {},
+		...projection ? {
+			metadata: projection.metadata,
+			columns: projection.columns
+		} : {},
 		...opts.filter ? {
 			filter: opts.filter,
 			useBloomFilters: true,
@@ -202,6 +208,14 @@ async function decodeParquetToRows(bytes, opts = {}) {
 		} : {}
 	}));
 }
+async function presentColumns(file, requested) {
+	const metadata = await parquetMetadataAsync(file);
+	const present = new Set(parquetSchema(metadata).children.map((child) => child.element.name));
+	return {
+		metadata,
+		columns: requested.filter((name) => present.has(name))
+	};
+}
 function normalizeDecodedDates(rows) {
 	if (rows.length === 0) return rows;
 	const dateCols = [];

package/dist/iceberg/catalog.d.mts CHANGED Viewed

@@ -1,7 +1,7 @@
 import { EngineError } from "../errors.mjs";
 import { IcebergTableName, PartitionKeyEncoding } from "./schema.mjs";
-import { icebergCreateTable } from "../node_modules/.pnpm/icebird@0.8.26_patch_hash_75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244/node_modules/icebird/types/write/write.mjs";
-import "../node_modules/.pnpm/icebird@0.8.26_patch_hash_75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244/node_modules/icebird/types/index.mjs";
+import { icebergCreateTable } from "../node_modules/.pnpm/icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/types/write/write.mjs";
+import "../node_modules/.pnpm/icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/types/index.mjs";
 import { Result } from "gscdump/result";
 import { CommitRetryOptions, ConnectIcebergOptions, IcebergCatalogConfig, IcebergConnection, IcebergListedDataFile, QueryProfiler } from "@gscdump/lakehouse";
 type IcebirdCreateTableOptions = Parameters<typeof icebergCreateTable>[0];

package/dist/iceberg/catalog.mjs CHANGED Viewed

@@ -2,8 +2,8 @@ import { engineErrors } from "../errors.mjs";
 import { TABLE_METADATA } from "../drizzle-schema.mjs";
 import "../schema.mjs";
 import { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_PARTITION_SPEC, ICEBERG_TABLES, icebergSchemasFor } from "./schema.mjs";
-import { icebergCreateTable } from "../node_modules/.pnpm/icebird@0.8.26_patch_hash_75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244/node_modules/icebird/src/write/write.mjs";
-import "../node_modules/.pnpm/icebird@0.8.26_patch_hash_75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244/node_modules/icebird/src/index.mjs";
+import { icebergCreateTable } from "../node_modules/.pnpm/icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/src/write/write.mjs";
+import "../node_modules/.pnpm/icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/src/index.mjs";
 import { err, ok } from "gscdump/result";
 import { ICEBERG_TYPE_MAP, connectIcebergCatalog, dropIcebergTables, resolveIcebergDataFiles } from "@gscdump/lakehouse";
 import { icebergAppendRetrying } from "@gscdump/lakehouse/unsafe-raw";

package/dist/node_modules/.pnpm/{hyparquet-writer@0.16.6_patch_hash_549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a → hyparquet-writer@0.16.9_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f}/node_modules/hyparquet-writer/src/bloom.mjs RENAMED Viewed

@@ -1,4 +1,4 @@
-import { hashParquetValue } from "../../../../hyparquet@1.28.1/node_modules/hyparquet/src/bloom.mjs";
+import { hashParquetValue } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/bloom.mjs";
 import { serializeTCompactProtocol } from "./thrift.mjs";
 const SALT = new Uint32Array([
 	1203114875,

@@ -22,37 +22,37 @@ var ByteWriter = class {
 		return new Uint8Array(this.buffer, 0, this.index);
 	}
 	appendUint8(value) {
-		this.ensure(this.index + 1);
+		this.ensure(1);
 		this.view.setUint8(this.index, value);
 		this.offset++;
 		this.index++;
 	}
 	appendUint32(value) {
-		this.ensure(this.index + 4);
+		this.ensure(4);
 		this.view.setUint32(this.index, value, true);
 		this.offset += 4;
 		this.index += 4;
 	}
 	appendInt32(value) {
-		this.ensure(this.index + 4);
+		this.ensure(4);
 		this.view.setInt32(this.index, value, true);
 		this.offset += 4;
 		this.index += 4;
 	}
 	appendInt64(value) {
-		this.ensure(this.index + 8);
+		this.ensure(8);
 		this.view.setBigInt64(this.index, BigInt(value), true);
 		this.offset += 8;
 		this.index += 8;
 	}
 	appendFloat32(value) {
-		this.ensure(this.index + 8);
+		this.ensure(4);
 		this.view.setFloat32(this.index, value, true);
 		this.offset += 4;
 		this.index += 4;
 	}
 	appendFloat64(value) {
-		this.ensure(this.index + 8);
+		this.ensure(8);
 		this.view.setFloat64(this.index, value, true);
 		this.offset += 8;
 		this.index += 8;
@@ -61,7 +61,7 @@ var ByteWriter = class {
 		this.appendBytes(new Uint8Array(value));
 	}
 	appendBytes(value) {
-		this.ensure(this.index + value.length);
+		this.ensure(value.length);
 		new Uint8Array(this.buffer, this.index, value.length).set(value);
 		this.offset += value.length;
 		this.index += value.length;

@@ -11,7 +11,7 @@ function writeColumn({ writer, column, pageData }) {
 	const offsetStart = writer.offset;
 	const encodings = [];
 	const isGeospatial = element?.logical_type?.type === "GEOMETRY" || element?.logical_type?.type === "GEOGRAPHY";
-	const statistics = stats ? getStatistics(values) : void 0;
+	const statistics = stats ? getStatistics(values, element) : void 0;
 	const geospatial_statistics = stats && isGeospatial ? geospatialStatistics(values) : void 0;
 	let bloomFilter;
 	if (column.bloomFilter) {
@@ -70,7 +70,7 @@ function writeColumn({ writer, column, pageData }) {
 			pageData: pageChunk
 		});
 		if (columnIndex) {
-			const { min_value, max_value, null_count = 0n } = getStatistics(values.slice(start, end));
+			const { min_value, max_value, null_count = 0n } = getStatistics(values.slice(start, end), element);
 			columnIndex.null_pages.push(null_count === BigInt(end - start));
 			columnIndex.min_values.push(unconvertMinMax(min_value, element, false) ?? /* @__PURE__ */ new Uint8Array());
 			columnIndex.max_values.push(unconvertMinMax(max_value, element, true) ?? /* @__PURE__ */ new Uint8Array());
@@ -167,7 +167,7 @@ function getPageBoundaries(values, type, type_length, pageSize) {
 	});
 	return boundaries;
 }
-function getStatistics(values) {
+function getStatistics(values, element) {
 	let min_value = void 0;
 	let max_value = void 0;
 	let null_count = 0n;
@@ -178,8 +178,9 @@ function getStatistics(values) {
 		}
 		if (typeof value === "object" && !(value instanceof Uint8Array)) continue;
 		if (typeof value === "number" && Number.isNaN(value)) continue;
-		if (min_value === void 0 || compareValues(value, min_value) < 0) min_value = value;
-		if (max_value === void 0 || compareValues(value, max_value) > 0) max_value = value;
+		const statisticValue = element.converted_type === "DECIMAL" && typeof value === "number" ? BigInt(Math.round(value * 10 ** (element.scale || 0))) : value;
+		if (min_value === void 0 || compareValues(statisticValue, min_value) < 0) min_value = statisticValue;
+		if (max_value === void 0 || compareValues(statisticValue, max_value) > 0) max_value = statisticValue;
 	}
 	if (min_value === 0) min_value = -0;
 	if (max_value === 0) max_value = 0;

@@ -1,6 +1,6 @@
+import { Encodings, PageTypes } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/constants.mjs";
 import { ByteWriter } from "./bytewriter.mjs";
 import { serializeTCompactProtocol } from "./thrift.mjs";
-import { Encodings, PageTypes } from "../../../../hyparquet@1.28.1/node_modules/hyparquet/src/constants.mjs";
 import { deltaBinaryPack, deltaByteArray, deltaLengthByteArray } from "./delta.mjs";
 import { writeRleBitPackedHybrid } from "./encoding.mjs";
 import { writePlain } from "./plain.mjs";

@@ -1,4 +1,4 @@
-import { getMaxDefinitionLevel, isListLike, isMapLike } from "../../../../hyparquet@1.28.1/node_modules/hyparquet/src/schema.mjs";
+import { getMaxDefinitionLevel, isListLike, isMapLike } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/schema.mjs";
 function encodeNestedValues(treePath, rows) {
 	const schemaPath = treePath.map((n) => n.element);
 	if (treePath.length < 2) throw new Error("parquet schema path must include column");

@@ -1,5 +1,5 @@
+import { BoundaryOrders } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/constants.mjs";
 import { serializeTCompactProtocol } from "./thrift.mjs";
-import { BoundaryOrders } from "../../../../hyparquet@1.28.1/node_modules/hyparquet/src/constants.mjs";
 function writeIndexes(writer, pageIndexes) {
 	for (const { chunk, columnIndex } of pageIndexes) writeColumnIndex(writer, chunk, columnIndex);
 	for (const { chunk, offsetIndex } of pageIndexes) writeOffsetIndex(writer, chunk, offsetIndex);

@@ -1,6 +1,6 @@
-import { getSchemaPath } from "../../../../hyparquet@1.28.1/node_modules/hyparquet/src/schema.mjs";
+import { CompressionCodecs, ConvertedTypes, EdgeInterpolationAlgorithms, Encodings, FieldRepetitionTypes, PageTypes, ParquetTypes } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/constants.mjs";
+import { getSchemaPath } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/schema.mjs";
 import { ThriftStruct, serializeTCompactProtocol } from "./thrift.mjs";
-import { CompressionCodecs, ConvertedTypes, EdgeInterpolationAlgorithms, Encodings, FieldRepetitionTypes, PageTypes, ParquetTypes } from "../../../../hyparquet@1.28.1/node_modules/hyparquet/src/constants.mjs";
 import { unconvertStatistics } from "./unconvert.mjs";
 function writeMetadata(writer, metadata) {
 	const compact = {

@@ -1,4 +1,4 @@
-import { getSchemaPath } from "../../../../hyparquet@1.28.1/node_modules/hyparquet/src/schema.mjs";
+import { getSchemaPath } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/schema.mjs";
 import { writeBlooms } from "./bloom.mjs";
 import { encodeVariantColumn } from "./variant.mjs";
 import { writeColumn } from "./column.mjs";

@@ -1,20 +1,15 @@
 import { ByteWriter } from "./bytewriter.mjs";
-const BLOCK_SIZE = 65536;
 const MAX_HASH_TABLE_BITS = 14;
+const MAX_OFFSET = 65535;
 const globalHashTables = new Array(15);
 function snappyCompress(input) {
 	const writer = new ByteWriter();
 	writer.appendVarInt(input.length);
-	let pos = 0;
-	while (pos < input.length) {
-		const fragmentSize = Math.min(input.length - pos, BLOCK_SIZE);
-		compressFragment(writer, input, pos, fragmentSize);
-		pos += fragmentSize;
-	}
+	compressInput(writer, input);
 	return writer.getBytes();
 }
 function hashFunc(key, hashFuncShift) {
-	return key * 506832829 >>> hashFuncShift;
+	return Math.imul(key, 506832829) >>> hashFuncShift;
 }
 function load32(array, pos) {
 	return array[pos] + (array[pos + 1] << 8) + (array[pos + 2] << 16) + (array[pos + 3] << 24);
@@ -24,13 +19,11 @@ function equals32(array, pos1, pos2) {
 }
 function emitLiteral(writer, input, ip, len) {
 	if (len <= 60) writer.appendUint8(len - 1 << 2);
-	else if (len < 256) {
-		writer.appendUint8(240);
-		writer.appendUint8(len - 1);
-	} else {
-		writer.appendUint8(244);
-		writer.appendUint8(len - 1 & 255);
-		writer.appendUint8(len - 1 >>> 8);
+	else {
+		const value = len - 1;
+		const bytes = value < 256 ? 1 : value < 65536 ? 2 : value < 16777216 ? 3 : 4;
+		writer.appendUint8(59 + bytes << 2);
+		for (let i = 0; i < bytes; i++) writer.appendUint8(value >>> i * 8);
 	}
 	writer.appendBytes(input.subarray(ip, ip + len));
 }
@@ -55,18 +48,19 @@ function emitCopy(writer, offset, len) {
 	}
 	emitCopyLessThan64(writer, offset, len);
 }
-function compressFragment(writer, input, ip, inputSize) {
+function compressInput(writer, input) {
+	const inputSize = input.length;
 	let hashTableBits = 1;
 	while (1 << hashTableBits <= inputSize && hashTableBits <= MAX_HASH_TABLE_BITS) hashTableBits++;
 	hashTableBits--;
 	const hashFuncShift = 32 - hashTableBits;
-	globalHashTables[hashTableBits] ??= new Uint16Array(1 << hashTableBits);
+	globalHashTables[hashTableBits] ??= new Uint32Array(1 << hashTableBits);
 	const hashTable = globalHashTables[hashTableBits];
 	hashTable.fill(0);
-	const ipEnd = ip + inputSize;
+	const ipEnd = inputSize;
 	let ipLimit;
-	const baseIp = ip;
-	let nextEmit = ip;
+	let ip = 0;
+	let nextEmit = 0;
 	let hash, nextHash;
 	let nextIp, candidate, skip;
 	let bytesBetweenHashLookups;
@@ -92,9 +86,9 @@ function compressFragment(writer, input, ip, inputSize) {
 					break;
 				}
 				nextHash = hashFunc(load32(input, nextIp), hashFuncShift);
-				candidate = baseIp + hashTable[hash];
-				hashTable[hash] = ip - baseIp;
-			} while (!equals32(input, ip, candidate));
+				candidate = hashTable[hash];
+				hashTable[hash] = ip;
+			} while (ip - candidate > MAX_OFFSET || !equals32(input, ip, candidate));
 			if (!flag) break;
 			emitLiteral(writer, input, nextEmit, ip - nextEmit);
 			do {
@@ -110,11 +104,11 @@ function compressFragment(writer, input, ip, inputSize) {
 					break;
 				}
 				prevHash = hashFunc(load32(input, ip - 1), hashFuncShift);
-				hashTable[prevHash] = ip - 1 - baseIp;
+				hashTable[prevHash] = ip - 1;
 				curHash = hashFunc(load32(input, ip), hashFuncShift);
-				candidate = baseIp + hashTable[curHash];
-				hashTable[curHash] = ip - baseIp;
-			} while (equals32(input, ip, candidate));
+				candidate = hashTable[curHash];
+				hashTable[curHash] = ip;
+			} while (ip - candidate <= MAX_OFFSET && equals32(input, ip, candidate));
 			if (!flag) break;
 			ip++;
 			nextHash = hashFunc(load32(input, ip), hashFuncShift);

@@ -1,4 +1,4 @@
-import "../../../../hyparquet@1.28.1/node_modules/hyparquet/types/node.mjs";
+import "../../../../hyparquet@1.29.2/node_modules/hyparquet/types/node.mjs";
 interface Writer {
   buffer: ArrayBuffer;
   view: DataView;

@@ -1,5 +1,5 @@
-import { toJson } from "../../../../hyparquet@1.28.1/node_modules/hyparquet/src/utils.mjs";
-import "../../../../hyparquet@1.28.1/node_modules/hyparquet/src/node.mjs";
+import { toJson } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/utils.mjs";
+import "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/node.mjs";
 import { geojsonToWkb } from "./wkb.mjs";
 const dayMillis = 864e5;
 function mapConverted(values, convert) {
@@ -15,9 +15,10 @@ function unconvert(element, values) {
 	const { type, converted_type: ctype, logical_type: ltype } = element;
 	if (ctype === "DECIMAL") {
 		const factor = 10 ** (element.scale || 0);
-		return values.map((v) => {
+		return Array.from(values, (v) => {
 			if (v === null || v === void 0) return v;
-			if (typeof v !== "number") throw new Error("DECIMAL must be a number");
+			if (typeof v === "bigint") return unconvertDecimal(element, v);
+			if (typeof v !== "number") throw new Error("DECIMAL must be a number or bigint");
 			return unconvertDecimal(element, BigInt(Math.round(v * factor)));
 		});
 	}
@@ -120,6 +121,7 @@ function minMaxIsExact(value, element) {
 	const { type } = element;
 	if (type !== "BYTE_ARRAY" && type !== "FIXED_LEN_BYTE_ARRAY") return void 0;
 	if (element.logical_type?.type === "UUID") return void 0;
+	if (element.converted_type === "DECIMAL") return void 0;
 	return (value instanceof Uint8Array ? value : new TextEncoder().encode(value.toString())).length > STATS_TRUNCATE_LENGTH ? false : void 0;
 }
 function unconvertMinMax(value, element, isMax) {
@@ -128,13 +130,13 @@ function unconvertMinMax(value, element, isMax) {
 	if (type === "BOOLEAN") return new Uint8Array([value ? 1 : 0]);
 	if (element.logical_type?.type === "UUID" && (typeof value === "string" || value instanceof Uint8Array)) return unconvertUuid(value);
 	if (converted_type === "DECIMAL") {
-		if (typeof value !== "number") throw new Error("DECIMAL must be a number");
+		if (typeof value !== "number" && typeof value !== "bigint") throw new Error("DECIMAL must be a number or bigint");
 		const factor = 10 ** (element.scale || 0);
-		const out = unconvertDecimal(element, BigInt(Math.round(value * factor)));
+		const out = unconvertDecimal(element, typeof value === "bigint" ? value : BigInt(Math.round(value * factor)));
 		if (out instanceof Uint8Array) return out;
 		if (typeof out === "number") {
 			const buffer = /* @__PURE__ */ new ArrayBuffer(4);
-			new DataView(buffer).setFloat32(0, out, true);
+			new DataView(buffer).setInt32(0, out, true);
 			return new Uint8Array(buffer);
 		}
 		if (typeof out === "bigint") {

@@ -10,6 +10,7 @@ function encodeVariantColumn(values, shredding, column) {
 	if (column?.required) {
 		for (let i = 0; i < values.length; i++) if (values[i] === void 0) throw new Error(`required variant column ${column.name} has undefined value at index ${i}`);
 	}
+	const scratchWriters = [];
 	const shreddingConfig = shredding && normalizeShreddingConfig(shredding);
 	if (shreddingConfig) {
 		const metadataCache = /* @__PURE__ */ new Map();
@@ -20,7 +21,7 @@ function encodeVariantColumn(values, shredding, column) {
 			const { metadata, keyIndex } = getVariantRowMetadata(keys, metadataCache);
 			return {
 				metadata,
-				...encodeShredded(value, shreddingConfig, keyIndex, true)
+				...encodeShredded(value, shreddingConfig, keyIndex, true, scratchWriters)
 			};
 		});
 	}
@@ -32,29 +33,29 @@ function encodeVariantColumn(values, shredding, column) {
 		if (value === void 0) return null;
 		return {
 			metadata,
-			value: writeVariantValue(value, keyIndex)
+			value: writeVariantValue(value, keyIndex, scratchWriters)
 		};
 	});
 }
-function encodeShredded(value, shredType, keyIndex, allowPartialObjects) {
+function encodeShredded(value, shredType, keyIndex, allowPartialObjects, scratchWriters) {
 	if (value === null || value === void 0) return {
 		value: VARIANT_NULL,
 		typed_value: null
 	};
 	if (Array.isArray(shredType)) {
 		if (!Array.isArray(value)) return {
-			value: writeVariantValue(value, keyIndex),
+			value: writeVariantValue(value, keyIndex, scratchWriters),
 			typed_value: null
 		};
 		const elemShred = shredType[0];
 		return {
 			value: null,
-			typed_value: value.map((el) => encodeShredded(el, elemShred, keyIndex, false))
+			typed_value: value.map((el) => encodeShredded(el, elemShred, keyIndex, false, scratchWriters))
 		};
 	}
 	if (typeof shredType === "object") {
 		if (typeof value !== "object" || Array.isArray(value) || value instanceof Date || value instanceof Uint8Array) return {
-			value: writeVariantValue(value, keyIndex),
+			value: writeVariantValue(value, keyIndex, scratchWriters),
 			typed_value: null
 		};
 		const remaining = {};
@@ -65,21 +66,21 @@ function encodeShredded(value, shredType, keyIndex, allowPartialObjects) {
 			hasRemaining = true;
 		}
 		if (hasRemaining && !allowPartialObjects) return {
-			value: writeVariantValue(value, keyIndex),
+			value: writeVariantValue(value, keyIndex, scratchWriters),
 			typed_value: null
 		};
 		const fieldNames = Object.keys(shredType);
 		if (fieldNames.some((fieldName) => (!Object.prototype.hasOwnProperty.call(value, fieldName) || value[fieldName] === void 0) && keyIndex.has(fieldName))) return {
-			value: writeVariantValue(value, keyIndex),
+			value: writeVariantValue(value, keyIndex, scratchWriters),
 			typed_value: null
 		};
 		const typedValue = {};
 		for (const fieldName of fieldNames) {
 			if (!Object.prototype.hasOwnProperty.call(value, fieldName) || value[fieldName] === void 0) continue;
-			typedValue[fieldName] = encodeShredded(value[fieldName], shredType[fieldName], keyIndex, false);
+			typedValue[fieldName] = encodeShredded(value[fieldName], shredType[fieldName], keyIndex, false, scratchWriters);
 		}
 		return {
-			value: hasRemaining ? writeVariantValue(remaining, keyIndex) : null,
+			value: hasRemaining ? writeVariantValue(remaining, keyIndex, scratchWriters) : null,
 			typed_value: typedValue
 		};
 	}
@@ -88,7 +89,7 @@ function encodeShredded(value, shredType, keyIndex, allowPartialObjects) {
 		typed_value: value
 	};
 	return {
-		value: writeVariantValue(value, keyIndex),
+		value: writeVariantValue(value, keyIndex, scratchWriters),
 		typed_value: null
 	};
 }
@@ -185,12 +186,12 @@ function writeVariantMetadata(dictionary) {
 	}
 	return bytes;
 }
-function writeVariantValue(value, keyIndex) {
+function writeVariantValue(value, keyIndex, scratchWriters) {
 	const writer = new ByteWriter(8);
-	writeValue(value, writer, keyIndex);
+	writeValue(value, writer, keyIndex, scratchWriters);
 	return writer.getBytes();
 }
-function writeValue(val, writer, keyIndex) {
+function writeValue(val, writer, keyIndex, scratchWriters) {
 	if (val === null || val === void 0) {
 		writer.appendUint8(0);
 		return;
@@ -255,16 +256,16 @@ function writeValue(val, writer, keyIndex) {
 		return;
 	}
 	if (Array.isArray(val)) {
-		writeVariantArray(val, writer, keyIndex);
+		writeVariantArray(val, writer, keyIndex, scratchWriters);
 		return;
 	}
 	if (typeof val === "object") {
-		writeVariantObject(val, writer, keyIndex);
+		writeVariantObject(val, writer, keyIndex, scratchWriters);
 		return;
 	}
 	throw new Error(`variant cannot encode value: ${val}`);
 }
-function writeVariantObject(obj, writer, keyIndex) {
+function writeVariantObject(obj, writer, keyIndex, scratchWriters) {
 	const entries = Object.keys(obj).filter((key) => obj[key] !== void 0).map((key) => {
 		const id = keyIndex.get(key);
 		if (id === void 0) throw new Error(`variant key not in dictionary: ${key}`);
@@ -276,11 +277,11 @@ function writeVariantObject(obj, writer, keyIndex) {
 	entries.sort((a, b) => a.id - b.id);
 	const numElements = entries.length;
 	const idWidth = byteWidth(numElements > 0 ? entries[numElements - 1].id : 0);
-	const scratch = new ByteWriter(8);
+	const scratch = scratchWriters.pop() ?? new ByteWriter(8);
 	const offsets = new Array(numElements + 1);
 	offsets[0] = 0;
 	for (let i = 0; i < numElements; i++) {
-		writeValue(obj[entries[i].key], scratch, keyIndex);
+		writeValue(obj[entries[i].key], scratch, keyIndex, scratchWriters);
 		offsets[i + 1] = scratch.index;
 	}
 	const offsetWidth = byteWidth(offsets[numElements]);
@@ -291,14 +292,17 @@ function writeVariantObject(obj, writer, keyIndex) {
 	for (const { id } of entries) appendUnsignedLE(writer, id, idWidth);
 	for (const off of offsets) appendUnsignedLE(writer, off, offsetWidth);
 	writer.appendBytes(scratch.getBytes());
+	scratch.index = 0;
+	scratch.offset = 0;
+	scratchWriters.push(scratch);
 }
-function writeVariantArray(arr, writer, keyIndex) {
+function writeVariantArray(arr, writer, keyIndex, scratchWriters) {
 	const numElements = arr.length;
-	const scratch = new ByteWriter(8);
+	const scratch = scratchWriters.pop() ?? new ByteWriter(8);
 	const offsets = new Array(numElements + 1);
 	offsets[0] = 0;
 	for (let i = 0; i < numElements; i++) {
-		writeValue(arr[i], scratch, keyIndex);
+		writeValue(arr[i], scratch, keyIndex, scratchWriters);
 		offsets[i + 1] = scratch.index;
 	}
 	const offsetWidth = byteWidth(offsets[numElements]);
@@ -308,6 +312,9 @@ function writeVariantArray(arr, writer, keyIndex) {
 	else writer.appendUint8(numElements);
 	for (const off of offsets) appendUnsignedLE(writer, off, offsetWidth);
 	writer.appendBytes(scratch.getBytes());
+	scratch.index = 0;
+	scratch.offset = 0;
+	scratchWriters.push(scratch);
 }
 function byteWidth(maxValue) {
 	if (maxValue <= 255) return 1;

@@ -1,4 +1,4 @@
-import "../../../../hyparquet@1.28.1/node_modules/hyparquet/types/node.mjs";
+import "../../../../hyparquet@1.29.2/node_modules/hyparquet/types/node.mjs";
 import { Writer as Writer$1 } from "../src/types.mjs";
 import "./write.mjs";
 import "./write-rows.mjs";

package/dist/node_modules/.pnpm/hyparquet-writer@0.16.9_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/types/parquet-writer.d.mts ADDED Viewed

	@@ -0,0 +1,2 @@
1	+ import "../../../../hyparquet@1.29.2/node_modules/hyparquet/types/node.mjs";
2	+ import "../src/types.mjs";

package/dist/node_modules/.pnpm/hyparquet-writer@0.16.9_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/types/schema.d.mts ADDED Viewed

	@@ -0,0 +1,2 @@
1	+ import "../../../../hyparquet@1.29.2/node_modules/hyparquet/types/node.mjs";
2	+ import "../src/types.mjs";

package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/index.mjs CHANGED Viewed

@@ -1,8 +1,8 @@
-import "./metadata.mjs";
+import { parquetMetadataAsync, parquetSchema } from "./metadata.mjs";
 import "./indexes.mjs";
-import "./utils.mjs";
+import { toJson } from "./utils.mjs";
 import "./snappy.mjs";
 import "./scan.mjs";
 import { parquetReadObjects } from "./read.mjs";
 import "./query.mjs";
-export { parquetReadObjects };
+export { parquetMetadataAsync, parquetReadObjects, parquetSchema, toJson };

package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/node.mjs CHANGED Viewed

@@ -1,7 +1,7 @@
-import "./metadata.mjs";
+import { parquetMetadataAsync, parquetSchema } from "./metadata.mjs";
 import "./indexes.mjs";
-import "./utils.mjs";
+import { toJson } from "./utils.mjs";
 import "./snappy.mjs";
 import { parquetReadObjects } from "./read.mjs";
 import "./index.mjs";
-export { parquetReadObjects };
+export { parquetMetadataAsync, parquetReadObjects, parquetSchema, toJson };

package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/types.d.mts CHANGED Viewed

@@ -23,4 +23,12 @@ type ParquetQueryOperator = {
   $nin?: ParquetQueryValue[];
   $not?: ParquetQueryOperator;
 };
-export { ParquetQueryFilter, ParquetQueryOperator, ParquetQueryValue };
+/**
+ * File-like object that can read slices of a file asynchronously.
+ */
+interface AsyncBuffer {
+  byteLength: number;
+  slice(start: number, end?: number): Awaitable<ArrayBuffer>;
+}
+type Awaitable<T> = T | Promise<T>;
+export { AsyncBuffer, Awaitable, ParquetQueryFilter, ParquetQueryOperator, ParquetQueryValue };