@gscdump/engine 7.5.1 → 7.7.0

package/README.md

@@ -42,6 +42,19 @@ options produces identical bytes regardless of producer batch boundaries.
 This supports a counting/digest pass followed by a separately coordinated
 fixed-length upload, without the codec owning transport or commit semantics.
 
+The streaming encoder normally refuses when retained footer/index descriptors
+exceed `limits.maxRetainedMetadataBytes` (8 MiB by default). For wide values,
+callers can explicitly choose `metadataOverflow: 'omit-statistics'`. On the
+first overflow it omits column-chunk statistics and ColumnIndexes throughout
+that file, including earlier groups. OffsetIndexes, encoding descriptors,
+schema, row counts, data/dictionary pages and explicit `fileMetadata` remain
+unchanged. Required descriptors must still fit the same budget, and the
+group/footer output-buffer limit still applies. Missing statistics mean unknown
+bounds, so readers may scan more pages or groups; they do not mean empty data.
+If no overflow occurs, this option preserves the normal bytes. It bounds
+retained optional descriptors, not their transient construction or total heap.
+The array encoders and all omitted-option behavior remain unchanged.
+
 `decodeParquetRowGroups({ byteLength, slice }, options)` reads exact byte ranges
 and by default yields one physical row group at a time. The caller must pin every range
 to the same immutable object or ETag. It preserves the existing date, column

package/dist/adapters/hyparquet.d.mts

@@ -29,6 +29,13 @@ export interface EncodeParquetBatchesOptions extends EncodeParquetOptions {
   columns: readonly ColumnDef[];
   /** Maximum rows per output group. Default 25000, matching the array codec. */
   rowGroupSize?: number;
+  /**
+   * Opt in to omitting all footer column statistics and ColumnIndexes if they
+   * exceed the retained metadata limit. Default: refuse. OffsetIndexes,
+   * required descriptors and explicit file metadata remain bounded/preserved.
+   * Missing statistics reduce pruning; this does not bound transient codec heap.
+   */
+  metadataOverflow?: 'omit-statistics';
   limits?: {
     /** Refuse a producer batch with more rows. Default 25000. */
     maxBatchRows?: number;

package/dist/adapters/hyparquet.mjs

@@ -219,6 +219,11 @@ function streamLimit(value, fallback, name) {
 function scalarBytes(value) {
 	return typeof value === "string" ? value.length * 3 + 8 : 16;
 }
+var RetainedMetadataLimitError = class extends RangeError {
+	constructor() {
+		super("parquet stream exceeds maxRetainedMetadataBytes");
+	}
+};
 function retainedDescriptorBytes(value, remaining) {
 	const seen = /* @__PURE__ */ new Set();
 	let bytes = 0;
@@ -233,12 +238,14 @@ function retainedDescriptorBytes(value, remaining) {
 				visit(value[key]);
 			}
 		} else bytes += scalarBytes(value);
-		if (bytes > remaining) throw new RangeError("parquet stream exceeds maxRetainedMetadataBytes");
+		if (bytes > remaining) throw new RetainedMetadataLimitError();
 	};
 	visit(value);
 	return bytes;
 }
 async function encodeParquetRowBatches(batches, opts, write) {
+	const metadataOverflow = opts.metadataOverflow;
+	if (metadataOverflow !== void 0 && metadataOverflow !== "omit-statistics") throw new RangeError("parquet stream metadataOverflow must be omit-statistics when provided");
 	const rowGroupSize = streamLimit(opts.rowGroupSize, ROW_GROUP_SIZE, "rowGroupSize");
 	const maxBatchRows = streamLimit(opts.limits?.maxBatchRows, ROW_GROUP_SIZE, "maxBatchRows");
 	const maxBatchBytes = streamLimit(opts.limits?.maxBatchBytes, 8388608, "maxBatchBytes");
@@ -260,7 +267,8 @@ async function encodeParquetRowBatches(batches, opts, write) {
 		metadataBytes += scalarBytes(key) + scalarBytes(opts.fileMetadata[key]);
 		if (metadataBytes > maxOutputBufferBytes) throw new RangeError("parquet stream file metadata exceeds maxOutputBufferBytes");
 	}
-	let retainedMetadataBytes = retainedDescriptorBytes([schema, opts.fileMetadata], maxRetainedMetadataBytes);
+	const initialMetadataBytes = retainedDescriptorBytes([schema, opts.fileMetadata], maxRetainedMetadataBytes);
+	let retainedMetadataBytes = initialMetadataBytes;
 	const writer = new ByteWriter(Math.min(65536, maxOutputBufferBytes));
 	writer.ensure = (size) => {
 		const required = writer.index + size;
@@ -293,6 +301,14 @@ async function encodeParquetRowBatches(batches, opts, write) {
 	let groupBytes = 0;
 	let groups = 0;
 	let rowCount = 0;
+	let omitStatistics = false;
+	const omitOptionalMetadata = (firstIndex) => {
+		for (let index = firstIndex; index < parquet.pendingIndexes.length; index++) {
+			const pending = parquet.pendingIndexes[index];
+			if (pending.chunk.meta_data) delete pending.chunk.meta_data.statistics;
+			delete pending.columnIndex;
+		}
+	};
 	const flushGroup = async () => {
 		if (!groupRows) return;
 		if (++groups > maxRowGroups) throw new RangeError("parquet stream exceeds maxRowGroups");
@@ -306,7 +322,21 @@ async function encodeParquetRowBatches(batches, opts, write) {
 			})),
 			rowGroupSize: groupRows
 		});
-		retainedMetadataBytes += retainedDescriptorBytes([parquet.row_groups[parquet.row_groups.length - 1], parquet.pendingIndexes.slice(firstIndex)], maxRetainedMetadataBytes - retainedMetadataBytes);
+		if (omitStatistics) omitOptionalMetadata(firstIndex);
+		try {
+			retainedMetadataBytes += retainedDescriptorBytes([parquet.row_groups[parquet.row_groups.length - 1], parquet.pendingIndexes.slice(firstIndex)], maxRetainedMetadataBytes - retainedMetadataBytes);
+		} catch (error) {
+			if (!(error instanceof RetainedMetadataLimitError) || metadataOverflow !== "omit-statistics" || omitStatistics) throw error;
+			omitStatistics = true;
+			omitOptionalMetadata(0);
+			retainedMetadataBytes = initialMetadataBytes;
+			let index = 0;
+			for (const group of parquet.row_groups) {
+				const end = index + group.columns.length;
+				retainedMetadataBytes += retainedDescriptorBytes([group, parquet.pendingIndexes.slice(index, end)], maxRetainedMetadataBytes - retainedMetadataBytes);
+				index = end;
+			}
+		}
 		values = codecs.map(() => []);
 		groupRows = 0;
 		groupBytes = 0;

package/dist/iceberg/catalog.d.mts

@@ -1,7 +1,7 @@
 import { EngineError } from "../errors.mjs";
 import { IcebergTableName, PartitionKeyEncoding } from "./schema.mjs";
-import { icebergCreateTable } from "../node_modules/.pnpm/icebird@0.8.30_patch_hash_a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/types/write/write.mjs";
-import "../node_modules/.pnpm/icebird@0.8.30_patch_hash_a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/types/index.mjs";
+import { icebergCreateTable } from "../node_modules/.pnpm/icebird@0.8.30_patch_hash_1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/write/write.mjs";
+import "../node_modules/.pnpm/icebird@0.8.30_patch_hash_1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/index.mjs";
 import { Result } from "gscdump/result";
 import { CommitRetryOptions, ConnectIcebergOptions, IcebergCatalogConfig, IcebergConnection, IcebergListedDataFile, QueryProfiler } from "@gscdump/lakehouse";
 type IcebirdCreateTableOptions = Parameters<typeof icebergCreateTable>[0];

package/dist/iceberg/catalog.mjs

@@ -2,8 +2,8 @@ import { engineErrors } from "../errors.mjs";
 import { TABLE_METADATA } from "../drizzle-schema.mjs";
 import "../schema.mjs";
 import { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_PARTITION_SPEC, ICEBERG_TABLES, icebergSchemasFor } from "./schema.mjs";
-import { icebergCreateTable } from "../node_modules/.pnpm/icebird@0.8.30_patch_hash_a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/write/write.mjs";
-import "../node_modules/.pnpm/icebird@0.8.30_patch_hash_a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/index.mjs";
+import { icebergCreateTable } from "../node_modules/.pnpm/icebird@0.8.30_patch_hash_1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/write/write.mjs";
+import "../node_modules/.pnpm/icebird@0.8.30_patch_hash_1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/index.mjs";
 import { err, ok } from "gscdump/result";
 import { ICEBERG_TYPE_MAP, connectIcebergCatalog, dropIcebergTables, resolveIcebergDataFiles } from "@gscdump/lakehouse";
 import { icebergAppendRetrying } from "@gscdump/lakehouse/unsafe-raw";

package/dist/node_modules/.pnpm/{hyparquet-compressors@1.1.2 → hyparquet-compressors@1.1.2_patch_hash_821c89dbb6181f0c34d9abebd844451458a4f16c04e7aa7a19abca0c09c2238d}/node_modules/hyparquet-compressors/src/gzip.mjs

@@ -39,12 +39,15 @@ function gzipStart(input, i) {
 	for (let zs = (flag >> 3 & 1) + (flag >> 4 & 1); zs > 0; zs -= Number(!input[i++]));
 	return i + (flag & 2);
 }
-function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
-	let out = output ?? /* @__PURE__ */ new Uint8Array(1024);
+function gunzip(input, output, inputIndex = 0, outputIndex = 0, limits) {
+	const maximum = limits?.maxOutputBytes;
+	if (limits && (limits.raw !== true || !Number.isSafeInteger(maximum) || maximum < 0 || output || inputIndex !== 0 || outputIndex !== 0)) throw new RangeError("Invalid bounded raw DEFLATE options");
+	let out = output ?? new Uint8Array(limits ? Math.min(1024, maximum) : 1024);
+	if (limits && !input.length) throw new Error("unexpected EOF");
 	if (!(input.length - inputIndex)) return out;
-	const payloadStart = gzipStart(input, inputIndex);
-	if (payloadStart === input.length - 8) return out;
-	if (payloadStart > input.length - 8) throw new Error("unexpected EOF");
+	const payloadStart = limits ? 0 : gzipStart(input, inputIndex);
+	if (!limits && payloadStart === input.length - 8) return out;
+	if (!limits && payloadStart > input.length - 8) throw new Error("unexpected EOF");
 	let pos = payloadStart * 8;
 	let final = 0;
 	let lengthBits = 0;
@@ -53,9 +56,10 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 	let distMap;
 	const totalBits = input.length * 8;
 	function ensureSize(length) {
+		if (limits && (!Number.isSafeInteger(length) || length > maximum)) throw new RangeError("DEFLATE output byte limit exceeded");
 		if (!output && length > out.length) {
 			const old = out;
-			out = new Uint8Array(Math.max(old.length * 2, length));
+			out = new Uint8Array(limits ? Math.min(maximum, Math.max(old.length * 2, length)) : Math.max(old.length * 2, length));
 			out.set(old);
 		}
 	}
@@ -67,6 +71,7 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 			if (!type) {
 				const s = shft(pos) + 4;
 				const l = input[s - 4] | input[s - 3] << 8;
+				if (limits && (s > input.length || ((input[s - 2] | input[s - 1] << 8) ^ l) !== 65535)) throw new Error("invalid stored block length");
 				const t = s + l;
 				if (t > input.length) throw new Error("unexpected EOF");
 				ensureSize(outputIndex + l);
@@ -93,6 +98,7 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 				const codeLengthMap = huffMap(codeLengthTree, codeLengthBits, 1);
 				for (let i = 0; i < tl;) {
 					const r = codeLengthMap[bits(input, pos, clbMask)];
+					if (limits && !r) throw new Error("invalid code length");
 					pos += r & 15;
 					const symbol = r >> 4;
 					if (symbol < 16) lengthDistanceTree[i++] = symbol;
@@ -100,6 +106,7 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 						let copy = 0;
 						let n = 0;
 						if (symbol === 16) {
+							if (limits && i === 0) throw new Error("invalid code length repeat");
 							n = 3 + bits(input, pos, 3);
 							pos += 2;
 							copy = lengthDistanceTree[i - 1];
@@ -110,6 +117,7 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 							n = 11 + bits(input, pos, 127);
 							pos += 7;
 						}
+						if (limits && i + n > tl) throw new Error("invalid code length repeat");
 						while (n--) lengthDistanceTree[i++] = copy;
 					}
 				}
@@ -122,7 +130,7 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 			} else throw new Error("invalid block type");
 			if (pos > totalBits) throw new Error("unexpected EOF");
 		}
-		ensureSize(outputIndex + 131072);
+		ensureSize(limits ? Math.min(maximum, outputIndex + 131072) : outputIndex + 131072);
 		const lms = (1 << lengthBits) - 1;
 		const dms = (1 << distBits) - 1;
 		for (;;) {
@@ -131,8 +139,10 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 			pos += code & 15;
 			if (pos > totalBits) throw new Error("unexpected EOF");
 			if (!code) throw new Error("invalid length/literal");
-			if (sym < 256) out[outputIndex++] = sym;
-			else if (sym === 256) {
+			if (sym < 256) {
+				if (limits) ensureSize(outputIndex + 1);
+				out[outputIndex++] = sym;
+			} else if (sym === 256) {
 				lengthMap = void 0;
 				break;
 			} else {
@@ -147,6 +157,7 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 				const d = distMap[bits16(input, pos) & dms];
 				const dsym = d >> 4;
 				if (!d) throw new Error("invalid distance");
+				if (limits && dsym > 29) throw new Error("invalid distance symbol");
 				pos += d & 15;
 				let dt = fixedDistance[dsym];
 				if (dsym > 3) {
@@ -163,10 +174,12 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 		}
 		if (lengthMap) final = 1;
 	} while (!final);
-	if (outputIndex < out.length) {
+	if (limits && Math.ceil(pos / 8) !== input.length) throw new Error("trailing raw DEFLATE bytes");
+	if (!limits && outputIndex < out.length) {
 		const nextBlock = Math.ceil(pos / 8) + 8;
 		gunzip(input, out, nextBlock, outputIndex);
 	}
+	if (limits && outputIndex !== out.length) return out.slice(0, outputIndex);
 	if (!output) return out.subarray(0, outputIndex);
 	return out;
 }

package/dist/node_modules/.pnpm/{icebird@0.8.30_patch_hash_a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f → icebird@0.8.30_patch_hash_1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471}/node_modules/icebird/src/avro/avro.metadata.mjs

@@ -1,2 +1,3 @@
+import "./avro.budget.mjs";
 Array.from({ length: 128 }, (_, i) => BigInt(i >>> 1 ^ -(i & 1)));
 export {};

package/dist/node_modules/.pnpm/icebird@0.8.30_patch_hash_1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/avro/avro.read.mjs

@@ -0,0 +1,6 @@
+import "../../../../../hyparquet@1.31.1/node_modules/hyparquet/src/convert.mjs";
+import "../../../../../hyparquet-compressors@1.1.2_patch_hash_821c89dbb6181f0c34d9abebd844451458a4f16c04e7aa7a19abca0c09c2238d/node_modules/hyparquet-compressors/src/index.mjs";
+import "./avro.budget.mjs";
+import "./avro.metadata.mjs";
+new TextDecoder();
+export {};

package/dist/node_modules/.pnpm/{icebird@0.8.30_patch_hash_a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f → icebird@0.8.30_patch_hash_1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471}/node_modules/icebird/src/fetch.mjs

@@ -1,6 +1,6 @@
 import "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/node.mjs";
 import "../../../../hyparquet-writer@0.16.10_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/node.mjs";
-import "../../../../hyparquet-compressors@1.1.2/node_modules/hyparquet-compressors/src/index.mjs";
+import "../../../../hyparquet-compressors@1.1.2_patch_hash_821c89dbb6181f0c34d9abebd844451458a4f16c04e7aa7a19abca0c09c2238d/node_modules/hyparquet-compressors/src/index.mjs";
 import "./avro/avro.metadata.mjs";
 import "./avro/avro.read.mjs";
 import "./utils.mjs";

package/dist/node_modules/.pnpm/{icebird@0.8.30_patch_hash_a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f → icebird@0.8.30_patch_hash_1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471}/node_modules/icebird/src/manifest.mjs

@@ -1,3 +1,4 @@
+import "./avro/avro.budget.mjs";
 import "./avro/avro.metadata.mjs";
 import "./avro/avro.read.mjs";
 import "./fetch.mjs";

package/dist/node_modules/.pnpm/{icebird@0.8.30_patch_hash_a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f → icebird@0.8.30_patch_hash_1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471}/node_modules/icebird/src/read.mjs

@@ -2,7 +2,7 @@ import "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/schema.mjs";
 import "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/utils.mjs";
 import "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/filter.mjs";
 import "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/node.mjs";
-import "../../../../hyparquet-compressors@1.1.2/node_modules/hyparquet-compressors/src/index.mjs";
+import "../../../../hyparquet-compressors@1.1.2_patch_hash_821c89dbb6181f0c34d9abebd844451458a4f16c04e7aa7a19abca0c09c2238d/node_modules/hyparquet-compressors/src/index.mjs";
 import "./utils.mjs";
 import "./delete.mjs";
 import "./fetch.mjs";

package/dist/node_modules/.pnpm/icebird@0.8.30_patch_hash_1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/write/geospatial.mjs

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "7.5.1",
+  "version": "7.7.0",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -189,10 +189,10 @@
     }
   },
   "dependencies": {
-    "@gscdump/contracts": "^7.5.1",
-    "@gscdump/lakehouse": "^7.5.1",
+    "@gscdump/contracts": "^7.7.0",
+    "@gscdump/lakehouse": "^7.7.0",
     "drizzle-orm": "1.0.0-rc.4",
-    "gscdump": "^7.5.1",
+    "gscdump": "^7.7.0",
     "pathe": "^2.0.3",
     "proper-lockfile": "^4.1.2"
   },

package/dist/node_modules/.pnpm/icebird@0.8.30_patch_hash_a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/avro/avro.read.mjs

@@ -1,5 +0,0 @@
-import "../../../../../hyparquet@1.31.1/node_modules/hyparquet/src/convert.mjs";
-import "../../../../../hyparquet-compressors@1.1.2/node_modules/hyparquet-compressors/src/index.mjs";
-import "./avro.metadata.mjs";
-new TextDecoder();
-export {};