@gscdump/engine 7.5.1 → 7.6.0

package/README.md

@@ -42,6 +42,19 @@ options produces identical bytes regardless of producer batch boundaries.
 This supports a counting/digest pass followed by a separately coordinated
 fixed-length upload, without the codec owning transport or commit semantics.
 
+The streaming encoder normally refuses when retained footer/index descriptors
+exceed `limits.maxRetainedMetadataBytes` (8 MiB by default). For wide values,
+callers can explicitly choose `metadataOverflow: 'omit-statistics'`. On the
+first overflow it omits column-chunk statistics and ColumnIndexes throughout
+that file, including earlier groups. OffsetIndexes, encoding descriptors,
+schema, row counts, data/dictionary pages and explicit `fileMetadata` remain
+unchanged. Required descriptors must still fit the same budget, and the
+group/footer output-buffer limit still applies. Missing statistics mean unknown
+bounds, so readers may scan more pages or groups; they do not mean empty data.
+If no overflow occurs, this option preserves the normal bytes. It bounds
+retained optional descriptors, not their transient construction or total heap.
+The array encoders and all omitted-option behavior remain unchanged.
+
 `decodeParquetRowGroups({ byteLength, slice }, options)` reads exact byte ranges
 and by default yields one physical row group at a time. The caller must pin every range
 to the same immutable object or ETag. It preserves the existing date, column

package/dist/adapters/hyparquet.d.mts

@@ -29,6 +29,13 @@ export interface EncodeParquetBatchesOptions extends EncodeParquetOptions {
   columns: readonly ColumnDef[];
   /** Maximum rows per output group. Default 25000, matching the array codec. */
   rowGroupSize?: number;
+  /**
+   * Opt in to omitting all footer column statistics and ColumnIndexes if they
+   * exceed the retained metadata limit. Default: refuse. OffsetIndexes,
+   * required descriptors and explicit file metadata remain bounded/preserved.
+   * Missing statistics reduce pruning; this does not bound transient codec heap.
+   */
+  metadataOverflow?: 'omit-statistics';
   limits?: {
     /** Refuse a producer batch with more rows. Default 25000. */
     maxBatchRows?: number;

package/dist/adapters/hyparquet.mjs

@@ -219,6 +219,11 @@ function streamLimit(value, fallback, name) {
 function scalarBytes(value) {
 	return typeof value === "string" ? value.length * 3 + 8 : 16;
 }
+var RetainedMetadataLimitError = class extends RangeError {
+	constructor() {
+		super("parquet stream exceeds maxRetainedMetadataBytes");
+	}
+};
 function retainedDescriptorBytes(value, remaining) {
 	const seen = /* @__PURE__ */ new Set();
 	let bytes = 0;
@@ -233,12 +238,14 @@ function retainedDescriptorBytes(value, remaining) {
 				visit(value[key]);
 			}
 		} else bytes += scalarBytes(value);
-		if (bytes > remaining) throw new RangeError("parquet stream exceeds maxRetainedMetadataBytes");
+		if (bytes > remaining) throw new RetainedMetadataLimitError();
 	};
 	visit(value);
 	return bytes;
 }
 async function encodeParquetRowBatches(batches, opts, write) {
+	const metadataOverflow = opts.metadataOverflow;
+	if (metadataOverflow !== void 0 && metadataOverflow !== "omit-statistics") throw new RangeError("parquet stream metadataOverflow must be omit-statistics when provided");
 	const rowGroupSize = streamLimit(opts.rowGroupSize, ROW_GROUP_SIZE, "rowGroupSize");
 	const maxBatchRows = streamLimit(opts.limits?.maxBatchRows, ROW_GROUP_SIZE, "maxBatchRows");
 	const maxBatchBytes = streamLimit(opts.limits?.maxBatchBytes, 8388608, "maxBatchBytes");
@@ -260,7 +267,8 @@ async function encodeParquetRowBatches(batches, opts, write) {
 		metadataBytes += scalarBytes(key) + scalarBytes(opts.fileMetadata[key]);
 		if (metadataBytes > maxOutputBufferBytes) throw new RangeError("parquet stream file metadata exceeds maxOutputBufferBytes");
 	}
-	let retainedMetadataBytes = retainedDescriptorBytes([schema, opts.fileMetadata], maxRetainedMetadataBytes);
+	const initialMetadataBytes = retainedDescriptorBytes([schema, opts.fileMetadata], maxRetainedMetadataBytes);
+	let retainedMetadataBytes = initialMetadataBytes;
 	const writer = new ByteWriter(Math.min(65536, maxOutputBufferBytes));
 	writer.ensure = (size) => {
 		const required = writer.index + size;
@@ -293,6 +301,14 @@ async function encodeParquetRowBatches(batches, opts, write) {
 	let groupBytes = 0;
 	let groups = 0;
 	let rowCount = 0;
+	let omitStatistics = false;
+	const omitOptionalMetadata = (firstIndex) => {
+		for (let index = firstIndex; index < parquet.pendingIndexes.length; index++) {
+			const pending = parquet.pendingIndexes[index];
+			if (pending.chunk.meta_data) delete pending.chunk.meta_data.statistics;
+			delete pending.columnIndex;
+		}
+	};
 	const flushGroup = async () => {
 		if (!groupRows) return;
 		if (++groups > maxRowGroups) throw new RangeError("parquet stream exceeds maxRowGroups");
@@ -306,7 +322,21 @@ async function encodeParquetRowBatches(batches, opts, write) {
 			})),
 			rowGroupSize: groupRows
 		});
-		retainedMetadataBytes += retainedDescriptorBytes([parquet.row_groups[parquet.row_groups.length - 1], parquet.pendingIndexes.slice(firstIndex)], maxRetainedMetadataBytes - retainedMetadataBytes);
+		if (omitStatistics) omitOptionalMetadata(firstIndex);
+		try {
+			retainedMetadataBytes += retainedDescriptorBytes([parquet.row_groups[parquet.row_groups.length - 1], parquet.pendingIndexes.slice(firstIndex)], maxRetainedMetadataBytes - retainedMetadataBytes);
+		} catch (error) {
+			if (!(error instanceof RetainedMetadataLimitError) || metadataOverflow !== "omit-statistics" || omitStatistics) throw error;
+			omitStatistics = true;
+			omitOptionalMetadata(0);
+			retainedMetadataBytes = initialMetadataBytes;
+			let index = 0;
+			for (const group of parquet.row_groups) {
+				const end = index + group.columns.length;
+				retainedMetadataBytes += retainedDescriptorBytes([group, parquet.pendingIndexes.slice(index, end)], maxRetainedMetadataBytes - retainedMetadataBytes);
+				index = end;
+			}
+		}
 		values = codecs.map(() => []);
 		groupRows = 0;
 		groupBytes = 0;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "7.5.1",
+  "version": "7.6.0",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -189,10 +189,10 @@
     }
   },
   "dependencies": {
-    "@gscdump/contracts": "^7.5.1",
-    "@gscdump/lakehouse": "^7.5.1",
+    "@gscdump/contracts": "^7.6.0",
+    "@gscdump/lakehouse": "^7.6.0",
     "drizzle-orm": "1.0.0-rc.4",
-    "gscdump": "^7.5.1",
+    "gscdump": "^7.6.0",
     "pathe": "^2.0.3",
     "proper-lockfile": "^4.1.2"
   },