@gscdump/engine 7.3.2 → 7.4.0

package/README.md

@@ -30,6 +30,35 @@ different publications produce different bytes. It must retain and replay the
 exact encoded bytes for retries; this option does not provide publication or
 recovery coordination itself.
 
+For a bounded, already ordered source, `/hyparquet` also exports
+`encodeParquetRowBatches(batches, options, write)`. It consumes sync or async
+batches, coerces the declared `columns` with the same physical types as the
+array encoder, and emits one Parquet file through an awaited byte callback.
+It does not sort, deduplicate, retry, or publish an object. The callback borrows
+its bytes until its promise settles; copy a chunk if retaining it. The caller
+must abort/discard the entire destination on any error, including an error
+while writing the footer. Reopening the same immutable source with identical
+options produces identical bytes regardless of producer batch boundaries.
+This supports a counting/digest pass followed by a separately coordinated
+fixed-length upload, without the codec owning transport or commit semantics.
+
+`decodeParquetRowGroups({ byteLength, slice }, options)` reads exact byte ranges
+and yields one physical row group at a time. The caller must pin every range
+to the same immutable object or ETag. It preserves the existing date, column
+projection, row-window and filter behavior. An absent, corrupt, oversized or
+incompletely read file throws; only valid empty Parquet yields no rows.
+
+Both APIs have configurable admission limits. The encoder bounds producer
+batch rows/value bytes, output group rows/value bytes, retained descriptor counts
+and scalar bytes (including statistics), and the current encoded group/footer
+buffer. The decoder checks footer size,
+descriptor count, declared group rows/uncompressed bytes, and every range request.
+The default decoder accepts the existing encoder's 25,000-row groups. These
+limits do not establish a Worker heap guarantee: caller-retained rows, codec
+transient allocations, footer descriptors, and concurrent requests still use
+memory. Metadata limits are not protection against malicious Parquet contents.
+The existing array APIs and their default bytes are unchanged.
+
 ## Subpath exports
 
 | Subpath | Purpose |

package/dist/adapters/hyparquet.d.mts

@@ -1,6 +1,6 @@
 import { ParquetQueryFilter } from "../node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/types/index.mjs";
 import "../node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/types/node.mjs";
-import { CodecCtx, DataSource, ParquetCodec, Row, TableName } from "../storage.mjs";
+import { CodecCtx, DataSource, ParquetCodec, Row, TableName, WriteResult } from "../storage.mjs";
 import { ColumnDef } from "../schema.mjs";
 export interface EncodeParquetOptions {
   /** File footer metadata, without adding fact columns. Keys are encoded in sorted order. */
@@ -23,6 +23,36 @@ export interface EncodeFlexOptions extends EncodeParquetOptions {
  * merges cleanly with fact-table reads.
  */
 export declare function encodeRowsToParquetFlex(rows: readonly Row[], opts: EncodeFlexOptions): Uint8Array;
+export interface EncodeParquetBatchesOptions extends EncodeParquetOptions {
+  /** Output schema and column order. Input rows must already be ordered/deduplicated. */
+  columns: readonly ColumnDef[];
+  /** Maximum rows per output group. Default 25000, matching the array codec. */
+  rowGroupSize?: number;
+  limits?: {
+    /** Refuse a producer batch with more rows. Default 25000. */
+    maxBatchRows?: number;
+    /** Conservative scalar-value bytes per producer batch. Default 8 MiB. */
+    maxBatchBytes?: number;
+    /** Split groups at this conservative scalar-value budget. Default 2 MiB. */
+    maxRowGroupBytes?: number;
+    /** Bound retained footer/index descriptors. Default 4096. */
+    maxRowGroups?: number;
+    /** Bound retained descriptor/scalar bytes, including statistics. Default 8 MiB. */
+    maxRetainedMetadataBytes?: number;
+    /** Bound the current encoded group or footer buffer. Default 8 MiB. */
+    maxOutputBufferBytes?: number;
+  };
+}
+/**
+ * Encode ordered batches into ONE Parquet file with sequential backpressure.
+ * Group boundaries depend on rows/options, never producer batch boundaries.
+ * `write` borrows its bytes only until its returned promise settles; copy them
+ * if retaining them. This function does not close/commit a destination. On ANY
+ * rejection the caller must abort/discard the entire partial output, including
+ * after a footer callback failure. It performs no sorting, dedupe or retries.
+ * Memory includes one input batch, one group, and bounded footer descriptors.
+ */
+export declare function encodeParquetRowBatches(batches: AsyncIterable<readonly Row[]> | Iterable<readonly Row[]>, opts: EncodeParquetBatchesOptions, write: (chunk: Uint8Array) => Promise<void>): Promise<WriteResult>;
 export interface DecodeParquetOptions {
   /** First physical row to decode, inclusive. */
   rowStart?: number;
@@ -49,6 +79,32 @@ export interface DecodeParquetOptions {
   columns?: readonly string[];
 }
 export declare function decodeParquetToRows(bytes: Uint8Array, opts?: DecodeParquetOptions): Promise<Row[]>;
+/** A caller-owned, immutable/pinned object; ranges must return exactly the requested bytes. */
+export interface ParquetRangeSource {
+  byteLength: number;
+  slice: (start: number, end?: number) => ArrayBuffer | Promise<ArrayBuffer>;
+}
+export interface DecodeParquetGroupsOptions extends DecodeParquetOptions {
+  limits?: {
+    /** Footer bytes before deserialization. Default 1 MiB. */
+    maxMetadataBytes?: number;
+    /** Includes the existing encoder's 25000-row groups. Default 25000. */
+    maxRowGroupRows?: number;
+    /** Declared uncompressed bytes across all columns in one group. Default 16 MiB. */
+    maxRowGroupBytes?: number;
+    /** Bound metadata descriptors. Default 4096. */
+    maxRowGroups?: number;
+    /** Maximum individual range request. Default 16 MiB. */
+    maxReadBytes?: number;
+  };
+}
+/**
+ * Pull one physical group at a time without downloading the complete object.
+ * The limits are admission bounds over Parquet metadata, not a memory-safety
+ * guarantee for malicious files or caller-retained batches. No cross-group
+ * read-ahead is started. Missing/corrupt/range-error input throws; it is not empty.
+ */
+export declare function decodeParquetRowGroups(file: ParquetRangeSource, opts?: DecodeParquetGroupsOptions): AsyncGenerator<Row[]>;
 export interface HyparquetCodecOptions {
   /**
    * Override `readRows`. Useful when reads should be delegated to a faster

package/dist/adapters/hyparquet.mjs

@@ -1,10 +1,11 @@
 import { TABLE_METADATA } from "../drizzle-schema.mjs";
 import { SCHEMAS } from "../schema.mjs";
 import { decompress } from "../node_modules/.pnpm/fzstd@0.1.1/node_modules/fzstd/esm/index.mjs";
-import { parquetMetadataAsync, parquetSchema } from "../node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/src/metadata.mjs";
+import { parquetMetadata, parquetMetadataAsync, parquetSchema } from "../node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/src/metadata.mjs";
 import { parquetReadObjects } from "../node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/src/read.mjs";
 import "../node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/src/node.mjs";
 import { ByteWriter } from "../node_modules/.pnpm/hyparquet-writer@0.16.10_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/bytewriter.mjs";
+import { ParquetWriter } from "../node_modules/.pnpm/hyparquet-writer@0.16.10_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/parquet-writer.mjs";
 import { parquetWriteRows } from "../node_modules/.pnpm/hyparquet-writer@0.16.10_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/write-rows.mjs";
 import "../node_modules/.pnpm/hyparquet-writer@0.16.10_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/node.mjs";
 const ROW_GROUP_SIZE = 25e3;
@@ -206,6 +207,139 @@ function encodeRowsToParquetFlex(rows, opts) {
 		return 0;
 	}), columns, rowGroupSize, opts.fileMetadata);
 }
+function streamLimit(value, fallback, name) {
+	const limit = value ?? fallback;
+	if (!Number.isSafeInteger(limit) || limit <= 0) throw new RangeError(`parquet stream ${name} must be a positive safe integer`);
+	return limit;
+}
+function scalarBytes(value) {
+	return typeof value === "string" ? value.length * 3 + 8 : 16;
+}
+function retainedDescriptorBytes(value, remaining) {
+	const seen = /* @__PURE__ */ new Set();
+	let bytes = 0;
+	const visit = (value) => {
+		if (value !== null && typeof value === "object") {
+			if (seen.has(value)) return;
+			seen.add(value);
+			bytes += 64;
+			if (ArrayBuffer.isView(value) || value instanceof ArrayBuffer) bytes += value.byteLength;
+			else for (const key in value) if (Object.hasOwn(value, key)) {
+				bytes += scalarBytes(key);
+				visit(value[key]);
+			}
+		} else bytes += scalarBytes(value);
+		if (bytes > remaining) throw new RangeError("parquet stream exceeds maxRetainedMetadataBytes");
+	};
+	visit(value);
+	return bytes;
+}
+async function encodeParquetRowBatches(batches, opts, write) {
+	const rowGroupSize = streamLimit(opts.rowGroupSize, ROW_GROUP_SIZE, "rowGroupSize");
+	const maxBatchRows = streamLimit(opts.limits?.maxBatchRows, ROW_GROUP_SIZE, "maxBatchRows");
+	const maxBatchBytes = streamLimit(opts.limits?.maxBatchBytes, 8388608, "maxBatchBytes");
+	const maxRowGroupBytes = streamLimit(opts.limits?.maxRowGroupBytes, 2097152, "maxRowGroupBytes");
+	const maxRowGroups = streamLimit(opts.limits?.maxRowGroups, 4096, "maxRowGroups");
+	const maxRetainedMetadataBytes = streamLimit(opts.limits?.maxRetainedMetadataBytes, 8388608, "maxRetainedMetadataBytes");
+	const maxOutputBufferBytes = streamLimit(opts.limits?.maxOutputBufferBytes, 8388608, "maxOutputBufferBytes");
+	if (opts.columns.length === 0 || opts.columns.length > 64 || new Set(opts.columns.map((col) => col.name)).size !== opts.columns.length) throw new RangeError("parquet stream requires 1 to 64 uniquely named columns");
+	const codecs = opts.columns.map((col) => ({
+		name: col.name,
+		isDate: col.type === "DATE",
+		type: basicTypeFor(col.type),
+		nullable: col.nullable
+	}));
+	const schema = buildWriteSchema(opts.columns);
+	const metadataKeys = Object.keys(opts.fileMetadata ?? {}).sort();
+	let metadataBytes = 0;
+	for (const key of metadataKeys) {
+		metadataBytes += scalarBytes(key) + scalarBytes(opts.fileMetadata[key]);
+		if (metadataBytes > maxOutputBufferBytes) throw new RangeError("parquet stream file metadata exceeds maxOutputBufferBytes");
+	}
+	let retainedMetadataBytes = retainedDescriptorBytes([schema, opts.fileMetadata], maxRetainedMetadataBytes);
+	const writer = new ByteWriter(Math.min(65536, maxOutputBufferBytes));
+	writer.ensure = (size) => {
+		const required = writer.index + size;
+		if (required > maxOutputBufferBytes) throw new RangeError("parquet stream encoded group/footer exceeds maxOutputBufferBytes");
+		if (required > writer.buffer.byteLength) {
+			const next = new ArrayBuffer(Math.min(maxOutputBufferBytes, Math.max(required, writer.buffer.byteLength * 2)));
+			new Uint8Array(next).set(new Uint8Array(writer.buffer, 0, writer.index));
+			writer.buffer = next;
+			writer.view = new DataView(next);
+		}
+	};
+	const flush = async () => {
+		if (writer.index > 0) await write(new Uint8Array(writer.buffer, 0, writer.index));
+		writer.index = 0;
+	};
+	const output = Object.assign(writer, {
+		flush,
+		finish: flush
+	});
+	const parquet = new ParquetWriter({
+		writer: output,
+		schema,
+		kvMetadata: metadataKeys.length ? metadataKeys.map((key) => ({
+			key,
+			value: opts.fileMetadata[key]
+		})) : void 0
+	});
+	let values = codecs.map(() => []);
+	let groupRows = 0;
+	let groupBytes = 0;
+	let groups = 0;
+	let rowCount = 0;
+	const flushGroup = async () => {
+		if (!groupRows) return;
+		if (++groups > maxRowGroups) throw new RangeError("parquet stream exceeds maxRowGroups");
+		const firstIndex = parquet.pendingIndexes.length;
+		await parquet.write({
+			columnData: codecs.map((col, index) => ({
+				name: col.name,
+				nullable: col.nullable,
+				columnIndex: true,
+				data: values[index]
+			})),
+			rowGroupSize: groupRows
+		});
+		retainedMetadataBytes += retainedDescriptorBytes([parquet.row_groups[parquet.row_groups.length - 1], parquet.pendingIndexes.slice(firstIndex)], maxRetainedMetadataBytes - retainedMetadataBytes);
+		values = codecs.map(() => []);
+		groupRows = 0;
+		groupBytes = 0;
+	};
+	try {
+		for await (const batch of batches) {
+			if (batch.length > maxBatchRows) throw new RangeError("parquet stream exceeds maxBatchRows");
+			let batchBytes = 0;
+			for (const row of batch) {
+				const converted = codecs.map((col) => col.isDate ? toEpochDays(row[col.name]) : coerceValue(row[col.name], col.type));
+				const bytes = converted.reduce((sum, value) => sum + scalarBytes(value), 0);
+				batchBytes += bytes;
+				if (batchBytes > maxBatchBytes) throw new RangeError("parquet stream exceeds maxBatchBytes");
+				if (bytes > maxRowGroupBytes) throw new RangeError("parquet stream single row exceeds maxRowGroupBytes");
+				if (groupRows && (groupRows >= rowGroupSize || groupBytes + bytes > maxRowGroupBytes)) await flushGroup();
+				for (let index = 0; index < converted.length; index++) values[index].push(converted[index]);
+				groupRows++;
+				groupBytes += bytes;
+				rowCount++;
+				if (groupRows === rowGroupSize) await flushGroup();
+			}
+		}
+		await flushGroup();
+		await parquet.finish();
+		return {
+			bytes: writer.offset,
+			rowCount
+		};
+	} finally {
+		values = [];
+		parquet.row_groups.length = 0;
+		parquet.pendingIndexes.length = 0;
+		writer.buffer = /* @__PURE__ */ new ArrayBuffer(0);
+		writer.view = new DataView(writer.buffer);
+		writer.index = 0;
+	}
+}
 function asyncBufferFromBytes(bytes) {
 	const base = bytes.byteOffset;
 	const buf = bytes.buffer;
@@ -226,7 +360,45 @@ async function decodeParquetToRows(bytes, opts = {}) {
 async function* decodeParquetGroups(bytes, opts = {}) {
 	if (bytes.byteLength === 0) return;
 	const file = asyncBufferFromBytes(bytes);
-	const metadata = await parquetMetadataAsync(file);
+	yield* decodeGroupsFromMetadata(file, await parquetMetadataAsync(file), opts);
+}
+async function* decodeParquetRowGroups(file, opts = {}) {
+	const maxMetadataBytes = streamLimit(opts.limits?.maxMetadataBytes, 1048576, "maxMetadataBytes");
+	const maxRowGroupRows = streamLimit(opts.limits?.maxRowGroupRows, ROW_GROUP_SIZE, "maxRowGroupRows");
+	const maxRowGroupBytes = streamLimit(opts.limits?.maxRowGroupBytes, 16777216, "maxRowGroupBytes");
+	const maxRowGroups = streamLimit(opts.limits?.maxRowGroups, 4096, "maxRowGroups");
+	const maxReadBytes = streamLimit(opts.limits?.maxReadBytes, 16777216, "maxReadBytes");
+	if (!Number.isSafeInteger(file.byteLength) || file.byteLength < 12) throw new RangeError("parquet stream requires a valid file length");
+	const bounded = {
+		byteLength: file.byteLength,
+		async slice(start, end = file.byteLength) {
+			if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || end < start || end > file.byteLength || end - start > maxReadBytes) throw new RangeError("parquet stream range exceeds maxReadBytes or file bounds");
+			const bytes = await file.slice(start, end);
+			if (bytes.byteLength !== end - start) throw new Error("parquet stream returned an incomplete range");
+			return bytes;
+		}
+	};
+	const trailer = new DataView(await bounded.slice(file.byteLength - 8, file.byteLength));
+	if (trailer.getUint32(4, true) !== 827474256) throw new Error("parquet stream invalid footer");
+	const metadataBytes = trailer.getUint32(0, true);
+	if (metadataBytes > maxMetadataBytes || metadataBytes + 12 > file.byteLength) throw new RangeError("parquet stream footer exceeds maxMetadataBytes or file bounds");
+	const metadata = parquetMetadata(await bounded.slice(file.byteLength - metadataBytes - 8, file.byteLength));
+	if (metadata.row_groups.length > maxRowGroups || metadata.schema.length > 65) throw new RangeError("parquet stream exceeds row-group/schema descriptor limits");
+	let totalRows = 0;
+	for (const group of metadata.row_groups) {
+		if (group.columns.length > 64) throw new RangeError("parquet stream exceeds column descriptor limits");
+		const rows = Number(group.num_rows);
+		if (group.columns.some((col) => [col.meta_data?.total_uncompressed_size, col.meta_data?.total_compressed_size].some((size) => !Number.isSafeInteger(Number(size)) || Number(size) < 0))) throw new RangeError("parquet stream invalid declared group bytes");
+		const uncompressed = group.columns.reduce((sum, col) => sum + Number(col.meta_data?.total_uncompressed_size ?? NaN), 0);
+		const compressed = group.columns.reduce((sum, col) => sum + Number(col.meta_data?.total_compressed_size ?? NaN), 0);
+		if (!Number.isSafeInteger(rows) || rows < 0 || rows > maxRowGroupRows) throw new RangeError("parquet stream exceeds maxRowGroupRows");
+		if (!Number.isSafeInteger(uncompressed) || uncompressed < 0 || uncompressed > maxRowGroupBytes || !Number.isSafeInteger(compressed) || compressed < 0 || compressed > maxRowGroupBytes) throw new RangeError("parquet stream declared group bytes exceed maxRowGroupBytes");
+		totalRows += rows;
+	}
+	if (!Number.isSafeInteger(totalRows) || BigInt(totalRows) !== metadata.num_rows) throw new Error("parquet stream row-count metadata mismatch");
+	yield* decodeGroupsFromMetadata(bounded, metadata, opts);
+}
+async function* decodeGroupsFromMetadata(file, metadata, opts) {
 	const tree = parquetSchema(metadata);
 	const present = /* @__PURE__ */ new Set();
 	const dateColumns = [];
@@ -339,4 +511,4 @@ function createHyparquetCodec(options = {}) {
 		}
 	};
 }
-export { createHyparquetCodec, decodeParquetToRows, encodeRowsToParquet, encodeRowsToParquetFlex };
+export { createHyparquetCodec, decodeParquetRowGroups, decodeParquetToRows, encodeParquetRowBatches, encodeRowsToParquet, encodeRowsToParquetFlex };

package/dist/node_modules/.pnpm/hyparquet-writer@0.16.10_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/index.mjs

@@ -1,7 +1,7 @@
 import { ByteWriter } from "./bytewriter.mjs";
 import "./schema.mjs";
 import "./wkb.mjs";
-import "./parquet-writer.mjs";
+import { ParquetWriter } from "./parquet-writer.mjs";
 import "./write.mjs";
 import { parquetWriteRows } from "./write-rows.mjs";
-export { ByteWriter, parquetWriteRows };
+export { ByteWriter, ParquetWriter, parquetWriteRows };

package/dist/node_modules/.pnpm/hyparquet-writer@0.16.10_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/node.mjs

@@ -1,8 +1,8 @@
 import { ByteWriter } from "./bytewriter.mjs";
 import "./schema.mjs";
 import "./wkb.mjs";
-import "./parquet-writer.mjs";
+import { ParquetWriter } from "./parquet-writer.mjs";
 import "./write.mjs";
 import { parquetWriteRows } from "./write-rows.mjs";
 import "./index.mjs";
-export { ByteWriter, parquetWriteRows };
+export { ByteWriter, ParquetWriter, parquetWriteRows };

package/dist/node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/src/index.mjs

@@ -1,8 +1,8 @@
-import { parquetMetadataAsync, parquetSchema } from "./metadata.mjs";
+import { parquetMetadata, parquetMetadataAsync, parquetSchema } from "./metadata.mjs";
 import "./indexes.mjs";
 import { toJson } from "./utils.mjs";
 import "./snappy.mjs";
 import "./scan.mjs";
 import { parquetReadObjects } from "./read.mjs";
 import "./query.mjs";
-export { parquetMetadataAsync, parquetReadObjects, parquetSchema, toJson };
+export { parquetMetadata, parquetMetadataAsync, parquetReadObjects, parquetSchema, toJson };

package/dist/node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/src/node.mjs

@@ -1,7 +1,7 @@
-import { parquetMetadataAsync, parquetSchema } from "./metadata.mjs";
+import { parquetMetadata, parquetMetadataAsync, parquetSchema } from "./metadata.mjs";
 import "./indexes.mjs";
 import { toJson } from "./utils.mjs";
 import "./snappy.mjs";
 import { parquetReadObjects } from "./read.mjs";
 import "./index.mjs";
-export { parquetMetadataAsync, parquetReadObjects, parquetSchema, toJson };
+export { parquetMetadata, parquetMetadataAsync, parquetReadObjects, parquetSchema, toJson };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "7.3.2",
+  "version": "7.4.0",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -189,10 +189,10 @@
     }
   },
   "dependencies": {
-    "@gscdump/contracts": "^7.3.2",
-    "@gscdump/lakehouse": "^7.3.2",
+    "@gscdump/contracts": "^7.4.0",
+    "@gscdump/lakehouse": "^7.4.0",
     "drizzle-orm": "1.0.0-rc.4",
-    "gscdump": "^7.3.2",
+    "gscdump": "^7.4.0",
     "pathe": "^2.0.3",
     "proper-lockfile": "^4.1.2"
   },