@gscdump/engine 7.3.1 → 7.3.2

package/README.md

@@ -20,6 +20,16 @@ npm install @gscdump/engine
 The CLI installs this peer. Browser consumers use `@gscdump/engine-duckdb-wasm`.
 The package includes `hyparquet` and `hyparquet-writer` for JavaScript Parquet encoding and decoding.
 
+The `/hyparquet` encoders accept optional `fileMetadata: Record<string, string>`:
+use `encodeRowsToParquet(table, rows, { fileMetadata })`, or include it in the
+options for `encodeRowsToParquetFlex`. Metadata is stored in the file footer, not as fact
+columns, including for empty files. Keys are sorted for deterministic encoding;
+omitting metadata or passing an empty object preserves the existing bytes.
+A publication coordinator can supply a unique identifier so identical rows from
+different publications produce different bytes. It must retain and replay the
+exact encoded bytes for retries; this option does not provide publication or
+recovery coordination itself.
+
 ## Subpath exports
 
 | Subpath | Purpose |

package/dist/adapters/hyparquet.d.mts

@@ -2,8 +2,12 @@ import { ParquetQueryFilter } from "../node_modules/.pnpm/hyparquet@1.31.1/node_
 import "../node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/types/node.mjs";
 import { CodecCtx, DataSource, ParquetCodec, Row, TableName } from "../storage.mjs";
 import { ColumnDef } from "../schema.mjs";
-export declare function encodeRowsToParquet(table: TableName, rows: readonly Row[]): Uint8Array;
-export interface EncodeFlexOptions {
+export interface EncodeParquetOptions {
+  /** File footer metadata, without adding fact columns. Keys are encoded in sorted order. */
+  fileMetadata?: Readonly<Record<string, string>>;
+}
+export declare function encodeRowsToParquet(table: TableName, rows: readonly Row[], opts?: EncodeParquetOptions): Uint8Array;
+export interface EncodeFlexOptions extends EncodeParquetOptions {
   /** Columns defining the output schema + order. */
   columns: readonly ColumnDef[];
   /** Sort key columns (subset of `columns` by name). Empty = preserve input order. */

package/dist/adapters/hyparquet.mjs

@@ -157,7 +157,7 @@ function indexRow(index, key, row) {
 	} else if (sameNaturalKey(key, existing, row)) index.set(hash, row);
 	else index.set(hash, [existing, row]);
 }
-function encodeOrderedRows(rows, columns, rowGroupSize) {
+function encodeOrderedRows(rows, columns, rowGroupSize, fileMetadata) {
 	const schema = buildWriteSchema(columns);
 	const codecs = columns.map((col) => ({
 		name: col.name,
@@ -177,18 +177,24 @@ function encodeOrderedRows(rows, columns, rowGroupSize) {
 		}
 	}
 	const writer = new ByteWriter();
+	const metadataKeys = Object.keys(fileMetadata ?? {}).sort();
+	const kvMetadata = metadataKeys.length > 0 ? metadataKeys.map((key) => ({
+		key,
+		value: fileMetadata[key]
+	})) : void 0;
 	parquetWriteRows({
 		writer,
 		rows: coercedRows(),
 		columns: columnSpecs,
 		schema,
-		rowGroupSize
+		rowGroupSize,
+		kvMetadata
 	});
 	return new Uint8Array(writer.getBuffer());
 }
-function encodeRowsToParquet(table, rows) {
+function encodeRowsToParquet(table, rows, opts = {}) {
 	const schema = SCHEMAS[table];
-	return encodeOrderedRows(sortRowsByClusterKey(table, rows), schema.columns, ROW_GROUP_SIZE);
+	return encodeOrderedRows(sortRowsByClusterKey(table, rows), schema.columns, ROW_GROUP_SIZE, opts.fileMetadata);
 }
 function encodeRowsToParquetFlex(rows, opts) {
 	const { columns, sortKey = [], rowGroupSize = ROW_GROUP_SIZE } = opts;
@@ -198,7 +204,7 @@ function encodeRowsToParquetFlex(rows, opts) {
 			if (cmp !== 0) return cmp;
 		}
 		return 0;
-	}), columns, rowGroupSize);
+	}), columns, rowGroupSize, opts.fileMetadata);
 }
 function asyncBufferFromBytes(bytes) {
 	const base = bytes.byteOffset;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "7.3.1",
+  "version": "7.3.2",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -189,10 +189,10 @@
     }
   },
   "dependencies": {
-    "@gscdump/contracts": "^7.3.1",
-    "@gscdump/lakehouse": "^7.3.1",
+    "@gscdump/contracts": "^7.3.2",
+    "@gscdump/lakehouse": "^7.3.2",
     "drizzle-orm": "1.0.0-rc.4",
-    "gscdump": "^7.3.1",
+    "gscdump": "^7.3.2",
     "pathe": "^2.0.3",
     "proper-lockfile": "^4.1.2"
   },