@gscdump/engine 3.5.0 → 3.6.0

package/README.md

@@ -4,9 +4,11 @@
 [![npm downloads](https://img.shields.io/npm/dm/@gscdump/engine?color=yellow)](https://npm.chart.dev/@gscdump/engine)
 [![license](https://img.shields.io/github/license/harlan-zw/gscdump?color=yellow)](https://github.com/harlan-zw/gscdump/blob/main/LICENSE)
 
-> Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.
+Store Search Console data in Parquet files and query it with DuckDB.
+This package also provides Source contracts, SQL resolution, and runtime adapters.
 
-Owns the storage runtime, planner, schema, and adapters that were previously bundled into `gscdump`. Edge consumers stay on [`gscdump`](../gscdump); anything that needs to read/write Parquet, run the DuckDB executor, or attach a snapshot lives here.
+Choose an export for your runtime.
+Node filesystem and DuckDB helpers live on dedicated subpaths.
 
 ## Install
 
@@ -14,10 +16,9 @@ Owns the storage runtime, planner, schema, and adapters that were previously bun
 npm install @gscdump/engine
 ```
 
-Optional peers (install only what your runtime needs):
-
-- `@duckdb/duckdb-wasm` — DuckDB-WASM executor (browser + node-blocking shim).
-- `hyparquet`, `hyparquet-writer` — pure-JS Parquet codec for environments without a native build.
+`@duckdb/node-api` is an optional peer for the Node runtime.
+The CLI installs this peer. Browser consumers use `@gscdump/engine-duckdb-wasm`.
+The package includes `hyparquet` and `hyparquet-writer` for JavaScript Parquet encoding and decoding.
 
 ## Subpath exports
 
@@ -31,39 +32,48 @@ Optional peers (install only what your runtime needs):
 | `@gscdump/engine/ingest-accumulator` | Stateful ingest flushing behind injected storage and recovery hooks. |
 | `@gscdump/engine/sql` | SQL literal binding helpers (`bindLiterals`, `formatLiteral`). |
 | `@gscdump/engine/sql-fragments` | Reusable SQL fragments shared across analyzers. |
-| `@gscdump/engine/rollups` | Pre-aggregated rollup contracts + helpers. |
+| `@gscdump/engine/rollups` | Pre-aggregated rollup contracts and helpers. |
 | `@gscdump/engine/entities` | Parquet-backed entity stores and their record contracts. |
 | `@gscdump/engine/entity-keys` | Pure entity object-key and URL-hash helpers, with no Parquet runtime. |
 | `@gscdump/engine/resolver` | Dialect-neutral SQL composition: `ResolverAdapter`, `pgResolverAdapter`, `resolveToSQL`. |
 | `@gscdump/engine/scope` | Multi-tenant scope predicates. |
 | `@gscdump/engine/arrow` | Apache Arrow utilities for engine result conversion. |
 | `@gscdump/engine/node` | Node-only DuckDB handle. |
-| `@gscdump/engine/filesystem` | Node-only `DataSource` + `ManifestStore` adapters. |
+| `@gscdump/engine/filesystem` | Node-only `DataSource` and `ManifestStore` adapters. |
 | `@gscdump/engine/hyparquet` | Pure-JS `ParquetCodec`. |
-| `@gscdump/engine/r2` | Cloudflare R2 `DataSource` (structurally typed against `R2Bucket`). |
+| `@gscdump/engine/r2` | Cloudflare R2 `DataSource` and `createR2ManifestStore` (structurally typed against `R2Bucket`). |
 | `@gscdump/engine/iceberg` | Edge-safe GSC dataset schemas, catalog wrappers, and append sink. Generic Iceberg APIs live in `@gscdump/lakehouse`. |
 | `@gscdump/engine/sink-node` | Node-only Iceberg overwrite/delete recovery writer. |
-| `@gscdump/engine/source` | Query-source contracts and factories. |
+| `@gscdump/engine/source` | Source contracts and factories. |
+| `@gscdump/engine/analyzer` | Analyzer contracts, registry construction, and dispatch. |
+| `@gscdump/engine/report` | Report definitions and contracts. |
+| `@gscdump/engine/period` | Date windows and comparison ranges. |
+| `@gscdump/engine/analysis-types` | Analysis parameters and results. |
+| `@gscdump/engine/analysis-range` | Analysis date-range resolution. |
+| `@gscdump/engine/errors` | Tagged Engine failures. |
+| `@gscdump/engine/profile` | Query timing and profiling helpers. |
+| `@gscdump/engine/sync-config` | Sync limits and table/search-type configuration. |
+| `@gscdump/engine/vendor/hysnappy` | Worker-compatible compression alias. |
 
 ## Stability
 
 | Surface | Stability |
 |---|---|
 | `createStorageEngine` and storage contracts (`StorageEngine`, `Row`, `WriteCtx`, ...) | Public |
-| Adapters (`/node`, `/filesystem`, `/hyparquet`, `/r2`, `/r2-manifest`) | Public |
+| Adapters (`/node`, `/filesystem`, `/hyparquet`, `/r2`) | Public |
 | Planner (`resolveToSQL`, `enumeratePartitions`) | Public |
 | Schema (`SCHEMAS`, `allTables`, ...) | Public |
-| Internals reached through `@gscdump/engine/<file>` paths not listed above | Private — may break between minors |
+| Files outside the declared package exports | Private |
 
 ## Related
 
-- [`gscdump`](../gscdump) — REST client + query builder (edge-safe peer dep).
-- [`@gscdump/analysis`](../analysis) — analyzers; consumes storage through `AnalysisQuerySource` adapters.
-- [`@gscdump/engine-duckdb-wasm`](../engine-duckdb-wasm) — DuckDB-WASM browser adapter.
-- [`@gscdump/engine-sqlite`](../engine-sqlite) — SQLite / D1 adapter.
-- [`@gscdump/lakehouse`](../lakehouse) — dataset-agnostic Iceberg catalog and
+- [`gscdump`](../gscdump) : REST client and query builder (edge-safe peer dep).
+- [`@gscdump/analysis`](../analysis) : analyzers; consumes storage through `AnalysisQuerySource` adapters.
+- [`@gscdump/engine-duckdb-wasm`](../engine-duckdb-wasm) : DuckDB-WASM browser adapter.
+- [`@gscdump/engine-sqlite`](../engine-sqlite) : SQLite / D1 adapter.
+- [`@gscdump/lakehouse`](../lakehouse) : dataset-agnostic Iceberg catalog and
   dataset registry APIs.
-- [`@gscdump/cli`](../cli) — CLI wrapping engine + analysis.
+- [`@gscdump/cli`](../cli) : CLI wrapping engine and analysis.
 
 ## License
 

package/dist/THIRD-PARTY-LICENSES.md

@@ -35,11 +35,11 @@ Repository: https://github.com/101arrowz/fzstd
 
 ---------------------------------------
 
-## hyparquet, hyparquet-compressors, hyparquet-writer, icebird, squirreling
+## hyparquet, hyparquet, hyparquet-compressors, hyparquet-writer, icebird, squirreling
 
 License: MIT
 By: Hyperparam
-Repositories: https://github.com/hyparam/hyparquet, https://github.com/hyparam/hyparquet-compressors, https://github.com/hyparam/hyparquet-writer, https://github.com/hyparam/icebird, https://github.com/hyparam/squirreling
+Repositories: https://github.com/hyparam/hyparquet, https://github.com/hyparam/hyparquet, https://github.com/hyparam/hyparquet-compressors, https://github.com/hyparam/hyparquet-writer, https://github.com/hyparam/icebird, https://github.com/hyparam/squirreling
 
 > The MIT License (MIT)
 > 

package/dist/adapters/duckdb-node.d.mts

@@ -1,7 +1,7 @@
 import { DuckDBHandle } from "../duckdb.mjs";
-interface NodeDuckDBOptions {
+export interface NodeDuckDBOptions {
   verbose?: boolean;
 }
-declare function createNodeDuckDBHandle(opts?: NodeDuckDBOptions): DuckDBHandle;
-declare function resetNodeDuckDB(): void;
-export { NodeDuckDBOptions, createNodeDuckDBHandle, resetNodeDuckDB };
+/** Node uses one native connection. Browser and Worker adapters own their WASM runtimes. */
+export declare function createNodeDuckDBHandle(opts?: NodeDuckDBOptions): DuckDBHandle;
+export declare function resetNodeDuckDB(): void;

package/dist/adapters/duckdb-node.mjs

@@ -1,98 +1,124 @@
-import { arrowToRows } from "../arrow-utils.mjs";
-import { createRequire } from "node:module";
 import process from "node:process";
-import { unlinkSync } from "node:fs";
+import { randomUUID } from "node:crypto";
+import { mkdtempSync, rmSync } from "node:fs";
+import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { fileURLToPath } from "node:url";
-import { ConsoleLogger, NODE_RUNTIME, VoidLogger, createDuckDB } from "@duckdb/duckdb-wasm/dist/duckdb-node-blocking.cjs";
-const require_ = createRequire(typeof __filename !== "undefined" ? __filename : typeof import.meta !== "undefined" ? fileURLToPath(import.meta.url) : process.cwd());
-let singleton = null;
-let singletonOpts = null;
-function bundles() {
-	return {
-		mvp: {
-			mainModule: require_.resolve("@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm"),
-			mainWorker: null
-		},
-		eh: {
-			mainModule: require_.resolve("@duckdb/duckdb-wasm/dist/duckdb-eh.wasm"),
-			mainWorker: null
-		}
+import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
+import { DuckDBInstance, blobValue, timestampValue } from "@duckdb/node-api";
+let singleton;
+function getRuntime() {
+	if (singleton) return singleton;
+	const directory = mkdtempSync(join(tmpdir(), "gscdump-duckdb-"));
+	const onExit = () => rmSync(directory, {
+		recursive: true,
+		force: true
+	});
+	process.once("exit", onExit);
+	singleton = {
+		directory,
+		onExit,
+		tail: Promise.resolve(),
+		state: { _tag: "Unopened" }
 	};
+	return singleton;
 }
-async function initialize(opts) {
-	const logger = opts.verbose ? new ConsoleLogger() : new VoidLogger();
-	const db = await createDuckDB(bundles(), logger, NODE_RUNTIME);
-	await db.instantiate();
-	return {
-		db,
-		conn: db.connect()
-	};
+function enqueue(runtime, operation) {
+	const result = runtime.tail.then(operation);
+	runtime.tail = result.then(() => void 0, () => void 0);
+	return result;
 }
-function getSingleton(opts) {
-	if (!singleton) {
-		singleton = initialize(opts);
-		singletonOpts = opts;
+async function connect(runtime, opts) {
+	if (runtime.state._tag === "Opened") return runtime.state.connection;
+	const instance = await DuckDBInstance.create(":memory:", { temp_directory: join(runtime.directory, "spill") });
+	try {
+		const connection = await instance.connect();
+		runtime.state = {
+			_tag: "Opened",
+			instance,
+			connection
+		};
+		await connection.run("SET file_search_path = $1", [runtime.directory]);
+		if (opts.verbose) console.warn("[gscdump] Native DuckDB initialized");
+		return connection;
+	} catch (error) {
+		if (runtime.state._tag === "Opened") runtime.state.connection.closeSync();
+		runtime.state = { _tag: "Unopened" };
+		instance.closeSync();
+		throw error;
 	}
-	return singleton;
+}
+function temporaryPath(runtime, name) {
+	const path = resolve(runtime.directory, name);
+	const child = relative(runtime.directory, path);
+	if (!child || child === ".." || child.startsWith(`..${sep}`) || isAbsolute(child)) throw new TypeError("DuckDB temporary files must stay inside the temporary directory.");
+	return path;
+}
+function parameter(value) {
+	if (value === null || typeof value === "string" || typeof value === "boolean" || typeof value === "bigint") return value;
+	if (typeof value === "number" && Number.isFinite(value)) return value;
+	if (value instanceof Date && Number.isFinite(value.getTime())) return timestampValue(BigInt(value.getTime()) * 1000n);
+	if (value instanceof Uint8Array) return blobValue(value);
+	throw new TypeError("DuckDB parameters must be finite scalars, dates, byte arrays, or null.");
+}
+function rowValue(value) {
+	if (value instanceof Date) return value.getTime();
+	if (Array.isArray(value)) return value.map(rowValue);
+	if (value instanceof Uint8Array) return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
+	if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, rowValue(item)]));
+	return value;
 }
 function createNodeDuckDBHandle(opts = {}) {
-	if (singleton && opts.verbose !== void 0 && opts.verbose !== (singletonOpts?.verbose ?? false)) console.warn(`[gscdump] createNodeDuckDBHandle: ignoring verbose=${opts.verbose} — a shared DuckDB instance was already initialized with verbose=${singletonOpts?.verbose ?? false}. Call resetNodeDuckDB() before re-initializing to change it.`);
-	getSingleton(opts);
 	return {
-		async query(sql, params) {
-			const { conn } = await getSingleton(opts);
-			if (!params || params.length === 0) {
-				const result = conn.query(sql);
-				return arrowToRows(result);
-			}
-			const stmt = conn.prepare(sql);
-			try {
-				const result = stmt.query(...params);
-				return arrowToRows(result);
-			} finally {
-				stmt.close();
-			}
+		query(sql, params) {
+			const runtime = getRuntime();
+			return enqueue(runtime, async () => {
+				const result = await (await connect(runtime, opts)).runAndReadAll(sql, params?.length ? params.map(parameter) : void 0);
+				const names = result.columnNames();
+				return result.getRowsJS().map((values) => Object.fromEntries(names.map((name, index) => [name, rowValue(values[index])])));
+			});
 		},
-		async registerFileBuffer(name, bytes) {
-			const { db } = await getSingleton(opts);
-			db.registerFileBuffer(name, bytes);
+		registerFileBuffer(name, bytes) {
+			const runtime = getRuntime();
+			return enqueue(runtime, async () => {
+				const path = temporaryPath(runtime, name);
+				await mkdir(dirname(path), { recursive: true });
+				await writeFile(path, bytes);
+			});
 		},
-		async copyFileToBuffer(name) {
-			const { db } = await getSingleton(opts);
-			return db.copyFileToBuffer(name);
+		copyFileToBuffer(name) {
+			const runtime = getRuntime();
+			return enqueue(runtime, () => readFile(temporaryPath(runtime, name)));
 		},
-		async dropFiles(names) {
-			const { db } = await getSingleton(opts);
-			for (const name of names) {
-				try {
-					db.dropFile(name);
-				} catch (error) {
-					const message = error instanceof Error ? error.message : String(error);
-					if (!/not found|does not exist|unknown file/i.test(message)) throw error;
-				}
-				try {
-					unlinkSync(name);
-				} catch (error) {
-					if (error.code !== "ENOENT") throw error;
-				}
-			}
+		dropFiles(names) {
+			const runtime = getRuntime();
+			return enqueue(runtime, async () => {
+				for (const name of names) await rm(temporaryPath(runtime, name), { force: true });
+			});
 		},
 		makeTempPath(ext) {
-			return join(tmpdir(), `gscdump-${Math.random().toString(36).slice(2, 10)}.${ext}`);
+			return temporaryPath(getRuntime(), `${randomUUID()}.${ext}`);
 		}
 	};
 }
 function resetNodeDuckDB() {
-	const pending = singleton;
-	singleton = null;
-	singletonOpts = null;
-	pending?.then(({ db, conn }) => {
-		conn.close();
-		db.reset();
-	}).catch((err) => {
-		console.warn("[gscdump] resetNodeDuckDB: failed to release DuckDB instance", err);
+	const runtime = singleton;
+	singleton = void 0;
+	if (!runtime) return;
+	enqueue(runtime, async () => {
+		try {
+			if (runtime.state._tag === "Opened") {
+				runtime.state.connection.closeSync();
+				runtime.state.instance.closeSync();
+			}
+		} finally {
+			await rm(runtime.directory, {
+				recursive: true,
+				force: true
+			});
+			process.removeListener("exit", runtime.onExit);
+		}
+	}).catch((error) => {
+		console.warn("[gscdump] Failed to release DuckDB resources", error);
 	});
 }
 export { createNodeDuckDBHandle, resetNodeDuckDB };

package/dist/adapters/filesystem.d.mts

@@ -1,14 +1,13 @@
 import { DataSource, ManifestStore } from "../storage.mjs";
-interface FilesystemDataSourceOptions {
+export interface FilesystemDataSourceOptions {
   rootDir: string;
 }
-declare function createFilesystemDataSource(opts: FilesystemDataSourceOptions): DataSource;
-interface FilesystemManifestStoreOptions {
+export declare function createFilesystemDataSource(opts: FilesystemDataSourceOptions): DataSource;
+export interface FilesystemManifestStoreOptions {
   path: string;
 }
-declare function createFilesystemManifestStore(opts: FilesystemManifestStoreOptions): ManifestStore;
-declare function filesystemStats(rootDir: string): Promise<{
+export declare function createFilesystemManifestStore(opts: FilesystemManifestStoreOptions): ManifestStore;
+export declare function filesystemStats(rootDir: string): Promise<{
   files: number;
   bytes: number;
-}>;
-export { FilesystemDataSourceOptions, FilesystemManifestStoreOptions, createFilesystemDataSource, createFilesystemManifestStore, filesystemStats };
+}>;

package/dist/adapters/filesystem.mjs

@@ -1,15 +1,20 @@
 import { manifestEntryKey, matchesManifestEntryFilter, matchesSyncStateFilter, matchesWatermarkFilter, mergeSyncState, syncStateKey, watermarkKey } from "../manifest-store-utils.mjs";
-import { dirname, join, resolve } from "node:path";
-import { Buffer } from "node:buffer";
 import { randomBytes } from "node:crypto";
 import { mkdir, open, readFile, readdir, rename, rm, stat, unlink, writeFile } from "node:fs/promises";
+import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
+import { Buffer } from "node:buffer";
+import { normalize } from "pathe";
 import { lock } from "proper-lockfile";
+function portablePath(path) {
+	return sep === "\\" ? normalize(path) : path;
+}
 function createFilesystemDataSource(opts) {
 	const root = resolve(opts.rootDir);
 	const readyDirectories = /* @__PURE__ */ new Map();
 	function pathFor(key) {
 		const resolved = resolve(root, key);
-		if (!resolved.startsWith(`${root}/`) && resolved !== root) throw new Error(`path escapes root: ${key}`);
+		const child = relative(root, resolved);
+		if (child === ".." || child.startsWith(`..${sep}`) || isAbsolute(child)) throw new Error(`path escapes root: ${key}`);
 		return resolved;
 	}
 	async function ensureDirectory(path) {
@@ -72,11 +77,11 @@ function createFilesystemDataSource(opts) {
 			const full = pathFor(prefix);
 			const out = [];
 			await walk(full, out);
-			return out.map((p) => p.slice(root.length + 1));
+			return out.map((p) => portablePath(relative(root, p)));
 		},
 		async *streamList(prefix) {
 			const full = pathFor(prefix);
-			for await (const p of walkStream(full)) yield p.slice(root.length + 1);
+			for await (const p of walkStream(full)) yield portablePath(relative(root, p));
 		},
 		async head(key) {
 			const path = pathFor(key);
@@ -86,7 +91,7 @@ function createFilesystemDataSource(opts) {
 			});
 		},
 		uri(key) {
-			return pathFor(key);
+			return portablePath(pathFor(key));
 		}
 	};
 }

package/dist/adapters/hyparquet.d.mts

@@ -1,9 +1,9 @@
-import { ParquetQueryFilter } from "../node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/types/index.mjs";
-import "../node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/types/node.mjs";
+import { ParquetQueryFilter } from "../node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/types/index.mjs";
+import "../node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/types/node.mjs";
 import { CodecCtx, DataSource, ParquetCodec, Row, TableName } from "../storage.mjs";
 import { ColumnDef } from "../schema.mjs";
-declare function encodeRowsToParquet(table: TableName, rows: readonly Row[]): Uint8Array;
-interface EncodeFlexOptions {
+export declare function encodeRowsToParquet(table: TableName, rows: readonly Row[]): Uint8Array;
+export interface EncodeFlexOptions {
   /** Columns defining the output schema + order. */
   columns: readonly ColumnDef[];
   /** Sort key columns (subset of `columns` by name). Empty = preserve input order. */
@@ -18,8 +18,8 @@ interface EncodeFlexOptions {
  * the canonical encoder so DuckDB `read_parquet(union_by_name = true)`
  * merges cleanly with fact-table reads.
  */
-declare function encodeRowsToParquetFlex(rows: readonly Row[], opts: EncodeFlexOptions): Uint8Array;
-interface DecodeParquetOptions {
+export declare function encodeRowsToParquetFlex(rows: readonly Row[], opts: EncodeFlexOptions): Uint8Array;
+export interface DecodeParquetOptions {
   /** First physical row to decode, inclusive. */
   rowStart?: number;
   /** Last physical row to decode, exclusive. */
@@ -44,8 +44,8 @@ interface DecodeParquetOptions {
    */
   columns?: readonly string[];
 }
-declare function decodeParquetToRows(bytes: Uint8Array, opts?: DecodeParquetOptions): Promise<Row[]>;
-interface HyparquetCodecOptions {
+export declare function decodeParquetToRows(bytes: Uint8Array, opts?: DecodeParquetOptions): Promise<Row[]>;
+export interface HyparquetCodecOptions {
   /**
    * Override `readRows`. Useful when reads should be delegated to a faster
    * engine (e.g. DuckDB-WASM via httpfs) while writes + compaction stay on
@@ -58,5 +58,4 @@ interface HyparquetCodecOptions {
    */
   compactionReadConcurrency?: number;
 }
-declare function createHyparquetCodec(options?: HyparquetCodecOptions): ParquetCodec;
-export { DecodeParquetOptions, EncodeFlexOptions, HyparquetCodecOptions, createHyparquetCodec, decodeParquetToRows, encodeRowsToParquet, encodeRowsToParquetFlex };
+export declare function createHyparquetCodec(options?: HyparquetCodecOptions): ParquetCodec;

package/dist/adapters/hyparquet.mjs

@@ -1,8 +1,8 @@
 import { TABLE_METADATA } from "../drizzle-schema.mjs";
 import { SCHEMAS } from "../schema.mjs";
-import { parquetMetadataAsync, parquetSchema } from "../node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/metadata.mjs";
-import { parquetReadObjects } from "../node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/read.mjs";
-import "../node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/node.mjs";
+import { parquetMetadataAsync, parquetSchema } from "../node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/metadata.mjs";
+import { parquetReadObjects } from "../node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/read.mjs";
+import "../node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/node.mjs";
 import { ByteWriter } from "../node_modules/.pnpm/hyparquet-writer@0.16.9_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/bytewriter.mjs";
 import { parquetWriteRows } from "../node_modules/.pnpm/hyparquet-writer@0.16.9_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/write-rows.mjs";
 import "../node_modules/.pnpm/hyparquet-writer@0.16.9_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/node.mjs";
@@ -124,14 +124,37 @@ function sortRowsByClusterKey(table, rows) {
 	});
 	return copy;
 }
-function naturalKeyFor(table, row) {
-	const key = TABLE_METADATA[table].sortKey;
-	let value = "";
-	for (let index = 0; index < key.length; index++) {
-		if (index > 0) value += "\0";
-		value += `${row[key[index]] ?? ""}`;
+function naturalKeyHash(key, row) {
+	let hash = 2166136261;
+	for (const column of key) {
+		const value = String(row[column] ?? "");
+		for (let i = 0; i < value.length; i++) hash = Math.imul(hash ^ value.charCodeAt(i), 16777619);
+		hash = Math.imul(hash, 16777619);
 	}
-	return value;
+	return hash & 1073741823;
+}
+function sameNaturalKey(key, a, b) {
+	return key.every((column) => String(a[column] ?? "") === String(b[column] ?? ""));
+}
+function naturalKeyTuple(key, row) {
+	return JSON.stringify(key.map((column) => String(row[column] ?? "")));
+}
+function indexRow(index, key, row) {
+	const hash = naturalKeyHash(key, row);
+	const existing = index.get(hash);
+	if (existing === void 0) index.set(hash, row);
+	else if (existing instanceof Map) existing.set(naturalKeyTuple(key, row), row);
+	else if (Array.isArray(existing)) {
+		const match = existing.findIndex((candidate) => sameNaturalKey(key, candidate, row));
+		if (match >= 0) existing[match] = row;
+		else if (existing.length < 8) existing.push(row);
+		else {
+			const bucket = new Map(existing.map((candidate) => [naturalKeyTuple(key, candidate), candidate]));
+			bucket.set(naturalKeyTuple(key, row), row);
+			index.set(hash, bucket);
+		}
+	} else if (sameNaturalKey(key, existing, row)) index.set(hash, row);
+	else index.set(hash, [existing, row]);
 }
 function encodeOrderedRows(rows, columns, rowGroupSize) {
 	const schema = buildWriteSchema(columns);
@@ -189,44 +212,57 @@ function asyncBufferFromBytes(bytes) {
 	};
 }
 async function decodeParquetToRows(bytes, opts = {}) {
-	if (bytes.byteLength === 0) return [];
-	const file = asyncBufferFromBytes(bytes);
-	const projection = opts.columns ? await presentColumns(file, opts.columns) : null;
-	return normalizeDecodedDates(await parquetReadObjects({
-		file,
-		...opts.rowStart === void 0 ? {} : { rowStart: opts.rowStart },
-		...opts.rowEnd === void 0 ? {} : { rowEnd: opts.rowEnd },
-		...opts.rowStart !== void 0 || opts.rowEnd !== void 0 ? { useOffsetIndex: true } : {},
-		...projection ? {
-			metadata: projection.metadata,
-			columns: projection.columns
-		} : {},
-		...opts.filter ? {
-			filter: opts.filter,
-			useBloomFilters: true,
-			usePageIndex: true
-		} : {}
-	}));
+	const rows = [];
+	for await (const group of decodeParquetGroups(bytes, opts)) for (const row of group) rows.push(row);
+	return rows;
 }
-async function presentColumns(file, requested) {
+async function* decodeParquetGroups(bytes, opts = {}) {
+	if (bytes.byteLength === 0) return;
+	const file = asyncBufferFromBytes(bytes);
 	const metadata = await parquetMetadataAsync(file);
 	const present = new Set(parquetSchema(metadata).children.map((child) => child.element.name));
-	return {
-		metadata,
-		columns: requested.filter((name) => present.has(name))
-	};
+	const columns = opts.columns?.filter((name) => present.has(name));
+	let groupStart = 0;
+	for (const group of metadata.row_groups) {
+		const groupEnd = groupStart + Number(group.num_rows);
+		const rowStart = Math.max(groupStart, opts.rowStart ?? 0);
+		const rowEnd = Math.min(groupEnd, opts.rowEnd ?? Infinity);
+		if (rowStart < rowEnd) yield normalizeDecodedDates(await parquetReadObjects({
+			file,
+			metadata: {
+				...metadata,
+				row_groups: [group],
+				num_rows: group.num_rows
+			},
+			columns,
+			rowStart: rowStart - groupStart,
+			rowEnd: rowEnd - groupStart,
+			useOffsetIndex: rowStart > groupStart || rowEnd < groupEnd,
+			...opts.filter ? {
+				filter: opts.filter,
+				useBloomFilters: true,
+				usePageIndex: true
+			} : {}
+		}));
+		groupStart = groupEnd;
+	}
 }
 function normalizeDecodedDates(rows) {
-	if (rows.length === 0) return rows;
-	const dateCols = [];
-	const first = rows[0];
-	for (const k in first) if (first[k] instanceof Date) dateCols.push(k);
-	if (dateCols.length === 0) return rows;
+	const dates = /* @__PURE__ */ new Map();
 	for (const row of rows) {
 		const r = row;
-		for (const k of dateCols) {
+		for (const k in r) {
 			const v = r[k];
-			if (v instanceof Date) r[k] = isoFromDate(v);
+			if (v instanceof Date) {
+				const day = Math.floor(v.getTime() / EPOCH_DAY_MS);
+				let iso = dates.get(day);
+				if (iso === void 0) {
+					iso = isoFromDate(v);
+					if (dates.size >= 1024) dates.clear();
+					dates.set(day, iso);
+				}
+				r[k] = iso;
+			}
 		}
 	}
 	return rows;
@@ -256,16 +292,20 @@ function createHyparquetCodec(options = {}) {
 					rowCount: 0
 				};
 			}
+			const key = TABLE_METADATA[ctx.table].sortKey;
 			const byNaturalKey = /* @__PURE__ */ new Map();
 			for (let offset = 0; offset < inputKeys.length; offset += compactionReadConcurrency) {
 				const batch = inputKeys.slice(offset, offset + compactionReadConcurrency);
 				const inputs = await Promise.all(batch.map((key) => dataSource.read(key)));
-				for (const input of inputs) {
-					const rows = await decodeParquetToRows(input);
-					for (let i = 0; i < rows.length; i++) byNaturalKey.set(naturalKeyFor(ctx.table, rows[i]), rows[i]);
+				while (inputs.length > 0) {
+					const input = inputs.shift();
+					for await (const rows of decodeParquetGroups(input)) for (const row of rows) indexRow(byNaturalKey, key, row);
 				}
 			}
-			const rows = [...byNaturalKey.values()];
+			const rows = [];
+			for (const entry of byNaturalKey.values()) if (entry instanceof Map || Array.isArray(entry)) for (const row of entry.values()) rows.push(row);
+			else rows.push(entry);
+			byNaturalKey.clear();
 			const bytes = encodeRowsToParquet(ctx.table, rows);
 			await dataSource.write(outputKey, bytes);
 			return {

package/dist/adapters/node-harness.d.mts

@@ -1,14 +1,14 @@
 import { DataSource, StorageEngine } from "../storage.mjs";
 import { Row, TableName, TenantCtx } from "@gscdump/contracts";
 import { SearchType } from "gscdump/query";
-interface NodeHarnessOptions {
+export interface NodeHarnessOptions {
   dataDir: string;
   /** Tenant user id. Defaults to `'local'` for single-user CLI installs. */
   userId?: string;
   /** Name of the manifest file under `dataDir`. Defaults to `manifest.json`. */
   manifestFilename?: string;
 }
-interface NodeHarness {
+export interface NodeHarness {
   engine: StorageEngine;
   /**
    * Underlying filesystem-backed DataSource. Exposed so commands that write
@@ -35,5 +35,4 @@ interface NodeHarness {
     keys: string[];
   }>;
 }
-declare function createNodeHarness(opts: NodeHarnessOptions): NodeHarness;
-export { NodeHarness, NodeHarnessOptions, createNodeHarness };
+export declare function createNodeHarness(opts: NodeHarnessOptions): NodeHarness;