@gscdump/engine 0.37.3 → 0.37.4

package/dist/_chunks/coerce.mjs

@@ -1,9 +1,13 @@
 import { coerceBigIntToNumber } from "@gscdump/lakehouse";
 function coerceRow(row) {
 	let mutated = null;
-	for (const [k, v] of Object.entries(row)) if (typeof v === "bigint") {
-		if (!mutated) mutated = { ...row };
-		mutated[k] = coerceBigIntToNumber(v);
+	for (const k in row) {
+		if (!Object.hasOwn(row, k)) continue;
+		const v = row[k];
+		if (typeof v === "bigint") {
+			if (!mutated) mutated = { ...row };
+			mutated[k] = coerceBigIntToNumber(v);
+		}
 	}
 	return mutated ?? row;
 }

package/dist/_chunks/compaction.mjs

@@ -1,6 +1,6 @@
 import { dayPartition, inferSearchType, mondayOfWeek, monthPartition, objectKey, quarterOfMonth, quarterPartition, weekPartition } from "./layout.mjs";
 import { currentSchemaVersion } from "./schema.mjs";
-import { MS_PER_DAY } from "gscdump";
+import { MS_PER_DAY } from "gscdump/dates";
 const DAILY_PARTITION_RE = /^daily\/(\d{4}-\d{2}-\d{2})$/;
 const WEEKLY_PARTITION_RE = /^weekly\/(\d{4}-\d{2}-\d{2})$/;
 const MONTHLY_PARTITION_RE = /^monthly\/(\d{4}-\d{2})$/;

package/dist/_chunks/engine.mjs

@@ -414,8 +414,13 @@ function createStorageEngine(opts) {
 				if (probed) bytes = probed.bytes;
 			}
 			if (bytes > 104857600) {
-				await dataSource.delete([key]).catch(() => {});
-				throw new Error(`writeDay payload ${bytes} bytes exceeds ${MAX_DAY_BYTES} hard ceiling (table=${ctx.table}, key=${key})`);
+				const sizeError = /* @__PURE__ */ new Error(`writeDay payload ${bytes} bytes exceeds ${MAX_DAY_BYTES} hard ceiling (table=${ctx.table}, key=${key})`);
+				try {
+					await dataSource.delete([key]);
+				} catch (deleteError) {
+					throw new AggregateError([sizeError, deleteError], `${sizeError.message}; failed to delete oversized object`);
+				}
+				throw sizeError;
 			}
 			const entry = {
 				userId: ctx.userId,
@@ -482,8 +487,13 @@ function createStorageEngine(opts) {
 				if (probed) bytes = probed.bytes;
 			}
 			if (bytes > 104857600) {
-				await dataSource.delete([key]).catch(() => {});
-				throw new Error(`writeHour payload ${bytes} bytes exceeds ${MAX_DAY_BYTES} hard ceiling (table=${ctx.table}, key=${key})`);
+				const sizeError = /* @__PURE__ */ new Error(`writeHour payload ${bytes} bytes exceeds ${MAX_DAY_BYTES} hard ceiling (table=${ctx.table}, key=${key})`);
+				try {
+					await dataSource.delete([key]);
+				} catch (deleteError) {
+					throw new AggregateError([sizeError, deleteError], `${sizeError.message}; failed to delete oversized object`);
+				}
+				throw sizeError;
 			}
 			const entry = {
 				userId: ctx.userId,

package/dist/_chunks/entities.mjs

@@ -569,7 +569,25 @@ function isoDate(ms) {
 	return new Date(ms).toISOString().slice(0, 10);
 }
 function hashUrlList(urls) {
-	return hashUrl(urls.map((u) => u.loc).sort().join("\n"));
+	return hashSortedUrlList(urls.map((u) => u.loc).sort());
+}
+function hashSortedUrlList(locs) {
+	let hi = 2166136261;
+	let lo = 3421674724;
+	for (let locIndex = 0; locIndex < locs.length; locIndex++) {
+		const loc = locs[locIndex];
+		const length = loc.length + (locIndex < locs.length - 1 ? 1 : 0);
+		for (let i = 0; i < length; i++) {
+			const c = i < loc.length ? loc.charCodeAt(i) : 10;
+			lo ^= c;
+			const loMul = Math.imul(lo, 435) >>> 0;
+			const carry = Math.floor(lo * 435 / 4294967296);
+			const hiMul = Math.imul(hi, 435) + Math.imul(lo, 1) + carry >>> 0;
+			lo = loMul;
+			hi = hiMul;
+		}
+	}
+	return (hi >>> 0).toString(16).padStart(8, "0") + (lo >>> 0).toString(16).padStart(8, "0");
 }
 function createSitemapStore(opts) {
 	const ds = opts.dataSource;
@@ -621,7 +639,7 @@ function createSitemapStore(opts) {
 			for await (const rec of this.loadUrls(ctx, feedpath, { includeRemoved: true })) priorByHash.set(rec.urlHash, rec);
 			const livePrior = Array.from(priorByHash.values()).filter((r) => r.removedAt == null);
 			if (livePrior.length > 0) {
-				if (hashUrl(livePrior.map((r) => String(r.loc)).sort().join("\n")) === contentHash) return {
+				if (hashSortedUrlList(livePrior.map((r) => String(r.loc)).sort()) === contentHash) return {
 					added: 0,
 					removed: 0,
 					kept: livePrior.length,

package/dist/_chunks/layout.mjs

@@ -1,4 +1,4 @@
-import { MS_PER_DAY, toIsoDate } from "gscdump";
+import { MS_PER_DAY, toIsoDate } from "gscdump/dates";
 function dayPartition(date) {
 	return `daily/${date}`;
 }

package/dist/adapters/duckdb-node.mjs

@@ -1,9 +1,9 @@
 import { arrowToRows } from "../arrow-utils.mjs";
 import { createRequire } from "node:module";
+import process from "node:process";
 import { unlinkSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import process from "node:process";
 import { fileURLToPath } from "node:url";
 import { ConsoleLogger, NODE_RUNTIME, VoidLogger, createDuckDB } from "@duckdb/duckdb-wasm/dist/duckdb-node-blocking.cjs";
 const require_ = createRequire(typeof __filename !== "undefined" ? __filename : typeof import.meta !== "undefined" ? fileURLToPath(import.meta.url) : process.cwd());
@@ -64,10 +64,15 @@ function createNodeDuckDBHandle(opts = {}) {
 			for (const name of names) {
 				try {
 					db.dropFile(name);
-				} catch {}
+				} catch (error) {
+					const message = error instanceof Error ? error.message : String(error);
+					if (!/not found|does not exist|unknown file/i.test(message)) throw error;
+				}
 				try {
 					unlinkSync(name);
-				} catch {}
+				} catch (error) {
+					if (error.code !== "ENOENT") throw error;
+				}
 			}
 		},
 		makeTempPath(ext) {

package/dist/adapters/filesystem.mjs

@@ -95,7 +95,11 @@ function createFilesystemManifestStore(opts) {
 		const tmp = `${manifestPath}.${randomBytes(6).toString("hex")}.tmp`;
 		await writeFile(tmp, JSON.stringify(data), "utf8");
 		await rename(tmp, manifestPath).catch(async (err) => {
-			await unlink(tmp).catch(() => {});
+			try {
+				await unlink(tmp);
+			} catch (cleanupError) {
+				throw new AggregateError([err, cleanupError], `failed to replace manifest and remove temporary file ${tmp}`);
+			}
 			throw err;
 		});
 	}
@@ -112,7 +116,7 @@ function createFilesystemManifestStore(opts) {
 	async function drain() {
 		if (running) return;
 		running = true;
-		while (queue.length > 0) await queue.shift()().catch(() => {});
+		while (queue.length > 0) await queue.shift()();
 		running = false;
 	}
 	async function registerVersionsImpl(newEntries, superseding) {

package/dist/index.mjs

@@ -41,6 +41,16 @@ function createNoopIngestAccumulator() {
 function createIngestAccumulator(opts) {
 	const { engine, ctx, hooks, ...accOpts } = opts;
 	const acc = createRowAccumulator(accOpts);
+	function reportHookFailure(hook, error) {
+		console.warn(`[gscdump/engine] ${hook} hook failed`, error);
+	}
+	async function notifyWriteError(info) {
+		try {
+			await hooks.onWriteError(info);
+		} catch (hookError) {
+			reportHookFailure("onWriteError", hookError);
+		}
+	}
 	async function writeOne(table, date, rows) {
 		const scope = scopeOf(ctx, table, date);
 		return (ctx.grain === "hour" ? engine.writeHour ?? (() => Promise.reject(/* @__PURE__ */ new Error("ingest accumulator: grain=hour requires engine.writeHour"))) : engine.writeDay)(scope, rows).then(() => engine.setSyncState(scope, "done")).then(async () => {
@@ -54,23 +64,31 @@ function createIngestAccumulator(opts) {
 				rows: rows.length
 			};
 		}).catch(async (err) => {
-			await hooks.onWriteError({
+			await notifyWriteError({
 				table,
 				date,
 				error: err
-			}).catch(() => {});
+			});
 			return { ok: false };
 		});
 	}
 	async function recover(table, date) {
 		const scope = scopeOf(ctx, table, date);
-		await engine.setSyncState(scope, "failed", { error: "mid-continuation-skip" }).catch(() => {});
+		try {
+			await engine.setSyncState(scope, "failed", { error: "mid-continuation-skip" });
+		} catch (stateError) {
+			await notifyWriteError({
+				table,
+				date,
+				error: stateError
+			});
+		}
 		return hooks.onRecover(table, date).catch(async (err) => {
-			await hooks.onWriteError({
+			await notifyWriteError({
 				table,
 				date,
 				error: err
-			}).catch(() => {});
+			});
 			return false;
 		});
 	}
@@ -86,18 +104,18 @@ function createIngestAccumulator(opts) {
 				const tasks = [];
 				for (const [table, byDate] of buckets) for (const date of byDate.keys()) tasks.push(recover(table, date));
 				const results = await Promise.all(tasks).catch(async (err) => {
-					await hooks.onWriteError({
+					await notifyWriteError({
 						table: null,
 						date: null,
 						error: err
-					}).catch(() => {});
+					});
 					return [];
 				});
-				if (overflowed) await hooks.onWriteError({
+				if (overflowed) await notifyWriteError({
 					table: null,
 					date: null,
 					error: /* @__PURE__ */ new Error(`ingest accumulator overflow at ${totalRows} rows; recovering via forced re-sync`)
-				}).catch(() => {});
+				});
 				return {
 					flushed: 0,
 					recovered: results.filter(Boolean).length,
@@ -115,10 +133,14 @@ function createIngestAccumulator(opts) {
 				flushed++;
 				rowsWritten += o.rows;
 			} else failed++;
-			if (flushed > 0) await hooks.onJobComplete?.({
-				flushed,
-				rowsWritten
-			}).catch(() => {});
+			if (flushed > 0 && hooks.onJobComplete) try {
+				await hooks.onJobComplete({
+					flushed,
+					rowsWritten
+				});
+			} catch (hookError) {
+				reportHookFailure("onJobComplete", hookError);
+			}
 			return {
 				flushed,
 				recovered: 0,

package/dist/ingest.mjs

@@ -18,7 +18,37 @@ const TABLE_DIMS = {
 	],
 	hourly_pages: ["hour", "page"]
 };
+const COMMON_HTTP_AUTHORITY_RE = /^([\w.~-]+)(?::(\d{1,5}))?$/;
+const IPV4_HOST_RE = /^\d+(?:\.\d+){3}$/;
+function isCommonHttpAuthority(value) {
+	const match = COMMON_HTTP_AUTHORITY_RE.exec(value);
+	if (!match) return false;
+	if (match[2] !== void 0 && Number(match[2]) > 65535) return false;
+	const host = match[1];
+	if (IPV4_HOST_RE.test(host) && host.split(".").some((part) => Number(part) > 255)) return false;
+	return true;
+}
 function toPath(gscUrl) {
+	const authorityStart = gscUrl.startsWith("https://") ? 8 : gscUrl.startsWith("http://") ? 7 : -1;
+	if (authorityStart > 0) {
+		const slash = gscUrl.indexOf("/", authorityStart);
+		const query = gscUrl.indexOf("?", authorityStart);
+		const hash = gscUrl.indexOf("#", authorityStart);
+		const backslash = gscUrl.indexOf("\\", authorityStart);
+		let firstDelimiter = gscUrl.length;
+		if (slash >= 0 && slash < firstDelimiter) firstDelimiter = slash;
+		if (query >= 0 && query < firstDelimiter) firstDelimiter = query;
+		if (hash >= 0 && hash < firstDelimiter) firstDelimiter = hash;
+		const authority = gscUrl.slice(authorityStart, firstDelimiter);
+		if (backslash < 0 && !/\s/.test(gscUrl) && firstDelimiter > authorityStart && isCommonHttpAuthority(authority)) {
+			if (firstDelimiter !== slash) return "/";
+			let pathEnd = gscUrl.length;
+			if (query > slash && query < pathEnd) pathEnd = query;
+			if (hash > slash && hash < pathEnd) pathEnd = hash;
+			const path = gscUrl.slice(slash, pathEnd);
+			if (!/[^\x20-\x7E]/.test(path) && !/(?:^|\/)(?:\.|%2e){1,2}(?:\/|$)/i.test(path)) return path;
+		}
+	}
 	try {
 		return new URL(gscUrl).pathname;
 	} catch {
@@ -209,19 +239,6 @@ function createRowAccumulator(options = {}) {
 	const latestDate = /* @__PURE__ */ new Map();
 	let total = 0;
 	let overflowed = false;
-	function bucketFor(table, date) {
-		let byDate = buckets.get(table);
-		if (!byDate) {
-			byDate = /* @__PURE__ */ new Map();
-			buckets.set(table, byDate);
-		}
-		let rows = byDate.get(date);
-		if (!rows) {
-			rows = [];
-			byDate.set(date, rows);
-		}
-		return rows;
-	}
 	return {
 		get totalRows() {
 			return total;
@@ -231,20 +248,30 @@ function createRowAccumulator(options = {}) {
 		},
 		push(table, rows) {
 			if (overflowed) return false;
+			let byDate = buckets.get(table);
+			let newestDate = trackDateBoundary ? latestDate.get(table) : void 0;
 			for (const r of rows) {
 				const t = transformGscRow(table, r, options);
 				if (!t || !t.date) continue;
-				bucketFor(table, t.date).push(t.row);
-				total++;
-				if (trackDateBoundary) {
-					const prev = latestDate.get(table);
-					if (!prev || t.date > prev) latestDate.set(table, t.date);
+				if (!byDate) {
+					byDate = /* @__PURE__ */ new Map();
+					buckets.set(table, byDate);
 				}
+				let dateRows = byDate.get(t.date);
+				if (!dateRows) {
+					dateRows = [];
+					byDate.set(t.date, dateRows);
+				}
+				dateRows.push(t.row);
+				total++;
+				if (trackDateBoundary && (!newestDate || t.date > newestDate)) newestDate = t.date;
 				if (total > maxRows) {
+					if (newestDate) latestDate.set(table, newestDate);
 					overflowed = true;
 					return false;
 				}
 			}
+			if (newestDate) latestDate.set(table, newestDate);
 			return true;
 		},
 		drain() {

package/dist/period/index.mjs

@@ -1,9 +1,9 @@
-import { MS_PER_DAY, daysAgo, toIsoDate } from "gscdump";
+import { MS_PER_DAY, daysAgoUtc, toIsoDate } from "gscdump/dates";
 function defaultEndDate() {
-	return daysAgo(3);
+	return daysAgoUtc(3);
 }
 function defaultStartDate() {
-	return daysAgo(31);
+	return daysAgoUtc(31);
 }
 function periodOf(params) {
 	return {

package/dist/rollups.d.mts

@@ -332,26 +332,6 @@ declare const topKeywords28dRollup: RollupDef;
  * coexist during a migration.
  */
 declare const topKeywords28dParquetRollup: RollupDef;
-/**
- * Materialises canonical-query variant grouping so the read path
- * (`buildExtrasQueries` in `resolver/compile.ts`) becomes a passthrough scan
- * instead of two window passes (`ROW_NUMBER`/`COUNT` over `PARTITION BY
- * query_canonical`) plus a `GROUP_CONCAT` over the whole `queries` table on
- * every request — work that is single-threaded under DuckDB-WASM/Workers and
- * scales with table size. See ADR-0017.
- *
- * One row per `query_canonical` group, columns named 1:1 with the live query's
- * output (`joinKey`, `variantCount`, `canonicalName`, `variants`) so
- * `mergeExtras` consumes either source unchanged. `variants` packs the top-10
- * variants as `query:::clicks:::impressions:::position` joined by `||`,
- * identical to the live composer.
- *
- * Full history (`windowDays: null`), not a trailing window: grouping metadata
- * is global (which variant is canonical, how many variants exist) and stays
- * stable across requests rather than shifting with each query's date range.
- * Reflects the last sync/compaction, not the live tail — readers that need the
- * tail can layer a recent-overlay later (the envelope carries `builtAt`).
- */
 declare const queryCanonicalVariantsRollup: RollupDef;
 /**
  * Canonical-grained fact aggregate (ADR-0018 Gap 2): pre-sums the raw

package/dist/rollups.mjs

@@ -3,7 +3,7 @@ import { engineErrors } from "./errors.mjs";
 import { encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
 import { createIndexingMetadataStore, createQueryDimStore, createSitemapStore, inspectionParquetKey, sitemapUrlsIndexPrefix } from "./_chunks/entities.mjs";
 import { encodeJsonBigintSafe } from "@gscdump/lakehouse";
-import { MS_PER_DAY } from "gscdump";
+import { MS_PER_DAY } from "gscdump/dates";
 function rollupPrefix(ctx, searchType) {
 	const base = ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/rollups` : `u_${ctx.userId}/rollups`;
 	return searchType !== void 0 && searchType !== "web" ? `${base}/${searchType}` : base;
@@ -569,6 +569,24 @@ const topKeywords28dParquetRollup = {
 		}));
 	}
 };
+const CANONICAL_VARIANT_LIMIT = 10;
+function retainCanonicalVariant(bucket, query, clicks, impressions, sumPos) {
+	bucket.count++;
+	let insertAt = 0;
+	while (insertAt < bucket.top.length) {
+		const existing = bucket.top[insertAt];
+		if (clicks > existing.clicks || clicks === existing.clicks && query.localeCompare(existing.query) < 0) break;
+		insertAt++;
+	}
+	if (insertAt >= CANONICAL_VARIANT_LIMIT) return;
+	bucket.top.splice(insertAt, 0, {
+		query,
+		clicks,
+		impressions,
+		sumPos
+	});
+	if (bucket.top.length > CANONICAL_VARIANT_LIMIT) bucket.top.pop();
+}
 const queryCanonicalVariantsRollup = {
 	id: "query_canonical_variants",
 	windowDays: null,
@@ -641,28 +659,27 @@ const queryCanonicalVariantsRollup = {
 			});
 			for (const r of rows) {
 				const joinKey = String(r.joinKey);
-				const list = byCanonical.get(joinKey);
-				const v = {
-					query: String(r.query),
-					clicks: Number(r.clicks),
-					impressions: Number(r.impressions),
-					sumPos: Number(r.sum_pos)
-				};
-				if (list) list.push(v);
-				else byCanonical.set(joinKey, [v]);
+				let bucket = byCanonical.get(joinKey);
+				if (!bucket) {
+					bucket = {
+						count: 0,
+						top: []
+					};
+					byCanonical.set(joinKey, bucket);
+				}
+				retainCanonicalVariant(bucket, String(r.query), Number(r.clicks), Number(r.impressions), Number(r.sum_pos));
 			}
 			if (rows.length < 2e4) break;
 			cursor = String(rows[rows.length - 1].query);
 		}
 		const out = [];
-		for (const [joinKey, variants] of byCanonical) {
-			variants.sort((a, b) => b.clicks - a.clicks || a.query.localeCompare(b.query));
-			const canonicalName = variants[0]?.query ?? null;
-			const top = variants.slice(0, 10).filter((v) => v.impressions > 0);
+		for (const [joinKey, bucket] of byCanonical) {
+			const canonicalName = bucket.top[0]?.query ?? null;
+			const top = bucket.top.filter((v) => v.impressions > 0);
 			const variantsStr = top.length === 0 ? null : top.map((v) => `${v.query}:::${v.clicks}:::${v.impressions}:::${(v.sumPos / v.impressions + 1).toFixed(1)}`).join("||");
 			out.push({
 				joinKey,
-				variantCount: BigInt(variants.length),
+				variantCount: BigInt(bucket.count),
 				canonicalName,
 				variants: variantsStr
 			});
@@ -1083,6 +1100,40 @@ const sitemapHealthRollup = {
 		};
 	}
 };
+const RECENT_SITEMAP_CHANGE_LIMIT = 200;
+function isLessRecent(a, b) {
+	return a.at < b.at || a.at === b.at && a.sequence > b.sequence;
+}
+function retainRecentSitemapChange(heap, change) {
+	if (heap.length < RECENT_SITEMAP_CHANGE_LIMIT) {
+		heap.push(change);
+		let index = heap.length - 1;
+		while (index > 0) {
+			const parent = index - 1 >> 1;
+			if (!isLessRecent(heap[index], heap[parent])) break;
+			const parentValue = heap[parent];
+			heap[parent] = heap[index];
+			heap[index] = parentValue;
+			index = parent;
+		}
+		return;
+	}
+	if (!isLessRecent(heap[0], change)) return;
+	heap[0] = change;
+	let index = 0;
+	for (;;) {
+		const left = index * 2 + 1;
+		const right = left + 1;
+		let leastRecent = index;
+		if (left < heap.length && isLessRecent(heap[left], heap[leastRecent])) leastRecent = left;
+		if (right < heap.length && isLessRecent(heap[right], heap[leastRecent])) leastRecent = right;
+		if (leastRecent === index) return;
+		const childValue = heap[leastRecent];
+		heap[leastRecent] = heap[index];
+		heap[index] = childValue;
+		index = leastRecent;
+	}
+}
 const sitemapChanges28dRollup = {
 	id: "sitemap_changes_28d",
 	windowDays: 28,
@@ -1094,6 +1145,7 @@ const sitemapChanges28dRollup = {
 		const counts = /* @__PURE__ */ new Map();
 		const addedTop = [];
 		const removedTop = [];
+		let sequence = 0;
 		function key(k) {
 			return `${k.day}\x00${k.feedpath}`;
 		}
@@ -1112,20 +1164,18 @@ const sitemapChanges28dRollup = {
 				added: 0,
 				removed: 0
 			};
+			const change = {
+				loc: d.loc,
+				feedpath: d.feedpath,
+				at: d.at,
+				sequence: sequence++
+			};
 			if (d.op === "added") {
 				cur.added += 1;
-				addedTop.push({
-					loc: d.loc,
-					feedpath: d.feedpath,
-					at: d.at
-				});
+				retainRecentSitemapChange(addedTop, change);
 			} else {
 				cur.removed += 1;
-				removedTop.push({
-					loc: d.loc,
-					feedpath: d.feedpath,
-					at: d.at
-				});
+				retainRecentSitemapChange(removedTop, change);
 			}
 			counts.set(k, cur);
 		}
@@ -1133,12 +1183,15 @@ const sitemapChanges28dRollup = {
 			if (a.day !== b.day) return a.day < b.day ? -1 : 1;
 			return a.feedpath < b.feedpath ? -1 : 1;
 		});
-		addedTop.sort((a, b) => b.at - a.at);
-		removedTop.sort((a, b) => b.at - a.at);
+		const toRecentList = (heap) => heap.sort((a, b) => b.at - a.at || a.sequence - b.sequence).map(({ loc, feedpath, at }) => ({
+			loc,
+			feedpath,
+			at
+		}));
 		return {
 			days,
-			topAdded: addedTop.slice(0, 200),
-			topRemoved: removedTop.slice(0, 200)
+			topAdded: toRecentList(addedTop),
+			topRemoved: toRecentList(removedTop)
 		};
 	}
 };

package/dist/sink-node.mjs

@@ -1,5 +1,37 @@
 import { ICEBERG_SCHEMAS, assertIcebergTable } from "./_chunks/schema2.mjs";
-import { resolvePyIcebergPython, runPyIcebergWriter } from "@gscdump/lakehouse";
+import process from "node:process";
+const PYICEBERG_PYTHON_ENV = "GSCDUMP_ICEBERG_PYTHON";
+const DEFAULT_PYICEBERG_PYTHON = "python3";
+function resolvePyIcebergPython(override) {
+	return override ?? process.env[PYICEBERG_PYTHON_ENV] ?? DEFAULT_PYICEBERG_PYTHON;
+}
+async function runPyIcebergWriter(options) {
+	const { execFile } = await import("node:child_process");
+	return new Promise((resolve, reject) => {
+		execFile(options.python, [options.script], { maxBuffer: 64 * 1024 * 1024 }, (err, stdout, stderr) => {
+			let parsed;
+			let parseError;
+			if (stdout.trim()) try {
+				parsed = JSON.parse(stdout);
+			} catch (error) {
+				parseError = error;
+			}
+			if (parsed && !(err && options.rejectOnProcessError)) {
+				resolve(parsed);
+				return;
+			}
+			if (err) {
+				if (options.processErrorAsParseFailure) {
+					reject(/* @__PURE__ */ new Error(`${options.label} produced no parseable output (${err.message})${stderr ? `: ${stderr}` : ""}`));
+					return;
+				}
+				reject(/* @__PURE__ */ new Error(`${options.label} process failed (${err.message})${stderr ? `: ${stderr}` : ""}`));
+				return;
+			}
+			reject(new Error(`${options.label} produced no parseable output: ${stdout || stderr}`, { cause: parseError }));
+		}).stdin?.end(JSON.stringify(options.job));
+	});
+}
 function createIcebergOverwriteWriter(opts) {
 	const { catalog, backend } = opts;
 	async function overwriteSlice(slice, rows) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.37.3",
+  "version": "0.37.4",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,12 +181,13 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/lakehouse": "0.37.3",
-    "@gscdump/contracts": "0.37.3",
-    "gscdump": "0.37.3"
+    "gscdump": "0.37.4",
+    "@gscdump/contracts": "0.37.4",
+    "@gscdump/lakehouse": "0.37.4"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",
+    "@types/node": "^26.1.0",
     "@types/proper-lockfile": "^4.1.4",
     "aws4fetch": "^1.0.20",
     "hyparquet": "^1.26.2",