@gscdump/engine 0.26.9 → 0.27.0

package/dist/adapters/duckdb-node.mjs

@@ -1,8 +1,8 @@
 import { arrowToRows } from "../arrow-utils.mjs";
 import { createRequire } from "node:module";
 import { join } from "node:path";
-import process from "node:process";
 import { fileURLToPath } from "node:url";
+import process from "node:process";
 import { unlinkSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { ConsoleLogger, NODE_RUNTIME, VoidLogger, createDuckDB } from "@duckdb/duckdb-wasm/dist/duckdb-node-blocking.cjs";

package/dist/sink-node.d.mts

@@ -8,6 +8,7 @@ interface LocalIcebergSinkFullOptions extends LocalIcebergSinkOptions {
   /** Override the writer-script path. Defaults to `scripts/iceberg-writer.py`. */
   writerScript?: string;
 }
+/** Run the PyIceberg writer subprocess for one job, return its parsed result. */
 interface LocalIcebergSink extends Sink {
   /** The catalog namespace the 5 tables live under. */
   readonly namespace: string;

package/dist/sink-node.mjs

@@ -1,44 +1,47 @@
 import { ICEBERG_SCHEMAS } from "./_chunks/schema2.mjs";
-import { execFile } from "node:child_process";
 import { dirname, join } from "node:path";
-import process from "node:process";
 import { fileURLToPath } from "node:url";
-const POC_S3 = {
-	endpoint: "localhost:9100",
-	accessKeyId: "poc",
-	secretAccessKey: "pocpocpoc",
-	region: "us-east-1"
-};
-function resolveWriterScript(override) {
-	if (override) return override;
-	return join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "iceberg-writer.py");
+import process from "node:process";
+function resolvePyIcebergPython(override) {
+	return override ?? process.env["GSCDUMP_ICEBERG_PYTHON"] ?? "python3";
 }
-function runWriter(python, script, job) {
+async function runPyIcebergWriter(options) {
+	const { execFile } = await import("node:child_process");
 	return new Promise((resolve, reject) => {
-		execFile(python, [script], { maxBuffer: 64 * 1024 * 1024 }, (err, stdout, stderr) => {
+		execFile(options.python, [options.script], { maxBuffer: 64 * 1024 * 1024 }, (err, stdout, stderr) => {
 			let parsed;
 			if (stdout.trim()) try {
 				parsed = JSON.parse(stdout);
 			} catch {}
-			if (parsed?.error) {
-				reject(/* @__PURE__ */ new Error(`LocalIcebergSink writer failed: ${parsed.error}`));
+			if (parsed && !(err && options.rejectOnProcessError)) {
+				resolve(parsed);
 				return;
 			}
 			if (err) {
-				reject(/* @__PURE__ */ new Error(`LocalIcebergSink writer process failed (${err.message})${stderr ? `: ${stderr}` : ""}`));
+				if (options.processErrorAsParseFailure) {
+					reject(/* @__PURE__ */ new Error(`${options.label} produced no parseable output (${err.message})${stderr ? `: ${stderr}` : ""}`));
+					return;
+				}
+				reject(/* @__PURE__ */ new Error(`${options.label} process failed (${err.message})${stderr ? `: ${stderr}` : ""}`));
 				return;
 			}
-			if (!parsed) {
-				reject(/* @__PURE__ */ new Error(`LocalIcebergSink writer produced no parseable output: ${stdout || stderr}`));
-				return;
-			}
-			resolve(parsed);
-		}).stdin?.end(JSON.stringify(job));
+			reject(/* @__PURE__ */ new Error(`${options.label} produced no parseable output: ${stdout || stderr}`));
+		}).stdin?.end(JSON.stringify(options.job));
 	});
 }
+const POC_S3 = {
+	endpoint: "localhost:9100",
+	accessKeyId: "poc",
+	secretAccessKey: "pocpocpoc",
+	region: "us-east-1"
+};
+function resolveWriterScript(override) {
+	if (override) return override;
+	return join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "iceberg-writer.py");
+}
 function createLocalIcebergSink(options) {
 	const s3 = options.s3 ?? POC_S3;
-	const python = options.python ?? process.env.GSCDUMP_ICEBERG_PYTHON ?? "python3";
+	const python = resolvePyIcebergPython(options.python);
 	const script = resolveWriterScript(options.writerScript);
 	function buildJob(op, slice, rows) {
 		return {
@@ -61,7 +64,14 @@ function createLocalIcebergSink(options) {
 		capabilities: { appendOnly: true },
 		async emit(slice, rows) {
 			if (rows.length === 0) return { rowCount: 0 };
-			const res = await runWriter(python, script, buildJob("emit", slice, rows));
+			const res = await runPyIcebergWriter({
+				python,
+				script,
+				job: buildJob("emit", slice, rows),
+				label: "LocalIcebergSink writer",
+				rejectOnProcessError: true
+			});
+			if (res.error) throw new Error(`LocalIcebergSink writer failed: ${res.error}`);
 			touched.add(slice.table);
 			return { rowCount: res.rowCount ?? 0 };
 		},

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.26.9",
+  "version": "0.27.0",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -190,8 +190,8 @@
     "drizzle-orm": "1.0.0-rc.3",
     "icebird": "^0.8.10",
     "proper-lockfile": "^4.1.2",
-    "gscdump": "0.26.9",
-    "@gscdump/contracts": "0.26.9"
+    "@gscdump/contracts": "0.27.0",
+    "gscdump": "0.27.0"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",
@@ -200,7 +200,7 @@
     "hyparquet": "^1.26.0",
     "hyparquet-writer": "^0.15.6",
     "tsx": "^4.22.4",
-    "vitest": "^4.1.8"
+    "vitest": "^4.1.9"
   },
   "scripts": {
     "dev": "obuild --stub",