@gscdump/engine 0.33.10 → 0.35.2

package/dist/iceberg/index.mjs

@@ -1,85 +1,11 @@
 import { TABLE_METADATA } from "../_chunks/schema.mjs";
 import { engineErrors } from "../errors.mjs";
-import { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable } from "../_chunks/schema2.mjs";
-import { cachingResolver, icebergAppend, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver } from "../_chunks/libs/icebird.mjs";
+import { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, gscDataset, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable } from "../_chunks/schema2.mjs";
+import { icebergCreateTable, icebergManifests, restCatalogLoadTable } from "../_chunks/libs/icebird.mjs";
 import { err, ok } from "gscdump/result";
-async function cacheGet(cache, key, now) {
-	const boxed = await cache.storage.getItem(key).catch(() => null);
-	if (!boxed || typeof boxed.exp !== "number" || boxed.exp <= now) return void 0;
-	return boxed.v;
-}
-function cachePut(cache, key, value, ttlMs, now) {
-	const boxed = {
-		v: value,
-		exp: now + ttlMs
-	};
-	const write = cache.storage.setItem(key, boxed, { ttl: Math.ceil(ttlMs / 1e3) }).catch(() => {});
-	if (cache.defer) {
-		cache.defer(write);
-		return Promise.resolve();
-	}
-	return write;
-}
-const SITE_ID_FIELD_INDEX = ICEBERG_PARTITION_SPEC.findIndex((f) => f.sourceColumn === "site_id" && f.transform === "identity");
-const SEARCH_TYPE_FIELD_INDEX = ICEBERG_PARTITION_SPEC.findIndex((f) => f.sourceColumn === "search_type" && f.transform === "identity");
-const DATE_MONTH_FIELD_INDEX = ICEBERG_PARTITION_SPEC.findIndex((f) => f.transform === "month");
-function toUint8(bytes) {
-	if (bytes == null) return null;
-	return bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes);
-}
-function decodeString(bytes) {
-	const u = toUint8(bytes);
-	return u == null ? null : new TextDecoder().decode(u);
-}
-function decodeInt(bytes) {
-	const u = toUint8(bytes);
-	if (u == null) return null;
-	return new DataView(u.buffer, u.byteOffset, u.byteLength).getInt32(0, true);
-}
-function buildPartitionFilter(siteId, searchType, wantedMonths, encoding = "int") {
-	return (partitions) => {
-		const parts = partitions;
-		if (!parts || parts.length === 0) return true;
-		if (encoding === "string") {
-			const siteStr = String(siteId);
-			const searchStr = String(searchType);
-			const siteSummary = parts[SITE_ID_FIELD_INDEX];
-			if (siteSummary && (siteSummary.lower_bound != null || siteSummary.upper_bound != null)) {
-				const lo = decodeString(siteSummary.lower_bound);
-				const hi = decodeString(siteSummary.upper_bound);
-				if (lo != null && hi != null && (siteStr < lo || siteStr > hi)) return false;
-			}
-			const searchTypeSummary = parts[SEARCH_TYPE_FIELD_INDEX];
-			if (searchTypeSummary && (searchTypeSummary.lower_bound != null || searchTypeSummary.upper_bound != null)) {
-				const lo = decodeString(searchTypeSummary.lower_bound);
-				const hi = decodeString(searchTypeSummary.upper_bound);
-				if (lo != null && hi != null && (searchStr < lo || searchStr > hi)) return false;
-			}
-		}
-		const monthSummary = parts[DATE_MONTH_FIELD_INDEX];
-		if (monthSummary && (monthSummary.lower_bound != null || monthSummary.upper_bound != null)) {
-			const lo = decodeInt(monthSummary.lower_bound);
-			const hi = decodeInt(monthSummary.upper_bound);
-			if (lo != null && hi != null) {
-				let anyInRange = false;
-				for (const wm of wantedMonths) if (wm >= lo && wm <= hi) {
-					anyInRange = true;
-					break;
-				}
-				if (!anyInRange) return false;
-			}
-		}
-		return true;
-	};
-}
-const ICEBERG_TYPE_MAP = {
-	STRING: "string",
-	INT: "int",
-	LONG: "long",
-	DOUBLE: "double",
-	DATE: "date"
-};
-function icebergSchemaFor(table, encoding = "int") {
+import { ICEBERG_TYPE_MAP, connectIcebergCatalog, dropIcebergTables as dropIcebergTables$1, ensureIcebergNamespace, listIcebergTables, resolveIcebergDataFiles } from "@gscdump/lakehouse";
+import { icebergAppendRetrying, isCommitRateLimited } from "@gscdump/lakehouse/unsafe-raw";
+function icebergSchemaFor(table, encoding = DEFAULT_PARTITION_KEY_ENCODING) {
 	return {
 		"type": "struct",
 		"schema-id": 0,
@@ -91,7 +17,7 @@ function icebergSchemaFor(table, encoding = "int") {
 		}))
 	};
 }
-function icebergPartitionSpecFor(table, encoding = "int") {
+function icebergPartitionSpecFor(table, encoding = DEFAULT_PARTITION_KEY_ENCODING) {
 	const fields = icebergSchemasFor(encoding)[table].columns;
 	const fieldId = (name) => {
 		const col = fields.find((c) => c.name === name);
@@ -108,7 +34,7 @@ function icebergPartitionSpecFor(table, encoding = "int") {
 		}))
 	};
 }
-function icebergSortOrderFor(table, encoding = "int") {
+function icebergSortOrderFor(table, encoding = DEFAULT_PARTITION_KEY_ENCODING) {
 	const fields = icebergSchemasFor(encoding)[table].columns;
 	const fieldId = (name) => {
 		const col = fields.find((c) => c.name === name);
@@ -125,112 +51,7 @@ function icebergSortOrderFor(table, encoding = "int") {
 		}))
 	};
 }
-const CATALOG_CONFIG_TTL_MS = 3600 * 1e3;
-function catalogConfigKey(config) {
-	return `gsc-catalog-cfg\0${config.catalogUri}\0${config.warehouse}`;
-}
-async function connectIcebergCatalog(config, opts = {}) {
-	const now = (opts.clock ?? Date.now)();
-	const requestInit = { headers: { Authorization: `Bearer ${config.catalogToken}` } };
-	let catalog;
-	if (opts.cache) {
-		const cached = await cacheGet(opts.cache, catalogConfigKey(config), now);
-		if (cached) catalog = Object.freeze({
-			type: "rest",
-			url: cached.url,
-			prefix: cached.prefix,
-			defaults: cached.defaults,
-			overrides: cached.overrides,
-			requestInit
-		});
-	}
-	if (!catalog) {
-		catalog = await restCatalogConnect({
-			url: config.catalogUri,
-			warehouse: config.warehouse,
-			requestInit
-		});
-		if (opts.cache) {
-			const toCache = {
-				url: catalog.url,
-				prefix: catalog.prefix,
-				defaults: catalog.defaults,
-				overrides: catalog.overrides
-			};
-			await cachePut(opts.cache, catalogConfigKey(config), toCache, CATALOG_CONFIG_TTL_MS, now);
-		}
-	}
-	const resolver = cachingResolver(s3SignedResolver({
-		accessKeyId: config.s3.accessKeyId,
-		secretAccessKey: config.s3.secretAccessKey,
-		region: config.s3.region ?? "auto",
-		endpoint: config.s3.endpoint,
-		pathStyle: true
-	}));
-	return {
-		catalog,
-		resolver,
-		namespace: config.namespace
-	};
-}
-const APPEND_ID_SUMMARY_KEY = "gscdump.append-id";
-const APPEND_LANDED_SCAN_DEPTH = 25;
-function isCommitRateLimited(err) {
-	if (err && typeof err === "object" && err.status === 429) return true;
-	const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
-	return msg.includes("429") || msg.includes("too many commits") || msg.includes("rate limit");
-}
-function defaultCommitSleep(ms) {
-	return new Promise((resolve) => setTimeout(resolve, ms));
-}
-async function icebergAppendRetrying(args, options = {}) {
-	const maxAttempts = options.maxAttempts ?? 6;
-	const baseDelayMs = options.baseDelayMs ?? 1e3;
-	const maxDelayMs = options.maxDelayMs ?? 2e4;
-	const sleep = options.sleep ?? defaultCommitSleep;
-	const random = options.random ?? Math.random;
-	const appendId = options.appendId ?? await deriveAppendId(args);
-	const stampedArgs = {
-		...args,
-		snapshotProperties: {
-			...args.snapshotProperties,
-			[APPEND_ID_SUMMARY_KEY]: appendId
-		}
-	};
-	if (await appendAlreadyLanded(args, appendId).catch(() => false)) return;
-	for (let attempt = 0; attempt < maxAttempts; attempt++) {
-		const err = await icebergAppend(stampedArgs).then(() => void 0, (e) => e);
-		if (err === void 0) return;
-		if (await appendAlreadyLanded(args, appendId).catch(() => false)) return;
-		if (!isCommitRateLimited(err) || attempt === maxAttempts - 1) throw err;
-		const ceiling = Math.min(maxDelayMs, baseDelayMs * 2 ** attempt);
-		await sleep(Math.floor(random() * ceiling));
-	}
-}
-async function deriveAppendId(args) {
-	const records = args.records ?? [];
-	if (records.length === 0) return globalThis.crypto.randomUUID();
-	const rowSig = (r) => Object.keys(r).sort().map((k) => `${k}=${String(r[k])}`).join("
");
-	const body = records.map(rowSig).sort().join("");
-	const digest = await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(body));
-	return Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, "0")).join("");
-}
-async function appendAlreadyLanded(args, appendId) {
-	const a = args;
-	if (a.catalog?.type !== "rest" || a.namespace == null || a.table == null) return false;
-	const { metadata } = await restCatalogLoadTable(a.catalog, {
-		namespace: a.namespace,
-		table: a.table
-	});
-	const snapshots = metadata.snapshots ?? [];
-	const from = Math.max(0, snapshots.length - APPEND_LANDED_SCAN_DEPTH);
-	for (let i = snapshots.length - 1; i >= from; i--) if (snapshots[i]?.summary?.[APPEND_ID_SUMMARY_KEY] === appendId) return true;
-	return false;
-}
-async function ensureIcebergNamespace(conn) {
-	await restCatalogCreateNamespace(conn.catalog, { namespace: conn.namespace }).catch(() => {});
-}
-async function createIcebergTables(conn, tables = ICEBERG_TABLES, encoding = "int") {
+async function createIcebergTables(conn, tables = ICEBERG_TABLES, encoding = DEFAULT_PARTITION_KEY_ENCODING) {
 	const results = [];
 	for (const table of tables) await icebergCreateTable({
 		catalog: conn.catalog,
@@ -248,153 +69,32 @@ async function createIcebergTables(conn, tables = ICEBERG_TABLES, encoding = "in
 	}));
 	return results;
 }
-async function listIcebergTables(conn) {
-	return (await restCatalogListTables(conn.catalog, { namespace: conn.namespace })).map((t) => t.name).sort();
-}
-const SNAPSHOT_REF_TTL_MS = 1800 * 1e3;
-const RESOLVED_FILES_TTL_MS = 1440 * 60 * 1e3;
-const METADATA_TTL_MS = 1440 * 60 * 1e3;
-const MAX_CACHED_METADATA_BYTES = 2 * 1024 * 1024;
-function snapshotRefKey(namespace, table) {
-	return `gsc-snapref\0${namespace}\0${table}`;
-}
-function metadataRefKey(namespace, table, snapshotId) {
-	return `gsc-snapmeta\0${namespace}\0${table}\0${snapshotId}`;
-}
-function resolvedFilesKey(namespace, table, snapshotId, siteId, searchType, wantedMonths) {
-	return `gsc-files\0${namespace}\0${table}\0${snapshotId}\0${siteId}\0${searchType}\0${[...wantedMonths].sort((a, b) => a - b).join(",")}`;
-}
-function monthsInRange(range) {
-	const [sy, sm] = range.start.split("-").map(Number);
-	const [ey, em] = range.end.split("-").map(Number);
-	const out = [];
-	let y = sy;
-	let m = sm;
-	while (y < ey || y === ey && m <= em) {
-		out.push(`${y}-${String(m).padStart(2, "0")}`);
-		m++;
-		if (m > 12) {
-			m = 1;
-			y++;
-		}
-	}
-	return out;
-}
-function monthsSinceEpoch(ym) {
-	const [y, m] = ym.split("-").map(Number);
-	return (y - 1970) * 12 + (m - 1);
-}
-function stripBucket(filePath) {
-	if (!filePath.startsWith("s3://")) return filePath;
-	const rest = filePath.slice(5);
-	const slash = rest.indexOf("/");
-	return slash >= 0 ? rest.slice(slash + 1) : rest;
-}
-async function loadSnapshotId(conn, opts, now) {
-	if (opts.cache) {
-		const cached = await cacheGet(opts.cache, snapshotRefKey(conn.namespace, opts.table), now);
-		if (cached !== void 0) return {
-			snapshotId: cached,
-			metadata: null
-		};
-	}
-	const { metadata } = await restCatalogLoadTable(conn.catalog, {
-		namespace: conn.namespace,
-		table: opts.table
-	});
-	const raw = metadata["current-snapshot-id"];
-	const snapshotId = raw == null ? null : String(raw);
-	if (opts.cache) {
-		await cachePut(opts.cache, snapshotRefKey(conn.namespace, opts.table), snapshotId, SNAPSHOT_REF_TTL_MS, now);
-		if (snapshotId != null) {
-			if (JSON.stringify(metadata).length <= MAX_CACHED_METADATA_BYTES) await cachePut(opts.cache, metadataRefKey(conn.namespace, opts.table, snapshotId), metadata, METADATA_TTL_MS, now);
-		}
-	}
-	return {
-		snapshotId,
-		metadata
-	};
+async function dropIcebergTables(conn, tables) {
+	return (await dropIcebergTables$1(conn, tables)).map((r) => ({
+		table: r.table,
+		outcome: r.ok ? ok(void 0) : err(engineErrors.icebergTableOpFailed("drop", r.table, r.error))
+	}));
 }
 async function listIcebergDataFiles(conn, opts) {
-	const profiler = opts.profiler;
-	const now = (opts.clock ?? Date.now)();
-	const wantedMonths = new Set(monthsInRange(opts.range).map(monthsSinceEpoch));
-	const endSnapshot = profiler?.start("iceberg.snapshot");
-	let { snapshotId, metadata } = await loadSnapshotId(conn, opts, now);
-	endSnapshot?.({ cached: metadata == null && snapshotId != null });
-	if (snapshotId == null) return [];
-	const filesKey = resolvedFilesKey(conn.namespace, opts.table, snapshotId, opts.siteId, opts.searchType, wantedMonths);
-	if (opts.cache) {
-		const endCache = profiler?.start("iceberg.cache");
-		const cached = await cacheGet(opts.cache, filesKey, now);
-		endCache?.({ hit: cached !== void 0 });
-		if (cached !== void 0) return cached;
-	}
-	if (!metadata && opts.cache) {
-		const cachedMeta = await cacheGet(opts.cache, metadataRefKey(conn.namespace, opts.table, snapshotId), now);
-		if (cachedMeta != null) metadata = cachedMeta;
-	}
-	if (!metadata) {
-		const reloaded = await loadSnapshotId(conn, {
-			...opts,
-			cache: void 0
-		}, now);
-		snapshotId = reloaded.snapshotId;
-		metadata = reloaded.metadata;
-		if (snapshotId == null || !metadata) return [];
-	}
-	const endWalk = profiler?.start("iceberg.walk");
-	const partitionFilter = buildPartitionFilter(opts.siteId, opts.searchType, wantedMonths, opts.encoding ?? "int");
-	const manifests = await icebergManifests({
-		metadata,
-		resolver: conn.resolver,
-		partitionFilter
-	});
-	const wantSite = String(opts.siteId);
-	const wantSearch = String(opts.searchType);
-	const out = [];
-	for (const m of manifests) for (const entry of m.entries) {
-		if (entry.status === 2) continue;
-		const df = entry.data_file;
-		if (df.content !== 0) continue;
-		const part = df.partition;
-		if (String(part.site_id) !== wantSite) continue;
-		if (String(part.search_type) !== wantSearch) continue;
-		const month = part.date_month;
-		if (typeof month !== "number" || !wantedMonths.has(month)) continue;
-		out.push({
-			filePath: df.file_path,
-			objectKey: stripBucket(df.file_path),
-			bytes: Number(df.file_size_in_bytes),
-			rowCount: Number(df.record_count)
-		});
-	}
-	endWalk?.({
-		manifests: manifests.length,
-		files: out.length
-	});
-	if (opts.cache) {
-		const freshKey = resolvedFilesKey(conn.namespace, opts.table, snapshotId, opts.siteId, opts.searchType, wantedMonths);
-		await cachePut(opts.cache, freshKey, out, RESOLVED_FILES_TTL_MS, now);
-	}
-	return out;
-}
-async function dropIcebergTables(conn, tables) {
-	const targets = tables ?? (await restCatalogListTables(conn.catalog, { namespace: conn.namespace })).map((t) => t.name);
-	const results = [];
-	for (const table of targets) await icebergDropTable({
-		catalog: conn.catalog,
+	const matches = (opts.encoding ?? DEFAULT_PARTITION_KEY_ENCODING) === "string" ? [{
+		field: "site_id",
+		value: opts.siteId,
+		encoding: "string"
+	}, {
+		field: "search_type",
+		value: opts.searchType,
+		encoding: "string"
+	}] : [];
+	return resolveIcebergDataFiles(conn, {
 		namespace: conn.namespace,
-		table,
-		purgeRequested: true
-	}).then(() => results.push({
-		table,
-		outcome: ok(void 0)
-	}), (e) => results.push({
-		table,
-		outcome: err(engineErrors.icebergTableOpFailed("drop", table, e))
-	}));
-	return results;
+		table: opts.table,
+		partitionSpec: ICEBERG_PARTITION_SPEC,
+		matches,
+		range: opts.range,
+		cache: opts.cache,
+		clock: opts.clock,
+		profiler: opts.profiler
+	});
 }
 const DAY_MILLIS = 864e5;
 const INT32_MIN = -2147483648;
@@ -426,7 +126,7 @@ function toIntPartitionSiteId(value) {
 }
 function dedupeByIdentity(table, records) {
 	if (records.length < 2) return records;
-	const key = ICEBERG_SCHEMAS[table].identityColumns;
+	const key = gscDataset(table, "int").tableSpec.identityColumns;
 	const seen = /* @__PURE__ */ new Map();
 	for (const rec of records) {
 		const k = key.map((col) => `${rec[col] ?? ""}`).join("\0");
@@ -435,7 +135,7 @@ function dedupeByIdentity(table, records) {
 	return seen.size === records.length ? records : [...seen.values()];
 }
 function sortByClusterKey(table, records) {
-	const cols = TABLE_METADATA[table].clusterKey;
+	const cols = gscDataset(table, "int").tableSpec.clusterKey ?? [];
 	if (cols.length === 0 || records.length < 2) return records;
 	return records.slice().sort((a, b) => {
 		for (const col of cols) {
@@ -454,7 +154,7 @@ function sortByClusterKey(table, records) {
 }
 function toRecords(slice, rows, encoding) {
 	const siteVal = encoding === "int" ? toIntPartitionSiteId(slice.ctx.siteId) : slice.ctx.siteId ?? "";
-	const searchVal = encoding === "int" ? SEARCH_TYPE_INT[slice.searchType] : slice.searchType;
+	const searchVal = gscDataset(slice.table, encoding).def.dims.search_type.toPartitionValue(slice.searchType);
 	return rows.map((row) => {
 		const out = {};
 		for (const k in row) out[k] = coerceJsonSafe(row[k]);
@@ -466,7 +166,7 @@ function toRecords(slice, rows, encoding) {
 }
 function createIcebergAppendSink(options) {
 	let connection;
-	const encoding = options.encoding ?? "int";
+	const encoding = options.encoding ?? DEFAULT_PARTITION_KEY_ENCODING;
 	const buffers = /* @__PURE__ */ new Map();
 	function connect() {
 		connection ??= connectIcebergCatalog(options.catalog);

package/dist/sink-node.d.mts

@@ -1,27 +1,5 @@
 import { Row } from "./_chunks/storage.mjs";
-import { IcebergS3Config, LocalIcebergSinkOptions, Sink, SinkCloseResult, SliceOverwriteWriter } from "./_chunks/sink.mjs";
-/** Full `LocalIcebergSink` options — extends the frozen contract options. */
-interface LocalIcebergSinkFullOptions extends LocalIcebergSinkOptions {
-  /** S3 credentials for the warehouse. Defaults to the POC MinIO creds. */
-  s3?: IcebergS3Config;
-  /** Python interpreter. Defaults to `$GSCDUMP_ICEBERG_PYTHON` then `python3`. */
-  python?: string;
-  /** Override the writer-script path. Defaults to `scripts/iceberg-writer.py`. */
-  writerScript?: string;
-}
-/** Run the PyIceberg writer subprocess for one job, return its parsed result. */
-interface LocalIcebergSink extends Sink {
-  /** The catalog namespace the 5 tables live under. */
-  readonly namespace: string;
-}
-/**
- * Create a `LocalIcebergSink` pointed at a local Iceberg REST catalog.
- *
- * Requires the POC docker stack (`poc/iceberg/docker-compose.iceberg.yml`)
- * running and a Python env with `pyiceberg` + `pyarrow` available. Tests that
- * use this sink must skip when the stack is unreachable.
- */
-declare function createLocalIcebergSink(options: LocalIcebergSinkFullOptions): LocalIcebergSink;
+import { IcebergS3Config, Sink, SinkCloseResult, SliceOverwriteWriter } from "./_chunks/sink.mjs";
 /** Connection details for the Iceberg REST catalog the writer targets. */
 interface OverwriteWriterCatalogConfig {
   /**
@@ -180,4 +158,4 @@ declare function deleteSiteFromShard(args: {
   siteId: string | number;
   tables: readonly string[];
 }): Promise<DeleteSiteResult[]>;
-export { type DeleteJob, type DeleteSiteResult, type IcebergOverwriteWriter, type IcebergOverwriteWriterOptions, type LocalIcebergSink, type LocalIcebergSinkFullOptions, type LocalIcebergSinkOptions, type OverwriteBackend, type OverwriteJob, type OverwriteJobResult, type OverwriteWriterCatalogConfig, type PyIcebergJob, createIcebergOverwriteWriter, createLocalIcebergSink, deleteSiteFromShard, httpBackend, overwriteWriterAsSink, subprocessBackend };
+export { type DeleteJob, type DeleteSiteResult, type IcebergOverwriteWriter, type IcebergOverwriteWriterOptions, type OverwriteBackend, type OverwriteJob, type OverwriteJobResult, type OverwriteWriterCatalogConfig, type PyIcebergJob, createIcebergOverwriteWriter, deleteSiteFromShard, httpBackend, overwriteWriterAsSink, subprocessBackend };

package/dist/sink-node.mjs

@@ -1,88 +1,5 @@
 import { ICEBERG_SCHEMAS, assertIcebergTable } from "./_chunks/schema2.mjs";
-import { dirname, join } from "node:path";
-import { fileURLToPath } from "node:url";
-import process from "node:process";
-function resolvePyIcebergPython(override) {
-	return override ?? process.env["GSCDUMP_ICEBERG_PYTHON"] ?? "python3";
-}
-async function runPyIcebergWriter(options) {
-	const { execFile } = await import("node:child_process");
-	return new Promise((resolve, reject) => {
-		execFile(options.python, [options.script], { maxBuffer: 64 * 1024 * 1024 }, (err, stdout, stderr) => {
-			let parsed;
-			if (stdout.trim()) try {
-				parsed = JSON.parse(stdout);
-			} catch {}
-			if (parsed && !(err && options.rejectOnProcessError)) {
-				resolve(parsed);
-				return;
-			}
-			if (err) {
-				if (options.processErrorAsParseFailure) {
-					reject(/* @__PURE__ */ new Error(`${options.label} produced no parseable output (${err.message})${stderr ? `: ${stderr}` : ""}`));
-					return;
-				}
-				reject(/* @__PURE__ */ new Error(`${options.label} process failed (${err.message})${stderr ? `: ${stderr}` : ""}`));
-				return;
-			}
-			reject(/* @__PURE__ */ new Error(`${options.label} produced no parseable output: ${stdout || stderr}`));
-		}).stdin?.end(JSON.stringify(options.job));
-	});
-}
-const POC_S3 = {
-	endpoint: "localhost:9100",
-	accessKeyId: "poc",
-	secretAccessKey: "pocpocpoc",
-	region: "us-east-1"
-};
-function resolveWriterScript(override) {
-	if (override) return override;
-	return join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "iceberg-writer.py");
-}
-function createLocalIcebergSink(options) {
-	const s3 = options.s3 ?? POC_S3;
-	const python = resolvePyIcebergPython(options.python);
-	const script = resolveWriterScript(options.writerScript);
-	function buildJob(op, slice, rows) {
-		return {
-			op,
-			catalogUri: options.catalogUri,
-			namespace: options.namespace,
-			warehouse: options.warehouse,
-			s3,
-			table: slice.table,
-			spec: ICEBERG_SCHEMAS[slice.table],
-			siteId: slice.ctx.siteId ?? "",
-			searchType: slice.searchType,
-			date: slice.date,
-			rows
-		};
-	}
-	const touched = /* @__PURE__ */ new Set();
-	return {
-		namespace: options.namespace,
-		capabilities: { appendOnly: true },
-		async emit(slice, rows) {
-			if (rows.length === 0) return { rowCount: 0 };
-			const res = await runPyIcebergWriter({
-				python,
-				script,
-				job: buildJob("emit", slice, rows),
-				label: "LocalIcebergSink writer",
-				rejectOnProcessError: true
-			});
-			if (res.error) throw new Error(`LocalIcebergSink writer failed: ${res.error}`);
-			touched.add(slice.table);
-			return { rowCount: res.rowCount ?? 0 };
-		},
-		async close() {
-			return {
-				flushed: [...touched],
-				failed: []
-			};
-		}
-	};
-}
+import { resolvePyIcebergPython, runPyIcebergWriter } from "@gscdump/lakehouse";
 function createIcebergOverwriteWriter(opts) {
 	const { catalog, backend } = opts;
 	async function overwriteSlice(slice, rows) {
@@ -181,4 +98,4 @@ async function deleteSiteFromShard(args) {
 	}
 	return results;
 }
-export { createIcebergOverwriteWriter, createLocalIcebergSink, deleteSiteFromShard, httpBackend, overwriteWriterAsSink, subprocessBackend };
+export { createIcebergOverwriteWriter, deleteSiteFromShard, httpBackend, overwriteWriterAsSink, subprocessBackend };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.33.10",
+  "version": "0.35.2",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,8 +181,9 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "0.33.10",
-    "gscdump": "0.33.10"
+    "@gscdump/contracts": "0.35.2",
+    "@gscdump/lakehouse": "0.35.2",
+    "gscdump": "0.35.2"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",

package/dist/_chunks/libs/chokidar.d.mts

@@ -1 +0,0 @@
-export { };

package/dist/_chunks/libs/db0.d.mts

@@ -1 +0,0 @@
-export { };

package/dist/_chunks/libs/denque.d.mts

@@ -1 +0,0 @@
-export { };

package/dist/_chunks/libs/ioredis.d.mts

@@ -1 +0,0 @@
-export { };

package/dist/_chunks/libs/lru-cache.d.mts

@@ -1 +0,0 @@
-export { };