@gscdump/engine 0.32.7 → 0.32.9

package/dist/_chunks/sink.d.mts

@@ -285,10 +285,26 @@ interface CommitRetryOptions {
   random?: () => number;
   /**
    * Idempotency token stamped into the appended snapshot's summary
-   * (`gscdump.append-id`) and matched by the landed-check before any outer
-   * retry. Generated per call (`crypto.randomUUID`) so it is STABLE across this
-   * call's retries but UNIQUE per call — a content hash would risk a false
-   * "already landed" skip (silent data loss). Injectable for deterministic tests.
+   * (`gscdump.append-id`) and matched by the landed-check. When omitted it is
+   * DERIVED from the records' content (a SHA-256 over each row's full
+   * key/value, order-independent — see {@link deriveAppendId}), making the
+   * token STABLE across PROCESSES, not just within one call's retry loop.
+   *
+   * Why content-derived (superseding the old "random per call" choice): the
+   * double we actually hit is a CROSS-RUN re-append — a job commits the
+   * Iceberg snapshot, then dies/evicts before its D1 ledger write, so a queue
+   * RETRY (or operator re-resync) re-runs from scratch, re-emits the identical
+   * buffer, and appends a second copy. A random per-call id can't catch that
+   * (the retry is a new process → new id). A content hash CAN: the retry
+   * re-derives the same id, the pre-append landed-check finds it, and skips.
+   *
+   * The old "content hash risks a false skip / data loss" fear does NOT apply
+   * to this fact model: a date is appended exactly once when finalized and
+   * never revised (revisions go through the OVERWRITE path), and pagination
+   * pages carry DIFFERENT rows → a different content hash → never falsely
+   * skipped. Two appends collide only if their full row-sets are byte-identical
+   * — which is precisely a duplicate that SHOULD be skipped. Injectable for
+   * deterministic tests.
    */
   appendId?: string;
 }
@@ -454,6 +470,10 @@ interface SinkCapabilities {
   /** Whether the sink exposes partition overwrite semantics. */
   canOverwrite?: boolean;
 }
+/** Partition-overwrite writer for revision paths. */
+interface SliceOverwriteWriter {
+  overwriteSlice: (slice: SinkSlice, rows: readonly Row$1[]) => Promise<SinkWriteResult>;
+}
 /**
  * Outcome of `Sink.close()` — which tables' buffered rows reached durable
  * storage and which failed.
@@ -539,4 +559,4 @@ interface LocalIcebergSinkOptions extends SinkOptions {
   /** S3-compatible warehouse location (POC: MinIO). */
   warehouse: string;
 }
-export { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, LocalIcebergSinkOptions, PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };
+export { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, LocalIcebergSinkOptions, PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, SliceOverwriteWriter, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };

package/dist/iceberg/index.mjs

@@ -189,7 +189,7 @@ async function icebergAppendRetrying(args, options = {}) {
 	const maxDelayMs = options.maxDelayMs ?? 2e4;
 	const sleep = options.sleep ?? defaultCommitSleep;
 	const random = options.random ?? Math.random;
-	const appendId = options.appendId ?? globalThis.crypto.randomUUID();
+	const appendId = options.appendId ?? await deriveAppendId(args);
 	const stampedArgs = {
 		...args,
 		snapshotProperties: {
@@ -197,6 +197,7 @@ async function icebergAppendRetrying(args, options = {}) {
 			[APPEND_ID_SUMMARY_KEY]: appendId
 		}
 	};
+	if (await appendAlreadyLanded(args, appendId).catch(() => false)) return;
 	for (let attempt = 0; attempt < maxAttempts; attempt++) {
 		const err = await icebergAppend(stampedArgs).then(() => void 0, (e) => e);
 		if (err === void 0) return;
@@ -206,6 +207,14 @@ async function icebergAppendRetrying(args, options = {}) {
 		await sleep(Math.floor(random() * ceiling));
 	}
 }
+async function deriveAppendId(args) {
+	const records = args.records ?? [];
+	if (records.length === 0) return globalThis.crypto.randomUUID();
+	const rowSig = (r) => Object.keys(r).sort().map((k) => `${k}=${String(r[k])}`).join("
");
+	const body = records.map(rowSig).sort().join("");
+	const digest = await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(body));
+	return Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, "0")).join("");
+}
 async function appendAlreadyLanded(args, appendId) {
 	const a = args;
 	if (a.catalog?.type !== "rest" || a.namespace == null || a.table == null) return false;

package/dist/sink-node.d.mts

@@ -1,4 +1,5 @@
-import { IcebergS3Config, LocalIcebergSinkOptions, Sink } from "./_chunks/sink.mjs";
+import { Row } from "./_chunks/storage.mjs";
+import { IcebergS3Config, LocalIcebergSinkOptions, Sink, SinkCloseResult, SliceOverwriteWriter } from "./_chunks/sink.mjs";
 /** Full `LocalIcebergSink` options — extends the frozen contract options. */
 interface LocalIcebergSinkFullOptions extends LocalIcebergSinkOptions {
   /** S3 credentials for the warehouse. Defaults to the POC MinIO creds. */
@@ -21,4 +22,162 @@ interface LocalIcebergSink extends Sink {
  * use this sink must skip when the stack is unreachable.
  */
 declare function createLocalIcebergSink(options: LocalIcebergSinkFullOptions): LocalIcebergSink;
-export { type LocalIcebergSink, type LocalIcebergSinkFullOptions, type LocalIcebergSinkOptions, createLocalIcebergSink };
+/** Connection details for the Iceberg REST catalog the writer targets. */
+interface OverwriteWriterCatalogConfig {
+  /**
+   * Iceberg REST catalog endpoint, e.g. `http://localhost:8181` (POC) or the
+   *  R2 Data Catalog endpoint (prod).
+   */
+  catalogUri: string;
+  /** Catalog namespace the 5 fact tables live under, e.g. `gsc`. */
+  namespace: string;
+  /** Warehouse identifier the catalog resolves table locations under. */
+  warehouse: string;
+  /** S3-compatible object store backing the catalog. */
+  s3: IcebergS3Config;
+  /** Bearer token for a token-authed REST catalog (R2 Data Catalog). */
+  catalogToken?: string;
+}
+/**
+ * The JSON job wire-format shared with `scripts/iceberg-writer.py` — identical
+ * to the shape `LocalIcebergSink` already emits, so one PyIceberg script backs
+ * both the local sink and this writer. `op` is always `overwrite` here.
+ */
+interface OverwriteJob {
+  op: 'overwrite';
+  catalogUri: string;
+  namespace: string;
+  warehouse: string;
+  s3: IcebergS3Config;
+  table: string;
+  spec: unknown;
+  siteId: string;
+  searchType: string;
+  date: string;
+  /** Data columns only — the script injects `site_id` / `search_type`. */
+  rows: readonly Row[];
+}
+/**
+ * Per-site delete job — removes EVERY row for one `siteId` across a whole
+ * table (all dates / search types). Recovery op for clearing a corrupt site
+ * (e.g. a cross-run append double) from a shared per-team shard before a clean
+ * re-backfill, without touching sibling sites. Carries no rows. `siteId` is a
+ * `number` for an INT-encoded shard, a `string` for STRING encoding.
+ */
+interface DeleteJob {
+  op: 'delete';
+  catalogUri: string;
+  namespace: string;
+  warehouse: string;
+  s3: IcebergS3Config;
+  /** Bearer token for a token-authed REST catalog (R2 Data Catalog). */
+  catalogToken?: string;
+  table: string;
+  siteId: string | number;
+}
+/** Any job the shared PyIceberg backend can execute. */
+type PyIcebergJob = OverwriteJob | DeleteJob;
+/** Result wire-format from the PyIceberg backend. */
+interface OverwriteJobResult {
+  /** Rows written (emit/overwrite) or removed (delete). */
+  rowCount?: number;
+  error?: string;
+}
+/**
+ * A transport that executes one PyIceberg job and returns its result.
+ * `subprocessBackend` and `httpBackend` are the two shipped implementations;
+ * tests inject a fake. The param is the `OverwriteJob | DeleteJob` union, so a
+ * backend value is still assignable where the narrower `OverwriteBackend` is
+ * expected (a wider-param function accepts the narrower call).
+ */
+type OverwriteBackend = (job: PyIcebergJob) => Promise<OverwriteJobResult>;
+interface IcebergOverwriteWriterOptions {
+  catalog: OverwriteWriterCatalogConfig;
+  /** The transport that runs the PyIceberg job. */
+  backend: OverwriteBackend;
+}
+/**
+ * The trailing-window Iceberg overwrite writer. Conforms to the frozen
+ * `SliceOverwriteWriter` contract; `overwriteSlice` builds one PyIceberg
+ * overwrite job per call and runs it through the configured backend.
+ */
+interface IcebergOverwriteWriter extends SliceOverwriteWriter {
+  /** Release any backend resources. Idempotent. */
+  close: () => Promise<SinkCloseResult>;
+}
+/**
+ * Build a trailing-window Iceberg overwrite writer over the given backend.
+ * Pure — all I/O is the backend's; the writer only shapes the job.
+ */
+declare function createIcebergOverwriteWriter(opts: IcebergOverwriteWriterOptions): IcebergOverwriteWriter;
+/** Options for the PyIceberg subprocess backend. */
+interface SubprocessBackendOptions {
+  /** Python interpreter. Defaults to `$GSCDUMP_ICEBERG_PYTHON` then `python3`. */
+  python?: string;
+  /**
+   * Path to the PyIceberg writer script. Defaults to
+   *  `<engine>/scripts/iceberg-writer.py`.
+   */
+  writerScript?: string;
+}
+/**
+ * Backend that runs the PyIceberg overwrite by spawning the shared writer
+ * script (`scripts/iceberg-writer.py`). Node-only — used by local integration
+ * tests and a Node-side scheduled job box. The script reads the job JSON on
+ * stdin and writes `{rowCount}` / `{error}` on stdout.
+ */
+declare function subprocessBackend(opts?: SubprocessBackendOptions): OverwriteBackend;
+/** Options for the PyIceberg HTTP backend (the Cloudflare Container in prod). */
+interface HttpBackendOptions {
+  /**
+   * URL of the PyIceberg overwrite service — a POST endpoint accepting the
+   *  `OverwriteJob` JSON and returning `OverwriteJobResult`.
+   */
+  endpoint: string;
+  /** Optional bearer token for the service. */
+  token?: string;
+  /** Injectable fetch — defaults to the global. */
+  fetch?: typeof fetch;
+}
+/**
+ * Backend that POSTs the overwrite job to a PyIceberg HTTP service — the
+ * Cloudflare Container running `iceberg-writer.py` behind an HTTP shim. This
+ * is the prod transport: the request-path Worker holds no Python and never
+ * does an Iceberg metadata commit itself.
+ */
+declare function httpBackend(opts: HttpBackendOptions): OverwriteBackend;
+/**
+ * Adapt an `IcebergOverwriteWriter` into a full `Sink` whose `emit` is also a
+ * partition-overwrite — convenient for tests so a single Iceberg writer backs
+ * both append and revision paths against the local stack. Prod keeps `emit`
+ * on the Pipeline; only `overwriteSlice` delegates here.
+ */
+declare function overwriteWriterAsSink(writer: IcebergOverwriteWriter): Sink & SliceOverwriteWriter;
+/** Per-table outcome of a {@link deleteSiteFromShard} run. */
+interface DeleteSiteResult {
+  table: string;
+  /** Rows removed, when the backend reported a count. */
+  rowCount?: number;
+  /** Present when this table's delete failed; the sweep continues regardless. */
+  error?: string;
+}
+/**
+ * Remove every row for one `siteId` from a per-team Iceberg shard, across the
+ * given tables. Recovery op: clear a corrupt site (e.g. a cross-run append
+ * double) before a clean re-backfill, leaving sibling sites in the shared shard
+ * untouched. `site_id` is an identity partition column, so PyIceberg prunes
+ * whole data files (no row-level delete files).
+ *
+ * Runs SEQUENTIALLY (one commit per table at a time) so the per-table catalog
+ * commit-rate ceiling is never tripped. A table failure is captured in its
+ * result row, not thrown — the sweep always attempts every table so a partial
+ * failure is visible and re-runnable (delete is idempotent: re-deleting an
+ * already-empty site is a no-op).
+ */
+declare function deleteSiteFromShard(args: {
+  catalog: OverwriteWriterCatalogConfig;
+  backend: OverwriteBackend;
+  siteId: string | number;
+  tables: readonly string[];
+}): Promise<DeleteSiteResult[]>;
+export { type DeleteJob, type DeleteSiteResult, type IcebergOverwriteWriter, type IcebergOverwriteWriterOptions, type LocalIcebergSink, type LocalIcebergSinkFullOptions, type LocalIcebergSinkOptions, type OverwriteBackend, type OverwriteJob, type OverwriteJobResult, type OverwriteWriterCatalogConfig, type PyIcebergJob, createIcebergOverwriteWriter, createLocalIcebergSink, deleteSiteFromShard, httpBackend, overwriteWriterAsSink, subprocessBackend };

package/dist/sink-node.mjs

@@ -1,4 +1,4 @@
-import { ICEBERG_SCHEMAS } from "./_chunks/schema2.mjs";
+import { ICEBERG_SCHEMAS, assertIcebergTable } from "./_chunks/schema2.mjs";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 import process from "node:process";
@@ -83,4 +83,102 @@ function createLocalIcebergSink(options) {
 		}
 	};
 }
-export { createLocalIcebergSink };
+function createIcebergOverwriteWriter(opts) {
+	const { catalog, backend } = opts;
+	async function overwriteSlice(slice, rows) {
+		const siteId = slice.ctx.siteId;
+		if (!siteId) throw new Error("overwriteSlice: slice.ctx.siteId is required for the Iceberg partition key");
+		const table = assertIcebergTable(slice.table);
+		const res = await backend({
+			op: "overwrite",
+			catalogUri: catalog.catalogUri,
+			namespace: catalog.namespace,
+			warehouse: catalog.warehouse,
+			s3: catalog.s3,
+			table,
+			spec: ICEBERG_SCHEMAS[table],
+			siteId,
+			searchType: slice.searchType,
+			date: slice.date,
+			rows
+		});
+		if (res.error) throw new Error(`overwriteSlice: PyIceberg backend failed: ${res.error}`);
+		return { rowCount: res.rowCount ?? rows.length };
+	}
+	return {
+		overwriteSlice,
+		async close() {
+			return {
+				flushed: [],
+				failed: []
+			};
+		}
+	};
+}
+function subprocessBackend(opts = {}) {
+	return async (job) => {
+		const { dirname, join } = await import("node:path");
+		const { fileURLToPath } = await import("node:url");
+		return runPyIcebergWriter({
+			python: resolvePyIcebergPython(opts.python),
+			script: opts.writerScript ?? join(dirname(fileURLToPath(import.meta.url)), "..", "scripts", "iceberg-writer.py"),
+			job,
+			label: "iceberg overwrite subprocess",
+			processErrorAsParseFailure: true
+		});
+	};
+}
+function httpBackend(opts) {
+	const doFetch = opts.fetch ?? fetch;
+	return async (job) => {
+		const res = await doFetch(opts.endpoint, {
+			method: "POST",
+			headers: {
+				"content-type": "application/json",
+				...opts.token ? { authorization: `Bearer ${opts.token}` } : {}
+			},
+			body: JSON.stringify(job)
+		});
+		if (!res.ok) {
+			const body = await res.text().catch(() => "");
+			return { error: `HTTP ${res.status}${body ? `: ${body}` : ""}` };
+		}
+		return await res.json();
+	};
+}
+function overwriteWriterAsSink(writer) {
+	return {
+		capabilities: {
+			canOverwrite: true,
+			appendOnly: false
+		},
+		emit: (slice, rows) => writer.overwriteSlice(slice, rows),
+		overwriteSlice: (slice, rows) => writer.overwriteSlice(slice, rows),
+		close: () => writer.close()
+	};
+}
+async function deleteSiteFromShard(args) {
+	const { catalog, backend, siteId, tables } = args;
+	const results = [];
+	for (const table of tables) {
+		const res = await backend({
+			op: "delete",
+			catalogUri: catalog.catalogUri,
+			namespace: catalog.namespace,
+			warehouse: catalog.warehouse,
+			s3: catalog.s3,
+			...catalog.catalogToken ? { catalogToken: catalog.catalogToken } : {},
+			table,
+			siteId
+		}).catch((err) => ({ error: err instanceof Error ? err.message : String(err) }));
+		results.push(res.error ? {
+			table,
+			error: res.error
+		} : {
+			table,
+			rowCount: res.rowCount
+		});
+	}
+	return results;
+}
+export { createIcebergOverwriteWriter, createLocalIcebergSink, deleteSiteFromShard, httpBackend, overwriteWriterAsSink, subprocessBackend };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.32.7",
+  "version": "0.32.9",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,8 +181,8 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "0.32.7",
-    "gscdump": "0.32.7"
+    "@gscdump/contracts": "0.32.9",
+    "gscdump": "0.32.9"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",