@gscdump/engine 0.32.8 → 0.32.10

package/dist/_chunks/sink.d.mts

@@ -285,10 +285,26 @@ interface CommitRetryOptions {
   random?: () => number;
   /**
    * Idempotency token stamped into the appended snapshot's summary
-   * (`gscdump.append-id`) and matched by the landed-check before any outer
-   * retry. Generated per call (`crypto.randomUUID`) so it is STABLE across this
-   * call's retries but UNIQUE per call — a content hash would risk a false
-   * "already landed" skip (silent data loss). Injectable for deterministic tests.
+   * (`gscdump.append-id`) and matched by the landed-check. When omitted it is
+   * DERIVED from the records' content (a SHA-256 over each row's full
+   * key/value, order-independent — see {@link deriveAppendId}), making the
+   * token STABLE across PROCESSES, not just within one call's retry loop.
+   *
+   * Why content-derived (superseding the old "random per call" choice): the
+   * double we actually hit is a CROSS-RUN re-append — a job commits the
+   * Iceberg snapshot, then dies/evicts before its D1 ledger write, so a queue
+   * RETRY (or operator re-resync) re-runs from scratch, re-emits the identical
+   * buffer, and appends a second copy. A random per-call id can't catch that
+   * (the retry is a new process → new id). A content hash CAN: the retry
+   * re-derives the same id, the pre-append landed-check finds it, and skips.
+   *
+   * The old "content hash risks a false skip / data loss" fear does NOT apply
+   * to this fact model: a date is appended exactly once when finalized and
+   * never revised (revisions go through the OVERWRITE path), and pagination
+   * pages carry DIFFERENT rows → a different content hash → never falsely
+   * skipped. Two appends collide only if their full row-sets are byte-identical
+   * — which is precisely a duplicate that SHOULD be skipped. Injectable for
+   * deterministic tests.
    */
   appendId?: string;
 }

package/dist/iceberg/index.mjs

@@ -189,7 +189,7 @@ async function icebergAppendRetrying(args, options = {}) {
 	const maxDelayMs = options.maxDelayMs ?? 2e4;
 	const sleep = options.sleep ?? defaultCommitSleep;
 	const random = options.random ?? Math.random;
-	const appendId = options.appendId ?? globalThis.crypto.randomUUID();
+	const appendId = options.appendId ?? await deriveAppendId(args);
 	const stampedArgs = {
 		...args,
 		snapshotProperties: {
@@ -197,6 +197,7 @@ async function icebergAppendRetrying(args, options = {}) {
 			[APPEND_ID_SUMMARY_KEY]: appendId
 		}
 	};
+	if (await appendAlreadyLanded(args, appendId).catch(() => false)) return;
 	for (let attempt = 0; attempt < maxAttempts; attempt++) {
 		const err = await icebergAppend(stampedArgs).then(() => void 0, (e) => e);
 		if (err === void 0) return;
@@ -206,6 +207,14 @@ async function icebergAppendRetrying(args, options = {}) {
 		await sleep(Math.floor(random() * ceiling));
 	}
 }
+async function deriveAppendId(args) {
+	const records = args.records ?? [];
+	if (records.length === 0) return globalThis.crypto.randomUUID();
+	const rowSig = (r) => Object.keys(r).sort().map((k) => `${k}=${String(r[k])}`).join("
");
+	const body = records.map(rowSig).sort().join("");
+	const digest = await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(body));
+	return Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, "0")).join("");
+}
 async function appendAlreadyLanded(args, appendId) {
 	const a = args;
 	if (a.catalog?.type !== "rest" || a.namespace == null || a.table == null) return false;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.32.8",
+  "version": "0.32.10",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,8 +181,8 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "gscdump": "0.32.8",
-    "@gscdump/contracts": "0.32.8"
+    "@gscdump/contracts": "0.32.10",
+    "gscdump": "0.32.10"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",