@gscdump/engine 0.32.5 → 0.32.7

package/dist/_chunks/libs/icebird.d.mts

@@ -303,7 +303,8 @@ declare function icebergAppend({
   tableUrl,
   resolver,
   records,
-  sortOrderId
+  sortOrderId,
+  snapshotProperties
 }: {
   catalog: Catalog;
   namespace?: string | string[] | undefined;
@@ -312,6 +313,7 @@ declare function icebergAppend({
   resolver?: Resolver | undefined;
   records: Record<string, any>[];
   sortOrderId?: number | undefined;
+  snapshotProperties?: Record<string, string> | undefined;
 }): Promise<TableMetadata>;
 /**
  * Create a new table. REST: delegates to the catalog's create endpoint.

package/dist/_chunks/libs/icebird.mjs

@@ -3216,7 +3216,7 @@ async function prepareAppend({ tableUrl, metadata, records, resolver, sortOrderI
 		writtenFiles: [...writtenDataFiles.map((f) => f.path), manifestPath]
 	};
 }
-async function stageSnapshotForAppend({ tableUrl, metadata, prepared, resolver }) {
+async function stageSnapshotForAppend({ tableUrl, metadata, prepared, resolver, snapshotProperties }) {
 	if (!tableUrl) throw new Error("tableUrl is required");
 	if (!resolver?.writer) throw new Error("resolver.writer is required");
 	const sequenceNumber = BigInt(metadata["last-sequence-number"] ?? 0) + 1n;
@@ -3254,7 +3254,8 @@ async function stageSnapshotForAppend({ tableUrl, metadata, prepared, resolver }
 		"total-data-files": String(prevTotals.files + BigInt(prepared.addedDataFilesCount)),
 		"total-delete-files": "0",
 		"total-position-deletes": "0",
-		"total-equality-deletes": "0"
+		"total-equality-deletes": "0",
+		...snapshotProperties ?? {}
 	};
 	return await buildSnapshotUpdate({
 		tableUrl,
@@ -3370,7 +3371,7 @@ const DEFAULT_RETRY = Object.freeze({
 	factor: 2,
 	totalTimeoutMs: 1800 * 1e3
 });
-async function icebergAppend({ catalog, namespace, table, tableUrl, resolver, records, sortOrderId }) {
+async function icebergAppend({ catalog, namespace, table, tableUrl, resolver, records, sortOrderId, snapshotProperties }) {
 	const ctx = await loadTable({
 		catalog,
 		namespace,
@@ -3396,7 +3397,8 @@ async function icebergAppend({ catalog, namespace, table, tableUrl, resolver, re
 			tableUrl: workingCtx.tableUrl,
 			metadata: workingCtx.metadata,
 			prepared,
-			resolver: requireResolver(workingCtx.resolver, "icebergAppend")
+			resolver: requireResolver(workingCtx.resolver, "icebergAppend"),
+			snapshotProperties
 		})
 	});
 }

package/dist/_chunks/sink.d.mts

@@ -283,6 +283,14 @@ interface CommitRetryOptions {
   sleep?: (ms: number) => Promise<void>;
   /** Injectable RNG for the jitter — tests pass a deterministic value. */
   random?: () => number;
+  /**
+   * Idempotency token stamped into the appended snapshot's summary
+   * (`gscdump.append-id`) and matched by the landed-check before any outer
+   * retry. Generated per call (`crypto.randomUUID`) so it is STABLE across this
+   * call's retries but UNIQUE per call — a content hash would risk a false
+   * "already landed" skip (silent data loss). Injectable for deterministic tests.
+   */
+  appendId?: string;
 }
 /**
  * True when `err` is an R2 Data Catalog commit rate-limit response
@@ -297,15 +305,18 @@ declare function isCommitRateLimited(err: unknown): boolean;
  * retries 412/409 internally; 429 is the gap this closes. Non-429 errors
  * (and 429s that survive every attempt) propagate unchanged.
  *
- * RESIDUAL RISK — re-upload orphans. icebird's `icebergAppend` prepares the
- * data + manifest files ONCE, outside its internal 412/409 retry loop, so
- * those retries never re-upload data. A 429 that escapes that loop and is
- * retried HERE re-runs the whole `icebergAppend` call, which re-prepares and
- * re-uploads the data files; the previous attempt's parquet objects become
- * orphans (referenced by no snapshot). 429s should be rare and clear within
- * an attempt or two, so orphan volume is small, and R2 orphan-file cleanup
- * reclaims them. Eliminating the 429 source entirely (per-table commit
- * coalescing) is assessed in the Phase-1.5 report.
+ * IDEMPOTENT RE-RUN. A 429 that escapes icebird's internal loop is retried HERE
+ * by re-running the whole `icebergAppend`, which re-prepares + re-uploads the
+ * data files. That is safe ONLY if the prior attempt's commit did NOT land — but
+ * R2 Data Catalog can apply a commit and STILL return 429 (rate-limit on the
+ * response path), and a lost ack looks identical. A blind re-run then appends a
+ * SECOND copy of the same rows → silent double-count (observed: a re-resynced
+ * site read 2× its true totals). To close this, every attempt stamps a per-call
+ * `gscdump.append-id` into the snapshot summary, and BEFORE retrying or giving
+ * up we reload the table and check whether that id already landed
+ * ({@link appendAlreadyLanded}); if so the append succeeded and we return without
+ * re-appending. A non-landed 429 still re-runs (the previous attempt's parquet is
+ * an orphan, reclaimed by R2 cleanup) — same as before, no double.
  */
 declare function icebergAppendRetrying(args: Parameters<typeof icebergAppend>[0], options?: CommitRetryOptions): Promise<void>;
 /**

package/dist/iceberg/index.mjs

@@ -173,6 +173,8 @@ async function connectIcebergCatalog(config, opts = {}) {
 		namespace: config.namespace
 	};
 }
+const APPEND_ID_SUMMARY_KEY = "gscdump.append-id";
+const APPEND_LANDED_SCAN_DEPTH = 25;
 function isCommitRateLimited(err) {
 	if (err && typeof err === "object" && err.status === 429) return true;
 	const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
@@ -187,14 +189,35 @@ async function icebergAppendRetrying(args, options = {}) {
 	const maxDelayMs = options.maxDelayMs ?? 2e4;
 	const sleep = options.sleep ?? defaultCommitSleep;
 	const random = options.random ?? Math.random;
+	const appendId = options.appendId ?? globalThis.crypto.randomUUID();
+	const stampedArgs = {
+		...args,
+		snapshotProperties: {
+			...args.snapshotProperties,
+			[APPEND_ID_SUMMARY_KEY]: appendId
+		}
+	};
 	for (let attempt = 0; attempt < maxAttempts; attempt++) {
-		const err = await icebergAppend(args).then(() => void 0, (e) => e);
+		const err = await icebergAppend(stampedArgs).then(() => void 0, (e) => e);
 		if (err === void 0) return;
+		if (await appendAlreadyLanded(args, appendId).catch(() => false)) return;
 		if (!isCommitRateLimited(err) || attempt === maxAttempts - 1) throw err;
 		const ceiling = Math.min(maxDelayMs, baseDelayMs * 2 ** attempt);
 		await sleep(Math.floor(random() * ceiling));
 	}
 }
+async function appendAlreadyLanded(args, appendId) {
+	const a = args;
+	if (a.catalog?.type !== "rest" || a.namespace == null || a.table == null) return false;
+	const { metadata } = await restCatalogLoadTable(a.catalog, {
+		namespace: a.namespace,
+		table: a.table
+	});
+	const snapshots = metadata.snapshots ?? [];
+	const from = Math.max(0, snapshots.length - APPEND_LANDED_SCAN_DEPTH);
+	for (let i = snapshots.length - 1; i >= from; i--) if (snapshots[i]?.summary?.[APPEND_ID_SUMMARY_KEY] === appendId) return true;
+	return false;
+}
 async function ensureIcebergNamespace(conn) {
 	await restCatalogCreateNamespace(conn.catalog, { namespace: conn.namespace }).catch(() => {});
 }
@@ -219,7 +242,7 @@ async function createIcebergTables(conn, tables = ICEBERG_TABLES, encoding = "st
 async function listIcebergTables(conn) {
 	return (await restCatalogListTables(conn.catalog, { namespace: conn.namespace })).map((t) => t.name).sort();
 }
-const SNAPSHOT_REF_TTL_MS = 300 * 1e3;
+const SNAPSHOT_REF_TTL_MS = 1800 * 1e3;
 const RESOLVED_FILES_TTL_MS = 1440 * 60 * 1e3;
 const METADATA_TTL_MS = 1440 * 60 * 1e3;
 const MAX_CACHED_METADATA_BYTES = 2 * 1024 * 1024;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.32.5",
+  "version": "0.32.7",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,8 +181,8 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "gscdump": "0.32.5",
-    "@gscdump/contracts": "0.32.5"
+    "@gscdump/contracts": "0.32.7",
+    "gscdump": "0.32.7"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",