@gscdump/lakehouse 5.9.1 → 7.0.0

package/README.md

@@ -45,6 +45,27 @@ For retry decisions, import `isCommitRateLimited`, `isCommitServerError`, or
 Use the dataset registry to define datasets.
 Raw Icebird table creation and append functions live under `unsafe-raw` for package adapters and diagnostics.
 
+## Append retries
+
+Supply a durable `commitRetry.appendId` when a dataset append belongs to a job or receipt.
+Reuse that ID when the job retries, including retries after package upgrades.
+
+Without an explicit ID, row appends derive a versioned ID from framed, typed record content.
+Row order and object key order do not change the ID.
+
+If only a legacy content ID matches, the append throws `LegacyAppendIdentityUnverifiable`.
+The error includes `legacyAppendId`.
+The legacy ID cannot prove whether the committed rows match the retry.
+Verify the committed rows before supplying that legacy ID as an explicit `appendId`.
+If the rows differ, use a new durable ID for the new append.
+Do not retry with the error's legacy ID without checking the rows.
+
+Before upgrading producers that derive IDs, drain their existing write tasks.
+Do not run old and new derived-ID writers concurrently against the same table.
+Older writers cannot recognize the new content ID format.
+
+Batch appends already require an explicit `appendId` and retain that behavior.
+
 ## License
 
 [MIT](../../LICENSE)

package/dist/append-identity.mjs

@@ -0,0 +1,42 @@
+function encodeValue(value) {
+	if (value === null) return "null";
+	if (typeof value !== "object") {
+		if (typeof value === "function" || typeof value === "symbol") throw new TypeError("Append records contain an unsupported value.");
+		return `${typeof value}:${Object.is(value, -0) ? "-0" : String(value)}`;
+	}
+	if (value instanceof Date) return `date:${value.toISOString()}`;
+	if (value instanceof Uint8Array) return `bytes:${Array.from(value, (byte) => byte.toString(16).padStart(2, "0")).join("")}`;
+	if (Array.isArray(value)) return ["array", Array.from(value, encodeValue)];
+	if (value instanceof Map) return ["map", Array.from(value, ([key, entry]) => JSON.stringify([encodeValue(key), encodeValue(entry)])).sort()];
+	if (Object.getPrototypeOf(value) !== Object.prototype && Object.getPrototypeOf(value) !== null) throw new TypeError("Append records contain an unsupported value.");
+	return ["object", Object.keys(value).sort().map((key) => [key, encodeValue(value[key])])];
+}
+async function hashContent(content) {
+	const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(content));
+	return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join("");
+}
+async function deriveAppendIdentity(records) {
+	if (!records.length) return {
+		_tag: "Explicit",
+		appendId: crypto.randomUUID()
+	};
+	const legacyAppendId = await hashContent(records.map((record) => Object.keys(record).sort().map((key) => `${key}=${String(record[key])}`).join("")).sort().join(""));
+	const groups = /* @__PURE__ */ new Map();
+	for (const record of records) {
+		if (Object.getPrototypeOf(record) !== Object.prototype && Object.getPrototypeOf(record) !== null) throw new TypeError("Append records contain an unsupported value.");
+		const keys = Object.keys(record).sort();
+		const schema = JSON.stringify(keys);
+		const row = JSON.stringify(keys.map((key) => encodeValue(record[key])));
+		const group = groups.get(schema);
+		if (group) group.push(row);
+		else groups.set(schema, [row]);
+	}
+	const content = `[${Array.from(groups, ([schema, rows]) => `[${schema},[${rows.sort().join(",")}]]`).sort().join(",")}]`;
+	groups.clear();
+	return {
+		_tag: "Content",
+		appendId: `content-v2:${await hashContent(content)}`,
+		legacyAppendId
+	};
+}
+export { deriveAppendIdentity };

package/dist/catalog.d.mts

@@ -153,8 +153,9 @@ export interface CommitRetryOptions {
   /**
    * Idempotency token stamped into the appended snapshot's summary
    * (`lakehouse.append-id`) and matched by the landed-check. When omitted it is
-   * DERIVED from the records' content, making the token STABLE across
-   * PROCESSES, not just within one call's retry loop.
+   * Derived from framed, typed record content when omitted. Row and object
+   * key order do not change the token. A matching legacy content token throws
+   * `LegacyAppendIdentityUnverifiable` until committed rows are verified.
    */
   appendId?: string;
 }
@@ -192,7 +193,7 @@ export declare function isCommitTransient(err: unknown): boolean;
  * ({@link isCommitTransient}: 429 rate-limits and R2 5xx blips), using
  * full-jitter exponential back-off, plus a landed-check idempotency guard so a
  * failure whose commit actually landed is never re-applied (see
- * `deriveAppendId`/`appendAlreadyLanded`).
+ * `deriveAppendIdentity`/`checkAppendLanded`).
  *
  * Retrying is safe because `icebergAppend` writes data + manifest files BEFORE
  * the atomic catalog pointer swap, so a 5xx during the upload phase aborts

package/dist/catalog.mjs

@@ -3,6 +3,7 @@ import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables,
 import { cachingResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/fetch.mjs";
 import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/manifest.mjs";
 import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/s3.mjs";
+import { deriveAppendIdentity } from "./append-identity.mjs";
 import { cacheGet, cachePut, reportCatalogCacheError } from "./catalog-cache.mjs";
 import { toIcebergDayCount } from "./date.mjs";
 import { MULTI_MONTH_MANIFEST, buildManifestPartitionFilter, manifestMonthBucket } from "./partition-prune.mjs";
@@ -228,16 +229,20 @@ async function icebergAppendRetrying(args, options = {}) {
 	const maxDelayMs = options.maxDelayMs ?? 2e4;
 	const sleep = options.sleep ?? defaultCommitSleep;
 	const random = options.random ?? Math.random;
-	const appendId = options.appendId ?? await deriveAppendId(args);
+	const identity = options.appendId === void 0 ? await deriveAppendIdentity(args.records ?? []) : {
+		_tag: "Explicit",
+		appendId: options.appendId
+	};
+	const appendId = identity.appendId;
 	const stampedArgs = {
 		...args,
 		snapshotProperties: {
 			...args.snapshotProperties,
 			[APPEND_ID_SUMMARY_KEY]: appendId
 		},
-		isAlreadyApplied: (metadata) => hasAppendId(metadata, appendId)
+		isAlreadyApplied: (metadata) => hasAppendIdentity(metadata, identity)
 	};
-	let check = await checkAppendLanded(args, appendId);
+	let check = await checkAppendLanded(args, identity);
 	if (check.landed) return;
 	for (let attempt = 0; attempt < maxAttempts; attempt++) {
 		const err = await icebergAppend({
@@ -246,7 +251,7 @@ async function icebergAppendRetrying(args, options = {}) {
 		}).then(() => void 0, (e) => e);
 		if (err === void 0) return;
 		if (!isCommitTransient(err)) throw err;
-		check = await checkAppendLanded(args, appendId);
+		check = await checkAppendLanded(args, identity);
 		if (check.landed) return;
 		if (attempt === maxAttempts - 1) throw err;
 		const ceiling = Math.min(maxDelayMs, baseDelayMs * 2 ** attempt);
@@ -270,7 +275,10 @@ async function icebergAppendBatchesRetrying(args, options) {
 		},
 		isAlreadyApplied: (metadata) => hasAppendId(metadata, appendId)
 	};
-	let check = await checkAppendLanded(args, appendId);
+	let check = await checkAppendLanded(args, {
+		_tag: "Explicit",
+		appendId
+	});
 	if (check.landed) return false;
 	for (let attempt = 0; attempt < maxAttempts; attempt++) {
 		const err = await icebergAppendBatches({
@@ -280,7 +288,10 @@ async function icebergAppendBatchesRetrying(args, options) {
 		}).then(() => void 0, (e) => e);
 		if (err === void 0) return true;
 		if (!isCommitTransient(err)) throw err;
-		check = await checkAppendLanded(args, appendId);
+		check = await checkAppendLanded(args, {
+			_tag: "Explicit",
+			appendId
+		});
 		if (check.landed) return true;
 		if (attempt === maxAttempts - 1) throw err;
 		const ceiling = Math.min(maxDelayMs, baseDelayMs * 2 ** attempt);
@@ -288,15 +299,7 @@ async function icebergAppendBatchesRetrying(args, options) {
 	}
 	return false;
 }
-async function deriveAppendId(args) {
-	const records = args.records ?? [];
-	if (records.length === 0) return globalThis.crypto.randomUUID();
-	const rowSig = (r) => Object.keys(r).sort().map((k) => `${k}=${String(r[k])}`).join("");
-	const body = records.map(rowSig).sort().join("");
-	const digest = await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(body));
-	return Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, "0")).join("");
-}
-async function checkAppendLanded(args, appendId) {
+async function checkAppendLanded(args, identity) {
 	const a = args;
 	if (a.catalog?.type !== "rest" || a.namespace == null || a.table == null) return {
 		landed: false,
@@ -306,7 +309,7 @@ async function checkAppendLanded(args, appendId) {
 		namespace: a.namespace,
 		table: a.table
 	});
-	return hasAppendId(metadata, appendId) ? { landed: true } : {
+	return hasAppendIdentity(metadata, identity) ? { landed: true } : {
 		landed: false,
 		metadata
 	};
@@ -314,6 +317,14 @@ async function checkAppendLanded(args, appendId) {
 function hasAppendId(metadata, appendId) {
 	return metadata.snapshots?.some((snapshot) => snapshot.summary?.[APPEND_ID_SUMMARY_KEY] === appendId) ?? false;
 }
+function hasAppendIdentity(metadata, identity) {
+	if (hasAppendId(metadata, identity.appendId)) return true;
+	if (identity._tag === "Content" && hasAppendId(metadata, identity.legacyAppendId)) throw Object.assign(/* @__PURE__ */ new Error("A legacy append ID matched. Verify committed rows before retrying with an explicit append ID."), {
+		_tag: "LegacyAppendIdentityUnverifiable",
+		legacyAppendId: identity.legacyAppendId
+	});
+	return false;
+}
 const SNAPSHOT_REF_TTL_MS = 18e5;
 const RESOLVED_FILES_TTL_MS = 864e5;
 const METADATA_TTL_MS = 864e5;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/lakehouse",
   "type": "module",
-  "version": "5.9.1",
+  "version": "7.0.0",
   "description": "Dataset-agnostic Iceberg lakehouse layer + dataset registry for R2 Data Catalog producers (ADR-0021).",
   "author": {
     "name": "Harlan Wilton",