@gscdump/lakehouse 2.0.3 → 2.0.5

package/README.md

@@ -29,7 +29,7 @@ Node.js 22 or newer is required.
 ```ts
 import { connectIcebergCatalog, listIcebergTables } from '@gscdump/lakehouse'
 import { encodeJsonBigintSafe } from '@gscdump/lakehouse/bigint'
-import { isCommitRateLimited } from '@gscdump/lakehouse/maintenance'
+import { isCommitRateLimited, isCommitTransient } from '@gscdump/lakehouse/maintenance'
 
 const connection = await connectIcebergCatalog(config)
 const tables = await listIcebergTables(connection)
@@ -39,11 +39,18 @@ try {
 }
 catch (error) {
   if (isCommitRateLimited(error)) {
-    // Retry using the caller's bounded backoff policy.
+    // 429 specifically: decorrelate concurrent writers (defer off-slot).
+  }
+  else if (isCommitTransient(error)) {
+    // 429 or an R2 5xx blip: worth another attempt at all.
   }
 }
 ```
 
+`isCommitRateLimited` stays 429-only; `isCommitServerError` covers transient
+R2 5xx responses; `isCommitTransient` is the union and is what the package's
+own append-retry loop uses.
+
 The dataset registry is the normal authoring boundary. Raw Icebird table
 creation and append primitives are intentionally excluded from the package
 root so producers cannot silently bypass registered dataset definitions.

package/dist/catalog.d.mts

@@ -142,13 +142,43 @@ type IcebergAppendArgs = Parameters<typeof icebergAppend>[0];
 /**
  * True when `err` is an R2 Data Catalog commit rate-limit response
  * (`429 too many commits to this table`).
+ *
+ * Deliberately 429-ONLY. Callers use this to decide whether to decorrelate
+ * concurrent committers (defer the whole unit of work off-slot), which is the
+ * wrong response to a one-off server blip — see {@link isCommitServerError}.
  */
 declare function isCommitRateLimited(err: unknown): boolean;
 /**
- * `icebergAppend` wrapped with retry on R2 Data Catalog 429 commit
- * rate-limits, using full-jitter exponential back-off, plus a landed-check
- * idempotency guard so a 429 whose commit actually landed is never re-applied
- * (see `deriveAppendId`/`appendAlreadyLanded`).
+ * True when `err` is a transient server-side 5xx from R2 — a bare
+ * `500 Internal Server Error` on a data/metadata object PUT, or a 502/503/504
+ * from the catalog REST API.
+ *
+ * These carry no S3 error code and no body; they are single-request blips that
+ * clear on the next call. Distinct from {@link isCommitRateLimited} so a caller
+ * that must decorrelate writers on a 429 does not also do so on a server error.
+ */
+declare function isCommitServerError(err: unknown): boolean;
+/**
+ * The retry predicate: is `err` worth another append attempt at all?
+ *
+ * Union of {@link isCommitRateLimited} and {@link isCommitServerError}.
+ * Everything else — 4xx, catalog conflicts icebird already exhausted its own
+ * 412/409 retries on, schema/validation failures — is permanent and must
+ * propagate on the first attempt.
+ */
+declare function isCommitTransient(err: unknown): boolean;
+/**
+ * `icebergAppend` wrapped with retry on transient commit failures
+ * ({@link isCommitTransient}: 429 rate-limits and R2 5xx blips), using
+ * full-jitter exponential back-off, plus a landed-check idempotency guard so a
+ * failure whose commit actually landed is never re-applied (see
+ * `deriveAppendId`/`appendAlreadyLanded`).
+ *
+ * Retrying is safe because `icebergAppend` writes data + manifest files BEFORE
+ * the atomic catalog pointer swap, so a 5xx during the upload phase aborts
+ * before anything is referenced by a snapshot; the retry writes fresh files and
+ * commits once. A 5xx on the pointer swap itself is covered by the
+ * landed-check.
  */
 declare function icebergAppendRetrying(args: IcebergAppendArgs, options?: CommitRetryOptions): Promise<void>;
 /** A data file in the current snapshot's manifest, scoped to one partition. */
@@ -210,4 +240,4 @@ declare function invalidateSnapshotRef(cache: CatalogCache, namespace: string, t
  * the def's own spec + identity/dims values.
  */
 declare function resolveIcebergDataFiles(conn: IcebergConnection, opts: ResolveIcebergDataFilesOptions): Promise<IcebergListedDataFile[]>;
-export { CommitRetryOptions, ConnectIcebergOptions, IcebergAppendArgs, IcebergCatalogConfig, IcebergConnection, IcebergListedDataFile, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableOpResult, QueryProfiler, ResolveIcebergDataFilesOptions, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, invalidateSnapshotRef, isCommitRateLimited, listIcebergTables, resolveIcebergDataFiles };
+export { CommitRetryOptions, ConnectIcebergOptions, IcebergAppendArgs, IcebergCatalogConfig, IcebergConnection, IcebergListedDataFile, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableOpResult, QueryProfiler, ResolveIcebergDataFilesOptions, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, invalidateSnapshotRef, isCommitRateLimited, isCommitServerError, isCommitTransient, listIcebergTables, resolveIcebergDataFiles };

package/dist/catalog.mjs

@@ -129,6 +129,22 @@ function isCommitRateLimited(err) {
 	const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
 	return msg.includes("429") || msg.includes("too many commits") || msg.includes("rate limit");
 }
+const TRANSIENT_SERVER_STATUSES = /* @__PURE__ */ new Set([
+	500,
+	502,
+	503,
+	504
+]);
+const TRANSIENT_SERVER_MESSAGE_RE = /\b(?:500 internal server error|502 bad gateway|503 service unavailable|504 gateway time-?out)\b/i;
+function isCommitServerError(err) {
+	const status = err && typeof err === "object" ? err.status : void 0;
+	if (typeof status === "number") return TRANSIENT_SERVER_STATUSES.has(status);
+	const msg = err instanceof Error ? err.message : String(err);
+	return TRANSIENT_SERVER_MESSAGE_RE.test(msg);
+}
+function isCommitTransient(err) {
+	return isCommitRateLimited(err) || isCommitServerError(err);
+}
 function defaultCommitSleep(ms) {
 	return new Promise((resolve) => setTimeout(resolve, ms));
 }
@@ -151,7 +167,7 @@ async function icebergAppendRetrying(args, options = {}) {
 	for (let attempt = 0; attempt < maxAttempts; attempt++) {
 		const err = await icebergAppend(stampedArgs).then(() => void 0, (e) => e);
 		if (err === void 0) return;
-		if (!isCommitRateLimited(err)) throw err;
+		if (!isCommitTransient(err)) throw err;
 		if (await appendAlreadyLanded(args, appendId)) return;
 		if (attempt === maxAttempts - 1) throw err;
 		const ceiling = Math.min(maxDelayMs, baseDelayMs * 2 ** attempt);
@@ -181,7 +197,7 @@ async function icebergAppendBatchesRetrying(args, options) {
 			batches: batchFactory()
 		}).then(() => void 0, (e) => e);
 		if (err === void 0) return true;
-		if (!isCommitRateLimited(err)) throw err;
+		if (!isCommitTransient(err)) throw err;
 		if (await appendAlreadyLanded(args, appendId)) return true;
 		if (attempt === maxAttempts - 1) throw err;
 		const ceiling = Math.min(maxDelayMs, baseDelayMs * 2 ** attempt);
@@ -272,7 +288,11 @@ async function loadSnapshotId(conn, namespace, table, cache, now) {
 	if (cache) {
 		await cachePut(cache, snapshotRefKey(scope, namespace, table), snapshotId, SNAPSHOT_REF_TTL_MS, now);
 		if (snapshotId != null) {
-			if (stringifyBigintSafe(metadata).length <= MAX_CACHED_METADATA_BYTES) await cachePut(cache, metadataRefKey(scope, namespace, table, snapshotId), metadata, METADATA_TTL_MS, now);
+			const serialized = stringifyBigintSafe(metadata);
+			if (serialized.length <= MAX_CACHED_METADATA_BYTES) {
+				const cacheableMetadata = JSON.parse(serialized);
+				await cachePut(cache, metadataRefKey(scope, namespace, table, snapshotId), cacheableMetadata, METADATA_TTL_MS, now);
+			}
 		}
 	}
 	return {
@@ -348,4 +368,4 @@ async function resolveIcebergDataFiles(conn, opts) {
 	}
 	return out;
 }
-export { catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, icebergAppendBatchesRetrying, icebergAppendRetrying, invalidateSnapshotRef, isCommitRateLimited, listIcebergTables, resolveIcebergDataFiles };
+export { catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, icebergAppendBatchesRetrying, icebergAppendRetrying, invalidateSnapshotRef, isCommitRateLimited, isCommitServerError, isCommitTransient, listIcebergTables, resolveIcebergDataFiles };

package/dist/maintenance.d.mts

@@ -1,3 +1,3 @@
-import { isCommitRateLimited } from "./catalog.mjs";
+import { isCommitRateLimited, isCommitServerError, isCommitTransient } from "./catalog.mjs";
 import { SweepListPage, SweepListedObject, SweepStorageClient, SweepUncommittedOrphansOptions, SweepUncommittedOrphansResult, sweepUncommittedOrphans } from "./orphan-sweep.mjs";
-export { type SweepListPage, type SweepListedObject, type SweepStorageClient, type SweepUncommittedOrphansOptions, type SweepUncommittedOrphansResult, isCommitRateLimited, sweepUncommittedOrphans };
+export { type SweepListPage, type SweepListedObject, type SweepStorageClient, type SweepUncommittedOrphansOptions, type SweepUncommittedOrphansResult, isCommitRateLimited, isCommitServerError, isCommitTransient, sweepUncommittedOrphans };

package/dist/maintenance.mjs

@@ -1,3 +1,3 @@
-import { isCommitRateLimited } from "./catalog.mjs";
+import { isCommitRateLimited, isCommitServerError, isCommitTransient } from "./catalog.mjs";
 import { sweepUncommittedOrphans } from "./orphan-sweep.mjs";
-export { isCommitRateLimited, sweepUncommittedOrphans };
+export { isCommitRateLimited, isCommitServerError, isCommitTransient, sweepUncommittedOrphans };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/lakehouse",
   "type": "module",
-  "version": "2.0.3",
+  "version": "2.0.5",
   "description": "Dataset-agnostic Iceberg lakehouse layer + dataset registry for R2 Data Catalog producers (ADR-0021).",
   "author": {
     "name": "Harlan Wilton",