@gscdump/lakehouse 7.10.0 → 7.11.1

package/README.md

@@ -197,6 +197,35 @@ An uncertain response may follow a successful commit, so deleting staged files c
 The operation never expires historical snapshots or deletes historical manifests.
 Keep those objects while authenticated receipts or append retries still need them.
 
+## Table restoration
+
+Use `restoreIcebergTable(connection, table, options)` from `@gscdump/lakehouse/maintenance` for an empty Catalog target.
+Restore every referenced metadata, manifest, and data object before registering the table.
+Preserve their S3 locations, table UUID, snapshot IDs, and immutable bytes.
+The caller owns the target and excludes concurrent writers through settlement.
+
+Supply the saved metadata location, SHA-256, table UUID, head snapshot ID, and complete snapshot ID inventory.
+The operation checks the target, saved metadata bytes, and identity before registration.
+Metadata can use plain JSON or gzip. The byte limit applies to stored and expanded bytes.
+The SHA-256 always authenticates the stored object bytes before decompression.
+It registers with `overwrite: false` and verifies the returned identity.
+Existing tables produce `target-exists`, including tables with matching saved identities.
+Malformed bytes or mismatched identities produce `backup-invalid` before registration.
+
+Supply `confirmAuthority` to recheck ownership before access, immediately before registration, and after acknowledgement.
+This callback does not create an atomic transaction with external state.
+`Restored` confirms the acknowledged table identity. It does not verify every referenced data object.
+After restoration, run the application's receipt, object hash, and query checks through fresh read sessions.
+Supply `cache` to invalidate its saved snapshot pointer after success.
+
+`Unknown` means registration may have committed.
+Keep the restored objects and inspect fresh Catalog metadata before deciding the next action.
+The operation never retries, overwrites, or removes objects after an uncertain response.
+Other infrastructure failures propagate.
+
+To detach a registration while retaining its objects, use `dropIcebergTables(connection, tables, { purge: false })`.
+The default still requests purging. Use explicit table names for a restore drill.
+
 ## License
 
 [MIT](../../LICENSE)

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=3323fb6fb33622328d6d10f450feec7039468e8cad2e8667b7d8e59d48ae701b/node_modules/icebird/src/catalog/rest.mjs

@@ -64,6 +64,25 @@ async function restCatalogCreateTable(ctx, { namespace, table, schema, location,
 		config: responseBody.config ?? {}
 	};
 }
+async function restCatalogRegisterTable(ctx, { namespace, table, metadataLocation, overwrite }) {
+	const ns = encodeNamespace(namespace);
+	const body = {
+		name: table,
+		"metadata-location": metadataLocation
+	};
+	if (overwrite !== void 0) body.overwrite = overwrite;
+	const res = await restFetch(ctx, `namespaces/${ns}/register`, {
+		method: "POST",
+		headers: { "content-type": "application/json" },
+		body: stringifyIcebergJson(body)
+	});
+	const responseBody = parseIcebergJson(await res.text());
+	return {
+		metadataLocation: responseBody["metadata-location"],
+		metadata: responseBody.metadata,
+		config: responseBody.config ?? {}
+	};
+}
 async function restCatalogUpdateTable(ctx, { namespace, table, requirements, updates }) {
 	const res = await restFetch(ctx, `namespaces/${encodeNamespace(namespace)}/tables/${encodeURIComponent(table)}`, {
 		method: "POST",
@@ -161,4 +180,4 @@ async function paginate(baseParams, fetchPage) {
 		pageToken = encodeURIComponent(nextPageToken);
 	}
 }
-export { restCatalogConnect, restCatalogCreateNamespace, restCatalogCreateTable, restCatalogDropTable, restCatalogListTables, restCatalogLoadTable, restCatalogUpdateTable };
+export { restCatalogConnect, restCatalogCreateNamespace, restCatalogCreateTable, restCatalogDropTable, restCatalogListTables, restCatalogLoadTable, restCatalogRegisterTable, restCatalogUpdateTable };

package/dist/catalog.d.mts

@@ -135,10 +135,12 @@ export interface IcebergTableOpResult {
   error?: string;
 }
 /**
- * Drop tables from the catalog namespace, purging their data objects.
+ * Drop tables from the catalog namespace. Set purge to false to preserve their objects.
  * Defaults to every table currently in the namespace.
  */
-export declare function dropIcebergTables(conn: IcebergConnection, tables?: readonly string[]): Promise<IcebergTableOpResult[]>;
+export declare function dropIcebergTables(conn: IcebergConnection, tables?: readonly string[], options?: {
+  purge: boolean;
+}): Promise<IcebergTableOpResult[]>;
 /** Tunable retry policy for {@link icebergAppendRetrying}. */
 export interface CommitRetryOptions {
   /** Total attempts, including the first. Default 6. */

package/dist/catalog.mjs

@@ -190,7 +190,7 @@ async function ensureIcebergNamespace(conn) {
 async function listIcebergTables(conn) {
 	return (await restCatalogListTables(conn.catalog, { namespace: conn.namespace })).map((t) => t.name).sort();
 }
-async function dropIcebergTables(conn, tables) {
+async function dropIcebergTables(conn, tables, options = { purge: true }) {
 	const targets = tables ?? (await restCatalogListTables(conn.catalog, { namespace: conn.namespace })).map((t) => t.name);
 	const results = [];
 	const { icebergDropTable } = await useIcebirdWriteModule();
@@ -198,7 +198,7 @@ async function dropIcebergTables(conn, tables) {
 		catalog: conn.catalog,
 		namespace: conn.namespace,
 		table,
-		purgeRequested: true
+		purgeRequested: options.purge
 	}).then(() => results.push({
 		table,
 		ok: true

package/dist/maintenance.d.mts

@@ -2,4 +2,5 @@ import { isCommitRateLimited, isCommitServerError, isCommitTransient } from "./c
 import { MergeIcebergManifestsOptions, MergeIcebergManifestsResult, mergeIcebergManifests } from "./manifest-merge.mjs";
 import { SweepListPage, SweepListedObject, SweepStorageClient, SweepUncommittedOrphansOptions, SweepUncommittedOrphansResult, sweepUncommittedOrphans } from "./orphan-sweep.mjs";
 import { ExpireIcebergSnapshotsOptions, ExpireIcebergSnapshotsResult, PlanIcebergSnapshotRetentionOptions, PlanIcebergSnapshotRetentionResult, expireIcebergSnapshots, planIcebergSnapshotRetention } from "./snapshot-retention.mjs";
-export { type ExpireIcebergSnapshotsOptions, type ExpireIcebergSnapshotsResult, type MergeIcebergManifestsOptions, type MergeIcebergManifestsResult, type PlanIcebergSnapshotRetentionOptions, type PlanIcebergSnapshotRetentionResult, type SweepListPage, type SweepListedObject, type SweepStorageClient, type SweepUncommittedOrphansOptions, type SweepUncommittedOrphansResult, expireIcebergSnapshots, isCommitRateLimited, isCommitServerError, isCommitTransient, mergeIcebergManifests, planIcebergSnapshotRetention, sweepUncommittedOrphans };
+import { RestoreIcebergTableOptions, RestoreIcebergTableResult, restoreIcebergTable } from "./table-restore.mjs";
+export { type ExpireIcebergSnapshotsOptions, type ExpireIcebergSnapshotsResult, type MergeIcebergManifestsOptions, type MergeIcebergManifestsResult, type PlanIcebergSnapshotRetentionOptions, type PlanIcebergSnapshotRetentionResult, type RestoreIcebergTableOptions, type RestoreIcebergTableResult, type SweepListPage, type SweepListedObject, type SweepStorageClient, type SweepUncommittedOrphansOptions, type SweepUncommittedOrphansResult, expireIcebergSnapshots, isCommitRateLimited, isCommitServerError, isCommitTransient, mergeIcebergManifests, planIcebergSnapshotRetention, restoreIcebergTable, sweepUncommittedOrphans };

package/dist/maintenance.mjs

@@ -2,4 +2,5 @@ import { isCommitRateLimited, isCommitServerError, isCommitTransient } from "./c
 import { mergeIcebergManifests } from "./manifest-merge.mjs";
 import { sweepUncommittedOrphans } from "./orphan-sweep.mjs";
 import { expireIcebergSnapshots, planIcebergSnapshotRetention } from "./snapshot-retention.mjs";
-export { expireIcebergSnapshots, isCommitRateLimited, isCommitServerError, isCommitTransient, mergeIcebergManifests, planIcebergSnapshotRetention, sweepUncommittedOrphans };
+import { restoreIcebergTable } from "./table-restore.mjs";
+export { expireIcebergSnapshots, isCommitRateLimited, isCommitServerError, isCommitTransient, mergeIcebergManifests, planIcebergSnapshotRetention, restoreIcebergTable, sweepUncommittedOrphans };

package/dist/table-restore.d.mts

@@ -0,0 +1,33 @@
+import { CatalogCache } from "./catalog-cache.mjs";
+import { IcebergConnection } from "./catalog.mjs";
+export interface RestoreIcebergTableOptions {
+  /** Immutable metadata object restored before its Catalog registration. */
+  metadataLocation: string;
+  metadataSha256: string;
+  tableUuid: string;
+  headSnapshotId: string | null;
+  /** Complete snapshot identity inventory from the saved metadata. */
+  snapshotIds: readonly string[];
+  /** The caller excludes writers and owns the target through settlement. */
+  confirmAuthority: () => Promise<boolean>;
+  /** Bounds stored and expanded metadata. Defaults to 16 MiB. The maximum is 64 MiB. */
+  maxMetadataBytes?: number;
+  cache?: CatalogCache;
+}
+export type RestoreIcebergTableResult = {
+  _tag: 'Restored';
+  tableUuid: string;
+  snapshotIds: string[];
+} | {
+  _tag: 'Err';
+  reason: 'invalid-options' | 'target-exists' | 'backup-invalid' | 'authority-changed';
+} | {
+  _tag: 'Unknown';
+  reason: 'request-failed';
+  cause: unknown;
+} | {
+  _tag: 'Unknown';
+  reason: 'acknowledgement-invalid' | 'authority-changed';
+};
+/** Restore a verified metadata pointer into an empty target. Never replace an existing table. */
+export declare function restoreIcebergTable(conn: IcebergConnection, table: string, options: RestoreIcebergTableOptions): Promise<RestoreIcebergTableResult>;

package/dist/table-restore.mjs

@@ -0,0 +1,140 @@
+import { parseIcebergJson } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=3323fb6fb33622328d6d10f450feec7039468e8cad2e8667b7d8e59d48ae701b/node_modules/icebird/src/json.mjs";
+import { restCatalogLoadTable, restCatalogRegisterTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=3323fb6fb33622328d6d10f450feec7039468e8cad2e8667b7d8e59d48ae701b/node_modules/icebird/src/catalog/rest.mjs";
+import { invalidateSnapshotRef } from "./catalog.mjs";
+function validOptions(table, options) {
+	const maxBytes = options.maxMetadataBytes ?? 16777216;
+	return table.length > 0 && table.length <= 255 && /^s3:\/\/[a-z0-9][a-z0-9.-]+\/[^\s?#]+$/.test(options.metadataLocation) && /^[a-f0-9]{64}$/.test(options.metadataSha256) && /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(options.tableUuid) && Number.isSafeInteger(maxBytes) && maxBytes > 0 && maxBytes <= 67108864 && options.snapshotIds.length <= 1e5 && options.snapshotIds.every((id) => /^(?:0|[1-9]\d{0,18})$/.test(id)) && new Set(options.snapshotIds).size === options.snapshotIds.length && (options.headSnapshotId === null ? options.snapshotIds.length === 0 : options.snapshotIds.includes(options.headSnapshotId));
+}
+function matchesBackup(loaded, options) {
+	if (!loaded.metadata || typeof loaded.metadata !== "object" || Array.isArray(loaded.metadata)) return false;
+	const metadata = loaded.metadata;
+	const snapshots = metadata.snapshots ?? [];
+	if (!Array.isArray(snapshots) || snapshots.some((snapshot) => !snapshot || typeof snapshot !== "object")) return false;
+	const ids = snapshots.map((snapshot) => String(snapshot["snapshot-id"]));
+	const expected = new Set(options.snapshotIds);
+	const rawHead = metadata["current-snapshot-id"];
+	const head = rawHead == null || String(rawHead) === "-1" ? null : String(rawHead);
+	return loaded.metadataLocation === options.metadataLocation && metadata["table-uuid"] === options.tableUuid && head === options.headSnapshotId && ids.length === expected.size && new Set(ids).size === expected.size && ids.every((id) => expected.has(id));
+}
+async function expandMetadata(bytes, maxBytes) {
+	if (bytes[0] !== 31 || bytes[1] !== 139) return bytes;
+	const reader = new Blob([bytes]).stream().pipeThrough(new DecompressionStream("gzip")).getReader();
+	try {
+		const chunks = [];
+		let total = 0;
+		for (;;) {
+			const chunk = await reader.read();
+			if (chunk.done) break;
+			total += chunk.value.byteLength;
+			if (total > maxBytes) {
+				await reader.cancel();
+				return null;
+			}
+			chunks.push(chunk.value);
+		}
+		const expanded = new Uint8Array(total);
+		let offset = 0;
+		for (const chunk of chunks) {
+			expanded.set(chunk, offset);
+			offset += chunk.byteLength;
+		}
+		return expanded;
+	} catch {
+		return null;
+	} finally {
+		reader.releaseLock();
+	}
+}
+async function restoreIcebergTable(conn, table, options) {
+	if (!validOptions(table, options)) return {
+		_tag: "Err",
+		reason: "invalid-options"
+	};
+	if (!await options.confirmAuthority()) return {
+		_tag: "Err",
+		reason: "authority-changed"
+	};
+	if (await restCatalogLoadTable(conn.catalog, {
+		namespace: conn.namespace,
+		table
+	}).then(() => true, (error) => {
+		if (error && typeof error === "object" && "status" in error && error.status === 404) return false;
+		throw error;
+	})) return {
+		_tag: "Err",
+		reason: "target-exists"
+	};
+	const reader = await conn.resolver.reader(options.metadataLocation);
+	const maxBytes = options.maxMetadataBytes ?? 16777216;
+	if (!Number.isSafeInteger(reader.byteLength) || reader.byteLength <= 0 || reader.byteLength > maxBytes) return {
+		_tag: "Err",
+		reason: "backup-invalid"
+	};
+	const bytes = new Uint8Array(await reader.slice(0, reader.byteLength));
+	if (bytes.byteLength !== reader.byteLength) return {
+		_tag: "Err",
+		reason: "backup-invalid"
+	};
+	const digest = await crypto.subtle.digest("SHA-256", bytes);
+	if ([...new Uint8Array(digest)].map((value) => value.toString(16).padStart(2, "0")).join("") !== options.metadataSha256) return {
+		_tag: "Err",
+		reason: "backup-invalid"
+	};
+	const expanded = await expandMetadata(bytes, maxBytes);
+	if (!expanded) return {
+		_tag: "Err",
+		reason: "backup-invalid"
+	};
+	let metadata;
+	try {
+		metadata = parseIcebergJson(new TextDecoder("utf-8", { fatal: true }).decode(expanded));
+	} catch {
+		return {
+			_tag: "Err",
+			reason: "backup-invalid"
+		};
+	}
+	if (!matchesBackup({
+		metadataLocation: options.metadataLocation,
+		metadata
+	}, options)) return {
+		_tag: "Err",
+		reason: "backup-invalid"
+	};
+	if (!await options.confirmAuthority()) return {
+		_tag: "Err",
+		reason: "authority-changed"
+	};
+	const registered = await restCatalogRegisterTable(conn.catalog, {
+		namespace: conn.namespace,
+		table,
+		metadataLocation: options.metadataLocation,
+		overwrite: false
+	}).then((value) => ({
+		_tag: "Acknowledged",
+		value
+	}), (cause) => ({
+		_tag: "Unknown",
+		reason: "request-failed",
+		cause
+	}));
+	if (registered._tag === "Unknown") return registered;
+	if (!matchesBackup({
+		metadataLocation: registered.value.metadataLocation,
+		metadata: registered.value.metadata
+	}, options)) return {
+		_tag: "Unknown",
+		reason: "acknowledgement-invalid"
+	};
+	if (!await options.confirmAuthority()) return {
+		_tag: "Unknown",
+		reason: "authority-changed"
+	};
+	if (options.cache) await invalidateSnapshotRef(options.cache, conn.namespace, table, conn.cacheScope);
+	return {
+		_tag: "Restored",
+		tableUuid: options.tableUuid,
+		snapshotIds: [...options.snapshotIds]
+	};
+}
+export { restoreIcebergTable };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/lakehouse",
   "type": "module",
-  "version": "7.10.0",
+  "version": "7.11.1",
   "description": "Dataset-agnostic Iceberg lakehouse layer + dataset registry for R2 Data Catalog producers (ADR-0021).",
   "author": {
     "name": "Harlan Wilton",