@gscdump/lakehouse 3.6.2 → 3.7.0

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.28_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae}/node_modules/icebird/src/write/write.mjs

@@ -14,14 +14,30 @@ const DEFAULT_RETRY = Object.freeze({
 	factor: 2,
 	totalTimeoutMs: 18e5
 });
-async function icebergAppend({ catalog, namespace, table, tableUrl, resolver, records, sortOrderId, snapshotProperties }) {
-	const ctx = await loadTable({
+async function loadTableOrReuse({ catalog, namespace, table, tableUrl, resolver, metadata }) {
+	if (metadata && catalog.type === "rest") return {
+		metadata,
+		metadataFileName: void 0,
+		tableUrl: metadata.location,
+		resolver
+	};
+	return await loadTable({
 		catalog,
 		namespace,
 		table,
 		tableUrl,
 		resolver
 	});
+}
+async function icebergAppend({ catalog, namespace, table, tableUrl, resolver, records, sortOrderId, snapshotProperties, metadata }) {
+	const ctx = await loadTableOrReuse({
+		catalog,
+		namespace,
+		table,
+		tableUrl,
+		resolver,
+		metadata
+	});
 	const prepared = await prepareAppend({
 		tableUrl: ctx.tableUrl,
 		metadata: ctx.metadata,
@@ -45,13 +61,14 @@ async function icebergAppend({ catalog, namespace, table, tableUrl, resolver, re
 		})
 	});
 }
-async function icebergAppendBatches({ catalog, namespace, table, tableUrl, resolver, batches, sortOrderId, snapshotProperties }) {
-	const ctx = await loadTable({
+async function icebergAppendBatches({ catalog, namespace, table, tableUrl, resolver, batches, sortOrderId, snapshotProperties, metadata }) {
+	const ctx = await loadTableOrReuse({
 		catalog,
 		namespace,
 		table,
 		tableUrl,
-		resolver
+		resolver,
+		metadata
 	});
 	const writer = requireResolver(ctx.resolver, "icebergAppendBatches");
 	const prepared = [];

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.28_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae}/node_modules/icebird/types/write/write.d.mts

@@ -22,6 +22,7 @@ export declare function icebergAppend({ catalog, namespace, table, tableUrl, res
   records: Record<string, any>[];
   sortOrderId?: number;
   snapshotProperties?: Record<string, string>;
+  metadata?: TableMetadata;
 }): Promise<TableMetadata>;
 /**
  * Create a new table. REST: delegates to the catalog's create endpoint.

package/dist/catalog.d.mts

@@ -1,9 +1,9 @@
-import { icebergAppend } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/types/write/write.mjs";
+import { icebergAppend } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/types/write/write.mjs";
 import { CatalogCache } from "./catalog-cache.mjs";
 import { IcebergPrimitiveType, IcebergS3Config } from "./schema.mjs";
 import { PartitionValueMatch } from "./partition-prune.mjs";
-import { restCatalogConnect } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/types/catalog/rest.mjs";
-import { cachingResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/types/fetch.mjs";
+import { restCatalogConnect } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/types/catalog/rest.mjs";
+import { cachingResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/types/fetch.mjs";
 /** A field in an icebird table `Schema`. */
 export interface IcebergSchemaField {
   id: number;

package/dist/catalog.mjs

@@ -1,14 +1,14 @@
 import { stringifyBigintSafe } from "./bigint.mjs";
-import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/src/catalog/rest.mjs";
-import { cachingResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/src/fetch.mjs";
-import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/src/manifest.mjs";
-import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/src/s3.mjs";
+import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/catalog/rest.mjs";
+import { cachingResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/fetch.mjs";
+import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/manifest.mjs";
+import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/s3.mjs";
 import { cacheGet, cachePut, reportCatalogCacheError } from "./catalog-cache.mjs";
 import { toIcebergDayCount } from "./date.mjs";
 import { MULTI_MONTH_MANIFEST, buildManifestPartitionFilter, manifestMonthBucket } from "./partition-prune.mjs";
 let icebirdWriteModule;
 function useIcebirdWriteModule() {
-	icebirdWriteModule ??= import("./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/src/write/write.mjs");
+	icebirdWriteModule ??= import("./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/write/write.mjs");
 	return icebirdWriteModule;
 }
 const CATALOG_CONFIG_TTL_MS = 36e5;
@@ -166,12 +166,17 @@ async function icebergAppendRetrying(args, options = {}) {
 			[APPEND_ID_SUMMARY_KEY]: appendId
 		}
 	};
-	if (await appendAlreadyLanded(args, appendId)) return;
+	let check = await checkAppendLanded(args, appendId);
+	if (check.landed) return;
 	for (let attempt = 0; attempt < maxAttempts; attempt++) {
-		const err = await icebergAppend(stampedArgs).then(() => void 0, (e) => e);
+		const err = await icebergAppend({
+			...stampedArgs,
+			metadata: check.metadata
+		}).then(() => void 0, (e) => e);
 		if (err === void 0) return;
 		if (!isCommitTransient(err)) throw err;
-		if (await appendAlreadyLanded(args, appendId)) return;
+		check = await checkAppendLanded(args, appendId);
+		if (check.landed) return;
 		if (attempt === maxAttempts - 1) throw err;
 		const ceiling = Math.min(maxDelayMs, baseDelayMs * 2 ** attempt);
 		await sleep(Math.floor(random() * ceiling));
@@ -193,15 +198,18 @@ async function icebergAppendBatchesRetrying(args, options) {
 			[APPEND_ID_SUMMARY_KEY]: appendId
 		}
 	};
-	if (await appendAlreadyLanded(args, appendId)) return false;
+	let check = await checkAppendLanded(args, appendId);
+	if (check.landed) return false;
 	for (let attempt = 0; attempt < maxAttempts; attempt++) {
 		const err = await icebergAppendBatches({
 			...stampedArgs,
+			metadata: check.metadata,
 			batches: batchFactory()
 		}).then(() => void 0, (e) => e);
 		if (err === void 0) return true;
 		if (!isCommitTransient(err)) throw err;
-		if (await appendAlreadyLanded(args, appendId)) return true;
+		check = await checkAppendLanded(args, appendId);
+		if (check.landed) return true;
 		if (attempt === maxAttempts - 1) throw err;
 		const ceiling = Math.min(maxDelayMs, baseDelayMs * 2 ** attempt);
 		await sleep(Math.floor(random() * ceiling));
@@ -216,17 +224,23 @@ async function deriveAppendId(args) {
 	const digest = await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(body));
 	return Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, "0")).join("");
 }
-async function appendAlreadyLanded(args, appendId) {
+async function checkAppendLanded(args, appendId) {
 	const a = args;
-	if (a.catalog?.type !== "rest" || a.namespace == null || a.table == null) return false;
+	if (a.catalog?.type !== "rest" || a.namespace == null || a.table == null) return {
+		landed: false,
+		metadata: void 0
+	};
 	const { metadata } = await restCatalogLoadTable(a.catalog, {
 		namespace: a.namespace,
 		table: a.table
 	});
 	const snapshots = metadata.snapshots ?? [];
 	const from = Math.max(0, snapshots.length - APPEND_LANDED_SCAN_DEPTH);
-	for (let i = snapshots.length - 1; i >= from; i--) if (snapshots[i]?.summary?.[APPEND_ID_SUMMARY_KEY] === appendId) return true;
-	return false;
+	for (let i = snapshots.length - 1; i >= from; i--) if (snapshots[i]?.summary?.[APPEND_ID_SUMMARY_KEY] === appendId) return { landed: true };
+	return {
+		landed: false,
+		metadata
+	};
 }
 const SNAPSHOT_REF_TTL_MS = 18e5;
 const RESOLVED_FILES_TTL_MS = 864e5;

package/dist/dataset-runtime.mjs

@@ -1,5 +1,5 @@
 import { connectIcebergCatalog, ensureIcebergNamespace, icebergAppendBatchesRetrying, icebergAppendRetrying, resolveIcebergDataFiles } from "./catalog.mjs";
-import { icebergCreateTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/src/write/write.mjs";
+import { icebergCreateTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/write/write.mjs";
 async function createDatasetTable(conn, table, schema, partitionSpec, sortOrder) {
 	const results = [];
 	await icebergCreateTable({

package/dist/orphan-sweep.mjs

@@ -1,6 +1,6 @@
-import { restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/src/catalog/rest.mjs";
-import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/src/manifest.mjs";
-import "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/src/index.mjs";
+import { restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/catalog/rest.mjs";
+import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/manifest.mjs";
+import "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/index.mjs";
 const DEFAULT_GRACE_HOURS = 48;
 const DEFAULT_MAX_DELETES = 500;
 const DEFAULT_IO_CONCURRENCY = 4;

package/dist/unsafe-raw.d.mts

@@ -1,7 +1,7 @@
-import { icebergAppend, icebergCreateTable, icebergDropTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/types/write/write.mjs";
-import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/types/catalog/rest.mjs";
+import { icebergAppend, icebergCreateTable, icebergDropTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/types/write/write.mjs";
+import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/types/catalog/rest.mjs";
 import { icebergAppendRetrying } from "./catalog.mjs";
-import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/types/s3.mjs";
-import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/types/manifest.mjs";
-import "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/types/index.mjs";
+import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/types/s3.mjs";
+import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/types/manifest.mjs";
+import "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/types/index.mjs";
 export { icebergAppend, icebergAppendRetrying, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver };

package/dist/unsafe-raw.mjs

@@ -1,7 +1,7 @@
-import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/src/catalog/rest.mjs";
-import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/src/manifest.mjs";
-import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/src/s3.mjs";
+import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/catalog/rest.mjs";
+import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/manifest.mjs";
+import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/s3.mjs";
 import { icebergAppendRetrying } from "./catalog.mjs";
-import { icebergAppend, icebergCreateTable, icebergDropTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/src/write/write.mjs";
-import "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/src/index.mjs";
+import { icebergAppend, icebergCreateTable, icebergDropTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/write/write.mjs";
+import "./_virtual/node_modules/.pnpm/icebird@0.8.28_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/index.mjs";
 export { icebergAppend, icebergAppendRetrying, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/lakehouse",
   "type": "module",
-  "version": "3.6.2",
+  "version": "3.7.0",
   "description": "Dataset-agnostic Iceberg lakehouse layer + dataset registry for R2 Data Catalog producers (ADR-0021).",
   "author": {
     "name": "Harlan Wilton",