@gscdump/lakehouse 4.0.1 → 4.1.0

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.30_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae → icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7}/node_modules/icebird/src/write/write.mjs

@@ -29,7 +29,7 @@ async function loadTableOrReuse({ catalog, namespace, table, tableUrl, resolver,
 		resolver
 	});
 }
-async function icebergAppend({ catalog, namespace, table, tableUrl, resolver, records, sortOrderId, snapshotProperties, metadata }) {
+async function icebergAppend({ catalog, namespace, table, tableUrl, resolver, records, sortOrderId, snapshotProperties, metadata, isAlreadyApplied }) {
 	const ctx = await loadTableOrReuse({
 		catalog,
 		namespace,
@@ -52,6 +52,7 @@ async function icebergAppend({ catalog, namespace, table, tableUrl, resolver, re
 			table
 		},
 		ctx,
+		isAlreadyApplied,
 		stage: (workingCtx) => stageSnapshotForAppend({
 			tableUrl: workingCtx.tableUrl,
 			metadata: workingCtx.metadata,
@@ -61,7 +62,7 @@ async function icebergAppend({ catalog, namespace, table, tableUrl, resolver, re
 		})
 	});
 }
-async function icebergAppendBatches({ catalog, namespace, table, tableUrl, resolver, batches, sortOrderId, snapshotProperties, metadata }) {
+async function icebergAppendBatches({ catalog, namespace, table, tableUrl, resolver, batches, sortOrderId, snapshotProperties, metadata, isAlreadyApplied }) {
 	const ctx = await loadTableOrReuse({
 		catalog,
 		namespace,
@@ -94,6 +95,7 @@ async function icebergAppendBatches({ catalog, namespace, table, tableUrl, resol
 			table
 		},
 		ctx,
+		isAlreadyApplied,
 		stage: (workingCtx) => stageSnapshotForAppendBatches({
 			tableUrl: workingCtx.tableUrl,
 			metadata: workingCtx.metadata,
@@ -406,12 +408,13 @@ async function commitStaged(catalog, target, ctx, staged) {
 		conditionalCommits: catalog.type === "file" && catalog.conditionalCommits
 	});
 }
-async function commitWithRetry({ catalog, target, ctx, stage }) {
+async function commitWithRetry({ catalog, target, ctx, stage, isAlreadyApplied }) {
 	const retryEnabled = catalog.type === "rest" || catalog.type === "file" && catalog.conditionalCommits === true;
 	const policy = resolveRetryPolicy(ctx.metadata);
 	const startedAt = Date.now();
 	let workingCtx = ctx;
 	for (let attempt = 1; attempt <= policy.maxAttempts; attempt++) {
+		if (isAlreadyApplied?.(workingCtx.metadata)) return workingCtx.metadata;
 		const staged = await stage(workingCtx);
 		try {
 			return await commitStaged(catalog, target, workingCtx, staged);

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.30_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae → icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7}/node_modules/icebird/types/write/write.d.mts

@@ -13,7 +13,7 @@ import { Catalog, Lister, PartitionSpec, Resolver, Schema, SortOrder, TableMetad
  * @param {number} [options.sortOrderId] - Sort order id to apply; defaults to the table default.
  * @returns {Promise<TableMetadata>}
  */
-export declare function icebergAppend({ catalog, namespace, table, tableUrl, resolver, records, sortOrderId, snapshotProperties }: {
+export declare function icebergAppend({ catalog, namespace, table, tableUrl, resolver, records, sortOrderId, snapshotProperties, metadata, isAlreadyApplied }: {
   catalog: Catalog;
   namespace?: string | string[];
   table?: string;
@@ -23,6 +23,7 @@ export declare function icebergAppend({ catalog, namespace, table, tableUrl, res
   sortOrderId?: number;
   snapshotProperties?: Record<string, string>;
   metadata?: TableMetadata;
+  isAlreadyApplied?: (metadata: TableMetadata) => boolean;
 }): Promise<TableMetadata>;
 /**
  * Create a new table. REST: delegates to the catalog's create endpoint.

package/dist/catalog.d.mts

@@ -1,9 +1,9 @@
-import { icebergAppend } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/types/write/write.mjs";
+import { icebergAppend } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/types/write/write.mjs";
 import { CatalogCache } from "./catalog-cache.mjs";
 import { IcebergPrimitiveType, IcebergS3Config } from "./schema.mjs";
 import { PartitionValueMatch } from "./partition-prune.mjs";
-import { restCatalogConnect } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/types/catalog/rest.mjs";
-import { cachingResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/types/fetch.mjs";
+import { restCatalogConnect } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/types/catalog/rest.mjs";
+import { cachingResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/types/fetch.mjs";
 /** A field in an icebird table `Schema`. */
 export interface IcebergSchemaField {
   id: number;

package/dist/catalog.mjs

@@ -1,14 +1,14 @@
 import { stringifyBigintSafe } from "./bigint.mjs";
-import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/catalog/rest.mjs";
-import { cachingResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/fetch.mjs";
-import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/manifest.mjs";
-import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/s3.mjs";
+import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/catalog/rest.mjs";
+import { cachingResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/fetch.mjs";
+import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/manifest.mjs";
+import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/s3.mjs";
 import { cacheGet, cachePut, reportCatalogCacheError } from "./catalog-cache.mjs";
 import { toIcebergDayCount } from "./date.mjs";
 import { MULTI_MONTH_MANIFEST, buildManifestPartitionFilter, manifestMonthBucket } from "./partition-prune.mjs";
 let icebirdWriteModule;
 function useIcebirdWriteModule() {
-	icebirdWriteModule ??= import("./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/write/write.mjs");
+	icebirdWriteModule ??= import("./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/write/write.mjs");
 	return icebirdWriteModule;
 }
 const CATALOG_CONFIG_TTL_MS = 36e5;
@@ -126,7 +126,6 @@ async function dropIcebergTables(conn, tables) {
 	return results;
 }
 const APPEND_ID_SUMMARY_KEY = "lakehouse.append-id";
-const APPEND_LANDED_SCAN_DEPTH = 25;
 function isCommitRateLimited(err) {
 	if (err && typeof err === "object" && err.status === 429) return true;
 	const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
@@ -164,7 +163,8 @@ async function icebergAppendRetrying(args, options = {}) {
 		snapshotProperties: {
 			...args.snapshotProperties,
 			[APPEND_ID_SUMMARY_KEY]: appendId
-		}
+		},
+		isAlreadyApplied: (metadata) => hasAppendId(metadata, appendId)
 	};
 	let check = await checkAppendLanded(args, appendId);
 	if (check.landed) return;
@@ -196,7 +196,8 @@ async function icebergAppendBatchesRetrying(args, options) {
 		snapshotProperties: {
 			...appendArgs.snapshotProperties,
 			[APPEND_ID_SUMMARY_KEY]: appendId
-		}
+		},
+		isAlreadyApplied: (metadata) => hasAppendId(metadata, appendId)
 	};
 	let check = await checkAppendLanded(args, appendId);
 	if (check.landed) return false;
@@ -234,14 +235,14 @@ async function checkAppendLanded(args, appendId) {
 		namespace: a.namespace,
 		table: a.table
 	});
-	const snapshots = metadata.snapshots ?? [];
-	const from = Math.max(0, snapshots.length - APPEND_LANDED_SCAN_DEPTH);
-	for (let i = snapshots.length - 1; i >= from; i--) if (snapshots[i]?.summary?.[APPEND_ID_SUMMARY_KEY] === appendId) return { landed: true };
-	return {
+	return hasAppendId(metadata, appendId) ? { landed: true } : {
 		landed: false,
 		metadata
 	};
 }
+function hasAppendId(metadata, appendId) {
+	return metadata.snapshots?.some((snapshot) => snapshot.summary?.[APPEND_ID_SUMMARY_KEY] === appendId) ?? false;
+}
 const SNAPSHOT_REF_TTL_MS = 18e5;
 const RESOLVED_FILES_TTL_MS = 864e5;
 const METADATA_TTL_MS = 864e5;

package/dist/dataset-runtime.mjs

@@ -1,5 +1,5 @@
 import { connectIcebergCatalog, ensureIcebergNamespace, icebergAppendBatchesRetrying, icebergAppendRetrying, resolveIcebergDataFiles } from "./catalog.mjs";
-import { icebergCreateTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/write/write.mjs";
+import { icebergCreateTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/write/write.mjs";
 async function createDatasetTable(conn, table, schema, partitionSpec, sortOrder) {
 	const results = [];
 	await icebergCreateTable({

package/dist/orphan-sweep.mjs

@@ -1,6 +1,6 @@
-import { restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/catalog/rest.mjs";
-import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/manifest.mjs";
-import "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/index.mjs";
+import { restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/catalog/rest.mjs";
+import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/manifest.mjs";
+import "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/index.mjs";
 const DEFAULT_GRACE_HOURS = 48;
 const DEFAULT_MAX_DELETES = 500;
 const DEFAULT_IO_CONCURRENCY = 4;

package/dist/unsafe-raw.d.mts

@@ -1,7 +1,7 @@
-import { icebergAppend, icebergCreateTable, icebergDropTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/types/write/write.mjs";
-import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/types/catalog/rest.mjs";
+import { icebergAppend, icebergCreateTable, icebergDropTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/types/write/write.mjs";
+import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/types/catalog/rest.mjs";
 import { icebergAppendRetrying } from "./catalog.mjs";
-import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/types/s3.mjs";
-import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/types/manifest.mjs";
-import "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/types/index.mjs";
+import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/types/s3.mjs";
+import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/types/manifest.mjs";
+import "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/types/index.mjs";
 export { icebergAppend, icebergAppendRetrying, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver };

package/dist/unsafe-raw.mjs

@@ -1,7 +1,7 @@
-import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/catalog/rest.mjs";
-import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/manifest.mjs";
-import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/s3.mjs";
+import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/catalog/rest.mjs";
+import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/manifest.mjs";
+import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/s3.mjs";
 import { icebergAppendRetrying } from "./catalog.mjs";
-import { icebergAppend, icebergCreateTable, icebergDropTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/write/write.mjs";
-import "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/index.mjs";
+import { icebergAppend, icebergCreateTable, icebergDropTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/write/write.mjs";
+import "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/index.mjs";
 export { icebergAppend, icebergAppendRetrying, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/lakehouse",
   "type": "module",
-  "version": "4.0.1",
+  "version": "4.1.0",
   "description": "Dataset-agnostic Iceberg lakehouse layer + dataset registry for R2 Data Catalog producers (ADR-0021).",
   "author": {
     "name": "Harlan Wilton",