@beam-network/sdk 0.5.0 → 0.5.1

package/dist/client.js

@@ -1,10 +1,17 @@
 import { abortMultipartUpload, createMultipartUpload, expiresAtIso, prepareProviderDestination, prepareProviderSource, signAbortMultipartUpload, signCompleteMultipartUpload, signDeleteObject, signDestinationRoute, signFinalObjectHead, signListMultipartUpload, signListObjects, signUploadPartCopy } from "./provider-signing.js";
-import { BeamTransferControl, BEAM_DEFAULT_NATS_URL, compactSignedRoutes } from "./nats-control.js";
+import { BeamTransferControl, BEAM_DEFAULT_NATS_URL, compactSignedRoutes, isRecoverableRouteStreamError } from "./nats-control.js";
 import { multipartPartNumber } from "./multipart-limits.js";
 export { BEAM_DEFAULT_NATS_URL };
-// The error classes live in ./errors.js and are re-exported by ./index.js.
-// Re-exporting them here too would make `export *` in index.ts ambiguous, which
-// silently drops the names from the package's public types.
+export class BeamApiError extends Error {
+    status;
+    body;
+    constructor(message, status, body) {
+        super(message);
+        this.status = status;
+        this.body = body;
+        this.name = "BeamApiError";
+    }
+}
 export class BeamClient {
     apiKey;
     natsUrl;
@@ -12,10 +19,6 @@ export class BeamClient {
     control;
     routeSigningConcurrency;
     routeSigningConcurrencyOverridden;
-    /** Last credit state Beam reported. Null until the first request or connect. */
-    get creditStatus() {
-        return this.control.creditStatus;
-    }
     constructor(options = {}) {
         if (!options.apiKey?.trim()) {
             throw new Error("apiKey is required.");
@@ -32,8 +35,7 @@ export class BeamClient {
             subjectPrefix: options.transferClientSubjectPrefix,
             transferRuntimeShardCount: options.transferRuntimeShardCount,
             requestTimeoutMs: options.requestTimeoutMs,
-            maxPayloadBytes: options.maxPayloadBytes,
-            onCreditWarning: options.onCreditWarning
+            maxPayloadBytes: options.maxPayloadBytes
         });
         this.fetchImpl = options.fetch ?? globalThis.fetch;
         if (!this.fetchImpl) {
@@ -71,9 +73,12 @@ export class BeamClient {
     createTransfer(input) {
         return this.prepareProviderTransfer(input);
     }
-    transferStatus(transferId) {
+    async transferStatus(transferId) {
         validateId(transferId, "transferId");
-        return this.control.request("transfer.status", { transfer_id: transferId }, { transferId });
+        const status = await this.control.request("transfer.status", { transfer_id: transferId }, { transferId });
+        if (["completed", "failed", "cancelled"].includes(status.status))
+            this.control.releaseRecoveryLease(transferId);
+        return status;
     }
     distributeTransfer(transferId) {
         validateId(transferId, "transferId");
@@ -88,6 +93,7 @@ export class BeamClient {
             transferId,
             idempotencyKey: `transfer:${transferId}:cancel`
         });
+        this.control.releaseRecoveryLease(transferId);
         return result;
     }
     async planTransfer(input) {
@@ -105,8 +111,10 @@ export class BeamClient {
     }
     async prepareTransfer(input) {
         const transferId = await transferIdForIdempotencyKey(input.idempotencyKey);
+        const routeGenerationId = input.routeGenerationId ?? randomUuid();
         const result = await this.control.request("transfer.prepare", compact({
             transfer_id: transferId,
+            route_generation_id: routeGenerationId,
             sources: input.sources,
             destinations: input.destinations,
             name: input.name,
@@ -125,56 +133,97 @@ export class BeamClient {
         validateId(transferId, "transferId");
         validateMultipartGroupManifest(input.multipartGroupManifest, transferId);
         validateSignedRouteManifestContract(input.chunkRoutes, input.multipartGroupManifest);
-        const destinationCount = new Set(input.chunkRoutes.map((route) => route.destination_id)).size;
-        const orderedRoutes = input.chunkRoutes
-            .map((route) => {
-            if (signedRouteDeliveryIndex(route) !== undefined)
-                return route;
-            if (destinationCount !== 1) {
-                throw new Error("delivery_index is required when manually attaching routes for multiple destinations");
-            }
-            return { ...route, delivery_index: route.chunk_index };
-        })
-            .sort(compareSignedRoutesByDelivery);
-        const streamId = await stableRouteStreamId({
-            transferId,
-            planIdentity: await routeCoordinateChecksum(orderedRoutes),
-            totalRoutes: orderedRoutes.length,
-            totalChunks: countDistinctRouteChunks(orderedRoutes),
-            signedUrlFlow: input.multipartGroupManifest.length ? "signed_url_v2" : "signed_url_v1"
-        });
-        const stream = new RouteStreamSender(this.control, {
-            streamId,
+        const streamRoutes = async (chunkRoutes, multipartGroupManifest, routeGenerationId, urlsExpiresAt) => {
+            validateMultipartGroupManifest(multipartGroupManifest, transferId);
+            validateSignedRouteManifestContract(chunkRoutes, multipartGroupManifest);
+            const destinationCount = new Set(chunkRoutes.map((route) => route.destination_id)).size;
+            const orderedRoutes = chunkRoutes.map((route) => {
+                if (signedRouteDeliveryIndex(route) !== undefined)
+                    return route;
+                if (destinationCount !== 1) {
+                    throw new Error("delivery_index is required when manually attaching routes for multiple destinations");
+                }
+                return { ...route, delivery_index: route.chunk_index };
+            })
+                .sort(compareSignedRoutesByDelivery);
+            const streamId = await stableRouteStreamId({
+                transferId,
+                planIdentity: `${await routeCoordinateChecksum(orderedRoutes)}:${routeGenerationId}`,
+                totalRoutes: orderedRoutes.length,
+                totalChunks: countDistinctRouteChunks(orderedRoutes),
+                signedUrlFlow: multipartGroupManifest.length ? "signed_url_v2" : "signed_url_v1"
+            });
+            const stream = new RouteStreamSender(this.control, {
+                streamId,
+                transferId,
+                routeGenerationId,
+                totalRoutes: orderedRoutes.length,
+                totalChunks: countDistinctRouteChunks(orderedRoutes),
+                urlsExpiresAt,
+                signedUrlFlow: multipartGroupManifest.length ? "signed_url_v2" : "signed_url_v1",
+                autoDistribute: input.autoDistribute ?? true
+            });
+            await stream.begin();
+            if (multipartGroupManifest.length)
+                await stream.addManifestGroups(multipartGroupManifest);
+            for (const route of orderedRoutes)
+                await stream.addRoute(route);
+            return stream.complete();
+        };
+        const routeStreamLock = new AsyncMutex();
+        const releaseInitialStream = await routeStreamLock.acquire();
+        this.control.registerRecoveryLease({
             transferId,
-            totalRoutes: orderedRoutes.length,
-            totalChunks: countDistinctRouteChunks(orderedRoutes),
-            urlsExpiresAt: input.urlsExpiresAt,
-            signedUrlFlow: input.multipartGroupManifest.length ? "signed_url_v2" : "signed_url_v1",
-            autoDistribute: input.autoDistribute ?? true
+            planFingerprint: input.planFingerprint,
+            coordinateChecksum: input.coordinateChecksum,
+            replayRoutes: async (routeGenerationId) => {
+                await routeStreamLock.run(async () => {
+                    const fresh = await input.recoveryFactory(routeGenerationId);
+                    const attached = await streamRoutes(fresh.chunkRoutes, fresh.multipartGroupManifest, routeGenerationId, fresh.urlsExpiresAt);
+                    if (!attached.success)
+                        throw new Error(attached.error ?? attached.message ?? "route replay failed");
+                });
+            }
         });
-        await stream.begin();
-        if (input.multipartGroupManifest.length) {
-            await stream.addManifestGroups(input.multipartGroupManifest);
+        try {
+            const attached = await streamRoutes(input.chunkRoutes, input.multipartGroupManifest, input.routeGenerationId, input.urlsExpiresAt);
+            if (!attached.success)
+                this.control.releaseRecoveryLease(transferId);
+            return attached;
         }
-        for (const route of orderedRoutes) {
-            await stream.addRoute(route);
+        catch (error) {
+            if (isRecoverableRouteStreamError(error))
+                this.control.continueRecoveryLease(transferId);
+            else
+                this.control.releaseRecoveryLease(transferId);
+            throw error;
+        }
+        finally {
+            releaseInitialStream();
         }
-        return stream.complete();
     }
     async prepareProviderTransfer(input) {
         const expiresIn = input.expiresIn ?? 3600;
         const requestedSignedUrlFlow = input.signedUrlFlow ?? "signed_url_v1";
-        const preparedDestinations = input.destinations.map((destination, index) => prepareProviderDestination(destination, { index }));
+        const retainedRecoveryInput = {
+            ...input,
+            sources: input.sources.map((source) => ({ ...source })),
+            destinations: input.destinations.map((destination) => ({ ...destination })),
+            onBeforeTransferPrepare: undefined,
+            onPrepared: undefined,
+            throwIfCancelled: undefined
+        };
+        const preparedDestinations = retainedRecoveryInput.destinations.map((destination, index) => prepareProviderDestination(destination, { index }));
         const destinationsById = new Map();
         preparedDestinations.forEach((preparedDestination, index) => {
-            const destination = input.destinations[index];
+            const destination = retainedRecoveryInput.destinations[index];
             if (!destination) {
                 throw new Error(`missing provider destination config for prepared destination ${preparedDestination.destination_id}`);
             }
             destinationsById.set(preparedDestination.destination_id, destination);
         });
         await input.throwIfCancelled?.();
-        const preparedSources = await Promise.all(input.sources.map((source, index) => prepareProviderSource(source, { index, expiresIn, fetchImpl: this.fetchImpl })));
+        const preparedSources = await Promise.all(retainedRecoveryInput.sources.map((source, index) => prepareProviderSource(source, { index, expiresIn, fetchImpl: this.fetchImpl })));
         await input.throwIfCancelled?.();
         await input.onBeforeTransferPrepare?.();
         const prepared = await this.prepareTransfer({
@@ -183,145 +232,201 @@ export class BeamClient {
             name: input.name,
             testMode: input.testMode,
             signedUrlFlow: requestedSignedUrlFlow,
-            idempotencyKey: input.idempotencyKey
+            idempotencyKey: input.idempotencyKey,
+            routeGenerationId: input.routeGenerationId
         });
         if (!prepared.success) {
             return prepared;
         }
-        await input.onPrepared?.(prepared);
-        await input.throwIfCancelled?.(prepared.transfer_id);
         const sourcesById = new Map();
         preparedSources.forEach((preparedSource, index) => {
-            const source = input.sources[index];
+            const source = retainedRecoveryInput.sources[index];
             if (!source) {
                 throw new Error(`missing provider source config for prepared source ${preparedSource.source_id}`);
             }
             sourcesById.set(preparedSource.source_id, source);
         });
         const multipartUploads = new Map();
-        const pendingRoutes = new Set();
-        let signingConcurrency = this.routeSigningConcurrency;
-        let signedInWindow = 0;
-        let signingWindowStartedAt = performance.now();
-        let routeStream = null;
-        let routeStreamBeginAttempted = false;
-        let multipartManifestTask = null;
-        let multipartGroupWaiters = new Map();
-        try {
-            const streamId = await stableRouteStreamId({
-                transferId: prepared.transfer_id,
-                planIdentity: prepared.plan_descriptor.plan_nonce,
-                totalRoutes: prepared.plan_descriptor.delivery_route_count,
-                totalChunks: prepared.plan_descriptor.logical_chunk_count,
-                signedUrlFlow: requestedSignedUrlFlow
-            });
-            routeStream = new RouteStreamSender(this.control, {
-                streamId,
-                transferId: prepared.transfer_id,
-                totalRoutes: prepared.plan_descriptor.delivery_route_count,
-                totalChunks: prepared.plan_descriptor.logical_chunk_count,
-                signedUrlFlow: requestedSignedUrlFlow,
-                autoDistribute: input.distribute !== false
-            });
-            routeStreamBeginAttempted = true;
-            await routeStream.begin();
-            multipartGroupWaiters = createMultipartGroupWaiters(prepared, destinationsById);
-            multipartManifestTask = createMultipartGroupManifest({
-                prepared,
-                destinationsById,
-                multipartUploads,
-                expiresIn,
-                signedUrlFlow: requestedSignedUrlFlow,
-                concurrency: this.routeSigningConcurrency,
-                onGroupReady: async (state) => {
-                    if (requestedSignedUrlFlow === "signed_url_v2") {
-                        validateMultipartGroupManifest([state.manifest], prepared.transfer_id);
-                        await routeStream.addManifestGroups([state.manifest]);
+        const autoDistribute = input.distribute !== false;
+        let initialThrowIfCancelled = input.throwIfCancelled;
+        const routeStreamLock = new AsyncMutex();
+        const releaseInitialStream = await routeStreamLock.acquire();
+        const streamPreparedRoutes = async (routeGenerationId, recoveryReplay) => {
+            const pendingRoutes = new Set();
+            let signingConcurrency = this.routeSigningConcurrency;
+            let signedInWindow = 0;
+            let signingWindowStartedAt = performance.now();
+            let routeStream = null;
+            let routeStreamBeginAttempted = false;
+            let multipartManifestTask = null;
+            let multipartGroupWaiters = new Map();
+            let foregroundCancelled = false;
+            const throwIfCancelled = recoveryReplay || !initialThrowIfCancelled ? undefined : async (transferId) => {
+                try {
+                    await initialThrowIfCancelled?.(transferId);
+                }
+                catch (error) {
+                    foregroundCancelled = true;
+                    throw error;
+                }
+            };
+            try {
+                const streamId = await stableRouteStreamId({
+                    transferId: prepared.transfer_id,
+                    planIdentity: `${prepared.plan_descriptor.plan_nonce}:${routeGenerationId}`,
+                    totalRoutes: prepared.plan_descriptor.delivery_route_count,
+                    totalChunks: prepared.plan_descriptor.logical_chunk_count,
+                    signedUrlFlow: requestedSignedUrlFlow
+                });
+                routeStream = new RouteStreamSender(this.control, {
+                    streamId,
+                    transferId: prepared.transfer_id,
+                    routeGenerationId,
+                    totalRoutes: prepared.plan_descriptor.delivery_route_count,
+                    totalChunks: prepared.plan_descriptor.logical_chunk_count,
+                    signedUrlFlow: requestedSignedUrlFlow,
+                    autoDistribute,
+                    urlsExpiresAt: expiresAtIso(expiresIn)
+                });
+                routeStreamBeginAttempted = true;
+                await routeStream.begin();
+                multipartGroupWaiters = createMultipartGroupWaiters(prepared, destinationsById);
+                multipartManifestTask = createMultipartGroupManifest({
+                    prepared,
+                    destinationsById,
+                    multipartUploads,
+                    expiresIn,
+                    signedUrlFlow: requestedSignedUrlFlow,
+                    concurrency: this.routeSigningConcurrency,
+                    preserveUploadsOnFailure: recoveryReplay,
+                    onGroupReady: async (state) => {
+                        if (requestedSignedUrlFlow === "signed_url_v2") {
+                            validateMultipartGroupManifest([state.manifest], prepared.transfer_id);
+                            await routeStream.addManifestGroups([state.manifest]);
+                        }
+                        multipartGroupWaiters.get(state.manifest.multipart_group_id)?.resolve(state);
+                    },
+                    onGroupFailed: (groupId, error) => multipartGroupWaiters.get(groupId)?.reject(error)
+                });
+                const streamNextCompletedRoute = async () => {
+                    const settled = await Promise.race([...pendingRoutes].map((pending) => pending.then((route) => ({ pending, route }))));
+                    pendingRoutes.delete(settled.pending);
+                    await routeStream.addRoute(settled.route);
+                    signedInWindow += 1;
+                    if (signedInWindow === ROUTE_STREAM_BATCH_ROUTES) {
+                        const elapsedMs = performance.now() - signingWindowStartedAt;
+                        if (!this.routeSigningConcurrencyOverridden && elapsedMs > 4_000 && signingConcurrency < 256) {
+                            signingConcurrency = Math.min(256, signingConcurrency * 2);
+                        }
+                        signedInWindow = 0;
+                        signingWindowStartedAt = performance.now();
                     }
-                    multipartGroupWaiters.get(state.manifest.multipart_group_id)?.resolve(state);
-                },
-                onGroupFailed: (groupId, error) => multipartGroupWaiters.get(groupId)?.reject(error)
-            });
-            const streamNextCompletedRoute = async () => {
-                const settled = await Promise.race([...pendingRoutes].map((pending) => pending.then((route) => ({ pending, route }))));
-                pendingRoutes.delete(settled.pending);
-                await routeStream.addRoute(settled.route);
-                signedInWindow += 1;
-                if (signedInWindow === ROUTE_STREAM_BATCH_ROUTES) {
-                    const elapsedMs = performance.now() - signingWindowStartedAt;
-                    if (!this.routeSigningConcurrencyOverridden && elapsedMs > 4_000 && signingConcurrency < 256) {
-                        signingConcurrency = Math.min(256, signingConcurrency * 2);
+                };
+                for (const chunk of materializePlanChunks(prepared.plan_descriptor, prepared.transfer_id)) {
+                    await throwIfCancelled?.(prepared.transfer_id);
+                    for (const target of chunk.destinations) {
+                        const pendingRoute = (async () => {
+                            await throwIfCancelled?.(prepared.transfer_id);
+                            const destination = destinationsById.get(target.destination_id);
+                            if (!destination)
+                                throw new Error(`BeamCore returned unknown destination_id: ${target.destination_id}`);
+                            const source = sourcesById.get(chunk.source_id);
+                            if (!source)
+                                throw new Error(`BeamCore returned unknown source_id: ${chunk.source_id}`);
+                            const metadata = target.metadata ?? {};
+                            const finalObjectKey = typeof metadata.final_object_key === "string" ? metadata.final_object_key : target.object_key;
+                            if (!finalObjectKey)
+                                throw new Error("destination signing target is missing object_key");
+                            const multipartGroupId = multipartGroupStateKey(prepared.transfer_id, target.destination_id, chunk.source_id, finalObjectKey);
+                            const upload = isHippiusDestination(destination)
+                                ? undefined
+                                : await multipartGroupWaiters.get(multipartGroupId)?.promise;
+                            if (!isHippiusDestination(destination) && !upload)
+                                throw new Error(`multipart group manifest is missing for ${chunk.source_id}:${target.destination_id}`);
+                            return this.signProviderRoute({
+                                chunk,
+                                target,
+                                source,
+                                destination,
+                                expiresIn,
+                                ...(upload ? { upload } : {}),
+                                transferId: prepared.transfer_id,
+                                finalObjectKey,
+                                signedUrlFlow: requestedSignedUrlFlow,
+                                partNumber: typeof metadata.part_number === "number"
+                                    ? metadata.part_number
+                                    : multipartPartNumber(chunk.source_chunk_index),
+                            });
+                        })();
+                        pendingRoutes.add(pendingRoute);
+                        if (pendingRoutes.size >= signingConcurrency)
+                            await streamNextCompletedRoute();
                     }
-                    signedInWindow = 0;
-                    signingWindowStartedAt = performance.now();
                 }
-            };
-            for (const chunk of materializePlanChunks(prepared.plan_descriptor, prepared.transfer_id)) {
-                await input.throwIfCancelled?.(prepared.transfer_id);
-                for (const target of chunk.destinations) {
-                    const pendingRoute = (async () => {
-                        await input.throwIfCancelled?.(prepared.transfer_id);
-                        const destination = destinationsById.get(target.destination_id);
-                        if (!destination)
-                            throw new Error(`BeamCore returned unknown destination_id: ${target.destination_id}`);
-                        const source = sourcesById.get(chunk.source_id);
-                        if (!source)
-                            throw new Error(`BeamCore returned unknown source_id: ${chunk.source_id}`);
-                        const metadata = target.metadata ?? {};
-                        const finalObjectKey = typeof metadata.final_object_key === "string" ? metadata.final_object_key : target.object_key;
-                        if (!finalObjectKey)
-                            throw new Error("destination signing target is missing object_key");
-                        const multipartGroupId = multipartGroupStateKey(prepared.transfer_id, target.destination_id, chunk.source_id, finalObjectKey);
-                        const upload = isHippiusDestination(destination)
-                            ? undefined
-                            : await multipartGroupWaiters.get(multipartGroupId)?.promise;
-                        if (!isHippiusDestination(destination) && !upload)
-                            throw new Error(`multipart group manifest is missing for ${chunk.source_id}:${target.destination_id}`);
-                        return this.signProviderRoute({
-                            chunk,
-                            target,
-                            source,
-                            destination,
-                            expiresIn,
-                            ...(upload ? { upload } : {}),
+                while (pendingRoutes.size)
+                    await streamNextCompletedRoute();
+                const multipartGroupManifest = await multipartManifestTask;
+                if (requestedSignedUrlFlow === "signed_url_v2") {
+                    validateMultipartGroupManifest(multipartGroupManifest, prepared.transfer_id);
+                }
+                const attached = await routeStream.complete();
+                if (!attached.success) {
+                    throw new Error(attached.error ?? attached.message ?? "route stream failed");
+                }
+            }
+            catch (error) {
+                if (routeStream)
+                    await routeStream.abort().catch(() => undefined);
+                await Promise.allSettled(pendingRoutes);
+                if (multipartManifestTask)
+                    await Promise.allSettled([multipartManifestTask]);
+                if (!recoveryReplay) {
+                    if (foregroundCancelled || isRecoverableRouteStreamError(error)) {
+                        this.control.continueRecoveryLease(prepared.transfer_id);
+                    }
+                    else {
+                        this.control.releaseRecoveryLease(prepared.transfer_id);
+                        await cancelAndAbortProviderFailure({
+                            client: this,
                             transferId: prepared.transfer_id,
-                            finalObjectKey,
-                            signedUrlFlow: requestedSignedUrlFlow,
-                            partNumber: typeof metadata.part_number === "number"
-                                ? metadata.part_number
-                                : multipartPartNumber(chunk.source_chunk_index),
+                            cause: error,
+                            uploads: multipartUploads,
+                            abortBeforeCancel: !routeStreamBeginAttempted
                         });
-                    })();
-                    pendingRoutes.add(pendingRoute);
-                    if (pendingRoutes.size >= signingConcurrency)
-                        await streamNextCompletedRoute();
+                    }
                 }
+                throw error;
+            }
+        };
+        this.control.registerRecoveryLease({
+            transferId: prepared.transfer_id,
+            planFingerprint: prepared.plan_fingerprint,
+            coordinateChecksum: prepared.coordinate_checksum,
+            replayRoutes: async (routeGenerationId) => {
+                await routeStreamLock.run(() => streamPreparedRoutes(routeGenerationId, true));
+            },
+            disposeSecrets: () => clearRecoverySecrets(retainedRecoveryInput)
+        });
+        try {
+            try {
+                await input.onPrepared?.(prepared);
+            }
+            catch (error) {
+                this.control.continueRecoveryLease(prepared.transfer_id);
+                throw error;
             }
-            while (pendingRoutes.size)
-                await streamNextCompletedRoute();
-            const multipartGroupManifest = await multipartManifestTask;
-            if (requestedSignedUrlFlow === "signed_url_v2") {
-                validateMultipartGroupManifest(multipartGroupManifest, prepared.transfer_id);
+            try {
+                await input.throwIfCancelled?.(prepared.transfer_id);
             }
-            const attached = await routeStream.complete();
-            if (!attached.success) {
-                throw new Error(attached.error ?? attached.message ?? "route stream failed");
+            catch (error) {
+                this.control.continueRecoveryLease(prepared.transfer_id);
+                throw error;
             }
+            await streamPreparedRoutes(prepared.route_generation_id, false);
+            initialThrowIfCancelled = undefined;
         }
-        catch (error) {
-            if (routeStream)
-                await routeStream.abort();
-            await Promise.allSettled(pendingRoutes);
-            if (multipartManifestTask)
-                await Promise.allSettled([multipartManifestTask]);
-            await cancelAndAbortProviderFailure({
-                client: this,
-                transferId: prepared.transfer_id,
-                cause: error,
-                uploads: multipartUploads,
-                abortBeforeCancel: !routeStreamBeginAttempted
-            });
-            throw error;
+        finally {
+            releaseInitialStream();
         }
         return prepared;
     }
@@ -486,6 +591,27 @@ export class BeamClient {
     }
 }
 const ROUTE_STREAM_BATCH_ROUTES = 2_048;
+class AsyncMutex {
+    tail = Promise.resolve();
+    async acquire() {
+        const previous = this.tail;
+        let release;
+        this.tail = new Promise((resolve) => {
+            release = resolve;
+        });
+        await previous;
+        return release;
+    }
+    async run(operation) {
+        const release = await this.acquire();
+        try {
+            return await operation();
+        }
+        finally {
+            release();
+        }
+    }
+}
 class RouteStreamSender {
     control;
     options;
@@ -505,6 +631,7 @@ class RouteStreamSender {
     async begin() {
         await this.control.request("transfer.route_stream.begin", compact({
             transfer_id: this.options.transferId,
+            route_generation_id: this.options.routeGenerationId,
             stream_id: this.streamId,
             total_routes: this.options.totalRoutes,
             total_chunks: this.options.totalChunks,
@@ -524,6 +651,7 @@ class RouteStreamSender {
             const identity = await stableManifestBatchIdentity([group]);
             await this.control.request("transfer.route_stream.manifest", {
                 transfer_id: this.options.transferId,
+                route_generation_id: this.options.routeGenerationId,
                 stream_id: this.streamId,
                 manifest_batch_id: identity,
                 groups: [group]
@@ -562,6 +690,7 @@ class RouteStreamSender {
             throw this.sendError;
         return this.control.request("transfer.route_stream.complete", {
             transfer_id: this.options.transferId,
+            route_generation_id: this.options.routeGenerationId,
             stream_id: this.streamId,
             expected_batches: this.batchIndex,
             expected_routes: this.options.totalRoutes,
@@ -580,6 +709,7 @@ class RouteStreamSender {
         const routes = this.batch.splice(0, this.batch.length);
         const chunks = this.control.splitRoutesForPayload("transfer.route_stream.batch", {
             transfer_id: this.options.transferId,
+            route_generation_id: this.options.routeGenerationId,
             stream_id: this.streamId,
             batch_id: `${this.streamId}:estimate`,
             batch_index: this.batchIndex
@@ -599,6 +729,7 @@ class RouteStreamSender {
                     const batchId = `${this.streamId}:${batchIndex}:${coordinateChecksum}`;
                     await this.control.request("transfer.route_stream.batch", {
                         transfer_id: this.options.transferId,
+                        route_generation_id: this.options.routeGenerationId,
                         stream_id: this.streamId,
                         batch_id: batchId,
                         batch_index: batchIndex,
@@ -878,11 +1009,13 @@ async function createMultipartGroupManifest(input) {
                 "beam-transfer-id": input.prepared.transfer_id,
                 ...(input.signedUrlFlow === "signed_url_v2" ? { "beam-multipart-group-id": multipartGroupId } : {})
             };
-            const uploadId = await createMultipartUpload({
+            const retainedUpload = input.multipartUploads.get(multipartGroupId);
+            const uploadId = retainedUpload?.uploadId ?? await createMultipartUpload({
                 destination: destinationConfig,
                 objectKey: finalObjectKey,
                 metadata: finalObjectMetadata
             });
+            const createdUpload = !retainedUpload;
             try {
                 const urlsExpiresAt = expiresAtIso(input.expiresIn);
                 const stagingObjectPrefix = buildStagingObjectPrefix({
@@ -940,11 +1073,13 @@ async function createMultipartGroupManifest(input) {
                 return { ok: true, state };
             }
             catch (error) {
-                try {
-                    await abortMultipartUpload(destinationConfig, finalObjectKey, uploadId);
-                }
-                catch (abortError) {
-                    throw new AggregateError([error, abortError], `multipart group setup and cleanup failed for ${multipartGroupId}`);
+                if (createdUpload) {
+                    try {
+                        await abortMultipartUpload(destinationConfig, finalObjectKey, uploadId);
+                    }
+                    catch (abortError) {
+                        throw new AggregateError([error, abortError], `multipart group setup and cleanup failed for ${multipartGroupId}`);
+                    }
                 }
                 throw error;
             }
@@ -957,6 +1092,8 @@ async function createMultipartGroupManifest(input) {
     });
     const failure = results.find((result) => !result.ok);
     if (failure && !failure.ok) {
+        if (input.preserveUploadsOnFailure)
+            throw failure.error;
         try {
             await abortCreatedUploads(input.multipartUploads);
         }
@@ -1235,3 +1372,13 @@ function isHippiusDestination(destination) {
 function sleep(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
 }
+function clearRecoverySecrets(input) {
+    const secretKeys = new Set(["access_key_id", "secret_access_key", "session_token", "api_token", "token", "password"]);
+    for (const provider of [...input.sources, ...input.destinations]) {
+        const record = provider;
+        for (const key of secretKeys) {
+            if (key in record)
+                record[key] = "";
+        }
+    }
+}