@beam-network/sdk 0.6.0 → 0.7.0

package/dist/client.js

@@ -1,4 +1,5 @@
-import { signMultipartRecovery, abortMultipartUpload, createMultipartUpload, expiresAtIso, prepareProviderDestination, prepareProviderSource, signAbortMultipartUpload, signCompleteMultipartUpload, signDestinationReadRange, signDestinationRoute, signFinalObjectHead, signListMultipartUpload, signSourceReadRange, isHuggingFaceProvider } from "./provider-signing.js";
+import { SdkPerformanceCollector, SourceSignatureHistory, currentSdkPerformance, measureSdkPhase } from "./performance.js";
+import { signMultipartRecovery, abortMultipartUpload, createMultipartUpload, expiresAtIso, prepareProviderDestination, prepareProviderSource, releaseProviderClients, signAbortMultipartUpload, signCompleteMultipartUpload, signDestinationReadRange, signDestinationRoute, signFinalObjectHead, signListMultipartUpload, signSourceReadRange, signSourceChunk, boundedGrantExpiry, isHuggingFaceProvider } from "./provider-signing.js";
 import { describe as describeHuggingFace, hashSourceStream, huggingFaceCommit, huggingFaceCompleteLfsUpload, huggingFaceLfsBatch, huggingFacePreupload, huggingFaceVerifyLfsUpload, readSourceSample } from "./huggingface.js";
 import { BeamTransferControl, BEAM_DEFAULT_NATS_URL, compactSignedRoutes, isRecoverableRouteStreamError } from "./nats-control.js";
 export class BeamRouteRecoveryPendingError extends Error {
@@ -53,11 +54,15 @@ export class BeamClient {
     routeSigningConcurrency;
     routeSigningConcurrencyOverridden;
     multipartControlConcurrency;
+    onDiagnostics;
+    diagnosticPending = false;
     recoverySigners = new Map();
     integrityAuditSigners = new Map();
+    integrityGrantCache = new Map();
     integrityAuditSubmissions = new Map();
     huggingFaceUploads = new Map();
     constructor(options = {}) {
+        this.onDiagnostics = options.onDiagnostics;
         if (!options.apiKey?.trim()) {
             throw new Error("apiKey is required.");
         }
@@ -237,7 +242,7 @@ export class BeamClient {
         };
         const routeStreamLock = new AsyncMutex();
         const releaseInitialStream = await routeStreamLock.acquire();
-        this.control.registerRecoveryLease({
+        const recoveryLease = {
             transferId,
             planFingerprint: input.planFingerprint,
             coordinateChecksum: input.coordinateChecksum,
@@ -249,19 +254,20 @@ export class BeamClient {
                         throw new Error(attached.error ?? attached.message ?? "route replay failed");
                 });
             }
-        });
+        };
+        this.control.registerRecoveryLease(recoveryLease);
         try {
             const attached = await streamRoutes(input.chunkRoutes, input.multipartGroupManifest, input.routeGenerationId, input.urlsExpiresAt);
             if (!attached.success)
-                this.control.releaseRecoveryLease(transferId);
+                this.control.releaseRecoveryLease(transferId, recoveryLease);
             return attached;
         }
         catch (error) {
             if (isRecoverableRouteStreamError(error)) {
-                this.control.continueRecoveryLease(transferId);
+                this.control.continueRecoveryLease(transferId, recoveryLease);
                 throw new BeamRouteRecoveryPendingError(transferId, error);
             }
-            this.control.releaseRecoveryLease(transferId);
+            this.control.releaseRecoveryLease(transferId, recoveryLease);
             throw error;
         }
         finally {
@@ -294,7 +300,8 @@ export class BeamClient {
             destinationsById.set(preparedDestination.destination_id, destination);
         });
         await input.throwIfCancelled?.();
-        const preparedSources = await Promise.all(retainedRecoveryInput.sources.map((source, index) => prepareProviderSource(source, { index, expiresIn, fetchImpl: this.fetchImpl, signal: input.signal })));
+        const discoveryTelemetry = new SdkPerformanceCollector();
+        const preparedSources = await discoveryTelemetry.measure("sdk.discovery", () => Promise.all(retainedRecoveryInput.sources.map((source, index) => prepareProviderSource(source, { index, expiresIn, fetchImpl: this.fetchImpl, signal: input.signal }))));
         await input.throwIfCancelled?.();
         const huggingFace = await this.planHuggingFaceUploads({
             sources: retainedRecoveryInput.sources,
@@ -345,10 +352,18 @@ export class BeamClient {
         const assertOwnership = () => input.signal?.throwIfAborted();
         const routeStreamLock = new AsyncMutex();
         const releaseInitialStream = await routeStreamLock.acquire();
+        const sourceSignatureHistory = new SourceSignatureHistory(!resume);
         const streamPreparedRoutes = async (routeGenerationId, recoveryReplay) => {
+            const telemetry = new SdkPerformanceCollector(sourceSignatureHistory);
+            telemetry.seedDiscovery(discoveryTelemetry);
+            telemetry.gauge("signing_configured_limit", this.routeSigningConcurrency);
+            telemetry.gauge("multipart_configured_limit", this.multipartControlConcurrency);
+            telemetry.gauge("signing_limit", this.routeSigningConcurrency);
+            telemetry.gauge("multipart_limit", this.multipartControlConcurrency);
             const pendingRoutes = new Set();
             let signingConcurrency = this.routeSigningConcurrency;
             let signedInWindow = 0;
+            let activeSigning = 0;
             let signingWindowStartedAt = performance.now();
             let routeStream = null;
             let routeStreamBeginAttempted = false;
@@ -376,6 +391,15 @@ export class BeamClient {
                     signedUrlFlow: requestedSignedUrlFlow
                 });
                 routeStream = new RouteStreamSender(this.control, {
+                    telemetry,
+                    onDiagnostics: (summary) => {
+                        if (!this.onDiagnostics || this.diagnosticPending)
+                            return;
+                        this.diagnosticPending = true;
+                        setTimeout(() => {
+                            void Promise.resolve().then(() => this.onDiagnostics?.(summary)).catch(() => { }).finally(() => { this.diagnosticPending = false; });
+                        }, 0);
+                    },
                     streamId,
                     transferId: prepared.transfer_id,
                     routeGenerationId,
@@ -388,7 +412,7 @@ export class BeamClient {
                 routeStreamBeginAttempted = true;
                 await routeStream.begin();
                 multipartGroupWaiters = createMultipartGroupWaiters(prepared, destinationsById);
-                multipartManifestTask = createMultipartGroupManifest({
+                multipartManifestTask = telemetry.measure("sdk.multipart_create", () => createMultipartGroupManifest({
                     prepared,
                     destinationsById,
                     multipartUploads,
@@ -400,14 +424,14 @@ export class BeamClient {
                         assertOwnership();
                         recoveryMultipartUploads.set(state.manifest.multipart_group_id, state);
                         validateMultipartGroupManifest([state.manifest], prepared.transfer_id);
-                        await input.onMultipartGroupReady?.(multipartGroupIdentity(prepared.transfer_id, state.manifest));
+                        await telemetry.measure("sdk.multipart_callback", async () => input.onMultipartGroupReady?.(multipartGroupIdentity(prepared.transfer_id, state.manifest)));
                         await routeStream.addManifestGroups([state.manifest]);
                         multipartGroupWaiters.get(state.manifest.multipart_group_id)?.resolve(state);
                     },
                     onGroupFailed: (groupId, error) => multipartGroupWaiters.get(groupId)?.reject(error)
-                });
+                }));
                 const streamNextCompletedRoute = async () => {
-                    const settled = await Promise.race([...pendingRoutes].map((pending) => pending.then((route) => ({ pending, route }))));
+                    const settled = await telemetry.measure("sdk.producer_wait", () => Promise.race([...pendingRoutes].map((pending) => pending.then((route) => ({ pending, route })))));
                     pendingRoutes.delete(settled.pending);
                     assertOwnership();
                     await routeStream.addRoute(settled.route);
@@ -416,6 +440,8 @@ export class BeamClient {
                         const elapsedMs = performance.now() - signingWindowStartedAt;
                         if (!this.routeSigningConcurrencyOverridden && elapsedMs > 4_000 && signingConcurrency < 256) {
                             signingConcurrency = Math.min(256, signingConcurrency * 2);
+                            telemetry.increment("concurrency_changes");
+                            telemetry.gauge("signing_limit", signingConcurrency);
                         }
                         signedInWindow = 0;
                         signingWindowStartedAt = performance.now();
@@ -423,6 +449,11 @@ export class BeamClient {
                 };
                 for (const chunk of materializePlanChunks(prepared.plan_descriptor, prepared.transfer_id)) {
                     await throwIfCancelled?.(prepared.transfer_id);
+                    // This promise belongs to this chunk and signing generation only. Its
+                    // immutable credential/range identity and actual expiry cannot be renewed.
+                    let sourceUses = 0;
+                    const sourceGrant = telemetry.measure("sdk.source_signing", () => signSourceChunk({ source: sourcesById.get(chunk.source_id), fallbackUrl: chunk.source_url, chunk, expiresIn, fetchImpl: this.fetchImpl })).then(grant => { telemetry.sourceCreated(chunk.chunk_index); return grant; });
+                    void sourceGrant.catch(() => { });
                     for (const target of chunk.destinations) {
                         const pendingRoute = (async () => {
                             await throwIfCancelled?.(prepared.transfer_id);
@@ -439,13 +470,16 @@ export class BeamClient {
                             const multipartGroupId = multipartGroupStateKey(prepared.transfer_id, target.destination_id, chunk.source_id, finalObjectKey);
                             const upload = isDirectPutDestination(destination)
                                 ? undefined
-                                : await multipartGroupWaiters.get(multipartGroupId)?.promise;
+                                : await telemetry.measure("sdk.multipart_ready_wait", async () => multipartGroupWaiters.get(multipartGroupId)?.promise);
                             if (!isDirectPutDestination(destination) && !upload)
                                 throw new Error(`multipart group manifest is missing for ${chunk.source_id}:${target.destination_id}`);
-                            return this.signProviderRoute({
+                            telemetry.gauge("signing_active_peak", ++activeSigning);
+                            return telemetry.measure("sdk.signing", () => this.signProviderRoute({
                                 chunk,
                                 target,
                                 source,
+                                sourceGrant: telemetry.measure("sdk.source_grant_wait", () => sourceGrant).then(grant => { if (sourceUses++ > 0)
+                                    telemetry.counters.source_reuses++; return grant; }),
                                 destination,
                                 expiresIn,
                                 ...(upload ? { upload } : {}),
@@ -456,11 +490,12 @@ export class BeamClient {
                                 partNumber: typeof metadata.part_number === "number"
                                     ? metadata.part_number
                                     : multipartPartNumber(chunk.source_chunk_index),
-                            });
+                            })).finally(() => { activeSigning--; });
                         })();
                         pendingRoutes.add(pendingRoute);
+                        telemetry.gauge("signing_pending_peak", pendingRoutes.size);
                         if (pendingRoutes.size >= signingConcurrency)
-                            await streamNextCompletedRoute();
+                            await telemetry.measure("sdk.signing_queue", streamNextCompletedRoute);
                     }
                 }
                 while (pendingRoutes.size)
@@ -475,8 +510,8 @@ export class BeamClient {
             }
             catch (error) {
                 if (input.signal?.aborted) {
-                    this.stopRecoverySigner(prepared.transfer_id);
-                    this.control.releaseRecoveryLease(prepared.transfer_id);
+                    this.stopRecoverySigner(prepared.transfer_id, recoveryLease);
+                    this.control.releaseRecoveryLease(prepared.transfer_id, recoveryLease);
                     await Promise.allSettled(pendingRoutes);
                     if (multipartManifestTask)
                         await Promise.allSettled([multipartManifestTask]);
@@ -489,10 +524,10 @@ export class BeamClient {
                     await Promise.allSettled([multipartManifestTask]);
                 if (!recoveryReplay) {
                     if (foregroundCancelled) {
-                        this.control.continueRecoveryLease(prepared.transfer_id);
+                        this.control.continueRecoveryLease(prepared.transfer_id, recoveryLease);
                     }
                     else if (isRecoverableRouteStreamError(error)) {
-                        this.control.continueRecoveryLease(prepared.transfer_id);
+                        this.control.continueRecoveryLease(prepared.transfer_id, recoveryLease);
                         throw new BeamRouteRecoveryPendingError(prepared.transfer_id, error);
                     }
                     else {
@@ -510,27 +545,16 @@ export class BeamClient {
                             // The retained recovery input owns the same destination objects used by
                             // multipart cleanup. Releasing it earlier scrubs their scoped credentials
                             // before abortMultipartUpload can clean up already-created uploads.
-                            this.control.releaseRecoveryLease(prepared.transfer_id);
+                            this.control.releaseRecoveryLease(prepared.transfer_id, recoveryLease);
                         }
                     }
                 }
                 throw error;
             }
         };
-        await this.startProviderRouteRecoverySigner({
-            prepared,
-            sourcesById,
-            destinationsById,
-            multipartUploads: recoveryMultipartUploads,
-            signal: input.signal,
-            expiresIn
-        });
-        const stopOwnedRecovery = () => {
-            this.stopRecoverySigner(prepared.transfer_id);
-            this.control.releaseRecoveryLease(prepared.transfer_id);
-        };
-        input.signal?.addEventListener("abort", stopOwnedRecovery, { once: true });
-        this.control.registerRecoveryLease({
+        // The lease object is this owner's identity: releases, recovery requests, and
+        // signer shutdowns below only act while it is still the registered lease.
+        const recoveryLease = {
             transferId: prepared.transfer_id,
             planFingerprint: prepared.plan_fingerprint,
             coordinateChecksum: prepared.coordinate_checksum,
@@ -539,23 +563,39 @@ export class BeamClient {
             },
             disposeSecrets: () => {
                 input.signal?.removeEventListener("abort", stopOwnedRecovery);
+                releaseProviderClients([...retainedRecoveryInput.sources, ...retainedRecoveryInput.destinations]);
                 clearRecoverySecrets(retainedRecoveryInput);
             }
+        };
+        await this.startProviderRouteRecoverySigner({
+            owner: recoveryLease,
+            prepared,
+            sourcesById,
+            destinationsById,
+            multipartUploads: recoveryMultipartUploads,
+            signal: input.signal,
+            expiresIn
         });
+        const stopOwnedRecovery = () => {
+            this.stopRecoverySigner(prepared.transfer_id, recoveryLease);
+            this.control.releaseRecoveryLease(prepared.transfer_id, recoveryLease);
+        };
+        input.signal?.addEventListener("abort", stopOwnedRecovery, { once: true });
+        this.control.registerRecoveryLease(recoveryLease);
         try {
             try {
                 assertOwnership();
                 await input.onPrepared?.(prepared);
             }
             catch (error) {
-                this.control.continueRecoveryLease(prepared.transfer_id);
+                this.control.continueRecoveryLease(prepared.transfer_id, recoveryLease);
                 throw error;
             }
             try {
                 await input.throwIfCancelled?.(prepared.transfer_id);
             }
             catch (error) {
-                this.control.continueRecoveryLease(prepared.transfer_id);
+                this.control.continueRecoveryLease(prepared.transfer_id, recoveryLease);
                 throw error;
             }
             await streamPreparedRoutes(prepared.route_generation_id, false);
@@ -740,6 +780,7 @@ export class BeamClient {
                     metadata: directPutRouteMetadata(input.target.metadata)
                 },
                 source: input.source,
+                sourceGrant: input.sourceGrant,
                 destination: input.destination,
                 expiresIn: input.expiresIn,
                 destUrl,
@@ -751,6 +792,7 @@ export class BeamClient {
                 chunk: input.chunk,
                 target: { ...input.target, metadata: partRouteMetadata(input.target.metadata) },
                 source: input.source,
+                sourceGrant: input.sourceGrant,
                 destination: input.destination,
                 expiresIn: input.expiresIn,
                 fetchImpl: this.fetchImpl
@@ -768,6 +810,7 @@ export class BeamClient {
                 metadata: partRouteMetadata(input.target.metadata)
             },
             source: input.source,
+            sourceGrant: input.sourceGrant,
             destination: input.destination,
             expiresIn: input.expiresIn,
             uploadId: upload.uploadId,
@@ -791,6 +834,7 @@ export class BeamClient {
             input.signal?.throwIfAborted();
             if (payload.route_generation_id.length === 0)
                 throw new Error("route recovery generation is required");
+            const sourceGrants = new Map();
             const routes = await mapOrderedWithConcurrency(payload.chunks, this.routeSigningConcurrency, async (requested) => {
                 input.signal?.throwIfAborted();
                 if (requested.route_generation_id !== payload.route_generation_id) {
@@ -821,11 +865,22 @@ export class BeamClient {
                         multipartUploads: input.multipartUploads,
                         expiresIn: input.expiresIn
                     });
+                const sourceKey = JSON.stringify([chunk.source_id, chunk.chunk_index]);
+                let sourceGrant = sourceGrants.get(sourceKey);
+                if (!sourceGrant) {
+                    sourceGrant = signSourceChunk({ source, chunk, fallbackUrl: chunk.source_url, expiresIn: input.expiresIn, fetchImpl: this.fetchImpl });
+                    sourceGrants.set(sourceKey, sourceGrant);
+                    void sourceGrant.catch(() => { });
+                }
                 const route = await this.signProviderRoute({
+                    sourceGrant,
                     chunk,
                     target: {
                         ...target,
-                        object_key: requested.final_object_key,
+                        // Direct-PUT destinations (Hippius, Hugging Face) keep the planned
+                        // per-chunk key exactly as initial materialization does; only
+                        // multipart destinations sign parts against the final object key.
+                        object_key: isDirectPutDestination(destination) ? target.object_key : requested.final_object_key,
                         metadata: {
                             ...(target.metadata ?? {}),
                             ...(requested.destination_metadata ?? {}),
@@ -861,16 +916,33 @@ export class BeamClient {
                 chunk_routes: routes
             };
         });
-        this.recoverySigners.set(input.prepared.transfer_id, stop);
-        this.integrityAuditSigners.set(input.prepared.transfer_id, async (challenge) => {
-            await this.submitProviderIntegrityAuditGrants({
-                challenge,
-                prepared: input.prepared,
-                sourcesById: input.sourcesById,
-                destinationsById: input.destinationsById,
-                expiresIn: input.expiresIn
+        if (input.signal?.aborted) {
+            stop();
+            return;
+        }
+        this.stopRecoverySigner(input.prepared.transfer_id);
+        this.integrityAuditSigners.set(input.prepared.transfer_id, (challenge) => this.buildProviderIntegrityAuditGrants({
+            challenge, prepared: input.prepared, sourcesById: input.sourcesById, destinationsById: input.destinationsById, expiresIn: input.expiresIn
+        }));
+        const registration = { owner: input.owner, stop };
+        this.recoverySigners.set(input.prepared.transfer_id, registration);
+        try {
+            const stopIntegrity = await this.control.serveIntegritySigner(input.prepared.transfer_id, challenge => {
+                input.signal?.throwIfAborted();
+                return this.signedIntegrityGrants(challenge);
             });
-        });
+            if (input.signal?.aborted || this.recoverySigners.get(input.prepared.transfer_id) !== registration) {
+                stop();
+                stopIntegrity();
+                return;
+            }
+            registration.stop = () => { stop(); stopIntegrity(); };
+        }
+        catch {
+            // Status-driven signing remains available if the optional subscription fails.
+            if (input.signal?.aborted)
+                this.stopRecoverySigner(input.prepared.transfer_id, input.owner);
+        }
     }
     async recoveryMultipartUploadState(input) {
         const existing = input.multipartUploads.get(input.requested.multipart_group_id);
@@ -937,21 +1009,56 @@ export class BeamClient {
         if (!signer)
             throw new Error("integrity audit signer unavailable");
         const existing = this.integrityAuditSubmissions.get(challenge.audit_id);
-        if (existing) {
+        const cached = this.integrityGrantCache.get(challenge.audit_id);
+        if (cached && cached.fingerprint !== JSON.stringify(challenge))
+            throw new Error("integrity audit identity changed");
+        if (existing && cached && cached.expiresAt > Date.now() + 30_000) {
             await existing;
             return;
         }
-        const submission = signer(challenge);
+        const submission = this.signedIntegrityGrants(challenge).then(async (payload) => {
+            const result = await this.control.request("transfer.integrity_audit_grants", payload, {
+                transferId: challenge.transfer_id, idempotencyKey: `transfer:${challenge.transfer_id}:integrity-audit:${challenge.audit_id}:${payload.submitted_at}`
+            });
+            if (result.published !== true)
+                throw new Error("integrity audit delivery unavailable");
+        });
         this.integrityAuditSubmissions.set(challenge.audit_id, submission);
         try {
             await submission;
         }
         catch (error) {
-            this.integrityAuditSubmissions.delete(challenge.audit_id);
+            if (this.integrityAuditSubmissions.get(challenge.audit_id) === submission)
+                this.integrityAuditSubmissions.delete(challenge.audit_id);
             throw error;
         }
     }
-    async submitProviderIntegrityAuditGrants(input) {
+    signedIntegrityGrants(challenge) {
+        const fingerprint = JSON.stringify(challenge);
+        const cached = this.integrityGrantCache.get(challenge.audit_id);
+        if (cached && cached.fingerprint !== fingerprint)
+            return Promise.reject(new Error("integrity audit identity changed"));
+        if (cached && cached.expiresAt > Date.now() + 30_000)
+            return cached.payload;
+        if (!cached && this.integrityGrantCache.size >= 1024)
+            return Promise.reject(new Error("integrity signer capacity unavailable"));
+        const signer = this.integrityAuditSigners.get(challenge.transfer_id);
+        if (!signer)
+            return Promise.reject(new Error("integrity audit signer unavailable"));
+        this.integrityAuditSubmissions.delete(challenge.audit_id);
+        const entry = { transferId: challenge.transfer_id, expiresAt: 0, fingerprint, payload: Promise.resolve({}) };
+        entry.payload = signer(challenge).then(payload => {
+            const chunks = payload.chunks;
+            entry.expiresAt = Math.min(...chunks.flatMap(chunk => [Date.parse(chunk.source.expires_at), Date.parse(chunk.destination.expires_at)]));
+            return payload;
+        }).catch(error => { if (this.integrityGrantCache.get(challenge.audit_id) === entry)
+            this.integrityGrantCache.delete(challenge.audit_id); throw error; });
+        // Pending grants are shared too. Actual expiry replaces this sentinel on completion.
+        entry.expiresAt = Infinity;
+        this.integrityGrantCache.set(challenge.audit_id, entry);
+        return entry.payload;
+    }
+    async buildProviderIntegrityAuditGrants(input) {
         if (input.challenge.transfer_id !== input.prepared.transfer_id) {
             throw new Error("integrity audit challenge transfer mismatch");
         }
@@ -978,6 +1085,7 @@ export class BeamClient {
             }
             const sourceEtag = plannedSource?.metadata?.etag;
             const sourceVersionId = plannedSource?.metadata?.version_id;
+            const grantExpiresAt = new Date(Math.floor(Date.now() / 1000) * 1000 + input.expiresIn * 1000).toISOString();
             const [sourceGrant, destinationGrant] = await Promise.all([
                 signSourceReadRange({
                     source,
@@ -998,31 +1106,30 @@ export class BeamClient {
                     fetchImpl: this.fetchImpl
                 })
             ]);
-            const grantExpiresAt = expiresAtIso(input.expiresIn);
             const { orchestrator_id: _orchestratorId, orchestrator_hotkey: _orchestratorHotkey, worker_id: _workerId, ...grantChunk } = chunk;
             return {
                 ...grantChunk,
-                source: { ...sourceGrant, expires_at: grantExpiresAt },
-                destination: { ...destinationGrant, expires_at: grantExpiresAt }
+                source: { ...sourceGrant, expires_at: boundedGrantExpiry(sourceGrant.url, grantExpiresAt) },
+                destination: { ...destinationGrant, expires_at: boundedGrantExpiry(destinationGrant.url, grantExpiresAt) }
             };
         });
-        await this.control.request("transfer.integrity_audit_grants", {
-            transfer_id: input.challenge.transfer_id,
-            audit_id: input.challenge.audit_id,
-            submitted_at: submittedAt,
-            chunks
-        }, {
-            transferId: input.challenge.transfer_id,
-            idempotencyKey: `transfer:${input.challenge.transfer_id}:integrity-audit:${input.challenge.audit_id}`
-        });
+        return { transfer_id: input.challenge.transfer_id, audit_id: input.challenge.audit_id, submitted_at: submittedAt, chunks };
     }
-    stopRecoverySigner(transferId) {
-        const stop = this.recoverySigners.get(transferId);
-        if (stop) {
-            stop();
+    /** Stop a transfer's recovery and integrity signers; with `owner`, only if that owner installed them. */
+    stopRecoverySigner(transferId, owner) {
+        const signer = this.recoverySigners.get(transferId);
+        if (owner && signer?.owner !== owner)
+            return;
+        if (signer) {
+            signer.stop();
             this.recoverySigners.delete(transferId);
         }
         this.integrityAuditSigners.delete(transferId);
+        for (const [auditId, entry] of this.integrityGrantCache)
+            if (entry.transferId === transferId) {
+                this.integrityGrantCache.delete(auditId);
+                this.integrityAuditSubmissions.delete(auditId);
+            }
     }
     stopAllRecoverySigners() {
         for (const transferId of [...this.recoverySigners.keys()])
@@ -1118,7 +1225,7 @@ export class BeamClient {
         }
     }
 }
-const ROUTE_STREAM_BATCH_ROUTES = 2_048;
+const ROUTE_STREAM_BATCH_ROUTES = 1_024;
 class AsyncMutex {
     tail = Promise.resolve();
     async acquire() {
@@ -1143,6 +1250,8 @@ class AsyncMutex {
 class RouteStreamSender {
     control;
     options;
+    started = performance.now();
+    telemetry;
     streamId;
     checksum = new RouteKeysChecksum();
     batch = [];
@@ -1151,12 +1260,15 @@ class RouteStreamSender {
     routeCount = 0;
     sendTail = Promise.resolve();
     sendError = null;
+    sending = false;
     constructor(control, options) {
         this.control = control;
         this.options = options;
         this.streamId = options.streamId;
+        this.telemetry = options.telemetry ?? new SdkPerformanceCollector();
     }
     async begin() {
+        this.telemetry.startDelaySampling();
         await this.control.request("transfer.route_stream.begin", compact({
             transfer_id: this.options.transferId,
             route_generation_id: this.options.routeGenerationId,
@@ -1173,6 +1285,7 @@ class RouteStreamSender {
         });
     }
     async addManifestGroups(groups) {
+        const started = performance.now();
         if (!groups.length)
             return;
         await Promise.all(groups.map(async (group) => {
@@ -1188,8 +1301,12 @@ class RouteStreamSender {
                 idempotencyKey: `transfer:${this.options.transferId}:route-stream:${this.streamId}:manifest:${identity}`
             });
         }));
+        this.telemetry.observe("sdk.manifest_ack", performance.now() - started);
+        this.telemetry.mark("first_manifest_ms");
     }
     async addRoute(route) {
+        this.telemetry.mark("first_route_ms");
+        const assemblyStarted = performance.now();
         const deliveryIndex = signedRouteDeliveryIndex(route);
         if (deliveryIndex === undefined) {
             throw new Error("route delivery_index is required");
@@ -1203,6 +1320,8 @@ class RouteStreamSender {
         this.seenDeliveryIndices.add(deliveryIndex);
         this.routeCount += 1;
         this.batch.push({ ...route, delivery_index: deliveryIndex });
+        this.telemetry.gauge("buffered_batches_peak", 1 + Number(this.sending));
+        this.telemetry.observe("sdk.route_assembly", performance.now() - assemblyStarted);
         if (this.batch.length >= ROUTE_STREAM_BATCH_ROUTES) {
             await this.enqueueFlush();
         }
@@ -1212,11 +1331,17 @@ class RouteStreamSender {
             this.seenDeliveryIndices.size !== this.options.totalRoutes) {
             throw new Error(`route stream has incomplete or duplicate delivery indices: received ${this.routeCount} of ${this.options.totalRoutes} routes`);
         }
+        this.telemetry.mark("final_flush_ms");
         await this.enqueueFlush();
         await this.sendTail;
         if (this.sendError)
             throw this.sendError;
+        this.telemetry.observe("sdk.preparation", performance.now() - this.started);
+        this.telemetry.finish();
+        const sdkPerformance = this.telemetry.snapshot(this.control.supportsPerformanceV2?.(this.options.transferId) ?? false);
+        this.options.onDiagnostics?.(sdkPerformance);
         return this.control.request("transfer.route_stream.complete", {
+            sdk_performance: sdkPerformance,
             transfer_id: this.options.transferId,
             route_generation_id: this.options.routeGenerationId,
             stream_id: this.streamId,
@@ -1231,10 +1356,11 @@ class RouteStreamSender {
     async enqueueFlush() {
         if (!this.batch.length)
             return;
-        await this.sendTail;
+        await this.telemetry.measure("sdk.buffer_wait", () => this.sendTail);
         if (this.sendError)
             throw this.sendError;
         const routes = this.batch.splice(0, this.batch.length);
+        const encodingStarted = performance.now();
         const chunks = this.control.splitRoutesForPayload("transfer.route_stream.batch", {
             transfer_id: this.options.transferId,
             route_generation_id: this.options.routeGenerationId,
@@ -1242,20 +1368,29 @@ class RouteStreamSender {
             batch_id: `${this.streamId}:estimate`,
             batch_index: this.batchIndex
         }, routes);
+        this.telemetry.observe("sdk.batch_encode", performance.now() - encodingStarted);
+        this.sending = true;
         for (const chunk of chunks) {
+            const queuedAt = performance.now();
             const batchIndex = this.batchIndex;
             this.batchIndex += 1;
             this.sendTail = this.sendTail.then(async () => {
+                this.telemetry.observe("sdk.batch_queue", performance.now() - queuedAt);
                 if (this.sendError)
                     return;
                 try {
+                    const checksumStarted = performance.now();
                     const chunkChecksum = new RouteKeysChecksum();
                     await chunkChecksum.addMany(chunk.map(routeKeyForSignedRoute));
                     this.checksum.merge(chunkChecksum);
                     const routeChecksum = chunkChecksum.value();
                     const coordinateChecksum = await routeCoordinateChecksum(chunk);
                     const batchId = `${this.streamId}:${batchIndex}:${coordinateChecksum}`;
-                    await this.control.request("transfer.route_stream.batch", {
+                    this.telemetry.observe("sdk.batch_checksum", performance.now() - checksumStarted);
+                    this.telemetry.gauge("batch_routes_max", chunk.length);
+                    this.telemetry.mark("first_batch_ms");
+                    this.telemetry.counters.route_batches++;
+                    await this.telemetry.measure("sdk.batch_ack", () => this.control.request("transfer.route_stream.batch", {
                         transfer_id: this.options.transferId,
                         route_generation_id: this.options.routeGenerationId,
                         stream_id: this.streamId,
@@ -1267,15 +1402,17 @@ class RouteStreamSender {
                     }, {
                         transferId: this.options.transferId,
                         idempotencyKey: `transfer:${this.options.transferId}:route-stream:${this.streamId}:batch:${batchIndex}:${coordinateChecksum}`
-                    });
+                    }));
                 }
                 catch (error) {
                     this.sendError = error;
                 }
             });
         }
+        this.sendTail = this.sendTail.finally(() => { this.sending = false; });
     }
     async abort() {
+        this.telemetry.finish();
         this.sendError ??= new Error("route_stream_aborted");
         await this.sendTail;
     }
@@ -1511,6 +1648,9 @@ function createDeferred() {
         resolve = resolvePromise;
         reject = rejectPromise;
     });
+    // A waiter may be rejected after its route loop already stopped (for example when
+    // ownership is fenced mid-stream); awaiting callers still observe the rejection.
+    promise.catch(() => undefined);
     return { promise, resolve, reject };
 }
 function createMultipartGroupWaiters(prepared, destinationsById) {
@@ -1562,7 +1702,12 @@ function restoreProviderMultipartIdentities(prepared, destinations, identities,
 }
 async function createMultipartGroupManifest(input) {
     const groups = input.prepared.plan_descriptor.sources.flatMap((source) => input.prepared.plan_descriptor.destinations.map((destination) => ({ source, destination })));
+    const queuedAt = performance.now();
+    const telemetry = currentSdkPerformance();
+    let active = 0;
     const results = await mapOrderedWithConcurrency(groups, input.concurrency, async ({ source, destination }) => {
+        telemetry?.observe("sdk.multipart_queue", performance.now() - queuedAt);
+        telemetry?.gauge("multipart_active_peak", ++active);
         let multipartGroupId = null;
         try {
             const destinationConfig = input.destinationsById.get(destination.destination_id);
@@ -1580,12 +1725,12 @@ async function createMultipartGroupManifest(input) {
             const finalObjectMetadata = { "beam-transfer-id": input.prepared.transfer_id };
             input.signal?.throwIfAborted();
             const retainedUpload = input.multipartUploads.get(multipartGroupId);
-            const uploadId = retainedUpload?.uploadId ?? await createMultipartUpload({
+            const uploadId = retainedUpload?.uploadId ?? await measureSdkPhase("sdk.multipart_provider", () => createMultipartUpload({
                 destination: destinationConfig,
                 objectKey: finalObjectKey,
                 metadata: finalObjectMetadata,
                 signal: input.signal
-            });
+            }));
             const createdUpload = !retainedUpload;
             if (createdUpload) {
                 input.multipartUploads.set(multipartGroupId, {
@@ -1648,6 +1793,9 @@ async function createMultipartGroupManifest(input) {
                 input.onGroupFailed(multipartGroupId, error);
             return { ok: false, error };
         }
+        finally {
+            active--;
+        }
     });
     const failure = results.find((result) => !result.ok);
     if (failure && !failure.ok) {