@beam-network/sdk 0.6.1 → 0.8.0

package/README.md

@@ -39,8 +39,7 @@ const transfer = await beam.createTransfer({
       secret_access_key: "aws-secret-key"
     })
   ],
-  name: "r2-to-s3-report",
-  testMode: true
+  name: "r2-to-s3-report"
 });
 
 const status = await beam.waitForTransfer(transfer.transfer_id);
@@ -49,7 +48,32 @@ await beam.close();
 ```
 
 The main `createTransfer` API is provider-aware and strictly typed for S3, R2, S3-compatible, Hippius, and Hugging Face configs.
-Use `testMode: true` to create a BeamCore test-mode transfer.
+
+## Storage Credentials
+
+Source and destination credentials must not be restricted to specific IP addresses or networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different networks, so restricted credentials make the transfer fail.
+
+## Failed Transfers
+
+`waitForTransfer` rejects a failed transfer with `BeamTransferFailedError`, whose `errorMessage` is BeamCore's `error_message` verbatim. When the storage refused Beam's requests, the error is the `BeamStorageAccessError` subclass, with `code` set to `source_access_denied` or `destination_access_denied`:
+
+```ts
+import { BeamStorageAccessError, BeamTransferFailedError } from "@beam-network/sdk";
+
+try {
+  await beam.waitForTransfer(transfer.transfer_id);
+} catch (error) {
+  if (error instanceof BeamStorageAccessError) {
+    // For example: "destination_access_denied: The destination storage refused Beam's requests (403 AccessDenied). ..."
+    console.error(error.code, error.errorMessage);
+  } else if (error instanceof BeamTransferFailedError) {
+    console.error(error.errorMessage);
+  }
+  throw error;
+}
+```
+
+Callers that poll `transferStatus` themselves can pass a failed status to `transferFailedError(transferId, status.error_message)` to get the same classification.
 
 ## S3-Compatible Providers
 
@@ -195,3 +219,24 @@ Multipart control helpers (`createMultipartUpload`, `listMultipartParts`, `compl
 Route streaming flushes at 1,024 routes, with no timer, and sends the final partial batch at stream completion. Signing overlaps acknowledged publication through bounded buffering; existing encoded-payload limits still apply. A source grant is reused across destinations only within the same signing generation and actual expiry.
 
 The optional `onDiagnostics` callback receives bounded preparation measurements and source-grant reuse counts. Callbacks are best effort, may be dropped under load, and do not affect transfer outcomes. Durations may overlap; do not sum them into elapsed transfer time. Transfer status also exposes a typed `performance` summary when supported by Core.
+
+Optional `storage_location` (`StorageLocation` in Go) describes the physical
+storage location. It is separate from the region used to sign provider requests.
+Leave it unset when unknown; a signing region such as R2's `auto` is not a location.
+
+When supported by Core, diagnostics use `sdk-performance/v2`: bounded histograms,
+preparation milestones, concurrency high-water marks, and separate provider,
+callback, manifest, signing, and transport waits. Histogram bins are noncumulative,
+with upper bounds in milliseconds of 0.1, 0.5, 1, 2, 5, 10, 25, 50, 100, 250,
+500, 1000, 2500, 5000, 10000, 30000, 120000, then overflow. `unmeasured` explicitly
+identifies unavailable measurements. Process CPU includes other concurrent work
+in the SDK process; it is not transfer-exclusive CPU. Detailed reporting can be
+disabled with `BEAM_SDK_PERFORMANCE_DETAILS=false`. Older peers retain v1 reports.
+No credentials or storage grants are included in these diagnostics.
+
+`sdk.producer_wait` measures waiting for a signed route separately from publication
+backpressure. Configured-limit gauges preserve starting limits; effective-limit
+gauges report the highest limit reached. Optional `source_renewals` counts source
+grants recreated in later preparation generations. A bounded 16 KiB history tracks
+131,072 chunk indices; the counter is omitted after takeover or beyond that bound.
+It does not include separate recovery-control signing operations.

package/dist/client.d.ts

@@ -19,6 +19,33 @@ export declare class BeamProviderTransferError extends AggregateError {
         cleanupError?: unknown;
     });
 }
+/** Failure codes BeamCore reports when source or destination storage refuses Beam's requests. */
+export type BeamStorageAccessErrorCode = "source_access_denied" | "destination_access_denied";
+/** A transfer that BeamCore reported as failed. */
+export declare class BeamTransferFailedError extends Error {
+    readonly transferId: string;
+    /** The transfer's `error_message` from BeamCore, verbatim; null when BeamCore sent none. */
+    readonly errorMessage: string | null;
+    constructor(transferId: string, errorMessage: string | null);
+}
+/**
+ * The source or destination storage refused Beam's requests (`source_access_denied` or
+ * `destination_access_denied`). `errorMessage` carries BeamCore's explanation verbatim.
+ *
+ * Check that the credentials allow the operation on this bucket and path and are not restricted
+ * to specific IP addresses or networks (for example Cloudflare R2 API-token client IP filtering,
+ * S3 bucket policies with `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many
+ * workers on different networks, so restricted credentials make the transfer fail.
+ */
+export declare class BeamStorageAccessError extends BeamTransferFailedError {
+    readonly code: BeamStorageAccessErrorCode;
+    constructor(transferId: string, errorMessage: string, code: BeamStorageAccessErrorCode);
+}
+/**
+ * Builds the error for a failed transfer status: a {@link BeamStorageAccessError} when the
+ * server message starts with a storage access code, otherwise a {@link BeamTransferFailedError}.
+ */
+export declare function transferFailedError(transferId: string, errorMessage: string | null): BeamTransferFailedError;
 export declare class BeamApiError extends Error {
     readonly status: number;
     readonly body: string;
@@ -29,7 +56,6 @@ type TransferPrepareInput = {
     sources: PreparedHttpSource[];
     destinations: PreparedDestination[];
     name?: string;
-    testMode?: boolean;
     urlsExpiresAt?: string;
     signedUrlFlow?: SignedUrlFlow;
     idempotencyKey?: string;
@@ -54,22 +80,62 @@ export declare class BeamClient {
     constructor(options?: BeamClientOptions);
     close(): Promise<void>;
     openTransferTerminalWaiter(transferId: string): Promise<TransferTerminalSignalWaiter>;
+    /**
+     * Creates a transfer from raw source and destination configs (`transfer.create`).
+     *
+     * Source and destination credentials must not be restricted to specific IP addresses or
+     * networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
+     * `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
+     * networks, so restricted credentials make the transfer fail.
+     */
     createRawTransfer(input: RawTransferCreateInput): Promise<TransferCreateResponse>;
+    /**
+     * Signs provider sources and destinations, prepares the transfer, streams its signed routes
+     * and, unless `distribute` is false, starts it. Alias of {@link prepareProviderTransfer}.
+     *
+     * Source and destination credentials must not be restricted to specific IP addresses or
+     * networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
+     * `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
+     * networks, so restricted credentials make the transfer fail.
+     */
     createTransfer(input: ProviderTransferCreateInput): Promise<TransferPrepareResponse>;
+    /**
+     * Takes over a prepared provider transfer in a new process, reusing its multipart uploads.
+     *
+     * Source and destination credentials must not be restricted to specific IP addresses or
+     * networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
+     * `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
+     * networks, so restricted credentials make the transfer fail.
+     */
     resumeProviderTransfer(input: ProviderTransferResumeInput): Promise<TransferPrepareResponse>;
     transferStatus(transferId: string): Promise<TransferStatusInfo>;
     distributeTransfer(transferId: string): Promise<DistributeResponse>;
     private requestTransferCancellation;
     cancelTransfer(transferId: string): Promise<TransferCancelResponse>;
+    /**
+     * Asks BeamCore for the compact plan a transfer would use without creating it (`transfer.plan`).
+     *
+     * Source and destination credentials must not be restricted to specific IP addresses or
+     * networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
+     * `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
+     * networks, so restricted credentials make the transfer fail.
+     */
     planTransfer(input: {
         sources: PlanningHttpSource[];
         destinations: PreparedDestination[];
         name?: string;
-        testMode?: boolean;
         urlsExpiresAt?: string;
         signedUrlFlow?: SignedUrlFlow;
         chunkSize?: number;
     }): Promise<TransferPlanResponse>;
+    /**
+     * Prepares a transfer from already-signed HTTP sources and destinations (`transfer.prepare`).
+     *
+     * Source and destination credentials must not be restricted to specific IP addresses or
+     * networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
+     * `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
+     * networks, so restricted credentials make the transfer fail.
+     */
     prepareTransfer(input: TransferPrepareInput): Promise<TransferPrepareResponse>;
     private prepareTransferWithRequestKey;
     attachSignedUrls(transferId: string, input: {
@@ -87,6 +153,15 @@ export declare class BeamClient {
         urlsExpiresAt?: string;
         autoDistribute?: boolean;
     }): Promise<AttachSignedUrlsResponse>;
+    /**
+     * Signs provider sources and destinations, prepares the transfer, streams its signed routes
+     * and, unless `distribute` is false, starts it.
+     *
+     * Source and destination credentials must not be restricted to specific IP addresses or
+     * networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
+     * `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
+     * networks, so restricted credentials make the transfer fail.
+     */
     prepareProviderTransfer(input: ProviderTransferCreateInput): Promise<TransferPrepareResponse>;
     private executeProviderTransfer;
     /**
@@ -113,7 +188,21 @@ export declare class BeamClient {
     /** Stop a transfer's recovery and integrity signers; with `owner`, only if that owner installed them. */
     private stopRecoverySigner;
     private stopAllRecoverySigners;
+    /**
+     * Creates a raw transfer and distributes it.
+     *
+     * Source and destination credentials must not be restricted to specific IP addresses or
+     * networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
+     * `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
+     * networks, so restricted credentials make the transfer fail.
+     */
     createAndDistribute(input: RawTransferCreateInput): Promise<TransferCreateResponse>;
+    /**
+     * Waits until the transfer completes. A failed transfer rejects with a
+     * {@link BeamTransferFailedError} carrying BeamCore's `error_message` verbatim, or with its
+     * {@link BeamStorageAccessError} subclass when the source or destination storage refused
+     * Beam's requests (`source_access_denied`, `destination_access_denied`).
+     */
     waitForTransfer(transferId: string, options?: {
         timeoutMs?: number;
         pollIntervalMs?: number;

package/dist/client.js

@@ -1,4 +1,4 @@
-import { SdkPerformanceCollector } from "./performance.js";
+import { SdkPerformanceCollector, SourceSignatureHistory, currentSdkPerformance, measureSdkPhase } from "./performance.js";
 import { signMultipartRecovery, abortMultipartUpload, createMultipartUpload, expiresAtIso, prepareProviderDestination, prepareProviderSource, releaseProviderClients, signAbortMultipartUpload, signCompleteMultipartUpload, signDestinationReadRange, signDestinationRoute, signFinalObjectHead, signListMultipartUpload, signSourceReadRange, signSourceChunk, boundedGrantExpiry, isHuggingFaceProvider } from "./provider-signing.js";
 import { describe as describeHuggingFace, hashSourceStream, huggingFaceCommit, huggingFaceCompleteLfsUpload, huggingFaceLfsBatch, huggingFacePreupload, huggingFaceVerifyLfsUpload, readSourceSample } from "./huggingface.js";
 import { BeamTransferControl, BEAM_DEFAULT_NATS_URL, compactSignedRoutes, isRecoverableRouteStreamError } from "./nats-control.js";
@@ -36,6 +36,54 @@ export class BeamProviderTransferError extends AggregateError {
         this.cause = input.cause;
     }
 }
+const STORAGE_ACCESS_ERROR_CODES = [
+    "source_access_denied",
+    "destination_access_denied"
+];
+/** A transfer that BeamCore reported as failed. */
+export class BeamTransferFailedError extends Error {
+    transferId;
+    /** The transfer's `error_message` from BeamCore, verbatim; null when BeamCore sent none. */
+    errorMessage;
+    constructor(transferId, errorMessage) {
+        super(`Transfer ${transferId} failed: ${errorMessage ?? "unknown error"}`);
+        this.name = "BeamTransferFailedError";
+        this.transferId = transferId;
+        this.errorMessage = errorMessage;
+    }
+}
+/**
+ * The source or destination storage refused Beam's requests (`source_access_denied` or
+ * `destination_access_denied`). `errorMessage` carries BeamCore's explanation verbatim.
+ *
+ * Check that the credentials allow the operation on this bucket and path and are not restricted
+ * to specific IP addresses or networks (for example Cloudflare R2 API-token client IP filtering,
+ * S3 bucket policies with `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many
+ * workers on different networks, so restricted credentials make the transfer fail.
+ */
+export class BeamStorageAccessError extends BeamTransferFailedError {
+    code;
+    constructor(transferId, errorMessage, code) {
+        super(transferId, errorMessage);
+        this.name = "BeamStorageAccessError";
+        this.code = code;
+    }
+}
+/**
+ * Builds the error for a failed transfer status: a {@link BeamStorageAccessError} when the
+ * server message starts with a storage access code, otherwise a {@link BeamTransferFailedError}.
+ */
+export function transferFailedError(transferId, errorMessage) {
+    const code = errorMessage === null ? undefined : storageAccessErrorCode(errorMessage);
+    return code && errorMessage !== null
+        ? new BeamStorageAccessError(transferId, errorMessage, code)
+        : new BeamTransferFailedError(transferId, errorMessage);
+}
+function storageAccessErrorCode(errorMessage) {
+    const separator = errorMessage.indexOf(":");
+    const prefix = (separator === -1 ? errorMessage : errorMessage.slice(0, separator)).trim();
+    return STORAGE_ACCESS_ERROR_CODES.find((code) => code === prefix);
+}
 export class BeamApiError extends Error {
     status;
     body;
@@ -94,6 +142,14 @@ export class BeamClient {
         validateId(transferId, "transferId");
         return this.control.openTerminalSignalWaiter(transferId);
     }
+    /**
+     * Creates a transfer from raw source and destination configs (`transfer.create`).
+     *
+     * Source and destination credentials must not be restricted to specific IP addresses or
+     * networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
+     * `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
+     * networks, so restricted credentials make the transfer fail.
+     */
     async createRawTransfer(input) {
         const transferId = await transferIdForIdempotencyKey(input.idempotencyKey);
         const body = compact({
@@ -106,7 +162,6 @@ export class BeamClient {
             merkle_root: input.merkleRoot,
             chunk_hashes: input.chunkHashes,
             callbacks: input.callbacks,
-            test_mode: input.testMode || undefined,
             progressive_mode: input.progressiveMode || undefined,
             signed_url_flow: input.signedUrlFlow ?? "signed_url"
         });
@@ -115,9 +170,26 @@ export class BeamClient {
             idempotencyKey: `transfer:${transferId}:create`
         });
     }
+    /**
+     * Signs provider sources and destinations, prepares the transfer, streams its signed routes
+     * and, unless `distribute` is false, starts it. Alias of {@link prepareProviderTransfer}.
+     *
+     * Source and destination credentials must not be restricted to specific IP addresses or
+     * networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
+     * `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
+     * networks, so restricted credentials make the transfer fail.
+     */
     createTransfer(input) {
         return this.prepareProviderTransfer(input);
     }
+    /**
+     * Takes over a prepared provider transfer in a new process, reusing its multipart uploads.
+     *
+     * Source and destination credentials must not be restricted to specific IP addresses or
+     * networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
+     * `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
+     * networks, so restricted credentials make the transfer fail.
+     */
     async resumeProviderTransfer(input) {
         validateId(input.transferId, "transferId");
         return this.executeProviderTransfer(input, input);
@@ -159,12 +231,19 @@ export class BeamClient {
         this.stopRecoverySigner(transferId);
         return result;
     }
+    /**
+     * Asks BeamCore for the compact plan a transfer would use without creating it (`transfer.plan`).
+     *
+     * Source and destination credentials must not be restricted to specific IP addresses or
+     * networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
+     * `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
+     * networks, so restricted credentials make the transfer fail.
+     */
     async planTransfer(input) {
         const result = await this.control.request("transfer.plan", compact({
             sources: input.sources,
             destinations: input.destinations,
             name: input.name,
-            test_mode: input.testMode || undefined,
             chunk_size: input.chunkSize,
             urls_expires_at: input.urlsExpiresAt,
             signed_url_flow: input.signedUrlFlow ?? "signed_url"
@@ -173,6 +252,14 @@ export class BeamClient {
             validateCompactTransferPlan(result.plan_descriptor, result.signed_url_flow);
         return result;
     }
+    /**
+     * Prepares a transfer from already-signed HTTP sources and destinations (`transfer.prepare`).
+     *
+     * Source and destination credentials must not be restricted to specific IP addresses or
+     * networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
+     * `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
+     * networks, so restricted credentials make the transfer fail.
+     */
     prepareTransfer(input) {
         return this.prepareTransferWithRequestKey(input);
     }
@@ -187,7 +274,6 @@ export class BeamClient {
             sources: input.sources,
             destinations: input.destinations,
             name: input.name,
-            test_mode: input.testMode || undefined,
             chunk_size: input.chunkSize,
             urls_expires_at: input.urlsExpiresAt,
             signed_url_flow: input.signedUrlFlow ?? "signed_url"
@@ -274,6 +360,15 @@ export class BeamClient {
             releaseInitialStream();
         }
     }
+    /**
+     * Signs provider sources and destinations, prepares the transfer, streams its signed routes
+     * and, unless `distribute` is false, starts it.
+     *
+     * Source and destination credentials must not be restricted to specific IP addresses or
+     * networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
+     * `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
+     * networks, so restricted credentials make the transfer fail.
+     */
     async prepareProviderTransfer(input) {
         return this.executeProviderTransfer(input);
     }
@@ -300,10 +395,9 @@ export class BeamClient {
             destinationsById.set(preparedDestination.destination_id, destination);
         });
         await input.throwIfCancelled?.();
-        const discoveryStarted = performance.now();
-        const preparedSources = await Promise.all(retainedRecoveryInput.sources.map((source, index) => prepareProviderSource(source, { index, expiresIn, fetchImpl: this.fetchImpl, signal: input.signal })));
+        const discoveryTelemetry = new SdkPerformanceCollector();
+        const preparedSources = await discoveryTelemetry.measure("sdk.discovery", () => Promise.all(retainedRecoveryInput.sources.map((source, index) => prepareProviderSource(source, { index, expiresIn, fetchImpl: this.fetchImpl, signal: input.signal }))));
         await input.throwIfCancelled?.();
-        const discoveryMs = performance.now() - discoveryStarted;
         const huggingFace = await this.planHuggingFaceUploads({
             sources: retainedRecoveryInput.sources,
             preparedSources,
@@ -317,7 +411,6 @@ export class BeamClient {
             sources: preparedSources,
             destinations: preparedDestinations,
             name: input.name,
-            testMode: input.testMode,
             chunkSize: huggingFace.chunkSize ?? input.chunkSize,
             signedUrlFlow: requestedSignedUrlFlow,
             ...(resume ? { transferId: resume.transferId } : { idempotencyKey: input.idempotencyKey, routeGenerationId: input.routeGenerationId })
@@ -353,13 +446,18 @@ export class BeamClient {
         const assertOwnership = () => input.signal?.throwIfAborted();
         const routeStreamLock = new AsyncMutex();
         const releaseInitialStream = await routeStreamLock.acquire();
+        const sourceSignatureHistory = new SourceSignatureHistory(!resume);
         const streamPreparedRoutes = async (routeGenerationId, recoveryReplay) => {
-            const telemetry = new SdkPerformanceCollector();
-            if (!recoveryReplay)
-                telemetry.observe("sdk.discovery", discoveryMs);
+            const telemetry = new SdkPerformanceCollector(sourceSignatureHistory);
+            telemetry.seedDiscovery(discoveryTelemetry);
+            telemetry.gauge("signing_configured_limit", this.routeSigningConcurrency);
+            telemetry.gauge("multipart_configured_limit", this.multipartControlConcurrency);
+            telemetry.gauge("signing_limit", this.routeSigningConcurrency);
+            telemetry.gauge("multipart_limit", this.multipartControlConcurrency);
             const pendingRoutes = new Set();
             let signingConcurrency = this.routeSigningConcurrency;
             let signedInWindow = 0;
+            let activeSigning = 0;
             let signingWindowStartedAt = performance.now();
             let routeStream = null;
             let routeStreamBeginAttempted = false;
@@ -420,14 +518,14 @@ export class BeamClient {
                         assertOwnership();
                         recoveryMultipartUploads.set(state.manifest.multipart_group_id, state);
                         validateMultipartGroupManifest([state.manifest], prepared.transfer_id);
-                        await input.onMultipartGroupReady?.(multipartGroupIdentity(prepared.transfer_id, state.manifest));
+                        await telemetry.measure("sdk.multipart_callback", async () => input.onMultipartGroupReady?.(multipartGroupIdentity(prepared.transfer_id, state.manifest)));
                         await routeStream.addManifestGroups([state.manifest]);
                         multipartGroupWaiters.get(state.manifest.multipart_group_id)?.resolve(state);
                     },
                     onGroupFailed: (groupId, error) => multipartGroupWaiters.get(groupId)?.reject(error)
                 }));
                 const streamNextCompletedRoute = async () => {
-                    const settled = await Promise.race([...pendingRoutes].map((pending) => pending.then((route) => ({ pending, route }))));
+                    const settled = await telemetry.measure("sdk.producer_wait", () => Promise.race([...pendingRoutes].map((pending) => pending.then((route) => ({ pending, route })))));
                     pendingRoutes.delete(settled.pending);
                     assertOwnership();
                     await routeStream.addRoute(settled.route);
@@ -436,6 +534,8 @@ export class BeamClient {
                         const elapsedMs = performance.now() - signingWindowStartedAt;
                         if (!this.routeSigningConcurrencyOverridden && elapsedMs > 4_000 && signingConcurrency < 256) {
                             signingConcurrency = Math.min(256, signingConcurrency * 2);
+                            telemetry.increment("concurrency_changes");
+                            telemetry.gauge("signing_limit", signingConcurrency);
                         }
                         signedInWindow = 0;
                         signingWindowStartedAt = performance.now();
@@ -445,10 +545,9 @@ export class BeamClient {
                     await throwIfCancelled?.(prepared.transfer_id);
                     // This promise belongs to this chunk and signing generation only. Its
                     // immutable credential/range identity and actual expiry cannot be renewed.
-                    const sourceGrant = signSourceChunk({ source: sourcesById.get(chunk.source_id), fallbackUrl: chunk.source_url, chunk, expiresIn, fetchImpl: this.fetchImpl });
+                    let sourceUses = 0;
+                    const sourceGrant = telemetry.measure("sdk.source_signing", () => signSourceChunk({ source: sourcesById.get(chunk.source_id), fallbackUrl: chunk.source_url, chunk, expiresIn, fetchImpl: this.fetchImpl })).then(grant => { telemetry.sourceCreated(chunk.chunk_index); return grant; });
                     void sourceGrant.catch(() => { });
-                    telemetry.counters.source_signatures++;
-                    telemetry.counters.source_reuses += Math.max(0, chunk.destinations.length - 1);
                     for (const target of chunk.destinations) {
                         const pendingRoute = (async () => {
                             await throwIfCancelled?.(prepared.transfer_id);
@@ -465,14 +564,16 @@ export class BeamClient {
                             const multipartGroupId = multipartGroupStateKey(prepared.transfer_id, target.destination_id, chunk.source_id, finalObjectKey);
                             const upload = isDirectPutDestination(destination)
                                 ? undefined
-                                : await multipartGroupWaiters.get(multipartGroupId)?.promise;
+                                : await telemetry.measure("sdk.multipart_ready_wait", async () => multipartGroupWaiters.get(multipartGroupId)?.promise);
                             if (!isDirectPutDestination(destination) && !upload)
                                 throw new Error(`multipart group manifest is missing for ${chunk.source_id}:${target.destination_id}`);
+                            telemetry.gauge("signing_active_peak", ++activeSigning);
                             return telemetry.measure("sdk.signing", () => this.signProviderRoute({
                                 chunk,
                                 target,
                                 source,
-                                sourceGrant,
+                                sourceGrant: telemetry.measure("sdk.source_grant_wait", () => sourceGrant).then(grant => { if (sourceUses++ > 0)
+                                    telemetry.counters.source_reuses++; return grant; }),
                                 destination,
                                 expiresIn,
                                 ...(upload ? { upload } : {}),
@@ -483,11 +584,12 @@ export class BeamClient {
                                 partNumber: typeof metadata.part_number === "number"
                                     ? metadata.part_number
                                     : multipartPartNumber(chunk.source_chunk_index),
-                            }));
+                            })).finally(() => { activeSigning--; });
                         })();
                         pendingRoutes.add(pendingRoute);
+                        telemetry.gauge("signing_pending_peak", pendingRoutes.size);
                         if (pendingRoutes.size >= signingConcurrency)
-                            await streamNextCompletedRoute();
+                            await telemetry.measure("sdk.signing_queue", streamNextCompletedRoute);
                     }
                 }
                 while (pendingRoutes.size)
@@ -1127,6 +1229,14 @@ export class BeamClient {
         for (const transferId of [...this.recoverySigners.keys()])
             this.stopRecoverySigner(transferId);
     }
+    /**
+     * Creates a raw transfer and distributes it.
+     *
+     * Source and destination credentials must not be restricted to specific IP addresses or
+     * networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
+     * `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
+     * networks, so restricted credentials make the transfer fail.
+     */
     async createAndDistribute(input) {
         const transfer = await this.createRawTransfer(input);
         if (transfer.success) {
@@ -1134,6 +1244,12 @@ export class BeamClient {
         }
         return transfer;
     }
+    /**
+     * Waits until the transfer completes. A failed transfer rejects with a
+     * {@link BeamTransferFailedError} carrying BeamCore's `error_message` verbatim, or with its
+     * {@link BeamStorageAccessError} subclass when the source or destination storage refused
+     * Beam's requests (`source_access_denied`, `destination_access_denied`).
+     */
     async waitForTransfer(transferId, options = {}) {
         const timeoutMs = options.timeoutMs ?? 300_000;
         const pollIntervalMs = options.pollIntervalMs ?? 15_000;
@@ -1166,7 +1282,7 @@ export class BeamClient {
                     return status;
                 }
                 if (status.status === "failed") {
-                    throw new Error(`Transfer failed: ${status.error_message ?? "unknown error"}`);
+                    throw transferFailedError(transferId, status.error_message);
                 }
                 if (status.status === "cancelled") {
                     throw new Error("Transfer cancelled");
@@ -1252,6 +1368,7 @@ class RouteStreamSender {
     routeCount = 0;
     sendTail = Promise.resolve();
     sendError = null;
+    sending = false;
     constructor(control, options) {
         this.control = control;
         this.options = options;
@@ -1259,6 +1376,7 @@ class RouteStreamSender {
         this.telemetry = options.telemetry ?? new SdkPerformanceCollector();
     }
     async begin() {
+        this.telemetry.startDelaySampling();
         await this.control.request("transfer.route_stream.begin", compact({
             transfer_id: this.options.transferId,
             route_generation_id: this.options.routeGenerationId,
@@ -1275,6 +1393,7 @@ class RouteStreamSender {
         });
     }
     async addManifestGroups(groups) {
+        const started = performance.now();
         if (!groups.length)
             return;
         await Promise.all(groups.map(async (group) => {
@@ -1290,8 +1409,12 @@ class RouteStreamSender {
                 idempotencyKey: `transfer:${this.options.transferId}:route-stream:${this.streamId}:manifest:${identity}`
             });
         }));
+        this.telemetry.observe("sdk.manifest_ack", performance.now() - started);
+        this.telemetry.mark("first_manifest_ms");
     }
     async addRoute(route) {
+        this.telemetry.mark("first_route_ms");
+        const assemblyStarted = performance.now();
         const deliveryIndex = signedRouteDeliveryIndex(route);
         if (deliveryIndex === undefined) {
             throw new Error("route delivery_index is required");
@@ -1305,6 +1428,8 @@ class RouteStreamSender {
         this.seenDeliveryIndices.add(deliveryIndex);
         this.routeCount += 1;
         this.batch.push({ ...route, delivery_index: deliveryIndex });
+        this.telemetry.gauge("buffered_batches_peak", 1 + Number(this.sending));
+        this.telemetry.observe("sdk.route_assembly", performance.now() - assemblyStarted);
         if (this.batch.length >= ROUTE_STREAM_BATCH_ROUTES) {
             await this.enqueueFlush();
         }
@@ -1314,12 +1439,14 @@ class RouteStreamSender {
             this.seenDeliveryIndices.size !== this.options.totalRoutes) {
             throw new Error(`route stream has incomplete or duplicate delivery indices: received ${this.routeCount} of ${this.options.totalRoutes} routes`);
         }
+        this.telemetry.mark("final_flush_ms");
         await this.enqueueFlush();
         await this.sendTail;
         if (this.sendError)
             throw this.sendError;
         this.telemetry.observe("sdk.preparation", performance.now() - this.started);
-        const sdkPerformance = this.telemetry.snapshot();
+        this.telemetry.finish();
+        const sdkPerformance = this.telemetry.snapshot(this.control.supportsPerformanceV2?.(this.options.transferId) ?? false);
         this.options.onDiagnostics?.(sdkPerformance);
         return this.control.request("transfer.route_stream.complete", {
             sdk_performance: sdkPerformance,
@@ -1341,6 +1468,7 @@ class RouteStreamSender {
         if (this.sendError)
             throw this.sendError;
         const routes = this.batch.splice(0, this.batch.length);
+        const encodingStarted = performance.now();
         const chunks = this.control.splitRoutesForPayload("transfer.route_stream.batch", {
             transfer_id: this.options.transferId,
             route_generation_id: this.options.routeGenerationId,
@@ -1348,19 +1476,27 @@ class RouteStreamSender {
             batch_id: `${this.streamId}:estimate`,
             batch_index: this.batchIndex
         }, routes);
+        this.telemetry.observe("sdk.batch_encode", performance.now() - encodingStarted);
+        this.sending = true;
         for (const chunk of chunks) {
+            const queuedAt = performance.now();
             const batchIndex = this.batchIndex;
             this.batchIndex += 1;
             this.sendTail = this.sendTail.then(async () => {
+                this.telemetry.observe("sdk.batch_queue", performance.now() - queuedAt);
                 if (this.sendError)
                     return;
                 try {
+                    const checksumStarted = performance.now();
                     const chunkChecksum = new RouteKeysChecksum();
                     await chunkChecksum.addMany(chunk.map(routeKeyForSignedRoute));
                     this.checksum.merge(chunkChecksum);
                     const routeChecksum = chunkChecksum.value();
                     const coordinateChecksum = await routeCoordinateChecksum(chunk);
                     const batchId = `${this.streamId}:${batchIndex}:${coordinateChecksum}`;
+                    this.telemetry.observe("sdk.batch_checksum", performance.now() - checksumStarted);
+                    this.telemetry.gauge("batch_routes_max", chunk.length);
+                    this.telemetry.mark("first_batch_ms");
                     this.telemetry.counters.route_batches++;
                     await this.telemetry.measure("sdk.batch_ack", () => this.control.request("transfer.route_stream.batch", {
                         transfer_id: this.options.transferId,
@@ -1381,8 +1517,10 @@ class RouteStreamSender {
                 }
             });
         }
+        this.sendTail = this.sendTail.finally(() => { this.sending = false; });
     }
     async abort() {
+        this.telemetry.finish();
         this.sendError ??= new Error("route_stream_aborted");
         await this.sendTail;
     }
@@ -1672,7 +1810,12 @@ function restoreProviderMultipartIdentities(prepared, destinations, identities,
 }
 async function createMultipartGroupManifest(input) {
     const groups = input.prepared.plan_descriptor.sources.flatMap((source) => input.prepared.plan_descriptor.destinations.map((destination) => ({ source, destination })));
+    const queuedAt = performance.now();
+    const telemetry = currentSdkPerformance();
+    let active = 0;
     const results = await mapOrderedWithConcurrency(groups, input.concurrency, async ({ source, destination }) => {
+        telemetry?.observe("sdk.multipart_queue", performance.now() - queuedAt);
+        telemetry?.gauge("multipart_active_peak", ++active);
         let multipartGroupId = null;
         try {
             const destinationConfig = input.destinationsById.get(destination.destination_id);
@@ -1690,12 +1833,12 @@ async function createMultipartGroupManifest(input) {
             const finalObjectMetadata = { "beam-transfer-id": input.prepared.transfer_id };
             input.signal?.throwIfAborted();
             const retainedUpload = input.multipartUploads.get(multipartGroupId);
-            const uploadId = retainedUpload?.uploadId ?? await createMultipartUpload({
+            const uploadId = retainedUpload?.uploadId ?? await measureSdkPhase("sdk.multipart_provider", () => createMultipartUpload({
                 destination: destinationConfig,
                 objectKey: finalObjectKey,
                 metadata: finalObjectMetadata,
                 signal: input.signal
-            });
+            }));
             const createdUpload = !retainedUpload;
             if (createdUpload) {
                 input.multipartUploads.set(multipartGroupId, {
@@ -1758,6 +1901,9 @@ async function createMultipartGroupManifest(input) {
                 input.onGroupFailed(multipartGroupId, error);
             return { ok: false, error };
         }
+        finally {
+            active--;
+        }
     });
     const failure = results.find((result) => !result.ok);
     if (failure && !failure.ok) {

package/dist/models.d.ts

@@ -1,16 +1,22 @@
 export type SignedUrlFlow = "signed_url";
 export interface SdkPerformanceSummary {
-    schema_version: "sdk-performance/v1";
+    gauges?: Record<string, number>;
+    milestones?: Record<string, number>;
+    unmeasured?: string[];
+    detail_dropped?: number;
+    schema_version: "sdk-performance/v1" | "sdk-performance/v2";
     measurements: Array<{
-        name: "sdk.discovery" | "sdk.multipart_create" | "sdk.signing" | "sdk.batch_ack" | "sdk.buffer_wait" | "sdk.preparation";
+        name: string;
         count: number;
         work_ms: number;
         max_ms: number;
+        histogram?: number[];
     }>;
     counters: {
         source_signatures: number;
         source_reuses: number;
         route_batches: number;
+        source_renewals?: number;
     };
 }
 export interface BeamClientOptions {
@@ -28,6 +34,14 @@ export interface BeamClientOptions {
     multipartControlConcurrency?: number;
     fetch?: typeof fetch;
 }
+/**
+ * Raw transfer source.
+ *
+ * Source credentials must not be restricted to specific IP addresses or networks (for example
+ * Cloudflare R2 API-token client IP filtering, S3 bucket policies with `aws:SourceIp`, or
+ * VPC-only endpoints). Beam moves data through many workers on different networks, so
+ * restricted credentials make the transfer fail.
+ */
 export interface SourceConfig {
     type: string;
     bucket?: string;
@@ -41,6 +55,14 @@ export interface SourceConfig {
     url?: string;
     headers?: Record<string, string>;
 }
+/**
+ * Raw transfer destination.
+ *
+ * Destination credentials must not be restricted to specific IP addresses or networks (for
+ * example Cloudflare R2 API-token client IP filtering, S3 bucket policies with `aws:SourceIp`, or
+ * VPC-only endpoints). Beam moves data through many workers on different networks, so
+ * restricted credentials make the transfer fail.
+ */
 export interface DestConfig {
     type: string;
     bucket?: string;
@@ -67,7 +89,6 @@ export interface TransferCreateRequest {
     merkle_root?: string;
     chunk_hashes?: string[];
     callbacks?: CallbackConfig[];
-    test_mode?: boolean;
     progressive_mode?: boolean;
     signed_url_flow: SignedUrlFlow;
 }
@@ -80,10 +101,6 @@ export interface RawTransferCreateInput {
     merkleRoot?: string;
     chunkHashes?: string[];
     callbacks?: CallbackConfig[];
-    /**
-     * Maps to BeamCore test_mode.
-     */
-    testMode?: boolean;
     progressiveMode?: boolean;
     signedUrlFlow?: SignedUrlFlow;
     idempotencyKey?: string;
@@ -158,6 +175,21 @@ export interface IntegrityAuditChallenge {
     chunks: IntegrityAuditChallengeChunk[];
 }
 export interface TransferPerformance {
+    recovery_bandwidth?: {
+        excluded_tasks: number;
+        accepted_bytes: number;
+    };
+    deadline?: {
+        version: "assignment-deadline/v1";
+        decisions: number;
+        selected_min_ms: number;
+        selected_max_ms: number;
+        estimated_max_ms: number;
+        sample_count_max: number;
+        data_age_max_ms: number | null;
+        reasons: Record<string, number>;
+    };
+    sdk_detail?: SdkPerformanceSummary;
     schema_version: "transfer-performance/v1";
     runtime_epoch: string;
     coverage: "complete" | "restarted";
@@ -177,6 +209,7 @@ export interface TransferPerformance {
         source_signatures: number;
         source_reuses: number;
         route_batches: number;
+        source_renewals?: number;
     };
     detail_dropped: number;
     unmeasured?: string[];
@@ -232,7 +265,17 @@ export interface TransferTerminalSignalWaiter {
     wait(timeoutMs: number): Promise<TransferTerminalEvent | null>;
     close(): Promise<void>;
 }
+/**
+ * Amazon S3 source or destination.
+ *
+ * Credentials must not be restricted to specific IP addresses or networks (for example
+ * Cloudflare R2 API-token client IP filtering, S3 bucket policies with `aws:SourceIp`, or
+ * VPC-only endpoints). Beam moves data through many workers on different networks, so
+ * restricted credentials make the transfer fail.
+ */
 export interface S3ProviderConfig {
+    /** Physical storage location; independent of the signing region. */
+    storage_location?: string;
     provider: "s3";
     id?: string;
     bucket: string;
@@ -243,7 +286,17 @@ export interface S3ProviderConfig {
     session_token?: string;
     endpoint_url?: string;
 }
+/**
+ * Cloudflare R2 source or destination.
+ *
+ * Credentials must not be restricted to specific IP addresses or networks (for example
+ * Cloudflare R2 API-token client IP filtering, S3 bucket policies with `aws:SourceIp`, or
+ * VPC-only endpoints). Beam moves data through many workers on different networks, so
+ * restricted credentials make the transfer fail.
+ */
 export interface R2ProviderConfig {
+    /** Physical storage location; independent of the signing region. */
+    storage_location?: string;
     provider: "r2";
     id?: string;
     bucket: string;
@@ -253,7 +306,17 @@ export interface R2ProviderConfig {
     account_id?: string;
     endpoint_url?: string;
 }
+/**
+ * S3-compatible source or destination (MinIO, Wasabi, Backblaze B2, DigitalOcean Spaces and others).
+ *
+ * Credentials must not be restricted to specific IP addresses or networks (for example
+ * Cloudflare R2 API-token client IP filtering, S3 bucket policies with `aws:SourceIp`, or
+ * VPC-only endpoints). Beam moves data through many workers on different networks, so
+ * restricted credentials make the transfer fail.
+ */
 export interface S3CompatibleProviderConfig {
+    /** Physical storage location; independent of the signing region. */
+    storage_location?: string;
     provider: string;
     driver?: "s3-compatible";
     id?: string;
@@ -267,7 +330,17 @@ export interface S3CompatibleProviderConfig {
     force_path_style?: boolean;
     account_id?: string;
 }
+/**
+ * Hippius source or destination.
+ *
+ * Credentials must not be restricted to specific IP addresses or networks (for example
+ * Cloudflare R2 API-token client IP filtering, S3 bucket policies with `aws:SourceIp`, or
+ * VPC-only endpoints). Beam moves data through many workers on different networks, so
+ * restricted credentials make the transfer fail.
+ */
 export interface HippiusProviderConfig {
+    /** Physical storage location; independent of the signing region. */
+    storage_location?: string;
     provider: "hippius";
     id?: string;
     bucket: string;
@@ -276,7 +349,17 @@ export interface HippiusProviderConfig {
     base_url?: string;
 }
 export type HuggingFaceRepoType = "model" | "dataset" | "space" | "kernel" | "bucket";
+/**
+ * Hugging Face Hub source or destination.
+ *
+ * The token and any credentials must not be restricted to specific IP addresses or networks (for example
+ * Cloudflare R2 API-token client IP filtering, S3 bucket policies with `aws:SourceIp`, or
+ * VPC-only endpoints). Beam moves data through many workers on different networks, so
+ * restricted credentials make the transfer fail.
+ */
 export interface HuggingFaceProviderConfig {
+    /** Physical storage location; independent of the signing region. */
+    storage_location?: string;
     provider: "huggingface";
     id?: string;
     /** Namespace and repo or bucket name separated by a slash, for example `org/dataset`. */
@@ -328,13 +411,19 @@ export declare const HuggingFaceProviderConfig: Readonly<{
 export interface ProviderTransferCreateInput {
     /** Ownership fence. Aborting stops signing/replay without cancelling a replacement owner. */
     signal?: AbortSignal;
+    /**
+     * Storage to read from. Source credentials must not be restricted to specific IP addresses or
+     * networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
+     * `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
+     * networks, so restricted credentials make the transfer fail.
+     */
     sources: ProviderSourceConfig[];
-    destinations: ProviderDestinationConfig[];
-    name?: string;
     /**
-     * Maps to BeamCore test_mode.
+     * Storage to write to. Destination credentials must not be restricted to specific IP addresses
+     * or networks, for the same reason as `sources`.
      */
-    testMode?: boolean;
+    destinations: ProviderDestinationConfig[];
+    name?: string;
     expiresIn?: number;
     /**
      * Defaults to true. Set to false to only prepare and stream signed routes without distribution.
@@ -368,6 +457,11 @@ export interface ProviderMultipartGroupIdentity {
     expectedPartCount: number;
     expiresAt: string;
 }
+/**
+ * HTTP source for planning and preparing. Its URL and headers must work from any network: Beam
+ * moves data through many workers on different networks, so IP- or network-restricted URLs make
+ * the transfer fail.
+ */
 export interface PlanningHttpSource {
     source_id: string;
     type: "http";
@@ -479,7 +573,6 @@ export interface TransferPrepareResponse {
     success: boolean;
     transfer_id: string;
     transfer_key?: string;
-    test_mode?: boolean;
     chunk_size?: number;
     total_size?: number;
     total_sources?: number;

package/dist/nats-control.d.ts

@@ -135,6 +135,7 @@ export declare class BeamTransferControl {
     private readonly recoveryOperations;
     private readonly recoveryRequestedEpoch;
     private readonly runtimeEpochs;
+    private readonly runtimeCapabilities;
     private helloTimer;
     constructor(options: TransferControlOptions);
     close(): Promise<void>;
@@ -155,6 +156,7 @@ export declare class BeamTransferControl {
     splitRoutesForPayload(messageType: TransferClientMessageType, basePayload: Record<string, unknown>, routes: SignedChunkRoute[]): SignedChunkRoute[][];
     private connection;
     private ensureHelloMonitor;
+    supportsPerformanceV2(transferId: string): boolean;
     private pollRuntimeHello;
     private observeRuntimeEpoch;
     private recoverLease;

package/dist/nats-control.js

@@ -1,3 +1,4 @@
+import { currentSdkPerformance } from "./performance.js";
 import { encode, decode } from "@msgpack/msgpack";
 import { connect } from "nats";
 export const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/v7";
@@ -65,6 +66,7 @@ export class BeamTransferControl {
     recoveryOperations = new Map();
     recoveryRequestedEpoch = new Map();
     runtimeEpochs = new Map();
+    runtimeCapabilities = new Map();
     helloTimer = null;
     constructor(options) {
         this.apiKey = options.apiKey;
@@ -277,7 +279,12 @@ export class BeamTransferControl {
                     producer: "sdk",
                     payload
                 };
+                const encodeStarted = performance.now();
                 const bytes = encode(envelope);
+                if (messageType === "transfer.route_stream.batch") {
+                    currentSdkPerformance()?.observe("sdk.batch_encode", performance.now() - encodeStarted);
+                    currentSdkPerformance()?.gauge("batch_bytes_max", bytes.byteLength);
+                }
                 if (bytes.byteLength > this.maxPayloadBytes) {
                     throw new Error(`NATS lifecycle request is ${bytes.byteLength} bytes, above maxPayloadBytes=${this.maxPayloadBytes}`);
                 }
@@ -488,10 +495,15 @@ export class BeamTransferControl {
         }, 5_000);
         this.helloTimer.unref?.();
     }
+    supportsPerformanceV2(transferId) {
+        const value = this.runtimeCapabilities.get(transferShardId(transferId, this.shardCount));
+        return !!value && value.until > Date.now() && value.capabilities.includes("sdk-performance/v2");
+    }
     async pollRuntimeHello(shardId) {
         try {
             try {
-                await this.request("runtime.hello", { capabilities: ["integrity-signing/v1"] }, { idempotencyKey: `runtime:hello:${shardId}:integrity-signing`, shardId });
+                const hello = await this.request("runtime.hello", { capabilities: ["integrity-signing/v1", "sdk-performance/v2"] }, { idempotencyKey: `runtime:hello:${shardId}:integrity-signing`, shardId });
+                this.runtimeCapabilities.set(shardId, { until: Date.now() + 15_000, capabilities: hello.capabilities ?? [] });
             }
             catch {
                 await this.request("runtime.hello", {}, { idempotencyKey: `runtime:hello:${shardId}`, shardId });
@@ -510,6 +522,7 @@ export class BeamTransferControl {
             return;
         if (previous.runtimeEpoch === runtimeEpoch && previous.transportEpoch === transportEpoch)
             return;
+        this.runtimeCapabilities.delete(shardId);
         if (previous.runtimeEpoch !== runtimeEpoch) {
             this.authToken = null;
             this.authTokenExpiresAt = 0;

package/dist/performance.d.ts

@@ -1,12 +1,39 @@
 import type { SdkPerformanceSummary } from './models.js';
+export declare const currentSdkPerformance: () => SdkPerformanceCollector | undefined;
+export declare const measureSdkPhase: <T>(name: string, work: () => Promise<T>) => Promise<T>;
+/** Bounded identities only: no URLs, credentials or grant bodies. */
+export declare class SourceSignatureHistory {
+    complete: boolean;
+    private readonly bits;
+    constructor(complete?: boolean);
+    created(index: number): boolean;
+}
 export declare class SdkPerformanceCollector {
+    private readonly sourceHistory?;
+    constructor(sourceHistory?: SourceSignatureHistory | undefined);
+    private sourceRenewals;
+    sourceCreated(index: number): void;
+    private readonly started;
+    private readonly details;
+    private readonly cpu;
+    private finished;
+    private delayTimer?;
     private readonly measurements;
+    private readonly gauges;
+    private readonly milestones;
+    private dropped;
     readonly counters: {
         source_signatures: number;
         source_reuses: number;
         route_batches: number;
     };
-    observe(name: SdkPerformanceSummary['measurements'][number]['name'], milliseconds: number): void;
-    measure<T>(name: SdkPerformanceSummary['measurements'][number]['name'], work: () => Promise<T>): Promise<T>;
-    snapshot(): SdkPerformanceSummary;
+    observe(name: string, milliseconds: number): void;
+    gauge(name: string, value: number): void;
+    increment(name: string): void;
+    seedDiscovery(other: SdkPerformanceCollector): void;
+    mark(name: string): void;
+    measure<T>(name: string, work: () => Promise<T>): Promise<T>;
+    startDelaySampling(): void;
+    finish(): void;
+    snapshot(v2?: boolean): SdkPerformanceSummary;
 }

package/dist/performance.js

@@ -1,25 +1,124 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+const NAMES = ["sdk.discovery", "sdk.multipart_create", "sdk.signing", "sdk.batch_ack", "sdk.buffer_wait", "sdk.preparation", "sdk.source_signing", "sdk.destination_signing", "sdk.source_grant_wait", "sdk.multipart_ready_wait", "sdk.multipart_provider", "sdk.multipart_queue", "sdk.multipart_callback", "sdk.manifest_ack", "sdk.provider_client_setup", "sdk.metadata_request", "sdk.route_assembly", "sdk.batch_encode", "sdk.batch_checksum", "sdk.batch_queue", "sdk.signing_queue", "sdk.process_cpu", "sdk.event_loop_delay", "sdk.producer_wait"];
+const BOUNDS = [0.1, .5, 1, 2, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000, 120000];
+const GAUGES = new Set(["signing_configured_limit", "multipart_configured_limit", "signing_limit", "multipart_limit", "signing_active_peak", "signing_pending_peak", "multipart_active_peak", "buffered_batches_peak", "batch_routes_max", "batch_bytes_max", "concurrency_changes", "source_waiters", "provider_clients_created", "provider_clients_reused"]);
+const MILESTONES = new Set(["first_manifest_ms", "first_route_ms", "first_batch_ms", "final_flush_ms", "prepared_ms"]);
+const context = new AsyncLocalStorage();
+let activeDelaySamplers = 0;
+export const currentSdkPerformance = () => context.getStore();
+export const measureSdkPhase = (name, work) => context.getStore()?.measure(name, work) ?? work();
+/** Bounded identities only: no URLs, credentials or grant bodies. */
+export class SourceSignatureHistory {
+    complete;
+    bits = new Uint8Array(16_384);
+    constructor(complete = true) {
+        this.complete = complete;
+    }
+    created(index) {
+        if (!Number.isSafeInteger(index) || index < 0 || index >= this.bits.length * 8) {
+            this.complete = false;
+            return false;
+        }
+        const byte = index >>> 3, mask = 1 << (index & 7);
+        const renewed = (this.bits[byte] & mask) !== 0;
+        this.bits[byte] |= mask;
+        return renewed;
+    }
+}
 export class SdkPerformanceCollector {
+    sourceHistory;
+    constructor(sourceHistory) {
+        this.sourceHistory = sourceHistory;
+    }
+    sourceRenewals = 0;
+    sourceCreated(index) {
+        this.counters.source_signatures++;
+        if (this.sourceHistory?.created(index))
+            this.sourceRenewals++;
+    }
+    started = performance.now();
+    details = process.env.BEAM_SDK_PERFORMANCE_DETAILS !== 'false';
+    cpu = process.cpuUsage();
+    finished = false;
+    delayTimer;
     measurements = new Map();
+    gauges = {};
+    milestones = {};
+    dropped = 0;
     counters = { source_signatures: 0, source_reuses: 0, route_batches: 0 };
     observe(name, milliseconds) {
-        if (!Number.isFinite(milliseconds) || milliseconds < 0)
+        const index = NAMES.indexOf(name);
+        if (index < 0 || (!this.details && index >= 6) || !Number.isFinite(milliseconds) || milliseconds < 0 || milliseconds > 86_400_000)
+            return;
+        const value = this.measurements.get(name) ?? { count: 0, work_ms: 0, max_ms: 0, histogram: Array(18).fill(0) };
+        if (value.count >= 1_000_000 || value.work_ms + milliseconds > 86_400_000) {
+            this.dropped++;
             return;
-        const value = this.measurements.get(name) ?? { count: 0, work_ms: 0, max_ms: 0 };
+        }
         value.count++;
         value.work_ms += milliseconds;
         value.max_ms = Math.max(value.max_ms, milliseconds);
+        const bin = BOUNDS.findIndex(bound => milliseconds <= bound);
+        value.histogram[bin < 0 ? 17 : bin]++;
         this.measurements.set(name, value);
     }
+    gauge(name, value) {
+        if (this.details && GAUGES.has(name) && Number.isFinite(value) && value >= 0)
+            this.gauges[name] = Math.min(1_000_000_000, Math.max(this.gauges[name] ?? 0, value));
+    }
+    increment(name) { this.gauge(name, (this.gauges[name] ?? 0) + 1); }
+    seedDiscovery(other) {
+        for (const name of ["sdk.discovery", "sdk.metadata_request", "sdk.provider_client_setup"]) {
+            const value = other.measurements.get(name);
+            if (value)
+                this.measurements.set(name, { ...value, histogram: [...value.histogram] });
+        }
+        for (const name of ["provider_clients_created", "provider_clients_reused"])
+            if (other.gauges[name] !== undefined)
+                this.gauges[name] = other.gauges[name];
+    }
+    mark(name) {
+        if (this.details && MILESTONES.has(name) && this.milestones[name] === undefined)
+            this.milestones[name] = performance.now() - this.started;
+    }
     async measure(name, work) {
         const start = performance.now();
         try {
-            return await work();
+            return await context.run(this, work);
         }
         finally {
             this.observe(name, performance.now() - start);
         }
     }
-    snapshot() {
-        return { schema_version: 'sdk-performance/v1', measurements: [...this.measurements].map(([name, value]) => ({ name: name, ...value })), counters: { ...this.counters } };
+    startDelaySampling() {
+        if (!this.details || this.delayTimer || this.finished || activeDelaySamplers >= 16)
+            return;
+        activeDelaySamplers++;
+        let last = performance.now();
+        this.delayTimer = setInterval(() => {
+            const now = performance.now();
+            this.observe('sdk.event_loop_delay', Math.max(0, now - last - 100));
+            last = now;
+        }, 100);
+        this.delayTimer.unref();
+    }
+    finish() {
+        if (this.finished)
+            return;
+        this.finished = true;
+        if (this.delayTimer) {
+            clearInterval(this.delayTimer);
+            this.delayTimer = undefined;
+            activeDelaySamplers--;
+        }
+        const used = process.cpuUsage(this.cpu);
+        this.observe('sdk.process_cpu', (used.user + used.system) / 1000);
+        this.mark('prepared_ms');
+    }
+    snapshot(v2 = true) {
+        const measurements = [...this.measurements].filter(([name]) => v2 || NAMES.slice(0, 6).includes(name))
+            .map(([name, value]) => ({ name, count: value.count, work_ms: value.work_ms, max_ms: value.max_ms, ...(v2 ? { histogram: [...value.histogram] } : {}) }));
+        return { schema_version: v2 ? 'sdk-performance/v2' : 'sdk-performance/v1', measurements, counters: { ...this.counters, ...(v2 && this.sourceHistory?.complete ? { source_renewals: this.sourceRenewals } : {}) },
+            ...(v2 ? { gauges: { ...this.gauges }, milestones: { ...this.milestones }, unmeasured: NAMES.slice(6).filter(name => !this.measurements.has(name)), detail_dropped: this.dropped } : {}) };
     }
 }

package/dist/provider-signing.js

@@ -1,3 +1,4 @@
+import { currentSdkPerformance, measureSdkPhase } from "./performance.js";
 import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, GetObjectCommand, HeadObjectCommand, ListPartsCommand, PutObjectCommand, S3Client, UploadPartCommand } from "@aws-sdk/client-s3";
 import { UploadPartCopyCommand, DeleteObjectCommand, ListObjectsV2Command } from "@aws-sdk/client-s3";
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
@@ -82,7 +83,7 @@ export async function prepareProviderSource(source, options = {}) {
     if (isS3CompatibleProvider(source)) {
         const endpoint = s3CompatibleEndpoint(source);
         const client = createS3CompatibleClient(source, endpoint);
-        const head = await client.send(new HeadObjectCommand({ Bucket: source.bucket, Key: source.key }), { abortSignal: options.signal });
+        const head = await measureSdkPhase("sdk.metadata_request", () => client.send(new HeadObjectCommand({ Bucket: source.bucket, Key: source.key }), { abortSignal: options.signal }));
         const getUrl = await getSignedUrl(client, new GetObjectCommand({ Bucket: source.bucket, Key: source.key }), { expiresIn });
         return {
             source_id: source.id ?? `src_${index}`,
@@ -113,7 +114,7 @@ export async function prepareProviderSource(source, options = {}) {
             size,
             filename: filename(source.key),
             expires_at: expiresAtIso(expiresIn),
-            metadata: { bucket: source.bucket, key: source.key, base_url: baseUrl }
+            metadata: compactMetadata({ bucket: source.bucket, key: source.key, base_url: baseUrl, storage_location: source.storage_location })
         };
     }
     if (isHuggingFaceProvider(source)) {
@@ -161,7 +162,7 @@ export async function prepareProviderSourceForPlan(source, options = {}) {
             provider: "hippius",
             size,
             filename: filename(source.key),
-            metadata: { bucket: source.bucket, key: source.key, base_url: baseUrl }
+            metadata: compactMetadata({ bucket: source.bucket, key: source.key, base_url: baseUrl, storage_location: source.storage_location })
         };
     }
     if (isHuggingFaceProvider(source)) {
@@ -199,6 +200,7 @@ export function prepareProviderDestination(destination, options = {}) {
             metadata: {
                 bucket: destination.bucket,
                 key: destination.key,
+                storage_location: destination.storage_location,
                 base_url: destination.base_url ?? "https://api.hippius.com"
             }
         };
@@ -336,14 +338,14 @@ export async function signDestinationRoute(input) {
     }
     const destinationExpiresAt = expiresAtIso(input.expiresIn);
     const [destUrl, sourceRoute] = await Promise.all([
-        input.destUrl ?? signDestinationUrl({
+        input.destUrl ?? measureSdkPhase("sdk.destination_signing", () => signDestinationUrl({
             destination: input.destination,
             objectKey: targetObjectKey,
             uploadId: input.uploadId,
             partNumber: input.partNumber,
             expiresIn: input.expiresIn,
             fetchImpl: input.fetchImpl,
-        }),
+        })),
         input.sourceGrant ?? signSourceChunk({
             source: input.source,
             fallbackUrl: input.chunk.source_url,
@@ -522,8 +524,11 @@ function s3ClientCacheKey(source, endpoint) {
 function createS3CompatibleClient(source, endpoint = s3CompatibleEndpoint(source)) {
     const cacheKey = s3ClientCacheKey(source, endpoint);
     const cached = s3ClientCache.get(source);
-    if (cached?.identity === cacheKey)
+    if (cached?.identity === cacheKey) {
+        currentSdkPerformance()?.increment("provider_clients_reused");
         return cached.client;
+    }
+    const setupStarted = performance.now();
     const client = new S3Client({
         region: s3CompatibleRegion(source),
         endpoint,
@@ -536,6 +541,8 @@ function createS3CompatibleClient(source, endpoint = s3CompatibleEndpoint(source
         }
     });
     s3ClientCache.set(source, { identity: cacheKey, client });
+    currentSdkPerformance()?.increment("provider_clients_created");
+    currentSdkPerformance()?.observe("sdk.provider_client_setup", performance.now() - setupStarted);
     return client;
 }
 export function isS3CompatibleProvider(config) {
@@ -586,6 +593,7 @@ function s3CompatibleMetadata(source, endpoint = s3CompatibleEndpoint(source)) {
         bucket: source.bucket,
         key: source.key,
         region: s3CompatibleRegion(source),
+        storage_location: source.storage_location,
         endpoint_url: endpoint,
         account_id: "account_id" in source ? source.account_id : undefined
     });
@@ -599,6 +607,7 @@ export function isHuggingFaceProvider(config) {
 function huggingFaceMetadata(config, etag, commitHash) {
     return compactMetadata({
         driver: "huggingface",
+        storage_location: config.storage_location,
         repo_id: config.repo_id,
         repo_type: huggingFaceRepoType(config),
         revision: huggingFaceRevision(config),

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@beam-network/sdk",
-    "version": "0.6.1",
+    "version": "0.8.0",
     "description": "TypeScript SDK for BEAM transfer creation and management.",
     "type": "module",
     "license": "MIT",