@beam-network/sdk 0.6.1 → 0.7.0

package/README.md

@@ -195,3 +195,24 @@ Multipart control helpers (`createMultipartUpload`, `listMultipartParts`, `compl
 Route streaming flushes at 1,024 routes, with no timer, and sends the final partial batch at stream completion. Signing overlaps acknowledged publication through bounded buffering; existing encoded-payload limits still apply. A source grant is reused across destinations only within the same signing generation and actual expiry.
 
 The optional `onDiagnostics` callback receives bounded preparation measurements and source-grant reuse counts. Callbacks are best effort, may be dropped under load, and do not affect transfer outcomes. Durations may overlap; do not sum them into elapsed transfer time. Transfer status also exposes a typed `performance` summary when supported by Core.
+
+Optional `storage_location` (`StorageLocation` in Go) describes the physical
+storage location. It is separate from the region used to sign provider requests.
+Leave it unset when unknown; a signing region such as R2's `auto` is not a location.
+
+When supported by Core, diagnostics use `sdk-performance/v2`: bounded histograms,
+preparation milestones, concurrency high-water marks, and separate provider,
+callback, manifest, signing, and transport waits. Histogram bins are noncumulative,
+with upper bounds in milliseconds of 0.1, 0.5, 1, 2, 5, 10, 25, 50, 100, 250,
+500, 1000, 2500, 5000, 10000, 30000, 120000, then overflow. `unmeasured` explicitly
+identifies unavailable measurements. Process CPU includes other concurrent work
+in the SDK process; it is not transfer-exclusive CPU. Detailed reporting can be
+disabled with `BEAM_SDK_PERFORMANCE_DETAILS=false`. Older peers retain v1 reports.
+No credentials or storage grants are included in these diagnostics.
+
+`sdk.producer_wait` measures waiting for a signed route separately from publication
+backpressure. Configured-limit gauges preserve starting limits; effective-limit
+gauges report the highest limit reached. Optional `source_renewals` counts source
+grants recreated in later preparation generations. A bounded 16 KiB history tracks
+131,072 chunk indices; the counter is omitted after takeover or beyond that bound.
+It does not include separate recovery-control signing operations.

package/dist/client.js

@@ -1,4 +1,4 @@
-import { SdkPerformanceCollector } from "./performance.js";
+import { SdkPerformanceCollector, SourceSignatureHistory, currentSdkPerformance, measureSdkPhase } from "./performance.js";
 import { signMultipartRecovery, abortMultipartUpload, createMultipartUpload, expiresAtIso, prepareProviderDestination, prepareProviderSource, releaseProviderClients, signAbortMultipartUpload, signCompleteMultipartUpload, signDestinationReadRange, signDestinationRoute, signFinalObjectHead, signListMultipartUpload, signSourceReadRange, signSourceChunk, boundedGrantExpiry, isHuggingFaceProvider } from "./provider-signing.js";
 import { describe as describeHuggingFace, hashSourceStream, huggingFaceCommit, huggingFaceCompleteLfsUpload, huggingFaceLfsBatch, huggingFacePreupload, huggingFaceVerifyLfsUpload, readSourceSample } from "./huggingface.js";
 import { BeamTransferControl, BEAM_DEFAULT_NATS_URL, compactSignedRoutes, isRecoverableRouteStreamError } from "./nats-control.js";
@@ -300,10 +300,9 @@ export class BeamClient {
             destinationsById.set(preparedDestination.destination_id, destination);
         });
         await input.throwIfCancelled?.();
-        const discoveryStarted = performance.now();
-        const preparedSources = await Promise.all(retainedRecoveryInput.sources.map((source, index) => prepareProviderSource(source, { index, expiresIn, fetchImpl: this.fetchImpl, signal: input.signal })));
+        const discoveryTelemetry = new SdkPerformanceCollector();
+        const preparedSources = await discoveryTelemetry.measure("sdk.discovery", () => Promise.all(retainedRecoveryInput.sources.map((source, index) => prepareProviderSource(source, { index, expiresIn, fetchImpl: this.fetchImpl, signal: input.signal }))));
         await input.throwIfCancelled?.();
-        const discoveryMs = performance.now() - discoveryStarted;
         const huggingFace = await this.planHuggingFaceUploads({
             sources: retainedRecoveryInput.sources,
             preparedSources,
@@ -353,13 +352,18 @@ export class BeamClient {
         const assertOwnership = () => input.signal?.throwIfAborted();
         const routeStreamLock = new AsyncMutex();
         const releaseInitialStream = await routeStreamLock.acquire();
+        const sourceSignatureHistory = new SourceSignatureHistory(!resume);
         const streamPreparedRoutes = async (routeGenerationId, recoveryReplay) => {
-            const telemetry = new SdkPerformanceCollector();
-            if (!recoveryReplay)
-                telemetry.observe("sdk.discovery", discoveryMs);
+            const telemetry = new SdkPerformanceCollector(sourceSignatureHistory);
+            telemetry.seedDiscovery(discoveryTelemetry);
+            telemetry.gauge("signing_configured_limit", this.routeSigningConcurrency);
+            telemetry.gauge("multipart_configured_limit", this.multipartControlConcurrency);
+            telemetry.gauge("signing_limit", this.routeSigningConcurrency);
+            telemetry.gauge("multipart_limit", this.multipartControlConcurrency);
             const pendingRoutes = new Set();
             let signingConcurrency = this.routeSigningConcurrency;
             let signedInWindow = 0;
+            let activeSigning = 0;
             let signingWindowStartedAt = performance.now();
             let routeStream = null;
             let routeStreamBeginAttempted = false;
@@ -420,14 +424,14 @@ export class BeamClient {
                         assertOwnership();
                         recoveryMultipartUploads.set(state.manifest.multipart_group_id, state);
                         validateMultipartGroupManifest([state.manifest], prepared.transfer_id);
-                        await input.onMultipartGroupReady?.(multipartGroupIdentity(prepared.transfer_id, state.manifest));
+                        await telemetry.measure("sdk.multipart_callback", async () => input.onMultipartGroupReady?.(multipartGroupIdentity(prepared.transfer_id, state.manifest)));
                         await routeStream.addManifestGroups([state.manifest]);
                         multipartGroupWaiters.get(state.manifest.multipart_group_id)?.resolve(state);
                     },
                     onGroupFailed: (groupId, error) => multipartGroupWaiters.get(groupId)?.reject(error)
                 }));
                 const streamNextCompletedRoute = async () => {
-                    const settled = await Promise.race([...pendingRoutes].map((pending) => pending.then((route) => ({ pending, route }))));
+                    const settled = await telemetry.measure("sdk.producer_wait", () => Promise.race([...pendingRoutes].map((pending) => pending.then((route) => ({ pending, route })))));
                     pendingRoutes.delete(settled.pending);
                     assertOwnership();
                     await routeStream.addRoute(settled.route);
@@ -436,6 +440,8 @@ export class BeamClient {
                         const elapsedMs = performance.now() - signingWindowStartedAt;
                         if (!this.routeSigningConcurrencyOverridden && elapsedMs > 4_000 && signingConcurrency < 256) {
                             signingConcurrency = Math.min(256, signingConcurrency * 2);
+                            telemetry.increment("concurrency_changes");
+                            telemetry.gauge("signing_limit", signingConcurrency);
                         }
                         signedInWindow = 0;
                         signingWindowStartedAt = performance.now();
@@ -445,10 +451,9 @@ export class BeamClient {
                     await throwIfCancelled?.(prepared.transfer_id);
                     // This promise belongs to this chunk and signing generation only. Its
                     // immutable credential/range identity and actual expiry cannot be renewed.
-                    const sourceGrant = signSourceChunk({ source: sourcesById.get(chunk.source_id), fallbackUrl: chunk.source_url, chunk, expiresIn, fetchImpl: this.fetchImpl });
+                    let sourceUses = 0;
+                    const sourceGrant = telemetry.measure("sdk.source_signing", () => signSourceChunk({ source: sourcesById.get(chunk.source_id), fallbackUrl: chunk.source_url, chunk, expiresIn, fetchImpl: this.fetchImpl })).then(grant => { telemetry.sourceCreated(chunk.chunk_index); return grant; });
                     void sourceGrant.catch(() => { });
-                    telemetry.counters.source_signatures++;
-                    telemetry.counters.source_reuses += Math.max(0, chunk.destinations.length - 1);
                     for (const target of chunk.destinations) {
                         const pendingRoute = (async () => {
                             await throwIfCancelled?.(prepared.transfer_id);
@@ -465,14 +470,16 @@ export class BeamClient {
                             const multipartGroupId = multipartGroupStateKey(prepared.transfer_id, target.destination_id, chunk.source_id, finalObjectKey);
                             const upload = isDirectPutDestination(destination)
                                 ? undefined
-                                : await multipartGroupWaiters.get(multipartGroupId)?.promise;
+                                : await telemetry.measure("sdk.multipart_ready_wait", async () => multipartGroupWaiters.get(multipartGroupId)?.promise);
                             if (!isDirectPutDestination(destination) && !upload)
                                 throw new Error(`multipart group manifest is missing for ${chunk.source_id}:${target.destination_id}`);
+                            telemetry.gauge("signing_active_peak", ++activeSigning);
                             return telemetry.measure("sdk.signing", () => this.signProviderRoute({
                                 chunk,
                                 target,
                                 source,
-                                sourceGrant,
+                                sourceGrant: telemetry.measure("sdk.source_grant_wait", () => sourceGrant).then(grant => { if (sourceUses++ > 0)
+                                    telemetry.counters.source_reuses++; return grant; }),
                                 destination,
                                 expiresIn,
                                 ...(upload ? { upload } : {}),
@@ -483,11 +490,12 @@ export class BeamClient {
                                 partNumber: typeof metadata.part_number === "number"
                                     ? metadata.part_number
                                     : multipartPartNumber(chunk.source_chunk_index),
-                            }));
+                            })).finally(() => { activeSigning--; });
                         })();
                         pendingRoutes.add(pendingRoute);
+                        telemetry.gauge("signing_pending_peak", pendingRoutes.size);
                         if (pendingRoutes.size >= signingConcurrency)
-                            await streamNextCompletedRoute();
+                            await telemetry.measure("sdk.signing_queue", streamNextCompletedRoute);
                     }
                 }
                 while (pendingRoutes.size)
@@ -1252,6 +1260,7 @@ class RouteStreamSender {
     routeCount = 0;
     sendTail = Promise.resolve();
     sendError = null;
+    sending = false;
     constructor(control, options) {
         this.control = control;
         this.options = options;
@@ -1259,6 +1268,7 @@ class RouteStreamSender {
         this.telemetry = options.telemetry ?? new SdkPerformanceCollector();
     }
     async begin() {
+        this.telemetry.startDelaySampling();
         await this.control.request("transfer.route_stream.begin", compact({
             transfer_id: this.options.transferId,
             route_generation_id: this.options.routeGenerationId,
@@ -1275,6 +1285,7 @@ class RouteStreamSender {
         });
     }
     async addManifestGroups(groups) {
+        const started = performance.now();
         if (!groups.length)
             return;
         await Promise.all(groups.map(async (group) => {
@@ -1290,8 +1301,12 @@ class RouteStreamSender {
                 idempotencyKey: `transfer:${this.options.transferId}:route-stream:${this.streamId}:manifest:${identity}`
             });
         }));
+        this.telemetry.observe("sdk.manifest_ack", performance.now() - started);
+        this.telemetry.mark("first_manifest_ms");
     }
     async addRoute(route) {
+        this.telemetry.mark("first_route_ms");
+        const assemblyStarted = performance.now();
         const deliveryIndex = signedRouteDeliveryIndex(route);
         if (deliveryIndex === undefined) {
             throw new Error("route delivery_index is required");
@@ -1305,6 +1320,8 @@ class RouteStreamSender {
         this.seenDeliveryIndices.add(deliveryIndex);
         this.routeCount += 1;
         this.batch.push({ ...route, delivery_index: deliveryIndex });
+        this.telemetry.gauge("buffered_batches_peak", 1 + Number(this.sending));
+        this.telemetry.observe("sdk.route_assembly", performance.now() - assemblyStarted);
         if (this.batch.length >= ROUTE_STREAM_BATCH_ROUTES) {
             await this.enqueueFlush();
         }
@@ -1314,12 +1331,14 @@ class RouteStreamSender {
             this.seenDeliveryIndices.size !== this.options.totalRoutes) {
             throw new Error(`route stream has incomplete or duplicate delivery indices: received ${this.routeCount} of ${this.options.totalRoutes} routes`);
         }
+        this.telemetry.mark("final_flush_ms");
         await this.enqueueFlush();
         await this.sendTail;
         if (this.sendError)
             throw this.sendError;
         this.telemetry.observe("sdk.preparation", performance.now() - this.started);
-        const sdkPerformance = this.telemetry.snapshot();
+        this.telemetry.finish();
+        const sdkPerformance = this.telemetry.snapshot(this.control.supportsPerformanceV2?.(this.options.transferId) ?? false);
         this.options.onDiagnostics?.(sdkPerformance);
         return this.control.request("transfer.route_stream.complete", {
             sdk_performance: sdkPerformance,
@@ -1341,6 +1360,7 @@ class RouteStreamSender {
         if (this.sendError)
             throw this.sendError;
         const routes = this.batch.splice(0, this.batch.length);
+        const encodingStarted = performance.now();
         const chunks = this.control.splitRoutesForPayload("transfer.route_stream.batch", {
             transfer_id: this.options.transferId,
             route_generation_id: this.options.routeGenerationId,
@@ -1348,19 +1368,27 @@ class RouteStreamSender {
             batch_id: `${this.streamId}:estimate`,
             batch_index: this.batchIndex
         }, routes);
+        this.telemetry.observe("sdk.batch_encode", performance.now() - encodingStarted);
+        this.sending = true;
         for (const chunk of chunks) {
+            const queuedAt = performance.now();
             const batchIndex = this.batchIndex;
             this.batchIndex += 1;
             this.sendTail = this.sendTail.then(async () => {
+                this.telemetry.observe("sdk.batch_queue", performance.now() - queuedAt);
                 if (this.sendError)
                     return;
                 try {
+                    const checksumStarted = performance.now();
                     const chunkChecksum = new RouteKeysChecksum();
                     await chunkChecksum.addMany(chunk.map(routeKeyForSignedRoute));
                     this.checksum.merge(chunkChecksum);
                     const routeChecksum = chunkChecksum.value();
                     const coordinateChecksum = await routeCoordinateChecksum(chunk);
                     const batchId = `${this.streamId}:${batchIndex}:${coordinateChecksum}`;
+                    this.telemetry.observe("sdk.batch_checksum", performance.now() - checksumStarted);
+                    this.telemetry.gauge("batch_routes_max", chunk.length);
+                    this.telemetry.mark("first_batch_ms");
                     this.telemetry.counters.route_batches++;
                     await this.telemetry.measure("sdk.batch_ack", () => this.control.request("transfer.route_stream.batch", {
                         transfer_id: this.options.transferId,
@@ -1381,8 +1409,10 @@ class RouteStreamSender {
                 }
             });
         }
+        this.sendTail = this.sendTail.finally(() => { this.sending = false; });
     }
     async abort() {
+        this.telemetry.finish();
         this.sendError ??= new Error("route_stream_aborted");
         await this.sendTail;
     }
@@ -1672,7 +1702,12 @@ function restoreProviderMultipartIdentities(prepared, destinations, identities,
 }
 async function createMultipartGroupManifest(input) {
     const groups = input.prepared.plan_descriptor.sources.flatMap((source) => input.prepared.plan_descriptor.destinations.map((destination) => ({ source, destination })));
+    const queuedAt = performance.now();
+    const telemetry = currentSdkPerformance();
+    let active = 0;
     const results = await mapOrderedWithConcurrency(groups, input.concurrency, async ({ source, destination }) => {
+        telemetry?.observe("sdk.multipart_queue", performance.now() - queuedAt);
+        telemetry?.gauge("multipart_active_peak", ++active);
         let multipartGroupId = null;
         try {
             const destinationConfig = input.destinationsById.get(destination.destination_id);
@@ -1690,12 +1725,12 @@ async function createMultipartGroupManifest(input) {
             const finalObjectMetadata = { "beam-transfer-id": input.prepared.transfer_id };
             input.signal?.throwIfAborted();
             const retainedUpload = input.multipartUploads.get(multipartGroupId);
-            const uploadId = retainedUpload?.uploadId ?? await createMultipartUpload({
+            const uploadId = retainedUpload?.uploadId ?? await measureSdkPhase("sdk.multipart_provider", () => createMultipartUpload({
                 destination: destinationConfig,
                 objectKey: finalObjectKey,
                 metadata: finalObjectMetadata,
                 signal: input.signal
-            });
+            }));
             const createdUpload = !retainedUpload;
             if (createdUpload) {
                 input.multipartUploads.set(multipartGroupId, {
@@ -1758,6 +1793,9 @@ async function createMultipartGroupManifest(input) {
                 input.onGroupFailed(multipartGroupId, error);
             return { ok: false, error };
         }
+        finally {
+            active--;
+        }
     });
     const failure = results.find((result) => !result.ok);
     if (failure && !failure.ok) {

package/dist/models.d.ts

@@ -1,16 +1,22 @@
 export type SignedUrlFlow = "signed_url";
 export interface SdkPerformanceSummary {
-    schema_version: "sdk-performance/v1";
+    gauges?: Record<string, number>;
+    milestones?: Record<string, number>;
+    unmeasured?: string[];
+    detail_dropped?: number;
+    schema_version: "sdk-performance/v1" | "sdk-performance/v2";
     measurements: Array<{
-        name: "sdk.discovery" | "sdk.multipart_create" | "sdk.signing" | "sdk.batch_ack" | "sdk.buffer_wait" | "sdk.preparation";
+        name: string;
         count: number;
         work_ms: number;
         max_ms: number;
+        histogram?: number[];
     }>;
     counters: {
         source_signatures: number;
         source_reuses: number;
         route_batches: number;
+        source_renewals?: number;
     };
 }
 export interface BeamClientOptions {
@@ -158,6 +164,21 @@ export interface IntegrityAuditChallenge {
     chunks: IntegrityAuditChallengeChunk[];
 }
 export interface TransferPerformance {
+    recovery_bandwidth?: {
+        excluded_tasks: number;
+        accepted_bytes: number;
+    };
+    deadline?: {
+        version: "assignment-deadline/v1";
+        decisions: number;
+        selected_min_ms: number;
+        selected_max_ms: number;
+        estimated_max_ms: number;
+        sample_count_max: number;
+        data_age_max_ms: number | null;
+        reasons: Record<string, number>;
+    };
+    sdk_detail?: SdkPerformanceSummary;
     schema_version: "transfer-performance/v1";
     runtime_epoch: string;
     coverage: "complete" | "restarted";
@@ -177,6 +198,7 @@ export interface TransferPerformance {
         source_signatures: number;
         source_reuses: number;
         route_batches: number;
+        source_renewals?: number;
     };
     detail_dropped: number;
     unmeasured?: string[];
@@ -233,6 +255,8 @@ export interface TransferTerminalSignalWaiter {
     close(): Promise<void>;
 }
 export interface S3ProviderConfig {
+    /** Physical storage location; independent of the signing region. */
+    storage_location?: string;
     provider: "s3";
     id?: string;
     bucket: string;
@@ -244,6 +268,8 @@ export interface S3ProviderConfig {
     endpoint_url?: string;
 }
 export interface R2ProviderConfig {
+    /** Physical storage location; independent of the signing region. */
+    storage_location?: string;
     provider: "r2";
     id?: string;
     bucket: string;
@@ -254,6 +280,8 @@ export interface R2ProviderConfig {
     endpoint_url?: string;
 }
 export interface S3CompatibleProviderConfig {
+    /** Physical storage location; independent of the signing region. */
+    storage_location?: string;
     provider: string;
     driver?: "s3-compatible";
     id?: string;
@@ -268,6 +296,8 @@ export interface S3CompatibleProviderConfig {
     account_id?: string;
 }
 export interface HippiusProviderConfig {
+    /** Physical storage location; independent of the signing region. */
+    storage_location?: string;
     provider: "hippius";
     id?: string;
     bucket: string;
@@ -277,6 +307,8 @@ export interface HippiusProviderConfig {
 }
 export type HuggingFaceRepoType = "model" | "dataset" | "space" | "kernel" | "bucket";
 export interface HuggingFaceProviderConfig {
+    /** Physical storage location; independent of the signing region. */
+    storage_location?: string;
     provider: "huggingface";
     id?: string;
     /** Namespace and repo or bucket name separated by a slash, for example `org/dataset`. */

package/dist/nats-control.d.ts

@@ -135,6 +135,7 @@ export declare class BeamTransferControl {
     private readonly recoveryOperations;
     private readonly recoveryRequestedEpoch;
     private readonly runtimeEpochs;
+    private readonly runtimeCapabilities;
     private helloTimer;
     constructor(options: TransferControlOptions);
     close(): Promise<void>;
@@ -155,6 +156,7 @@ export declare class BeamTransferControl {
     splitRoutesForPayload(messageType: TransferClientMessageType, basePayload: Record<string, unknown>, routes: SignedChunkRoute[]): SignedChunkRoute[][];
     private connection;
     private ensureHelloMonitor;
+    supportsPerformanceV2(transferId: string): boolean;
     private pollRuntimeHello;
     private observeRuntimeEpoch;
     private recoverLease;

package/dist/nats-control.js

@@ -1,3 +1,4 @@
+import { currentSdkPerformance } from "./performance.js";
 import { encode, decode } from "@msgpack/msgpack";
 import { connect } from "nats";
 export const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/v7";
@@ -65,6 +66,7 @@ export class BeamTransferControl {
     recoveryOperations = new Map();
     recoveryRequestedEpoch = new Map();
     runtimeEpochs = new Map();
+    runtimeCapabilities = new Map();
     helloTimer = null;
     constructor(options) {
         this.apiKey = options.apiKey;
@@ -277,7 +279,12 @@ export class BeamTransferControl {
                     producer: "sdk",
                     payload
                 };
+                const encodeStarted = performance.now();
                 const bytes = encode(envelope);
+                if (messageType === "transfer.route_stream.batch") {
+                    currentSdkPerformance()?.observe("sdk.batch_encode", performance.now() - encodeStarted);
+                    currentSdkPerformance()?.gauge("batch_bytes_max", bytes.byteLength);
+                }
                 if (bytes.byteLength > this.maxPayloadBytes) {
                     throw new Error(`NATS lifecycle request is ${bytes.byteLength} bytes, above maxPayloadBytes=${this.maxPayloadBytes}`);
                 }
@@ -488,10 +495,15 @@ export class BeamTransferControl {
         }, 5_000);
         this.helloTimer.unref?.();
     }
+    supportsPerformanceV2(transferId) {
+        const value = this.runtimeCapabilities.get(transferShardId(transferId, this.shardCount));
+        return !!value && value.until > Date.now() && value.capabilities.includes("sdk-performance/v2");
+    }
     async pollRuntimeHello(shardId) {
         try {
             try {
-                await this.request("runtime.hello", { capabilities: ["integrity-signing/v1"] }, { idempotencyKey: `runtime:hello:${shardId}:integrity-signing`, shardId });
+                const hello = await this.request("runtime.hello", { capabilities: ["integrity-signing/v1", "sdk-performance/v2"] }, { idempotencyKey: `runtime:hello:${shardId}:integrity-signing`, shardId });
+                this.runtimeCapabilities.set(shardId, { until: Date.now() + 15_000, capabilities: hello.capabilities ?? [] });
             }
             catch {
                 await this.request("runtime.hello", {}, { idempotencyKey: `runtime:hello:${shardId}`, shardId });
@@ -510,6 +522,7 @@ export class BeamTransferControl {
             return;
         if (previous.runtimeEpoch === runtimeEpoch && previous.transportEpoch === transportEpoch)
             return;
+        this.runtimeCapabilities.delete(shardId);
         if (previous.runtimeEpoch !== runtimeEpoch) {
             this.authToken = null;
             this.authTokenExpiresAt = 0;

package/dist/performance.d.ts

@@ -1,12 +1,39 @@
 import type { SdkPerformanceSummary } from './models.js';
+export declare const currentSdkPerformance: () => SdkPerformanceCollector | undefined;
+export declare const measureSdkPhase: <T>(name: string, work: () => Promise<T>) => Promise<T>;
+/** Bounded identities only: no URLs, credentials or grant bodies. */
+export declare class SourceSignatureHistory {
+    complete: boolean;
+    private readonly bits;
+    constructor(complete?: boolean);
+    created(index: number): boolean;
+}
 export declare class SdkPerformanceCollector {
+    private readonly sourceHistory?;
+    constructor(sourceHistory?: SourceSignatureHistory | undefined);
+    private sourceRenewals;
+    sourceCreated(index: number): void;
+    private readonly started;
+    private readonly details;
+    private readonly cpu;
+    private finished;
+    private delayTimer?;
     private readonly measurements;
+    private readonly gauges;
+    private readonly milestones;
+    private dropped;
     readonly counters: {
         source_signatures: number;
         source_reuses: number;
         route_batches: number;
     };
-    observe(name: SdkPerformanceSummary['measurements'][number]['name'], milliseconds: number): void;
-    measure<T>(name: SdkPerformanceSummary['measurements'][number]['name'], work: () => Promise<T>): Promise<T>;
-    snapshot(): SdkPerformanceSummary;
+    observe(name: string, milliseconds: number): void;
+    gauge(name: string, value: number): void;
+    increment(name: string): void;
+    seedDiscovery(other: SdkPerformanceCollector): void;
+    mark(name: string): void;
+    measure<T>(name: string, work: () => Promise<T>): Promise<T>;
+    startDelaySampling(): void;
+    finish(): void;
+    snapshot(v2?: boolean): SdkPerformanceSummary;
 }

package/dist/performance.js

@@ -1,25 +1,124 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+const NAMES = ["sdk.discovery", "sdk.multipart_create", "sdk.signing", "sdk.batch_ack", "sdk.buffer_wait", "sdk.preparation", "sdk.source_signing", "sdk.destination_signing", "sdk.source_grant_wait", "sdk.multipart_ready_wait", "sdk.multipart_provider", "sdk.multipart_queue", "sdk.multipart_callback", "sdk.manifest_ack", "sdk.provider_client_setup", "sdk.metadata_request", "sdk.route_assembly", "sdk.batch_encode", "sdk.batch_checksum", "sdk.batch_queue", "sdk.signing_queue", "sdk.process_cpu", "sdk.event_loop_delay", "sdk.producer_wait"];
+const BOUNDS = [0.1, .5, 1, 2, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000, 120000];
+const GAUGES = new Set(["signing_configured_limit", "multipart_configured_limit", "signing_limit", "multipart_limit", "signing_active_peak", "signing_pending_peak", "multipart_active_peak", "buffered_batches_peak", "batch_routes_max", "batch_bytes_max", "concurrency_changes", "source_waiters", "provider_clients_created", "provider_clients_reused"]);
+const MILESTONES = new Set(["first_manifest_ms", "first_route_ms", "first_batch_ms", "final_flush_ms", "prepared_ms"]);
+const context = new AsyncLocalStorage();
+let activeDelaySamplers = 0;
+export const currentSdkPerformance = () => context.getStore();
+export const measureSdkPhase = (name, work) => context.getStore()?.measure(name, work) ?? work();
+/** Bounded identities only: no URLs, credentials or grant bodies. */
+export class SourceSignatureHistory {
+    complete;
+    bits = new Uint8Array(16_384);
+    constructor(complete = true) {
+        this.complete = complete;
+    }
+    created(index) {
+        if (!Number.isSafeInteger(index) || index < 0 || index >= this.bits.length * 8) {
+            this.complete = false;
+            return false;
+        }
+        const byte = index >>> 3, mask = 1 << (index & 7);
+        const renewed = (this.bits[byte] & mask) !== 0;
+        this.bits[byte] |= mask;
+        return renewed;
+    }
+}
 export class SdkPerformanceCollector {
+    sourceHistory;
+    constructor(sourceHistory) {
+        this.sourceHistory = sourceHistory;
+    }
+    sourceRenewals = 0;
+    sourceCreated(index) {
+        this.counters.source_signatures++;
+        if (this.sourceHistory?.created(index))
+            this.sourceRenewals++;
+    }
+    started = performance.now();
+    details = process.env.BEAM_SDK_PERFORMANCE_DETAILS !== 'false';
+    cpu = process.cpuUsage();
+    finished = false;
+    delayTimer;
     measurements = new Map();
+    gauges = {};
+    milestones = {};
+    dropped = 0;
     counters = { source_signatures: 0, source_reuses: 0, route_batches: 0 };
     observe(name, milliseconds) {
-        if (!Number.isFinite(milliseconds) || milliseconds < 0)
+        const index = NAMES.indexOf(name);
+        if (index < 0 || (!this.details && index >= 6) || !Number.isFinite(milliseconds) || milliseconds < 0 || milliseconds > 86_400_000)
+            return;
+        const value = this.measurements.get(name) ?? { count: 0, work_ms: 0, max_ms: 0, histogram: Array(18).fill(0) };
+        if (value.count >= 1_000_000 || value.work_ms + milliseconds > 86_400_000) {
+            this.dropped++;
             return;
-        const value = this.measurements.get(name) ?? { count: 0, work_ms: 0, max_ms: 0 };
+        }
         value.count++;
         value.work_ms += milliseconds;
         value.max_ms = Math.max(value.max_ms, milliseconds);
+        const bin = BOUNDS.findIndex(bound => milliseconds <= bound);
+        value.histogram[bin < 0 ? 17 : bin]++;
         this.measurements.set(name, value);
     }
+    gauge(name, value) {
+        if (this.details && GAUGES.has(name) && Number.isFinite(value) && value >= 0)
+            this.gauges[name] = Math.min(1_000_000_000, Math.max(this.gauges[name] ?? 0, value));
+    }
+    increment(name) { this.gauge(name, (this.gauges[name] ?? 0) + 1); }
+    seedDiscovery(other) {
+        for (const name of ["sdk.discovery", "sdk.metadata_request", "sdk.provider_client_setup"]) {
+            const value = other.measurements.get(name);
+            if (value)
+                this.measurements.set(name, { ...value, histogram: [...value.histogram] });
+        }
+        for (const name of ["provider_clients_created", "provider_clients_reused"])
+            if (other.gauges[name] !== undefined)
+                this.gauges[name] = other.gauges[name];
+    }
+    mark(name) {
+        if (this.details && MILESTONES.has(name) && this.milestones[name] === undefined)
+            this.milestones[name] = performance.now() - this.started;
+    }
     async measure(name, work) {
         const start = performance.now();
         try {
-            return await work();
+            return await context.run(this, work);
         }
         finally {
             this.observe(name, performance.now() - start);
         }
     }
-    snapshot() {
-        return { schema_version: 'sdk-performance/v1', measurements: [...this.measurements].map(([name, value]) => ({ name: name, ...value })), counters: { ...this.counters } };
+    startDelaySampling() {
+        if (!this.details || this.delayTimer || this.finished || activeDelaySamplers >= 16)
+            return;
+        activeDelaySamplers++;
+        let last = performance.now();
+        this.delayTimer = setInterval(() => {
+            const now = performance.now();
+            this.observe('sdk.event_loop_delay', Math.max(0, now - last - 100));
+            last = now;
+        }, 100);
+        this.delayTimer.unref();
+    }
+    finish() {
+        if (this.finished)
+            return;
+        this.finished = true;
+        if (this.delayTimer) {
+            clearInterval(this.delayTimer);
+            this.delayTimer = undefined;
+            activeDelaySamplers--;
+        }
+        const used = process.cpuUsage(this.cpu);
+        this.observe('sdk.process_cpu', (used.user + used.system) / 1000);
+        this.mark('prepared_ms');
+    }
+    snapshot(v2 = true) {
+        const measurements = [...this.measurements].filter(([name]) => v2 || NAMES.slice(0, 6).includes(name))
+            .map(([name, value]) => ({ name, count: value.count, work_ms: value.work_ms, max_ms: value.max_ms, ...(v2 ? { histogram: [...value.histogram] } : {}) }));
+        return { schema_version: v2 ? 'sdk-performance/v2' : 'sdk-performance/v1', measurements, counters: { ...this.counters, ...(v2 && this.sourceHistory?.complete ? { source_renewals: this.sourceRenewals } : {}) },
+            ...(v2 ? { gauges: { ...this.gauges }, milestones: { ...this.milestones }, unmeasured: NAMES.slice(6).filter(name => !this.measurements.has(name)), detail_dropped: this.dropped } : {}) };
     }
 }

package/dist/provider-signing.js

@@ -1,3 +1,4 @@
+import { currentSdkPerformance, measureSdkPhase } from "./performance.js";
 import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, GetObjectCommand, HeadObjectCommand, ListPartsCommand, PutObjectCommand, S3Client, UploadPartCommand } from "@aws-sdk/client-s3";
 import { UploadPartCopyCommand, DeleteObjectCommand, ListObjectsV2Command } from "@aws-sdk/client-s3";
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
@@ -82,7 +83,7 @@ export async function prepareProviderSource(source, options = {}) {
     if (isS3CompatibleProvider(source)) {
         const endpoint = s3CompatibleEndpoint(source);
         const client = createS3CompatibleClient(source, endpoint);
-        const head = await client.send(new HeadObjectCommand({ Bucket: source.bucket, Key: source.key }), { abortSignal: options.signal });
+        const head = await measureSdkPhase("sdk.metadata_request", () => client.send(new HeadObjectCommand({ Bucket: source.bucket, Key: source.key }), { abortSignal: options.signal }));
         const getUrl = await getSignedUrl(client, new GetObjectCommand({ Bucket: source.bucket, Key: source.key }), { expiresIn });
         return {
             source_id: source.id ?? `src_${index}`,
@@ -113,7 +114,7 @@ export async function prepareProviderSource(source, options = {}) {
             size,
             filename: filename(source.key),
             expires_at: expiresAtIso(expiresIn),
-            metadata: { bucket: source.bucket, key: source.key, base_url: baseUrl }
+            metadata: compactMetadata({ bucket: source.bucket, key: source.key, base_url: baseUrl, storage_location: source.storage_location })
         };
     }
     if (isHuggingFaceProvider(source)) {
@@ -161,7 +162,7 @@ export async function prepareProviderSourceForPlan(source, options = {}) {
             provider: "hippius",
             size,
             filename: filename(source.key),
-            metadata: { bucket: source.bucket, key: source.key, base_url: baseUrl }
+            metadata: compactMetadata({ bucket: source.bucket, key: source.key, base_url: baseUrl, storage_location: source.storage_location })
         };
     }
     if (isHuggingFaceProvider(source)) {
@@ -199,6 +200,7 @@ export function prepareProviderDestination(destination, options = {}) {
             metadata: {
                 bucket: destination.bucket,
                 key: destination.key,
+                storage_location: destination.storage_location,
                 base_url: destination.base_url ?? "https://api.hippius.com"
             }
         };
@@ -336,14 +338,14 @@ export async function signDestinationRoute(input) {
     }
     const destinationExpiresAt = expiresAtIso(input.expiresIn);
     const [destUrl, sourceRoute] = await Promise.all([
-        input.destUrl ?? signDestinationUrl({
+        input.destUrl ?? measureSdkPhase("sdk.destination_signing", () => signDestinationUrl({
             destination: input.destination,
             objectKey: targetObjectKey,
             uploadId: input.uploadId,
             partNumber: input.partNumber,
             expiresIn: input.expiresIn,
             fetchImpl: input.fetchImpl,
-        }),
+        })),
         input.sourceGrant ?? signSourceChunk({
             source: input.source,
             fallbackUrl: input.chunk.source_url,
@@ -522,8 +524,11 @@ function s3ClientCacheKey(source, endpoint) {
 function createS3CompatibleClient(source, endpoint = s3CompatibleEndpoint(source)) {
     const cacheKey = s3ClientCacheKey(source, endpoint);
     const cached = s3ClientCache.get(source);
-    if (cached?.identity === cacheKey)
+    if (cached?.identity === cacheKey) {
+        currentSdkPerformance()?.increment("provider_clients_reused");
         return cached.client;
+    }
+    const setupStarted = performance.now();
     const client = new S3Client({
         region: s3CompatibleRegion(source),
         endpoint,
@@ -536,6 +541,8 @@ function createS3CompatibleClient(source, endpoint = s3CompatibleEndpoint(source
         }
     });
     s3ClientCache.set(source, { identity: cacheKey, client });
+    currentSdkPerformance()?.increment("provider_clients_created");
+    currentSdkPerformance()?.observe("sdk.provider_client_setup", performance.now() - setupStarted);
     return client;
 }
 export function isS3CompatibleProvider(config) {
@@ -586,6 +593,7 @@ function s3CompatibleMetadata(source, endpoint = s3CompatibleEndpoint(source)) {
         bucket: source.bucket,
         key: source.key,
         region: s3CompatibleRegion(source),
+        storage_location: source.storage_location,
         endpoint_url: endpoint,
         account_id: "account_id" in source ? source.account_id : undefined
     });
@@ -599,6 +607,7 @@ export function isHuggingFaceProvider(config) {
 function huggingFaceMetadata(config, etag, commitHash) {
     return compactMetadata({
         driver: "huggingface",
+        storage_location: config.storage_location,
         repo_id: config.repo_id,
         repo_type: huggingFaceRepoType(config),
         revision: huggingFaceRevision(config),

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@beam-network/sdk",
-    "version": "0.6.1",
+    "version": "0.7.0",
     "description": "TypeScript SDK for BEAM transfer creation and management.",
     "type": "module",
     "license": "MIT",