@beam-network/sdk 0.3.0 → 0.4.1

package/dist/client.d.ts

@@ -12,6 +12,7 @@ export declare class BeamClient {
     private readonly fetchImpl;
     private readonly control;
     private readonly routeSigningConcurrency;
+    private readonly routeSigningConcurrencyOverridden;
     constructor(options?: BeamClientOptions);
     close(): Promise<void>;
     openTransferTerminalWaiter(transferId: string): Promise<TransferTerminalSignalWaiter>;

package/dist/client.js

@@ -18,6 +18,7 @@ export class BeamClient {
     fetchImpl;
     control;
     routeSigningConcurrency;
+    routeSigningConcurrencyOverridden;
     constructor(options = {}) {
         if (!options.apiKey?.trim()) {
             throw new Error("apiKey is required.");
@@ -25,6 +26,7 @@ export class BeamClient {
         this.apiKey = options.apiKey;
         this.natsUrl = options.natsUrl ?? options.natsWsUrl ?? BEAM_DEFAULT_NATS_URL;
         this.routeSigningConcurrency = positiveInteger(options.routeSigningConcurrency ?? 64, "routeSigningConcurrency");
+        this.routeSigningConcurrencyOverridden = options.routeSigningConcurrency !== undefined;
         this.control = new BeamTransferControl({
             apiKey: this.apiKey,
             natsUrl: options.natsUrl,
@@ -148,12 +150,14 @@ export class BeamClient {
             totalRoutes: orderedRoutes.length,
             totalChunks: countDistinctRouteChunks(orderedRoutes),
             urlsExpiresAt: input.urlsExpiresAt,
-            multipartGroupManifest: input.multipartGroupManifest,
             autoDistribute: input.autoDistribute ?? true
         });
         await stream.begin();
+        if (input.multipartGroupManifest.length) {
+            await stream.addManifestGroups(input.multipartGroupManifest);
+        }
         for (const route of orderedRoutes) {
-            stream.addRoute(route);
+            await stream.addRoute(route);
         }
         return stream.complete();
     }
@@ -196,17 +200,14 @@ export class BeamClient {
         });
         const multipartUploads = new Map();
         const pendingRoutes = new Set();
+        let signingConcurrency = this.routeSigningConcurrency;
+        let signedInWindow = 0;
+        let signingWindowStartedAt = performance.now();
         let routeStream = null;
         let routeStreamBeginAttempted = false;
+        let multipartManifestTask = null;
+        let multipartGroupWaiters = new Map();
         try {
-            const multipartGroupManifest = await createMultipartGroupManifest({
-                prepared,
-                destinationsById,
-                multipartUploads,
-                expiresIn,
-                concurrency: this.routeSigningConcurrency
-            });
-            validateMultipartGroupManifest(multipartGroupManifest, prepared.transfer_id);
             const streamId = await stableRouteStreamId({
                 transferId: prepared.transfer_id,
                 planIdentity: prepared.plan_descriptor.plan_nonce,
@@ -218,15 +219,37 @@ export class BeamClient {
                 transferId: prepared.transfer_id,
                 totalRoutes: prepared.plan_descriptor.delivery_route_count,
                 totalChunks: prepared.plan_descriptor.logical_chunk_count,
-                multipartGroupManifest,
                 autoDistribute: input.distribute !== false
             });
             routeStreamBeginAttempted = true;
             await routeStream.begin();
+            multipartGroupWaiters = createMultipartGroupWaiters(prepared, destinationsById);
+            multipartManifestTask = createMultipartGroupManifest({
+                prepared,
+                destinationsById,
+                multipartUploads,
+                expiresIn,
+                concurrency: this.routeSigningConcurrency,
+                onGroupReady: async (state) => {
+                    validateMultipartGroupManifest([state.manifest], prepared.transfer_id);
+                    await routeStream.addManifestGroups([state.manifest]);
+                    multipartGroupWaiters.get(state.manifest.multipart_group_id)?.resolve(state);
+                },
+                onGroupFailed: (groupId, error) => multipartGroupWaiters.get(groupId)?.reject(error)
+            });
             const streamNextCompletedRoute = async () => {
                 const settled = await Promise.race([...pendingRoutes].map((pending) => pending.then((route) => ({ pending, route }))));
                 pendingRoutes.delete(settled.pending);
-                routeStream.addRoute(settled.route);
+                await routeStream.addRoute(settled.route);
+                signedInWindow += 1;
+                if (signedInWindow === ROUTE_STREAM_BATCH_ROUTES) {
+                    const elapsedMs = performance.now() - signingWindowStartedAt;
+                    if (!this.routeSigningConcurrencyOverridden && elapsedMs > 4_000 && signingConcurrency < 256) {
+                        signingConcurrency = Math.min(256, signingConcurrency * 2);
+                    }
+                    signedInWindow = 0;
+                    signingWindowStartedAt = performance.now();
+                }
             };
             for (const chunk of materializePlanChunks(prepared.plan_descriptor, prepared.transfer_id)) {
                 await input.throwIfCancelled?.(prepared.transfer_id);
@@ -243,10 +266,12 @@ export class BeamClient {
                         const finalObjectKey = typeof metadata.final_object_key === "string" ? metadata.final_object_key : target.object_key;
                         if (!finalObjectKey)
                             throw new Error("destination signing target is missing object_key");
-                        const upload = multipartUploads.get(multipartGroupStateKey(prepared.transfer_id, target.destination_id, chunk.source_id, finalObjectKey));
-                        if (!isHippiusDestination(destination) && !upload) {
+                        const multipartGroupId = multipartGroupStateKey(prepared.transfer_id, target.destination_id, chunk.source_id, finalObjectKey);
+                        const upload = isHippiusDestination(destination)
+                            ? undefined
+                            : await multipartGroupWaiters.get(multipartGroupId)?.promise;
+                        if (!isHippiusDestination(destination) && !upload)
                             throw new Error(`multipart group manifest is missing for ${chunk.source_id}:${target.destination_id}`);
-                        }
                         return this.signProviderRoute({
                             chunk,
                             target,
@@ -262,12 +287,14 @@ export class BeamClient {
                         });
                     })();
                     pendingRoutes.add(pendingRoute);
-                    if (pendingRoutes.size >= this.routeSigningConcurrency)
+                    if (pendingRoutes.size >= signingConcurrency)
                         await streamNextCompletedRoute();
                 }
             }
             while (pendingRoutes.size)
                 await streamNextCompletedRoute();
+            const multipartGroupManifest = await multipartManifestTask;
+            validateMultipartGroupManifest(multipartGroupManifest, prepared.transfer_id);
             const attached = await routeStream.complete();
             if (!attached.success) {
                 throw new Error(attached.error ?? attached.message ?? "route stream failed");
@@ -277,6 +304,8 @@ export class BeamClient {
             if (routeStream)
                 await routeStream.abort();
             await Promise.allSettled(pendingRoutes);
+            if (multipartManifestTask)
+                await Promise.allSettled([multipartManifestTask]);
             await cancelAndAbortProviderFailure({
                 client: this,
                 transferId: prepared.transfer_id,
@@ -423,16 +452,14 @@ export class BeamClient {
         }
     }
 }
-const ROUTE_STREAM_BOOTSTRAP_BATCH_ROUTES = 32;
-const ROUTE_STREAM_STEADY_BATCH_ROUTES = 256;
+const ROUTE_STREAM_BATCH_ROUTES = 2_048;
 class RouteStreamSender {
     control;
     options;
     streamId;
     checksum = new RouteKeysChecksum();
     batch = [];
-    pendingRoutes = new Map();
-    nextDeliveryIndex = 0;
+    seenDeliveryIndices = new Set();
     batchIndex = 0;
     routeCount = 0;
     sendTail = Promise.resolve();
@@ -449,7 +476,6 @@ class RouteStreamSender {
             total_routes: this.options.totalRoutes,
             total_chunks: this.options.totalChunks,
             route_contract_version: "signed_url_v2",
-            multipart_group_manifest: this.options.multipartGroupManifest,
             urls_expires_at: this.options.urlsExpiresAt,
             signed_url_flow: "signed_url_v2",
             auto_distribute: this.options.autoDistribute
@@ -458,7 +484,23 @@ class RouteStreamSender {
             idempotencyKey: `transfer:${this.options.transferId}:route-stream:${this.streamId}:begin`
         });
     }
-    addRoute(route) {
+    async addManifestGroups(groups) {
+        if (!groups.length)
+            return;
+        await Promise.all(groups.map(async (group) => {
+            const identity = await stableManifestBatchIdentity([group]);
+            await this.control.request("transfer.route_stream.manifest", {
+                transfer_id: this.options.transferId,
+                stream_id: this.streamId,
+                manifest_batch_id: identity,
+                groups: [group]
+            }, {
+                transferId: this.options.transferId,
+                idempotencyKey: `transfer:${this.options.transferId}:route-stream:${this.streamId}:manifest:${identity}`
+            });
+        }));
+    }
+    async addRoute(route) {
         const deliveryIndex = signedRouteDeliveryIndex(route);
         if (deliveryIndex === undefined) {
             throw new Error("route delivery_index is required");
@@ -466,31 +508,22 @@ class RouteStreamSender {
         if (!Number.isSafeInteger(deliveryIndex) || deliveryIndex < 0 || deliveryIndex >= this.options.totalRoutes) {
             throw new Error(`route delivery_index ${deliveryIndex} is outside the declared route stream`);
         }
-        if (deliveryIndex < this.nextDeliveryIndex || this.pendingRoutes.has(deliveryIndex)) {
+        if (this.seenDeliveryIndices.has(deliveryIndex)) {
             throw new Error(`duplicate route delivery_index ${deliveryIndex}`);
         }
-        this.pendingRoutes.set(deliveryIndex, route);
+        this.seenDeliveryIndices.add(deliveryIndex);
         this.routeCount += 1;
-        while (this.pendingRoutes.has(this.nextDeliveryIndex)) {
-            const next = this.pendingRoutes.get(this.nextDeliveryIndex);
-            this.pendingRoutes.delete(this.nextDeliveryIndex);
-            this.batch.push({ ...next, delivery_index: this.nextDeliveryIndex });
-            this.nextDeliveryIndex += 1;
-            const target = this.batchIndex === 0
-                ? ROUTE_STREAM_BOOTSTRAP_BATCH_ROUTES
-                : ROUTE_STREAM_STEADY_BATCH_ROUTES;
-            if (this.batch.length >= target) {
-                this.enqueueFlush();
-            }
+        this.batch.push({ ...route, delivery_index: deliveryIndex });
+        if (this.batch.length >= ROUTE_STREAM_BATCH_ROUTES) {
+            await this.enqueueFlush();
         }
     }
     async complete() {
         if (this.routeCount !== this.options.totalRoutes ||
-            this.pendingRoutes.size !== 0 ||
-            this.nextDeliveryIndex !== this.options.totalRoutes) {
-            throw new Error(`route stream has non-contiguous delivery indices: received ${this.routeCount} of ${this.options.totalRoutes} routes`);
+            this.seenDeliveryIndices.size !== this.options.totalRoutes) {
+            throw new Error(`route stream has incomplete or duplicate delivery indices: received ${this.routeCount} of ${this.options.totalRoutes} routes`);
         }
-        this.enqueueFlush();
+        await this.enqueueFlush();
         await this.sendTail;
         if (this.sendError)
             throw this.sendError;
@@ -505,9 +538,12 @@ class RouteStreamSender {
             idempotencyKey: `transfer:${this.options.transferId}:route-stream:${this.streamId}:complete`
         });
     }
-    enqueueFlush() {
+    async enqueueFlush() {
         if (!this.batch.length)
             return;
+        await this.sendTail;
+        if (this.sendError)
+            throw this.sendError;
         const routes = this.batch.splice(0, this.batch.length);
         const chunks = this.control.splitRoutesForPayload("transfer.route_stream.batch", {
             transfer_id: this.options.transferId,
@@ -604,6 +640,12 @@ async function stableRouteStreamId(input) {
     const value = hex(bytes);
     return `${value.slice(0, 8)}-${value.slice(8, 12)}-${value.slice(12, 16)}-${value.slice(16, 20)}-${value.slice(20)}`;
 }
+async function stableManifestBatchIdentity(groups) {
+    const identities = groups
+        .map((group) => `${group.multipart_group_id}:${group.source_id}:${group.destination_id}`)
+        .sort();
+    return hex((await sha256Bytes(identities.join("\n"))).slice(0, 16));
+}
 function countDistinctRouteChunks(routes) {
     return new Set(routes.map((route) => `${route.source_id}:${route.chunk_index}`)).size;
 }
@@ -704,7 +746,7 @@ async function cancelTransferAfterProviderFailure(client, transferId, cause) {
         }
     }
     catch (cancelError) {
-        throw new Error(`provider transfer failed (${errorMessage(cause)}) and transfer cancellation failed (${errorMessage(cancelError)})`);
+        throw new Error(`provider transfer failed (${safeErrorCode(cause)}) and transfer cancellation failed (${safeErrorCode(cancelError)})`);
     }
 }
 async function cancelAndAbortProviderFailure(input) {
@@ -746,12 +788,46 @@ async function cancelAndAbortProviderFailure(input) {
         throw new AggregateError([input.cause, cleanupError], `provider transfer failed and multipart cleanup failed for ${input.transferId}`);
     }
 }
-function errorMessage(error) {
-    return error instanceof Error ? error.message : String(error);
+function safeErrorCode(error) {
+    if (!(error instanceof Error))
+        return "UnknownError";
+    const record = error;
+    const code = typeof record.code === "string" && /^[A-Za-z0-9_.-]{1,64}$/.test(record.code)
+        ? record.code
+        : error.name;
+    const status = [record.status, record.statusCode]
+        .find((value) => Number.isInteger(value) && Number(value) >= 100 && Number(value) <= 599);
+    return status === undefined ? code : `${code}:status=${status}`;
+}
+function createDeferred() {
+    let resolve;
+    let reject;
+    const promise = new Promise((resolvePromise, rejectPromise) => {
+        resolve = resolvePromise;
+        reject = rejectPromise;
+    });
+    return { promise, resolve, reject };
+}
+function createMultipartGroupWaiters(prepared, destinationsById) {
+    const waiters = new Map();
+    for (const source of prepared.plan_descriptor.sources) {
+        for (const destination of prepared.plan_descriptor.destinations) {
+            const destinationConfig = destinationsById.get(destination.destination_id);
+            if (!destinationConfig || isHippiusDestination(destinationConfig))
+                continue;
+            const finalObjectKey = destination.final_object_keys[source.source_id];
+            if (!finalObjectKey)
+                throw new Error(`plan final object key not found: ${source.source_id}:${destination.destination_id}`);
+            const groupId = multipartGroupStateKey(prepared.transfer_id, destination.destination_id, source.source_id, finalObjectKey);
+            waiters.set(groupId, createDeferred());
+        }
+    }
+    return waiters;
 }
 async function createMultipartGroupManifest(input) {
     const groups = input.prepared.plan_descriptor.sources.flatMap((source) => input.prepared.plan_descriptor.destinations.map((destination) => ({ source, destination })));
     const results = await mapOrderedWithConcurrency(groups, input.concurrency, async ({ source, destination }) => {
+        let multipartGroupId = null;
         try {
             const destinationConfig = input.destinationsById.get(destination.destination_id);
             if (!destinationConfig) {
@@ -764,7 +840,7 @@ async function createMultipartGroupManifest(input) {
             if (!finalObjectKey) {
                 throw new Error(`plan final object key not found: ${source.source_id}:${destination.destination_id}`);
             }
-            const multipartGroupId = multipartGroupStateKey(input.prepared.transfer_id, destination.destination_id, source.source_id, finalObjectKey);
+            multipartGroupId = multipartGroupStateKey(input.prepared.transfer_id, destination.destination_id, source.source_id, finalObjectKey);
             const finalObjectMetadata = {
                 "beam-transfer-id": input.prepared.transfer_id,
                 "beam-multipart-group-id": multipartGroupId
@@ -825,6 +901,7 @@ async function createMultipartGroupManifest(input) {
                     manifest
                 };
                 input.multipartUploads.set(multipartGroupId, state);
+                await input.onGroupReady(state);
                 return { ok: true, state };
             }
             catch (error) {
@@ -838,6 +915,8 @@ async function createMultipartGroupManifest(input) {
             }
         }
         catch (error) {
+            if (multipartGroupId)
+                input.onGroupFailed(multipartGroupId, error);
             return { ok: false, error };
         }
     });

package/dist/models.d.ts

@@ -122,22 +122,30 @@ export interface TransferStatusInfo {
     started_at: string | null;
     completed_at: string | null;
     runtime?: boolean;
-    total_bytes?: number;
-    total_size?: number;
-    total_chunks?: number;
-    chunks_completed?: number;
-    total_destinations?: number;
-    destinations_completed?: number;
+    source_bytes_total: number;
+    delivery_bytes_total: number;
+    delivery_bytes_completed: number;
+    delivery_tasks_total: number;
+    delivery_tasks_completed: number;
+    destinations_total: number;
+    destinations_completed: number;
+    destination_progress: Array<{
+        destination_id: string;
+        delivery_bytes_total: number;
+        delivery_bytes_completed: number;
+        completion_verified: boolean;
+    }>;
     destination_groups?: {
         totalGroups: number;
         completedGroups: number;
         pendingGroups: number;
         totalDestinations: number;
         completedDestinations: number;
+        completedDestinationIds: string[];
     };
 }
 export interface TransferTerminalEvent {
-    schema_version: "transfer-client-control/v3";
+    schema_version: "transfer-client-control/v4";
     producer: "transfer-runtime";
     transfer_id: string;
     status: "completed" | "failed" | "cancelled";

package/dist/nats-control.d.ts

@@ -1,9 +1,9 @@
 import type { MultipartGroupManifest, SignedChunkRoute, TransferTerminalSignalWaiter } from "./models.js";
-export declare const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/v3";
+export declare const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/v4";
 export declare const BEAM_DEFAULT_NATS_URL = "tls://nats.b1m.ai:4222";
 export declare const BEAM_DEFAULT_NATS_WS_URL = "wss://nats.b1m.ai:443";
 export declare const BEAM_DEFAULT_MAX_PAYLOAD_BYTES: number;
-export type TransferClientMessageType = "transfer.plan" | "transfer.create" | "transfer.prepare" | "transfer.route_stream.begin" | "transfer.route_stream.batch" | "transfer.route_stream.complete" | "transfer.distribute" | "transfer.status" | "transfer.cancel";
+export type TransferClientMessageType = "transfer.plan" | "transfer.create" | "transfer.prepare" | "transfer.route_stream.begin" | "transfer.route_stream.manifest" | "transfer.route_stream.batch" | "transfer.route_stream.complete" | "transfer.distribute" | "transfer.status" | "transfer.cancel";
 export interface CompactSignedRouteBatch {
     source_chunks: Array<{
         source_ref: number;
@@ -32,11 +32,16 @@ export interface RouteStreamBeginPayload {
     total_routes: number;
     total_chunks: number;
     route_contract_version: "signed_url_v2";
-    multipart_group_manifest: MultipartGroupManifest[];
     urls_expires_at?: string;
     signed_url_flow: "signed_url_v2";
     auto_distribute: boolean;
 }
+export interface RouteStreamManifestPayload {
+    transfer_id: string;
+    stream_id: string;
+    manifest_batch_id: string;
+    groups: MultipartGroupManifest[];
+}
 export interface TransferControlOptions {
     apiKey: string;
     natsUrl?: string;

package/dist/nats-control.js

@@ -1,6 +1,6 @@
 import { encode, decode } from "@msgpack/msgpack";
 import { connect } from "nats";
-export const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/v3";
+export const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/v4";
 export const BEAM_DEFAULT_NATS_URL = "tls://nats.b1m.ai:4222";
 export const BEAM_DEFAULT_NATS_WS_URL = "wss://nats.b1m.ai:443";
 // NATS enforces max_payload per message. This guard splits signed-route control
@@ -214,7 +214,7 @@ export class BeamTransferControl {
         }
     }
     splitRoutesForPayload(messageType, basePayload, routes) {
-        const split = (candidate) => {
+        const encodedBytes = (candidate) => {
             const encoded = encode({
                 schema_version: TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION,
                 environment: this.environment,
@@ -227,15 +227,31 @@ export class BeamTransferControl {
                 producer: "sdk",
                 payload: { ...basePayload, route_batch: compactSignedRoutes(candidate) }
             });
-            if (encoded.byteLength <= this.maxPayloadBytes)
-                return [candidate];
-            if (candidate.length === 1) {
-                throw new Error(`single signed route is ${encoded.byteLength} bytes, above maxPayloadBytes=${this.maxPayloadBytes}`);
-            }
-            const midpoint = Math.ceil(candidate.length / 2);
-            return [...split(candidate.slice(0, midpoint)), ...split(candidate.slice(midpoint))];
+            return encoded.byteLength;
         };
-        return routes.length ? split(routes) : [];
+        const chunks = [];
+        let offset = 0;
+        while (offset < routes.length) {
+            let low = 1;
+            let high = routes.length - offset;
+            let accepted = 0;
+            while (low <= high) {
+                const count = Math.floor((low + high) / 2);
+                if (encodedBytes(routes.slice(offset, offset + count)) <= this.maxPayloadBytes) {
+                    accepted = count;
+                    low = count + 1;
+                }
+                else
+                    high = count - 1;
+            }
+            if (accepted === 0) {
+                const bytes = encodedBytes(routes.slice(offset, offset + 1));
+                throw new Error(`single signed route is ${bytes} bytes, above maxPayloadBytes=${this.maxPayloadBytes}`);
+            }
+            chunks.push(routes.slice(offset, offset + accepted));
+            offset += accepted;
+        }
+        return chunks;
     }
     async connection() {
         if (this.closed)

package/dist/provider-signing.js

@@ -384,7 +384,8 @@ function s3CompatibleMetadata(source, endpoint = s3CompatibleEndpoint(source)) {
         bucket: source.bucket,
         key: source.key,
         region: s3CompatibleRegion(source),
-        endpoint_url: endpoint
+        endpoint_url: endpoint,
+        account_id: "account_id" in source ? source.account_id : undefined
     });
 }
 function isHippiusProvider(config) {
@@ -397,7 +398,7 @@ async function hippiusPresign(fetchImpl, baseUrl, token, bucket, key, action, ex
     url.searchParams.set("expires_in", String(expiresIn));
     const response = await fetchImpl(url, { headers: { Authorization: `Token ${token}` } });
     if (!response.ok) {
-        throw new Error(`Hippius presigned URL failed with ${response.status}: ${await response.text()}`);
+        throw new Error(`Hippius presigned URL failed status=${response.status}`);
     }
     return String((await response.json()).url);
 }
@@ -407,12 +408,12 @@ async function hippiusObjectSize(fetchImpl, baseUrl, token, bucket, key) {
     url.searchParams.set("max_keys", "1");
     const response = await fetchImpl(url, { headers: { Authorization: `Token ${token}` } });
     if (!response.ok) {
-        throw new Error(`Hippius object lookup failed with ${response.status}: ${await response.text()}`);
+        throw new Error(`Hippius object lookup failed status=${response.status}`);
     }
     const payload = await response.json();
     const contents = payload.Contents ?? payload.contents ?? [];
     if (!contents.length) {
-        throw new Error(`Object not found: hippius://${bucket}/${key}`);
+        throw new Error("Hippius object not found");
     }
     return Number(contents[0].Size ?? contents[0].size);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@beam-network/sdk",
-    "version": "0.3.0",
+    "version": "0.4.1",
     "description": "TypeScript SDK for BEAM transfer creation and management.",
     "type": "module",
     "license": "MIT",