@beam-network/sdk 0.2.20 → 0.2.22

package/README.md

@@ -95,6 +95,6 @@ For low-level raw transfer configs, use `createRawTransfer`; lifecycle transport
 
 ## Route Streaming And Payload Size
 
-Provider transfers stream signed routes over NATS in target batches of 128 routes. Signing continues while completed batches are sent in order. The SDK still splits a batch earlier when the encoded MessagePack request would exceed `maxPayloadBytes`, which defaults to 24 MiB.
+Provider transfers materialize the compact Runtime plan lazily, sign up to 64 routes concurrently by default, and stream a 256-route bootstrap batch followed by 2,048-route steady batches. Set `routeSigningConcurrency` on `BeamClientOptions` to tune the signing bound. Completed batches are sent FIFO while signing continues. Shared route fields use compact dictionaries, and encoded MessagePack requests split under `maxPayloadBytes`, which defaults to 24 MiB.
 
 NATS requires this guard because the broker rejects messages above its configured `max_payload`. This limit applies only to lifecycle/control metadata; transfer file bytes do not flow through NATS.

package/dist/client.d.ts

@@ -11,6 +11,7 @@ export declare class BeamClient {
     readonly natsUrl: string;
     private readonly fetchImpl;
     private readonly control;
+    private readonly routeSigningConcurrency;
     private readonly recoverySigners;
     constructor(options?: BeamClientOptions);
     close(): Promise<void>;

package/dist/client.js

@@ -1,5 +1,6 @@
 import { abortMultipartUpload, createMultipartUpload, nowIso, prepareProviderDestination, prepareProviderSource, listPartsPageMarker, listPartsVerifyMarker, signAbortMultipartUpload, signCompleteMultipartUpload, signDeleteObject, signDestinationRoute, signListMultipartUpload, signListObjects, signUploadPartCopy } from "./provider-signing.js";
-import { BeamTransferControl, BEAM_DEFAULT_NATS_URL } from "./nats-control.js";
+import { BeamTransferControl, BEAM_DEFAULT_NATS_URL, compactSignedRoutes } from "./nats-control.js";
+import { directMultipartInitialPartNumber } from "./multipart-limits.js";
 export { BEAM_DEFAULT_NATS_URL };
 export class BeamApiError extends Error {
     status;
@@ -16,6 +17,7 @@ export class BeamClient {
     natsUrl;
     fetchImpl;
     control;
+    routeSigningConcurrency;
     recoverySigners = new Map();
     constructor(options = {}) {
         if (!options.apiKey?.trim()) {
@@ -23,6 +25,7 @@ export class BeamClient {
         }
         this.apiKey = options.apiKey;
         this.natsUrl = options.natsUrl ?? options.natsWsUrl ?? BEAM_DEFAULT_NATS_URL;
+        this.routeSigningConcurrency = positiveInteger(options.routeSigningConcurrency ?? 64, "routeSigningConcurrency");
         this.control = new BeamTransferControl({
             apiKey: this.apiKey,
             natsUrl: options.natsUrl,
@@ -156,12 +159,13 @@ export class BeamClient {
         const signedUrlFlow = prepared.signed_url_flow ?? requestedSignedUrlFlow;
         const routeStream = new RouteStreamSender(this.control, {
             transferId: prepared.transfer_id,
-            totalRoutes: countRoutesInChunkPlan(prepared.chunk_plan),
-            totalChunks: prepared.logical_chunks ?? prepared.total_chunks ?? prepared.chunk_plan.length,
+            totalRoutes: prepared.plan_descriptor.delivery_route_count,
+            totalChunks: prepared.plan_descriptor.logical_chunk_count,
             signedUrlFlow,
             autoDistribute: input.distribute !== false
         });
         const multipartUploads = new Map();
+        const multipartUploadPromises = new Map();
         await this.startProviderRouteRecoverySigner({
             prepared,
             sourcesById,
@@ -174,71 +178,83 @@ export class BeamClient {
         try {
             await routeStream.begin();
             routeStreamStarted = true;
-            for (const chunk of prepared.chunk_plan) {
+            const pendingRoutes = [];
+            const flushOldestRoute = async () => {
+                const pending = pendingRoutes.shift();
+                if (pending)
+                    routeStream.addRoute(await pending);
+            };
+            for (const chunk of materializePlanChunks(prepared.plan_descriptor)) {
                 await input.throwIfCancelled?.(prepared.transfer_id);
                 for (const target of chunk.destinations) {
-                    const destination = destinationsById.get(target.destination_id);
-                    if (!destination) {
-                        throw new Error(`BeamCore returned unknown destination_id: ${target.destination_id}`);
-                    }
-                    const source = sourcesById.get(chunk.source_id);
-                    if (!source) {
-                        throw new Error(`BeamCore returned unknown source_id: ${chunk.source_id}`);
-                    }
-                    const metadata = target.metadata ?? {};
-                    const finalObjectKey = typeof metadata.final_object_key === "string"
-                        ? metadata.final_object_key
-                        : target.object_key;
-                    if (!finalObjectKey) {
-                        throw new Error("destination signing target is missing object_key");
-                    }
-                    const multipartKey = `${prepared.transfer_id}:${target.destination_id}:${chunk.source_id}:${finalObjectKey}`;
-                    let upload = multipartUploads.get(multipartKey);
-                    if (!upload) {
-                        const multipartCreatedAt = nowIso();
-                        if (isHippiusDestination(destination)) {
-                            upload = {
-                                destination,
-                                objectKey: finalObjectKey,
-                                uploadId: "",
-                                completeUrl: "",
-                                abortUrl: "",
-                                listUrl: "",
-                                multipartGroupId: multipartKey,
-                                multipartCreatedAt
-                            };
-                        }
-                        else {
-                            const uploadId = await createMultipartUpload(destination, finalObjectKey);
-                            upload = {
-                                destination,
-                                objectKey: finalObjectKey,
-                                uploadId,
-                                completeUrl: await signCompleteMultipartUpload(destination, finalObjectKey, uploadId, expiresIn),
-                                abortUrl: await signAbortMultipartUpload(destination, finalObjectKey, uploadId, expiresIn),
-                                listUrl: await signListMultipartUpload(destination, finalObjectKey, uploadId, expiresIn),
-                                multipartGroupId: multipartKey,
-                                multipartCreatedAt
-                            };
+                    pendingRoutes.push((async () => {
+                        await input.throwIfCancelled?.(prepared.transfer_id);
+                        const destination = destinationsById.get(target.destination_id);
+                        if (!destination)
+                            throw new Error(`BeamCore returned unknown destination_id: ${target.destination_id}`);
+                        const source = sourcesById.get(chunk.source_id);
+                        if (!source)
+                            throw new Error(`BeamCore returned unknown source_id: ${chunk.source_id}`);
+                        const metadata = target.metadata ?? {};
+                        const finalObjectKey = typeof metadata.final_object_key === "string" ? metadata.final_object_key : target.object_key;
+                        if (!finalObjectKey)
+                            throw new Error("destination signing target is missing object_key");
+                        const multipartKey = `${prepared.transfer_id}:${target.destination_id}:${chunk.source_id}:${finalObjectKey}`;
+                        let uploadPromise = multipartUploadPromises.get(multipartKey);
+                        if (!uploadPromise) {
+                            uploadPromise = (async () => {
+                                const multipartCreatedAt = nowIso();
+                                const upload = isHippiusDestination(destination)
+                                    ? {
+                                        destination,
+                                        objectKey: finalObjectKey,
+                                        uploadId: "",
+                                        completeUrl: "",
+                                        abortUrl: "",
+                                        listUrl: "",
+                                        multipartGroupId: multipartKey,
+                                        multipartCreatedAt
+                                    }
+                                    : await (async () => {
+                                        const uploadId = await createMultipartUpload(destination, finalObjectKey);
+                                        return {
+                                            destination,
+                                            objectKey: finalObjectKey,
+                                            uploadId,
+                                            completeUrl: await signCompleteMultipartUpload(destination, finalObjectKey, uploadId, expiresIn),
+                                            abortUrl: await signAbortMultipartUpload(destination, finalObjectKey, uploadId, expiresIn),
+                                            listUrl: await signListMultipartUpload(destination, finalObjectKey, uploadId, expiresIn),
+                                            multipartGroupId: multipartKey,
+                                            multipartCreatedAt
+                                        };
+                                    })();
+                                multipartUploads.set(multipartKey, upload);
+                                return upload;
+                            })();
+                            multipartUploadPromises.set(multipartKey, uploadPromise);
                         }
-                        multipartUploads.set(multipartKey, upload);
-                    }
-                    routeStream.addRoute(await this.signProviderRoute({
-                        chunk,
-                        target,
-                        source,
-                        destination,
-                        expiresIn,
-                        upload,
-                        transferId: prepared.transfer_id,
-                        finalObjectKey,
-                        partNumber: typeof metadata.part_number === "number"
-                            ? metadata.part_number
-                            : chunk.chunk_index * 3 + 1,
-                        signedUrlFlow
-                    }));
+                        const upload = await uploadPromise;
+                        return this.signProviderRoute({
+                            chunk,
+                            target,
+                            source,
+                            destination,
+                            expiresIn,
+                            upload,
+                            transferId: prepared.transfer_id,
+                            finalObjectKey,
+                            partNumber: typeof metadata.part_number === "number"
+                                ? metadata.part_number
+                                : directMultipartInitialPartNumber(chunk.chunk_index),
+                            signedUrlFlow
+                        });
+                    })());
+                    if (pendingRoutes.length >= this.routeSigningConcurrency)
+                        await flushOldestRoute();
                 }
             }
+            while (pendingRoutes.length)
+                await flushOldestRoute();
             const attached = await routeStream.complete();
             if (!attached.success) {
                 throw new Error(attached.error ?? attached.message ?? "route stream failed");
@@ -338,9 +354,8 @@ export class BeamClient {
     async startProviderRouteRecoverySigner(input) {
         this.stopRecoverySigner(input.prepared.transfer_id);
         const stop = await this.control.serveRouteRecoverySigner(input.prepared.transfer_id, async (payload) => {
-            const routes = [];
-            for (const requested of payload.chunks) {
-                const chunk = input.prepared.chunk_plan.find((candidate) => candidate.chunk_index === requested.chunk_index && candidate.source_id === requested.source_id);
+            const routes = await mapOrderedWithConcurrency(payload.chunks, this.routeSigningConcurrency, async (requested) => {
+                const chunk = materializePlanChunk(input.prepared.plan_descriptor, requested.source_id, requested.chunk_index);
                 if (!chunk) {
                     throw new Error(`recovery signing chunk not found: ${requested.source_id}:${requested.chunk_index}`);
                 }
@@ -371,7 +386,7 @@ export class BeamClient {
                     attempt_slot: requested.attempt_slot,
                     route_generation_id: requested.route_generation_id
                 };
-                routes.push(await this.signProviderRoute({
+                return this.signProviderRoute({
                     chunk,
                     target: {
                         ...target,
@@ -386,13 +401,13 @@ export class BeamClient {
                     finalObjectKey: requested.final_object_key,
                     partNumber: requested.part_number,
                     signedUrlFlow: input.signedUrlFlow
-                }));
-            }
+                });
+            });
             return {
                 transfer_id: payload.transfer_id,
                 route_generation_id: payload.route_generation_id,
                 signed_at: nowIso(),
-                chunk_routes: routes
+                route_batch: compactSignedRoutes(routes)
             };
         });
         this.recoverySigners.set(input.prepared.transfer_id, stop);
@@ -441,9 +456,8 @@ export class BeamClient {
         }
     }
 }
-// Target route-stream batches at 128 signed routes. The transport still splits
-// earlier when the encoded NATS request would exceed maxPayloadBytes.
-const ROUTE_STREAM_TARGET_BATCH_ROUTES = 128;
+const ROUTE_STREAM_BOOTSTRAP_BATCH_ROUTES = 256;
+const ROUTE_STREAM_STEADY_BATCH_ROUTES = 2_048;
 class RouteStreamSender {
     control;
     options;
@@ -451,6 +465,7 @@ class RouteStreamSender {
     checksum = new RouteKeysChecksum();
     batch = [];
     batchIndex = 0;
+    routeCount = 0;
     sendTail = Promise.resolve();
     sendError = null;
     constructor(control, options) {
@@ -469,8 +484,15 @@ class RouteStreamSender {
         }), this.options.transferId);
     }
     addRoute(route) {
-        this.batch.push(route);
-        if (this.batch.length >= ROUTE_STREAM_TARGET_BATCH_ROUTES) {
+        const normalizedRoute = route.delivery_index === undefined
+            ? { ...route, delivery_index: this.routeCount }
+            : route;
+        this.routeCount += 1;
+        this.batch.push(normalizedRoute);
+        const target = this.batchIndex === 0
+            ? ROUTE_STREAM_BOOTSTRAP_BATCH_ROUTES
+            : ROUTE_STREAM_STEADY_BATCH_ROUTES;
+        if (this.batch.length >= target) {
             this.enqueueFlush();
         }
     }
@@ -512,7 +534,7 @@ class RouteStreamSender {
                         stream_id: this.streamId,
                         batch_id: `${this.streamId}:${batchIndex}`,
                         batch_index: batchIndex,
-                        chunk_routes: chunk,
+                        route_batch: compactSignedRoutes(chunk),
                         route_count: chunk.length,
                         route_keys_checksum: chunkChecksum.value()
                     }, this.options.transferId);
@@ -555,8 +577,72 @@ function routeKeyForSignedRoute(route) {
 function countDistinctRouteChunks(routes) {
     return new Set(routes.map((route) => `${route.source_id}:${route.chunk_index}`)).size;
 }
-function countRoutesInChunkPlan(chunks) {
-    return chunks.reduce((sum, chunk) => sum + chunk.destinations.length, 0);
+function* materializePlanChunks(descriptor) {
+    for (const source of descriptor.sources) {
+        for (let sourceChunkIndex = 0; sourceChunkIndex < source.chunk_count; sourceChunkIndex += 1) {
+            yield materializePlanChunk(descriptor, source.source_id, source.global_chunk_start + sourceChunkIndex);
+        }
+    }
+}
+async function mapOrderedWithConcurrency(values, concurrency, map) {
+    const results = new Array(values.length);
+    let nextIndex = 0;
+    const workers = Array.from({ length: Math.min(concurrency, values.length) }, async () => {
+        while (true) {
+            const index = nextIndex;
+            nextIndex += 1;
+            if (index >= values.length)
+                return;
+            results[index] = await map(values[index], index);
+        }
+    });
+    await Promise.all(workers);
+    return results;
+}
+function positiveInteger(value, label) {
+    if (!Number.isInteger(value) || value < 1)
+        throw new Error(`${label} must be a positive integer.`);
+    return value;
+}
+function materializePlanChunk(descriptor, sourceId, chunkIndex) {
+    const source = descriptor.sources.find((candidate) => candidate.source_id === sourceId);
+    if (!source)
+        throw new Error(`plan source not found: ${sourceId}`);
+    const sourceChunkIndex = chunkIndex - source.global_chunk_start;
+    if (sourceChunkIndex < 0 || sourceChunkIndex >= source.chunk_count) {
+        throw new Error(`plan chunk is outside source range: ${sourceId}:${chunkIndex}`);
+    }
+    const sourceOffset = sourceChunkIndex * descriptor.chunk_size;
+    const chunkSize = Math.min(descriptor.chunk_size, source.size - sourceOffset);
+    return {
+        chunk_index: chunkIndex,
+        source_id: sourceId,
+        source_chunk_index: sourceChunkIndex,
+        source_offset: sourceOffset,
+        chunk_size: chunkSize,
+        source_url: source.url,
+        destinations: descriptor.destinations.map((destination) => {
+            const finalObjectKey = destination.final_object_keys[sourceId];
+            if (!finalObjectKey)
+                throw new Error(`plan final object key not found: ${sourceId}:${destination.destination_id}`);
+            const deliveryIndex = chunkIndex * descriptor.destinations.length + destination.destination_index;
+            return {
+                destination_id: destination.destination_id,
+                delivery_index: deliveryIndex,
+                provider: destination.provider,
+                object_key: `${finalObjectKey}/${descriptor.plan_nonce}/chunk-${String(sourceChunkIndex).padStart(6, "0")}`,
+                metadata: {
+                    ...(destination.metadata ?? {}),
+                    final_object_key: finalObjectKey,
+                    part_number: directMultipartInitialPartNumber(chunkIndex),
+                    logical_attempt_index: 0,
+                    attempt_slot: 0,
+                    route_generation_id: `initial-${chunkIndex}-${destination.destination_id}`,
+                    delivery_index: deliveryIndex
+                }
+            };
+        })
+    };
 }
 async function sha256Bytes(value) {
     const input = new TextEncoder().encode(value);

package/dist/models.d.ts

@@ -8,6 +8,7 @@ export interface BeamClientOptions {
     transferRuntimeShardCount?: number;
     requestTimeoutMs?: number;
     maxPayloadBytes?: number;
+    routeSigningConcurrency?: number;
     fetch?: typeof fetch;
 }
 export interface SourceConfig {
@@ -240,10 +241,36 @@ export interface ChunkSigningPlanItem {
     source_url: string;
     destinations: ChunkDestinationSigningTarget[];
 }
+export interface CompactTransferPlanSource extends PreparedHttpSource {
+    global_chunk_start: number;
+    chunk_count: number;
+}
+export interface CompactTransferPlanDestination extends PreparedDestination {
+    destination_index: number;
+    final_object_keys: Record<string, string>;
+}
+export interface CompactTransferPlanDescriptor {
+    version: "compact-transfer-plan/v1";
+    plan_nonce: string;
+    chunk_size: number;
+    sources: CompactTransferPlanSource[];
+    destinations: CompactTransferPlanDestination[];
+    logical_chunk_count: number;
+    delivery_route_count: number;
+    multipart_attempt_slots: number;
+    formulas: {
+        source_offset: "source_chunk_index * chunk_size";
+        delivery_index: "chunk_index * destination_count + destination_index";
+        part_number: "chunk_index * multipart_attempt_slots + attempt_slot + 1";
+        route_generation_id: "initial-{chunk_index}-{destination_id}";
+        staging_object_key: "{final_object_key}/{plan_nonce}/chunk-{source_chunk_index:6}";
+    };
+}
 export interface SignedChunkRoute {
     source_id: string;
     destination_id: string;
     chunk_index: number;
+    delivery_index?: number;
     source_url: string;
     dest_url: string;
     source_offset: number;
@@ -261,7 +288,7 @@ export interface TransferPlanResponse {
     total_destinations?: number;
     logical_chunks?: number;
     total_chunks?: number;
-    chunk_plan: ChunkSigningPlanItem[];
+    plan_descriptor: CompactTransferPlanDescriptor;
     signed_url_flow?: SignedUrlFlow;
     error?: string;
     message?: string;
@@ -277,7 +304,7 @@ export interface TransferPrepareResponse {
     total_destinations?: number;
     logical_chunks?: number;
     total_chunks?: number;
-    chunk_plan: ChunkSigningPlanItem[];
+    plan_descriptor: CompactTransferPlanDescriptor;
     signed_url_flow?: SignedUrlFlow;
     error?: string;
     message?: string;

package/dist/multipart-limits.d.ts

@@ -0,0 +1,4 @@
+export declare const DIRECT_MULTIPART_ATTEMPT_SLOT_COUNT = 2;
+export declare const DIRECT_MULTIPART_MAX_PART_NUMBER = 10000;
+export declare const DIRECT_MULTIPART_MAX_LOGICAL_CHUNKS: number;
+export declare function directMultipartInitialPartNumber(chunkIndex: number): number;

package/dist/multipart-limits.js

@@ -0,0 +1,12 @@
+export const DIRECT_MULTIPART_ATTEMPT_SLOT_COUNT = 2;
+export const DIRECT_MULTIPART_MAX_PART_NUMBER = 10_000;
+export const DIRECT_MULTIPART_MAX_LOGICAL_CHUNKS = Math.floor(DIRECT_MULTIPART_MAX_PART_NUMBER / DIRECT_MULTIPART_ATTEMPT_SLOT_COUNT);
+export function directMultipartInitialPartNumber(chunkIndex) {
+    if (!Number.isInteger(chunkIndex) || chunkIndex < 0) {
+        throw new Error("chunk_index must be a non-negative integer");
+    }
+    if (chunkIndex >= DIRECT_MULTIPART_MAX_LOGICAL_CHUNKS) {
+        throw new Error(`chunk_index must be less than ${DIRECT_MULTIPART_MAX_LOGICAL_CHUNKS}`);
+    }
+    return chunkIndex * DIRECT_MULTIPART_ATTEMPT_SLOT_COUNT + 1;
+}

package/dist/nats-control.d.ts

@@ -1,4 +1,5 @@
-export declare const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/v1";
+import type { SignedChunkRoute } from "./models.js";
+export declare const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/v2";
 export declare const BEAM_DEFAULT_NATS_URL = "tls://nats.b1m.ai:4222";
 export declare const BEAM_DEFAULT_NATS_WS_URL = "wss://nats.b1m.ai:443";
 export declare const BEAM_DEFAULT_MAX_PAYLOAD_BYTES: number;
@@ -33,7 +34,33 @@ export interface RouteRecoverySignReplyPayload {
     transfer_id: string;
     route_generation_id: string;
     signed_at?: string;
-    chunk_routes: unknown[];
+    route_batch: CompactSignedRouteBatch;
+}
+export interface CompactSignedRouteBatch {
+    source_chunks: Array<{
+        source_ref: number;
+        source_id: string;
+        chunk_index: number;
+        source_url: string;
+        source_offset: number;
+        chunk_size: number;
+        expires_at?: string;
+        headers?: Record<string, string>;
+    }>;
+    multipart_groups: Array<{
+        group_ref: number;
+        metadata: Record<string, unknown>;
+    }>;
+    routes: Array<{
+        source_ref: number;
+        destination_id: string;
+        delivery_index: number;
+        dest_url: string;
+        expires_at?: string;
+        dest_headers?: Record<string, string>;
+        multipart_group_ref?: number;
+        metadata?: Record<string, unknown>;
+    }>;
 }
 export interface TransferControlOptions {
     apiKey: string;
@@ -61,7 +88,7 @@ export declare class BeamTransferControl {
     serveRouteRecoverySigner(transferId: string, handler: (payload: RouteRecoverySignRequestPayload) => Promise<RouteRecoverySignReplyPayload>): Promise<() => void>;
     close(): Promise<void>;
     request<T>(messageType: TransferClientMessageType, payload: Record<string, unknown>, transferId?: string): Promise<T>;
-    splitRoutesForPayload(messageType: TransferClientMessageType, basePayload: Record<string, unknown>, routes: unknown[]): unknown[][];
+    splitRoutesForPayload(messageType: TransferClientMessageType, basePayload: Record<string, unknown>, routes: SignedChunkRoute[]): SignedChunkRoute[][];
     private connection;
     private resolveAuthToken;
     private authSubject;
@@ -69,4 +96,5 @@ export declare class BeamTransferControl {
     private buildRouteRecoveryReply;
     private requestSubject;
 }
+export declare function compactSignedRoutes(routes: SignedChunkRoute[]): CompactSignedRouteBatch;
 export declare function transferShardId(transferId: string, shardCount: number): number;

package/dist/nats-control.js

@@ -1,6 +1,6 @@
 import { encode, decode } from "@msgpack/msgpack";
 import { connect } from "nats";
-export const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/v1";
+export const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/v2";
 export const BEAM_DEFAULT_NATS_URL = "tls://nats.b1m.ai:4222";
 export const BEAM_DEFAULT_NATS_WS_URL = "wss://nats.b1m.ai:443";
 // NATS enforces max_payload per message. This guard splits signed-route control
@@ -134,33 +134,28 @@ export class BeamTransferControl {
         return (decoded.payload ?? {});
     }
     splitRoutesForPayload(messageType, basePayload, routes) {
-        const chunks = [];
-        let current = [];
-        for (const route of routes) {
-            const candidate = [...current, route];
+        const split = (candidate) => {
             const encoded = encode({
                 schema_version: TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION,
                 environment: this.environment,
                 key_prefix: this.keyPrefix,
                 shard_id: 0,
                 message_type: messageType,
-                request_id: randomId(),
+                request_id: "00000000-0000-4000-8000-000000000000",
                 auth_token: "x".repeat(512),
                 occurred_at: new Date().toISOString(),
                 producer: "sdk",
-                payload: { ...basePayload, chunk_routes: candidate }
+                payload: { ...basePayload, route_batch: compactSignedRoutes(candidate) }
             });
-            if (current.length > 0 && encoded.byteLength > this.maxPayloadBytes) {
-                chunks.push(current);
-                current = [route];
+            if (encoded.byteLength <= this.maxPayloadBytes)
+                return [candidate];
+            if (candidate.length === 1) {
+                throw new Error(`single signed route is ${encoded.byteLength} bytes, above maxPayloadBytes=${this.maxPayloadBytes}`);
             }
-            else {
-                current = candidate;
-            }
-        }
-        if (current.length > 0)
-            chunks.push(current);
-        return chunks;
+            const midpoint = Math.ceil(candidate.length / 2);
+            return [...split(candidate.slice(0, midpoint)), ...split(candidate.slice(midpoint))];
+        };
+        return routes.length ? split(routes) : [];
     }
     async connection() {
         if (this.nc && !this.nc.isClosed())
@@ -214,6 +209,107 @@ export class BeamTransferControl {
         return `${this.subjectPrefix}.${this.environment}.sdk.${this.keyPrefix}.shard.${shardId}.${messageType.replace(/\./g, "_")}`;
     }
 }
+const ROUTE_ATTEMPT_METADATA_KEYS = new Set([
+    "part_number",
+    "logical_attempt_index",
+    "attempt_slot",
+    "route_generation_id",
+    "staging_object_key",
+    "copy_url",
+    "copy_source",
+    "copy_source_range",
+    "delete_url",
+    "part_verify_url",
+    "list_page_url"
+]);
+const ROUTE_IDENTITY_METADATA_KEYS = [
+    "source_id",
+    "destination_id",
+    "chunk_index",
+    "route_chunk_index",
+    "delivery_index"
+];
+export function compactSignedRoutes(routes) {
+    const source_chunks = [];
+    const sourceRefs = new Map();
+    const multipart_groups = [];
+    const groupRefs = new Map();
+    const compactRoutes = [];
+    for (const [routeIndex, route] of routes.entries()) {
+        const sourceKey = JSON.stringify([
+            route.source_id,
+            route.chunk_index,
+            route.source_url,
+            route.source_offset,
+            route.chunk_size,
+            route.expires_at ?? null,
+            route.headers ?? null
+        ]);
+        let sourceRef = sourceRefs.get(sourceKey);
+        if (sourceRef === undefined) {
+            sourceRef = source_chunks.length;
+            sourceRefs.set(sourceKey, sourceRef);
+            source_chunks.push(compact({
+                source_ref: sourceRef,
+                source_id: route.source_id,
+                chunk_index: route.chunk_index,
+                source_url: route.source_url,
+                source_offset: route.source_offset,
+                chunk_size: route.chunk_size,
+                expires_at: route.expires_at,
+                headers: route.headers
+            }));
+        }
+        const metadata = { ...(route.metadata ?? {}) };
+        const deliveryIndex = route.delivery_index ?? nonNegativeInteger(metadata.delivery_index) ?? routeIndex;
+        for (const key of ROUTE_IDENTITY_METADATA_KEYS)
+            delete metadata[key];
+        const groupBaseKey = multipartGroupKey(route, metadata);
+        const groupMetadata = Object.fromEntries(Object.entries(metadata).filter(([key]) => !ROUTE_ATTEMPT_METADATA_KEYS.has(key)));
+        const groupKey = groupBaseKey ? `${groupBaseKey}:${JSON.stringify(groupMetadata)}` : null;
+        let groupRef;
+        if (groupKey) {
+            groupRef = groupRefs.get(groupKey);
+            if (groupRef === undefined) {
+                groupRef = multipart_groups.length;
+                groupRefs.set(groupKey, groupRef);
+                multipart_groups.push({ group_ref: groupRef, metadata: groupMetadata });
+            }
+            for (const key of Object.keys(metadata)) {
+                if (!ROUTE_ATTEMPT_METADATA_KEYS.has(key))
+                    delete metadata[key];
+            }
+        }
+        compactRoutes.push(compact({
+            source_ref: sourceRef,
+            destination_id: route.destination_id,
+            delivery_index: deliveryIndex,
+            dest_url: route.dest_url,
+            expires_at: route.expires_at,
+            dest_headers: route.dest_headers,
+            multipart_group_ref: groupRef,
+            metadata: Object.keys(metadata).length ? metadata : undefined
+        }));
+    }
+    return { source_chunks, multipart_groups, routes: compactRoutes };
+}
+function multipartGroupKey(route, metadata) {
+    const groupId = typeof metadata.multipart_group_id === "string" ? metadata.multipart_group_id : "";
+    if (groupId)
+        return `id:${groupId}`;
+    const uploadId = typeof metadata.upload_id === "string" ? metadata.upload_id : "";
+    const finalObjectKey = typeof metadata.final_object_key === "string" ? metadata.final_object_key : "";
+    return uploadId || finalObjectKey
+        ? JSON.stringify([route.source_id, route.destination_id, uploadId, finalObjectKey])
+        : null;
+}
+function nonNegativeInteger(value) {
+    const parsed = typeof value === "number" ? value : Number(value);
+    return Number.isInteger(parsed) && parsed >= 0 ? parsed : null;
+}
+function compact(value) {
+    return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined));
+}
 export function transferShardId(transferId, shardCount) {
     let hash = 2166136261;
     for (let i = 0; i < transferId.length; i += 1) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@beam-network/sdk",
-    "version": "0.2.20",
+    "version": "0.2.22",
     "description": "TypeScript SDK for BEAM transfer creation and management.",
     "type": "module",
     "license": "MIT",