@beam-network/sdk 0.4.1 → 0.4.3

package/README.md

@@ -96,9 +96,9 @@ For low-level raw transfer configs, use `createRawTransfer`; lifecycle transport
 
 ## Route Streaming And Payload Size
 
-Provider transfers use `signed_url_v2` and `transfer-client-control/v3`. Before route streaming, S3, R2, and S3-compatible destinations create one metadata-bearing multipart upload per source/destination object. Its deduplicated manifest contains complete/abort and final HEAD controls, expected object size/count, required Beam object metadata, and exactly `ceil(max_part_number / 1000)` unique URLs in each of `list_page_urls` and `staging_list_urls`. Staging pages fix `MaxKeys=1000`; keys are ordered by unsigned UTF-8 bytes, page zero omits `StartAfter`, and later pages start after the preceding page's last key. Per-route batches carry only the stable group ID and route-attempt metadata. Part number is `source_chunk_index + 1`, and retries overwrite that same part.
+Provider transfers default to `signed_url_v1` and `transfer-client-control/v3`. S3, R2, and S3-compatible destinations create one final multipart upload per source/destination object, then each route signs direct `UploadPart` to that upload. Core receives per-route completion, abort, ListParts, and final HEAD controls and completes the destination after all winning ETags are known. `signed_url_v2` remains explicitly selectable for the legacy staged-copy manifest path.
 
-The client signs up to 64 routes concurrently by default and emits a 32-route bootstrap batch followed by 256-route steady batches. Out-of-order signing completions are buffered until the next deterministic delivery index, keeping batch contents reproducible. The stream ID derives from the transfer and immutable plan identity, and each ordered batch ID includes its route-coordinate checksum. Manual multi-destination attachment requires `delivery_index` on every route. Encoded MessagePack requests still split under `maxPayloadBytes` (24 MiB by default), and lifecycle mutations retry transient failures up to three times with the same request identity. `waitForTransfer` subscribes to the API-key-owned, at-most-once terminal signal before its first status read and reconciles every signal through authoritative status; subscription failure degrades to the same jittered 15-to-30-second status fallback. Call `close()` when the client is no longer needed; it also closes outstanding terminal waiters.
+The client signs up to 64 routes concurrently by default and emits 2,048-route logical batches, split only when encoded MessagePack requests exceed `maxPayloadBytes` (24 MiB by default). The stream ID derives from the transfer, selected flow, and immutable plan identity, and each ordered batch ID includes its route-coordinate checksum. Manual multi-destination attachment requires `delivery_index` on every route. Lifecycle mutations retry transient failures up to three times with the same request identity. `waitForTransfer` subscribes to the API-key-owned, at-most-once terminal signal before its first status read and reconciles every signal through authoritative status; subscription failure degrades to the same jittered 15-to-30-second status fallback. Call `close()` when the client is no longer needed; it also closes outstanding terminal waiters.
 
 Hippius uses genuine non-multipart v2 routes, so its manifest is empty and group-level final HEAD verification is not available from that provider flow.
 

package/dist/client.js

@@ -63,7 +63,7 @@ export class BeamClient {
             callbacks: input.callbacks,
             test_mode: input.testMode || undefined,
             progressive_mode: input.progressiveMode || undefined,
-            signed_url_flow: "signed_url_v2"
+            signed_url_flow: input.signedUrlFlow ?? "signed_url_v1"
         });
         return this.control.request("transfer.create", body, {
             transferId,
@@ -99,7 +99,7 @@ export class BeamClient {
             name: input.name,
             test_mode: input.testMode || undefined,
             urls_expires_at: input.urlsExpiresAt,
-            signed_url_flow: "signed_url_v2"
+            signed_url_flow: input.signedUrlFlow ?? "signed_url_v1"
         }));
         if (result.success)
             validateCompactTransferPlan(result.plan_descriptor, result.signed_url_flow);
@@ -114,7 +114,7 @@ export class BeamClient {
             name: input.name,
             test_mode: input.testMode || undefined,
             urls_expires_at: input.urlsExpiresAt,
-            signed_url_flow: "signed_url_v2"
+            signed_url_flow: input.signedUrlFlow ?? "signed_url_v1"
         }), {
             transferId,
             idempotencyKey: `transfer:${transferId}:prepare`
@@ -142,7 +142,8 @@ export class BeamClient {
             transferId,
             planIdentity: await routeCoordinateChecksum(orderedRoutes),
             totalRoutes: orderedRoutes.length,
-            totalChunks: countDistinctRouteChunks(orderedRoutes)
+            totalChunks: countDistinctRouteChunks(orderedRoutes),
+            signedUrlFlow: input.multipartGroupManifest.length ? "signed_url_v2" : "signed_url_v1"
         });
         const stream = new RouteStreamSender(this.control, {
             streamId,
@@ -150,6 +151,7 @@ export class BeamClient {
             totalRoutes: orderedRoutes.length,
             totalChunks: countDistinctRouteChunks(orderedRoutes),
             urlsExpiresAt: input.urlsExpiresAt,
+            signedUrlFlow: input.multipartGroupManifest.length ? "signed_url_v2" : "signed_url_v1",
             autoDistribute: input.autoDistribute ?? true
         });
         await stream.begin();
@@ -163,7 +165,7 @@ export class BeamClient {
     }
     async prepareProviderTransfer(input) {
         const expiresIn = input.expiresIn ?? 3600;
-        const requestedSignedUrlFlow = "signed_url_v2";
+        const requestedSignedUrlFlow = input.signedUrlFlow ?? "signed_url_v1";
         const preparedDestinations = input.destinations.map((destination, index) => prepareProviderDestination(destination, { index }));
         const destinationsById = new Map();
         preparedDestinations.forEach((preparedDestination, index) => {
@@ -212,13 +214,15 @@ export class BeamClient {
                 transferId: prepared.transfer_id,
                 planIdentity: prepared.plan_descriptor.plan_nonce,
                 totalRoutes: prepared.plan_descriptor.delivery_route_count,
-                totalChunks: prepared.plan_descriptor.logical_chunk_count
+                totalChunks: prepared.plan_descriptor.logical_chunk_count,
+                signedUrlFlow: requestedSignedUrlFlow
             });
             routeStream = new RouteStreamSender(this.control, {
                 streamId,
                 transferId: prepared.transfer_id,
                 totalRoutes: prepared.plan_descriptor.delivery_route_count,
                 totalChunks: prepared.plan_descriptor.logical_chunk_count,
+                signedUrlFlow: requestedSignedUrlFlow,
                 autoDistribute: input.distribute !== false
             });
             routeStreamBeginAttempted = true;
@@ -229,10 +233,13 @@ export class BeamClient {
                 destinationsById,
                 multipartUploads,
                 expiresIn,
+                signedUrlFlow: requestedSignedUrlFlow,
                 concurrency: this.routeSigningConcurrency,
                 onGroupReady: async (state) => {
-                    validateMultipartGroupManifest([state.manifest], prepared.transfer_id);
-                    await routeStream.addManifestGroups([state.manifest]);
+                    if (requestedSignedUrlFlow === "signed_url_v2") {
+                        validateMultipartGroupManifest([state.manifest], prepared.transfer_id);
+                        await routeStream.addManifestGroups([state.manifest]);
+                    }
                     multipartGroupWaiters.get(state.manifest.multipart_group_id)?.resolve(state);
                 },
                 onGroupFailed: (groupId, error) => multipartGroupWaiters.get(groupId)?.reject(error)
@@ -281,6 +288,7 @@ export class BeamClient {
                             ...(upload ? { upload } : {}),
                             transferId: prepared.transfer_id,
                             finalObjectKey,
+                            signedUrlFlow: requestedSignedUrlFlow,
                             partNumber: typeof metadata.part_number === "number"
                                 ? metadata.part_number
                                 : multipartPartNumber(chunk.source_chunk_index),
@@ -294,7 +302,9 @@ export class BeamClient {
             while (pendingRoutes.size)
                 await streamNextCompletedRoute();
             const multipartGroupManifest = await multipartManifestTask;
-            validateMultipartGroupManifest(multipartGroupManifest, prepared.transfer_id);
+            if (requestedSignedUrlFlow === "signed_url_v2") {
+                validateMultipartGroupManifest(multipartGroupManifest, prepared.transfer_id);
+            }
             const attached = await routeStream.complete();
             if (!attached.success) {
                 throw new Error(attached.error ?? attached.message ?? "route stream failed");
@@ -330,8 +340,33 @@ export class BeamClient {
         }
         const upload = input.upload;
         if (!upload)
-            throw new Error("signed_url_v2 route is missing its multipart group");
+            throw new Error(`${input.signedUrlFlow} route is missing its multipart upload`);
         validateMultipartPartNumber(input.partNumber, upload.manifest);
+        if (input.signedUrlFlow === "signed_url_v1") {
+            return signDestinationRoute({
+                chunk: input.chunk,
+                target: {
+                    ...input.target,
+                    object_key: input.finalObjectKey,
+                    metadata: partRouteMetadata(input.target.metadata)
+                },
+                source: input.source,
+                destination: input.destination,
+                expiresIn: input.expiresIn,
+                uploadId: upload.uploadId,
+                partNumber: input.partNumber,
+                finalObjectKey: input.finalObjectKey,
+                expectedObjectSize: upload.manifest.expected_object_size,
+                expectedPartCount: upload.manifest.expected_part_count,
+                maxPartNumber: upload.manifest.max_part_number,
+                completeUrl: upload.manifest.complete_url,
+                abortUrl: upload.manifest.abort_url,
+                listPageUrl: upload.manifest.list_page_urls[multipartListPageIndex(input.partNumber)],
+                finalHeadUrl: upload.manifest.final_head_url,
+                finalObjectMetadata: upload.manifest.final_object_metadata,
+                fetchImpl: this.fetchImpl
+            });
+        }
         const stagingObjectPrefix = buildStagingObjectPrefix({
             finalObjectKey: input.finalObjectKey,
             transferId: input.transferId,
@@ -475,9 +510,9 @@ class RouteStreamSender {
             stream_id: this.streamId,
             total_routes: this.options.totalRoutes,
             total_chunks: this.options.totalChunks,
-            route_contract_version: "signed_url_v2",
+            route_contract_version: this.options.signedUrlFlow,
             urls_expires_at: this.options.urlsExpiresAt,
-            signed_url_flow: "signed_url_v2",
+            signed_url_flow: this.options.signedUrlFlow,
             auto_distribute: this.options.autoDistribute
         }), {
             transferId: this.options.transferId,
@@ -633,7 +668,7 @@ async function routeCoordinateChecksum(routes) {
     return checksum.value();
 }
 async function stableRouteStreamId(input) {
-    const digest = await sha256Bytes(`beam:route-stream:signed_url_v2:${input.transferId}:${input.planIdentity}:${input.totalRoutes}:${input.totalChunks}`);
+    const digest = await sha256Bytes(`beam:route-stream:${input.signedUrlFlow}:${input.transferId}:${input.planIdentity}:${input.totalRoutes}:${input.totalChunks}`);
     const bytes = digest.slice(0, 16);
     bytes[6] = ((bytes[6] ?? 0) & 0x0f) | 0x50;
     bytes[8] = ((bytes[8] ?? 0) & 0x3f) | 0x80;
@@ -843,7 +878,7 @@ async function createMultipartGroupManifest(input) {
             multipartGroupId = multipartGroupStateKey(input.prepared.transfer_id, destination.destination_id, source.source_id, finalObjectKey);
             const finalObjectMetadata = {
                 "beam-transfer-id": input.prepared.transfer_id,
-                "beam-multipart-group-id": multipartGroupId
+                ...(input.signedUrlFlow === "signed_url_v2" ? { "beam-multipart-group-id": multipartGroupId } : {})
             };
             const uploadId = await createMultipartUpload({
                 destination: destinationConfig,
@@ -874,7 +909,9 @@ async function createMultipartGroupManifest(input) {
                     signCompleteMultipartUpload(destinationConfig, finalObjectKey, uploadId, input.expiresIn),
                     signAbortMultipartUpload(destinationConfig, finalObjectKey, uploadId, input.expiresIn),
                     Promise.all(listPageMarkers.map((partNumberMarker) => signListMultipartUpload(destinationConfig, finalObjectKey, uploadId, input.expiresIn, { maxParts: 1_000, ...(partNumberMarker ? { partNumberMarker } : {}) }))),
-                    Promise.all(stagingListStartAfter.map((startAfter) => signListObjects(destinationConfig, stagingListPrefix, input.expiresIn, startAfter ? { startAfter } : {}))),
+                    input.signedUrlFlow === "signed_url_v2"
+                        ? Promise.all(stagingListStartAfter.map((startAfter) => signListObjects(destinationConfig, stagingListPrefix, input.expiresIn, startAfter ? { startAfter } : {})))
+                        : Promise.resolve([]),
                     signFinalObjectHead(destinationConfig, finalObjectKey, input.expiresIn)
                 ]);
                 const manifest = {
@@ -980,6 +1017,9 @@ function validateMultipartPartNumber(partNumber, manifest) {
         throw new Error(`multipart part_number ${partNumber} is outside group ${manifest.multipart_group_id} range 1-${manifest.max_part_number}`);
     }
 }
+function multipartListPageIndex(partNumber) {
+    return Math.floor((partNumber - 1) / 1_000);
+}
 function validateSignedRouteManifestContract(routes, manifest) {
     const groups = new Map(manifest.map((group) => [group.multipart_group_id, group]));
     for (const route of routes) {
@@ -1019,7 +1059,7 @@ const COMPACT_TRANSFER_PLAN_FORMULAS = {
     staging_object_key: "{final_object_key}.beam-staging/{transfer_id}/{source_id}/{destination_id}/part-{part_number}-chunk-{chunk_index}"
 };
 function validateCompactTransferPlan(descriptor, signedUrlFlow) {
-    if (signedUrlFlow !== "signed_url_v2") {
+    if (signedUrlFlow !== "signed_url_v1" && signedUrlFlow !== "signed_url_v2") {
         throw new Error(`BeamCore returned unsupported signed_url_flow: ${String(signedUrlFlow)}`);
     }
     const keys = Object.keys(descriptor);
@@ -1094,15 +1134,21 @@ function validateMultipartGroupManifest(manifest, transferId) {
             throw new Error(`multipart group ${group.multipart_group_id} requires ${expectedListPageCount} unique list_page_urls`);
         }
         if (!Array.isArray(group.staging_list_urls)
-            || group.staging_list_urls.length !== expectedListPageCount
-            || new Set(group.staging_list_urls).size !== expectedListPageCount
             || group.staging_list_urls.some((url) => typeof url !== "string" || url.length === 0)) {
-            throw new Error(`multipart group ${group.multipart_group_id} requires ${expectedListPageCount} unique staging_list_urls`);
+            throw new Error(`multipart group ${group.multipart_group_id} has invalid staging_list_urls`);
+        }
+        if (group.staging_list_urls.length > 0
+            && (group.staging_list_urls.length !== expectedListPageCount
+                || new Set(group.staging_list_urls).size !== expectedListPageCount)) {
+            throw new Error(`multipart group ${group.multipart_group_id} requires ${expectedListPageCount} unique staging_list_urls when staging is used`);
         }
         const metadataKeys = Object.keys(group.final_object_metadata);
-        if (metadataKeys.length !== 2
+        const stagedCopyMetadata = group.staging_list_urls.length > 0;
+        if (metadataKeys.length !== (stagedCopyMetadata ? 2 : 1)
             || group.final_object_metadata["beam-transfer-id"] !== transferId
-            || group.final_object_metadata["beam-multipart-group-id"] !== group.multipart_group_id) {
+            || (stagedCopyMetadata
+                ? group.final_object_metadata["beam-multipart-group-id"] !== group.multipart_group_id
+                : group.final_object_metadata["beam-multipart-group-id"] !== undefined)) {
             throw new Error(`multipart group ${group.multipart_group_id} has invalid final_object_metadata`);
         }
     }

package/dist/models.d.ts

@@ -1,4 +1,4 @@
-export type SignedUrlFlow = "signed_url_v2";
+export type SignedUrlFlow = "signed_url_v1" | "signed_url_v2";
 export interface BeamClientOptions {
     apiKey?: string;
     natsUrl?: string;
@@ -176,6 +176,15 @@ export interface R2ProviderConfig {
     account_id?: string;
     endpoint_url?: string;
 }
+export interface S3CompatibleProviderCapabilities {
+    /**
+     * Whether the provider's ListParts / ListObjects response includes a per-part
+     * LastModified timestamp. Defaults to true (S3-spec compliant). Providers that
+     * omit it (e.g. Hippius) declare false so Beam finalization substitutes a
+     * Beam-side timestamp instead of looping into multipart recovery.
+     */
+    multipart_part_last_modified?: boolean;
+}
 export interface S3CompatibleProviderConfig {
     provider: string;
     driver?: "s3-compatible";
@@ -189,6 +198,7 @@ export interface S3CompatibleProviderConfig {
     session_token?: string;
     force_path_style?: boolean;
     account_id?: string;
+    capabilities?: S3CompatibleProviderCapabilities;
 }
 export interface HippiusProviderConfig {
     provider: "hippius";
@@ -325,9 +335,9 @@ export interface MultipartGroupManifest {
     list_page_urls: string[];
     staging_list_urls: string[];
     final_head_url: string;
-    final_object_metadata: {
+    final_object_metadata: Record<string, string> & {
         "beam-transfer-id": string;
-        "beam-multipart-group-id": string;
+        "beam-multipart-group-id"?: string;
     };
     urls_expires_at: string;
 }

package/dist/nats-control.d.ts

@@ -31,9 +31,9 @@ export interface RouteStreamBeginPayload {
     stream_id: string;
     total_routes: number;
     total_chunks: number;
-    route_contract_version: "signed_url_v2";
+    route_contract_version: "signed_url_v1" | "signed_url_v2";
     urls_expires_at?: string;
-    signed_url_flow: "signed_url_v2";
+    signed_url_flow: "signed_url_v1" | "signed_url_v2";
     auto_distribute: boolean;
 }
 export interface RouteStreamManifestPayload {

package/dist/nats-control.js

@@ -402,9 +402,11 @@ export function compactSignedRoutes(routes) {
         const multipartGroupId = typeof metadata.multipart_group_id === "string"
             ? metadata.multipart_group_id
             : undefined;
-        for (const key of Object.keys(metadata)) {
-            if (!ROUTE_ATTEMPT_METADATA_KEYS.has(key))
-                delete metadata[key];
+        if (multipartGroupId) {
+            for (const key of Object.keys(metadata)) {
+                if (!ROUTE_ATTEMPT_METADATA_KEYS.has(key))
+                    delete metadata[key];
+            }
         }
         compactRoutes.push(compact({
             source_ref: sourceRef,

package/dist/provider-signing.d.ts

@@ -42,6 +42,16 @@ export declare function signDestinationRoute(input: {
     destination: ProviderDestinationConfig;
     expiresIn: number;
     partNumber?: number;
+    uploadId?: string;
+    completeUrl?: string;
+    abortUrl?: string;
+    listPageUrl?: string;
+    finalHeadUrl?: string;
+    finalObjectKey?: string;
+    expectedObjectSize?: number;
+    expectedPartCount?: number;
+    maxPartNumber?: number;
+    finalObjectMetadata?: Record<string, string>;
     multipartGroupId?: string;
     commitMethod?: "upload_part_copy";
     stagingObjectKey?: string;

package/dist/provider-signing.js

@@ -1,4 +1,4 @@
-import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, ListPartsCommand, PutObjectCommand, S3Client, UploadPartCopyCommand } from "@aws-sdk/client-s3";
+import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, ListPartsCommand, PutObjectCommand, S3Client, UploadPartCommand, UploadPartCopyCommand } from "@aws-sdk/client-s3";
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
 export function nowIso() {
     return new Date().toISOString();
@@ -230,6 +230,8 @@ export async function signDestinationRoute(input) {
         signDestinationUrl({
             destination: input.destination,
             objectKey,
+            uploadId: input.uploadId,
+            partNumber: input.partNumber,
             expiresIn: input.expiresIn,
             fetchImpl: input.fetchImpl,
         }),
@@ -260,6 +262,16 @@ export async function signDestinationRoute(input) {
             ...(input.copySource ? { copy_source: input.copySource } : {}),
             ...(input.copySourceRange ? { copy_source_range: input.copySourceRange } : {}),
             ...(input.deleteUrl ? { delete_url: input.deleteUrl } : {}),
+            ...(input.uploadId ? { upload_id: input.uploadId } : {}),
+            ...(input.completeUrl ? { complete_url: input.completeUrl } : {}),
+            ...(input.abortUrl ? { abort_url: input.abortUrl } : {}),
+            ...(input.listPageUrl ? { list_page_url: input.listPageUrl } : {}),
+            ...(input.finalHeadUrl ? { final_head_url: input.finalHeadUrl } : {}),
+            ...(input.finalObjectKey ? { final_object_key: input.finalObjectKey } : {}),
+            ...(input.expectedObjectSize !== undefined ? { expected_object_size: input.expectedObjectSize } : {}),
+            ...(input.expectedPartCount !== undefined ? { expected_part_count: input.expectedPartCount } : {}),
+            ...(input.maxPartNumber !== undefined ? { max_part_number: input.maxPartNumber } : {}),
+            ...(input.finalObjectMetadata ? { final_object_metadata: input.finalObjectMetadata } : {}),
             ...(input.partNumber ? { part_number: input.partNumber } : {})
         }
     };
@@ -299,6 +311,14 @@ async function signSourceRoute(input) {
 async function signDestinationUrl(input) {
     if (isS3CompatibleProvider(input.destination)) {
         const client = createS3CompatibleClient(input.destination);
+        if (input.uploadId && input.partNumber) {
+            return getSignedUrl(client, new UploadPartCommand({
+                Bucket: input.destination.bucket,
+                Key: input.objectKey,
+                UploadId: input.uploadId,
+                PartNumber: input.partNumber
+            }), { expiresIn: input.expiresIn });
+        }
         return getSignedUrl(client, new PutObjectCommand({ Bucket: input.destination.bucket, Key: input.objectKey }), { expiresIn: input.expiresIn });
     }
     if (isHippiusProvider(input.destination)) {
@@ -385,7 +405,8 @@ function s3CompatibleMetadata(source, endpoint = s3CompatibleEndpoint(source)) {
         key: source.key,
         region: s3CompatibleRegion(source),
         endpoint_url: endpoint,
-        account_id: "account_id" in source ? source.account_id : undefined
+        account_id: "account_id" in source ? source.account_id : undefined,
+        capabilities: "capabilities" in source ? source.capabilities : undefined
     });
 }
 function isHippiusProvider(config) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@beam-network/sdk",
-    "version": "0.4.1",
+    "version": "0.4.3",
     "description": "TypeScript SDK for BEAM transfer creation and management.",
     "type": "module",
     "license": "MIT",