@beam-network/sdk 0.5.21 → 0.6.0

package/README.md

@@ -1,5 +1,10 @@
 # BEAM SDK for TypeScript
 
+This release requires Core's `transfer-client-control/v7` contract. Multipart uploads use consecutive parts;
+Core may request retained staging for recovery through the existing SDK signing connection. Workers receive
+ordinary upload URLs. Keep the client connected until the transfer reaches a terminal state so Core can renew
+grants, promote recovered data, and clean up staging. Drain active transfers before upgrading Core and SDK consumers.
+
 Install:
 
 ```bash
@@ -149,7 +154,7 @@ For low-level raw transfer configs, use `createRawTransfer`; lifecycle transport
 
 ## Route Streaming And Payload Size
 
-Provider transfers use `transfer-client-control/v6`. Every reply carries Runtime and transport epochs; prepare and every route-stream message carry a UUID route generation. S3, R2, and S3-compatible destinations retain direct multipart UploadPart/ListParts/HEAD handling. Hippius and Hugging Face destinations take a plain PUT per chunk instead, so they carry no multipart group manifest.
+Provider transfers use `transfer-client-control/v7`. Every reply carries Runtime and transport epochs; prepare and every route-stream message carry a UUID route generation. S3, R2, and S3-compatible destinations retain direct multipart UploadPart/ListParts/HEAD handling. Hippius and Hugging Face destinations take a plain PUT per chunk instead, so they carry no multipart group manifest.
 
 The client signs up to 64 routes concurrently by default and emits 2,048-route logical batches. Encoded MessagePack requests target 4 MiB physical batches under `maxPayloadBytes` (8 MiB by default; explicit positive overrides remain supported). The splitter reserves control-envelope headroom for the live auth token and stable request identity so the final encoded request remains below the configured guard. Lifecycle control refreshes cached SDK auth before the final 30 seconds of token lifetime and retries `auth_token_expired` replies with the same request identity and payload but a fresh token, preserving Core request-conflict protection during large route streams. A single route above the 4 MiB target but within the configured guard publishes alone; a single route larger than the configured limit fails before publication with both sizes in the error. Multipart create and abort requests use a separate `multipartControlConcurrency` limit of 2 by default, with explicit positive overrides supported, and S3-compatible control requests make up to five attempts for transient provider failures. This keeps URL signing throughput independent from provider control-plane pressure.
 

package/dist/client.js

@@ -1,4 +1,4 @@
-import { abortMultipartUpload, createMultipartUpload, expiresAtIso, prepareProviderDestination, prepareProviderSource, signAbortMultipartUpload, signCompleteMultipartUpload, signDestinationReadRange, signDestinationRoute, signFinalObjectHead, signListMultipartUpload, signSourceReadRange, isHuggingFaceProvider } from "./provider-signing.js";
+import { signMultipartRecovery, abortMultipartUpload, createMultipartUpload, expiresAtIso, prepareProviderDestination, prepareProviderSource, signAbortMultipartUpload, signCompleteMultipartUpload, signDestinationReadRange, signDestinationRoute, signFinalObjectHead, signListMultipartUpload, signSourceReadRange, isHuggingFaceProvider } from "./provider-signing.js";
 import { describe as describeHuggingFace, hashSourceStream, huggingFaceCommit, huggingFaceCompleteLfsUpload, huggingFaceLfsBatch, huggingFacePreupload, huggingFaceVerifyLfsUpload, readSourceSample } from "./huggingface.js";
 import { BeamTransferControl, BEAM_DEFAULT_NATS_URL, compactSignedRoutes, isRecoverableRouteStreamError } from "./nats-control.js";
 export class BeamRouteRecoveryPendingError extends Error {
@@ -821,7 +821,7 @@ export class BeamClient {
                         multipartUploads: input.multipartUploads,
                         expiresIn: input.expiresIn
                     });
-                return this.signProviderRoute({
+                const route = await this.signProviderRoute({
                     chunk,
                     target: {
                         ...target,
@@ -852,6 +852,7 @@ export class BeamClient {
                     signedUrlFlow: "signed_url",
                     partNumber: requested.part_number
                 });
+                return signMultipartRecovery({ destination, transferId: input.prepared.transfer_id, requested, route, expiresIn: input.expiresIn });
             });
             return {
                 transfer_id: payload.transfer_id,
@@ -1661,7 +1662,7 @@ function multipartListPageMarkers(maxPartNumber) {
     return markers;
 }
 function multipartMaxPartNumber(chunkCount) {
-    return multipartPartNumber(positiveInteger(chunkCount, "source chunk_count") - 1, 2);
+    return multipartPartNumber(positiveInteger(chunkCount, "source chunk_count") - 1);
 }
 function validateMultipartPartNumber(partNumber, manifest) {
     if (!Number.isInteger(partNumber) || partNumber < 1 || partNumber > manifest.max_part_number) {
@@ -1715,7 +1716,7 @@ const COMPACT_TRANSFER_PLAN_KEYS = new Set([
 const COMPACT_TRANSFER_PLAN_FORMULAS = {
     source_offset: "source_chunk_index * chunk_size",
     delivery_index: "chunk_index * destination_count + destination_index",
-    part_number: "source_chunk_index * 3 + attempt_slot + 1",
+    part_number: "source_chunk_index + 1",
     route_generation_id: "initial-{chunk_index}-{destination_id}"
 };
 function validateCompactTransferPlan(descriptor, signedUrlFlow) {
@@ -1729,7 +1730,7 @@ function validateCompactTransferPlan(descriptor, signedUrlFlow) {
     if (descriptor.version !== "compact-transfer-plan/v1") {
         throw new Error(`BeamCore returned unsupported plan version: ${descriptor.version}`);
     }
-    if (descriptor.multipart_attempt_slots !== 3) {
+    if (descriptor.multipart_attempt_slots !== 1) {
         throw new Error("BeamCore returned unsupported multipart attempt slot count");
     }
     const formulaKeys = Object.keys(descriptor.formulas);
@@ -1782,7 +1783,7 @@ function validateMultipartGroupManifest(manifest, transferId) {
         if (!Number.isInteger(group.expected_part_count) || group.expected_part_count < 1 || group.expected_part_count > 10_000) {
             throw new Error(`multipart group ${group.multipart_group_id} has invalid expected_part_count`);
         }
-        if (!Number.isInteger(group.max_part_number) || group.max_part_number < group.expected_part_count || group.max_part_number > 10_000) {
+        if (!Number.isInteger(group.max_part_number) || group.max_part_number !== group.expected_part_count || group.max_part_number > 10_000) {
             throw new Error(`multipart group ${group.multipart_group_id} has invalid max_part_number`);
         }
         if (!Number.isSafeInteger(group.expected_object_size) || group.expected_object_size <= 0) {
@@ -1810,7 +1811,8 @@ const PART_ROUTE_METADATA_KEYS = new Set([
     "logical_attempt_index",
     "attempt_slot",
     "route_generation_id",
-    "delivery_index"
+    "delivery_index",
+    "etag_required"
 ]);
 function partRouteMetadata(metadata) {
     return Object.fromEntries(Object.entries(metadata ?? {}).filter(([key]) => PART_ROUTE_METADATA_KEYS.has(key)));

package/dist/models.d.ts

@@ -179,7 +179,7 @@ export interface ProviderTransferResumeInput extends Omit<ProviderTransferCreate
     multipartGroups: ProviderMultipartGroupIdentity[];
 }
 export interface TransferTerminalEvent {
-    schema_version: "transfer-client-control/v6";
+    schema_version: "transfer-client-control/v7";
     producer: "transfer-runtime";
     transfer_id: string;
     status: "completed" | "failed" | "cancelled";
@@ -377,11 +377,11 @@ export interface CompactTransferPlanDescriptor {
     destinations: CompactTransferPlanDestination[];
     logical_chunk_count: number;
     delivery_route_count: number;
-    multipart_attempt_slots: 3;
+    multipart_attempt_slots: 1;
     formulas: {
         source_offset: "source_chunk_index * chunk_size";
         delivery_index: "chunk_index * destination_count + destination_index";
-        part_number: "source_chunk_index * 3 + attempt_slot + 1";
+        part_number: "source_chunk_index + 1";
         route_generation_id: "initial-{chunk_index}-{destination_id}";
     };
 }

package/dist/multipart-limits.d.ts

@@ -1,4 +1,4 @@
 export declare const MULTIPART_MAX_PART_NUMBER = 10000;
-export declare const MULTIPART_ATTEMPT_SLOT_COUNT = 3;
+export declare const MULTIPART_ATTEMPT_SLOT_COUNT = 1;
 export declare const MULTIPART_MAX_SOURCE_CHUNKS: number;
 export declare function multipartPartNumber(chunkIndex: number, attemptSlot?: number): number;

package/dist/multipart-limits.js

@@ -1,5 +1,5 @@
 export const MULTIPART_MAX_PART_NUMBER = 10_000;
-export const MULTIPART_ATTEMPT_SLOT_COUNT = 3;
+export const MULTIPART_ATTEMPT_SLOT_COUNT = 1;
 export const MULTIPART_MAX_SOURCE_CHUNKS = Math.floor(MULTIPART_MAX_PART_NUMBER / MULTIPART_ATTEMPT_SLOT_COUNT);
 export function multipartPartNumber(chunkIndex, attemptSlot = 0) {
     if (!Number.isInteger(chunkIndex) || chunkIndex < 0) {
@@ -9,8 +9,8 @@ export function multipartPartNumber(chunkIndex, attemptSlot = 0) {
         throw new Error(`source_chunk_index must be less than ${MULTIPART_MAX_SOURCE_CHUNKS}`);
     }
     if (!Number.isInteger(attemptSlot) || attemptSlot < 0 || attemptSlot >= MULTIPART_ATTEMPT_SLOT_COUNT) {
-        throw new Error("attempt_slot must be 0, 1, or 2");
+        throw new Error("attempt_slot must be 0 for consecutive multipart uploads");
     }
-    const partNumber = chunkIndex * MULTIPART_ATTEMPT_SLOT_COUNT + attemptSlot + 1;
+    const partNumber = chunkIndex + 1;
     return partNumber;
 }

package/dist/nats-control.d.ts

@@ -1,12 +1,20 @@
 import { type ConnectionOptions } from "nats";
 import type { MultipartGroupManifest, SignedChunkRoute, TransferTerminalSignalWaiter } from "./models.js";
-export declare const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/v6";
+export declare const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/v7";
 export declare const BEAM_DEFAULT_NATS_URL = "tls://orch-gateway.b1m.ai:4222";
 export declare const BEAM_DEFAULT_NATS_WS_URL = "wss://orch-gateway.b1m.ai:443";
 export declare const BEAM_DEFAULT_MAX_PAYLOAD_BYTES: number;
 export declare const BEAM_ROUTE_TARGET_PAYLOAD_BYTES: number;
 export declare const ROUTE_RECOVERY_SIGN_MESSAGE_TYPE: "transfer.route_recovery.sign";
 export interface RouteRecoverySignChunk {
+    recovery?: {
+        operation: "upload" | "controls" | "list" | "delete" | "renew";
+        mode: "staged" | "direct";
+        attempt_id: string;
+        object_key?: string;
+        etag?: string;
+        continuation_token?: string;
+    };
     source_id: string;
     destination_id: string;
     chunk_index: number;
@@ -14,7 +22,7 @@ export interface RouteRecoverySignChunk {
     source_offset: number;
     chunk_size: number;
     logical_attempt_index: number;
-    attempt_slot: 0 | 1 | 2;
+    attempt_slot: 0;
     part_number: number;
     route_generation_id: string;
     multipart_group_id: string;

package/dist/nats-control.js

@@ -1,6 +1,6 @@
 import { encode, decode } from "@msgpack/msgpack";
 import { connect } from "nats";
-export const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/v6";
+export const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/v7";
 export const BEAM_DEFAULT_NATS_URL = "tls://orch-gateway.b1m.ai:4222";
 export const BEAM_DEFAULT_NATS_WS_URL = "wss://orch-gateway.b1m.ai:443";
 // NATS enforces max_payload per message. This guard splits signed-route control
@@ -604,6 +604,7 @@ const ROUTE_ATTEMPT_METADATA_KEYS = new Set([
     "part_number",
     "logical_attempt_index",
     "attempt_slot",
+    "etag_required",
     "route_generation_id"
 ]);
 const ROUTE_IDENTITY_METADATA_KEYS = [

package/dist/provider-signing.d.ts

@@ -1,5 +1,14 @@
+import type { RouteRecoverySignChunk } from "./nats-control.js";
 import type { ChunkDestinationSigningTarget, ChunkSigningPlanItem, HuggingFaceProviderConfig, PlanningHttpSource, PreparedDestination, PreparedHttpSource, ProviderDestinationConfig, ProviderSourceConfig, R2ProviderConfig, S3CompatibleProviderConfig, S3ProviderConfig, SignedChunkRoute } from "./models.js";
 export type AnyS3CompatibleProviderConfig = S3ProviderConfig | R2ProviderConfig | S3CompatibleProviderConfig;
+/** Core-only grants: the worker receives just the ordinary upload URL. */
+export declare function signMultipartRecovery(input: {
+    destination: ProviderDestinationConfig;
+    transferId: string;
+    requested: RouteRecoverySignChunk;
+    route: SignedChunkRoute;
+    expiresIn: number;
+}): Promise<SignedChunkRoute>;
 export declare function nowIso(): string;
 export declare function expiresAtIso(expiresIn: number): string;
 export declare function prepareProviderSource(source: ProviderSourceConfig, options?: {

package/dist/provider-signing.js

@@ -1,6 +1,70 @@
 import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, GetObjectCommand, HeadObjectCommand, ListPartsCommand, PutObjectCommand, S3Client, UploadPartCommand } from "@aws-sdk/client-s3";
+import { UploadPartCopyCommand, DeleteObjectCommand, ListObjectsV2Command } from "@aws-sdk/client-s3";
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
 import { huggingFaceEndpoint, huggingFaceFileMetadata, huggingFaceRepoType, huggingFaceRevision } from "./huggingface.js";
+/** Core-only grants: the worker receives just the ordinary upload URL. */
+export async function signMultipartRecovery(input) {
+    const { requested: r, destination, expiresIn } = input;
+    const request = r.recovery;
+    if (request?.operation === "renew") {
+        const count = Number(input.route.metadata?.expected_part_count);
+        if (!Number.isInteger(count) || count < 1 || count > 10_000)
+            throw new Error("invalid multipart count");
+        const [complete, abort, head, pages] = await Promise.all([
+            signCompleteMultipartUpload(destination, r.final_object_key, r.upload_id, expiresIn),
+            signAbortMultipartUpload(destination, r.final_object_key, r.upload_id, expiresIn),
+            signFinalObjectHead(destination, r.final_object_key, expiresIn),
+            Promise.all(Array.from({ length: Math.ceil(count / 1000) }, (_, i) => signListMultipartUpload(destination, r.final_object_key, r.upload_id, expiresIn, { maxParts: 1000, partNumberMarker: i * 1000 })))
+        ]);
+        return { ...input.route, metadata: { ...input.route.metadata, complete_url: complete, abort_url: abort, final_head_url: head,
+                list_page_urls: pages, list_page_url: pages[Math.floor((r.part_number - 1) / 1000)], control_urls_expires_at: expiresAtIso(expiresIn) } };
+    }
+    if (!request || request.mode === "direct") {
+        const metadata = { ...input.route.metadata };
+        delete metadata.recovery_staging;
+        return { ...input.route, metadata };
+    }
+    if (!isS3CompatibleProvider(destination))
+        throw new Error("multipart recovery requires an S3-compatible destination");
+    const prefix = `${r.final_object_key}.beam-recovery/${input.transferId}/${encodeURIComponent(r.multipart_group_id).replace(/[!'()*]/g, c => `%${c.charCodeAt(0).toString(16).toUpperCase()}`)}/`;
+    const objectKey = `${prefix}${r.part_number}/${request.attempt_id}`;
+    if (request.object_key && request.object_key !== objectKey)
+        throw new Error("recovery staging identity mismatch");
+    const client = createS3CompatibleClient(destination);
+    const sign = (command) => {
+        if (command instanceof PutObjectCommand)
+            return getSignedUrl(client, command, { expiresIn });
+        if (command instanceof HeadObjectCommand)
+            return getSignedUrl(client, command, { expiresIn });
+        if (command instanceof UploadPartCopyCommand)
+            return getSignedUrl(client, command, { expiresIn, unhoistableHeaders: new Set(["x-amz-copy-source", "x-amz-copy-source-if-match"]) });
+        if (command instanceof DeleteObjectCommand)
+            return getSignedUrl(client, command, { expiresIn });
+        return getSignedUrl(client, command, { expiresIn });
+    };
+    const metadata = { ...input.route.metadata };
+    delete metadata.recovery_staging;
+    if (request.operation === "list") {
+        metadata.recovery_listing = { prefix, url: await sign(new ListObjectsV2Command({ Bucket: destination.bucket, Prefix: prefix, ContinuationToken: request.continuation_token, MaxKeys: 1000 })) };
+        return { ...input.route, metadata };
+    }
+    const params = { Bucket: destination.bucket, Key: objectKey };
+    if (request.operation === "delete") {
+        metadata.recovery_delete = { object_key: objectKey, url: await sign(new DeleteObjectCommand(params)) };
+        return { ...input.route, metadata };
+    }
+    const copySource = `${encodeURIComponent(destination.bucket)}/${objectKey.split("/").map(encodeURIComponent).join("/")}`;
+    // R2 does not promise to enforce copy source conditions. Each attempt has its own object.
+    const condition = destination.provider === "s3" && request.etag ? `"${request.etag.replace(/^"|"$/g, "")}"` : undefined;
+    const [head, copy, remove] = await Promise.all([
+        sign(new HeadObjectCommand(params)),
+        sign(new UploadPartCopyCommand({ Bucket: destination.bucket, Key: r.final_object_key, UploadId: r.upload_id, PartNumber: r.part_number, CopySource: copySource, CopySourceIfMatch: condition })),
+        sign(new DeleteObjectCommand(params))
+    ]);
+    metadata.recovery_staging = { object_key: objectKey, attempt_id: request.attempt_id, head_url: head, copy_url: copy, delete_url: remove,
+        copy_headers: { "x-amz-copy-source": copySource, ...(condition ? { "x-amz-copy-source-if-match": condition } : {}) }, expires_at: expiresAtIso(expiresIn) };
+    return { ...input.route, metadata, ...(request.operation === "upload" ? { dest_url: await sign(new PutObjectCommand(params)) } : {}) };
+}
 export function nowIso() {
     return new Date().toISOString();
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@beam-network/sdk",
-    "version": "0.5.21",
+    "version": "0.6.0",
     "description": "TypeScript SDK for BEAM transfer creation and management.",
     "type": "module",
     "license": "MIT",