@beam-network/sdk 0.2.13 → 0.2.15

package/dist/client.d.ts

@@ -48,6 +48,12 @@ export declare class BeamClient {
     private request;
 }
 export declare function calculateOptimalChunkSize(totalSize: number): number;
+export declare function buildStagingObjectPrefix(input: {
+    finalObjectKey: string;
+    transferId: string;
+    sourceId: string;
+    destinationId: string;
+}): string;
 export declare function buildStagingObjectKey(input: {
     finalObjectKey: string;
     transferId: string;

package/dist/client.js

@@ -1,4 +1,4 @@
-import { abortMultipartUpload, computeChunksContentMd5, createMultipartUpload, nowIso, prepareProviderDestination, prepareProviderSource, prepareProviderSourceForPlan, isS3CompatibleProvider, listPartsPageMarker, listPartsVerifyMarker, signAbortMultipartUpload, signCompleteMultipartUpload, signDeleteObject, signDestinationRoute, signListMultipartUpload, signUploadPartCopy } from "./provider-signing.js";
+import { abortMultipartUpload, computeChunksContentMd5, createMultipartUpload, nowIso, prepareProviderDestination, prepareProviderSource, prepareProviderSourceForPlan, isS3CompatibleProvider, listPartsPageMarker, listPartsVerifyMarker, signAbortMultipartUpload, signCompleteMultipartUpload, signDeleteObject, signDestinationRoute, signListMultipartUpload, signListObjects, signUploadPartCopy } from "./provider-signing.js";
 export const BEAM_DEFAULT_URL = "https://beamcore.b1m.ai";
 export class BeamApiError extends Error {
     status;
@@ -108,6 +108,7 @@ export class BeamClient {
             fetchImpl: this.fetchImpl,
             provider: input.checksumProvider,
             groupSizeBytes: input.checksumGroupSizeBytes ?? 256 * 1024 * 1024,
+            readTimeoutMs: input.checksumReadTimeoutMs,
             onEvent: input.onChecksumEvent,
             throwIfCancelled: () => input.throwIfCancelled?.()
         });
@@ -341,16 +342,19 @@ export class BeamClient {
                 fetchImpl: this.fetchImpl
             });
         }
-        const stagingObjectKey = buildStagingObjectKey({
+        const stagingObjectPrefix = buildStagingObjectPrefix({
             finalObjectKey: input.finalObjectKey,
             transferId: input.transferId,
             sourceId: input.chunk.source_id,
-            destinationId: input.target.destination_id,
-            chunkIndex: input.chunk.chunk_index,
-            partNumber: input.partNumber
+            destinationId: input.target.destination_id
         });
+        const stagingObjectKey = [
+            stagingObjectPrefix,
+            `part-${input.partNumber}-chunk-${input.chunk.chunk_index}`
+        ].join("/");
         const copySourceRange = `bytes=0-${Math.max(0, input.chunk.chunk_size - 1)}`;
         const { copyUrl, copySource } = await signUploadPartCopy(input.destination, input.finalObjectKey, stagingObjectKey, input.upload.uploadId, input.partNumber, input.expiresIn, copySourceRange);
+        const stagingListUrl = await signListObjects(input.destination, stagingObjectPrefix, input.expiresIn);
         const deleteUrl = await signDeleteObject(input.destination, stagingObjectKey, input.expiresIn);
         return signDestinationRoute({
             chunk: input.chunk,
@@ -369,6 +373,7 @@ export class BeamClient {
             multipartCreatedAt: input.upload.multipartCreatedAt,
             commitMethod: "upload_part_copy",
             stagingObjectKey,
+            stagingListUrl,
             copyUrl,
             copySource,
             copySourceRange,
@@ -576,7 +581,8 @@ class ChunkChecksumCache {
                 chunks: misses,
                 expiresIn: this.options.expiresIn,
                 fetchImpl: this.options.fetchImpl,
-                groupSizeBytes: this.options.groupSizeBytes
+                groupSizeBytes: this.options.groupSizeBytes,
+                readTimeoutMs: this.options.readTimeoutMs
             });
             for (const chunk of misses) {
                 await this.options.throwIfCancelled?.();
@@ -716,13 +722,18 @@ function normalizeBaseUrl(value) {
 function compact(input) {
     return Object.fromEntries(Object.entries(input).filter(([, value]) => value !== undefined));
 }
-export function buildStagingObjectKey(input) {
+export function buildStagingObjectPrefix(input) {
     const base = input.finalObjectKey.replace(/\/+$/, "") || "object";
     return [
         `${base}.beam-staging`,
         safeObjectKeySegment(input.transferId),
         safeObjectKeySegment(input.sourceId),
-        safeObjectKeySegment(input.destinationId),
+        safeObjectKeySegment(input.destinationId)
+    ].join("/");
+}
+export function buildStagingObjectKey(input) {
+    return [
+        buildStagingObjectPrefix(input),
         `part-${input.partNumber}-chunk-${input.chunkIndex}`
     ].join("/");
 }

package/dist/models.d.ts

@@ -226,6 +226,7 @@ export interface ProviderTransferCreateInput {
     checksumProvider?: ChunkChecksumProvider;
     checksumReadConcurrency?: number;
     checksumGroupSizeBytes?: number;
+    checksumReadTimeoutMs?: number;
     onChecksumEvent?: (event: ChunkChecksumEvent) => void | Promise<void>;
 }
 export interface PlanningHttpSource {

package/dist/provider-signing.d.ts

@@ -17,6 +17,7 @@ export declare function prepareProviderDestination(destination: ProviderDestinat
 export declare function createMultipartUpload(destination: ProviderDestinationConfig, objectKey: string): Promise<string>;
 export declare function signCompleteMultipartUpload(destination: ProviderDestinationConfig, objectKey: string, uploadId: string, expiresIn: number): Promise<string>;
 export declare function signAbortMultipartUpload(destination: ProviderDestinationConfig, objectKey: string, uploadId: string, expiresIn: number): Promise<string>;
+export declare function signListObjects(destination: ProviderDestinationConfig, prefix: string, expiresIn: number): Promise<string>;
 export declare function signListMultipartUpload(destination: ProviderDestinationConfig, objectKey: string, uploadId: string, expiresIn: number, options?: {
     maxParts?: number;
     partNumberMarker?: number;
@@ -49,6 +50,7 @@ export declare function signDestinationRoute(input: {
     partVerifyUrl?: string;
     listPageUrl?: string;
     stagingObjectKey?: string;
+    stagingListUrl?: string;
     copyUrl?: string;
     copySource?: string;
     copySourceRange?: string;
@@ -61,12 +63,14 @@ export declare function computeChunksContentMd5(input: {
     expiresIn: number;
     fetchImpl?: typeof fetch;
     groupSizeBytes?: number;
+    readTimeoutMs?: number;
 }): Promise<Map<number, string>>;
 export declare function computeChunkContentMd5(input: {
     source: ProviderSourceConfig;
     chunk: ChunkSigningPlanItem;
     expiresIn: number;
     fetchImpl?: typeof fetch;
+    readTimeoutMs?: number;
 }): Promise<string>;
 export declare function isS3CompatibleProvider(config: ProviderSourceConfig | ProviderDestinationConfig): config is AnyS3CompatibleProviderConfig;
 export declare function s3CompatibleEndpoint(source: AnyS3CompatibleProviderConfig): string | undefined;

package/dist/provider-signing.js

@@ -1,9 +1,10 @@
 import { createHash } from "node:crypto";
-import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListPartsCommand, PutObjectCommand, S3Client, UploadPartCopyCommand, UploadPartCommand } from "@aws-sdk/client-s3";
+import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, ListPartsCommand, PutObjectCommand, S3Client, UploadPartCopyCommand, UploadPartCommand } from "@aws-sdk/client-s3";
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
 export function nowIso() {
     return new Date().toISOString();
 }
+const DEFAULT_CHECKSUM_READ_TIMEOUT_MS = 300_000;
 export function expiresAtIso(expiresIn) {
     return new Date(Date.now() + expiresIn * 1000).toISOString();
 }
@@ -139,6 +140,13 @@ export async function signAbortMultipartUpload(destination, objectKey, uploadId,
     }
     throw new Error(`multipart abort is not supported for ${destination.provider}`);
 }
+export async function signListObjects(destination, prefix, expiresIn) {
+    if (isS3CompatibleProvider(destination)) {
+        const client = createS3CompatibleClient(destination);
+        return getSignedUrl(client, new ListObjectsV2Command({ Bucket: destination.bucket, Prefix: prefix }), { expiresIn });
+    }
+    throw new Error(`multipart staging list is not supported for ${destination.provider}`);
+}
 export async function signListMultipartUpload(destination, objectKey, uploadId, expiresIn, options = {}) {
     if (isS3CompatibleProvider(destination)) {
         const client = createS3CompatibleClient(destination);
@@ -247,6 +255,7 @@ export async function signDestinationRoute(input) {
             ...(input.multipartCreatedAt ? { multipart_created_at: input.multipartCreatedAt } : {}),
             ...(input.commitMethod ? { commit_method: input.commitMethod } : {}),
             ...(input.stagingObjectKey ? { staging_object_key: input.stagingObjectKey } : {}),
+            ...(input.stagingListUrl ? { staging_list_url: input.stagingListUrl } : {}),
             ...(input.copyUrl ? { copy_url: input.copyUrl } : {}),
             ...(input.copySource ? { copy_source: input.copySource } : {}),
             ...(input.copySourceRange ? { copy_source_range: input.copySourceRange } : {}),
@@ -339,12 +348,13 @@ export async function computeChunksContentMd5(input) {
     const chunks = [...input.chunks].sort((a, b) => a.source_offset - b.source_offset || a.source_chunk_index - b.source_chunk_index);
     const groups = groupContiguousChunks(chunks, input.groupSizeBytes ?? 256 * 1024 * 1024);
     for (const group of groups) {
-        const groupOutput = await retryChecksumRead(() => computeChunkGroupContentMd5({
+        const groupOutput = await computeChunkGroupContentMd5Adaptive({
             source: input.source,
             chunks: group,
             expiresIn: input.expiresIn,
-            fetchImpl: input.fetchImpl
-        }));
+            fetchImpl: input.fetchImpl,
+            readTimeoutMs: input.readTimeoutMs
+        });
         for (const [sourceChunkIndex, checksum] of groupOutput) {
             output.set(sourceChunkIndex, checksum);
         }
@@ -357,7 +367,8 @@ export async function computeChunkContentMd5(input) {
         chunks: [input.chunk],
         expiresIn: input.expiresIn,
         fetchImpl: input.fetchImpl,
-        groupSizeBytes: input.chunk.chunk_size
+        groupSizeBytes: input.chunk.chunk_size,
+        readTimeoutMs: input.readTimeoutMs
     });
     const checksum = checksums.get(input.chunk.source_chunk_index);
     if (!checksum) {
@@ -392,6 +403,25 @@ function groupContiguousChunks(chunks, maxGroupSizeBytes) {
         groups.push(current);
     return groups;
 }
+async function computeChunkGroupContentMd5Adaptive(input) {
+    try {
+        return await retryChecksumRead(() => computeChunkGroupContentMd5(input));
+    }
+    catch (error) {
+        if (input.chunks.length <= 1) {
+            throw error;
+        }
+        const midpoint = Math.ceil(input.chunks.length / 2);
+        const output = new Map();
+        for (const chunks of [input.chunks.slice(0, midpoint), input.chunks.slice(midpoint)]) {
+            const partial = await computeChunkGroupContentMd5Adaptive({ ...input, chunks });
+            for (const [sourceChunkIndex, checksum] of partial) {
+                output.set(sourceChunkIndex, checksum);
+            }
+        }
+        return output;
+    }
+}
 async function computeChunkGroupContentMd5(input) {
     if (!input.chunks.length)
         return new Map();
@@ -400,36 +430,73 @@ async function computeChunkGroupContentMd5(input) {
     const last = chunks[chunks.length - 1];
     const sourceOffset = first.source_offset;
     const totalSize = last.source_offset + last.chunk_size - sourceOffset;
-    if (isS3CompatibleProvider(input.source)) {
-        const client = createS3CompatibleClient(input.source);
-        const response = await client.send(new GetObjectCommand({
-            Bucket: input.source.bucket,
-            Key: input.source.key,
-            Range: `bytes=${sourceOffset}-${sourceOffset + totalSize - 1}`
-        }));
-        if (!response.Body) {
-            throw new Error(`chunk checksum read returned no body for ${input.source.bucket}/${input.source.key}`);
+    return withChecksumReadTimeout(input.readTimeoutMs, async (abortSignal) => {
+        if (isS3CompatibleProvider(input.source)) {
+            const client = createS3CompatibleClient(input.source);
+            const response = await client.send(new GetObjectCommand({
+                Bucket: input.source.bucket,
+                Key: input.source.key,
+                Range: `bytes=${sourceOffset}-${sourceOffset + totalSize - 1}`
+            }), abortSignal ? { abortSignal } : undefined);
+            if (!response.Body) {
+                throw new Error(`chunk checksum read returned no body for ${input.source.bucket}/${input.source.key}`);
+            }
+            return hashChunkGroupContentMd5(chunks, totalSize, bodyToAsyncBytes(response.Body));
         }
-        return hashChunkGroupContentMd5(chunks, totalSize, bodyToAsyncBytes(response.Body));
-    }
-    const sourceRoute = await signSourceRangeRoute({
-        source: input.source,
-        fallbackUrl: first.source_url,
-        sourceOffset,
-        chunkSize: totalSize,
-        expiresIn: input.expiresIn,
-        fetchImpl: input.fetchImpl
+        const sourceRoute = await signSourceRangeRoute({
+            source: input.source,
+            fallbackUrl: first.source_url,
+            sourceOffset,
+            chunkSize: totalSize,
+            expiresIn: input.expiresIn,
+            fetchImpl: input.fetchImpl
+        });
+        const fetchImpl = input.fetchImpl ?? globalThis.fetch;
+        const response = await fetchImpl(sourceRoute.url, { method: "GET", headers: sourceRoute.headers, signal: abortSignal });
+        if (!response.ok) {
+            const message = `chunk checksum fetch failed with ${response.status}: ${await response.text()}`;
+            throw Object.assign(new Error(message), { status: response.status });
+        }
+        if (response.body) {
+            return hashChunkGroupContentMd5(chunks, totalSize, readableStreamToAsyncBytes(response.body));
+        }
+        return hashChunkGroupContentMd5(chunks, totalSize, [new Uint8Array(await response.arrayBuffer())]);
     });
-    const fetchImpl = input.fetchImpl ?? globalThis.fetch;
-    const response = await fetchImpl(sourceRoute.url, { method: "GET", headers: sourceRoute.headers });
-    if (!response.ok) {
-        const message = `chunk checksum fetch failed with ${response.status}: ${await response.text()}`;
-        throw Object.assign(new Error(message), { status: response.status });
+}
+function normalizedChecksumReadTimeoutMs(value) {
+    if (value === undefined)
+        return DEFAULT_CHECKSUM_READ_TIMEOUT_MS;
+    if (!Number.isFinite(value) || value <= 0)
+        return 0;
+    return Math.floor(value);
+}
+async function withChecksumReadTimeout(readTimeoutMs, task) {
+    const timeoutMs = normalizedChecksumReadTimeoutMs(readTimeoutMs);
+    if (timeoutMs <= 0) {
+        return task(undefined);
+    }
+    const controller = new AbortController();
+    const timeoutError = Object.assign(new Error(`chunk checksum read timed out after ${timeoutMs}ms`), { code: "CHECKSUM_READ_TIMEOUT" });
+    let timer;
+    const timeout = new Promise((_resolve, reject) => {
+        timer = setTimeout(() => {
+            controller.abort(timeoutError);
+            reject(timeoutError);
+        }, timeoutMs);
+    });
+    try {
+        return await Promise.race([task(controller.signal), timeout]);
+    }
+    catch (error) {
+        if (controller.signal.aborted) {
+            throw timeoutError;
+        }
+        throw error;
     }
-    if (response.body) {
-        return hashChunkGroupContentMd5(chunks, totalSize, readableStreamToAsyncBytes(response.body));
+    finally {
+        if (timer)
+            clearTimeout(timer);
     }
-    return hashChunkGroupContentMd5(chunks, totalSize, [new Uint8Array(await response.arrayBuffer())]);
 }
 async function hashChunkGroupContentMd5(chunks, totalSize, body) {
     const output = new Map();
@@ -530,6 +597,8 @@ async function retryChecksumRead(task) {
         }
         catch (error) {
             lastError = error;
+            if (isChecksumReadTimeout(error))
+                break;
             const status = typeof error === "object" && error && "status" in error ? Number(error.status) : undefined;
             if (status && status >= 400 && status < 500 && status !== 408 && status !== 429)
                 break;
@@ -539,6 +608,12 @@ async function retryChecksumRead(task) {
     }
     throw lastError;
 }
+function isChecksumReadTimeout(error) {
+    return Boolean(error &&
+        typeof error === "object" &&
+        "code" in error &&
+        error.code === "CHECKSUM_READ_TIMEOUT");
+}
 function sleep(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@beam-network/sdk",
-  "version": "0.2.13",
+  "version": "0.2.15",
   "description": "TypeScript SDK for BEAM transfer creation and management.",
   "type": "module",
   "license": "MIT",