@beam-network/sdk 0.2.9 → 0.2.10

package/dist/client.d.ts

@@ -1,4 +1,4 @@
-import type { AttachSignedUrlsResponse, BeamClientOptions, DistributeResponse, PreparedDestination, PreparedHttpSource, ProviderTransferCreateInput, RawTransferCreateInput, SignedChunkRoute, TransferCancelResponse, TransferCreateResponse, TransferPrepareResponse, TransferStatusInfo } from "./models.js";
+import type { AttachSignedUrlsResponse, BeamClientOptions, DistributeResponse, PreparedDestination, PreparedHttpSource, ProviderTransferCreateInput, RawTransferCreateInput, SignedChunkRoute, SignedUrlFlow, TransferCancelResponse, TransferCreateResponse, TransferPrepareResponse, TransferStatusInfo } from "./models.js";
 export declare const BEAM_DEFAULT_URL = "https://beamcore.b1m.ai";
 export declare class BeamApiError extends Error {
     readonly status: number;
@@ -21,6 +21,7 @@ export declare class BeamClient {
         name?: string;
         testMode?: boolean;
         urlsExpiresAt?: string;
+        signedUrlFlow?: SignedUrlFlow;
     }): Promise<TransferPrepareResponse>;
     attachSignedUrls(transferId: string, input: {
         chunkRoutes: SignedChunkRoute[];

package/dist/client.js

@@ -1,4 +1,4 @@
-import { abortMultipartUpload, createMultipartUpload, nowIso, prepareProviderDestination, prepareProviderSource, signAbortMultipartUpload, signCompleteMultipartUpload, signDeleteObject, signDestinationRoute, signListMultipartUpload, signUploadPartCopy } from "./provider-signing.js";
+import { abortMultipartUpload, computeChunkContentMd5, createMultipartUpload, nowIso, prepareProviderDestination, prepareProviderSource, isS3CompatibleProvider, listPartsPageMarker, listPartsVerifyMarker, signAbortMultipartUpload, signCompleteMultipartUpload, signDeleteObject, signDestinationRoute, signListMultipartUpload, signUploadPartCopy } from "./provider-signing.js";
 export const BEAM_DEFAULT_URL = "https://beamcore.b1m.ai";
 export class BeamApiError extends Error {
     status;
@@ -36,7 +36,8 @@ export class BeamClient {
             chunk_hashes: input.chunkHashes,
             callbacks: input.callbacks,
             test_mode: input.testMode || undefined,
-            progressive_mode: input.progressiveMode || undefined
+            progressive_mode: input.progressiveMode || undefined,
+            signed_url_flow: input.signedUrlFlow
         });
         return this.request("POST", "/transfers/create", body);
     }
@@ -63,7 +64,8 @@ export class BeamClient {
             destinations: input.destinations,
             name: input.name,
             test_mode: input.testMode || undefined,
-            urls_expires_at: input.urlsExpiresAt
+            urls_expires_at: input.urlsExpiresAt,
+            signed_url_flow: input.signedUrlFlow
         }));
     }
     attachSignedUrls(transferId, input) {
@@ -83,7 +85,8 @@ export class BeamClient {
             sources: preparedSources,
             destinations: preparedDestinations,
             name: input.name,
-            testMode: input.testMode
+            testMode: input.testMode,
+            signedUrlFlow: input.signedUrlFlow
         });
         if (!prepared.success) {
             return prepared;
@@ -98,6 +101,25 @@ export class BeamClient {
         preparedSources.forEach((preparedSource, index) => {
             sourcesById.set(preparedSource.source_id, input.sources[index]);
         });
+        const signedUrlFlow = prepared.signed_url_flow ?? input.signedUrlFlow ?? "signed_url_v1";
+        const checksumCache = new ChunkChecksumCache({
+            concurrency: 4,
+            expiresIn,
+            fetchImpl: this.fetchImpl
+        });
+        if (signedUrlFlow === "signed_url_v1") {
+            for (const chunk of prepared.chunk_plan) {
+                const needsChecksum = chunk.destinations.some((target) => {
+                    const destination = destinationsById.get(target.destination_id);
+                    return destination ? isS3CompatibleProvider(destination) : false;
+                });
+                if (!needsChecksum)
+                    continue;
+                const source = sourcesById.get(chunk.source_id);
+                if (source)
+                    checksumCache.preload(source, chunk);
+            }
+        }
         const routes = [];
         const multipartUploads = new Map();
         try {
@@ -161,7 +183,9 @@ export class BeamClient {
                         finalObjectKey,
                         partNumber: typeof metadata.part_number === "number"
                             ? metadata.part_number
-                            : chunk.source_chunk_index + 1
+                            : chunk.source_chunk_index + 1,
+                        signedUrlFlow,
+                        checksumCache
                     }));
                 }
             }
@@ -207,6 +231,32 @@ export class BeamClient {
         if (!input.target.object_key) {
             throw new Error("destination signing target is missing object_key");
         }
+        if (input.signedUrlFlow === "signed_url_v1") {
+            const contentMd5 = await input.checksumCache.get(input.source, input.chunk);
+            const partVerifyUrl = await signListMultipartUpload(input.destination, input.finalObjectKey, input.upload.uploadId, input.expiresIn, { maxParts: 1, partNumberMarker: listPartsVerifyMarker(input.partNumber) });
+            const listPageUrl = await signListMultipartUpload(input.destination, input.finalObjectKey, input.upload.uploadId, input.expiresIn, { maxParts: 1000, partNumberMarker: listPartsPageMarker(input.partNumber) });
+            return signDestinationRoute({
+                chunk: input.chunk,
+                target: input.target,
+                source: input.source,
+                destination: input.destination,
+                expiresIn: input.expiresIn,
+                uploadId: input.upload.uploadId,
+                partNumber: input.partNumber,
+                multipartObjectKey: input.finalObjectKey,
+                completeUrl: input.upload.completeUrl,
+                abortUrl: input.upload.abortUrl,
+                listUrl: input.upload.listUrl,
+                partVerifyUrl,
+                listPageUrl,
+                contentMd5,
+                transferId: input.transferId,
+                multipartGroupId: input.upload.multipartGroupId,
+                multipartCreatedAt: input.upload.multipartCreatedAt,
+                commitMethod: "upload_part",
+                fetchImpl: this.fetchImpl
+            });
+        }
         const stagingObjectKey = buildStagingObjectKey({
             finalObjectKey: input.finalObjectKey,
             transferId: input.transferId,
@@ -292,6 +342,47 @@ export class BeamClient {
         return (text ? JSON.parse(text) : {});
     }
 }
+class ChunkChecksumCache {
+    options;
+    queue = [];
+    cache = new Map();
+    active = 0;
+    constructor(options) {
+        this.options = options;
+    }
+    preload(source, chunk) {
+        void this.get(source, chunk).catch(() => undefined);
+    }
+    get(source, chunk) {
+        const key = `${chunk.source_id}:${chunk.source_chunk_index}`;
+        let existing = this.cache.get(key);
+        if (!existing) {
+            existing = this.run(() => computeChunkContentMd5({
+                source,
+                chunk,
+                expiresIn: this.options.expiresIn,
+                fetchImpl: this.options.fetchImpl
+            }));
+            this.cache.set(key, existing);
+        }
+        return existing;
+    }
+    run(task) {
+        return new Promise((resolve, reject) => {
+            const start = () => {
+                this.active += 1;
+                task().then(resolve, reject).finally(() => {
+                    this.active -= 1;
+                    this.queue.shift()?.();
+                });
+            };
+            if (this.active < Math.max(1, this.options.concurrency))
+                start();
+            else
+                this.queue.push(start);
+        });
+    }
+}
 export function calculateOptimalChunkSize(totalSize) {
     const mb = 1024 * 1024;
     const gb = 1024 * mb;

package/dist/models.d.ts

@@ -1,3 +1,4 @@
+export type SignedUrlFlow = "signed_url_v1" | "signed_url_v2";
 export interface BeamClientOptions {
     apiKey?: string;
     baseUrl?: string;
@@ -44,6 +45,7 @@ export interface TransferCreateRequest {
     callbacks?: CallbackConfig[];
     test_mode?: boolean;
     progressive_mode?: boolean;
+    signed_url_flow?: SignedUrlFlow;
 }
 export interface RawTransferCreateInput {
     sources: SourceConfig[];
@@ -59,6 +61,7 @@ export interface RawTransferCreateInput {
      */
     testMode?: boolean;
     progressiveMode?: boolean;
+    signedUrlFlow?: SignedUrlFlow;
 }
 export interface TransferCreateResponse {
     success: boolean;
@@ -186,6 +189,7 @@ export interface ProviderTransferCreateInput {
     distribute?: boolean;
     onPrepared?: (prepared: TransferPrepareResponse) => void | Promise<void>;
     throwIfCancelled?: (transferId: string) => void | Promise<void>;
+    signedUrlFlow?: SignedUrlFlow;
 }
 export interface PreparedHttpSource {
     source_id: string;
@@ -230,6 +234,7 @@ export interface SignedChunkRoute {
     chunk_size: number;
     expires_at?: string;
     headers?: Record<string, string>;
+    dest_headers?: Record<string, string>;
     metadata?: Record<string, unknown>;
 }
 export interface TransferPrepareResponse {
@@ -244,6 +249,7 @@ export interface TransferPrepareResponse {
     logical_chunks?: number;
     total_chunks?: number;
     chunk_plan: ChunkSigningPlanItem[];
+    signed_url_flow?: SignedUrlFlow;
     error?: string;
     message?: string;
 }

package/dist/provider-signing.d.ts

@@ -13,7 +13,12 @@ export declare function prepareProviderDestination(destination: ProviderDestinat
 export declare function createMultipartUpload(destination: ProviderDestinationConfig, objectKey: string): Promise<string>;
 export declare function signCompleteMultipartUpload(destination: ProviderDestinationConfig, objectKey: string, uploadId: string, expiresIn: number): Promise<string>;
 export declare function signAbortMultipartUpload(destination: ProviderDestinationConfig, objectKey: string, uploadId: string, expiresIn: number): Promise<string>;
-export declare function signListMultipartUpload(destination: ProviderDestinationConfig, objectKey: string, uploadId: string, expiresIn: number): Promise<string>;
+export declare function signListMultipartUpload(destination: ProviderDestinationConfig, objectKey: string, uploadId: string, expiresIn: number, options?: {
+    maxParts?: number;
+    partNumberMarker?: number;
+}): Promise<string>;
+export declare function listPartsPageMarker(partNumber: number, pageSize?: number): number | undefined;
+export declare function listPartsVerifyMarker(partNumber: number): number | undefined;
 export declare function signUploadPartCopy(destination: ProviderDestinationConfig, finalObjectKey: string, stagingObjectKey: string, uploadId: string, partNumber: number, expiresIn: number, copySourceRange?: string): Promise<{
     copyUrl: string;
     copySource: string;
@@ -35,7 +40,10 @@ export declare function signDestinationRoute(input: {
     transferId?: string;
     multipartGroupId?: string;
     multipartCreatedAt?: string;
-    commitMethod?: "upload_part_copy";
+    commitMethod?: "upload_part" | "upload_part_copy";
+    contentMd5?: string;
+    partVerifyUrl?: string;
+    listPageUrl?: string;
     stagingObjectKey?: string;
     copyUrl?: string;
     copySource?: string;
@@ -43,6 +51,12 @@ export declare function signDestinationRoute(input: {
     deleteUrl?: string;
     fetchImpl?: typeof fetch;
 }): Promise<SignedChunkRoute>;
+export declare function computeChunkContentMd5(input: {
+    source: ProviderSourceConfig;
+    chunk: ChunkSigningPlanItem;
+    expiresIn: number;
+    fetchImpl?: typeof fetch;
+}): Promise<string>;
 export declare function isS3CompatibleProvider(config: ProviderSourceConfig | ProviderDestinationConfig): config is AnyS3CompatibleProviderConfig;
 export declare function s3CompatibleEndpoint(source: AnyS3CompatibleProviderConfig): string | undefined;
 export declare function s3CompatibleRegion(source: AnyS3CompatibleProviderConfig): string;

package/dist/provider-signing.js

@@ -1,3 +1,4 @@
+import { createHash } from "node:crypto";
 import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListPartsCommand, PutObjectCommand, S3Client, UploadPartCopyCommand, UploadPartCommand } from "@aws-sdk/client-s3";
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
 export function nowIso() {
@@ -96,13 +97,26 @@ export async function signAbortMultipartUpload(destination, objectKey, uploadId,
     }
     throw new Error(`multipart abort is not supported for ${destination.provider}`);
 }
-export async function signListMultipartUpload(destination, objectKey, uploadId, expiresIn) {
+export async function signListMultipartUpload(destination, objectKey, uploadId, expiresIn, options = {}) {
     if (isS3CompatibleProvider(destination)) {
         const client = createS3CompatibleClient(destination);
-        return getSignedUrl(client, new ListPartsCommand({ Bucket: destination.bucket, Key: objectKey, UploadId: uploadId }), { expiresIn });
+        return getSignedUrl(client, new ListPartsCommand({
+            Bucket: destination.bucket,
+            Key: objectKey,
+            UploadId: uploadId,
+            MaxParts: options.maxParts,
+            PartNumberMarker: options.partNumberMarker === undefined ? undefined : String(options.partNumberMarker)
+        }), { expiresIn });
     }
     throw new Error(`multipart list-parts is not supported for ${destination.provider}`);
 }
+export function listPartsPageMarker(partNumber, pageSize = 1000) {
+    const marker = Math.floor((Math.max(1, partNumber) - 1) / pageSize) * pageSize;
+    return marker > 0 ? marker : undefined;
+}
+export function listPartsVerifyMarker(partNumber) {
+    return partNumber > 1 ? partNumber - 1 : undefined;
+}
 export async function signUploadPartCopy(destination, finalObjectKey, stagingObjectKey, uploadId, partNumber, expiresIn, copySourceRange) {
     if (isS3CompatibleProvider(destination)) {
         const client = createS3CompatibleClient(destination);
@@ -163,7 +177,8 @@ export async function signDestinationRoute(input) {
         uploadId: input.uploadId,
         partNumber: input.partNumber,
         useStagingPut: input.commitMethod === "upload_part_copy",
-        fetchImpl: input.fetchImpl
+        fetchImpl: input.fetchImpl,
+        contentMd5: input.contentMd5
     });
     const sourceRoute = await signSourceRoute({
         source: input.source,
@@ -182,6 +197,7 @@ export async function signDestinationRoute(input) {
         chunk_size: input.chunk.chunk_size,
         expires_at: expiresAtIso(input.expiresIn),
         headers: sourceRoute.headers,
+        ...(input.contentMd5 ? { dest_headers: { "Content-MD5": input.contentMd5 } } : {}),
         metadata: {
             ...(input.target.metadata ?? {}),
             ...(input.transferId ? { transfer_id: input.transferId } : {}),
@@ -195,6 +211,9 @@ export async function signDestinationRoute(input) {
             ...(input.deleteUrl ? { delete_url: input.deleteUrl } : {}),
             ...(input.uploadId ? { upload_id: input.uploadId } : {}),
             ...(input.partNumber ? { part_number: input.partNumber } : {}),
+            ...(input.contentMd5 ? { checksum_algorithm: "content-md5", checksum_value: input.contentMd5 } : {}),
+            ...(input.partVerifyUrl ? { part_verify_url: input.partVerifyUrl } : {}),
+            ...(input.listPageUrl ? { list_page_url: input.listPageUrl } : {}),
             ...(input.completeUrl ? { complete_url: input.completeUrl } : {}),
             ...(input.abortUrl ? { abort_url: input.abortUrl } : {}),
             ...(input.listUrl ? { list_url: input.listUrl } : {})
@@ -238,7 +257,8 @@ async function signDestinationUrl(input) {
                 Bucket: input.destination.bucket,
                 Key: input.uploadPartObjectKey,
                 UploadId: input.uploadId,
-                PartNumber: input.partNumber
+                PartNumber: input.partNumber,
+                ContentMD5: input.contentMd5
             }), { expiresIn: input.expiresIn });
         }
         return getSignedUrl(client, new PutObjectCommand({ Bucket: input.destination.bucket, Key: input.objectKey }), { expiresIn: input.expiresIn });
@@ -248,6 +268,25 @@ async function signDestinationUrl(input) {
     }
     return unsupportedProviderConfig(input.destination);
 }
+export async function computeChunkContentMd5(input) {
+    const sourceRoute = await signSourceRoute({
+        source: input.source,
+        fallbackUrl: input.chunk.source_url,
+        chunk: input.chunk,
+        expiresIn: input.expiresIn,
+        fetchImpl: input.fetchImpl
+    });
+    const fetchImpl = input.fetchImpl ?? globalThis.fetch;
+    const response = await fetchImpl(sourceRoute.url, { method: "GET", headers: sourceRoute.headers });
+    if (!response.ok) {
+        throw new Error(`chunk checksum fetch failed with ${response.status}: ${await response.text()}`);
+    }
+    const body = Buffer.from(await response.arrayBuffer());
+    if (body.length !== input.chunk.chunk_size) {
+        throw new Error(`chunk checksum fetch size mismatch: expected ${input.chunk.chunk_size}, got ${body.length}`);
+    }
+    return createHash("md5").update(body).digest("base64");
+}
 function createS3CompatibleClient(source, endpoint = s3CompatibleEndpoint(source)) {
     return new S3Client({
         region: s3CompatibleRegion(source),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@beam-network/sdk",
-  "version": "0.2.9",
+  "version": "0.2.10",
   "description": "TypeScript SDK for BEAM transfer creation and management.",
   "type": "module",
   "license": "MIT",