@beam-network/sdk 0.2.10 → 0.2.12

package/dist/client.d.ts

@@ -1,4 +1,4 @@
-import type { AttachSignedUrlsResponse, BeamClientOptions, DistributeResponse, PreparedDestination, PreparedHttpSource, ProviderTransferCreateInput, RawTransferCreateInput, SignedChunkRoute, SignedUrlFlow, TransferCancelResponse, TransferCreateResponse, TransferPrepareResponse, TransferStatusInfo } from "./models.js";
+import type { AttachSignedUrlsResponse, BeamClientOptions, DistributeResponse, PlanningHttpSource, PreparedDestination, PreparedHttpSource, ProviderTransferCreateInput, RawTransferCreateInput, SignedChunkRoute, SignedUrlFlow, TransferCancelResponse, TransferCreateResponse, TransferPlanResponse, TransferPrepareResponse, TransferStatusInfo } from "./models.js";
 export declare const BEAM_DEFAULT_URL = "https://beamcore.b1m.ai";
 export declare class BeamApiError extends Error {
     readonly status: number;
@@ -15,11 +15,21 @@ export declare class BeamClient {
     transferStatus(transferId: string): Promise<TransferStatusInfo>;
     distributeTransfer(transferId: string): Promise<DistributeResponse>;
     cancelTransfer(transferId: string): Promise<TransferCancelResponse>;
+    planTransfer(input: {
+        sources: PlanningHttpSource[];
+        destinations: PreparedDestination[];
+        name?: string;
+        testMode?: boolean;
+        experimental?: boolean;
+        urlsExpiresAt?: string;
+        signedUrlFlow?: SignedUrlFlow;
+    }): Promise<TransferPlanResponse>;
     prepareTransfer(input: {
         sources: PreparedHttpSource[];
         destinations: PreparedDestination[];
         name?: string;
         testMode?: boolean;
+        experimental?: boolean;
         urlsExpiresAt?: string;
         signedUrlFlow?: SignedUrlFlow;
     }): Promise<TransferPrepareResponse>;

package/dist/client.js

@@ -1,4 +1,4 @@
-import { abortMultipartUpload, computeChunkContentMd5, createMultipartUpload, nowIso, prepareProviderDestination, prepareProviderSource, isS3CompatibleProvider, listPartsPageMarker, listPartsVerifyMarker, signAbortMultipartUpload, signCompleteMultipartUpload, signDeleteObject, signDestinationRoute, signListMultipartUpload, signUploadPartCopy } from "./provider-signing.js";
+import { abortMultipartUpload, computeChunksContentMd5, createMultipartUpload, nowIso, prepareProviderDestination, prepareProviderSource, prepareProviderSourceForPlan, isS3CompatibleProvider, listPartsPageMarker, listPartsVerifyMarker, signAbortMultipartUpload, signCompleteMultipartUpload, signDeleteObject, signDestinationRoute, signListMultipartUpload, signUploadPartCopy } from "./provider-signing.js";
 export const BEAM_DEFAULT_URL = "https://beamcore.b1m.ai";
 export class BeamApiError extends Error {
     status;
@@ -36,6 +36,7 @@ export class BeamClient {
             chunk_hashes: input.chunkHashes,
             callbacks: input.callbacks,
             test_mode: input.testMode || undefined,
+            experimental: input.experimental || undefined,
             progressive_mode: input.progressiveMode || undefined,
             signed_url_flow: input.signedUrlFlow
         });
@@ -58,12 +59,24 @@ export class BeamClient {
         validateId(transferId, "transferId");
         return this.request("DELETE", `/transfers/${encodeURIComponent(transferId)}`);
     }
+    planTransfer(input) {
+        return this.request("POST", "/transfers/plan", compact({
+            sources: input.sources,
+            destinations: input.destinations,
+            name: input.name,
+            test_mode: input.testMode || undefined,
+            experimental: input.experimental || undefined,
+            urls_expires_at: input.urlsExpiresAt,
+            signed_url_flow: input.signedUrlFlow
+        }));
+    }
     prepareTransfer(input) {
         return this.request("POST", "/transfers/prepare", compact({
             sources: input.sources,
             destinations: input.destinations,
             name: input.name,
             test_mode: input.testMode || undefined,
+            experimental: input.experimental || undefined,
             urls_expires_at: input.urlsExpiresAt,
             signed_url_flow: input.signedUrlFlow
         }));
@@ -79,45 +92,119 @@ export class BeamClient {
     }
     async prepareProviderTransfer(input) {
         const expiresIn = input.expiresIn ?? 3600;
-        const preparedSources = await Promise.all(input.sources.map((source, index) => prepareProviderSource(source, { index, expiresIn, fetchImpl: this.fetchImpl })));
+        const requestedSignedUrlFlow = input.signedUrlFlow ?? "signed_url_v1";
         const preparedDestinations = input.destinations.map((destination, index) => prepareProviderDestination(destination, { index }));
-        const prepared = await this.prepareTransfer({
-            sources: preparedSources,
-            destinations: preparedDestinations,
-            name: input.name,
-            testMode: input.testMode,
-            signedUrlFlow: input.signedUrlFlow
+        const destinationsById = new Map();
+        preparedDestinations.forEach((preparedDestination, index) => {
+            const destination = input.destinations[index];
+            if (!destination) {
+                throw new Error(`missing provider destination config for prepared destination ${preparedDestination.destination_id}`);
+            }
+            destinationsById.set(preparedDestination.destination_id, destination);
+        });
+        const checksumCache = new ChunkChecksumCache({
+            concurrency: input.checksumReadConcurrency ?? 8,
+            expiresIn,
+            fetchImpl: this.fetchImpl,
+            provider: input.checksumProvider,
+            groupSizeBytes: input.checksumGroupSizeBytes ?? 256 * 1024 * 1024,
+            onEvent: input.onChecksumEvent,
+            throwIfCancelled: () => input.throwIfCancelled?.()
         });
-        if (!prepared.success) {
-            return prepared;
+        let preparedSources;
+        let prepared;
+        if (requestedSignedUrlFlow === "signed_url_v1") {
+            await input.throwIfCancelled?.();
+            const plannedSources = await Promise.all(input.sources.map((source, index) => prepareProviderSourceForPlan(source, { index, fetchImpl: this.fetchImpl })));
+            const planned = await this.planTransfer({
+                sources: plannedSources,
+                destinations: preparedDestinations,
+                name: input.name,
+                testMode: input.testMode,
+                experimental: input.experimental,
+                signedUrlFlow: requestedSignedUrlFlow
+            });
+            if (!planned.success) {
+                return failedPlanAsPrepareResponse(planned);
+            }
+            await input.onPlanned?.(planned);
+            await input.throwIfCancelled?.();
+            const plannedSourcesById = mapSourcesByPreparedId(plannedSources, input.sources);
+            const chunksBySource = checksumChunksBySource(planned.chunk_plan, destinationsById);
+            const totalWarmupChunks = [...chunksBySource.values()].reduce((sum, chunks) => sum + chunks.length, 0);
+            const totalWarmupBytes = [...chunksBySource.values()].reduce((sum, chunks) => sum + chunks.reduce((chunkSum, chunk) => chunkSum + chunk.chunk_size, 0), 0);
+            if (totalWarmupChunks > 0) {
+                const startedAt = Date.now();
+                emitChecksumEvent(input, {
+                    event: "checksum_warmup_started",
+                    sourceId: "all",
+                    chunkCount: totalWarmupChunks,
+                    bytes: totalWarmupBytes
+                });
+                for (const [sourceId, chunks] of chunksBySource) {
+                    await input.throwIfCancelled?.();
+                    const source = plannedSourcesById.get(sourceId);
+                    if (!source) {
+                        throw new Error(`BeamCore planned unknown source_id: ${sourceId}`);
+                    }
+                    await checksumCache.ensureMany(source, chunks);
+                }
+                emitChecksumEvent(input, {
+                    event: "checksum_warmup_completed",
+                    sourceId: "all",
+                    chunkCount: totalWarmupChunks,
+                    bytes: totalWarmupBytes,
+                    durationMs: Date.now() - startedAt
+                });
+            }
+            await input.throwIfCancelled?.();
+            preparedSources = await Promise.all(input.sources.map((source, index) => prepareProviderSource(source, { index, expiresIn, fetchImpl: this.fetchImpl })));
+            await input.onBeforeTransferPrepare?.();
+            prepared = await this.prepareTransfer({
+                sources: preparedSources,
+                destinations: preparedDestinations,
+                name: input.name,
+                testMode: input.testMode,
+                experimental: input.experimental,
+                signedUrlFlow: requestedSignedUrlFlow
+            });
+            if (!prepared.success) {
+                return prepared;
+            }
+            assertPreparedTransferMatchesPlan(planned, prepared);
+        }
+        else {
+            preparedSources = await Promise.all(input.sources.map((source, index) => prepareProviderSource(source, { index, expiresIn, fetchImpl: this.fetchImpl })));
+            await input.onBeforeTransferPrepare?.();
+            prepared = await this.prepareTransfer({
+                sources: preparedSources,
+                destinations: preparedDestinations,
+                name: input.name,
+                testMode: input.testMode,
+                experimental: input.experimental,
+                signedUrlFlow: requestedSignedUrlFlow
+            });
+            if (!prepared.success) {
+                return prepared;
+            }
         }
         await input.onPrepared?.(prepared);
         await input.throwIfCancelled?.(prepared.transfer_id);
-        const destinationsById = new Map();
-        preparedDestinations.forEach((preparedDestination, index) => {
-            destinationsById.set(preparedDestination.destination_id, input.destinations[index]);
-        });
         const sourcesById = new Map();
         preparedSources.forEach((preparedSource, index) => {
-            sourcesById.set(preparedSource.source_id, input.sources[index]);
-        });
-        const signedUrlFlow = prepared.signed_url_flow ?? input.signedUrlFlow ?? "signed_url_v1";
-        const checksumCache = new ChunkChecksumCache({
-            concurrency: 4,
-            expiresIn,
-            fetchImpl: this.fetchImpl
+            const source = input.sources[index];
+            if (!source) {
+                throw new Error(`missing provider source config for prepared source ${preparedSource.source_id}`);
+            }
+            sourcesById.set(preparedSource.source_id, source);
         });
+        const signedUrlFlow = prepared.signed_url_flow ?? requestedSignedUrlFlow;
         if (signedUrlFlow === "signed_url_v1") {
-            for (const chunk of prepared.chunk_plan) {
-                const needsChecksum = chunk.destinations.some((target) => {
-                    const destination = destinationsById.get(target.destination_id);
-                    return destination ? isS3CompatibleProvider(destination) : false;
-                });
-                if (!needsChecksum)
-                    continue;
-                const source = sourcesById.get(chunk.source_id);
+            const chunksBySource = checksumChunksBySource(prepared.chunk_plan, destinationsById);
+            for (const [sourceId, chunks] of chunksBySource) {
+                const source = sourcesById.get(sourceId);
                 if (source)
-                    checksumCache.preload(source, chunk);
+                    checksumCache.preloadMany(source, chunks);
             }
         }
         const routes = [];
@@ -342,6 +429,83 @@ export class BeamClient {
         return (text ? JSON.parse(text) : {});
     }
 }
+function checksumChunksBySource(chunkPlan, destinationsById) {
+    const chunksBySource = new Map();
+    for (const chunk of chunkPlan) {
+        const needsChecksum = chunk.destinations.some((target) => {
+            const destination = destinationsById.get(target.destination_id);
+            return destination ? isS3CompatibleProvider(destination) : false;
+        });
+        if (!needsChecksum)
+            continue;
+        const chunks = chunksBySource.get(chunk.source_id) ?? [];
+        chunks.push(chunk);
+        chunksBySource.set(chunk.source_id, chunks);
+    }
+    return chunksBySource;
+}
+function mapSourcesByPreparedId(preparedSources, providerSources) {
+    const sourcesById = new Map();
+    preparedSources.forEach((preparedSource, index) => {
+        const source = providerSources[index];
+        if (!source) {
+            throw new Error(`missing provider source config for planned source ${preparedSource.source_id}`);
+        }
+        sourcesById.set(preparedSource.source_id, source);
+    });
+    return sourcesById;
+}
+function failedPlanAsPrepareResponse(plan) {
+    return {
+        success: false,
+        transfer_id: "",
+        chunk_size: plan.chunk_size,
+        total_size: plan.total_size,
+        total_sources: plan.total_sources,
+        total_destinations: plan.total_destinations,
+        logical_chunks: plan.logical_chunks,
+        total_chunks: plan.total_chunks,
+        chunk_plan: plan.chunk_plan ?? [],
+        signed_url_flow: plan.signed_url_flow,
+        error: plan.error,
+        message: plan.message
+    };
+}
+function assertPreparedTransferMatchesPlan(plan, prepared) {
+    const summaryMatches = plan.chunk_size === prepared.chunk_size &&
+        plan.total_size === prepared.total_size &&
+        plan.total_sources === prepared.total_sources &&
+        plan.total_destinations === prepared.total_destinations &&
+        plan.logical_chunks === prepared.logical_chunks &&
+        plan.total_chunks === prepared.total_chunks;
+    if (!summaryMatches || planSignature(plan.chunk_plan) !== planSignature(prepared.chunk_plan)) {
+        throw new Error("prepared transfer chunk plan does not match preflight checksum plan");
+    }
+}
+function planSignature(chunks) {
+    return JSON.stringify(chunks.map((chunk) => ({
+        chunk_index: chunk.chunk_index,
+        source_id: chunk.source_id,
+        source_chunk_index: chunk.source_chunk_index,
+        source_offset: chunk.source_offset,
+        chunk_size: chunk.chunk_size,
+        destinations: chunk.destinations.map((target) => {
+            const metadata = target.metadata ?? {};
+            return {
+                destination_id: target.destination_id,
+                provider: target.provider,
+                final_object_key: typeof metadata.final_object_key === "string" ? metadata.final_object_key : target.object_key,
+                part_number: typeof metadata.part_number === "number" ? metadata.part_number : undefined,
+                delivery_index: typeof metadata.delivery_index === "number" ? metadata.delivery_index : undefined
+            };
+        })
+    })));
+}
+function emitChecksumEvent(input, event) {
+    if (!input.onChecksumEvent)
+        return;
+    void Promise.resolve(input.onChecksumEvent(event)).catch(() => undefined);
+}
 class ChunkChecksumCache {
     options;
     queue = [];
@@ -350,22 +514,128 @@ class ChunkChecksumCache {
     constructor(options) {
         this.options = options;
     }
-    preload(source, chunk) {
-        void this.get(source, chunk).catch(() => undefined);
+    preloadMany(source, chunks) {
+        const pending = this.uniqueChunks(chunks).filter((chunk) => !this.cache.has(this.key(chunk)));
+        for (const chunk of pending) {
+            this.cache.set(this.key(chunk), deferredChecksum());
+        }
+        for (const group of this.groupChunks(pending)) {
+            void this.run(() => this.fillGroup(source, group)).catch(() => undefined);
+        }
+    }
+    async ensureMany(source, chunks) {
+        const unique = this.uniqueChunks(chunks);
+        this.preloadMany(source, unique);
+        await Promise.all(unique.map((chunk) => this.get(source, chunk)));
     }
     get(source, chunk) {
-        const key = `${chunk.source_id}:${chunk.source_chunk_index}`;
+        const key = this.key(chunk);
         let existing = this.cache.get(key);
         if (!existing) {
-            existing = this.run(() => computeChunkContentMd5({
+            existing = deferredChecksum();
+            this.cache.set(key, existing);
+            void this.run(() => this.fillGroup(source, [chunk])).catch(() => undefined);
+        }
+        return existing.promise;
+    }
+    async fillGroup(source, chunks) {
+        const unresolved = new Set(chunks.map((chunk) => this.key(chunk)));
+        try {
+            await this.options.throwIfCancelled?.();
+            const misses = [];
+            for (const chunk of chunks) {
+                const key = this.key(chunk);
+                const deferred = this.cache.get(key);
+                if (!deferred)
+                    continue;
+                const cached = await this.options.provider?.getContentMd5({ source, chunk, algorithm: "content-md5" }).catch(() => undefined);
+                if (cached) {
+                    this.emit({ event: "checksum_cache_hit", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
+                    deferred.resolve(cached);
+                    unresolved.delete(key);
+                }
+                else {
+                    this.emit({ event: "checksum_cache_miss", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
+                    misses.push(chunk);
+                }
+            }
+            if (!misses.length)
+                return;
+            await this.options.throwIfCancelled?.();
+            const startedAt = Date.now();
+            const checksums = await computeChunksContentMd5({
                 source,
-                chunk,
+                chunks: misses,
                 expiresIn: this.options.expiresIn,
-                fetchImpl: this.options.fetchImpl
-            }));
-            this.cache.set(key, existing);
+                fetchImpl: this.options.fetchImpl,
+                groupSizeBytes: this.options.groupSizeBytes
+            });
+            for (const chunk of misses) {
+                await this.options.throwIfCancelled?.();
+                const checksum = checksums.get(chunk.source_chunk_index);
+                if (!checksum)
+                    throw new Error(`checksum missing for source chunk ${chunk.source_chunk_index}`);
+                await this.options.provider?.setContentMd5?.({ source, chunk, algorithm: "content-md5", contentMd5: checksum });
+                const deferred = this.cache.get(this.key(chunk));
+                deferred?.resolve(checksum);
+                unresolved.delete(this.key(chunk));
+                this.emit({ event: "checksum_cold_chunk_cached", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
+            }
+            this.emit({
+                event: "checksum_cold_group_completed",
+                sourceId: misses[0]?.source_id ?? source.id ?? "source",
+                chunkCount: misses.length,
+                bytes: misses.reduce((sum, chunk) => sum + chunk.chunk_size, 0),
+                durationMs: Date.now() - startedAt
+            });
+        }
+        catch (error) {
+            for (const key of unresolved) {
+                this.cache.get(key)?.reject(error);
+            }
+            throw error;
         }
-        return existing;
+    }
+    uniqueChunks(chunks) {
+        const seen = new Set();
+        const output = [];
+        for (const chunk of chunks) {
+            const key = this.key(chunk);
+            if (seen.has(key))
+                continue;
+            seen.add(key);
+            output.push(chunk);
+        }
+        return output.sort((a, b) => a.source_offset - b.source_offset || a.source_chunk_index - b.source_chunk_index);
+    }
+    groupChunks(chunks) {
+        const groups = [];
+        let current = [];
+        let groupStart = 0;
+        let groupEnd = 0;
+        const maxBytes = Math.max(1, Math.floor(this.options.groupSizeBytes));
+        for (const chunk of chunks) {
+            const chunkStart = chunk.source_offset;
+            const chunkEnd = chunk.source_offset + chunk.chunk_size;
+            const startsNewGroup = current.length === 0 || chunkStart !== groupEnd || chunkEnd - groupStart > maxBytes;
+            if (startsNewGroup) {
+                if (current.length)
+                    groups.push(current);
+                current = [chunk];
+                groupStart = chunkStart;
+                groupEnd = chunkEnd;
+            }
+            else {
+                current.push(chunk);
+                groupEnd = chunkEnd;
+            }
+        }
+        if (current.length)
+            groups.push(current);
+        return groups;
+    }
+    key(chunk) {
+        return `${chunk.source_id}:${chunk.source_chunk_index}`;
     }
     run(task) {
         return new Promise((resolve, reject) => {
@@ -382,6 +652,20 @@ class ChunkChecksumCache {
                 this.queue.push(start);
         });
     }
+    emit(event) {
+        if (!this.options.onEvent)
+            return;
+        void Promise.resolve(this.options.onEvent(event)).catch(() => undefined);
+    }
+}
+function deferredChecksum() {
+    let resolve;
+    let reject;
+    const promise = new Promise((res, rej) => {
+        resolve = res;
+        reject = rej;
+    });
+    return { promise, resolve, reject };
 }
 export function calculateOptimalChunkSize(totalSize) {
     const mb = 1024 * 1024;

package/dist/models.d.ts

@@ -44,6 +44,7 @@ export interface TransferCreateRequest {
     chunk_hashes?: string[];
     callbacks?: CallbackConfig[];
     test_mode?: boolean;
+    experimental?: boolean;
     progressive_mode?: boolean;
     signed_url_flow?: SignedUrlFlow;
 }
@@ -60,12 +61,17 @@ export interface RawTransferCreateInput {
      * Maps to BeamCore test_mode.
      */
     testMode?: boolean;
+    /**
+     * Experimental transfers keep positive evidence but exclude failures and overseer interventions from PRISM.
+     */
+    experimental?: boolean;
     progressiveMode?: boolean;
     signedUrlFlow?: SignedUrlFlow;
 }
 export interface TransferCreateResponse {
     success: boolean;
     transfer_id: string;
+    experimental?: boolean;
     total_chunks: number;
     total_sources: number;
     total_destinations: number;
@@ -109,6 +115,7 @@ export interface TransferStatusInfo {
     transfer_id: string;
     status: string;
     error_message: string | null;
+    experimental?: boolean;
     started_at: string | null;
     completed_at: string | null;
 }
@@ -157,6 +164,26 @@ export interface HippiusProviderConfig {
 }
 export type ProviderConfig = S3ProviderConfig | R2ProviderConfig | S3CompatibleProviderConfig | HippiusProviderConfig;
 export type ProviderSourceConfig = ProviderConfig;
+export interface ChunkChecksumProviderGetInput {
+    source: ProviderSourceConfig;
+    chunk: ChunkSigningPlanItem;
+    algorithm: "content-md5";
+}
+export interface ChunkChecksumProviderSetInput extends ChunkChecksumProviderGetInput {
+    contentMd5: string;
+}
+export interface ChunkChecksumProvider {
+    getContentMd5(input: ChunkChecksumProviderGetInput): Promise<string | null | undefined>;
+    setContentMd5?(input: ChunkChecksumProviderSetInput): Promise<void>;
+}
+export interface ChunkChecksumEvent {
+    event: "checksum_warmup_started" | "checksum_warmup_completed" | "checksum_cache_hit" | "checksum_cache_miss" | "checksum_cold_chunk_cached" | "checksum_cold_group_completed";
+    sourceId: string;
+    sourceChunkIndex?: number;
+    chunkCount?: number;
+    bytes?: number;
+    durationMs?: number;
+}
 export type ProviderDestinationConfig = ProviderConfig;
 export type S3ProviderConfigInput = Omit<S3ProviderConfig, "provider">;
 export type R2ProviderConfigInput = Omit<R2ProviderConfig, "provider">;
@@ -182,26 +209,39 @@ export interface ProviderTransferCreateInput {
      * Maps to BeamCore test_mode.
      */
     testMode?: boolean;
+    /**
+     * Experimental transfers keep positive evidence but exclude failures and overseer interventions from PRISM.
+     */
+    experimental?: boolean;
     expiresIn?: number;
     /**
      * Defaults to true. Set to false to only prepare and attach signed URLs.
      */
     distribute?: boolean;
+    onPlanned?: (plan: TransferPlanResponse) => void | Promise<void>;
+    onBeforeTransferPrepare?: () => void | Promise<void>;
     onPrepared?: (prepared: TransferPrepareResponse) => void | Promise<void>;
-    throwIfCancelled?: (transferId: string) => void | Promise<void>;
+    throwIfCancelled?: (transferId?: string) => void | Promise<void>;
     signedUrlFlow?: SignedUrlFlow;
+    checksumProvider?: ChunkChecksumProvider;
+    checksumReadConcurrency?: number;
+    checksumGroupSizeBytes?: number;
+    onChecksumEvent?: (event: ChunkChecksumEvent) => void | Promise<void>;
 }
-export interface PreparedHttpSource {
+export interface PlanningHttpSource {
     source_id: string;
     type: "http";
     provider?: string;
-    url: string;
+    url?: string;
     size: number;
     filename?: string;
     headers?: Record<string, string>;
     expires_at?: string;
     metadata?: Record<string, unknown>;
 }
+export interface PreparedHttpSource extends PlanningHttpSource {
+    url: string;
+}
 export interface PreparedDestination {
     destination_id: string;
     provider: string;
@@ -237,11 +277,25 @@ export interface SignedChunkRoute {
     dest_headers?: Record<string, string>;
     metadata?: Record<string, unknown>;
 }
+export interface TransferPlanResponse {
+    success: boolean;
+    chunk_size?: number;
+    total_size?: number;
+    total_sources?: number;
+    total_destinations?: number;
+    logical_chunks?: number;
+    total_chunks?: number;
+    chunk_plan: ChunkSigningPlanItem[];
+    signed_url_flow?: SignedUrlFlow;
+    error?: string;
+    message?: string;
+}
 export interface TransferPrepareResponse {
     success: boolean;
     transfer_id: string;
     transfer_key?: string;
     test_mode?: boolean;
+    experimental?: boolean;
     chunk_size?: number;
     total_size?: number;
     total_sources?: number;

package/dist/provider-signing.d.ts

@@ -1,4 +1,4 @@
-import type { ChunkDestinationSigningTarget, ChunkSigningPlanItem, PreparedDestination, PreparedHttpSource, ProviderDestinationConfig, ProviderSourceConfig, R2ProviderConfig, S3CompatibleProviderConfig, S3ProviderConfig, SignedChunkRoute } from "./models.js";
+import type { ChunkDestinationSigningTarget, ChunkSigningPlanItem, PlanningHttpSource, PreparedDestination, PreparedHttpSource, ProviderDestinationConfig, ProviderSourceConfig, R2ProviderConfig, S3CompatibleProviderConfig, S3ProviderConfig, SignedChunkRoute } from "./models.js";
 export type AnyS3CompatibleProviderConfig = S3ProviderConfig | R2ProviderConfig | S3CompatibleProviderConfig;
 export declare function nowIso(): string;
 export declare function expiresAtIso(expiresIn: number): string;
@@ -7,6 +7,10 @@ export declare function prepareProviderSource(source: ProviderSourceConfig, opti
     expiresIn?: number;
     fetchImpl?: typeof fetch;
 }): Promise<PreparedHttpSource>;
+export declare function prepareProviderSourceForPlan(source: ProviderSourceConfig, options?: {
+    index?: number;
+    fetchImpl?: typeof fetch;
+}): Promise<PlanningHttpSource>;
 export declare function prepareProviderDestination(destination: ProviderDestinationConfig, options?: {
     index?: number;
 }): PreparedDestination;
@@ -51,6 +55,13 @@ export declare function signDestinationRoute(input: {
     deleteUrl?: string;
     fetchImpl?: typeof fetch;
 }): Promise<SignedChunkRoute>;
+export declare function computeChunksContentMd5(input: {
+    source: ProviderSourceConfig;
+    chunks: ChunkSigningPlanItem[];
+    expiresIn: number;
+    fetchImpl?: typeof fetch;
+    groupSizeBytes?: number;
+}): Promise<Map<number, string>>;
 export declare function computeChunkContentMd5(input: {
     source: ProviderSourceConfig;
     chunk: ChunkSigningPlanItem;

package/dist/provider-signing.js

@@ -23,7 +23,13 @@ export async function prepareProviderSource(source, options = {}) {
             size: Number(head.ContentLength ?? 0),
             filename: filename(source.key),
             expires_at: expiresAtIso(expiresIn),
-            metadata: s3CompatibleMetadata(source, endpoint)
+            metadata: {
+                ...s3CompatibleMetadata(source, endpoint),
+                content_length: Number(head.ContentLength ?? 0),
+                ...(head.ETag ? { etag: head.ETag } : {}),
+                ...(head.LastModified ? { last_modified: head.LastModified.toISOString() } : {}),
+                ...(head.VersionId ? { version_id: head.VersionId } : {})
+            }
         };
     }
     if (isHippiusProvider(source)) {
@@ -44,6 +50,42 @@ export async function prepareProviderSource(source, options = {}) {
     }
     return unsupportedProviderConfig(source);
 }
+export async function prepareProviderSourceForPlan(source, options = {}) {
+    const index = options.index ?? 0;
+    if (isS3CompatibleProvider(source)) {
+        const endpoint = s3CompatibleEndpoint(source);
+        const client = createS3CompatibleClient(source, endpoint);
+        const head = await client.send(new HeadObjectCommand({ Bucket: source.bucket, Key: source.key }));
+        return {
+            source_id: source.id ?? `src_${index}`,
+            type: "http",
+            provider: source.provider,
+            size: Number(head.ContentLength ?? 0),
+            filename: filename(source.key),
+            metadata: {
+                ...s3CompatibleMetadata(source, endpoint),
+                content_length: Number(head.ContentLength ?? 0),
+                ...(head.ETag ? { etag: head.ETag } : {}),
+                ...(head.LastModified ? { last_modified: head.LastModified.toISOString() } : {}),
+                ...(head.VersionId ? { version_id: head.VersionId } : {})
+            }
+        };
+    }
+    if (isHippiusProvider(source)) {
+        const fetchImpl = options.fetchImpl ?? globalThis.fetch;
+        const baseUrl = source.base_url ?? "https://api.hippius.com";
+        const size = await hippiusObjectSize(fetchImpl, baseUrl, source.api_token, source.bucket, source.key);
+        return {
+            source_id: source.id ?? `src_${index}`,
+            type: "http",
+            provider: "hippius",
+            size,
+            filename: filename(source.key),
+            metadata: { bucket: source.bucket, key: source.key, base_url: baseUrl }
+        };
+    }
+    return unsupportedProviderConfig(source);
+}
 export function prepareProviderDestination(destination, options = {}) {
     const index = options.index ?? 0;
     const destinationId = destination.id ?? `dst_${index}`;
@@ -221,8 +263,11 @@ export async function signDestinationRoute(input) {
     };
 }
 function rangeHeaderForChunk(chunk) {
-    const start = Math.max(0, Math.floor(chunk.source_offset));
-    const size = Math.max(1, Math.floor(chunk.chunk_size));
+    return rangeHeaderForRange(chunk.source_offset, chunk.chunk_size);
+}
+function rangeHeaderForRange(sourceOffset, chunkSize) {
+    const start = Math.max(0, Math.floor(sourceOffset));
+    const size = Math.max(1, Math.floor(chunkSize));
     return `bytes=${start}-${start + size - 1}`;
 }
 async function signSourceRoute(input) {
@@ -249,6 +294,27 @@ async function signSourceRoute(input) {
     }
     return unsupportedProviderConfig(input.source);
 }
+async function signSourceRangeRoute(input) {
+    const range = rangeHeaderForRange(input.sourceOffset, input.chunkSize);
+    if (isS3CompatibleProvider(input.source)) {
+        const client = createS3CompatibleClient(input.source);
+        return {
+            url: await getSignedUrl(client, new GetObjectCommand({
+                Bucket: input.source.bucket,
+                Key: input.source.key,
+                Range: range
+            }), { expiresIn: input.expiresIn }),
+            headers: { Range: range }
+        };
+    }
+    if (isHippiusProvider(input.source)) {
+        return {
+            url: await hippiusPresign(input.fetchImpl ?? globalThis.fetch, input.source.base_url ?? "https://api.hippius.com", input.source.api_token, input.source.bucket, input.source.key, "get", input.expiresIn),
+            headers: { Range: range }
+        };
+    }
+    return { url: input.fallbackUrl, headers: { Range: range } };
+}
 async function signDestinationUrl(input) {
     if (isS3CompatibleProvider(input.destination)) {
         const client = createS3CompatibleClient(input.destination);
@@ -268,27 +334,232 @@ async function signDestinationUrl(input) {
     }
     return unsupportedProviderConfig(input.destination);
 }
+export async function computeChunksContentMd5(input) {
+    const output = new Map();
+    const chunks = [...input.chunks].sort((a, b) => a.source_offset - b.source_offset || a.source_chunk_index - b.source_chunk_index);
+    const groups = groupContiguousChunks(chunks, input.groupSizeBytes ?? 256 * 1024 * 1024);
+    for (const group of groups) {
+        const groupOutput = await retryChecksumRead(() => computeChunkGroupContentMd5({
+            source: input.source,
+            chunks: group,
+            expiresIn: input.expiresIn,
+            fetchImpl: input.fetchImpl
+        }));
+        for (const [sourceChunkIndex, checksum] of groupOutput) {
+            output.set(sourceChunkIndex, checksum);
+        }
+    }
+    return output;
+}
 export async function computeChunkContentMd5(input) {
-    const sourceRoute = await signSourceRoute({
+    const checksums = await computeChunksContentMd5({
         source: input.source,
-        fallbackUrl: input.chunk.source_url,
-        chunk: input.chunk,
+        chunks: [input.chunk],
+        expiresIn: input.expiresIn,
+        fetchImpl: input.fetchImpl,
+        groupSizeBytes: input.chunk.chunk_size
+    });
+    const checksum = checksums.get(input.chunk.source_chunk_index);
+    if (!checksum) {
+        throw new Error(`chunk checksum missing for source chunk ${input.chunk.source_chunk_index}`);
+    }
+    return checksum;
+}
+function groupContiguousChunks(chunks, maxGroupSizeBytes) {
+    const groups = [];
+    let current = [];
+    let groupStart = 0;
+    let groupEnd = 0;
+    const maxBytes = Math.max(1, Math.floor(maxGroupSizeBytes));
+    for (const chunk of chunks) {
+        const chunkStart = Math.max(0, Math.floor(chunk.source_offset));
+        const chunkEnd = chunkStart + Math.max(1, Math.floor(chunk.chunk_size));
+        const startsNewGroup = current.length === 0 ||
+            chunkStart !== groupEnd ||
+            chunkEnd - groupStart > maxBytes;
+        if (startsNewGroup) {
+            if (current.length)
+                groups.push(current);
+            current = [chunk];
+            groupStart = chunkStart;
+            groupEnd = chunkEnd;
+            continue;
+        }
+        current.push(chunk);
+        groupEnd = chunkEnd;
+    }
+    if (current.length)
+        groups.push(current);
+    return groups;
+}
+async function computeChunkGroupContentMd5(input) {
+    if (!input.chunks.length)
+        return new Map();
+    const chunks = [...input.chunks].sort((a, b) => a.source_offset - b.source_offset || a.source_chunk_index - b.source_chunk_index);
+    const first = chunks[0];
+    const last = chunks[chunks.length - 1];
+    const sourceOffset = first.source_offset;
+    const totalSize = last.source_offset + last.chunk_size - sourceOffset;
+    if (isS3CompatibleProvider(input.source)) {
+        const client = createS3CompatibleClient(input.source);
+        const response = await client.send(new GetObjectCommand({
+            Bucket: input.source.bucket,
+            Key: input.source.key,
+            Range: `bytes=${sourceOffset}-${sourceOffset + totalSize - 1}`
+        }));
+        if (!response.Body) {
+            throw new Error(`chunk checksum read returned no body for ${input.source.bucket}/${input.source.key}`);
+        }
+        return hashChunkGroupContentMd5(chunks, totalSize, bodyToAsyncBytes(response.Body));
+    }
+    const sourceRoute = await signSourceRangeRoute({
+        source: input.source,
+        fallbackUrl: first.source_url,
+        sourceOffset,
+        chunkSize: totalSize,
         expiresIn: input.expiresIn,
         fetchImpl: input.fetchImpl
     });
     const fetchImpl = input.fetchImpl ?? globalThis.fetch;
     const response = await fetchImpl(sourceRoute.url, { method: "GET", headers: sourceRoute.headers });
     if (!response.ok) {
-        throw new Error(`chunk checksum fetch failed with ${response.status}: ${await response.text()}`);
+        const message = `chunk checksum fetch failed with ${response.status}: ${await response.text()}`;
+        throw Object.assign(new Error(message), { status: response.status });
+    }
+    if (response.body) {
+        return hashChunkGroupContentMd5(chunks, totalSize, readableStreamToAsyncBytes(response.body));
+    }
+    return hashChunkGroupContentMd5(chunks, totalSize, [new Uint8Array(await response.arrayBuffer())]);
+}
+async function hashChunkGroupContentMd5(chunks, totalSize, body) {
+    const output = new Map();
+    let chunkIndex = 0;
+    let bytesInChunk = 0;
+    let totalRead = 0;
+    let hash = createHash("md5");
+    const consume = (value) => {
+        let offset = 0;
+        while (offset < value.byteLength) {
+            const chunk = chunks[chunkIndex];
+            if (!chunk) {
+                throw new Error(`chunk checksum fetch size mismatch: read more than ${totalSize} bytes`);
+            }
+            const remainingInChunk = chunk.chunk_size - bytesInChunk;
+            const take = Math.min(remainingInChunk, value.byteLength - offset);
+            hash.update(value.subarray(offset, offset + take));
+            offset += take;
+            totalRead += take;
+            bytesInChunk += take;
+            if (bytesInChunk === chunk.chunk_size) {
+                output.set(chunk.source_chunk_index, hash.digest("base64"));
+                chunkIndex += 1;
+                bytesInChunk = 0;
+                hash = createHash("md5");
+            }
+        }
+    };
+    for await (const value of body) {
+        consume(value);
+    }
+    if (totalRead !== totalSize || output.size !== chunks.length) {
+        throw new Error(`chunk checksum fetch size mismatch: expected ${totalSize}, got ${totalRead}`);
+    }
+    return output;
+}
+async function* readableStreamToAsyncBytes(stream) {
+    const reader = stream.getReader();
+    try {
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            if (value)
+                yield value;
+        }
+    }
+    finally {
+        reader.releaseLock();
     }
-    const body = Buffer.from(await response.arrayBuffer());
-    if (body.length !== input.chunk.chunk_size) {
-        throw new Error(`chunk checksum fetch size mismatch: expected ${input.chunk.chunk_size}, got ${body.length}`);
+}
+async function* bodyToAsyncBytes(body) {
+    if (body instanceof Uint8Array) {
+        yield body;
+        return;
     }
-    return createHash("md5").update(body).digest("base64");
+    if (typeof body === "string") {
+        yield new TextEncoder().encode(body);
+        return;
+    }
+    if (body && typeof body.transformToByteArray === "function") {
+        yield new Uint8Array(await body.transformToByteArray());
+        return;
+    }
+    if (body && typeof body.arrayBuffer === "function") {
+        yield new Uint8Array(await body.arrayBuffer());
+        return;
+    }
+    if (body && typeof body.getReader === "function") {
+        yield* readableStreamToAsyncBytes(body);
+        return;
+    }
+    if (body && Symbol.asyncIterator in Object(body)) {
+        for await (const value of body) {
+            yield toUint8Array(value);
+        }
+        return;
+    }
+    throw new Error("provider checksum read returned an unsupported body stream");
+}
+function toUint8Array(value) {
+    if (value instanceof Uint8Array)
+        return value;
+    if (typeof value === "string")
+        return new TextEncoder().encode(value);
+    if (ArrayBuffer.isView(value)) {
+        return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
+    }
+    if (value instanceof ArrayBuffer)
+        return new Uint8Array(value);
+    throw new Error("provider checksum stream yielded an unsupported chunk");
+}
+async function retryChecksumRead(task) {
+    let lastError;
+    for (let attempt = 0; attempt < 3; attempt += 1) {
+        try {
+            return await task();
+        }
+        catch (error) {
+            lastError = error;
+            const status = typeof error === "object" && error && "status" in error ? Number(error.status) : undefined;
+            if (status && status >= 400 && status < 500 && status !== 408 && status !== 429)
+                break;
+            if (attempt < 2)
+                await sleep(250 * (attempt + 1));
+        }
+    }
+    throw lastError;
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+const s3ClientCache = new Map();
+function s3ClientCacheKey(source, endpoint) {
+    return JSON.stringify({
+        provider: source.provider,
+        endpoint,
+        region: s3CompatibleRegion(source),
+        forcePathStyle: s3CompatibleForcePathStyle(source, endpoint),
+        accessKeyId: source.access_key_id,
+        secretAccessKey: source.secret_access_key,
+        sessionToken: "session_token" in source ? source.session_token : undefined
+    });
 }
 function createS3CompatibleClient(source, endpoint = s3CompatibleEndpoint(source)) {
-    return new S3Client({
+    const cacheKey = s3ClientCacheKey(source, endpoint);
+    const cached = s3ClientCache.get(cacheKey);
+    if (cached)
+        return cached;
+    const client = new S3Client({
         region: s3CompatibleRegion(source),
         endpoint,
         forcePathStyle: s3CompatibleForcePathStyle(source, endpoint),
@@ -298,6 +569,8 @@ function createS3CompatibleClient(source, endpoint = s3CompatibleEndpoint(source
             sessionToken: "session_token" in source ? source.session_token : undefined
         }
     });
+    s3ClientCache.set(cacheKey, client);
+    return client;
 }
 export function isS3CompatibleProvider(config) {
     if (config.provider === "s3" || config.provider === "r2") {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@beam-network/sdk",
-  "version": "0.2.10",
+  "version": "0.2.12",
   "description": "TypeScript SDK for BEAM transfer creation and management.",
   "type": "module",
   "license": "MIT",