@beam-network/sdk 0.2.11 → 0.2.12

package/dist/client.d.ts

@@ -1,4 +1,4 @@
-import type { AttachSignedUrlsResponse, BeamClientOptions, DistributeResponse, PreparedDestination, PreparedHttpSource, ProviderTransferCreateInput, RawTransferCreateInput, SignedChunkRoute, SignedUrlFlow, TransferCancelResponse, TransferCreateResponse, TransferPlanResponse, TransferPrepareResponse, TransferStatusInfo } from "./models.js";
+import type { AttachSignedUrlsResponse, BeamClientOptions, DistributeResponse, PlanningHttpSource, PreparedDestination, PreparedHttpSource, ProviderTransferCreateInput, RawTransferCreateInput, SignedChunkRoute, SignedUrlFlow, TransferCancelResponse, TransferCreateResponse, TransferPlanResponse, TransferPrepareResponse, TransferStatusInfo } from "./models.js";
 export declare const BEAM_DEFAULT_URL = "https://beamcore.b1m.ai";
 export declare class BeamApiError extends Error {
     readonly status: number;
@@ -16,7 +16,7 @@ export declare class BeamClient {
     distributeTransfer(transferId: string): Promise<DistributeResponse>;
     cancelTransfer(transferId: string): Promise<TransferCancelResponse>;
     planTransfer(input: {
-        sources: PreparedHttpSource[];
+        sources: PlanningHttpSource[];
         destinations: PreparedDestination[];
         name?: string;
         testMode?: boolean;

package/dist/client.js

@@ -1,4 +1,4 @@
-import { abortMultipartUpload, computeChunksContentMd5, createMultipartUpload, nowIso, prepareProviderDestination, prepareProviderSource, isS3CompatibleProvider, listPartsPageMarker, listPartsVerifyMarker, signAbortMultipartUpload, signCompleteMultipartUpload, signDeleteObject, signDestinationRoute, signListMultipartUpload, signUploadPartCopy } from "./provider-signing.js";
+import { abortMultipartUpload, computeChunksContentMd5, createMultipartUpload, nowIso, prepareProviderDestination, prepareProviderSource, prepareProviderSourceForPlan, isS3CompatibleProvider, listPartsPageMarker, listPartsVerifyMarker, signAbortMultipartUpload, signCompleteMultipartUpload, signDeleteObject, signDestinationRoute, signListMultipartUpload, signUploadPartCopy } from "./provider-signing.js";
 export const BEAM_DEFAULT_URL = "https://beamcore.b1m.ai";
 export class BeamApiError extends Error {
     status;
@@ -92,21 +92,8 @@ export class BeamClient {
     }
     async prepareProviderTransfer(input) {
         const expiresIn = input.expiresIn ?? 3600;
-        const preparedSources = await Promise.all(input.sources.map((source, index) => prepareProviderSource(source, { index, expiresIn, fetchImpl: this.fetchImpl })));
+        const requestedSignedUrlFlow = input.signedUrlFlow ?? "signed_url_v1";
         const preparedDestinations = input.destinations.map((destination, index) => prepareProviderDestination(destination, { index }));
-        const prepared = await this.prepareTransfer({
-            sources: preparedSources,
-            destinations: preparedDestinations,
-            name: input.name,
-            testMode: input.testMode,
-            experimental: input.experimental,
-            signedUrlFlow: input.signedUrlFlow
-        });
-        if (!prepared.success) {
-            return prepared;
-        }
-        await input.onPrepared?.(prepared);
-        await input.throwIfCancelled?.(prepared.transfer_id);
         const destinationsById = new Map();
         preparedDestinations.forEach((preparedDestination, index) => {
             const destination = input.destinations[index];
@@ -115,36 +102,105 @@ export class BeamClient {
             }
             destinationsById.set(preparedDestination.destination_id, destination);
         });
-        const sourcesById = new Map();
-        preparedSources.forEach((preparedSource, index) => {
-            const source = input.sources[index];
-            if (!source) {
-                throw new Error(`missing provider source config for prepared source ${preparedSource.source_id}`);
-            }
-            sourcesById.set(preparedSource.source_id, source);
-        });
-        const signedUrlFlow = prepared.signed_url_flow ?? input.signedUrlFlow ?? "signed_url_v1";
         const checksumCache = new ChunkChecksumCache({
             concurrency: input.checksumReadConcurrency ?? 8,
             expiresIn,
             fetchImpl: this.fetchImpl,
             provider: input.checksumProvider,
             groupSizeBytes: input.checksumGroupSizeBytes ?? 256 * 1024 * 1024,
-            onEvent: input.onChecksumEvent
+            onEvent: input.onChecksumEvent,
+            throwIfCancelled: () => input.throwIfCancelled?.()
         });
-        if (signedUrlFlow === "signed_url_v1") {
-            const chunksBySource = new Map();
-            for (const chunk of prepared.chunk_plan) {
-                const needsChecksum = chunk.destinations.some((target) => {
-                    const destination = destinationsById.get(target.destination_id);
-                    return destination ? isS3CompatibleProvider(destination) : false;
+        let preparedSources;
+        let prepared;
+        if (requestedSignedUrlFlow === "signed_url_v1") {
+            await input.throwIfCancelled?.();
+            const plannedSources = await Promise.all(input.sources.map((source, index) => prepareProviderSourceForPlan(source, { index, fetchImpl: this.fetchImpl })));
+            const planned = await this.planTransfer({
+                sources: plannedSources,
+                destinations: preparedDestinations,
+                name: input.name,
+                testMode: input.testMode,
+                experimental: input.experimental,
+                signedUrlFlow: requestedSignedUrlFlow
+            });
+            if (!planned.success) {
+                return failedPlanAsPrepareResponse(planned);
+            }
+            await input.onPlanned?.(planned);
+            await input.throwIfCancelled?.();
+            const plannedSourcesById = mapSourcesByPreparedId(plannedSources, input.sources);
+            const chunksBySource = checksumChunksBySource(planned.chunk_plan, destinationsById);
+            const totalWarmupChunks = [...chunksBySource.values()].reduce((sum, chunks) => sum + chunks.length, 0);
+            const totalWarmupBytes = [...chunksBySource.values()].reduce((sum, chunks) => sum + chunks.reduce((chunkSum, chunk) => chunkSum + chunk.chunk_size, 0), 0);
+            if (totalWarmupChunks > 0) {
+                const startedAt = Date.now();
+                emitChecksumEvent(input, {
+                    event: "checksum_warmup_started",
+                    sourceId: "all",
+                    chunkCount: totalWarmupChunks,
+                    bytes: totalWarmupBytes
                 });
-                if (!needsChecksum)
-                    continue;
-                const chunks = chunksBySource.get(chunk.source_id) ?? [];
-                chunks.push(chunk);
-                chunksBySource.set(chunk.source_id, chunks);
+                for (const [sourceId, chunks] of chunksBySource) {
+                    await input.throwIfCancelled?.();
+                    const source = plannedSourcesById.get(sourceId);
+                    if (!source) {
+                        throw new Error(`BeamCore planned unknown source_id: ${sourceId}`);
+                    }
+                    await checksumCache.ensureMany(source, chunks);
+                }
+                emitChecksumEvent(input, {
+                    event: "checksum_warmup_completed",
+                    sourceId: "all",
+                    chunkCount: totalWarmupChunks,
+                    bytes: totalWarmupBytes,
+                    durationMs: Date.now() - startedAt
+                });
+            }
+            await input.throwIfCancelled?.();
+            preparedSources = await Promise.all(input.sources.map((source, index) => prepareProviderSource(source, { index, expiresIn, fetchImpl: this.fetchImpl })));
+            await input.onBeforeTransferPrepare?.();
+            prepared = await this.prepareTransfer({
+                sources: preparedSources,
+                destinations: preparedDestinations,
+                name: input.name,
+                testMode: input.testMode,
+                experimental: input.experimental,
+                signedUrlFlow: requestedSignedUrlFlow
+            });
+            if (!prepared.success) {
+                return prepared;
             }
+            assertPreparedTransferMatchesPlan(planned, prepared);
+        }
+        else {
+            preparedSources = await Promise.all(input.sources.map((source, index) => prepareProviderSource(source, { index, expiresIn, fetchImpl: this.fetchImpl })));
+            await input.onBeforeTransferPrepare?.();
+            prepared = await this.prepareTransfer({
+                sources: preparedSources,
+                destinations: preparedDestinations,
+                name: input.name,
+                testMode: input.testMode,
+                experimental: input.experimental,
+                signedUrlFlow: requestedSignedUrlFlow
+            });
+            if (!prepared.success) {
+                return prepared;
+            }
+        }
+        await input.onPrepared?.(prepared);
+        await input.throwIfCancelled?.(prepared.transfer_id);
+        const sourcesById = new Map();
+        preparedSources.forEach((preparedSource, index) => {
+            const source = input.sources[index];
+            if (!source) {
+                throw new Error(`missing provider source config for prepared source ${preparedSource.source_id}`);
+            }
+            sourcesById.set(preparedSource.source_id, source);
+        });
+        const signedUrlFlow = prepared.signed_url_flow ?? requestedSignedUrlFlow;
+        if (signedUrlFlow === "signed_url_v1") {
+            const chunksBySource = checksumChunksBySource(prepared.chunk_plan, destinationsById);
             for (const [sourceId, chunks] of chunksBySource) {
                 const source = sourcesById.get(sourceId);
                 if (source)
@@ -373,6 +429,83 @@ export class BeamClient {
         return (text ? JSON.parse(text) : {});
     }
 }
+function checksumChunksBySource(chunkPlan, destinationsById) {
+    const chunksBySource = new Map();
+    for (const chunk of chunkPlan) {
+        const needsChecksum = chunk.destinations.some((target) => {
+            const destination = destinationsById.get(target.destination_id);
+            return destination ? isS3CompatibleProvider(destination) : false;
+        });
+        if (!needsChecksum)
+            continue;
+        const chunks = chunksBySource.get(chunk.source_id) ?? [];
+        chunks.push(chunk);
+        chunksBySource.set(chunk.source_id, chunks);
+    }
+    return chunksBySource;
+}
+function mapSourcesByPreparedId(preparedSources, providerSources) {
+    const sourcesById = new Map();
+    preparedSources.forEach((preparedSource, index) => {
+        const source = providerSources[index];
+        if (!source) {
+            throw new Error(`missing provider source config for planned source ${preparedSource.source_id}`);
+        }
+        sourcesById.set(preparedSource.source_id, source);
+    });
+    return sourcesById;
+}
+function failedPlanAsPrepareResponse(plan) {
+    return {
+        success: false,
+        transfer_id: "",
+        chunk_size: plan.chunk_size,
+        total_size: plan.total_size,
+        total_sources: plan.total_sources,
+        total_destinations: plan.total_destinations,
+        logical_chunks: plan.logical_chunks,
+        total_chunks: plan.total_chunks,
+        chunk_plan: plan.chunk_plan ?? [],
+        signed_url_flow: plan.signed_url_flow,
+        error: plan.error,
+        message: plan.message
+    };
+}
+function assertPreparedTransferMatchesPlan(plan, prepared) {
+    const summaryMatches = plan.chunk_size === prepared.chunk_size &&
+        plan.total_size === prepared.total_size &&
+        plan.total_sources === prepared.total_sources &&
+        plan.total_destinations === prepared.total_destinations &&
+        plan.logical_chunks === prepared.logical_chunks &&
+        plan.total_chunks === prepared.total_chunks;
+    if (!summaryMatches || planSignature(plan.chunk_plan) !== planSignature(prepared.chunk_plan)) {
+        throw new Error("prepared transfer chunk plan does not match preflight checksum plan");
+    }
+}
+function planSignature(chunks) {
+    return JSON.stringify(chunks.map((chunk) => ({
+        chunk_index: chunk.chunk_index,
+        source_id: chunk.source_id,
+        source_chunk_index: chunk.source_chunk_index,
+        source_offset: chunk.source_offset,
+        chunk_size: chunk.chunk_size,
+        destinations: chunk.destinations.map((target) => {
+            const metadata = target.metadata ?? {};
+            return {
+                destination_id: target.destination_id,
+                provider: target.provider,
+                final_object_key: typeof metadata.final_object_key === "string" ? metadata.final_object_key : target.object_key,
+                part_number: typeof metadata.part_number === "number" ? metadata.part_number : undefined,
+                delivery_index: typeof metadata.delivery_index === "number" ? metadata.delivery_index : undefined
+            };
+        })
+    })));
+}
+function emitChecksumEvent(input, event) {
+    if (!input.onChecksumEvent)
+        return;
+    void Promise.resolve(input.onChecksumEvent(event)).catch(() => undefined);
+}
 class ChunkChecksumCache {
     options;
     queue = [];
@@ -390,6 +523,11 @@ class ChunkChecksumCache {
             void this.run(() => this.fillGroup(source, group)).catch(() => undefined);
         }
     }
+    async ensureMany(source, chunks) {
+        const unique = this.uniqueChunks(chunks);
+        this.preloadMany(source, unique);
+        await Promise.all(unique.map((chunk) => this.get(source, chunk)));
+    }
     get(source, chunk) {
         const key = this.key(chunk);
         let existing = this.cache.get(key);
@@ -403,6 +541,7 @@ class ChunkChecksumCache {
     async fillGroup(source, chunks) {
         const unresolved = new Set(chunks.map((chunk) => this.key(chunk)));
         try {
+            await this.options.throwIfCancelled?.();
             const misses = [];
             for (const chunk of chunks) {
                 const key = this.key(chunk);
@@ -422,6 +561,7 @@ class ChunkChecksumCache {
             }
             if (!misses.length)
                 return;
+            await this.options.throwIfCancelled?.();
             const startedAt = Date.now();
             const checksums = await computeChunksContentMd5({
                 source,
@@ -431,6 +571,7 @@ class ChunkChecksumCache {
                 groupSizeBytes: this.options.groupSizeBytes
             });
             for (const chunk of misses) {
+                await this.options.throwIfCancelled?.();
                 const checksum = checksums.get(chunk.source_chunk_index);
                 if (!checksum)
                     throw new Error(`checksum missing for source chunk ${chunk.source_chunk_index}`);

package/dist/models.d.ts

@@ -177,7 +177,7 @@ export interface ChunkChecksumProvider {
     setContentMd5?(input: ChunkChecksumProviderSetInput): Promise<void>;
 }
 export interface ChunkChecksumEvent {
-    event: "checksum_cache_hit" | "checksum_cache_miss" | "checksum_cold_chunk_cached" | "checksum_cold_group_completed";
+    event: "checksum_warmup_started" | "checksum_warmup_completed" | "checksum_cache_hit" | "checksum_cache_miss" | "checksum_cold_chunk_cached" | "checksum_cold_group_completed";
     sourceId: string;
     sourceChunkIndex?: number;
     chunkCount?: number;
@@ -218,25 +218,30 @@ export interface ProviderTransferCreateInput {
      * Defaults to true. Set to false to only prepare and attach signed URLs.
      */
     distribute?: boolean;
+    onPlanned?: (plan: TransferPlanResponse) => void | Promise<void>;
+    onBeforeTransferPrepare?: () => void | Promise<void>;
     onPrepared?: (prepared: TransferPrepareResponse) => void | Promise<void>;
-    throwIfCancelled?: (transferId: string) => void | Promise<void>;
+    throwIfCancelled?: (transferId?: string) => void | Promise<void>;
     signedUrlFlow?: SignedUrlFlow;
     checksumProvider?: ChunkChecksumProvider;
     checksumReadConcurrency?: number;
     checksumGroupSizeBytes?: number;
     onChecksumEvent?: (event: ChunkChecksumEvent) => void | Promise<void>;
 }
-export interface PreparedHttpSource {
+export interface PlanningHttpSource {
     source_id: string;
     type: "http";
     provider?: string;
-    url: string;
+    url?: string;
     size: number;
     filename?: string;
     headers?: Record<string, string>;
     expires_at?: string;
     metadata?: Record<string, unknown>;
 }
+export interface PreparedHttpSource extends PlanningHttpSource {
+    url: string;
+}
 export interface PreparedDestination {
     destination_id: string;
     provider: string;

package/dist/provider-signing.d.ts

@@ -1,4 +1,4 @@
-import type { ChunkDestinationSigningTarget, ChunkSigningPlanItem, PreparedDestination, PreparedHttpSource, ProviderDestinationConfig, ProviderSourceConfig, R2ProviderConfig, S3CompatibleProviderConfig, S3ProviderConfig, SignedChunkRoute } from "./models.js";
+import type { ChunkDestinationSigningTarget, ChunkSigningPlanItem, PlanningHttpSource, PreparedDestination, PreparedHttpSource, ProviderDestinationConfig, ProviderSourceConfig, R2ProviderConfig, S3CompatibleProviderConfig, S3ProviderConfig, SignedChunkRoute } from "./models.js";
 export type AnyS3CompatibleProviderConfig = S3ProviderConfig | R2ProviderConfig | S3CompatibleProviderConfig;
 export declare function nowIso(): string;
 export declare function expiresAtIso(expiresIn: number): string;
@@ -7,6 +7,10 @@ export declare function prepareProviderSource(source: ProviderSourceConfig, opti
     expiresIn?: number;
     fetchImpl?: typeof fetch;
 }): Promise<PreparedHttpSource>;
+export declare function prepareProviderSourceForPlan(source: ProviderSourceConfig, options?: {
+    index?: number;
+    fetchImpl?: typeof fetch;
+}): Promise<PlanningHttpSource>;
 export declare function prepareProviderDestination(destination: ProviderDestinationConfig, options?: {
     index?: number;
 }): PreparedDestination;

package/dist/provider-signing.js

@@ -50,6 +50,42 @@ export async function prepareProviderSource(source, options = {}) {
     }
     return unsupportedProviderConfig(source);
 }
+export async function prepareProviderSourceForPlan(source, options = {}) {
+    const index = options.index ?? 0;
+    if (isS3CompatibleProvider(source)) {
+        const endpoint = s3CompatibleEndpoint(source);
+        const client = createS3CompatibleClient(source, endpoint);
+        const head = await client.send(new HeadObjectCommand({ Bucket: source.bucket, Key: source.key }));
+        return {
+            source_id: source.id ?? `src_${index}`,
+            type: "http",
+            provider: source.provider,
+            size: Number(head.ContentLength ?? 0),
+            filename: filename(source.key),
+            metadata: {
+                ...s3CompatibleMetadata(source, endpoint),
+                content_length: Number(head.ContentLength ?? 0),
+                ...(head.ETag ? { etag: head.ETag } : {}),
+                ...(head.LastModified ? { last_modified: head.LastModified.toISOString() } : {}),
+                ...(head.VersionId ? { version_id: head.VersionId } : {})
+            }
+        };
+    }
+    if (isHippiusProvider(source)) {
+        const fetchImpl = options.fetchImpl ?? globalThis.fetch;
+        const baseUrl = source.base_url ?? "https://api.hippius.com";
+        const size = await hippiusObjectSize(fetchImpl, baseUrl, source.api_token, source.bucket, source.key);
+        return {
+            source_id: source.id ?? `src_${index}`,
+            type: "http",
+            provider: "hippius",
+            size,
+            filename: filename(source.key),
+            metadata: { bucket: source.bucket, key: source.key, base_url: baseUrl }
+        };
+    }
+    return unsupportedProviderConfig(source);
+}
 export function prepareProviderDestination(destination, options = {}) {
     const index = options.index ?? 0;
     const destinationId = destination.id ?? `dst_${index}`;
@@ -364,6 +400,18 @@ async function computeChunkGroupContentMd5(input) {
     const last = chunks[chunks.length - 1];
     const sourceOffset = first.source_offset;
     const totalSize = last.source_offset + last.chunk_size - sourceOffset;
+    if (isS3CompatibleProvider(input.source)) {
+        const client = createS3CompatibleClient(input.source);
+        const response = await client.send(new GetObjectCommand({
+            Bucket: input.source.bucket,
+            Key: input.source.key,
+            Range: `bytes=${sourceOffset}-${sourceOffset + totalSize - 1}`
+        }));
+        if (!response.Body) {
+            throw new Error(`chunk checksum read returned no body for ${input.source.bucket}/${input.source.key}`);
+        }
+        return hashChunkGroupContentMd5(chunks, totalSize, bodyToAsyncBytes(response.Body));
+    }
     const sourceRoute = await signSourceRangeRoute({
         source: input.source,
         fallbackUrl: first.source_url,
@@ -378,6 +426,12 @@ async function computeChunkGroupContentMd5(input) {
         const message = `chunk checksum fetch failed with ${response.status}: ${await response.text()}`;
         throw Object.assign(new Error(message), { status: response.status });
     }
+    if (response.body) {
+        return hashChunkGroupContentMd5(chunks, totalSize, readableStreamToAsyncBytes(response.body));
+    }
+    return hashChunkGroupContentMd5(chunks, totalSize, [new Uint8Array(await response.arrayBuffer())]);
+}
+async function hashChunkGroupContentMd5(chunks, totalSize, body) {
     const output = new Map();
     let chunkIndex = 0;
     let bytesInChunk = 0;
@@ -404,23 +458,69 @@ async function computeChunkGroupContentMd5(input) {
             }
         }
     };
-    if (response.body) {
-        const reader = response.body.getReader();
+    for await (const value of body) {
+        consume(value);
+    }
+    if (totalRead !== totalSize || output.size !== chunks.length) {
+        throw new Error(`chunk checksum fetch size mismatch: expected ${totalSize}, got ${totalRead}`);
+    }
+    return output;
+}
+async function* readableStreamToAsyncBytes(stream) {
+    const reader = stream.getReader();
+    try {
         while (true) {
             const { done, value } = await reader.read();
             if (done)
                 break;
             if (value)
-                consume(value);
+                yield value;
         }
     }
-    else {
-        consume(new Uint8Array(await response.arrayBuffer()));
+    finally {
+        reader.releaseLock();
     }
-    if (totalRead !== totalSize || output.size !== chunks.length) {
-        throw new Error(`chunk checksum fetch size mismatch: expected ${totalSize}, got ${totalRead}`);
+}
+async function* bodyToAsyncBytes(body) {
+    if (body instanceof Uint8Array) {
+        yield body;
+        return;
     }
-    return output;
+    if (typeof body === "string") {
+        yield new TextEncoder().encode(body);
+        return;
+    }
+    if (body && typeof body.transformToByteArray === "function") {
+        yield new Uint8Array(await body.transformToByteArray());
+        return;
+    }
+    if (body && typeof body.arrayBuffer === "function") {
+        yield new Uint8Array(await body.arrayBuffer());
+        return;
+    }
+    if (body && typeof body.getReader === "function") {
+        yield* readableStreamToAsyncBytes(body);
+        return;
+    }
+    if (body && Symbol.asyncIterator in Object(body)) {
+        for await (const value of body) {
+            yield toUint8Array(value);
+        }
+        return;
+    }
+    throw new Error("provider checksum read returned an unsupported body stream");
+}
+function toUint8Array(value) {
+    if (value instanceof Uint8Array)
+        return value;
+    if (typeof value === "string")
+        return new TextEncoder().encode(value);
+    if (ArrayBuffer.isView(value)) {
+        return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
+    }
+    if (value instanceof ArrayBuffer)
+        return new Uint8Array(value);
+    throw new Error("provider checksum stream yielded an unsupported chunk");
 }
 async function retryChecksumRead(task) {
     let lastError;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@beam-network/sdk",
-  "version": "0.2.11",
+  "version": "0.2.12",
   "description": "TypeScript SDK for BEAM transfer creation and management.",
   "type": "module",
   "license": "MIT",