@beam-network/sdk 0.2.12 → 0.2.14

package/dist/client.d.ts

@@ -48,6 +48,12 @@ export declare class BeamClient {
     private request;
 }
 export declare function calculateOptimalChunkSize(totalSize: number): number;
+export declare function buildStagingObjectPrefix(input: {
+    finalObjectKey: string;
+    transferId: string;
+    sourceId: string;
+    destinationId: string;
+}): string;
 export declare function buildStagingObjectKey(input: {
     finalObjectKey: string;
     transferId: string;

package/dist/client.js

@@ -1,4 +1,4 @@
-import { abortMultipartUpload, computeChunksContentMd5, createMultipartUpload, nowIso, prepareProviderDestination, prepareProviderSource, prepareProviderSourceForPlan, isS3CompatibleProvider, listPartsPageMarker, listPartsVerifyMarker, signAbortMultipartUpload, signCompleteMultipartUpload, signDeleteObject, signDestinationRoute, signListMultipartUpload, signUploadPartCopy } from "./provider-signing.js";
+import { abortMultipartUpload, computeChunksContentMd5, createMultipartUpload, nowIso, prepareProviderDestination, prepareProviderSource, prepareProviderSourceForPlan, isS3CompatibleProvider, listPartsPageMarker, listPartsVerifyMarker, signAbortMultipartUpload, signCompleteMultipartUpload, signDeleteObject, signDestinationRoute, signListMultipartUpload, signListObjects, signUploadPartCopy } from "./provider-signing.js";
 export const BEAM_DEFAULT_URL = "https://beamcore.b1m.ai";
 export class BeamApiError extends Error {
     status;
@@ -135,7 +135,7 @@ export class BeamClient {
             const totalWarmupBytes = [...chunksBySource.values()].reduce((sum, chunks) => sum + chunks.reduce((chunkSum, chunk) => chunkSum + chunk.chunk_size, 0), 0);
             if (totalWarmupChunks > 0) {
                 const startedAt = Date.now();
-                emitChecksumEvent(input, {
+                await emitChecksumEvent(input, {
                     event: "checksum_warmup_started",
                     sourceId: "all",
                     chunkCount: totalWarmupChunks,
@@ -149,7 +149,7 @@ export class BeamClient {
                     }
                     await checksumCache.ensureMany(source, chunks);
                 }
-                emitChecksumEvent(input, {
+                await emitChecksumEvent(input, {
                     event: "checksum_warmup_completed",
                     sourceId: "all",
                     chunkCount: totalWarmupChunks,
@@ -158,6 +158,7 @@ export class BeamClient {
                 });
             }
             await input.throwIfCancelled?.();
+            checksumCache.seal();
             preparedSources = await Promise.all(input.sources.map((source, index) => prepareProviderSource(source, { index, expiresIn, fetchImpl: this.fetchImpl })));
             await input.onBeforeTransferPrepare?.();
             prepared = await this.prepareTransfer({
@@ -201,11 +202,7 @@ export class BeamClient {
         const signedUrlFlow = prepared.signed_url_flow ?? requestedSignedUrlFlow;
         if (signedUrlFlow === "signed_url_v1") {
             const chunksBySource = checksumChunksBySource(prepared.chunk_plan, destinationsById);
-            for (const [sourceId, chunks] of chunksBySource) {
-                const source = sourcesById.get(sourceId);
-                if (source)
-                    checksumCache.preloadMany(source, chunks);
-            }
+            checksumCache.assertManyWarmed(chunksBySource);
         }
         const routes = [];
         const multipartUploads = new Map();
@@ -344,16 +341,19 @@ export class BeamClient {
                 fetchImpl: this.fetchImpl
             });
         }
-        const stagingObjectKey = buildStagingObjectKey({
+        const stagingObjectPrefix = buildStagingObjectPrefix({
             finalObjectKey: input.finalObjectKey,
             transferId: input.transferId,
             sourceId: input.chunk.source_id,
-            destinationId: input.target.destination_id,
-            chunkIndex: input.chunk.chunk_index,
-            partNumber: input.partNumber
+            destinationId: input.target.destination_id
         });
+        const stagingObjectKey = [
+            stagingObjectPrefix,
+            `part-${input.partNumber}-chunk-${input.chunk.chunk_index}`
+        ].join("/");
         const copySourceRange = `bytes=0-${Math.max(0, input.chunk.chunk_size - 1)}`;
         const { copyUrl, copySource } = await signUploadPartCopy(input.destination, input.finalObjectKey, stagingObjectKey, input.upload.uploadId, input.partNumber, input.expiresIn, copySourceRange);
+        const stagingListUrl = await signListObjects(input.destination, stagingObjectPrefix, input.expiresIn);
         const deleteUrl = await signDeleteObject(input.destination, stagingObjectKey, input.expiresIn);
         return signDestinationRoute({
             chunk: input.chunk,
@@ -372,6 +372,7 @@ export class BeamClient {
             multipartCreatedAt: input.upload.multipartCreatedAt,
             commitMethod: "upload_part_copy",
             stagingObjectKey,
+            stagingListUrl,
             copyUrl,
             copySource,
             copySourceRange,
@@ -501,20 +502,28 @@ function planSignature(chunks) {
         })
     })));
 }
-function emitChecksumEvent(input, event) {
+async function emitChecksumEvent(input, event) {
     if (!input.onChecksumEvent)
         return;
-    void Promise.resolve(input.onChecksumEvent(event)).catch(() => undefined);
+    await input.onChecksumEvent(event);
 }
 class ChunkChecksumCache {
     options;
     queue = [];
     cache = new Map();
     active = 0;
+    sealed = false;
     constructor(options) {
         this.options = options;
     }
     preloadMany(source, chunks) {
+        if (this.sealed) {
+            const missing = this.uniqueChunks(chunks).filter((chunk) => !this.cache.has(this.key(chunk)));
+            if (missing.length) {
+                throw new Error(`checksum cache missing ${missing.length} chunk(s) after preflight warmup; refusing post-prepare source reads`);
+            }
+            return;
+        }
         const pending = this.uniqueChunks(chunks).filter((chunk) => !this.cache.has(this.key(chunk)));
         for (const chunk of pending) {
             this.cache.set(this.key(chunk), deferredChecksum());
@@ -532,6 +541,9 @@ class ChunkChecksumCache {
         const key = this.key(chunk);
         let existing = this.cache.get(key);
         if (!existing) {
+            if (this.sealed) {
+                return Promise.reject(new Error(`checksum cache missing ${key} after preflight warmup; refusing post-prepare source read`));
+            }
             existing = deferredChecksum();
             this.cache.set(key, existing);
             void this.run(() => this.fillGroup(source, [chunk])).catch(() => undefined);
@@ -550,12 +562,12 @@ class ChunkChecksumCache {
                     continue;
                 const cached = await this.options.provider?.getContentMd5({ source, chunk, algorithm: "content-md5" }).catch(() => undefined);
                 if (cached) {
-                    this.emit({ event: "checksum_cache_hit", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
+                    await this.emit({ event: "checksum_cache_hit", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
                     deferred.resolve(cached);
                     unresolved.delete(key);
                 }
                 else {
-                    this.emit({ event: "checksum_cache_miss", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
+                    await this.emit({ event: "checksum_cache_miss", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
                     misses.push(chunk);
                 }
             }
@@ -579,9 +591,9 @@ class ChunkChecksumCache {
                 const deferred = this.cache.get(this.key(chunk));
                 deferred?.resolve(checksum);
                 unresolved.delete(this.key(chunk));
-                this.emit({ event: "checksum_cold_chunk_cached", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
+                await this.emit({ event: "checksum_cold_chunk_cached", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
             }
-            this.emit({
+            await this.emit({
                 event: "checksum_cold_group_completed",
                 sourceId: misses[0]?.source_id ?? source.id ?? "source",
                 chunkCount: misses.length,
@@ -637,6 +649,21 @@ class ChunkChecksumCache {
     key(chunk) {
         return `${chunk.source_id}:${chunk.source_chunk_index}`;
     }
+    seal() {
+        this.sealed = true;
+    }
+    assertManyWarmed(chunksBySource) {
+        const missing = [];
+        for (const chunks of chunksBySource.values()) {
+            for (const chunk of this.uniqueChunks(chunks)) {
+                if (!this.cache.has(this.key(chunk)))
+                    missing.push(this.key(chunk));
+            }
+        }
+        if (missing.length) {
+            throw new Error(`checksum cache missing ${missing.length} chunk(s) after preflight warmup; refusing post-prepare source reads`);
+        }
+    }
     run(task) {
         return new Promise((resolve, reject) => {
             const start = () => {
@@ -652,10 +679,10 @@ class ChunkChecksumCache {
                 this.queue.push(start);
         });
     }
-    emit(event) {
+    async emit(event) {
         if (!this.options.onEvent)
             return;
-        void Promise.resolve(this.options.onEvent(event)).catch(() => undefined);
+        await this.options.onEvent(event);
     }
 }
 function deferredChecksum() {
@@ -693,13 +720,18 @@ function normalizeBaseUrl(value) {
 function compact(input) {
     return Object.fromEntries(Object.entries(input).filter(([, value]) => value !== undefined));
 }
-export function buildStagingObjectKey(input) {
+export function buildStagingObjectPrefix(input) {
     const base = input.finalObjectKey.replace(/\/+$/, "") || "object";
     return [
         `${base}.beam-staging`,
         safeObjectKeySegment(input.transferId),
         safeObjectKeySegment(input.sourceId),
-        safeObjectKeySegment(input.destinationId),
+        safeObjectKeySegment(input.destinationId)
+    ].join("/");
+}
+export function buildStagingObjectKey(input) {
+    return [
+        buildStagingObjectPrefix(input),
         `part-${input.partNumber}-chunk-${input.chunkIndex}`
     ].join("/");
 }

package/dist/provider-signing.d.ts

@@ -17,6 +17,7 @@ export declare function prepareProviderDestination(destination: ProviderDestinat
 export declare function createMultipartUpload(destination: ProviderDestinationConfig, objectKey: string): Promise<string>;
 export declare function signCompleteMultipartUpload(destination: ProviderDestinationConfig, objectKey: string, uploadId: string, expiresIn: number): Promise<string>;
 export declare function signAbortMultipartUpload(destination: ProviderDestinationConfig, objectKey: string, uploadId: string, expiresIn: number): Promise<string>;
+export declare function signListObjects(destination: ProviderDestinationConfig, prefix: string, expiresIn: number): Promise<string>;
 export declare function signListMultipartUpload(destination: ProviderDestinationConfig, objectKey: string, uploadId: string, expiresIn: number, options?: {
     maxParts?: number;
     partNumberMarker?: number;
@@ -49,6 +50,7 @@ export declare function signDestinationRoute(input: {
     partVerifyUrl?: string;
     listPageUrl?: string;
     stagingObjectKey?: string;
+    stagingListUrl?: string;
     copyUrl?: string;
     copySource?: string;
     copySourceRange?: string;

package/dist/provider-signing.js

@@ -1,5 +1,5 @@
 import { createHash } from "node:crypto";
-import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListPartsCommand, PutObjectCommand, S3Client, UploadPartCopyCommand, UploadPartCommand } from "@aws-sdk/client-s3";
+import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, ListPartsCommand, PutObjectCommand, S3Client, UploadPartCopyCommand, UploadPartCommand } from "@aws-sdk/client-s3";
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
 export function nowIso() {
     return new Date().toISOString();
@@ -139,6 +139,13 @@ export async function signAbortMultipartUpload(destination, objectKey, uploadId,
     }
     throw new Error(`multipart abort is not supported for ${destination.provider}`);
 }
+export async function signListObjects(destination, prefix, expiresIn) {
+    if (isS3CompatibleProvider(destination)) {
+        const client = createS3CompatibleClient(destination);
+        return getSignedUrl(client, new ListObjectsV2Command({ Bucket: destination.bucket, Prefix: prefix }), { expiresIn });
+    }
+    throw new Error(`multipart staging list is not supported for ${destination.provider}`);
+}
 export async function signListMultipartUpload(destination, objectKey, uploadId, expiresIn, options = {}) {
     if (isS3CompatibleProvider(destination)) {
         const client = createS3CompatibleClient(destination);
@@ -247,6 +254,7 @@ export async function signDestinationRoute(input) {
             ...(input.multipartCreatedAt ? { multipart_created_at: input.multipartCreatedAt } : {}),
             ...(input.commitMethod ? { commit_method: input.commitMethod } : {}),
             ...(input.stagingObjectKey ? { staging_object_key: input.stagingObjectKey } : {}),
+            ...(input.stagingListUrl ? { staging_list_url: input.stagingListUrl } : {}),
             ...(input.copyUrl ? { copy_url: input.copyUrl } : {}),
             ...(input.copySource ? { copy_source: input.copySource } : {}),
             ...(input.copySourceRange ? { copy_source_range: input.copySourceRange } : {}),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@beam-network/sdk",
-  "version": "0.2.12",
+  "version": "0.2.14",
   "description": "TypeScript SDK for BEAM transfer creation and management.",
   "type": "module",
   "license": "MIT",