@beam-network/sdk 0.2.15 → 0.2.17

package/dist/provider-signing.js

@@ -1,10 +1,8 @@
-import { createHash } from "node:crypto";
 import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, ListPartsCommand, PutObjectCommand, S3Client, UploadPartCopyCommand, UploadPartCommand } from "@aws-sdk/client-s3";
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
 export function nowIso() {
     return new Date().toISOString();
 }
-const DEFAULT_CHECKSUM_READ_TIMEOUT_MS = 300_000;
 export function expiresAtIso(expiresIn) {
     return new Date(Date.now() + expiresIn * 1000).toISOString();
 }
@@ -228,7 +226,6 @@ export async function signDestinationRoute(input) {
         partNumber: input.partNumber,
         useStagingPut: input.commitMethod === "upload_part_copy",
         fetchImpl: input.fetchImpl,
-        contentMd5: input.contentMd5
     });
     const sourceRoute = await signSourceRoute({
         source: input.source,
@@ -247,7 +244,6 @@ export async function signDestinationRoute(input) {
         chunk_size: input.chunk.chunk_size,
         expires_at: expiresAtIso(input.expiresIn),
         headers: sourceRoute.headers,
-        ...(input.contentMd5 ? { dest_headers: { "Content-MD5": input.contentMd5 } } : {}),
         metadata: {
             ...(input.target.metadata ?? {}),
             ...(input.transferId ? { transfer_id: input.transferId } : {}),
@@ -262,7 +258,6 @@ export async function signDestinationRoute(input) {
             ...(input.deleteUrl ? { delete_url: input.deleteUrl } : {}),
             ...(input.uploadId ? { upload_id: input.uploadId } : {}),
             ...(input.partNumber ? { part_number: input.partNumber } : {}),
-            ...(input.contentMd5 ? { checksum_algorithm: "content-md5", checksum_value: input.contentMd5 } : {}),
             ...(input.partVerifyUrl ? { part_verify_url: input.partVerifyUrl } : {}),
             ...(input.listPageUrl ? { list_page_url: input.listPageUrl } : {}),
             ...(input.completeUrl ? { complete_url: input.completeUrl } : {}),
@@ -303,27 +298,6 @@ async function signSourceRoute(input) {
     }
     return unsupportedProviderConfig(input.source);
 }
-async function signSourceRangeRoute(input) {
-    const range = rangeHeaderForRange(input.sourceOffset, input.chunkSize);
-    if (isS3CompatibleProvider(input.source)) {
-        const client = createS3CompatibleClient(input.source);
-        return {
-            url: await getSignedUrl(client, new GetObjectCommand({
-                Bucket: input.source.bucket,
-                Key: input.source.key,
-                Range: range
-            }), { expiresIn: input.expiresIn }),
-            headers: { Range: range }
-        };
-    }
-    if (isHippiusProvider(input.source)) {
-        return {
-            url: await hippiusPresign(input.fetchImpl ?? globalThis.fetch, input.source.base_url ?? "https://api.hippius.com", input.source.api_token, input.source.bucket, input.source.key, "get", input.expiresIn),
-            headers: { Range: range }
-        };
-    }
-    return { url: input.fallbackUrl, headers: { Range: range } };
-}
 async function signDestinationUrl(input) {
     if (isS3CompatibleProvider(input.destination)) {
         const client = createS3CompatibleClient(input.destination);
@@ -333,7 +307,6 @@ async function signDestinationUrl(input) {
                 Key: input.uploadPartObjectKey,
                 UploadId: input.uploadId,
                 PartNumber: input.partNumber,
-                ContentMD5: input.contentMd5
             }), { expiresIn: input.expiresIn });
         }
         return getSignedUrl(client, new PutObjectCommand({ Bucket: input.destination.bucket, Key: input.objectKey }), { expiresIn: input.expiresIn });
@@ -343,280 +316,6 @@ async function signDestinationUrl(input) {
     }
     return unsupportedProviderConfig(input.destination);
 }
-export async function computeChunksContentMd5(input) {
-    const output = new Map();
-    const chunks = [...input.chunks].sort((a, b) => a.source_offset - b.source_offset || a.source_chunk_index - b.source_chunk_index);
-    const groups = groupContiguousChunks(chunks, input.groupSizeBytes ?? 256 * 1024 * 1024);
-    for (const group of groups) {
-        const groupOutput = await computeChunkGroupContentMd5Adaptive({
-            source: input.source,
-            chunks: group,
-            expiresIn: input.expiresIn,
-            fetchImpl: input.fetchImpl,
-            readTimeoutMs: input.readTimeoutMs
-        });
-        for (const [sourceChunkIndex, checksum] of groupOutput) {
-            output.set(sourceChunkIndex, checksum);
-        }
-    }
-    return output;
-}
-export async function computeChunkContentMd5(input) {
-    const checksums = await computeChunksContentMd5({
-        source: input.source,
-        chunks: [input.chunk],
-        expiresIn: input.expiresIn,
-        fetchImpl: input.fetchImpl,
-        groupSizeBytes: input.chunk.chunk_size,
-        readTimeoutMs: input.readTimeoutMs
-    });
-    const checksum = checksums.get(input.chunk.source_chunk_index);
-    if (!checksum) {
-        throw new Error(`chunk checksum missing for source chunk ${input.chunk.source_chunk_index}`);
-    }
-    return checksum;
-}
-function groupContiguousChunks(chunks, maxGroupSizeBytes) {
-    const groups = [];
-    let current = [];
-    let groupStart = 0;
-    let groupEnd = 0;
-    const maxBytes = Math.max(1, Math.floor(maxGroupSizeBytes));
-    for (const chunk of chunks) {
-        const chunkStart = Math.max(0, Math.floor(chunk.source_offset));
-        const chunkEnd = chunkStart + Math.max(1, Math.floor(chunk.chunk_size));
-        const startsNewGroup = current.length === 0 ||
-            chunkStart !== groupEnd ||
-            chunkEnd - groupStart > maxBytes;
-        if (startsNewGroup) {
-            if (current.length)
-                groups.push(current);
-            current = [chunk];
-            groupStart = chunkStart;
-            groupEnd = chunkEnd;
-            continue;
-        }
-        current.push(chunk);
-        groupEnd = chunkEnd;
-    }
-    if (current.length)
-        groups.push(current);
-    return groups;
-}
-async function computeChunkGroupContentMd5Adaptive(input) {
-    try {
-        return await retryChecksumRead(() => computeChunkGroupContentMd5(input));
-    }
-    catch (error) {
-        if (input.chunks.length <= 1) {
-            throw error;
-        }
-        const midpoint = Math.ceil(input.chunks.length / 2);
-        const output = new Map();
-        for (const chunks of [input.chunks.slice(0, midpoint), input.chunks.slice(midpoint)]) {
-            const partial = await computeChunkGroupContentMd5Adaptive({ ...input, chunks });
-            for (const [sourceChunkIndex, checksum] of partial) {
-                output.set(sourceChunkIndex, checksum);
-            }
-        }
-        return output;
-    }
-}
-async function computeChunkGroupContentMd5(input) {
-    if (!input.chunks.length)
-        return new Map();
-    const chunks = [...input.chunks].sort((a, b) => a.source_offset - b.source_offset || a.source_chunk_index - b.source_chunk_index);
-    const first = chunks[0];
-    const last = chunks[chunks.length - 1];
-    const sourceOffset = first.source_offset;
-    const totalSize = last.source_offset + last.chunk_size - sourceOffset;
-    return withChecksumReadTimeout(input.readTimeoutMs, async (abortSignal) => {
-        if (isS3CompatibleProvider(input.source)) {
-            const client = createS3CompatibleClient(input.source);
-            const response = await client.send(new GetObjectCommand({
-                Bucket: input.source.bucket,
-                Key: input.source.key,
-                Range: `bytes=${sourceOffset}-${sourceOffset + totalSize - 1}`
-            }), abortSignal ? { abortSignal } : undefined);
-            if (!response.Body) {
-                throw new Error(`chunk checksum read returned no body for ${input.source.bucket}/${input.source.key}`);
-            }
-            return hashChunkGroupContentMd5(chunks, totalSize, bodyToAsyncBytes(response.Body));
-        }
-        const sourceRoute = await signSourceRangeRoute({
-            source: input.source,
-            fallbackUrl: first.source_url,
-            sourceOffset,
-            chunkSize: totalSize,
-            expiresIn: input.expiresIn,
-            fetchImpl: input.fetchImpl
-        });
-        const fetchImpl = input.fetchImpl ?? globalThis.fetch;
-        const response = await fetchImpl(sourceRoute.url, { method: "GET", headers: sourceRoute.headers, signal: abortSignal });
-        if (!response.ok) {
-            const message = `chunk checksum fetch failed with ${response.status}: ${await response.text()}`;
-            throw Object.assign(new Error(message), { status: response.status });
-        }
-        if (response.body) {
-            return hashChunkGroupContentMd5(chunks, totalSize, readableStreamToAsyncBytes(response.body));
-        }
-        return hashChunkGroupContentMd5(chunks, totalSize, [new Uint8Array(await response.arrayBuffer())]);
-    });
-}
-function normalizedChecksumReadTimeoutMs(value) {
-    if (value === undefined)
-        return DEFAULT_CHECKSUM_READ_TIMEOUT_MS;
-    if (!Number.isFinite(value) || value <= 0)
-        return 0;
-    return Math.floor(value);
-}
-async function withChecksumReadTimeout(readTimeoutMs, task) {
-    const timeoutMs = normalizedChecksumReadTimeoutMs(readTimeoutMs);
-    if (timeoutMs <= 0) {
-        return task(undefined);
-    }
-    const controller = new AbortController();
-    const timeoutError = Object.assign(new Error(`chunk checksum read timed out after ${timeoutMs}ms`), { code: "CHECKSUM_READ_TIMEOUT" });
-    let timer;
-    const timeout = new Promise((_resolve, reject) => {
-        timer = setTimeout(() => {
-            controller.abort(timeoutError);
-            reject(timeoutError);
-        }, timeoutMs);
-    });
-    try {
-        return await Promise.race([task(controller.signal), timeout]);
-    }
-    catch (error) {
-        if (controller.signal.aborted) {
-            throw timeoutError;
-        }
-        throw error;
-    }
-    finally {
-        if (timer)
-            clearTimeout(timer);
-    }
-}
-async function hashChunkGroupContentMd5(chunks, totalSize, body) {
-    const output = new Map();
-    let chunkIndex = 0;
-    let bytesInChunk = 0;
-    let totalRead = 0;
-    let hash = createHash("md5");
-    const consume = (value) => {
-        let offset = 0;
-        while (offset < value.byteLength) {
-            const chunk = chunks[chunkIndex];
-            if (!chunk) {
-                throw new Error(`chunk checksum fetch size mismatch: read more than ${totalSize} bytes`);
-            }
-            const remainingInChunk = chunk.chunk_size - bytesInChunk;
-            const take = Math.min(remainingInChunk, value.byteLength - offset);
-            hash.update(value.subarray(offset, offset + take));
-            offset += take;
-            totalRead += take;
-            bytesInChunk += take;
-            if (bytesInChunk === chunk.chunk_size) {
-                output.set(chunk.source_chunk_index, hash.digest("base64"));
-                chunkIndex += 1;
-                bytesInChunk = 0;
-                hash = createHash("md5");
-            }
-        }
-    };
-    for await (const value of body) {
-        consume(value);
-    }
-    if (totalRead !== totalSize || output.size !== chunks.length) {
-        throw new Error(`chunk checksum fetch size mismatch: expected ${totalSize}, got ${totalRead}`);
-    }
-    return output;
-}
-async function* readableStreamToAsyncBytes(stream) {
-    const reader = stream.getReader();
-    try {
-        while (true) {
-            const { done, value } = await reader.read();
-            if (done)
-                break;
-            if (value)
-                yield value;
-        }
-    }
-    finally {
-        reader.releaseLock();
-    }
-}
-async function* bodyToAsyncBytes(body) {
-    if (body instanceof Uint8Array) {
-        yield body;
-        return;
-    }
-    if (typeof body === "string") {
-        yield new TextEncoder().encode(body);
-        return;
-    }
-    if (body && typeof body.transformToByteArray === "function") {
-        yield new Uint8Array(await body.transformToByteArray());
-        return;
-    }
-    if (body && typeof body.arrayBuffer === "function") {
-        yield new Uint8Array(await body.arrayBuffer());
-        return;
-    }
-    if (body && typeof body.getReader === "function") {
-        yield* readableStreamToAsyncBytes(body);
-        return;
-    }
-    if (body && Symbol.asyncIterator in Object(body)) {
-        for await (const value of body) {
-            yield toUint8Array(value);
-        }
-        return;
-    }
-    throw new Error("provider checksum read returned an unsupported body stream");
-}
-function toUint8Array(value) {
-    if (value instanceof Uint8Array)
-        return value;
-    if (typeof value === "string")
-        return new TextEncoder().encode(value);
-    if (ArrayBuffer.isView(value)) {
-        return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
-    }
-    if (value instanceof ArrayBuffer)
-        return new Uint8Array(value);
-    throw new Error("provider checksum stream yielded an unsupported chunk");
-}
-async function retryChecksumRead(task) {
-    let lastError;
-    for (let attempt = 0; attempt < 3; attempt += 1) {
-        try {
-            return await task();
-        }
-        catch (error) {
-            lastError = error;
-            if (isChecksumReadTimeout(error))
-                break;
-            const status = typeof error === "object" && error && "status" in error ? Number(error.status) : undefined;
-            if (status && status >= 400 && status < 500 && status !== 408 && status !== 429)
-                break;
-            if (attempt < 2)
-                await sleep(250 * (attempt + 1));
-        }
-    }
-    throw lastError;
-}
-function isChecksumReadTimeout(error) {
-    return Boolean(error &&
-        typeof error === "object" &&
-        "code" in error &&
-        error.code === "CHECKSUM_READ_TIMEOUT");
-}
-function sleep(ms) {
-    return new Promise((resolve) => setTimeout(resolve, ms));
-}
 const s3ClientCache = new Map();
 function s3ClientCacheKey(source, endpoint) {
     return JSON.stringify({

package/package.json

@@ -1,36 +1,38 @@
 {
-  "name": "@beam-network/sdk",
-  "version": "0.2.15",
-  "description": "TypeScript SDK for BEAM transfer creation and management.",
-  "type": "module",
-  "license": "MIT",
-  "sideEffects": false,
-  "main": "./dist/index.js",
-  "types": "./dist/index.d.ts",
-  "exports": {
-    ".": {
-      "types": "./dist/index.d.ts",
-      "import": "./dist/index.js"
+    "name": "@beam-network/sdk",
+    "version": "0.2.17",
+    "description": "TypeScript SDK for BEAM transfer creation and management.",
+    "type": "module",
+    "license": "MIT",
+    "sideEffects": false,
+    "main": "./dist/index.js",
+    "types": "./dist/index.d.ts",
+    "exports": {
+        ".": {
+            "types": "./dist/index.d.ts",
+            "import": "./dist/index.js"
+        }
+    },
+    "files": [
+        "dist",
+        "README.md"
+    ],
+    "scripts": {
+        "build": "tsc -p tsconfig.json",
+        "test": "tsc -p tsconfig.json --noEmit && npm run test:functional",
+        "test:functional": "npm run build && node --test test/client.functional.test.mjs",
+        "lint": "tsc -p tsconfig.json --noEmit"
+    },
+    "publishConfig": {
+        "access": "public"
+    },
+    "dependencies": {
+        "@aws-sdk/client-s3": "^3.806.0",
+        "@aws-sdk/s3-request-presigner": "^3.806.0",
+        "@msgpack/msgpack": "^3.0.0",
+        "nats": "^2.29.3"
+    },
+    "devDependencies": {
+        "typescript": "^5.5.0"
     }
-  },
-  "files": [
-    "dist",
-    "README.md"
-  ],
-  "scripts": {
-    "build": "tsc -p tsconfig.json",
-    "test": "tsc -p tsconfig.json --noEmit && npm run test:functional",
-    "test:functional": "npm run build && node --test test/*.test.mjs",
-    "lint": "tsc -p tsconfig.json --noEmit"
-  },
-  "publishConfig": {
-    "access": "public"
-  },
-  "dependencies": {
-    "@aws-sdk/client-s3": "^3.806.0",
-    "@aws-sdk/s3-request-presigner": "^3.806.0"
-  },
-  "devDependencies": {
-    "typescript": "^5.5.0"
-  }
 }