@beam-network/sdk 0.2.23 → 0.4.0

package/dist/nats-control.js

@@ -1,12 +1,11 @@
 import { encode, decode } from "@msgpack/msgpack";
 import { connect } from "nats";
-export const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/v2";
+export const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/v4";
 export const BEAM_DEFAULT_NATS_URL = "tls://nats.b1m.ai:4222";
 export const BEAM_DEFAULT_NATS_WS_URL = "wss://nats.b1m.ai:443";
 // NATS enforces max_payload per message. This guard splits signed-route control
 // messages before the broker rejects them; transfer bytes never flow through NATS.
 export const BEAM_DEFAULT_MAX_PAYLOAD_BYTES = 24 * 1024 * 1024;
-export const ROUTE_RECOVERY_SIGN_MESSAGE_TYPE = "transfer.route_recovery.sign";
 function buildConnectionOptions(options) {
     const connectionOptions = {
         servers: options.natsUrl,
@@ -36,8 +35,12 @@ export class BeamTransferControl {
     natsUrl;
     apiKey;
     nc = null;
+    connectionPromise = null;
+    closed = false;
     authToken = null;
     authTokenExpiresAt = 0;
+    authResolvePromise = null;
+    terminalWaitCancellations = new Set();
     constructor(options) {
         this.apiKey = options.apiKey;
         this.keyPrefix = options.apiKey.slice(0, 12);
@@ -54,59 +57,29 @@ export class BeamTransferControl {
             throw new Error("transferRuntimeShardCount must be a positive integer.");
         }
     }
-    async serveRouteRecoverySigner(transferId, handler) {
-        const nc = await this.connection();
-        const subject = this.routeRecoverySignSubject(transferId);
-        const sub = nc.subscribe(subject);
-        await nc.flush();
-        let active = true;
-        void (async () => {
-            for await (const msg of sub) {
-                if (!active)
-                    break;
-                let request = null;
-                try {
-                    request = decode(msg.data);
-                    if (request.schema_version !== TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION ||
-                        request.environment !== this.environment ||
-                        request.key_prefix !== this.keyPrefix ||
-                        request.transfer_id !== transferId ||
-                        request.message_type !== ROUTE_RECOVERY_SIGN_MESSAGE_TYPE ||
-                        request.producer !== "transfer-runtime") {
-                        throw new Error("route recovery request envelope mismatch");
-                    }
-                    const payload = await handler(request.payload);
-                    msg.respond(encode(this.buildRouteRecoveryReply(request, true, 200, payload)));
-                }
-                catch (error) {
-                    const fallback = request ?? {
-                        message_id: "unknown",
-                        request_id: "unknown",
-                        transfer_id: transferId
-                    };
-                    msg.respond(encode(this.buildRouteRecoveryReply(fallback, false, 500, undefined, {
-                        code: "route_recovery_sign_failed",
-                        message: error instanceof Error ? error.message : String(error)
-                    })));
-                }
-            }
-        })();
-        return () => {
-            active = false;
-            sub.unsubscribe();
-        };
-    }
     async close() {
-        if (!this.nc)
-            return;
-        await this.nc.drain();
+        this.closed = true;
+        for (const cancel of [...this.terminalWaitCancellations])
+            cancel();
+        if (this.connectionPromise) {
+            try {
+                await this.connectionPromise;
+            }
+            catch {
+                // A connection that loses the close race closes itself before rejecting.
+            }
+        }
+        const nc = this.nc;
         this.nc = null;
+        if (nc && !nc.isClosed())
+            await nc.drain();
     }
-    async request(messageType, payload, transferId) {
-        const nc = await this.connection();
+    async request(messageType, payload, options = {}) {
         const authToken = await this.resolveAuthToken();
-        const shardId = transferId ? transferShardId(transferId, this.shardCount) : 0;
-        const requestId = randomId();
+        const shardId = options.transferId ? transferShardId(options.transferId, this.shardCount) : 0;
+        const requestId = options.idempotencyKey
+            ? await stableRequestId(`${messageType}:${options.idempotencyKey}`)
+            : randomId();
         const envelope = {
             message_id: `${TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION}:${this.environment}:${this.keyPrefix}:${messageType}:${requestId}`,
             schema_version: TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION,
@@ -125,16 +98,123 @@ export class BeamTransferControl {
             throw new Error(`NATS lifecycle request is ${bytes.byteLength} bytes, above maxPayloadBytes=${this.maxPayloadBytes}`);
         }
         const subject = this.requestSubject(messageType, shardId);
-        const response = await nc.request(subject, bytes, { timeout: this.requestTimeoutMs });
-        const decoded = decode(response.data);
-        if (!decoded.ok) {
-            const body = JSON.stringify(decoded.error ?? {});
-            throw new Error(`Beam lifecycle request failed with ${decoded.status}: ${body}`);
+        let lastError;
+        for (let attempt = 0; attempt < LIFECYCLE_REQUEST_MAX_ATTEMPTS; attempt += 1) {
+            try {
+                const nc = await this.connection();
+                const response = await nc.request(subject, bytes, { timeout: this.requestTimeoutMs });
+                const decoded = decode(response.data);
+                if (decoded.ok)
+                    return (decoded.payload ?? {});
+                const body = JSON.stringify(decoded.error ?? {});
+                const error = new Error(`Beam lifecycle request failed with ${decoded.status}: ${body}`);
+                if (!isRetryableLifecycleStatus(decoded.status) || attempt + 1 >= LIFECYCLE_REQUEST_MAX_ATTEMPTS) {
+                    throw error;
+                }
+                lastError = error;
+            }
+            catch (error) {
+                if (!isRetryableLifecycleError(error) || attempt + 1 >= LIFECYCLE_REQUEST_MAX_ATTEMPTS) {
+                    throw error;
+                }
+                lastError = error;
+                if (this.nc?.isClosed())
+                    this.nc = null;
+            }
+            await sleepWithJitter(LIFECYCLE_REQUEST_RETRY_DELAYS_MS[attempt] ?? 1_000);
+        }
+        throw lastError instanceof Error ? lastError : new Error("Beam lifecycle request failed after retries");
+    }
+    async openTerminalSignalWaiter(transferId) {
+        const nc = await this.connection();
+        const pending = [];
+        let activeWait = null;
+        let closed = false;
+        let terminalError = null;
+        let subscription = null;
+        const settleActive = (event, error) => {
+            const wait = activeWait;
+            if (!wait)
+                return;
+            activeWait = null;
+            clearTimeout(wait.timer);
+            if (error !== undefined)
+                wait.reject(error);
+            else
+                wait.resolve(event);
+        };
+        const close = async () => {
+            if (closed)
+                return;
+            closed = true;
+            this.terminalWaitCancellations.delete(cancel);
+            subscription?.unsubscribe();
+            settleActive(null);
+        };
+        const cancel = () => { void close(); };
+        try {
+            subscription = nc.subscribe(this.terminalSubject(transferId), {
+                callback: (error, message) => {
+                    if (error) {
+                        terminalError = error;
+                        subscription?.unsubscribe();
+                        settleActive(null, error);
+                        return;
+                    }
+                    try {
+                        const event = parseTerminalEvent(decode(message.data), transferId);
+                        if (activeWait)
+                            settleActive(event);
+                        else if (pending.length === 0)
+                            pending.push(event);
+                    }
+                    catch (decodeError) {
+                        terminalError = decodeError;
+                        subscription?.unsubscribe();
+                        settleActive(null, decodeError);
+                    }
+                }
+            });
+            await nc.flush();
+            if (this.closed) {
+                await close();
+                throw new Error("NATS lifecycle control is closed");
+            }
+            this.terminalWaitCancellations.add(cancel);
+            if (this.closed) {
+                await close();
+                throw new Error("NATS lifecycle control is closed");
+            }
+            return {
+                wait: async (timeoutMs) => {
+                    if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
+                        throw new Error("timeoutMs must be a positive number.");
+                    }
+                    if (closed)
+                        return null;
+                    if (terminalError !== null)
+                        throw terminalError;
+                    const event = pending.shift();
+                    if (event)
+                        return event;
+                    if (activeWait)
+                        throw new Error("terminal signal waiter already has an active wait");
+                    return new Promise((resolve, reject) => {
+                        const timer = setTimeout(() => settleActive(null), timeoutMs);
+                        activeWait = { resolve, reject, timer };
+                    });
+                },
+                close
+            };
+        }
+        catch (error) {
+            subscription?.unsubscribe();
+            settleActive(null);
+            throw error;
         }
-        return (decoded.payload ?? {});
     }
     splitRoutesForPayload(messageType, basePayload, routes) {
-        const split = (candidate) => {
+        const encodedBytes = (candidate) => {
             const encoded = encode({
                 schema_version: TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION,
                 environment: this.environment,
@@ -147,67 +227,126 @@ export class BeamTransferControl {
                 producer: "sdk",
                 payload: { ...basePayload, route_batch: compactSignedRoutes(candidate) }
             });
-            if (encoded.byteLength <= this.maxPayloadBytes)
-                return [candidate];
-            if (candidate.length === 1) {
-                throw new Error(`single signed route is ${encoded.byteLength} bytes, above maxPayloadBytes=${this.maxPayloadBytes}`);
-            }
-            const midpoint = Math.ceil(candidate.length / 2);
-            return [...split(candidate.slice(0, midpoint)), ...split(candidate.slice(midpoint))];
+            return encoded.byteLength;
         };
-        return routes.length ? split(routes) : [];
+        const chunks = [];
+        let offset = 0;
+        while (offset < routes.length) {
+            let low = 1;
+            let high = routes.length - offset;
+            let accepted = 0;
+            while (low <= high) {
+                const count = Math.floor((low + high) / 2);
+                if (encodedBytes(routes.slice(offset, offset + count)) <= this.maxPayloadBytes) {
+                    accepted = count;
+                    low = count + 1;
+                }
+                else
+                    high = count - 1;
+            }
+            if (accepted === 0) {
+                const bytes = encodedBytes(routes.slice(offset, offset + 1));
+                throw new Error(`single signed route is ${bytes} bytes, above maxPayloadBytes=${this.maxPayloadBytes}`);
+            }
+            chunks.push(routes.slice(offset, offset + accepted));
+            offset += accepted;
+        }
+        return chunks;
     }
     async connection() {
+        if (this.closed)
+            throw new Error("NATS lifecycle control is closed");
         if (this.nc && !this.nc.isClosed())
             return this.nc;
-        this.nc = await connect(buildConnectionOptions({
-            natsUrl: this.natsUrl,
-            apiKey: this.apiKey,
-            keyPrefix: this.keyPrefix
-        }));
-        return this.nc;
+        if (this.connectionPromise)
+            return this.connectionPromise;
+        const pending = (async () => {
+            const nc = await connect(buildConnectionOptions({
+                natsUrl: this.natsUrl,
+                apiKey: this.apiKey,
+                keyPrefix: this.keyPrefix
+            }));
+            if (this.closed) {
+                await nc.drain();
+                throw new Error("NATS lifecycle control is closed");
+            }
+            this.nc = nc;
+            return nc;
+        })();
+        this.connectionPromise = pending;
+        try {
+            return await pending;
+        }
+        finally {
+            if (this.connectionPromise === pending)
+                this.connectionPromise = null;
+        }
     }
     async resolveAuthToken() {
         const now = Math.floor(Date.now() / 1000);
         if (this.authToken && this.authTokenExpiresAt - 5 > now)
             return this.authToken;
-        const nc = await this.connection();
-        const response = await nc.request(this.authSubject(), new TextEncoder().encode("{}"), { timeout: this.requestTimeoutMs });
-        const parsed = JSON.parse(new TextDecoder().decode(response.data));
-        if (!parsed.ok || !parsed.token) {
-            throw new Error(`Beam SDK NATS auth resolve failed: ${parsed.error ?? "unknown_error"}`);
+        if (this.authResolvePromise)
+            return this.authResolvePromise;
+        const pending = this.resolveAuthTokenFresh();
+        this.authResolvePromise = pending;
+        try {
+            return await pending;
+        }
+        finally {
+            if (this.authResolvePromise === pending)
+                this.authResolvePromise = null;
         }
-        const claims = decodeJwtPayload(parsed.token);
-        this.authToken = parsed.token;
-        this.authTokenExpiresAt = claims.exp;
-        return parsed.token;
+    }
+    async resolveAuthTokenFresh() {
+        let lastError;
+        for (let attempt = 0; attempt < LIFECYCLE_REQUEST_MAX_ATTEMPTS; attempt += 1) {
+            try {
+                const nc = await this.connection();
+                const response = await nc.request(this.authSubject(), new TextEncoder().encode("{}"), { timeout: this.requestTimeoutMs });
+                const parsed = JSON.parse(new TextDecoder().decode(response.data));
+                if (!parsed.ok || !parsed.token) {
+                    throw new Error(`Beam SDK NATS auth resolve failed: ${parsed.error ?? "unknown_error"}`);
+                }
+                const claims = decodeJwtPayload(parsed.token);
+                this.authToken = parsed.token;
+                this.authTokenExpiresAt = claims.exp;
+                return parsed.token;
+            }
+            catch (error) {
+                if (!isRetryableLifecycleError(error) || attempt + 1 >= LIFECYCLE_REQUEST_MAX_ATTEMPTS) {
+                    throw error;
+                }
+                lastError = error;
+                if (this.nc?.isClosed())
+                    this.nc = null;
+                await sleepWithJitter(LIFECYCLE_REQUEST_RETRY_DELAYS_MS[attempt] ?? 1_000);
+            }
+        }
+        throw lastError instanceof Error ? lastError : new Error("Beam SDK NATS auth resolve failed after retries");
     }
     authSubject() {
         return `${this.subjectPrefix}.${this.environment}.auth.${this.keyPrefix}.resolve`;
     }
-    routeRecoverySignSubject(transferId) {
-        return `${this.subjectPrefix}.${this.environment}.sdk.${this.keyPrefix}.transfer.${transferId}.route_recovery_sign`;
-    }
-    buildRouteRecoveryReply(request, ok, status, payload, error) {
-        return {
-            message_id: request.message_id,
-            schema_version: TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION,
-            environment: this.environment,
-            key_prefix: this.keyPrefix,
-            transfer_id: request.transfer_id,
-            message_type: ROUTE_RECOVERY_SIGN_MESSAGE_TYPE,
-            request_id: request.request_id,
-            occurred_at: new Date().toISOString(),
-            producer: "sdk",
-            ok,
-            status,
-            ...(payload ? { payload } : {}),
-            ...(error ? { error } : {})
-        };
-    }
     requestSubject(messageType, shardId) {
         return `${this.subjectPrefix}.${this.environment}.sdk.${this.keyPrefix}.shard.${shardId}.${messageType.replace(/\./g, "_")}`;
     }
+    terminalSubject(transferId) {
+        return `${this.subjectPrefix}.${this.environment}.events.${this.keyPrefix}.${transferId}.terminal`;
+    }
+}
+function parseTerminalEvent(value, transferId) {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw new Error("invalid transfer terminal event");
+    const event = value;
+    if (event.schema_version !== TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION ||
+        event.producer !== "transfer-runtime" ||
+        event.transfer_id !== transferId ||
+        !["completed", "failed", "cancelled"].includes(String(event.status)) ||
+        typeof event.occurred_at !== "string") {
+        throw new Error("invalid transfer terminal event");
+    }
+    return event;
 }
 const ROUTE_ATTEMPT_METADATA_KEYS = new Set([
     "part_number",
@@ -218,9 +357,7 @@ const ROUTE_ATTEMPT_METADATA_KEYS = new Set([
     "copy_url",
     "copy_source",
     "copy_source_range",
-    "delete_url",
-    "part_verify_url",
-    "list_page_url"
+    "delete_url"
 ]);
 const ROUTE_IDENTITY_METADATA_KEYS = [
     "source_id",
@@ -232,8 +369,6 @@ const ROUTE_IDENTITY_METADATA_KEYS = [
 export function compactSignedRoutes(routes) {
     const source_chunks = [];
     const sourceRefs = new Map();
-    const multipart_groups = [];
-    const groupRefs = new Map();
     const compactRoutes = [];
     for (const [routeIndex, route] of routes.entries()) {
         const sourceKey = JSON.stringify([
@@ -264,21 +399,12 @@ export function compactSignedRoutes(routes) {
         const deliveryIndex = route.delivery_index ?? nonNegativeInteger(metadata.delivery_index) ?? routeIndex;
         for (const key of ROUTE_IDENTITY_METADATA_KEYS)
             delete metadata[key];
-        const groupBaseKey = multipartGroupKey(route, metadata);
-        const groupMetadata = Object.fromEntries(Object.entries(metadata).filter(([key]) => !ROUTE_ATTEMPT_METADATA_KEYS.has(key)));
-        const groupKey = groupBaseKey ? `${groupBaseKey}:${JSON.stringify(groupMetadata)}` : null;
-        let groupRef;
-        if (groupKey) {
-            groupRef = groupRefs.get(groupKey);
-            if (groupRef === undefined) {
-                groupRef = multipart_groups.length;
-                groupRefs.set(groupKey, groupRef);
-                multipart_groups.push({ group_ref: groupRef, metadata: groupMetadata });
-            }
-            for (const key of Object.keys(metadata)) {
-                if (!ROUTE_ATTEMPT_METADATA_KEYS.has(key))
-                    delete metadata[key];
-            }
+        const multipartGroupId = typeof metadata.multipart_group_id === "string"
+            ? metadata.multipart_group_id
+            : undefined;
+        for (const key of Object.keys(metadata)) {
+            if (!ROUTE_ATTEMPT_METADATA_KEYS.has(key))
+                delete metadata[key];
         }
         compactRoutes.push(compact({
             source_ref: sourceRef,
@@ -287,21 +413,11 @@ export function compactSignedRoutes(routes) {
             dest_url: route.dest_url,
             expires_at: route.expires_at,
             dest_headers: route.dest_headers,
-            multipart_group_ref: groupRef,
+            multipart_group_id: multipartGroupId,
             metadata: Object.keys(metadata).length ? metadata : undefined
         }));
     }
-    return { source_chunks, multipart_groups, routes: compactRoutes };
-}
-function multipartGroupKey(route, metadata) {
-    const groupId = typeof metadata.multipart_group_id === "string" ? metadata.multipart_group_id : "";
-    if (groupId)
-        return `id:${groupId}`;
-    const uploadId = typeof metadata.upload_id === "string" ? metadata.upload_id : "";
-    const finalObjectKey = typeof metadata.final_object_key === "string" ? metadata.final_object_key : "";
-    return uploadId || finalObjectKey
-        ? JSON.stringify([route.source_id, route.destination_id, uploadId, finalObjectKey])
-        : null;
+    return { source_chunks, routes: compactRoutes };
 }
 function nonNegativeInteger(value) {
     const parsed = typeof value === "number" ? value : Number(value);
@@ -340,3 +456,41 @@ function randomId() {
         return cryptoApi.randomUUID();
     return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
 }
+const LIFECYCLE_REQUEST_MAX_ATTEMPTS = 3;
+const LIFECYCLE_REQUEST_RETRY_DELAYS_MS = [150, 500];
+function isRetryableLifecycleStatus(status) {
+    return status === 408 || status === 425 || status === 429 || status >= 500;
+}
+function isRetryableLifecycleError(error) {
+    const record = error && typeof error === "object" ? error : null;
+    const code = typeof record?.code === "string" ? record.code : "";
+    const message = error instanceof Error ? error.message : String(error);
+    const combined = `${code} ${message}`.toLowerCase();
+    return [
+        "timeout",
+        "no responders",
+        "connection closed",
+        "disconnected",
+        "econnreset",
+        "econnrefused",
+        "etimedout",
+        "socket",
+        "network"
+    ].some((token) => combined.includes(token));
+}
+async function sleepWithJitter(baseMs) {
+    const jitter = Math.floor(baseMs * (Math.random() * 0.4 - 0.2));
+    await new Promise((resolve) => setTimeout(resolve, Math.max(1, baseMs + jitter)));
+}
+async function stableRequestId(identity) {
+    const bytes = new TextEncoder().encode(identity);
+    let digest;
+    if (globalThis.crypto?.subtle) {
+        digest = new Uint8Array(await globalThis.crypto.subtle.digest("SHA-256", bytes));
+    }
+    else {
+        const { createHash } = await import("node:crypto");
+        digest = new Uint8Array(createHash("sha256").update(bytes).digest());
+    }
+    return `idempotent-${[...digest].map((byte) => byte.toString(16).padStart(2, "0")).join("")}`;
+}

package/dist/provider-signing.d.ts

@@ -14,16 +14,21 @@ export declare function prepareProviderSourceForPlan(source: ProviderSourceConfi
 export declare function prepareProviderDestination(destination: ProviderDestinationConfig, options?: {
     index?: number;
 }): PreparedDestination;
-export declare function createMultipartUpload(destination: ProviderDestinationConfig, objectKey: string): Promise<string>;
+export declare function createMultipartUpload(input: {
+    destination: ProviderDestinationConfig;
+    objectKey: string;
+    metadata: Record<string, string>;
+}): Promise<string>;
+export declare function signFinalObjectHead(destination: ProviderDestinationConfig, objectKey: string, expiresIn: number): Promise<string>;
 export declare function signCompleteMultipartUpload(destination: ProviderDestinationConfig, objectKey: string, uploadId: string, expiresIn: number): Promise<string>;
 export declare function signAbortMultipartUpload(destination: ProviderDestinationConfig, objectKey: string, uploadId: string, expiresIn: number): Promise<string>;
-export declare function signListObjects(destination: ProviderDestinationConfig, prefix: string, expiresIn: number): Promise<string>;
+export declare function signListObjects(destination: ProviderDestinationConfig, prefix: string, expiresIn: number, options?: {
+    startAfter?: string;
+}): Promise<string>;
 export declare function signListMultipartUpload(destination: ProviderDestinationConfig, objectKey: string, uploadId: string, expiresIn: number, options?: {
     maxParts?: number;
     partNumberMarker?: number;
 }): Promise<string>;
-export declare function listPartsPageMarker(partNumber: number, pageSize?: number): number | undefined;
-export declare function listPartsVerifyMarker(partNumber: number): number | undefined;
 export declare function signUploadPartCopy(destination: ProviderDestinationConfig, finalObjectKey: string, stagingObjectKey: string, uploadId: string, partNumber: number, expiresIn: number, copySourceRange?: string): Promise<{
     copyUrl: string;
     copySource: string;
@@ -36,20 +41,10 @@ export declare function signDestinationRoute(input: {
     source?: ProviderSourceConfig;
     destination: ProviderDestinationConfig;
     expiresIn: number;
-    uploadId?: string;
     partNumber?: number;
-    multipartObjectKey?: string;
-    completeUrl?: string;
-    abortUrl?: string;
-    listUrl?: string;
-    transferId?: string;
     multipartGroupId?: string;
-    multipartCreatedAt?: string;
-    commitMethod?: "upload_part" | "upload_part_copy";
-    partVerifyUrl?: string;
-    listPageUrl?: string;
+    commitMethod?: "upload_part_copy";
     stagingObjectKey?: string;
-    stagingListUrl?: string;
     copyUrl?: string;
     copySource?: string;
     copySourceRange?: string;