@beam-network/sdk 0.2.14 → 0.2.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/client.d.ts +5 -3
- package/dist/client.js +59 -48
- package/dist/models.d.ts +14 -1
- package/dist/nats-control.d.ts +36 -0
- package/dist/nats-control.js +162 -0
- package/dist/provider-signing.d.ts +2 -0
- package/dist/provider-signing.js +97 -30
- package/package.json +36 -34
package/README.md
CHANGED
|
@@ -91,4 +91,4 @@ Use `S3ProviderConfig` for ordinary AWS S3 if preferred, and keep using
|
|
|
91
91
|
`S3CompatibleProviderConfig` when a provider needs a custom S3-compatible
|
|
92
92
|
endpoint.
|
|
93
93
|
|
|
94
|
-
For low-level
|
|
94
|
+
For low-level raw transfer configs, use `createRawTransfer`; lifecycle transport still goes through NATS.
|
package/dist/client.d.ts
CHANGED
|
@@ -1,15 +1,18 @@
|
|
|
1
1
|
import type { AttachSignedUrlsResponse, BeamClientOptions, DistributeResponse, PlanningHttpSource, PreparedDestination, PreparedHttpSource, ProviderTransferCreateInput, RawTransferCreateInput, SignedChunkRoute, SignedUrlFlow, TransferCancelResponse, TransferCreateResponse, TransferPlanResponse, TransferPrepareResponse, TransferStatusInfo } from "./models.js";
|
|
2
|
-
|
|
2
|
+
import { BEAM_DEFAULT_NATS_URL } from "./nats-control.js";
|
|
3
|
+
export { BEAM_DEFAULT_NATS_URL };
|
|
3
4
|
export declare class BeamApiError extends Error {
|
|
4
5
|
readonly status: number;
|
|
5
6
|
readonly body: string;
|
|
6
7
|
constructor(message: string, status: number, body: string);
|
|
7
8
|
}
|
|
8
9
|
export declare class BeamClient {
|
|
9
|
-
readonly baseUrl: string;
|
|
10
10
|
readonly apiKey: string;
|
|
11
|
+
readonly natsUrl: string;
|
|
11
12
|
private readonly fetchImpl;
|
|
13
|
+
private readonly control;
|
|
12
14
|
constructor(options?: BeamClientOptions);
|
|
15
|
+
close(): Promise<void>;
|
|
13
16
|
createRawTransfer(input: RawTransferCreateInput): Promise<TransferCreateResponse>;
|
|
14
17
|
createTransfer(input: ProviderTransferCreateInput): Promise<TransferPrepareResponse>;
|
|
15
18
|
transferStatus(transferId: string): Promise<TransferStatusInfo>;
|
|
@@ -45,7 +48,6 @@ export declare class BeamClient {
|
|
|
45
48
|
timeoutMs?: number;
|
|
46
49
|
pollIntervalMs?: number;
|
|
47
50
|
}): Promise<TransferStatusInfo>;
|
|
48
|
-
private request;
|
|
49
51
|
}
|
|
50
52
|
export declare function calculateOptimalChunkSize(totalSize: number): number;
|
|
51
53
|
export declare function buildStagingObjectPrefix(input: {
|
package/dist/client.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { abortMultipartUpload, computeChunksContentMd5, createMultipartUpload, nowIso, prepareProviderDestination, prepareProviderSource, prepareProviderSourceForPlan, isS3CompatibleProvider, listPartsPageMarker, listPartsVerifyMarker, signAbortMultipartUpload, signCompleteMultipartUpload, signDeleteObject, signDestinationRoute, signListMultipartUpload, signListObjects, signUploadPartCopy } from "./provider-signing.js";
|
|
2
|
-
|
|
2
|
+
import { BeamTransferControl, BEAM_DEFAULT_NATS_URL } from "./nats-control.js";
|
|
3
|
+
export { BEAM_DEFAULT_NATS_URL };
|
|
3
4
|
export class BeamApiError extends Error {
|
|
4
5
|
status;
|
|
5
6
|
body;
|
|
@@ -11,26 +12,41 @@ export class BeamApiError extends Error {
|
|
|
11
12
|
}
|
|
12
13
|
}
|
|
13
14
|
export class BeamClient {
|
|
14
|
-
baseUrl;
|
|
15
15
|
apiKey;
|
|
16
|
+
natsUrl;
|
|
16
17
|
fetchImpl;
|
|
18
|
+
control;
|
|
17
19
|
constructor(options = {}) {
|
|
18
20
|
if (!options.apiKey?.trim()) {
|
|
19
21
|
throw new Error("apiKey is required.");
|
|
20
22
|
}
|
|
21
|
-
this.baseUrl = normalizeBaseUrl(resolveBaseUrl(options));
|
|
22
23
|
this.apiKey = options.apiKey;
|
|
24
|
+
this.natsUrl = options.natsUrl ?? options.natsWsUrl ?? BEAM_DEFAULT_NATS_URL;
|
|
25
|
+
this.control = new BeamTransferControl({
|
|
26
|
+
apiKey: this.apiKey,
|
|
27
|
+
natsUrl: options.natsUrl,
|
|
28
|
+
natsWsUrl: options.natsWsUrl,
|
|
29
|
+
environment: options.environment,
|
|
30
|
+
subjectPrefix: options.transferClientSubjectPrefix,
|
|
31
|
+
transferRuntimeShardCount: options.transferRuntimeShardCount,
|
|
32
|
+
requestTimeoutMs: options.requestTimeoutMs,
|
|
33
|
+
maxPayloadBytes: options.maxPayloadBytes
|
|
34
|
+
});
|
|
23
35
|
this.fetchImpl = options.fetch ?? globalThis.fetch;
|
|
24
36
|
if (!this.fetchImpl) {
|
|
25
37
|
throw new Error("A fetch implementation is required.");
|
|
26
38
|
}
|
|
27
39
|
}
|
|
40
|
+
close() {
|
|
41
|
+
return this.control.close();
|
|
42
|
+
}
|
|
28
43
|
createRawTransfer(input) {
|
|
29
44
|
const body = compact({
|
|
45
|
+
transfer_id: newTransferId(),
|
|
30
46
|
sources: input.sources,
|
|
31
47
|
destinations: input.destinations,
|
|
32
48
|
total_size: input.totalSize,
|
|
33
|
-
chunk_size: input.chunkSize
|
|
49
|
+
chunk_size: input.chunkSize,
|
|
34
50
|
name: input.name,
|
|
35
51
|
merkle_root: input.merkleRoot,
|
|
36
52
|
chunk_hashes: input.chunkHashes,
|
|
@@ -40,27 +56,25 @@ export class BeamClient {
|
|
|
40
56
|
progressive_mode: input.progressiveMode || undefined,
|
|
41
57
|
signed_url_flow: input.signedUrlFlow
|
|
42
58
|
});
|
|
43
|
-
return this.request("
|
|
59
|
+
return this.control.request("transfer.create", body);
|
|
44
60
|
}
|
|
45
61
|
createTransfer(input) {
|
|
46
62
|
return this.prepareProviderTransfer(input);
|
|
47
63
|
}
|
|
48
64
|
transferStatus(transferId) {
|
|
49
65
|
validateId(transferId, "transferId");
|
|
50
|
-
return this.request("
|
|
66
|
+
return this.control.request("transfer.status", { transfer_id: transferId }, transferId);
|
|
51
67
|
}
|
|
52
68
|
distributeTransfer(transferId) {
|
|
53
69
|
validateId(transferId, "transferId");
|
|
54
|
-
return this.request("
|
|
55
|
-
transfer_id: transferId
|
|
56
|
-
});
|
|
70
|
+
return this.control.request("transfer.distribute", { transfer_id: transferId }, transferId);
|
|
57
71
|
}
|
|
58
72
|
cancelTransfer(transferId) {
|
|
59
73
|
validateId(transferId, "transferId");
|
|
60
|
-
return this.request("
|
|
74
|
+
return this.control.request("transfer.cancel", { transfer_id: transferId }, transferId);
|
|
61
75
|
}
|
|
62
76
|
planTransfer(input) {
|
|
63
|
-
return this.request("
|
|
77
|
+
return this.control.request("transfer.plan", compact({
|
|
64
78
|
sources: input.sources,
|
|
65
79
|
destinations: input.destinations,
|
|
66
80
|
name: input.name,
|
|
@@ -71,7 +85,9 @@ export class BeamClient {
|
|
|
71
85
|
}));
|
|
72
86
|
}
|
|
73
87
|
prepareTransfer(input) {
|
|
74
|
-
|
|
88
|
+
const transferId = newTransferId();
|
|
89
|
+
return this.control.request("transfer.prepare", compact({
|
|
90
|
+
transfer_id: transferId,
|
|
75
91
|
sources: input.sources,
|
|
76
92
|
destinations: input.destinations,
|
|
77
93
|
name: input.name,
|
|
@@ -79,16 +95,33 @@ export class BeamClient {
|
|
|
79
95
|
experimental: input.experimental || undefined,
|
|
80
96
|
urls_expires_at: input.urlsExpiresAt,
|
|
81
97
|
signed_url_flow: input.signedUrlFlow
|
|
82
|
-
}));
|
|
98
|
+
}), transferId);
|
|
83
99
|
}
|
|
84
|
-
attachSignedUrls(transferId, input) {
|
|
100
|
+
async attachSignedUrls(transferId, input) {
|
|
85
101
|
validateId(transferId, "transferId");
|
|
86
|
-
|
|
102
|
+
const attachmentId = newTransferId();
|
|
103
|
+
await this.control.request("transfer.attach_signed_urls.begin", compact({
|
|
87
104
|
transfer_id: transferId,
|
|
88
|
-
|
|
89
|
-
|
|
105
|
+
attachment_id: attachmentId,
|
|
106
|
+
total_routes: input.chunkRoutes.length,
|
|
90
107
|
urls_expires_at: input.urlsExpiresAt
|
|
91
|
-
}));
|
|
108
|
+
}), transferId);
|
|
109
|
+
const chunks = this.control.splitRoutesForPayload("transfer.attach_signed_urls.chunk", {
|
|
110
|
+
transfer_id: transferId,
|
|
111
|
+
attachment_id: attachmentId
|
|
112
|
+
}, input.chunkRoutes);
|
|
113
|
+
for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex += 1) {
|
|
114
|
+
await this.control.request("transfer.attach_signed_urls.chunk", {
|
|
115
|
+
transfer_id: transferId,
|
|
116
|
+
attachment_id: attachmentId,
|
|
117
|
+
chunk_index: chunkIndex,
|
|
118
|
+
chunk_routes: chunks[chunkIndex]
|
|
119
|
+
}, transferId);
|
|
120
|
+
}
|
|
121
|
+
return this.control.request("transfer.attach_signed_urls.commit", {
|
|
122
|
+
transfer_id: transferId,
|
|
123
|
+
attachment_id: attachmentId
|
|
124
|
+
}, transferId);
|
|
92
125
|
}
|
|
93
126
|
async prepareProviderTransfer(input) {
|
|
94
127
|
const expiresIn = input.expiresIn ?? 3600;
|
|
@@ -108,6 +141,7 @@ export class BeamClient {
|
|
|
108
141
|
fetchImpl: this.fetchImpl,
|
|
109
142
|
provider: input.checksumProvider,
|
|
110
143
|
groupSizeBytes: input.checksumGroupSizeBytes ?? 256 * 1024 * 1024,
|
|
144
|
+
readTimeoutMs: input.checksumReadTimeoutMs,
|
|
111
145
|
onEvent: input.onChecksumEvent,
|
|
112
146
|
throwIfCancelled: () => input.throwIfCancelled?.()
|
|
113
147
|
});
|
|
@@ -408,27 +442,6 @@ export class BeamClient {
|
|
|
408
442
|
await sleep(pollIntervalMs);
|
|
409
443
|
}
|
|
410
444
|
}
|
|
411
|
-
async request(method, path, body) {
|
|
412
|
-
const headers = {
|
|
413
|
-
Accept: "application/json",
|
|
414
|
-
"X-Api-Key": this.apiKey
|
|
415
|
-
};
|
|
416
|
-
let payload;
|
|
417
|
-
if (body !== undefined) {
|
|
418
|
-
headers["Content-Type"] = "application/json";
|
|
419
|
-
payload = JSON.stringify(body);
|
|
420
|
-
}
|
|
421
|
-
const response = await this.fetchImpl(`${this.baseUrl}${path}`, {
|
|
422
|
-
method,
|
|
423
|
-
headers,
|
|
424
|
-
body: payload
|
|
425
|
-
});
|
|
426
|
-
const text = await response.text();
|
|
427
|
-
if (!response.ok) {
|
|
428
|
-
throw new BeamApiError(`BEAM API request failed with ${response.status}`, response.status, text);
|
|
429
|
-
}
|
|
430
|
-
return (text ? JSON.parse(text) : {});
|
|
431
|
-
}
|
|
432
445
|
}
|
|
433
446
|
function checksumChunksBySource(chunkPlan, destinationsById) {
|
|
434
447
|
const chunksBySource = new Map();
|
|
@@ -580,7 +593,8 @@ class ChunkChecksumCache {
|
|
|
580
593
|
chunks: misses,
|
|
581
594
|
expiresIn: this.options.expiresIn,
|
|
582
595
|
fetchImpl: this.options.fetchImpl,
|
|
583
|
-
groupSizeBytes: this.options.groupSizeBytes
|
|
596
|
+
groupSizeBytes: this.options.groupSizeBytes,
|
|
597
|
+
readTimeoutMs: this.options.readTimeoutMs
|
|
584
598
|
});
|
|
585
599
|
for (const chunk of misses) {
|
|
586
600
|
await this.options.throwIfCancelled?.();
|
|
@@ -708,14 +722,11 @@ export function calculateOptimalChunkSize(totalSize) {
|
|
|
708
722
|
}
|
|
709
723
|
return maxChunk;
|
|
710
724
|
}
|
|
711
|
-
function
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
return
|
|
716
|
-
}
|
|
717
|
-
function normalizeBaseUrl(value) {
|
|
718
|
-
return value.replace(/\/+$/, "");
|
|
725
|
+
function newTransferId() {
|
|
726
|
+
const cryptoApi = globalThis.crypto;
|
|
727
|
+
if (cryptoApi?.randomUUID)
|
|
728
|
+
return cryptoApi.randomUUID();
|
|
729
|
+
return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
|
|
719
730
|
}
|
|
720
731
|
function compact(input) {
|
|
721
732
|
return Object.fromEntries(Object.entries(input).filter(([, value]) => value !== undefined));
|
package/dist/models.d.ts
CHANGED
|
@@ -1,7 +1,13 @@
|
|
|
1
1
|
export type SignedUrlFlow = "signed_url_v1" | "signed_url_v2";
|
|
2
2
|
export interface BeamClientOptions {
|
|
3
3
|
apiKey?: string;
|
|
4
|
-
|
|
4
|
+
natsUrl?: string;
|
|
5
|
+
natsWsUrl?: string;
|
|
6
|
+
environment?: string;
|
|
7
|
+
transferClientSubjectPrefix?: string;
|
|
8
|
+
transferRuntimeShardCount?: number;
|
|
9
|
+
requestTimeoutMs?: number;
|
|
10
|
+
maxPayloadBytes?: number;
|
|
5
11
|
fetch?: typeof fetch;
|
|
6
12
|
}
|
|
7
13
|
export interface SourceConfig {
|
|
@@ -71,6 +77,7 @@ export interface RawTransferCreateInput {
|
|
|
71
77
|
export interface TransferCreateResponse {
|
|
72
78
|
success: boolean;
|
|
73
79
|
transfer_id: string;
|
|
80
|
+
transfer_key?: string;
|
|
74
81
|
experimental?: boolean;
|
|
75
82
|
total_chunks: number;
|
|
76
83
|
total_sources: number;
|
|
@@ -113,11 +120,16 @@ export interface DestinationStatusInfo {
|
|
|
113
120
|
}
|
|
114
121
|
export interface TransferStatusInfo {
|
|
115
122
|
transfer_id: string;
|
|
123
|
+
name?: string;
|
|
116
124
|
status: string;
|
|
117
125
|
error_message: string | null;
|
|
118
126
|
experimental?: boolean;
|
|
119
127
|
started_at: string | null;
|
|
120
128
|
completed_at: string | null;
|
|
129
|
+
total_bytes?: number;
|
|
130
|
+
total_size?: number;
|
|
131
|
+
total_chunks?: number;
|
|
132
|
+
chunks_completed?: number;
|
|
121
133
|
}
|
|
122
134
|
export interface S3ProviderConfig {
|
|
123
135
|
provider: "s3";
|
|
@@ -226,6 +238,7 @@ export interface ProviderTransferCreateInput {
|
|
|
226
238
|
checksumProvider?: ChunkChecksumProvider;
|
|
227
239
|
checksumReadConcurrency?: number;
|
|
228
240
|
checksumGroupSizeBytes?: number;
|
|
241
|
+
checksumReadTimeoutMs?: number;
|
|
229
242
|
onChecksumEvent?: (event: ChunkChecksumEvent) => void | Promise<void>;
|
|
230
243
|
}
|
|
231
244
|
export interface PlanningHttpSource {
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
export declare const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/v1";
|
|
2
|
+
export declare const BEAM_DEFAULT_NATS_URL = "tls://nats.b1m.ai:4222";
|
|
3
|
+
export declare const BEAM_DEFAULT_NATS_WS_URL = "wss://nats.b1m.ai:443";
|
|
4
|
+
export type TransferClientMessageType = "transfer.plan" | "transfer.create" | "transfer.prepare" | "transfer.attach_signed_urls.begin" | "transfer.attach_signed_urls.chunk" | "transfer.attach_signed_urls.commit" | "transfer.distribute" | "transfer.status" | "transfer.cancel";
|
|
5
|
+
export interface TransferControlOptions {
|
|
6
|
+
apiKey: string;
|
|
7
|
+
natsUrl?: string;
|
|
8
|
+
natsWsUrl?: string;
|
|
9
|
+
environment?: string;
|
|
10
|
+
subjectPrefix?: string;
|
|
11
|
+
transferRuntimeShardCount?: number;
|
|
12
|
+
requestTimeoutMs?: number;
|
|
13
|
+
maxPayloadBytes?: number;
|
|
14
|
+
}
|
|
15
|
+
export declare class BeamTransferControl {
|
|
16
|
+
readonly environment: string;
|
|
17
|
+
readonly keyPrefix: string;
|
|
18
|
+
readonly shardCount: number;
|
|
19
|
+
readonly maxPayloadBytes: number;
|
|
20
|
+
private readonly subjectPrefix;
|
|
21
|
+
private readonly requestTimeoutMs;
|
|
22
|
+
private readonly natsUrl;
|
|
23
|
+
private readonly apiKey;
|
|
24
|
+
private nc;
|
|
25
|
+
private authToken;
|
|
26
|
+
private authTokenExpiresAt;
|
|
27
|
+
constructor(options: TransferControlOptions);
|
|
28
|
+
close(): Promise<void>;
|
|
29
|
+
request<T>(messageType: TransferClientMessageType, payload: Record<string, unknown>, transferId?: string): Promise<T>;
|
|
30
|
+
splitRoutesForPayload(messageType: TransferClientMessageType, basePayload: Record<string, unknown>, routes: unknown[]): unknown[][];
|
|
31
|
+
private connection;
|
|
32
|
+
private resolveAuthToken;
|
|
33
|
+
private authSubject;
|
|
34
|
+
private requestSubject;
|
|
35
|
+
}
|
|
36
|
+
export declare function transferShardId(transferId: string, shardCount: number): number;
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
import { encode, decode } from "@msgpack/msgpack";
|
|
2
|
+
import { connect } from "nats";
|
|
3
|
+
export const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/v1";
|
|
4
|
+
export const BEAM_DEFAULT_NATS_URL = "tls://nats.b1m.ai:4222";
|
|
5
|
+
export const BEAM_DEFAULT_NATS_WS_URL = "wss://nats.b1m.ai:443";
|
|
6
|
+
export class BeamTransferControl {
|
|
7
|
+
environment;
|
|
8
|
+
keyPrefix;
|
|
9
|
+
shardCount;
|
|
10
|
+
maxPayloadBytes;
|
|
11
|
+
subjectPrefix;
|
|
12
|
+
requestTimeoutMs;
|
|
13
|
+
natsUrl;
|
|
14
|
+
apiKey;
|
|
15
|
+
nc = null;
|
|
16
|
+
authToken = null;
|
|
17
|
+
authTokenExpiresAt = 0;
|
|
18
|
+
constructor(options) {
|
|
19
|
+
this.apiKey = options.apiKey;
|
|
20
|
+
this.keyPrefix = options.apiKey.slice(0, 12);
|
|
21
|
+
this.environment = options.environment ?? "prod";
|
|
22
|
+
this.subjectPrefix = normalizeSubjectPrefix(options.subjectPrefix ?? "beam.transfer.client");
|
|
23
|
+
this.shardCount = options.transferRuntimeShardCount ?? 1;
|
|
24
|
+
this.requestTimeoutMs = options.requestTimeoutMs ?? 30_000;
|
|
25
|
+
this.maxPayloadBytes = options.maxPayloadBytes ?? 6 * 1024 * 1024;
|
|
26
|
+
this.natsUrl = options.natsUrl ?? options.natsWsUrl ?? BEAM_DEFAULT_NATS_URL;
|
|
27
|
+
if (/^https?:\/\//i.test(this.natsUrl) || /^wss?:\/\//i.test(this.natsUrl) && !options.natsWsUrl) {
|
|
28
|
+
throw new Error("Beam SDK lifecycle endpoint must be nats://, tls://, or configured as natsWsUrl.");
|
|
29
|
+
}
|
|
30
|
+
if (!Number.isInteger(this.shardCount) || this.shardCount < 1) {
|
|
31
|
+
throw new Error("transferRuntimeShardCount must be a positive integer.");
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
async close() {
|
|
35
|
+
if (!this.nc)
|
|
36
|
+
return;
|
|
37
|
+
await this.nc.drain();
|
|
38
|
+
this.nc = null;
|
|
39
|
+
}
|
|
40
|
+
async request(messageType, payload, transferId) {
|
|
41
|
+
const nc = await this.connection();
|
|
42
|
+
const authToken = await this.resolveAuthToken();
|
|
43
|
+
const shardId = transferId ? transferShardId(transferId, this.shardCount) : 0;
|
|
44
|
+
const requestId = randomId();
|
|
45
|
+
const envelope = {
|
|
46
|
+
message_id: `${TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION}:${this.environment}:${this.keyPrefix}:${messageType}:${requestId}`,
|
|
47
|
+
schema_version: TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION,
|
|
48
|
+
environment: this.environment,
|
|
49
|
+
key_prefix: this.keyPrefix,
|
|
50
|
+
shard_id: shardId,
|
|
51
|
+
message_type: messageType,
|
|
52
|
+
request_id: requestId,
|
|
53
|
+
auth_token: authToken,
|
|
54
|
+
occurred_at: new Date().toISOString(),
|
|
55
|
+
producer: "sdk",
|
|
56
|
+
payload
|
|
57
|
+
};
|
|
58
|
+
const bytes = encode(envelope);
|
|
59
|
+
if (bytes.byteLength > this.maxPayloadBytes) {
|
|
60
|
+
throw new Error(`NATS lifecycle request is ${bytes.byteLength} bytes, above maxPayloadBytes=${this.maxPayloadBytes}`);
|
|
61
|
+
}
|
|
62
|
+
const subject = this.requestSubject(messageType, shardId);
|
|
63
|
+
const response = await nc.request(subject, bytes, { timeout: this.requestTimeoutMs });
|
|
64
|
+
const decoded = decode(response.data);
|
|
65
|
+
if (!decoded.ok) {
|
|
66
|
+
const body = JSON.stringify(decoded.error ?? {});
|
|
67
|
+
throw new Error(`Beam lifecycle request failed with ${decoded.status}: ${body}`);
|
|
68
|
+
}
|
|
69
|
+
return (decoded.payload ?? {});
|
|
70
|
+
}
|
|
71
|
+
splitRoutesForPayload(messageType, basePayload, routes) {
|
|
72
|
+
const chunks = [];
|
|
73
|
+
let current = [];
|
|
74
|
+
for (const route of routes) {
|
|
75
|
+
const candidate = [...current, route];
|
|
76
|
+
const encoded = encode({
|
|
77
|
+
schema_version: TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION,
|
|
78
|
+
environment: this.environment,
|
|
79
|
+
key_prefix: this.keyPrefix,
|
|
80
|
+
shard_id: 0,
|
|
81
|
+
message_type: messageType,
|
|
82
|
+
request_id: randomId(),
|
|
83
|
+
auth_token: "x".repeat(512),
|
|
84
|
+
occurred_at: new Date().toISOString(),
|
|
85
|
+
producer: "sdk",
|
|
86
|
+
payload: { ...basePayload, chunk_routes: candidate }
|
|
87
|
+
});
|
|
88
|
+
if (current.length > 0 && encoded.byteLength > this.maxPayloadBytes) {
|
|
89
|
+
chunks.push(current);
|
|
90
|
+
current = [route];
|
|
91
|
+
}
|
|
92
|
+
else {
|
|
93
|
+
current = candidate;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
if (current.length > 0)
|
|
97
|
+
chunks.push(current);
|
|
98
|
+
return chunks;
|
|
99
|
+
}
|
|
100
|
+
async connection() {
|
|
101
|
+
if (this.nc && !this.nc.isClosed())
|
|
102
|
+
return this.nc;
|
|
103
|
+
this.nc = await connect({
|
|
104
|
+
servers: this.natsUrl,
|
|
105
|
+
user: this.keyPrefix,
|
|
106
|
+
pass: this.apiKey,
|
|
107
|
+
name: `beam-sdk-${this.keyPrefix}`
|
|
108
|
+
});
|
|
109
|
+
return this.nc;
|
|
110
|
+
}
|
|
111
|
+
async resolveAuthToken() {
|
|
112
|
+
const now = Math.floor(Date.now() / 1000);
|
|
113
|
+
if (this.authToken && this.authTokenExpiresAt - 5 > now)
|
|
114
|
+
return this.authToken;
|
|
115
|
+
const nc = await this.connection();
|
|
116
|
+
const response = await nc.request(this.authSubject(), new TextEncoder().encode("{}"), { timeout: this.requestTimeoutMs });
|
|
117
|
+
const parsed = JSON.parse(new TextDecoder().decode(response.data));
|
|
118
|
+
if (!parsed.ok || !parsed.token) {
|
|
119
|
+
throw new Error(`Beam SDK NATS auth resolve failed: ${parsed.error ?? "unknown_error"}`);
|
|
120
|
+
}
|
|
121
|
+
const claims = decodeJwtPayload(parsed.token);
|
|
122
|
+
this.authToken = parsed.token;
|
|
123
|
+
this.authTokenExpiresAt = claims.exp;
|
|
124
|
+
return parsed.token;
|
|
125
|
+
}
|
|
126
|
+
authSubject() {
|
|
127
|
+
return `${this.subjectPrefix}.${this.environment}.auth.${this.keyPrefix}.resolve`;
|
|
128
|
+
}
|
|
129
|
+
requestSubject(messageType, shardId) {
|
|
130
|
+
return `${this.subjectPrefix}.${this.environment}.sdk.${this.keyPrefix}.shard.${shardId}.${messageType.replace(/\./g, "_")}`;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
export function transferShardId(transferId, shardCount) {
|
|
134
|
+
let hash = 2166136261;
|
|
135
|
+
for (let i = 0; i < transferId.length; i += 1) {
|
|
136
|
+
hash ^= transferId.charCodeAt(i);
|
|
137
|
+
hash = Math.imul(hash, 16777619) >>> 0;
|
|
138
|
+
}
|
|
139
|
+
return hash % shardCount;
|
|
140
|
+
}
|
|
141
|
+
function decodeJwtPayload(jwt) {
|
|
142
|
+
const parts = jwt.split(".");
|
|
143
|
+
if (parts.length < 2)
|
|
144
|
+
throw new Error("invalid JWT");
|
|
145
|
+
return JSON.parse(base64UrlDecode(parts[1]));
|
|
146
|
+
}
|
|
147
|
+
function base64UrlDecode(value) {
|
|
148
|
+
const normalized = value.replace(/-/g, "+").replace(/_/g, "/");
|
|
149
|
+
const padded = normalized.padEnd(Math.ceil(normalized.length / 4) * 4, "=");
|
|
150
|
+
if (typeof Buffer !== "undefined")
|
|
151
|
+
return Buffer.from(padded, "base64").toString("utf8");
|
|
152
|
+
return atob(padded);
|
|
153
|
+
}
|
|
154
|
+
function normalizeSubjectPrefix(value) {
|
|
155
|
+
return value.replace(/^\.+|\.+$/g, "");
|
|
156
|
+
}
|
|
157
|
+
function randomId() {
|
|
158
|
+
const cryptoApi = globalThis.crypto;
|
|
159
|
+
if (cryptoApi?.randomUUID)
|
|
160
|
+
return cryptoApi.randomUUID();
|
|
161
|
+
return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
|
|
162
|
+
}
|
|
@@ -63,12 +63,14 @@ export declare function computeChunksContentMd5(input: {
|
|
|
63
63
|
expiresIn: number;
|
|
64
64
|
fetchImpl?: typeof fetch;
|
|
65
65
|
groupSizeBytes?: number;
|
|
66
|
+
readTimeoutMs?: number;
|
|
66
67
|
}): Promise<Map<number, string>>;
|
|
67
68
|
export declare function computeChunkContentMd5(input: {
|
|
68
69
|
source: ProviderSourceConfig;
|
|
69
70
|
chunk: ChunkSigningPlanItem;
|
|
70
71
|
expiresIn: number;
|
|
71
72
|
fetchImpl?: typeof fetch;
|
|
73
|
+
readTimeoutMs?: number;
|
|
72
74
|
}): Promise<string>;
|
|
73
75
|
export declare function isS3CompatibleProvider(config: ProviderSourceConfig | ProviderDestinationConfig): config is AnyS3CompatibleProviderConfig;
|
|
74
76
|
export declare function s3CompatibleEndpoint(source: AnyS3CompatibleProviderConfig): string | undefined;
|
package/dist/provider-signing.js
CHANGED
|
@@ -4,6 +4,7 @@ import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
|
|
4
4
|
export function nowIso() {
|
|
5
5
|
return new Date().toISOString();
|
|
6
6
|
}
|
|
7
|
+
const DEFAULT_CHECKSUM_READ_TIMEOUT_MS = 300_000;
|
|
7
8
|
export function expiresAtIso(expiresIn) {
|
|
8
9
|
return new Date(Date.now() + expiresIn * 1000).toISOString();
|
|
9
10
|
}
|
|
@@ -347,12 +348,13 @@ export async function computeChunksContentMd5(input) {
|
|
|
347
348
|
const chunks = [...input.chunks].sort((a, b) => a.source_offset - b.source_offset || a.source_chunk_index - b.source_chunk_index);
|
|
348
349
|
const groups = groupContiguousChunks(chunks, input.groupSizeBytes ?? 256 * 1024 * 1024);
|
|
349
350
|
for (const group of groups) {
|
|
350
|
-
const groupOutput = await
|
|
351
|
+
const groupOutput = await computeChunkGroupContentMd5Adaptive({
|
|
351
352
|
source: input.source,
|
|
352
353
|
chunks: group,
|
|
353
354
|
expiresIn: input.expiresIn,
|
|
354
|
-
fetchImpl: input.fetchImpl
|
|
355
|
-
|
|
355
|
+
fetchImpl: input.fetchImpl,
|
|
356
|
+
readTimeoutMs: input.readTimeoutMs
|
|
357
|
+
});
|
|
356
358
|
for (const [sourceChunkIndex, checksum] of groupOutput) {
|
|
357
359
|
output.set(sourceChunkIndex, checksum);
|
|
358
360
|
}
|
|
@@ -365,7 +367,8 @@ export async function computeChunkContentMd5(input) {
|
|
|
365
367
|
chunks: [input.chunk],
|
|
366
368
|
expiresIn: input.expiresIn,
|
|
367
369
|
fetchImpl: input.fetchImpl,
|
|
368
|
-
groupSizeBytes: input.chunk.chunk_size
|
|
370
|
+
groupSizeBytes: input.chunk.chunk_size,
|
|
371
|
+
readTimeoutMs: input.readTimeoutMs
|
|
369
372
|
});
|
|
370
373
|
const checksum = checksums.get(input.chunk.source_chunk_index);
|
|
371
374
|
if (!checksum) {
|
|
@@ -400,6 +403,25 @@ function groupContiguousChunks(chunks, maxGroupSizeBytes) {
|
|
|
400
403
|
groups.push(current);
|
|
401
404
|
return groups;
|
|
402
405
|
}
|
|
406
|
+
async function computeChunkGroupContentMd5Adaptive(input) {
|
|
407
|
+
try {
|
|
408
|
+
return await retryChecksumRead(() => computeChunkGroupContentMd5(input));
|
|
409
|
+
}
|
|
410
|
+
catch (error) {
|
|
411
|
+
if (input.chunks.length <= 1) {
|
|
412
|
+
throw error;
|
|
413
|
+
}
|
|
414
|
+
const midpoint = Math.ceil(input.chunks.length / 2);
|
|
415
|
+
const output = new Map();
|
|
416
|
+
for (const chunks of [input.chunks.slice(0, midpoint), input.chunks.slice(midpoint)]) {
|
|
417
|
+
const partial = await computeChunkGroupContentMd5Adaptive({ ...input, chunks });
|
|
418
|
+
for (const [sourceChunkIndex, checksum] of partial) {
|
|
419
|
+
output.set(sourceChunkIndex, checksum);
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
return output;
|
|
423
|
+
}
|
|
424
|
+
}
|
|
403
425
|
async function computeChunkGroupContentMd5(input) {
|
|
404
426
|
if (!input.chunks.length)
|
|
405
427
|
return new Map();
|
|
@@ -408,36 +430,73 @@ async function computeChunkGroupContentMd5(input) {
|
|
|
408
430
|
const last = chunks[chunks.length - 1];
|
|
409
431
|
const sourceOffset = first.source_offset;
|
|
410
432
|
const totalSize = last.source_offset + last.chunk_size - sourceOffset;
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
433
|
+
return withChecksumReadTimeout(input.readTimeoutMs, async (abortSignal) => {
|
|
434
|
+
if (isS3CompatibleProvider(input.source)) {
|
|
435
|
+
const client = createS3CompatibleClient(input.source);
|
|
436
|
+
const response = await client.send(new GetObjectCommand({
|
|
437
|
+
Bucket: input.source.bucket,
|
|
438
|
+
Key: input.source.key,
|
|
439
|
+
Range: `bytes=${sourceOffset}-${sourceOffset + totalSize - 1}`
|
|
440
|
+
}), abortSignal ? { abortSignal } : undefined);
|
|
441
|
+
if (!response.Body) {
|
|
442
|
+
throw new Error(`chunk checksum read returned no body for ${input.source.bucket}/${input.source.key}`);
|
|
443
|
+
}
|
|
444
|
+
return hashChunkGroupContentMd5(chunks, totalSize, bodyToAsyncBytes(response.Body));
|
|
420
445
|
}
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
fetchImpl
|
|
446
|
+
const sourceRoute = await signSourceRangeRoute({
|
|
447
|
+
source: input.source,
|
|
448
|
+
fallbackUrl: first.source_url,
|
|
449
|
+
sourceOffset,
|
|
450
|
+
chunkSize: totalSize,
|
|
451
|
+
expiresIn: input.expiresIn,
|
|
452
|
+
fetchImpl: input.fetchImpl
|
|
453
|
+
});
|
|
454
|
+
const fetchImpl = input.fetchImpl ?? globalThis.fetch;
|
|
455
|
+
const response = await fetchImpl(sourceRoute.url, { method: "GET", headers: sourceRoute.headers, signal: abortSignal });
|
|
456
|
+
if (!response.ok) {
|
|
457
|
+
const message = `chunk checksum fetch failed with ${response.status}: ${await response.text()}`;
|
|
458
|
+
throw Object.assign(new Error(message), { status: response.status });
|
|
459
|
+
}
|
|
460
|
+
if (response.body) {
|
|
461
|
+
return hashChunkGroupContentMd5(chunks, totalSize, readableStreamToAsyncBytes(response.body));
|
|
462
|
+
}
|
|
463
|
+
return hashChunkGroupContentMd5(chunks, totalSize, [new Uint8Array(await response.arrayBuffer())]);
|
|
430
464
|
});
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
if (
|
|
434
|
-
|
|
435
|
-
|
|
465
|
+
}
|
|
466
|
+
function normalizedChecksumReadTimeoutMs(value) {
|
|
467
|
+
if (value === undefined)
|
|
468
|
+
return DEFAULT_CHECKSUM_READ_TIMEOUT_MS;
|
|
469
|
+
if (!Number.isFinite(value) || value <= 0)
|
|
470
|
+
return 0;
|
|
471
|
+
return Math.floor(value);
|
|
472
|
+
}
|
|
473
|
+
async function withChecksumReadTimeout(readTimeoutMs, task) {
|
|
474
|
+
const timeoutMs = normalizedChecksumReadTimeoutMs(readTimeoutMs);
|
|
475
|
+
if (timeoutMs <= 0) {
|
|
476
|
+
return task(undefined);
|
|
477
|
+
}
|
|
478
|
+
const controller = new AbortController();
|
|
479
|
+
const timeoutError = Object.assign(new Error(`chunk checksum read timed out after ${timeoutMs}ms`), { code: "CHECKSUM_READ_TIMEOUT" });
|
|
480
|
+
let timer;
|
|
481
|
+
const timeout = new Promise((_resolve, reject) => {
|
|
482
|
+
timer = setTimeout(() => {
|
|
483
|
+
controller.abort(timeoutError);
|
|
484
|
+
reject(timeoutError);
|
|
485
|
+
}, timeoutMs);
|
|
486
|
+
});
|
|
487
|
+
try {
|
|
488
|
+
return await Promise.race([task(controller.signal), timeout]);
|
|
436
489
|
}
|
|
437
|
-
|
|
438
|
-
|
|
490
|
+
catch (error) {
|
|
491
|
+
if (controller.signal.aborted) {
|
|
492
|
+
throw timeoutError;
|
|
493
|
+
}
|
|
494
|
+
throw error;
|
|
495
|
+
}
|
|
496
|
+
finally {
|
|
497
|
+
if (timer)
|
|
498
|
+
clearTimeout(timer);
|
|
439
499
|
}
|
|
440
|
-
return hashChunkGroupContentMd5(chunks, totalSize, [new Uint8Array(await response.arrayBuffer())]);
|
|
441
500
|
}
|
|
442
501
|
async function hashChunkGroupContentMd5(chunks, totalSize, body) {
|
|
443
502
|
const output = new Map();
|
|
@@ -538,6 +597,8 @@ async function retryChecksumRead(task) {
|
|
|
538
597
|
}
|
|
539
598
|
catch (error) {
|
|
540
599
|
lastError = error;
|
|
600
|
+
if (isChecksumReadTimeout(error))
|
|
601
|
+
break;
|
|
541
602
|
const status = typeof error === "object" && error && "status" in error ? Number(error.status) : undefined;
|
|
542
603
|
if (status && status >= 400 && status < 500 && status !== 408 && status !== 429)
|
|
543
604
|
break;
|
|
@@ -547,6 +608,12 @@ async function retryChecksumRead(task) {
|
|
|
547
608
|
}
|
|
548
609
|
throw lastError;
|
|
549
610
|
}
|
|
611
|
+
function isChecksumReadTimeout(error) {
|
|
612
|
+
return Boolean(error &&
|
|
613
|
+
typeof error === "object" &&
|
|
614
|
+
"code" in error &&
|
|
615
|
+
error.code === "CHECKSUM_READ_TIMEOUT");
|
|
616
|
+
}
|
|
550
617
|
function sleep(ms) {
|
|
551
618
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
552
619
|
}
|
package/package.json
CHANGED
|
@@ -1,36 +1,38 @@
|
|
|
1
1
|
{
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
"
|
|
34
|
-
|
|
35
|
-
|
|
2
|
+
"name": "@beam-network/sdk",
|
|
3
|
+
"version": "0.2.16",
|
|
4
|
+
"description": "TypeScript SDK for BEAM transfer creation and management.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"sideEffects": false,
|
|
8
|
+
"main": "./dist/index.js",
|
|
9
|
+
"types": "./dist/index.d.ts",
|
|
10
|
+
"exports": {
|
|
11
|
+
".": {
|
|
12
|
+
"types": "./dist/index.d.ts",
|
|
13
|
+
"import": "./dist/index.js"
|
|
14
|
+
}
|
|
15
|
+
},
|
|
16
|
+
"files": [
|
|
17
|
+
"dist",
|
|
18
|
+
"README.md"
|
|
19
|
+
],
|
|
20
|
+
"scripts": {
|
|
21
|
+
"build": "tsc -p tsconfig.json",
|
|
22
|
+
"test": "tsc -p tsconfig.json --noEmit \u0026\u0026 npm run test:functional",
|
|
23
|
+
"test:functional": "npm run build \u0026\u0026 node --test test/client.functional.test.mjs",
|
|
24
|
+
"lint": "tsc -p tsconfig.json --noEmit"
|
|
25
|
+
},
|
|
26
|
+
"publishConfig": {
|
|
27
|
+
"access": "public"
|
|
28
|
+
},
|
|
29
|
+
"dependencies": {
|
|
30
|
+
"@aws-sdk/client-s3": "^3.806.0",
|
|
31
|
+
"@aws-sdk/s3-request-presigner": "^3.806.0",
|
|
32
|
+
"@msgpack/msgpack": "^3.0.0",
|
|
33
|
+
"nats": "^2.29.3"
|
|
34
|
+
},
|
|
35
|
+
"devDependencies": {
|
|
36
|
+
"typescript": "^5.5.0"
|
|
37
|
+
}
|
|
36
38
|
}
|