@beam-network/sdk 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -4
- package/dist/client.d.ts +93 -5
- package/dist/client.js +127 -20
- package/dist/models.d.ts +71 -14
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -39,8 +39,7 @@ const transfer = await beam.createTransfer({
|
|
|
39
39
|
secret_access_key: "aws-secret-key"
|
|
40
40
|
})
|
|
41
41
|
],
|
|
42
|
-
name: "r2-to-s3-report"
|
|
43
|
-
testMode: true
|
|
42
|
+
name: "r2-to-s3-report"
|
|
44
43
|
});
|
|
45
44
|
|
|
46
45
|
const status = await beam.waitForTransfer(transfer.transfer_id);
|
|
@@ -49,7 +48,32 @@ await beam.close();
|
|
|
49
48
|
```
|
|
50
49
|
|
|
51
50
|
The main `createTransfer` API is provider-aware and strictly typed for S3, R2, S3-compatible, Hippius, and Hugging Face configs.
|
|
52
|
-
|
|
51
|
+
|
|
52
|
+
## Storage Credentials
|
|
53
|
+
|
|
54
|
+
Source and destination credentials must not be restricted to specific IP addresses or networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different networks, so restricted credentials make the transfer fail.
|
|
55
|
+
|
|
56
|
+
## Failed Transfers
|
|
57
|
+
|
|
58
|
+
`waitForTransfer` rejects a failed transfer with `BeamTransferFailedError`, whose `errorMessage` is BeamCore's `error_message` verbatim. When the storage refused Beam's requests, the error is the `BeamStorageAccessError` subclass, with `code` set to `source_access_denied` or `destination_access_denied`:
|
|
59
|
+
|
|
60
|
+
```ts
|
|
61
|
+
import { BeamStorageAccessError, BeamTransferFailedError } from "@beam-network/sdk";
|
|
62
|
+
|
|
63
|
+
try {
|
|
64
|
+
await beam.waitForTransfer(transfer.transfer_id);
|
|
65
|
+
} catch (error) {
|
|
66
|
+
if (error instanceof BeamStorageAccessError) {
|
|
67
|
+
// For example: "destination_access_denied: The destination storage refused Beam's requests (403 AccessDenied). ..."
|
|
68
|
+
console.error(error.code, error.errorMessage);
|
|
69
|
+
} else if (error instanceof BeamTransferFailedError) {
|
|
70
|
+
console.error(error.errorMessage);
|
|
71
|
+
}
|
|
72
|
+
throw error;
|
|
73
|
+
}
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Callers that poll `transferStatus` themselves can pass a failed status to `transferFailedError(transferId, status.error_message)` to get the same classification.
|
|
53
77
|
|
|
54
78
|
## S3-Compatible Providers
|
|
55
79
|
|
|
@@ -145,7 +169,9 @@ Constraints, all of them the Hub's rather than Beam's — see
|
|
|
145
169
|
Hub file, set `allow_source_rehash: true` to let the SDK read the source once to
|
|
146
170
|
compute it.
|
|
147
171
|
- **One source per Hugging Face destination**, because the Hub dictates the part
|
|
148
|
-
size and a plan carries a single chunk size.
|
|
172
|
+
size and a plan carries a single chunk size. The SDK sends that part size to
|
|
173
|
+
Beam as `provider_part_size`; Beam chooses the chunk size of every other
|
|
174
|
+
transfer.
|
|
149
175
|
- If you do not call `waitForTransfer`, call `finalizeHuggingFaceUploads(transferId)`
|
|
150
176
|
yourself once the transfer completes — without it the parts are uploaded but no
|
|
151
177
|
commit is made and the file does not appear in the repo.
|
package/dist/client.d.ts
CHANGED
|
@@ -19,6 +19,33 @@ export declare class BeamProviderTransferError extends AggregateError {
|
|
|
19
19
|
cleanupError?: unknown;
|
|
20
20
|
});
|
|
21
21
|
}
|
|
22
|
+
/** Failure codes BeamCore reports when source or destination storage refuses Beam's requests. */
|
|
23
|
+
export type BeamStorageAccessErrorCode = "source_access_denied" | "destination_access_denied";
|
|
24
|
+
/** A transfer that BeamCore reported as failed. */
|
|
25
|
+
export declare class BeamTransferFailedError extends Error {
|
|
26
|
+
readonly transferId: string;
|
|
27
|
+
/** The transfer's `error_message` from BeamCore, verbatim; null when BeamCore sent none. */
|
|
28
|
+
readonly errorMessage: string | null;
|
|
29
|
+
constructor(transferId: string, errorMessage: string | null);
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* The source or destination storage refused Beam's requests (`source_access_denied` or
|
|
33
|
+
* `destination_access_denied`). `errorMessage` carries BeamCore's explanation verbatim.
|
|
34
|
+
*
|
|
35
|
+
* Check that the credentials allow the operation on this bucket and path and are not restricted
|
|
36
|
+
* to specific IP addresses or networks (for example Cloudflare R2 API-token client IP filtering,
|
|
37
|
+
* S3 bucket policies with `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many
|
|
38
|
+
* workers on different networks, so restricted credentials make the transfer fail.
|
|
39
|
+
*/
|
|
40
|
+
export declare class BeamStorageAccessError extends BeamTransferFailedError {
|
|
41
|
+
readonly code: BeamStorageAccessErrorCode;
|
|
42
|
+
constructor(transferId: string, errorMessage: string, code: BeamStorageAccessErrorCode);
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Builds the error for a failed transfer status: a {@link BeamStorageAccessError} when the
|
|
46
|
+
* server message starts with a storage access code, otherwise a {@link BeamTransferFailedError}.
|
|
47
|
+
*/
|
|
48
|
+
export declare function transferFailedError(transferId: string, errorMessage: string | null): BeamTransferFailedError;
|
|
22
49
|
export declare class BeamApiError extends Error {
|
|
23
50
|
readonly status: number;
|
|
24
51
|
readonly body: string;
|
|
@@ -29,12 +56,10 @@ type TransferPrepareInput = {
|
|
|
29
56
|
sources: PreparedHttpSource[];
|
|
30
57
|
destinations: PreparedDestination[];
|
|
31
58
|
name?: string;
|
|
32
|
-
testMode?: boolean;
|
|
33
59
|
urlsExpiresAt?: string;
|
|
34
60
|
signedUrlFlow?: SignedUrlFlow;
|
|
35
61
|
idempotencyKey?: string;
|
|
36
62
|
routeGenerationId?: string;
|
|
37
|
-
chunkSize?: number;
|
|
38
63
|
};
|
|
39
64
|
export declare class BeamClient {
|
|
40
65
|
readonly apiKey: string;
|
|
@@ -54,22 +79,61 @@ export declare class BeamClient {
|
|
|
54
79
|
constructor(options?: BeamClientOptions);
|
|
55
80
|
close(): Promise<void>;
|
|
56
81
|
openTransferTerminalWaiter(transferId: string): Promise<TransferTerminalSignalWaiter>;
|
|
82
|
+
/**
|
|
83
|
+
* Creates a transfer from raw source and destination configs (`transfer.create`).
|
|
84
|
+
*
|
|
85
|
+
* Source and destination credentials must not be restricted to specific IP addresses or
|
|
86
|
+
* networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
|
|
87
|
+
* `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
|
|
88
|
+
* networks, so restricted credentials make the transfer fail.
|
|
89
|
+
*/
|
|
57
90
|
createRawTransfer(input: RawTransferCreateInput): Promise<TransferCreateResponse>;
|
|
91
|
+
/**
|
|
92
|
+
* Signs provider sources and destinations, prepares the transfer, streams its signed routes
|
|
93
|
+
* and, unless `distribute` is false, starts it. Alias of {@link prepareProviderTransfer}.
|
|
94
|
+
*
|
|
95
|
+
* Source and destination credentials must not be restricted to specific IP addresses or
|
|
96
|
+
* networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
|
|
97
|
+
* `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
|
|
98
|
+
* networks, so restricted credentials make the transfer fail.
|
|
99
|
+
*/
|
|
58
100
|
createTransfer(input: ProviderTransferCreateInput): Promise<TransferPrepareResponse>;
|
|
101
|
+
/**
|
|
102
|
+
* Takes over a prepared provider transfer in a new process, reusing its multipart uploads.
|
|
103
|
+
*
|
|
104
|
+
* Source and destination credentials must not be restricted to specific IP addresses or
|
|
105
|
+
* networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
|
|
106
|
+
* `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
|
|
107
|
+
* networks, so restricted credentials make the transfer fail.
|
|
108
|
+
*/
|
|
59
109
|
resumeProviderTransfer(input: ProviderTransferResumeInput): Promise<TransferPrepareResponse>;
|
|
60
110
|
transferStatus(transferId: string): Promise<TransferStatusInfo>;
|
|
61
111
|
distributeTransfer(transferId: string): Promise<DistributeResponse>;
|
|
62
112
|
private requestTransferCancellation;
|
|
63
113
|
cancelTransfer(transferId: string): Promise<TransferCancelResponse>;
|
|
114
|
+
/**
|
|
115
|
+
* Asks BeamCore for the compact plan a transfer would use without creating it (`transfer.plan`).
|
|
116
|
+
*
|
|
117
|
+
* Source and destination credentials must not be restricted to specific IP addresses or
|
|
118
|
+
* networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
|
|
119
|
+
* `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
|
|
120
|
+
* networks, so restricted credentials make the transfer fail.
|
|
121
|
+
*/
|
|
64
122
|
planTransfer(input: {
|
|
65
123
|
sources: PlanningHttpSource[];
|
|
66
124
|
destinations: PreparedDestination[];
|
|
67
125
|
name?: string;
|
|
68
|
-
testMode?: boolean;
|
|
69
126
|
urlsExpiresAt?: string;
|
|
70
127
|
signedUrlFlow?: SignedUrlFlow;
|
|
71
|
-
chunkSize?: number;
|
|
72
128
|
}): Promise<TransferPlanResponse>;
|
|
129
|
+
/**
|
|
130
|
+
* Prepares a transfer from already-signed HTTP sources and destinations (`transfer.prepare`).
|
|
131
|
+
*
|
|
132
|
+
* Source and destination credentials must not be restricted to specific IP addresses or
|
|
133
|
+
* networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
|
|
134
|
+
* `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
|
|
135
|
+
* networks, so restricted credentials make the transfer fail.
|
|
136
|
+
*/
|
|
73
137
|
prepareTransfer(input: TransferPrepareInput): Promise<TransferPrepareResponse>;
|
|
74
138
|
private prepareTransferWithRequestKey;
|
|
75
139
|
attachSignedUrls(transferId: string, input: {
|
|
@@ -87,13 +151,23 @@ export declare class BeamClient {
|
|
|
87
151
|
urlsExpiresAt?: string;
|
|
88
152
|
autoDistribute?: boolean;
|
|
89
153
|
}): Promise<AttachSignedUrlsResponse>;
|
|
154
|
+
/**
|
|
155
|
+
* Signs provider sources and destinations, prepares the transfer, streams its signed routes
|
|
156
|
+
* and, unless `distribute` is false, starts it.
|
|
157
|
+
*
|
|
158
|
+
* Source and destination credentials must not be restricted to specific IP addresses or
|
|
159
|
+
* networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
|
|
160
|
+
* `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
|
|
161
|
+
* networks, so restricted credentials make the transfer fail.
|
|
162
|
+
*/
|
|
90
163
|
prepareProviderTransfer(input: ProviderTransferCreateInput): Promise<TransferPrepareResponse>;
|
|
91
164
|
private executeProviderTransfer;
|
|
92
165
|
/**
|
|
93
166
|
* Negotiate every Hugging Face destination before the plan exists.
|
|
94
167
|
*
|
|
95
168
|
* The Hub will not issue upload URLs without the object's sha256, and it chooses the part
|
|
96
|
-
* size itself, so this runs first and the
|
|
169
|
+
* size itself, so this runs first and the prepare request then carries the Hub's part size
|
|
170
|
+
* as `provider_part_size`.
|
|
97
171
|
*/
|
|
98
172
|
private planHuggingFaceUploads;
|
|
99
173
|
/** Fail before any byte moves if BeamCore did not adopt the Hub's part layout. */
|
|
@@ -113,7 +187,21 @@ export declare class BeamClient {
|
|
|
113
187
|
/** Stop a transfer's recovery and integrity signers; with `owner`, only if that owner installed them. */
|
|
114
188
|
private stopRecoverySigner;
|
|
115
189
|
private stopAllRecoverySigners;
|
|
190
|
+
/**
|
|
191
|
+
* Creates a raw transfer and distributes it.
|
|
192
|
+
*
|
|
193
|
+
* Source and destination credentials must not be restricted to specific IP addresses or
|
|
194
|
+
* networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
|
|
195
|
+
* `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
|
|
196
|
+
* networks, so restricted credentials make the transfer fail.
|
|
197
|
+
*/
|
|
116
198
|
createAndDistribute(input: RawTransferCreateInput): Promise<TransferCreateResponse>;
|
|
199
|
+
/**
|
|
200
|
+
* Waits until the transfer completes. A failed transfer rejects with a
|
|
201
|
+
* {@link BeamTransferFailedError} carrying BeamCore's `error_message` verbatim, or with its
|
|
202
|
+
* {@link BeamStorageAccessError} subclass when the source or destination storage refused
|
|
203
|
+
* Beam's requests (`source_access_denied`, `destination_access_denied`).
|
|
204
|
+
*/
|
|
117
205
|
waitForTransfer(transferId: string, options?: {
|
|
118
206
|
timeoutMs?: number;
|
|
119
207
|
pollIntervalMs?: number;
|
package/dist/client.js
CHANGED
|
@@ -36,6 +36,54 @@ export class BeamProviderTransferError extends AggregateError {
|
|
|
36
36
|
this.cause = input.cause;
|
|
37
37
|
}
|
|
38
38
|
}
|
|
39
|
+
const STORAGE_ACCESS_ERROR_CODES = [
|
|
40
|
+
"source_access_denied",
|
|
41
|
+
"destination_access_denied"
|
|
42
|
+
];
|
|
43
|
+
/** A transfer that BeamCore reported as failed. */
|
|
44
|
+
export class BeamTransferFailedError extends Error {
|
|
45
|
+
transferId;
|
|
46
|
+
/** The transfer's `error_message` from BeamCore, verbatim; null when BeamCore sent none. */
|
|
47
|
+
errorMessage;
|
|
48
|
+
constructor(transferId, errorMessage) {
|
|
49
|
+
super(`Transfer ${transferId} failed: ${errorMessage ?? "unknown error"}`);
|
|
50
|
+
this.name = "BeamTransferFailedError";
|
|
51
|
+
this.transferId = transferId;
|
|
52
|
+
this.errorMessage = errorMessage;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* The source or destination storage refused Beam's requests (`source_access_denied` or
|
|
57
|
+
* `destination_access_denied`). `errorMessage` carries BeamCore's explanation verbatim.
|
|
58
|
+
*
|
|
59
|
+
* Check that the credentials allow the operation on this bucket and path and are not restricted
|
|
60
|
+
* to specific IP addresses or networks (for example Cloudflare R2 API-token client IP filtering,
|
|
61
|
+
* S3 bucket policies with `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many
|
|
62
|
+
* workers on different networks, so restricted credentials make the transfer fail.
|
|
63
|
+
*/
|
|
64
|
+
export class BeamStorageAccessError extends BeamTransferFailedError {
|
|
65
|
+
code;
|
|
66
|
+
constructor(transferId, errorMessage, code) {
|
|
67
|
+
super(transferId, errorMessage);
|
|
68
|
+
this.name = "BeamStorageAccessError";
|
|
69
|
+
this.code = code;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Builds the error for a failed transfer status: a {@link BeamStorageAccessError} when the
|
|
74
|
+
* server message starts with a storage access code, otherwise a {@link BeamTransferFailedError}.
|
|
75
|
+
*/
|
|
76
|
+
export function transferFailedError(transferId, errorMessage) {
|
|
77
|
+
const code = errorMessage === null ? undefined : storageAccessErrorCode(errorMessage);
|
|
78
|
+
return code && errorMessage !== null
|
|
79
|
+
? new BeamStorageAccessError(transferId, errorMessage, code)
|
|
80
|
+
: new BeamTransferFailedError(transferId, errorMessage);
|
|
81
|
+
}
|
|
82
|
+
function storageAccessErrorCode(errorMessage) {
|
|
83
|
+
const separator = errorMessage.indexOf(":");
|
|
84
|
+
const prefix = (separator === -1 ? errorMessage : errorMessage.slice(0, separator)).trim();
|
|
85
|
+
return STORAGE_ACCESS_ERROR_CODES.find((code) => code === prefix);
|
|
86
|
+
}
|
|
39
87
|
export class BeamApiError extends Error {
|
|
40
88
|
status;
|
|
41
89
|
body;
|
|
@@ -94,6 +142,14 @@ export class BeamClient {
|
|
|
94
142
|
validateId(transferId, "transferId");
|
|
95
143
|
return this.control.openTerminalSignalWaiter(transferId);
|
|
96
144
|
}
|
|
145
|
+
/**
|
|
146
|
+
* Creates a transfer from raw source and destination configs (`transfer.create`).
|
|
147
|
+
*
|
|
148
|
+
* Source and destination credentials must not be restricted to specific IP addresses or
|
|
149
|
+
* networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
|
|
150
|
+
* `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
|
|
151
|
+
* networks, so restricted credentials make the transfer fail.
|
|
152
|
+
*/
|
|
97
153
|
async createRawTransfer(input) {
|
|
98
154
|
const transferId = await transferIdForIdempotencyKey(input.idempotencyKey);
|
|
99
155
|
const body = compact({
|
|
@@ -101,12 +157,10 @@ export class BeamClient {
|
|
|
101
157
|
sources: input.sources,
|
|
102
158
|
destinations: input.destinations,
|
|
103
159
|
total_size: input.totalSize,
|
|
104
|
-
chunk_size: input.chunkSize,
|
|
105
160
|
name: input.name,
|
|
106
161
|
merkle_root: input.merkleRoot,
|
|
107
162
|
chunk_hashes: input.chunkHashes,
|
|
108
163
|
callbacks: input.callbacks,
|
|
109
|
-
test_mode: input.testMode || undefined,
|
|
110
164
|
progressive_mode: input.progressiveMode || undefined,
|
|
111
165
|
signed_url_flow: input.signedUrlFlow ?? "signed_url"
|
|
112
166
|
});
|
|
@@ -115,9 +169,26 @@ export class BeamClient {
|
|
|
115
169
|
idempotencyKey: `transfer:${transferId}:create`
|
|
116
170
|
});
|
|
117
171
|
}
|
|
172
|
+
/**
|
|
173
|
+
* Signs provider sources and destinations, prepares the transfer, streams its signed routes
|
|
174
|
+
* and, unless `distribute` is false, starts it. Alias of {@link prepareProviderTransfer}.
|
|
175
|
+
*
|
|
176
|
+
* Source and destination credentials must not be restricted to specific IP addresses or
|
|
177
|
+
* networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
|
|
178
|
+
* `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
|
|
179
|
+
* networks, so restricted credentials make the transfer fail.
|
|
180
|
+
*/
|
|
118
181
|
createTransfer(input) {
|
|
119
182
|
return this.prepareProviderTransfer(input);
|
|
120
183
|
}
|
|
184
|
+
/**
|
|
185
|
+
* Takes over a prepared provider transfer in a new process, reusing its multipart uploads.
|
|
186
|
+
*
|
|
187
|
+
* Source and destination credentials must not be restricted to specific IP addresses or
|
|
188
|
+
* networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
|
|
189
|
+
* `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
|
|
190
|
+
* networks, so restricted credentials make the transfer fail.
|
|
191
|
+
*/
|
|
121
192
|
async resumeProviderTransfer(input) {
|
|
122
193
|
validateId(input.transferId, "transferId");
|
|
123
194
|
return this.executeProviderTransfer(input, input);
|
|
@@ -159,13 +230,19 @@ export class BeamClient {
|
|
|
159
230
|
this.stopRecoverySigner(transferId);
|
|
160
231
|
return result;
|
|
161
232
|
}
|
|
233
|
+
/**
|
|
234
|
+
* Asks BeamCore for the compact plan a transfer would use without creating it (`transfer.plan`).
|
|
235
|
+
*
|
|
236
|
+
* Source and destination credentials must not be restricted to specific IP addresses or
|
|
237
|
+
* networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
|
|
238
|
+
* `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
|
|
239
|
+
* networks, so restricted credentials make the transfer fail.
|
|
240
|
+
*/
|
|
162
241
|
async planTransfer(input) {
|
|
163
242
|
const result = await this.control.request("transfer.plan", compact({
|
|
164
243
|
sources: input.sources,
|
|
165
244
|
destinations: input.destinations,
|
|
166
245
|
name: input.name,
|
|
167
|
-
test_mode: input.testMode || undefined,
|
|
168
|
-
chunk_size: input.chunkSize,
|
|
169
246
|
urls_expires_at: input.urlsExpiresAt,
|
|
170
247
|
signed_url_flow: input.signedUrlFlow ?? "signed_url"
|
|
171
248
|
}));
|
|
@@ -173,13 +250,21 @@ export class BeamClient {
|
|
|
173
250
|
validateCompactTransferPlan(result.plan_descriptor, result.signed_url_flow);
|
|
174
251
|
return result;
|
|
175
252
|
}
|
|
253
|
+
/**
|
|
254
|
+
* Prepares a transfer from already-signed HTTP sources and destinations (`transfer.prepare`).
|
|
255
|
+
*
|
|
256
|
+
* Source and destination credentials must not be restricted to specific IP addresses or
|
|
257
|
+
* networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
|
|
258
|
+
* `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
|
|
259
|
+
* networks, so restricted credentials make the transfer fail.
|
|
260
|
+
*/
|
|
176
261
|
prepareTransfer(input) {
|
|
177
262
|
return this.prepareTransferWithRequestKey(input);
|
|
178
263
|
}
|
|
179
|
-
async prepareTransferWithRequestKey(input,
|
|
264
|
+
async prepareTransferWithRequestKey(input, options = {}) {
|
|
180
265
|
const transferId = input.transferId ?? await transferIdForIdempotencyKey(input.idempotencyKey);
|
|
181
266
|
validateId(transferId, "transferId");
|
|
182
|
-
const prepareIdempotencyKey = requestKey ?? `transfer:${transferId}:prepare`;
|
|
267
|
+
const prepareIdempotencyKey = options.requestKey ?? `transfer:${transferId}:prepare`;
|
|
183
268
|
const routeGenerationId = input.routeGenerationId ?? await routeGenerationIdForPrepareIdempotencyKey(prepareIdempotencyKey);
|
|
184
269
|
const result = await this.control.request("transfer.prepare", compact({
|
|
185
270
|
transfer_id: transferId,
|
|
@@ -187,8 +272,7 @@ export class BeamClient {
|
|
|
187
272
|
sources: input.sources,
|
|
188
273
|
destinations: input.destinations,
|
|
189
274
|
name: input.name,
|
|
190
|
-
|
|
191
|
-
chunk_size: input.chunkSize,
|
|
275
|
+
provider_part_size: options.providerPartSize,
|
|
192
276
|
urls_expires_at: input.urlsExpiresAt,
|
|
193
277
|
signed_url_flow: input.signedUrlFlow ?? "signed_url"
|
|
194
278
|
}), {
|
|
@@ -274,6 +358,15 @@ export class BeamClient {
|
|
|
274
358
|
releaseInitialStream();
|
|
275
359
|
}
|
|
276
360
|
}
|
|
361
|
+
/**
|
|
362
|
+
* Signs provider sources and destinations, prepares the transfer, streams its signed routes
|
|
363
|
+
* and, unless `distribute` is false, starts it.
|
|
364
|
+
*
|
|
365
|
+
* Source and destination credentials must not be restricted to specific IP addresses or
|
|
366
|
+
* networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
|
|
367
|
+
* `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
|
|
368
|
+
* networks, so restricted credentials make the transfer fail.
|
|
369
|
+
*/
|
|
277
370
|
async prepareProviderTransfer(input) {
|
|
278
371
|
return this.executeProviderTransfer(input);
|
|
279
372
|
}
|
|
@@ -316,14 +409,13 @@ export class BeamClient {
|
|
|
316
409
|
sources: preparedSources,
|
|
317
410
|
destinations: preparedDestinations,
|
|
318
411
|
name: input.name,
|
|
319
|
-
testMode: input.testMode,
|
|
320
|
-
chunkSize: huggingFace.chunkSize ?? input.chunkSize,
|
|
321
412
|
signedUrlFlow: requestedSignedUrlFlow,
|
|
322
413
|
...(resume ? { transferId: resume.transferId } : { idempotencyKey: input.idempotencyKey, routeGenerationId: input.routeGenerationId })
|
|
323
414
|
};
|
|
324
|
-
const prepared =
|
|
325
|
-
|
|
326
|
-
:
|
|
415
|
+
const prepared = await this.prepareTransferWithRequestKey(prepareInput, {
|
|
416
|
+
requestKey: resume ? `transfer:${resume.transferId}:prepare:resume:${randomUuid()}` : undefined,
|
|
417
|
+
providerPartSize: huggingFace.providerPartSize
|
|
418
|
+
});
|
|
327
419
|
if (resume && prepared.transfer_id !== resume.transferId)
|
|
328
420
|
throw new Error("resumed provider transfer id mismatch");
|
|
329
421
|
input.signal?.throwIfAborted();
|
|
@@ -610,7 +702,8 @@ export class BeamClient {
|
|
|
610
702
|
* Negotiate every Hugging Face destination before the plan exists.
|
|
611
703
|
*
|
|
612
704
|
* The Hub will not issue upload URLs without the object's sha256, and it chooses the part
|
|
613
|
-
* size itself, so this runs first and the
|
|
705
|
+
* size itself, so this runs first and the prepare request then carries the Hub's part size
|
|
706
|
+
* as `provider_part_size`.
|
|
614
707
|
*/
|
|
615
708
|
async planHuggingFaceUploads(input) {
|
|
616
709
|
const targets = input.destinations
|
|
@@ -629,7 +722,7 @@ export class BeamClient {
|
|
|
629
722
|
? typeof preparedSource.metadata?.sha256 === "string" ? preparedSource.metadata.sha256 : undefined
|
|
630
723
|
: undefined;
|
|
631
724
|
const states = [];
|
|
632
|
-
let
|
|
725
|
+
let providerPartSize;
|
|
633
726
|
for (const { destination, index } of targets) {
|
|
634
727
|
const preparedDestination = input.preparedDestinations[index];
|
|
635
728
|
if ((destination.repo_type ?? "model") === "bucket") {
|
|
@@ -667,11 +760,11 @@ export class BeamClient {
|
|
|
667
760
|
size: preparedSource.size
|
|
668
761
|
});
|
|
669
762
|
if (plan.upload?.chunkSize !== undefined) {
|
|
670
|
-
if (
|
|
671
|
-
throw new Error(`huggingface destinations disagree on part size (${
|
|
763
|
+
if (providerPartSize !== undefined && providerPartSize !== plan.upload.chunkSize) {
|
|
764
|
+
throw new Error(`huggingface destinations disagree on part size (${providerPartSize} vs ${plan.upload.chunkSize}); `
|
|
672
765
|
+ "the plan carries a single chunk size");
|
|
673
766
|
}
|
|
674
|
-
|
|
767
|
+
providerPartSize = plan.upload.chunkSize;
|
|
675
768
|
}
|
|
676
769
|
states.push({
|
|
677
770
|
destination: config,
|
|
@@ -692,7 +785,7 @@ export class BeamClient {
|
|
|
692
785
|
}).then((hashed) => hashed.partEtags)
|
|
693
786
|
});
|
|
694
787
|
}
|
|
695
|
-
return { states,
|
|
788
|
+
return { states, providerPartSize };
|
|
696
789
|
}
|
|
697
790
|
/** Fail before any byte moves if BeamCore did not adopt the Hub's part layout. */
|
|
698
791
|
assertHuggingFacePlan(prepared, states) {
|
|
@@ -1135,6 +1228,14 @@ export class BeamClient {
|
|
|
1135
1228
|
for (const transferId of [...this.recoverySigners.keys()])
|
|
1136
1229
|
this.stopRecoverySigner(transferId);
|
|
1137
1230
|
}
|
|
1231
|
+
/**
|
|
1232
|
+
* Creates a raw transfer and distributes it.
|
|
1233
|
+
*
|
|
1234
|
+
* Source and destination credentials must not be restricted to specific IP addresses or
|
|
1235
|
+
* networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
|
|
1236
|
+
* `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
|
|
1237
|
+
* networks, so restricted credentials make the transfer fail.
|
|
1238
|
+
*/
|
|
1138
1239
|
async createAndDistribute(input) {
|
|
1139
1240
|
const transfer = await this.createRawTransfer(input);
|
|
1140
1241
|
if (transfer.success) {
|
|
@@ -1142,6 +1243,12 @@ export class BeamClient {
|
|
|
1142
1243
|
}
|
|
1143
1244
|
return transfer;
|
|
1144
1245
|
}
|
|
1246
|
+
/**
|
|
1247
|
+
* Waits until the transfer completes. A failed transfer rejects with a
|
|
1248
|
+
* {@link BeamTransferFailedError} carrying BeamCore's `error_message` verbatim, or with its
|
|
1249
|
+
* {@link BeamStorageAccessError} subclass when the source or destination storage refused
|
|
1250
|
+
* Beam's requests (`source_access_denied`, `destination_access_denied`).
|
|
1251
|
+
*/
|
|
1145
1252
|
async waitForTransfer(transferId, options = {}) {
|
|
1146
1253
|
const timeoutMs = options.timeoutMs ?? 300_000;
|
|
1147
1254
|
const pollIntervalMs = options.pollIntervalMs ?? 15_000;
|
|
@@ -1174,7 +1281,7 @@ export class BeamClient {
|
|
|
1174
1281
|
return status;
|
|
1175
1282
|
}
|
|
1176
1283
|
if (status.status === "failed") {
|
|
1177
|
-
throw
|
|
1284
|
+
throw transferFailedError(transferId, status.error_message);
|
|
1178
1285
|
}
|
|
1179
1286
|
if (status.status === "cancelled") {
|
|
1180
1287
|
throw new Error("Transfer cancelled");
|
package/dist/models.d.ts
CHANGED
|
@@ -34,6 +34,14 @@ export interface BeamClientOptions {
|
|
|
34
34
|
multipartControlConcurrency?: number;
|
|
35
35
|
fetch?: typeof fetch;
|
|
36
36
|
}
|
|
37
|
+
/**
|
|
38
|
+
* Raw transfer source.
|
|
39
|
+
*
|
|
40
|
+
* Source credentials must not be restricted to specific IP addresses or networks (for example
|
|
41
|
+
* Cloudflare R2 API-token client IP filtering, S3 bucket policies with `aws:SourceIp`, or
|
|
42
|
+
* VPC-only endpoints). Beam moves data through many workers on different networks, so
|
|
43
|
+
* restricted credentials make the transfer fail.
|
|
44
|
+
*/
|
|
37
45
|
export interface SourceConfig {
|
|
38
46
|
type: string;
|
|
39
47
|
bucket?: string;
|
|
@@ -47,6 +55,14 @@ export interface SourceConfig {
|
|
|
47
55
|
url?: string;
|
|
48
56
|
headers?: Record<string, string>;
|
|
49
57
|
}
|
|
58
|
+
/**
|
|
59
|
+
* Raw transfer destination.
|
|
60
|
+
*
|
|
61
|
+
* Destination credentials must not be restricted to specific IP addresses or networks (for
|
|
62
|
+
* example Cloudflare R2 API-token client IP filtering, S3 bucket policies with `aws:SourceIp`, or
|
|
63
|
+
* VPC-only endpoints). Beam moves data through many workers on different networks, so
|
|
64
|
+
* restricted credentials make the transfer fail.
|
|
65
|
+
*/
|
|
50
66
|
export interface DestConfig {
|
|
51
67
|
type: string;
|
|
52
68
|
bucket?: string;
|
|
@@ -68,12 +84,10 @@ export interface TransferCreateRequest {
|
|
|
68
84
|
sources: SourceConfig[];
|
|
69
85
|
destinations: DestConfig[];
|
|
70
86
|
total_size: number;
|
|
71
|
-
chunk_size?: number;
|
|
72
87
|
name?: string;
|
|
73
88
|
merkle_root?: string;
|
|
74
89
|
chunk_hashes?: string[];
|
|
75
90
|
callbacks?: CallbackConfig[];
|
|
76
|
-
test_mode?: boolean;
|
|
77
91
|
progressive_mode?: boolean;
|
|
78
92
|
signed_url_flow: SignedUrlFlow;
|
|
79
93
|
}
|
|
@@ -81,15 +95,10 @@ export interface RawTransferCreateInput {
|
|
|
81
95
|
sources: SourceConfig[];
|
|
82
96
|
destinations: DestConfig[];
|
|
83
97
|
totalSize: number;
|
|
84
|
-
chunkSize?: number;
|
|
85
98
|
name?: string;
|
|
86
99
|
merkleRoot?: string;
|
|
87
100
|
chunkHashes?: string[];
|
|
88
101
|
callbacks?: CallbackConfig[];
|
|
89
|
-
/**
|
|
90
|
-
* Maps to BeamCore test_mode.
|
|
91
|
-
*/
|
|
92
|
-
testMode?: boolean;
|
|
93
102
|
progressiveMode?: boolean;
|
|
94
103
|
signedUrlFlow?: SignedUrlFlow;
|
|
95
104
|
idempotencyKey?: string;
|
|
@@ -254,6 +263,14 @@ export interface TransferTerminalSignalWaiter {
|
|
|
254
263
|
wait(timeoutMs: number): Promise<TransferTerminalEvent | null>;
|
|
255
264
|
close(): Promise<void>;
|
|
256
265
|
}
|
|
266
|
+
/**
|
|
267
|
+
* Amazon S3 source or destination.
|
|
268
|
+
*
|
|
269
|
+
* Credentials must not be restricted to specific IP addresses or networks (for example
|
|
270
|
+
* Cloudflare R2 API-token client IP filtering, S3 bucket policies with `aws:SourceIp`, or
|
|
271
|
+
* VPC-only endpoints). Beam moves data through many workers on different networks, so
|
|
272
|
+
* restricted credentials make the transfer fail.
|
|
273
|
+
*/
|
|
257
274
|
export interface S3ProviderConfig {
|
|
258
275
|
/** Physical storage location; independent of the signing region. */
|
|
259
276
|
storage_location?: string;
|
|
@@ -267,6 +284,14 @@ export interface S3ProviderConfig {
|
|
|
267
284
|
session_token?: string;
|
|
268
285
|
endpoint_url?: string;
|
|
269
286
|
}
|
|
287
|
+
/**
|
|
288
|
+
* Cloudflare R2 source or destination.
|
|
289
|
+
*
|
|
290
|
+
* Credentials must not be restricted to specific IP addresses or networks (for example
|
|
291
|
+
* Cloudflare R2 API-token client IP filtering, S3 bucket policies with `aws:SourceIp`, or
|
|
292
|
+
* VPC-only endpoints). Beam moves data through many workers on different networks, so
|
|
293
|
+
* restricted credentials make the transfer fail.
|
|
294
|
+
*/
|
|
270
295
|
export interface R2ProviderConfig {
|
|
271
296
|
/** Physical storage location; independent of the signing region. */
|
|
272
297
|
storage_location?: string;
|
|
@@ -279,6 +304,14 @@ export interface R2ProviderConfig {
|
|
|
279
304
|
account_id?: string;
|
|
280
305
|
endpoint_url?: string;
|
|
281
306
|
}
|
|
307
|
+
/**
|
|
308
|
+
* S3-compatible source or destination (MinIO, Wasabi, Backblaze B2, DigitalOcean Spaces and others).
|
|
309
|
+
*
|
|
310
|
+
* Credentials must not be restricted to specific IP addresses or networks (for example
|
|
311
|
+
* Cloudflare R2 API-token client IP filtering, S3 bucket policies with `aws:SourceIp`, or
|
|
312
|
+
* VPC-only endpoints). Beam moves data through many workers on different networks, so
|
|
313
|
+
* restricted credentials make the transfer fail.
|
|
314
|
+
*/
|
|
282
315
|
export interface S3CompatibleProviderConfig {
|
|
283
316
|
/** Physical storage location; independent of the signing region. */
|
|
284
317
|
storage_location?: string;
|
|
@@ -295,6 +328,14 @@ export interface S3CompatibleProviderConfig {
|
|
|
295
328
|
force_path_style?: boolean;
|
|
296
329
|
account_id?: string;
|
|
297
330
|
}
|
|
331
|
+
/**
|
|
332
|
+
* Hippius source or destination.
|
|
333
|
+
*
|
|
334
|
+
* Credentials must not be restricted to specific IP addresses or networks (for example
|
|
335
|
+
* Cloudflare R2 API-token client IP filtering, S3 bucket policies with `aws:SourceIp`, or
|
|
336
|
+
* VPC-only endpoints). Beam moves data through many workers on different networks, so
|
|
337
|
+
* restricted credentials make the transfer fail.
|
|
338
|
+
*/
|
|
298
339
|
export interface HippiusProviderConfig {
|
|
299
340
|
/** Physical storage location; independent of the signing region. */
|
|
300
341
|
storage_location?: string;
|
|
@@ -306,6 +347,14 @@ export interface HippiusProviderConfig {
|
|
|
306
347
|
base_url?: string;
|
|
307
348
|
}
|
|
308
349
|
export type HuggingFaceRepoType = "model" | "dataset" | "space" | "kernel" | "bucket";
|
|
350
|
+
/**
|
|
351
|
+
* Hugging Face Hub source or destination.
|
|
352
|
+
*
|
|
353
|
+
* The token and any credentials must not be restricted to specific IP addresses or networks (for example
|
|
354
|
+
* Cloudflare R2 API-token client IP filtering, S3 bucket policies with `aws:SourceIp`, or
|
|
355
|
+
* VPC-only endpoints). Beam moves data through many workers on different networks, so
|
|
356
|
+
* restricted credentials make the transfer fail.
|
|
357
|
+
*/
|
|
309
358
|
export interface HuggingFaceProviderConfig {
|
|
310
359
|
/** Physical storage location; independent of the signing region. */
|
|
311
360
|
storage_location?: string;
|
|
@@ -360,13 +409,19 @@ export declare const HuggingFaceProviderConfig: Readonly<{
|
|
|
360
409
|
export interface ProviderTransferCreateInput {
|
|
361
410
|
/** Ownership fence. Aborting stops signing/replay without cancelling a replacement owner. */
|
|
362
411
|
signal?: AbortSignal;
|
|
412
|
+
/**
|
|
413
|
+
* Storage to read from. Source credentials must not be restricted to specific IP addresses or
|
|
414
|
+
* networks (for example Cloudflare R2 API-token client IP filtering, S3 bucket policies with
|
|
415
|
+
* `aws:SourceIp`, or VPC-only endpoints). Beam moves data through many workers on different
|
|
416
|
+
* networks, so restricted credentials make the transfer fail.
|
|
417
|
+
*/
|
|
363
418
|
sources: ProviderSourceConfig[];
|
|
364
|
-
destinations: ProviderDestinationConfig[];
|
|
365
|
-
name?: string;
|
|
366
419
|
/**
|
|
367
|
-
*
|
|
420
|
+
* Storage to write to. Destination credentials must not be restricted to specific IP addresses
|
|
421
|
+
* or networks, for the same reason as `sources`.
|
|
368
422
|
*/
|
|
369
|
-
|
|
423
|
+
destinations: ProviderDestinationConfig[];
|
|
424
|
+
name?: string;
|
|
370
425
|
expiresIn?: number;
|
|
371
426
|
/**
|
|
372
427
|
* Defaults to true. Set to false to only prepare and stream signed routes without distribution.
|
|
@@ -386,8 +441,6 @@ export interface ProviderTransferCreateInput {
|
|
|
386
441
|
idempotencyKey?: string;
|
|
387
442
|
/** Internal recovery generation; callers normally omit this. */
|
|
388
443
|
routeGenerationId?: string;
|
|
389
|
-
/** Requested plan chunk size. BeamCore may raise it; the response carries the effective value. */
|
|
390
|
-
chunkSize?: number;
|
|
391
444
|
}
|
|
392
445
|
export interface ProviderMultipartGroupIdentity {
|
|
393
446
|
transferId: string;
|
|
@@ -400,6 +453,11 @@ export interface ProviderMultipartGroupIdentity {
|
|
|
400
453
|
expectedPartCount: number;
|
|
401
454
|
expiresAt: string;
|
|
402
455
|
}
|
|
456
|
+
/**
|
|
457
|
+
* HTTP source for planning and preparing. Its URL and headers must work from any network: Beam
|
|
458
|
+
* moves data through many workers on different networks, so IP- or network-restricted URLs make
|
|
459
|
+
* the transfer fail.
|
|
460
|
+
*/
|
|
403
461
|
export interface PlanningHttpSource {
|
|
404
462
|
source_id: string;
|
|
405
463
|
type: "http";
|
|
@@ -511,7 +569,6 @@ export interface TransferPrepareResponse {
|
|
|
511
569
|
success: boolean;
|
|
512
570
|
transfer_id: string;
|
|
513
571
|
transfer_key?: string;
|
|
514
|
-
test_mode?: boolean;
|
|
515
572
|
chunk_size?: number;
|
|
516
573
|
total_size?: number;
|
|
517
574
|
total_sources?: number;
|