@beam-network/sdk 0.5.17-dev.1 → 0.5.17-dev.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/dist/client.d.ts +2 -0
- package/dist/client.js +124 -92
- package/dist/models.d.ts +6 -12
- package/dist/provider-signing.d.ts +2 -0
- package/dist/provider-signing.js +12 -5
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -12,7 +12,7 @@ Example:
|
|
|
12
12
|
import { BeamClient, R2ProviderConfig, S3ProviderConfig } from "@beam-network/sdk";
|
|
13
13
|
|
|
14
14
|
const beam = new BeamClient({
|
|
15
|
-
apiKey: "
|
|
15
|
+
apiKey: "b1m_your_key"
|
|
16
16
|
});
|
|
17
17
|
|
|
18
18
|
const transfer = await beam.createTransfer({
|
|
@@ -155,9 +155,13 @@ The client signs up to 64 routes concurrently by default and emits 2,048-route l
|
|
|
155
155
|
|
|
156
156
|
Callers that need durable recovery accounting can provide `onMultipartGroupReady`. The SDK invokes it after creating a multipart upload and before streaming that group’s signed routes. Its payload contains transfer, source, destination, object-key, upload, size, part-count, and expiry identities only; it never includes credentials, request headers, or signed URLs. A rejected callback aborts the newly created upload and fails preparation closed.
|
|
157
157
|
|
|
158
|
+
`resumeProviderTransfer` requires the complete saved callback payloads in `multipartGroups`. It validates every transfer/source/destination/object/size/part coordinate, then uses the same route-stream implementation as creation with those upload IDs. Missing, duplicate, or changed identities fail before new provider uploads or route publication. If a process died before recording an upload identity, treat it as uncertain cleanup instead of inventing a replacement. The `onPrepared` callback also runs during resume, before routes are streamed.
|
|
159
|
+
|
|
160
|
+
Create and resume accept an ownership `signal`. Aborting it stops initial routes, multipart creation, and recovery signing/replay; it does not cancel the transfer or abort a replacement owner's uploads. User cancellation still uses `cancelTransfer` and verified provider cleanup. `prepareProviderSource` accepts the same signal for metadata requests.
|
|
161
|
+
|
|
158
162
|
The stream ID derives from the transfer, selected flow, and immutable plan identity, and each ordered batch ID includes its route-coordinate checksum. Manual multi-destination attachment requires `delivery_index` on every route. Lifecycle mutations retry transient failures up to three times with the same request identity. A non-recoverable provider setup or route failure raises `BeamProviderTransferError`, whose `errors`, `transferCancelled`, and `multipartCleanupComplete` fields retain the original provider, cancellation, and cleanup outcomes. `waitForTransfer` subscribes to the API-key-owned, at-most-once terminal signal before its first status read and reconciles every signal through authoritative status; subscription failure degrades to the same jittered 15-to-30-second status fallback. Call `close()` when the client is no longer needed; it also closes outstanding terminal waiters.
|
|
159
163
|
|
|
160
|
-
|
|
164
|
+
During the pre-completion `integrity_check` phase, Runtime may include an `integrity_audit_challenge` in status. The TypeScript client signs exact read-only source and final-destination GET ranges and submits `transfer.integrity_audit_grants`. S3-compatible source and destination grants use the prepared source ETag and Runtime-verified final-object ETag as `If-Match` conditions. Submission failures retry on the next status poll; the transfer completes with a warning if the check cannot finish by its deadline.
|
|
161
165
|
|
|
162
166
|
Hippius uses canonical non-multipart `signed_url` routes, so its manifest is empty and group-level final HEAD verification is not available from that provider flow.
|
|
163
167
|
|
package/dist/client.d.ts
CHANGED
|
@@ -56,6 +56,7 @@ export declare class BeamClient {
|
|
|
56
56
|
resumeProviderTransfer(input: ProviderTransferResumeInput): Promise<TransferPrepareResponse>;
|
|
57
57
|
transferStatus(transferId: string): Promise<TransferStatusInfo>;
|
|
58
58
|
distributeTransfer(transferId: string): Promise<DistributeResponse>;
|
|
59
|
+
private requestTransferCancellation;
|
|
59
60
|
cancelTransfer(transferId: string): Promise<TransferCancelResponse>;
|
|
60
61
|
planTransfer(input: {
|
|
61
62
|
sources: PlanningHttpSource[];
|
|
@@ -84,6 +85,7 @@ export declare class BeamClient {
|
|
|
84
85
|
autoDistribute?: boolean;
|
|
85
86
|
}): Promise<AttachSignedUrlsResponse>;
|
|
86
87
|
prepareProviderTransfer(input: ProviderTransferCreateInput): Promise<TransferPrepareResponse>;
|
|
88
|
+
private executeProviderTransfer;
|
|
87
89
|
/**
|
|
88
90
|
* Negotiate every Hugging Face destination before the plan exists.
|
|
89
91
|
*
|
package/dist/client.js
CHANGED
|
@@ -115,68 +115,15 @@ export class BeamClient {
|
|
|
115
115
|
}
|
|
116
116
|
async resumeProviderTransfer(input) {
|
|
117
117
|
validateId(input.transferId, "transferId");
|
|
118
|
-
|
|
119
|
-
const requestedSignedUrlFlow = input.signedUrlFlow ?? "signed_url";
|
|
120
|
-
const retainedRecoveryInput = {
|
|
121
|
-
...input,
|
|
122
|
-
sources: input.sources.map((source) => ({ ...source })),
|
|
123
|
-
destinations: input.destinations.map((destination) => ({ ...destination }))
|
|
124
|
-
};
|
|
125
|
-
const preparedDestinations = retainedRecoveryInput.destinations.map((destination, index) => prepareProviderDestination(destination, { index }));
|
|
126
|
-
const destinationsById = new Map();
|
|
127
|
-
preparedDestinations.forEach((preparedDestination, index) => {
|
|
128
|
-
const destination = retainedRecoveryInput.destinations[index];
|
|
129
|
-
if (!destination) {
|
|
130
|
-
throw new Error(`missing provider destination config for prepared destination ${preparedDestination.destination_id}`);
|
|
131
|
-
}
|
|
132
|
-
destinationsById.set(preparedDestination.destination_id, destination);
|
|
133
|
-
});
|
|
134
|
-
const preparedSources = await Promise.all(retainedRecoveryInput.sources.map((source, index) => prepareProviderSource(source, { index, expiresIn, fetchImpl: this.fetchImpl })));
|
|
135
|
-
const resumePrepareRequestKey = `transfer:${input.transferId}:prepare:resume:${randomUuid()}`;
|
|
136
|
-
const prepared = await this.prepareTransferWithRequestKey({
|
|
137
|
-
transferId: input.transferId,
|
|
138
|
-
sources: preparedSources,
|
|
139
|
-
destinations: preparedDestinations,
|
|
140
|
-
name: input.name,
|
|
141
|
-
testMode: input.testMode,
|
|
142
|
-
signedUrlFlow: requestedSignedUrlFlow
|
|
143
|
-
}, resumePrepareRequestKey);
|
|
144
|
-
if (!prepared.success)
|
|
145
|
-
return prepared;
|
|
146
|
-
if (prepared.transfer_id !== input.transferId) {
|
|
147
|
-
throw new Error("resumed provider transfer id mismatch");
|
|
148
|
-
}
|
|
149
|
-
const sourcesById = new Map();
|
|
150
|
-
preparedSources.forEach((preparedSource, index) => {
|
|
151
|
-
const source = retainedRecoveryInput.sources[index];
|
|
152
|
-
if (!source) {
|
|
153
|
-
throw new Error(`missing provider source config for prepared source ${preparedSource.source_id}`);
|
|
154
|
-
}
|
|
155
|
-
sourcesById.set(preparedSource.source_id, source);
|
|
156
|
-
});
|
|
157
|
-
await this.startProviderRouteRecoverySigner({
|
|
158
|
-
prepared,
|
|
159
|
-
sourcesById,
|
|
160
|
-
destinationsById,
|
|
161
|
-
multipartUploads: new Map(),
|
|
162
|
-
expiresIn
|
|
163
|
-
});
|
|
164
|
-
this.control.registerRecoveryLease({
|
|
165
|
-
transferId: prepared.transfer_id,
|
|
166
|
-
planFingerprint: prepared.plan_fingerprint,
|
|
167
|
-
coordinateChecksum: prepared.coordinate_checksum,
|
|
168
|
-
replayRoutes: async () => {
|
|
169
|
-
throw new Error("route replay is unavailable after provider action resume; route signing remains available");
|
|
170
|
-
},
|
|
171
|
-
disposeSecrets: () => clearProviderResumeSecrets(retainedRecoveryInput)
|
|
172
|
-
});
|
|
173
|
-
return prepared;
|
|
118
|
+
return this.executeProviderTransfer(input, input);
|
|
174
119
|
}
|
|
175
120
|
async transferStatus(transferId) {
|
|
176
121
|
validateId(transferId, "transferId");
|
|
177
122
|
const status = await this.control.request("transfer.status", { transfer_id: transferId }, { transferId });
|
|
178
|
-
if (
|
|
123
|
+
if (status.integrity_audit_challenge) {
|
|
179
124
|
await this.submitIntegrityAuditGrantsIfPresent(status).catch(() => undefined);
|
|
125
|
+
}
|
|
126
|
+
if (["completed", "failed", "cancelled"].includes(status.status)) {
|
|
180
127
|
this.control.releaseRecoveryLease(transferId);
|
|
181
128
|
this.stopRecoverySigner(transferId);
|
|
182
129
|
}
|
|
@@ -189,12 +136,15 @@ export class BeamClient {
|
|
|
189
136
|
idempotencyKey: `transfer:${transferId}:distribute`
|
|
190
137
|
});
|
|
191
138
|
}
|
|
192
|
-
|
|
139
|
+
requestTransferCancellation(transferId) {
|
|
193
140
|
validateId(transferId, "transferId");
|
|
194
|
-
|
|
141
|
+
return this.control.request("transfer.cancel", { transfer_id: transferId }, {
|
|
195
142
|
transferId,
|
|
196
143
|
idempotencyKey: `transfer:${transferId}:cancel`
|
|
197
144
|
});
|
|
145
|
+
}
|
|
146
|
+
async cancelTransfer(transferId) {
|
|
147
|
+
const result = await this.requestTransferCancellation(transferId);
|
|
198
148
|
this.control.releaseRecoveryLease(transferId);
|
|
199
149
|
this.stopRecoverySigner(transferId);
|
|
200
150
|
return result;
|
|
@@ -314,6 +264,10 @@ export class BeamClient {
|
|
|
314
264
|
}
|
|
315
265
|
}
|
|
316
266
|
async prepareProviderTransfer(input) {
|
|
267
|
+
return this.executeProviderTransfer(input);
|
|
268
|
+
}
|
|
269
|
+
async executeProviderTransfer(input, resume) {
|
|
270
|
+
input.signal?.throwIfAborted();
|
|
317
271
|
const expiresIn = input.expiresIn ?? 3600;
|
|
318
272
|
const requestedSignedUrlFlow = input.signedUrlFlow ?? "signed_url";
|
|
319
273
|
const retainedRecoveryInput = {
|
|
@@ -335,7 +289,7 @@ export class BeamClient {
|
|
|
335
289
|
destinationsById.set(preparedDestination.destination_id, destination);
|
|
336
290
|
});
|
|
337
291
|
await input.throwIfCancelled?.();
|
|
338
|
-
const preparedSources = await Promise.all(retainedRecoveryInput.sources.map((source, index) => prepareProviderSource(source, { index, expiresIn, fetchImpl: this.fetchImpl })));
|
|
292
|
+
const preparedSources = await Promise.all(retainedRecoveryInput.sources.map((source, index) => prepareProviderSource(source, { index, expiresIn, fetchImpl: this.fetchImpl, signal: input.signal })));
|
|
339
293
|
await input.throwIfCancelled?.();
|
|
340
294
|
const huggingFace = await this.planHuggingFaceUploads({
|
|
341
295
|
sources: retainedRecoveryInput.sources,
|
|
@@ -345,16 +299,22 @@ export class BeamClient {
|
|
|
345
299
|
});
|
|
346
300
|
await input.throwIfCancelled?.();
|
|
347
301
|
await input.onBeforeTransferPrepare?.();
|
|
348
|
-
|
|
302
|
+
input.signal?.throwIfAborted();
|
|
303
|
+
const prepareInput = {
|
|
349
304
|
sources: preparedSources,
|
|
350
305
|
destinations: preparedDestinations,
|
|
351
306
|
name: input.name,
|
|
352
307
|
testMode: input.testMode,
|
|
353
308
|
chunkSize: huggingFace.chunkSize ?? input.chunkSize,
|
|
354
309
|
signedUrlFlow: requestedSignedUrlFlow,
|
|
355
|
-
idempotencyKey: input.idempotencyKey,
|
|
356
|
-
|
|
357
|
-
|
|
310
|
+
...(resume ? { transferId: resume.transferId } : { idempotencyKey: input.idempotencyKey, routeGenerationId: input.routeGenerationId })
|
|
311
|
+
};
|
|
312
|
+
const prepared = resume
|
|
313
|
+
? await this.prepareTransferWithRequestKey(prepareInput, `transfer:${resume.transferId}:prepare:resume:${randomUuid()}`)
|
|
314
|
+
: await this.prepareTransfer(prepareInput);
|
|
315
|
+
if (resume && prepared.transfer_id !== resume.transferId)
|
|
316
|
+
throw new Error("resumed provider transfer id mismatch");
|
|
317
|
+
input.signal?.throwIfAborted();
|
|
358
318
|
if (!prepared.success) {
|
|
359
319
|
return prepared;
|
|
360
320
|
}
|
|
@@ -372,9 +332,12 @@ export class BeamClient {
|
|
|
372
332
|
sourcesById.set(preparedSource.source_id, source);
|
|
373
333
|
});
|
|
374
334
|
const multipartUploads = new Map();
|
|
335
|
+
if (resume)
|
|
336
|
+
restoreProviderMultipartIdentities(prepared, destinationsById, resume.multipartGroups, multipartUploads);
|
|
375
337
|
const recoveryMultipartUploads = new Map();
|
|
376
338
|
const autoDistribute = input.distribute !== false;
|
|
377
339
|
let initialThrowIfCancelled = input.throwIfCancelled;
|
|
340
|
+
const assertOwnership = () => input.signal?.throwIfAborted();
|
|
378
341
|
const routeStreamLock = new AsyncMutex();
|
|
379
342
|
const releaseInitialStream = await routeStreamLock.acquire();
|
|
380
343
|
const streamPreparedRoutes = async (routeGenerationId, recoveryReplay) => {
|
|
@@ -387,9 +350,12 @@ export class BeamClient {
|
|
|
387
350
|
let multipartManifestTask = null;
|
|
388
351
|
let multipartGroupWaiters = new Map();
|
|
389
352
|
let foregroundCancelled = false;
|
|
390
|
-
|
|
353
|
+
assertOwnership();
|
|
354
|
+
const throwIfCancelled = async (transferId) => {
|
|
391
355
|
try {
|
|
392
|
-
|
|
356
|
+
assertOwnership();
|
|
357
|
+
if (!recoveryReplay)
|
|
358
|
+
await initialThrowIfCancelled?.(transferId);
|
|
393
359
|
}
|
|
394
360
|
catch (error) {
|
|
395
361
|
foregroundCancelled = true;
|
|
@@ -424,7 +390,9 @@ export class BeamClient {
|
|
|
424
390
|
expiresIn,
|
|
425
391
|
signedUrlFlow: requestedSignedUrlFlow,
|
|
426
392
|
concurrency: this.multipartControlConcurrency,
|
|
393
|
+
signal: input.signal,
|
|
427
394
|
onGroupReady: async (state) => {
|
|
395
|
+
assertOwnership();
|
|
428
396
|
recoveryMultipartUploads.set(state.manifest.multipart_group_id, state);
|
|
429
397
|
validateMultipartGroupManifest([state.manifest], prepared.transfer_id);
|
|
430
398
|
await input.onMultipartGroupReady?.(multipartGroupIdentity(prepared.transfer_id, state.manifest));
|
|
@@ -436,6 +404,7 @@ export class BeamClient {
|
|
|
436
404
|
const streamNextCompletedRoute = async () => {
|
|
437
405
|
const settled = await Promise.race([...pendingRoutes].map((pending) => pending.then((route) => ({ pending, route }))));
|
|
438
406
|
pendingRoutes.delete(settled.pending);
|
|
407
|
+
assertOwnership();
|
|
439
408
|
await routeStream.addRoute(settled.route);
|
|
440
409
|
signedInWindow += 1;
|
|
441
410
|
if (signedInWindow === ROUTE_STREAM_BATCH_ROUTES) {
|
|
@@ -493,12 +462,21 @@ export class BeamClient {
|
|
|
493
462
|
await streamNextCompletedRoute();
|
|
494
463
|
const multipartGroupManifest = await multipartManifestTask;
|
|
495
464
|
validateMultipartGroupManifest(multipartGroupManifest, prepared.transfer_id);
|
|
465
|
+
assertOwnership();
|
|
496
466
|
const attached = await routeStream.complete();
|
|
497
467
|
if (!attached.success) {
|
|
498
468
|
throw new Error(attached.error ?? attached.message ?? "route stream failed");
|
|
499
469
|
}
|
|
500
470
|
}
|
|
501
471
|
catch (error) {
|
|
472
|
+
if (input.signal?.aborted) {
|
|
473
|
+
this.stopRecoverySigner(prepared.transfer_id);
|
|
474
|
+
this.control.releaseRecoveryLease(prepared.transfer_id);
|
|
475
|
+
await Promise.allSettled(pendingRoutes);
|
|
476
|
+
if (multipartManifestTask)
|
|
477
|
+
await Promise.allSettled([multipartManifestTask]);
|
|
478
|
+
throw input.signal.reason;
|
|
479
|
+
}
|
|
502
480
|
if (routeStream)
|
|
503
481
|
await routeStream.abort().catch(() => undefined);
|
|
504
482
|
await Promise.allSettled(pendingRoutes);
|
|
@@ -513,15 +491,22 @@ export class BeamClient {
|
|
|
513
491
|
throw new BeamRouteRecoveryPendingError(prepared.transfer_id, error);
|
|
514
492
|
}
|
|
515
493
|
else {
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
494
|
+
try {
|
|
495
|
+
await cancelAndAbortProviderFailure({
|
|
496
|
+
cancelTransfer: () => this.requestTransferCancellation(prepared.transfer_id),
|
|
497
|
+
transferId: prepared.transfer_id,
|
|
498
|
+
cause: error,
|
|
499
|
+
uploads: multipartUploads,
|
|
500
|
+
abortBeforeCancel: !routeStreamBeginAttempted,
|
|
501
|
+
cleanupConcurrency: this.multipartControlConcurrency
|
|
502
|
+
});
|
|
503
|
+
}
|
|
504
|
+
finally {
|
|
505
|
+
// The retained recovery input owns the same destination objects used by
|
|
506
|
+
// multipart cleanup. Releasing it earlier scrubs their scoped credentials
|
|
507
|
+
// before abortMultipartUpload can clean up already-created uploads.
|
|
508
|
+
this.control.releaseRecoveryLease(prepared.transfer_id);
|
|
509
|
+
}
|
|
525
510
|
}
|
|
526
511
|
}
|
|
527
512
|
throw error;
|
|
@@ -532,8 +517,14 @@ export class BeamClient {
|
|
|
532
517
|
sourcesById,
|
|
533
518
|
destinationsById,
|
|
534
519
|
multipartUploads: recoveryMultipartUploads,
|
|
520
|
+
signal: input.signal,
|
|
535
521
|
expiresIn
|
|
536
522
|
});
|
|
523
|
+
const stopOwnedRecovery = () => {
|
|
524
|
+
this.stopRecoverySigner(prepared.transfer_id);
|
|
525
|
+
this.control.releaseRecoveryLease(prepared.transfer_id);
|
|
526
|
+
};
|
|
527
|
+
input.signal?.addEventListener("abort", stopOwnedRecovery, { once: true });
|
|
537
528
|
this.control.registerRecoveryLease({
|
|
538
529
|
transferId: prepared.transfer_id,
|
|
539
530
|
planFingerprint: prepared.plan_fingerprint,
|
|
@@ -541,10 +532,14 @@ export class BeamClient {
|
|
|
541
532
|
replayRoutes: async (routeGenerationId) => {
|
|
542
533
|
await routeStreamLock.run(() => streamPreparedRoutes(routeGenerationId, true));
|
|
543
534
|
},
|
|
544
|
-
disposeSecrets: () =>
|
|
535
|
+
disposeSecrets: () => {
|
|
536
|
+
input.signal?.removeEventListener("abort", stopOwnedRecovery);
|
|
537
|
+
clearRecoverySecrets(retainedRecoveryInput);
|
|
538
|
+
}
|
|
545
539
|
});
|
|
546
540
|
try {
|
|
547
541
|
try {
|
|
542
|
+
assertOwnership();
|
|
548
543
|
await input.onPrepared?.(prepared);
|
|
549
544
|
}
|
|
550
545
|
catch (error) {
|
|
@@ -788,9 +783,11 @@ export class BeamClient {
|
|
|
788
783
|
async startProviderRouteRecoverySigner(input) {
|
|
789
784
|
this.stopRecoverySigner(input.prepared.transfer_id);
|
|
790
785
|
const stop = await this.control.serveRouteRecoverySigner(input.prepared.transfer_id, async (payload) => {
|
|
786
|
+
input.signal?.throwIfAborted();
|
|
791
787
|
if (payload.route_generation_id.length === 0)
|
|
792
788
|
throw new Error("route recovery generation is required");
|
|
793
789
|
const routes = await mapOrderedWithConcurrency(payload.chunks, this.routeSigningConcurrency, async (requested) => {
|
|
790
|
+
input.signal?.throwIfAborted();
|
|
794
791
|
if (requested.route_generation_id !== payload.route_generation_id) {
|
|
795
792
|
throw new Error("route recovery chunk generation mismatch");
|
|
796
793
|
}
|
|
@@ -940,7 +937,13 @@ export class BeamClient {
|
|
|
940
937
|
}
|
|
941
938
|
const submission = signer(challenge);
|
|
942
939
|
this.integrityAuditSubmissions.set(challenge.audit_id, submission);
|
|
943
|
-
|
|
940
|
+
try {
|
|
941
|
+
await submission;
|
|
942
|
+
}
|
|
943
|
+
catch (error) {
|
|
944
|
+
this.integrityAuditSubmissions.delete(challenge.audit_id);
|
|
945
|
+
throw error;
|
|
946
|
+
}
|
|
944
947
|
}
|
|
945
948
|
async submitProviderIntegrityAuditGrants(input) {
|
|
946
949
|
if (input.challenge.transfer_id !== input.prepared.transfer_id) {
|
|
@@ -963,12 +966,17 @@ export class BeamClient {
|
|
|
963
966
|
}
|
|
964
967
|
const source = input.sourcesById.get(chunk.source_id);
|
|
965
968
|
const destination = input.destinationsById.get(chunk.destination_id);
|
|
969
|
+
const plannedSource = input.prepared.plan_descriptor.sources.find((candidate) => candidate.source_id === chunk.source_id);
|
|
966
970
|
if (!source || !destination) {
|
|
967
971
|
throw new Error("integrity audit provider configuration is unavailable");
|
|
968
972
|
}
|
|
973
|
+
const sourceEtag = plannedSource?.metadata?.etag;
|
|
974
|
+
const sourceVersionId = plannedSource?.metadata?.version_id;
|
|
969
975
|
const [sourceGrant, destinationGrant] = await Promise.all([
|
|
970
976
|
signSourceReadRange({
|
|
971
977
|
source,
|
|
978
|
+
...(typeof sourceEtag === "string" ? { ifMatch: sourceEtag } : {}),
|
|
979
|
+
...(typeof sourceVersionId === "string" ? { versionId: sourceVersionId } : {}),
|
|
972
980
|
offset: chunk.source_offset,
|
|
973
981
|
length: chunk.range_length,
|
|
974
982
|
expiresIn: input.expiresIn,
|
|
@@ -977,6 +985,7 @@ export class BeamClient {
|
|
|
977
985
|
signDestinationReadRange({
|
|
978
986
|
destination,
|
|
979
987
|
objectKey: chunk.final_object_key,
|
|
988
|
+
...(chunk.final_object_etag ? { ifMatch: chunk.final_object_etag } : {}),
|
|
980
989
|
offset: chunk.destination_offset,
|
|
981
990
|
length: chunk.range_length,
|
|
982
991
|
expiresIn: input.expiresIn,
|
|
@@ -1408,9 +1417,9 @@ async function sha256Bytes(value) {
|
|
|
1408
1417
|
function hex(bytes) {
|
|
1409
1418
|
return [...bytes].map((byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
1410
1419
|
}
|
|
1411
|
-
async function cancelTransferAfterProviderFailure(
|
|
1420
|
+
async function cancelTransferAfterProviderFailure(cancelTransfer, transferId, cause) {
|
|
1412
1421
|
try {
|
|
1413
|
-
const result = await
|
|
1422
|
+
const result = await cancelTransfer();
|
|
1414
1423
|
if (!result.success) {
|
|
1415
1424
|
throw new Error(result.message ?? `Beam rejected cancellation for ${transferId}`);
|
|
1416
1425
|
}
|
|
@@ -1433,7 +1442,7 @@ async function cancelAndAbortProviderFailure(input) {
|
|
|
1433
1442
|
}
|
|
1434
1443
|
}
|
|
1435
1444
|
try {
|
|
1436
|
-
await cancelTransferAfterProviderFailure(input.
|
|
1445
|
+
await cancelTransferAfterProviderFailure(input.cancelTransfer, input.transferId, input.cause);
|
|
1437
1446
|
}
|
|
1438
1447
|
catch (error) {
|
|
1439
1448
|
cancelError = error;
|
|
@@ -1505,6 +1514,37 @@ function createMultipartGroupWaiters(prepared, destinationsById) {
|
|
|
1505
1514
|
}
|
|
1506
1515
|
return waiters;
|
|
1507
1516
|
}
|
|
1517
|
+
function restoreProviderMultipartIdentities(prepared, destinations, identities, uploads) {
|
|
1518
|
+
if (!Array.isArray(identities))
|
|
1519
|
+
throw new Error("provider_multipart_recovery_identity_required");
|
|
1520
|
+
const expected = new Map();
|
|
1521
|
+
for (const source of prepared.plan_descriptor.sources) {
|
|
1522
|
+
for (const target of prepared.plan_descriptor.destinations) {
|
|
1523
|
+
const destination = destinations.get(target.destination_id);
|
|
1524
|
+
if (!destination)
|
|
1525
|
+
throw new Error("provider_multipart_recovery_destination_invalid");
|
|
1526
|
+
if (isDirectPutDestination(destination))
|
|
1527
|
+
continue;
|
|
1528
|
+
const objectKey = target.final_object_keys[source.source_id];
|
|
1529
|
+
if (!objectKey)
|
|
1530
|
+
throw new Error("provider_multipart_recovery_object_invalid");
|
|
1531
|
+
const key = multipartGroupStateKey(prepared.transfer_id, target.destination_id, source.source_id, objectKey);
|
|
1532
|
+
expected.set(key, { destination, objectKey, source, destinationId: target.destination_id });
|
|
1533
|
+
}
|
|
1534
|
+
}
|
|
1535
|
+
if (expected.size !== identities.length)
|
|
1536
|
+
throw new Error("provider_multipart_recovery_incomplete");
|
|
1537
|
+
for (const identity of identities) {
|
|
1538
|
+
const group = expected.get(identity.multipartGroupId);
|
|
1539
|
+
if (!group || uploads.has(identity.multipartGroupId) || identity.transferId !== prepared.transfer_id ||
|
|
1540
|
+
identity.sourceId !== group.source.source_id || identity.destinationId !== group.destinationId ||
|
|
1541
|
+
identity.objectKey !== group.objectKey || identity.expectedObjectSize !== group.source.size ||
|
|
1542
|
+
identity.expectedPartCount !== group.source.chunk_count || !identity.uploadId?.trim()) {
|
|
1543
|
+
throw new Error("provider_multipart_recovery_identity_invalid");
|
|
1544
|
+
}
|
|
1545
|
+
uploads.set(identity.multipartGroupId, { destination: group.destination, objectKey: group.objectKey, uploadId: identity.uploadId });
|
|
1546
|
+
}
|
|
1547
|
+
}
|
|
1508
1548
|
async function createMultipartGroupManifest(input) {
|
|
1509
1549
|
const groups = input.prepared.plan_descriptor.sources.flatMap((source) => input.prepared.plan_descriptor.destinations.map((destination) => ({ source, destination })));
|
|
1510
1550
|
const results = await mapOrderedWithConcurrency(groups, input.concurrency, async ({ source, destination }) => {
|
|
@@ -1523,11 +1563,13 @@ async function createMultipartGroupManifest(input) {
|
|
|
1523
1563
|
}
|
|
1524
1564
|
multipartGroupId = multipartGroupStateKey(input.prepared.transfer_id, destination.destination_id, source.source_id, finalObjectKey);
|
|
1525
1565
|
const finalObjectMetadata = { "beam-transfer-id": input.prepared.transfer_id };
|
|
1566
|
+
input.signal?.throwIfAborted();
|
|
1526
1567
|
const retainedUpload = input.multipartUploads.get(multipartGroupId);
|
|
1527
1568
|
const uploadId = retainedUpload?.uploadId ?? await createMultipartUpload({
|
|
1528
1569
|
destination: destinationConfig,
|
|
1529
1570
|
objectKey: finalObjectKey,
|
|
1530
|
-
metadata: finalObjectMetadata
|
|
1571
|
+
metadata: finalObjectMetadata,
|
|
1572
|
+
signal: input.signal
|
|
1531
1573
|
});
|
|
1532
1574
|
const createdUpload = !retainedUpload;
|
|
1533
1575
|
if (createdUpload) {
|
|
@@ -1574,7 +1616,7 @@ async function createMultipartGroupManifest(input) {
|
|
|
1574
1616
|
return { ok: true, state };
|
|
1575
1617
|
}
|
|
1576
1618
|
catch (error) {
|
|
1577
|
-
if (createdUpload) {
|
|
1619
|
+
if (createdUpload && !input.signal?.aborted) {
|
|
1578
1620
|
try {
|
|
1579
1621
|
await abortMultipartUpload(destinationConfig, finalObjectKey, uploadId);
|
|
1580
1622
|
input.multipartUploads.delete(multipartGroupId);
|
|
@@ -1858,13 +1900,3 @@ function clearRecoverySecrets(input) {
|
|
|
1858
1900
|
}
|
|
1859
1901
|
}
|
|
1860
1902
|
}
|
|
1861
|
-
function clearProviderResumeSecrets(input) {
|
|
1862
|
-
const secretKeys = new Set(["access_key_id", "secret_access_key", "session_token", "api_token", "token", "password"]);
|
|
1863
|
-
for (const provider of [...input.sources, ...input.destinations]) {
|
|
1864
|
-
const record = provider;
|
|
1865
|
-
for (const key of secretKeys) {
|
|
1866
|
-
if (key in record)
|
|
1867
|
-
record[key] = "";
|
|
1868
|
-
}
|
|
1869
|
-
}
|
|
1870
|
-
}
|
package/dist/models.d.ts
CHANGED
|
@@ -132,6 +132,7 @@ export interface IntegrityAuditChallengeChunk {
|
|
|
132
132
|
destination_offset: number;
|
|
133
133
|
range_length: number;
|
|
134
134
|
final_object_key: string;
|
|
135
|
+
final_object_etag: string | null;
|
|
135
136
|
}
|
|
136
137
|
export interface IntegrityAuditChallenge {
|
|
137
138
|
audit_id: string;
|
|
@@ -171,19 +172,10 @@ export interface TransferStatusInfo {
|
|
|
171
172
|
};
|
|
172
173
|
integrity_audit_challenge?: IntegrityAuditChallenge;
|
|
173
174
|
}
|
|
174
|
-
export interface ProviderTransferResumeInput {
|
|
175
|
+
export interface ProviderTransferResumeInput extends Omit<ProviderTransferCreateInput, "idempotencyKey" | "routeGenerationId"> {
|
|
175
176
|
transferId: string;
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
name?: string;
|
|
179
|
-
/**
|
|
180
|
-
* Maps to BeamCore test_mode. Must match the original prepared transfer.
|
|
181
|
-
*/
|
|
182
|
-
testMode?: boolean;
|
|
183
|
-
expiresIn?: number;
|
|
184
|
-
signedUrlFlow?: SignedUrlFlow;
|
|
185
|
-
/** Requested plan chunk size. BeamCore may raise it; the response carries the effective value. */
|
|
186
|
-
chunkSize?: number;
|
|
177
|
+
/** Complete durable multipart identities from onMultipartGroupReady; resume never creates replacement uploads. */
|
|
178
|
+
multipartGroups: ProviderMultipartGroupIdentity[];
|
|
187
179
|
}
|
|
188
180
|
export interface TransferTerminalEvent {
|
|
189
181
|
schema_version: "transfer-client-control/v6";
|
|
@@ -290,6 +282,8 @@ export declare const HuggingFaceProviderConfig: Readonly<{
|
|
|
290
282
|
create(input: HuggingFaceProviderConfigInput): HuggingFaceProviderConfig;
|
|
291
283
|
}>;
|
|
292
284
|
export interface ProviderTransferCreateInput {
|
|
285
|
+
/** Ownership fence. Aborting stops signing/replay without cancelling a replacement owner. */
|
|
286
|
+
signal?: AbortSignal;
|
|
293
287
|
sources: ProviderSourceConfig[];
|
|
294
288
|
destinations: ProviderDestinationConfig[];
|
|
295
289
|
name?: string;
|
|
@@ -6,6 +6,7 @@ export declare function prepareProviderSource(source: ProviderSourceConfig, opti
|
|
|
6
6
|
index?: number;
|
|
7
7
|
expiresIn?: number;
|
|
8
8
|
fetchImpl?: typeof fetch;
|
|
9
|
+
signal?: AbortSignal;
|
|
9
10
|
}): Promise<PreparedHttpSource>;
|
|
10
11
|
export declare function prepareProviderSourceForPlan(source: ProviderSourceConfig, options?: {
|
|
11
12
|
index?: number;
|
|
@@ -90,6 +91,7 @@ export declare function signSourceReadRange(input: {
|
|
|
90
91
|
export declare function signDestinationReadRange(input: {
|
|
91
92
|
destination: ProviderDestinationConfig;
|
|
92
93
|
objectKey: string;
|
|
94
|
+
ifMatch?: string;
|
|
93
95
|
offset: number;
|
|
94
96
|
length: number;
|
|
95
97
|
expiresIn: number;
|
package/dist/provider-signing.js
CHANGED
|
@@ -10,10 +10,15 @@ export function expiresAtIso(expiresIn) {
|
|
|
10
10
|
export async function prepareProviderSource(source, options = {}) {
|
|
11
11
|
const index = options.index ?? 0;
|
|
12
12
|
const expiresIn = options.expiresIn ?? 3600;
|
|
13
|
+
options.signal?.throwIfAborted();
|
|
14
|
+
const baseFetch = options.fetchImpl ?? globalThis.fetch;
|
|
15
|
+
const fetchImpl = options.signal
|
|
16
|
+
? (input, init) => baseFetch(input, { ...init, signal: AbortSignal.any([options.signal, ...(init?.signal ? [init.signal] : [])]) })
|
|
17
|
+
: baseFetch;
|
|
13
18
|
if (isS3CompatibleProvider(source)) {
|
|
14
19
|
const endpoint = s3CompatibleEndpoint(source);
|
|
15
20
|
const client = createS3CompatibleClient(source, endpoint);
|
|
16
|
-
const head = await client.send(new HeadObjectCommand({ Bucket: source.bucket, Key: source.key }));
|
|
21
|
+
const head = await client.send(new HeadObjectCommand({ Bucket: source.bucket, Key: source.key }), { abortSignal: options.signal });
|
|
17
22
|
const getUrl = await getSignedUrl(client, new GetObjectCommand({ Bucket: source.bucket, Key: source.key }), { expiresIn });
|
|
18
23
|
return {
|
|
19
24
|
source_id: source.id ?? `src_${index}`,
|
|
@@ -33,7 +38,6 @@ export async function prepareProviderSource(source, options = {}) {
|
|
|
33
38
|
};
|
|
34
39
|
}
|
|
35
40
|
if (isHippiusProvider(source)) {
|
|
36
|
-
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
|
37
41
|
const baseUrl = source.base_url ?? "https://api.hippius.com";
|
|
38
42
|
const size = await hippiusObjectSize(fetchImpl, baseUrl, source.api_token, source.bucket, source.key);
|
|
39
43
|
const url = await hippiusPresign(fetchImpl, baseUrl, source.api_token, source.bucket, source.key, "get", expiresIn);
|
|
@@ -49,7 +53,7 @@ export async function prepareProviderSource(source, options = {}) {
|
|
|
49
53
|
};
|
|
50
54
|
}
|
|
51
55
|
if (isHuggingFaceProvider(source)) {
|
|
52
|
-
const metadata = await huggingFaceFileMetadata(
|
|
56
|
+
const metadata = await huggingFaceFileMetadata(fetchImpl, source);
|
|
53
57
|
return {
|
|
54
58
|
source_id: source.id ?? `src_${index}`,
|
|
55
59
|
type: "http",
|
|
@@ -333,11 +337,14 @@ export async function signDestinationReadRange(input) {
|
|
|
333
337
|
url: await getSignedUrl(client, new GetObjectCommand({
|
|
334
338
|
Bucket: input.destination.bucket,
|
|
335
339
|
Key: input.objectKey,
|
|
336
|
-
Range: range
|
|
340
|
+
Range: range,
|
|
341
|
+
IfMatch: input.ifMatch
|
|
337
342
|
}), { expiresIn: input.expiresIn }),
|
|
338
|
-
headers: { Range: range }
|
|
343
|
+
headers: { Range: range, ...(input.ifMatch ? { "If-Match": input.ifMatch } : {}) }
|
|
339
344
|
};
|
|
340
345
|
}
|
|
346
|
+
if (input.ifMatch)
|
|
347
|
+
throw new Error("conditional destination ranges require S3-compatible storage");
|
|
341
348
|
if (isHippiusProvider(input.destination)) {
|
|
342
349
|
return {
|
|
343
350
|
url: await hippiusPresign(input.fetchImpl ?? globalThis.fetch, input.destination.base_url ?? "https://api.hippius.com", input.destination.api_token, input.destination.bucket, input.objectKey, "get", input.expiresIn),
|