@beam-network/sdk 0.5.4 → 0.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/client.d.ts +4 -14
- package/dist/client.js +148 -168
- package/dist/models.d.ts +4 -6
- package/dist/multipart-limits.d.ts +3 -2
- package/dist/multipart-limits.js +10 -5
- package/dist/nats-control.d.ts +40 -3
- package/dist/nats-control.js +80 -9
- package/dist/provider-signing.d.ts +0 -14
- package/dist/provider-signing.js +2 -60
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -96,13 +96,13 @@ For low-level raw transfer configs, use `createRawTransfer`; lifecycle transport
|
|
|
96
96
|
|
|
97
97
|
## Route Streaming And Payload Size
|
|
98
98
|
|
|
99
|
-
Provider transfers use `transfer-client-control/
|
|
99
|
+
Provider transfers use `transfer-client-control/v6`. Every reply carries Runtime and transport epochs; prepare and every route-stream message carry a UUID route generation. S3, R2, and S3-compatible destinations retain direct multipart UploadPart/ListParts/HEAD handling.
|
|
100
100
|
|
|
101
|
-
The client signs up to 64 routes concurrently by default and emits 2,048-route logical batches
|
|
101
|
+
The client signs up to 64 routes concurrently by default and emits 2,048-route logical batches. Encoded MessagePack requests target 4 MiB physical batches under `maxPayloadBytes` (8 MiB by default; explicit positive overrides remain supported). The splitter reserves control-envelope headroom for the live auth token and stable request identity so the final encoded request remains below the configured guard. A single route above the 4 MiB target but within the configured guard publishes alone; a single route larger than the configured limit fails before publication with both sizes in the error. Multipart create and abort requests use a separate `multipartControlConcurrency` limit of 2 by default, with explicit positive overrides supported, and S3-compatible control requests make up to five attempts for transient provider failures. This keeps URL signing throughput independent from provider control-plane pressure.
|
|
102
102
|
|
|
103
103
|
The stream ID derives from the transfer, selected flow, and immutable plan identity, and each ordered batch ID includes its route-coordinate checksum. Manual multi-destination attachment requires `delivery_index` on every route. Lifecycle mutations retry transient failures up to three times with the same request identity. A non-recoverable provider setup or route failure raises `BeamProviderTransferError`, whose `errors`, `transferCancelled`, and `multipartCleanupComplete` fields retain the original provider, cancellation, and cleanup outcomes. `waitForTransfer` subscribes to the API-key-owned, at-most-once terminal signal before its first status read and reconciles every signal through authoritative status; subscription failure degrades to the same jittered 15-to-30-second status fallback. Call `close()` when the client is no longer needed; it also closes outstanding terminal waiters.
|
|
104
104
|
|
|
105
|
-
Hippius uses
|
|
105
|
+
Hippius uses canonical non-multipart `signed_url` routes, so its manifest is empty and group-level final HEAD verification is not available from that provider flow.
|
|
106
106
|
|
|
107
107
|
NATS requires this guard because the broker rejects messages above its configured `max_payload`. This limit applies only to lifecycle/control metadata; transfer file bytes do not flow through NATS.
|
|
108
108
|
|
package/dist/client.d.ts
CHANGED
|
@@ -32,6 +32,7 @@ export declare class BeamClient {
|
|
|
32
32
|
private readonly routeSigningConcurrency;
|
|
33
33
|
private readonly routeSigningConcurrencyOverridden;
|
|
34
34
|
private readonly multipartControlConcurrency;
|
|
35
|
+
private readonly recoverySigners;
|
|
35
36
|
constructor(options?: BeamClientOptions);
|
|
36
37
|
close(): Promise<void>;
|
|
37
38
|
openTransferTerminalWaiter(transferId: string): Promise<TransferTerminalSignalWaiter>;
|
|
@@ -75,6 +76,9 @@ export declare class BeamClient {
|
|
|
75
76
|
}): Promise<AttachSignedUrlsResponse>;
|
|
76
77
|
prepareProviderTransfer(input: ProviderTransferCreateInput): Promise<TransferPrepareResponse>;
|
|
77
78
|
private signProviderRoute;
|
|
79
|
+
private startProviderRouteRecoverySigner;
|
|
80
|
+
private stopRecoverySigner;
|
|
81
|
+
private stopAllRecoverySigners;
|
|
78
82
|
createAndDistribute(input: RawTransferCreateInput): Promise<TransferCreateResponse>;
|
|
79
83
|
waitForTransfer(transferId: string, options?: {
|
|
80
84
|
timeoutMs?: number;
|
|
@@ -82,17 +86,3 @@ export declare class BeamClient {
|
|
|
82
86
|
maxPollIntervalMs?: number;
|
|
83
87
|
}): Promise<TransferStatusInfo>;
|
|
84
88
|
}
|
|
85
|
-
export declare function buildStagingObjectPrefix(input: {
|
|
86
|
-
finalObjectKey: string;
|
|
87
|
-
transferId: string;
|
|
88
|
-
sourceId: string;
|
|
89
|
-
destinationId: string;
|
|
90
|
-
}): string;
|
|
91
|
-
export declare function buildStagingObjectKey(input: {
|
|
92
|
-
finalObjectKey: string;
|
|
93
|
-
transferId: string;
|
|
94
|
-
sourceId: string;
|
|
95
|
-
destinationId: string;
|
|
96
|
-
chunkIndex: number;
|
|
97
|
-
partNumber: number;
|
|
98
|
-
}): string;
|
package/dist/client.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { abortMultipartUpload, createMultipartUpload, expiresAtIso, prepareProviderDestination, prepareProviderSource, signAbortMultipartUpload, signCompleteMultipartUpload,
|
|
1
|
+
import { abortMultipartUpload, createMultipartUpload, expiresAtIso, prepareProviderDestination, prepareProviderSource, signAbortMultipartUpload, signCompleteMultipartUpload, signDestinationRoute, signFinalObjectHead, signListMultipartUpload } from "./provider-signing.js";
|
|
2
2
|
import { BeamTransferControl, BEAM_DEFAULT_NATS_URL, compactSignedRoutes, isRecoverableRouteStreamError } from "./nats-control.js";
|
|
3
3
|
export class BeamRouteRecoveryPendingError extends Error {
|
|
4
4
|
transferId;
|
|
@@ -52,6 +52,7 @@ export class BeamClient {
|
|
|
52
52
|
routeSigningConcurrency;
|
|
53
53
|
routeSigningConcurrencyOverridden;
|
|
54
54
|
multipartControlConcurrency;
|
|
55
|
+
recoverySigners = new Map();
|
|
55
56
|
constructor(options = {}) {
|
|
56
57
|
if (!options.apiKey?.trim()) {
|
|
57
58
|
throw new Error("apiKey is required.");
|
|
@@ -77,6 +78,7 @@ export class BeamClient {
|
|
|
77
78
|
}
|
|
78
79
|
}
|
|
79
80
|
async close() {
|
|
81
|
+
this.stopAllRecoverySigners();
|
|
80
82
|
return this.control.close();
|
|
81
83
|
}
|
|
82
84
|
openTransferTerminalWaiter(transferId) {
|
|
@@ -97,7 +99,7 @@ export class BeamClient {
|
|
|
97
99
|
callbacks: input.callbacks,
|
|
98
100
|
test_mode: input.testMode || undefined,
|
|
99
101
|
progressive_mode: input.progressiveMode || undefined,
|
|
100
|
-
signed_url_flow: input.signedUrlFlow ?? "
|
|
102
|
+
signed_url_flow: input.signedUrlFlow ?? "signed_url"
|
|
101
103
|
});
|
|
102
104
|
return this.control.request("transfer.create", body, {
|
|
103
105
|
transferId,
|
|
@@ -110,8 +112,10 @@ export class BeamClient {
|
|
|
110
112
|
async transferStatus(transferId) {
|
|
111
113
|
validateId(transferId, "transferId");
|
|
112
114
|
const status = await this.control.request("transfer.status", { transfer_id: transferId }, { transferId });
|
|
113
|
-
if (["completed", "failed", "cancelled"].includes(status.status))
|
|
115
|
+
if (["completed", "failed", "cancelled"].includes(status.status)) {
|
|
114
116
|
this.control.releaseRecoveryLease(transferId);
|
|
117
|
+
this.stopRecoverySigner(transferId);
|
|
118
|
+
}
|
|
115
119
|
return status;
|
|
116
120
|
}
|
|
117
121
|
distributeTransfer(transferId) {
|
|
@@ -128,6 +132,7 @@ export class BeamClient {
|
|
|
128
132
|
idempotencyKey: `transfer:${transferId}:cancel`
|
|
129
133
|
});
|
|
130
134
|
this.control.releaseRecoveryLease(transferId);
|
|
135
|
+
this.stopRecoverySigner(transferId);
|
|
131
136
|
return result;
|
|
132
137
|
}
|
|
133
138
|
async planTransfer(input) {
|
|
@@ -137,7 +142,7 @@ export class BeamClient {
|
|
|
137
142
|
name: input.name,
|
|
138
143
|
test_mode: input.testMode || undefined,
|
|
139
144
|
urls_expires_at: input.urlsExpiresAt,
|
|
140
|
-
signed_url_flow: input.signedUrlFlow ?? "
|
|
145
|
+
signed_url_flow: input.signedUrlFlow ?? "signed_url"
|
|
141
146
|
}));
|
|
142
147
|
if (result.success)
|
|
143
148
|
validateCompactTransferPlan(result.plan_descriptor, result.signed_url_flow);
|
|
@@ -145,7 +150,8 @@ export class BeamClient {
|
|
|
145
150
|
}
|
|
146
151
|
async prepareTransfer(input) {
|
|
147
152
|
const transferId = await transferIdForIdempotencyKey(input.idempotencyKey);
|
|
148
|
-
const
|
|
153
|
+
const prepareIdempotencyKey = `transfer:${transferId}:prepare`;
|
|
154
|
+
const routeGenerationId = input.routeGenerationId ?? await routeGenerationIdForPrepareIdempotencyKey(prepareIdempotencyKey);
|
|
149
155
|
const result = await this.control.request("transfer.prepare", compact({
|
|
150
156
|
transfer_id: transferId,
|
|
151
157
|
route_generation_id: routeGenerationId,
|
|
@@ -154,10 +160,10 @@ export class BeamClient {
|
|
|
154
160
|
name: input.name,
|
|
155
161
|
test_mode: input.testMode || undefined,
|
|
156
162
|
urls_expires_at: input.urlsExpiresAt,
|
|
157
|
-
signed_url_flow: input.signedUrlFlow ?? "
|
|
163
|
+
signed_url_flow: input.signedUrlFlow ?? "signed_url"
|
|
158
164
|
}), {
|
|
159
165
|
transferId,
|
|
160
|
-
idempotencyKey:
|
|
166
|
+
idempotencyKey: prepareIdempotencyKey
|
|
161
167
|
});
|
|
162
168
|
if (result.success)
|
|
163
169
|
validateCompactTransferPlan(result.plan_descriptor, result.signed_url_flow);
|
|
@@ -185,7 +191,7 @@ export class BeamClient {
|
|
|
185
191
|
planIdentity: `${await routeCoordinateChecksum(orderedRoutes)}:${routeGenerationId}`,
|
|
186
192
|
totalRoutes: orderedRoutes.length,
|
|
187
193
|
totalChunks: countDistinctRouteChunks(orderedRoutes),
|
|
188
|
-
signedUrlFlow:
|
|
194
|
+
signedUrlFlow: "signed_url"
|
|
189
195
|
});
|
|
190
196
|
const stream = new RouteStreamSender(this.control, {
|
|
191
197
|
streamId,
|
|
@@ -194,7 +200,7 @@ export class BeamClient {
|
|
|
194
200
|
totalRoutes: orderedRoutes.length,
|
|
195
201
|
totalChunks: countDistinctRouteChunks(orderedRoutes),
|
|
196
202
|
urlsExpiresAt,
|
|
197
|
-
signedUrlFlow:
|
|
203
|
+
signedUrlFlow: "signed_url",
|
|
198
204
|
autoDistribute: input.autoDistribute ?? true
|
|
199
205
|
});
|
|
200
206
|
await stream.begin();
|
|
@@ -239,7 +245,7 @@ export class BeamClient {
|
|
|
239
245
|
}
|
|
240
246
|
async prepareProviderTransfer(input) {
|
|
241
247
|
const expiresIn = input.expiresIn ?? 3600;
|
|
242
|
-
const requestedSignedUrlFlow = input.signedUrlFlow ?? "
|
|
248
|
+
const requestedSignedUrlFlow = input.signedUrlFlow ?? "signed_url";
|
|
243
249
|
const retainedRecoveryInput = {
|
|
244
250
|
...input,
|
|
245
251
|
sources: input.sources.map((source) => ({ ...source })),
|
|
@@ -282,6 +288,7 @@ export class BeamClient {
|
|
|
282
288
|
sourcesById.set(preparedSource.source_id, source);
|
|
283
289
|
});
|
|
284
290
|
const multipartUploads = new Map();
|
|
291
|
+
const recoveryMultipartUploads = new Map();
|
|
285
292
|
const autoDistribute = input.distribute !== false;
|
|
286
293
|
let initialThrowIfCancelled = input.throwIfCancelled;
|
|
287
294
|
const routeStreamLock = new AsyncMutex();
|
|
@@ -334,10 +341,9 @@ export class BeamClient {
|
|
|
334
341
|
signedUrlFlow: requestedSignedUrlFlow,
|
|
335
342
|
concurrency: this.multipartControlConcurrency,
|
|
336
343
|
onGroupReady: async (state) => {
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
}
|
|
344
|
+
recoveryMultipartUploads.set(state.manifest.multipart_group_id, state);
|
|
345
|
+
validateMultipartGroupManifest([state.manifest], prepared.transfer_id);
|
|
346
|
+
await routeStream.addManifestGroups([state.manifest]);
|
|
341
347
|
multipartGroupWaiters.get(state.manifest.multipart_group_id)?.resolve(state);
|
|
342
348
|
},
|
|
343
349
|
onGroupFailed: (groupId, error) => multipartGroupWaiters.get(groupId)?.reject(error)
|
|
@@ -400,9 +406,7 @@ export class BeamClient {
|
|
|
400
406
|
while (pendingRoutes.size)
|
|
401
407
|
await streamNextCompletedRoute();
|
|
402
408
|
const multipartGroupManifest = await multipartManifestTask;
|
|
403
|
-
|
|
404
|
-
validateMultipartGroupManifest(multipartGroupManifest, prepared.transfer_id);
|
|
405
|
-
}
|
|
409
|
+
validateMultipartGroupManifest(multipartGroupManifest, prepared.transfer_id);
|
|
406
410
|
const attached = await routeStream.complete();
|
|
407
411
|
if (!attached.success) {
|
|
408
412
|
throw new Error(attached.error ?? attached.message ?? "route stream failed");
|
|
@@ -437,6 +441,13 @@ export class BeamClient {
|
|
|
437
441
|
throw error;
|
|
438
442
|
}
|
|
439
443
|
};
|
|
444
|
+
await this.startProviderRouteRecoverySigner({
|
|
445
|
+
prepared,
|
|
446
|
+
sourcesById,
|
|
447
|
+
destinationsById,
|
|
448
|
+
multipartUploads: recoveryMultipartUploads,
|
|
449
|
+
expiresIn
|
|
450
|
+
});
|
|
440
451
|
this.control.registerRecoveryLease({
|
|
441
452
|
transferId: prepared.transfer_id,
|
|
442
453
|
planFingerprint: prepared.plan_fingerprint,
|
|
@@ -484,63 +495,113 @@ export class BeamClient {
|
|
|
484
495
|
if (!upload)
|
|
485
496
|
throw new Error(`${input.signedUrlFlow} route is missing its multipart upload`);
|
|
486
497
|
validateMultipartPartNumber(input.partNumber, upload.manifest);
|
|
487
|
-
if (input.signedUrlFlow === "signed_url_v1") {
|
|
488
|
-
return signDestinationRoute({
|
|
489
|
-
chunk: input.chunk,
|
|
490
|
-
target: {
|
|
491
|
-
...input.target,
|
|
492
|
-
object_key: input.finalObjectKey,
|
|
493
|
-
metadata: partRouteMetadata(input.target.metadata)
|
|
494
|
-
},
|
|
495
|
-
source: input.source,
|
|
496
|
-
destination: input.destination,
|
|
497
|
-
expiresIn: input.expiresIn,
|
|
498
|
-
uploadId: upload.uploadId,
|
|
499
|
-
partNumber: input.partNumber,
|
|
500
|
-
finalObjectKey: input.finalObjectKey,
|
|
501
|
-
expectedObjectSize: upload.manifest.expected_object_size,
|
|
502
|
-
expectedPartCount: upload.manifest.expected_part_count,
|
|
503
|
-
maxPartNumber: upload.manifest.max_part_number,
|
|
504
|
-
completeUrl: upload.manifest.complete_url,
|
|
505
|
-
abortUrl: upload.manifest.abort_url,
|
|
506
|
-
listPageUrl: upload.manifest.list_page_urls[multipartListPageIndex(input.partNumber)],
|
|
507
|
-
finalHeadUrl: upload.manifest.final_head_url,
|
|
508
|
-
finalObjectMetadata: upload.manifest.final_object_metadata,
|
|
509
|
-
fetchImpl: this.fetchImpl
|
|
510
|
-
});
|
|
511
|
-
}
|
|
512
|
-
const stagingObjectPrefix = buildStagingObjectPrefix({
|
|
513
|
-
finalObjectKey: input.finalObjectKey,
|
|
514
|
-
transferId: input.transferId,
|
|
515
|
-
sourceId: input.chunk.source_id,
|
|
516
|
-
destinationId: input.target.destination_id
|
|
517
|
-
});
|
|
518
|
-
const stagingObjectKey = [
|
|
519
|
-
stagingObjectPrefix,
|
|
520
|
-
`part-${input.partNumber}-chunk-${input.chunk.chunk_index}`
|
|
521
|
-
].join("/");
|
|
522
|
-
const copySourceRange = `bytes=0-${Math.max(0, input.chunk.chunk_size - 1)}`;
|
|
523
|
-
const [{ copyUrl, copySource }, deleteUrl] = await Promise.all([
|
|
524
|
-
signUploadPartCopy(input.destination, input.finalObjectKey, stagingObjectKey, upload.uploadId, input.partNumber, input.expiresIn, copySourceRange),
|
|
525
|
-
signDeleteObject(input.destination, stagingObjectKey, input.expiresIn)
|
|
526
|
-
]);
|
|
527
498
|
return signDestinationRoute({
|
|
528
499
|
chunk: input.chunk,
|
|
529
|
-
target: {
|
|
500
|
+
target: {
|
|
501
|
+
...input.target,
|
|
502
|
+
object_key: input.finalObjectKey,
|
|
503
|
+
metadata: partRouteMetadata(input.target.metadata)
|
|
504
|
+
},
|
|
530
505
|
source: input.source,
|
|
531
506
|
destination: input.destination,
|
|
532
507
|
expiresIn: input.expiresIn,
|
|
508
|
+
uploadId: upload.uploadId,
|
|
533
509
|
partNumber: input.partNumber,
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
510
|
+
finalObjectKey: input.finalObjectKey,
|
|
511
|
+
expectedObjectSize: upload.manifest.expected_object_size,
|
|
512
|
+
expectedPartCount: upload.manifest.expected_part_count,
|
|
513
|
+
maxPartNumber: upload.manifest.max_part_number,
|
|
514
|
+
completeUrl: upload.manifest.complete_url,
|
|
515
|
+
abortUrl: upload.manifest.abort_url,
|
|
516
|
+
listPageUrl: upload.manifest.list_page_urls[multipartListPageIndex(input.partNumber)],
|
|
517
|
+
finalHeadUrl: upload.manifest.final_head_url,
|
|
518
|
+
finalObjectMetadata: upload.manifest.final_object_metadata,
|
|
541
519
|
fetchImpl: this.fetchImpl
|
|
542
520
|
});
|
|
543
521
|
}
|
|
522
|
+
async startProviderRouteRecoverySigner(input) {
|
|
523
|
+
this.stopRecoverySigner(input.prepared.transfer_id);
|
|
524
|
+
const stop = await this.control.serveRouteRecoverySigner(input.prepared.transfer_id, async (payload) => {
|
|
525
|
+
if (payload.route_generation_id.length === 0)
|
|
526
|
+
throw new Error("route recovery generation is required");
|
|
527
|
+
const routes = await mapOrderedWithConcurrency(payload.chunks, this.routeSigningConcurrency, async (requested) => {
|
|
528
|
+
if (requested.route_generation_id !== payload.route_generation_id) {
|
|
529
|
+
throw new Error("route recovery chunk generation mismatch");
|
|
530
|
+
}
|
|
531
|
+
const chunk = materializePlanChunk(input.prepared.plan_descriptor, input.prepared.transfer_id, requested.source_id, requested.chunk_index);
|
|
532
|
+
if (requested.part_number !== multipartPartNumber(chunk.source_chunk_index, requested.attempt_slot)) {
|
|
533
|
+
throw new Error("route recovery multipart slot mapping mismatch");
|
|
534
|
+
}
|
|
535
|
+
if (chunk.source_offset !== requested.source_offset
|
|
536
|
+
|| chunk.chunk_size !== requested.chunk_size) {
|
|
537
|
+
throw new Error("route recovery source coordinate mismatch");
|
|
538
|
+
}
|
|
539
|
+
const target = chunk.destinations.find((candidate) => candidate.destination_id === requested.destination_id
|
|
540
|
+
&& Number(candidate.metadata?.delivery_index) === requested.delivery_index);
|
|
541
|
+
if (!target)
|
|
542
|
+
throw new Error("route recovery destination coordinate mismatch");
|
|
543
|
+
const source = input.sourcesById.get(requested.source_id);
|
|
544
|
+
const destination = input.destinationsById.get(requested.destination_id);
|
|
545
|
+
if (!source || !destination)
|
|
546
|
+
throw new Error("route recovery provider configuration is unavailable");
|
|
547
|
+
const upload = input.multipartUploads.get(requested.multipart_group_id);
|
|
548
|
+
if (!upload)
|
|
549
|
+
throw new Error(`route recovery multipart upload is unavailable: ${requested.multipart_group_id}`);
|
|
550
|
+
if (upload.uploadId !== requested.upload_id
|
|
551
|
+
|| upload.objectKey !== requested.final_object_key
|
|
552
|
+
|| upload.manifest.source_id !== requested.source_id
|
|
553
|
+
|| upload.manifest.destination_id !== requested.destination_id) {
|
|
554
|
+
throw new Error("route recovery multipart identity mismatch");
|
|
555
|
+
}
|
|
556
|
+
return this.signProviderRoute({
|
|
557
|
+
chunk,
|
|
558
|
+
target: {
|
|
559
|
+
...target,
|
|
560
|
+
object_key: requested.final_object_key,
|
|
561
|
+
metadata: {
|
|
562
|
+
...(target.metadata ?? {}),
|
|
563
|
+
...(requested.destination_metadata ?? {}),
|
|
564
|
+
transfer_id: input.prepared.transfer_id,
|
|
565
|
+
final_object_key: requested.final_object_key,
|
|
566
|
+
upload_id: requested.upload_id,
|
|
567
|
+
multipart_group_id: requested.multipart_group_id,
|
|
568
|
+
delivery_index: requested.delivery_index,
|
|
569
|
+
part_number: requested.part_number,
|
|
570
|
+
logical_attempt_index: requested.logical_attempt_index,
|
|
571
|
+
attempt_slot: requested.attempt_slot,
|
|
572
|
+
route_generation_id: requested.route_generation_id
|
|
573
|
+
}
|
|
574
|
+
},
|
|
575
|
+
source,
|
|
576
|
+
destination,
|
|
577
|
+
expiresIn: input.expiresIn,
|
|
578
|
+
upload,
|
|
579
|
+
transferId: input.prepared.transfer_id,
|
|
580
|
+
finalObjectKey: requested.final_object_key,
|
|
581
|
+
signedUrlFlow: "signed_url",
|
|
582
|
+
partNumber: requested.part_number
|
|
583
|
+
});
|
|
584
|
+
});
|
|
585
|
+
return {
|
|
586
|
+
transfer_id: payload.transfer_id,
|
|
587
|
+
route_generation_id: payload.route_generation_id,
|
|
588
|
+
signed_at: new Date().toISOString(),
|
|
589
|
+
chunk_routes: routes
|
|
590
|
+
};
|
|
591
|
+
});
|
|
592
|
+
this.recoverySigners.set(input.prepared.transfer_id, stop);
|
|
593
|
+
}
|
|
594
|
+
stopRecoverySigner(transferId) {
|
|
595
|
+
const stop = this.recoverySigners.get(transferId);
|
|
596
|
+
if (!stop)
|
|
597
|
+
return;
|
|
598
|
+
stop();
|
|
599
|
+
this.recoverySigners.delete(transferId);
|
|
600
|
+
}
|
|
601
|
+
stopAllRecoverySigners() {
|
|
602
|
+
for (const transferId of [...this.recoverySigners.keys()])
|
|
603
|
+
this.stopRecoverySigner(transferId);
|
|
604
|
+
}
|
|
544
605
|
async createAndDistribute(input) {
|
|
545
606
|
const transfer = await this.createRawTransfer(input);
|
|
546
607
|
if (transfer.success) {
|
|
@@ -904,14 +965,7 @@ function materializePlanChunk(descriptor, transferId, sourceId, chunkIndex) {
|
|
|
904
965
|
const partNumber = multipartPartNumber(sourceChunkIndex);
|
|
905
966
|
const objectKey = destination.provider.toLowerCase() === "hippius"
|
|
906
967
|
? `${finalObjectKey}/${descriptor.plan_nonce}/chunk-${String(sourceChunkIndex).padStart(6, "0")}`
|
|
907
|
-
:
|
|
908
|
-
finalObjectKey,
|
|
909
|
-
transferId,
|
|
910
|
-
sourceId,
|
|
911
|
-
destinationId: destination.destination_id,
|
|
912
|
-
chunkIndex,
|
|
913
|
-
partNumber
|
|
914
|
-
});
|
|
968
|
+
: finalObjectKey;
|
|
915
969
|
return {
|
|
916
970
|
destination_id: destination.destination_id,
|
|
917
971
|
delivery_index: deliveryIndex,
|
|
@@ -1042,10 +1096,7 @@ async function createMultipartGroupManifest(input) {
|
|
|
1042
1096
|
throw new Error(`plan final object key not found: ${source.source_id}:${destination.destination_id}`);
|
|
1043
1097
|
}
|
|
1044
1098
|
multipartGroupId = multipartGroupStateKey(input.prepared.transfer_id, destination.destination_id, source.source_id, finalObjectKey);
|
|
1045
|
-
const finalObjectMetadata = {
|
|
1046
|
-
"beam-transfer-id": input.prepared.transfer_id,
|
|
1047
|
-
...(input.signedUrlFlow === "signed_url_v2" ? { "beam-multipart-group-id": multipartGroupId } : {})
|
|
1048
|
-
};
|
|
1099
|
+
const finalObjectMetadata = { "beam-transfer-id": input.prepared.transfer_id };
|
|
1049
1100
|
const retainedUpload = input.multipartUploads.get(multipartGroupId);
|
|
1050
1101
|
const uploadId = retainedUpload?.uploadId ?? await createMultipartUpload({
|
|
1051
1102
|
destination: destinationConfig,
|
|
@@ -1062,31 +1113,12 @@ async function createMultipartGroupManifest(input) {
|
|
|
1062
1113
|
}
|
|
1063
1114
|
try {
|
|
1064
1115
|
const urlsExpiresAt = expiresAtIso(input.expiresIn);
|
|
1065
|
-
const stagingObjectPrefix = buildStagingObjectPrefix({
|
|
1066
|
-
finalObjectKey,
|
|
1067
|
-
transferId: input.prepared.transfer_id,
|
|
1068
|
-
sourceId: source.source_id,
|
|
1069
|
-
destinationId: destination.destination_id
|
|
1070
|
-
});
|
|
1071
|
-
const stagingListPrefix = `${stagingObjectPrefix}/`;
|
|
1072
1116
|
const maxPartNumber = multipartMaxPartNumber(source.chunk_count);
|
|
1073
1117
|
const listPageMarkers = multipartListPageMarkers(maxPartNumber);
|
|
1074
|
-
const
|
|
1075
|
-
finalObjectKey,
|
|
1076
|
-
transferId: input.prepared.transfer_id,
|
|
1077
|
-
sourceId: source.source_id,
|
|
1078
|
-
destinationId: destination.destination_id,
|
|
1079
|
-
globalChunkStart: source.global_chunk_start,
|
|
1080
|
-
chunkCount: source.chunk_count
|
|
1081
|
-
});
|
|
1082
|
-
const stagingListStartAfter = pagedListStartAfter(stagingObjectKeys);
|
|
1083
|
-
const [completeUrl, abortUrl, listPageUrls, stagingListUrls, finalHeadUrl] = await Promise.all([
|
|
1118
|
+
const [completeUrl, abortUrl, listPageUrls, finalHeadUrl] = await Promise.all([
|
|
1084
1119
|
signCompleteMultipartUpload(destinationConfig, finalObjectKey, uploadId, input.expiresIn),
|
|
1085
1120
|
signAbortMultipartUpload(destinationConfig, finalObjectKey, uploadId, input.expiresIn),
|
|
1086
1121
|
Promise.all(listPageMarkers.map((partNumberMarker) => signListMultipartUpload(destinationConfig, finalObjectKey, uploadId, input.expiresIn, { maxParts: 1_000, ...(partNumberMarker ? { partNumberMarker } : {}) }))),
|
|
1087
|
-
input.signedUrlFlow === "signed_url_v2"
|
|
1088
|
-
? Promise.all(stagingListStartAfter.map((startAfter) => signListObjects(destinationConfig, stagingListPrefix, input.expiresIn, startAfter ? { startAfter } : {})))
|
|
1089
|
-
: Promise.resolve([]),
|
|
1090
1122
|
signFinalObjectHead(destinationConfig, finalObjectKey, input.expiresIn)
|
|
1091
1123
|
]);
|
|
1092
1124
|
const manifest = {
|
|
@@ -1101,7 +1133,6 @@ async function createMultipartGroupManifest(input) {
|
|
|
1101
1133
|
complete_url: completeUrl,
|
|
1102
1134
|
abort_url: abortUrl,
|
|
1103
1135
|
list_page_urls: listPageUrls,
|
|
1104
|
-
staging_list_urls: stagingListUrls,
|
|
1105
1136
|
final_head_url: finalHeadUrl,
|
|
1106
1137
|
final_object_metadata: finalObjectMetadata,
|
|
1107
1138
|
urls_expires_at: urlsExpiresAt
|
|
@@ -1147,40 +1178,8 @@ function multipartListPageMarkers(maxPartNumber) {
|
|
|
1147
1178
|
markers.push(marker);
|
|
1148
1179
|
return markers;
|
|
1149
1180
|
}
|
|
1150
|
-
function multipartStagingObjectKeys(input) {
|
|
1151
|
-
const encoder = new TextEncoder();
|
|
1152
|
-
const keys = Array.from({ length: input.chunkCount }, (_, sourceChunkIndex) => {
|
|
1153
|
-
const key = buildStagingObjectKey({
|
|
1154
|
-
finalObjectKey: input.finalObjectKey,
|
|
1155
|
-
transferId: input.transferId,
|
|
1156
|
-
sourceId: input.sourceId,
|
|
1157
|
-
destinationId: input.destinationId,
|
|
1158
|
-
chunkIndex: input.globalChunkStart + sourceChunkIndex,
|
|
1159
|
-
partNumber: multipartPartNumber(sourceChunkIndex)
|
|
1160
|
-
});
|
|
1161
|
-
return { key, bytes: encoder.encode(key) };
|
|
1162
|
-
});
|
|
1163
|
-
keys.sort((left, right) => compareBytes(left.bytes, right.bytes));
|
|
1164
|
-
return keys.map(({ key }) => key);
|
|
1165
|
-
}
|
|
1166
|
-
function pagedListStartAfter(sortedKeys) {
|
|
1167
|
-
const boundaries = [];
|
|
1168
|
-
for (let pageStart = 0; pageStart < sortedKeys.length; pageStart += 1_000) {
|
|
1169
|
-
boundaries.push(pageStart === 0 ? undefined : sortedKeys[pageStart - 1]);
|
|
1170
|
-
}
|
|
1171
|
-
return boundaries;
|
|
1172
|
-
}
|
|
1173
|
-
function compareBytes(leftBytes, rightBytes) {
|
|
1174
|
-
const sharedLength = Math.min(leftBytes.length, rightBytes.length);
|
|
1175
|
-
for (let index = 0; index < sharedLength; index += 1) {
|
|
1176
|
-
const difference = (leftBytes[index] ?? 0) - (rightBytes[index] ?? 0);
|
|
1177
|
-
if (difference !== 0)
|
|
1178
|
-
return difference;
|
|
1179
|
-
}
|
|
1180
|
-
return leftBytes.length - rightBytes.length;
|
|
1181
|
-
}
|
|
1182
1181
|
function multipartMaxPartNumber(chunkCount) {
|
|
1183
|
-
return multipartPartNumber(positiveInteger(chunkCount, "source chunk_count") - 1);
|
|
1182
|
+
return multipartPartNumber(positiveInteger(chunkCount, "source chunk_count") - 1, 2);
|
|
1184
1183
|
}
|
|
1185
1184
|
function validateMultipartPartNumber(partNumber, manifest) {
|
|
1186
1185
|
if (!Number.isInteger(partNumber) || partNumber < 1 || partNumber > manifest.max_part_number) {
|
|
@@ -1219,17 +1218,17 @@ const COMPACT_TRANSFER_PLAN_KEYS = new Set([
|
|
|
1219
1218
|
"destinations",
|
|
1220
1219
|
"logical_chunk_count",
|
|
1221
1220
|
"delivery_route_count",
|
|
1221
|
+
"multipart_attempt_slots",
|
|
1222
1222
|
"formulas"
|
|
1223
1223
|
]);
|
|
1224
1224
|
const COMPACT_TRANSFER_PLAN_FORMULAS = {
|
|
1225
1225
|
source_offset: "source_chunk_index * chunk_size",
|
|
1226
1226
|
delivery_index: "chunk_index * destination_count + destination_index",
|
|
1227
|
-
part_number: "source_chunk_index + 1",
|
|
1228
|
-
route_generation_id: "initial-{chunk_index}-{destination_id}"
|
|
1229
|
-
staging_object_key: "{final_object_key}.beam-staging/{transfer_id}/{source_id}/{destination_id}/part-{part_number}-chunk-{chunk_index}"
|
|
1227
|
+
part_number: "source_chunk_index * 3 + attempt_slot + 1",
|
|
1228
|
+
route_generation_id: "initial-{chunk_index}-{destination_id}"
|
|
1230
1229
|
};
|
|
1231
1230
|
function validateCompactTransferPlan(descriptor, signedUrlFlow) {
|
|
1232
|
-
if (signedUrlFlow !== "
|
|
1231
|
+
if (signedUrlFlow !== "signed_url") {
|
|
1233
1232
|
throw new Error(`BeamCore returned unsupported signed_url_flow: ${String(signedUrlFlow)}`);
|
|
1234
1233
|
}
|
|
1235
1234
|
const keys = Object.keys(descriptor);
|
|
@@ -1239,6 +1238,9 @@ function validateCompactTransferPlan(descriptor, signedUrlFlow) {
|
|
|
1239
1238
|
if (descriptor.version !== "compact-transfer-plan/v1") {
|
|
1240
1239
|
throw new Error(`BeamCore returned unsupported plan version: ${descriptor.version}`);
|
|
1241
1240
|
}
|
|
1241
|
+
if (descriptor.multipart_attempt_slots !== 3) {
|
|
1242
|
+
throw new Error("BeamCore returned unsupported multipart attempt slot count");
|
|
1243
|
+
}
|
|
1242
1244
|
const formulaKeys = Object.keys(descriptor.formulas);
|
|
1243
1245
|
if (formulaKeys.length !== Object.keys(COMPACT_TRANSFER_PLAN_FORMULAS).length
|
|
1244
1246
|
|| Object.entries(COMPACT_TRANSFER_PLAN_FORMULAS).some(([key, value]) => descriptor.formulas[key] !== value)) {
|
|
@@ -1257,7 +1259,6 @@ const MULTIPART_GROUP_MANIFEST_KEYS = new Set([
|
|
|
1257
1259
|
"complete_url",
|
|
1258
1260
|
"abort_url",
|
|
1259
1261
|
"list_page_urls",
|
|
1260
|
-
"staging_list_urls",
|
|
1261
1262
|
"final_head_url",
|
|
1262
1263
|
"final_object_metadata",
|
|
1263
1264
|
"urls_expires_at"
|
|
@@ -1290,8 +1291,8 @@ function validateMultipartGroupManifest(manifest, transferId) {
|
|
|
1290
1291
|
if (!Number.isInteger(group.expected_part_count) || group.expected_part_count < 1 || group.expected_part_count > 10_000) {
|
|
1291
1292
|
throw new Error(`multipart group ${group.multipart_group_id} has invalid expected_part_count`);
|
|
1292
1293
|
}
|
|
1293
|
-
if (group.max_part_number
|
|
1294
|
-
throw new Error(`multipart group ${group.multipart_group_id}
|
|
1294
|
+
if (!Number.isInteger(group.max_part_number) || group.max_part_number < group.expected_part_count || group.max_part_number > 10_000) {
|
|
1295
|
+
throw new Error(`multipart group ${group.multipart_group_id} has invalid max_part_number`);
|
|
1295
1296
|
}
|
|
1296
1297
|
if (!Number.isSafeInteger(group.expected_object_size) || group.expected_object_size <= 0) {
|
|
1297
1298
|
throw new Error(`multipart group ${group.multipart_group_id} has invalid expected_object_size`);
|
|
@@ -1303,22 +1304,9 @@ function validateMultipartGroupManifest(manifest, transferId) {
|
|
|
1303
1304
|
|| group.list_page_urls.some((url) => typeof url !== "string" || url.length === 0)) {
|
|
1304
1305
|
throw new Error(`multipart group ${group.multipart_group_id} requires ${expectedListPageCount} unique list_page_urls`);
|
|
1305
1306
|
}
|
|
1306
|
-
if (!Array.isArray(group.staging_list_urls)
|
|
1307
|
-
|| group.staging_list_urls.some((url) => typeof url !== "string" || url.length === 0)) {
|
|
1308
|
-
throw new Error(`multipart group ${group.multipart_group_id} has invalid staging_list_urls`);
|
|
1309
|
-
}
|
|
1310
|
-
if (group.staging_list_urls.length > 0
|
|
1311
|
-
&& (group.staging_list_urls.length !== expectedListPageCount
|
|
1312
|
-
|| new Set(group.staging_list_urls).size !== expectedListPageCount)) {
|
|
1313
|
-
throw new Error(`multipart group ${group.multipart_group_id} requires ${expectedListPageCount} unique staging_list_urls when staging is used`);
|
|
1314
|
-
}
|
|
1315
1307
|
const metadataKeys = Object.keys(group.final_object_metadata);
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|| group.final_object_metadata["beam-transfer-id"] !== transferId
|
|
1319
|
-
|| (stagedCopyMetadata
|
|
1320
|
-
? group.final_object_metadata["beam-multipart-group-id"] !== group.multipart_group_id
|
|
1321
|
-
: group.final_object_metadata["beam-multipart-group-id"] !== undefined)) {
|
|
1308
|
+
if (metadataKeys.length !== 1
|
|
1309
|
+
|| group.final_object_metadata["beam-transfer-id"] !== transferId) {
|
|
1322
1310
|
throw new Error(`multipart group ${group.multipart_group_id} has invalid final_object_metadata`);
|
|
1323
1311
|
}
|
|
1324
1312
|
}
|
|
@@ -1345,7 +1333,13 @@ function randomUuid() {
|
|
|
1345
1333
|
async function transferIdForIdempotencyKey(idempotencyKey) {
|
|
1346
1334
|
if (!idempotencyKey?.trim())
|
|
1347
1335
|
return randomUuid();
|
|
1348
|
-
|
|
1336
|
+
return stableUuidFromIdentity(`beam-transfer:${idempotencyKey.trim()}`);
|
|
1337
|
+
}
|
|
1338
|
+
async function routeGenerationIdForPrepareIdempotencyKey(idempotencyKey) {
|
|
1339
|
+
return stableUuidFromIdentity(`beam-route-generation:${idempotencyKey.trim()}`);
|
|
1340
|
+
}
|
|
1341
|
+
async function stableUuidFromIdentity(identity) {
|
|
1342
|
+
const input = new TextEncoder().encode(identity);
|
|
1349
1343
|
let digest;
|
|
1350
1344
|
if (globalThis.crypto?.subtle) {
|
|
1351
1345
|
digest = new Uint8Array(await globalThis.crypto.subtle.digest("SHA-256", input));
|
|
@@ -1366,20 +1360,6 @@ function jitteredDelay(baseMs) {
|
|
|
1366
1360
|
function compact(input) {
|
|
1367
1361
|
return Object.fromEntries(Object.entries(input).filter(([, value]) => value !== undefined));
|
|
1368
1362
|
}
|
|
1369
|
-
export function buildStagingObjectPrefix(input) {
|
|
1370
|
-
return [
|
|
1371
|
-
`${input.finalObjectKey}.beam-staging`,
|
|
1372
|
-
input.transferId,
|
|
1373
|
-
input.sourceId,
|
|
1374
|
-
input.destinationId
|
|
1375
|
-
].join("/");
|
|
1376
|
-
}
|
|
1377
|
-
export function buildStagingObjectKey(input) {
|
|
1378
|
-
return [
|
|
1379
|
-
buildStagingObjectPrefix(input),
|
|
1380
|
-
`part-${input.partNumber}-chunk-${input.chunkIndex}`
|
|
1381
|
-
].join("/");
|
|
1382
|
-
}
|
|
1383
1363
|
function validateId(value, name) {
|
|
1384
1364
|
if (!value || value.includes("/") || value.includes("..")) {
|
|
1385
1365
|
throw new Error(`${name} is invalid`);
|
package/dist/models.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export type SignedUrlFlow = "
|
|
1
|
+
export type SignedUrlFlow = "signed_url";
|
|
2
2
|
export interface BeamClientOptions {
|
|
3
3
|
apiKey?: string;
|
|
4
4
|
natsUrl?: string;
|
|
@@ -148,7 +148,7 @@ export interface TransferStatusInfo {
|
|
|
148
148
|
};
|
|
149
149
|
}
|
|
150
150
|
export interface TransferTerminalEvent {
|
|
151
|
-
schema_version: "transfer-client-control/
|
|
151
|
+
schema_version: "transfer-client-control/v6";
|
|
152
152
|
producer: "transfer-runtime";
|
|
153
153
|
transfer_id: string;
|
|
154
154
|
status: "completed" | "failed" | "cancelled";
|
|
@@ -294,12 +294,12 @@ export interface CompactTransferPlanDescriptor {
|
|
|
294
294
|
destinations: CompactTransferPlanDestination[];
|
|
295
295
|
logical_chunk_count: number;
|
|
296
296
|
delivery_route_count: number;
|
|
297
|
+
multipart_attempt_slots: 3;
|
|
297
298
|
formulas: {
|
|
298
299
|
source_offset: "source_chunk_index * chunk_size";
|
|
299
300
|
delivery_index: "chunk_index * destination_count + destination_index";
|
|
300
|
-
part_number: "source_chunk_index + 1";
|
|
301
|
+
part_number: "source_chunk_index * 3 + attempt_slot + 1";
|
|
301
302
|
route_generation_id: "initial-{chunk_index}-{destination_id}";
|
|
302
|
-
staging_object_key: "{final_object_key}.beam-staging/{transfer_id}/{source_id}/{destination_id}/part-{part_number}-chunk-{chunk_index}";
|
|
303
303
|
};
|
|
304
304
|
}
|
|
305
305
|
export interface SignedChunkRoute {
|
|
@@ -328,11 +328,9 @@ export interface MultipartGroupManifest {
|
|
|
328
328
|
complete_url: string;
|
|
329
329
|
abort_url: string;
|
|
330
330
|
list_page_urls: string[];
|
|
331
|
-
staging_list_urls: string[];
|
|
332
331
|
final_head_url: string;
|
|
333
332
|
final_object_metadata: Record<string, string> & {
|
|
334
333
|
"beam-transfer-id": string;
|
|
335
|
-
"beam-multipart-group-id"?: string;
|
|
336
334
|
};
|
|
337
335
|
urls_expires_at: string;
|
|
338
336
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
1
|
export declare const MULTIPART_MAX_PART_NUMBER = 10000;
|
|
2
|
-
export declare const
|
|
3
|
-
export declare
|
|
2
|
+
export declare const MULTIPART_ATTEMPT_SLOT_COUNT = 3;
|
|
3
|
+
export declare const MULTIPART_MAX_SOURCE_CHUNKS: number;
|
|
4
|
+
export declare function multipartPartNumber(chunkIndex: number, attemptSlot?: number): number;
|
package/dist/multipart-limits.js
CHANGED
|
@@ -1,11 +1,16 @@
|
|
|
1
1
|
export const MULTIPART_MAX_PART_NUMBER = 10_000;
|
|
2
|
-
export const
|
|
3
|
-
export
|
|
4
|
-
|
|
2
|
+
export const MULTIPART_ATTEMPT_SLOT_COUNT = 3;
|
|
3
|
+
export const MULTIPART_MAX_SOURCE_CHUNKS = Math.floor(MULTIPART_MAX_PART_NUMBER / MULTIPART_ATTEMPT_SLOT_COUNT);
|
|
4
|
+
export function multipartPartNumber(chunkIndex, attemptSlot = 0) {
|
|
5
|
+
if (!Number.isInteger(chunkIndex) || chunkIndex < 0) {
|
|
5
6
|
throw new Error("source_chunk_index must be a non-negative integer");
|
|
6
7
|
}
|
|
7
|
-
if (
|
|
8
|
+
if (chunkIndex >= MULTIPART_MAX_SOURCE_CHUNKS) {
|
|
8
9
|
throw new Error(`source_chunk_index must be less than ${MULTIPART_MAX_SOURCE_CHUNKS}`);
|
|
9
10
|
}
|
|
10
|
-
|
|
11
|
+
if (!Number.isInteger(attemptSlot) || attemptSlot < 0 || attemptSlot >= MULTIPART_ATTEMPT_SLOT_COUNT) {
|
|
12
|
+
throw new Error("attempt_slot must be 0, 1, or 2");
|
|
13
|
+
}
|
|
14
|
+
const partNumber = chunkIndex * MULTIPART_ATTEMPT_SLOT_COUNT + attemptSlot + 1;
|
|
15
|
+
return partNumber;
|
|
11
16
|
}
|
package/dist/nats-control.d.ts
CHANGED
|
@@ -1,9 +1,42 @@
|
|
|
1
1
|
import { type ConnectionOptions } from "nats";
|
|
2
2
|
import type { MultipartGroupManifest, SignedChunkRoute, TransferTerminalSignalWaiter } from "./models.js";
|
|
3
|
-
export declare const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/
|
|
3
|
+
export declare const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/v6";
|
|
4
4
|
export declare const BEAM_DEFAULT_NATS_URL = "tls://nats.b1m.ai:4222";
|
|
5
5
|
export declare const BEAM_DEFAULT_NATS_WS_URL = "wss://nats.b1m.ai:443";
|
|
6
6
|
export declare const BEAM_DEFAULT_MAX_PAYLOAD_BYTES: number;
|
|
7
|
+
export declare const BEAM_ROUTE_TARGET_PAYLOAD_BYTES: number;
|
|
8
|
+
export declare const ROUTE_RECOVERY_SIGN_MESSAGE_TYPE: "transfer.route_recovery.sign";
|
|
9
|
+
export interface RouteRecoverySignChunk {
|
|
10
|
+
source_id: string;
|
|
11
|
+
destination_id: string;
|
|
12
|
+
chunk_index: number;
|
|
13
|
+
delivery_index: number;
|
|
14
|
+
source_offset: number;
|
|
15
|
+
chunk_size: number;
|
|
16
|
+
logical_attempt_index: number;
|
|
17
|
+
attempt_slot: 0 | 1 | 2;
|
|
18
|
+
part_number: number;
|
|
19
|
+
route_generation_id: string;
|
|
20
|
+
multipart_group_id: string;
|
|
21
|
+
final_object_key: string;
|
|
22
|
+
upload_id: string;
|
|
23
|
+
urls_expires_at?: string;
|
|
24
|
+
multipart_created_at?: string;
|
|
25
|
+
source_metadata?: Record<string, unknown>;
|
|
26
|
+
destination_metadata?: Record<string, unknown>;
|
|
27
|
+
}
|
|
28
|
+
export interface RouteRecoverySignRequestPayload {
|
|
29
|
+
transfer_id: string;
|
|
30
|
+
route_generation_id: string;
|
|
31
|
+
requested_at?: string;
|
|
32
|
+
chunks: RouteRecoverySignChunk[];
|
|
33
|
+
}
|
|
34
|
+
export interface RouteRecoverySignReplyPayload {
|
|
35
|
+
transfer_id: string;
|
|
36
|
+
route_generation_id: string;
|
|
37
|
+
signed_at?: string;
|
|
38
|
+
chunk_routes: SignedChunkRoute[];
|
|
39
|
+
}
|
|
7
40
|
export type TransferClientMessageType = "runtime.hello" | "transfer.plan" | "transfer.create" | "transfer.prepare" | "transfer.route_stream.begin" | "transfer.route_stream.manifest" | "transfer.route_stream.batch" | "transfer.route_stream.complete" | "transfer.distribute" | "transfer.status" | "transfer.cancel" | "transfer.resume";
|
|
8
41
|
export interface CompactSignedRouteBatch {
|
|
9
42
|
source_chunks: Array<{
|
|
@@ -33,9 +66,9 @@ export interface RouteStreamBeginPayload {
|
|
|
33
66
|
stream_id: string;
|
|
34
67
|
total_routes: number;
|
|
35
68
|
total_chunks: number;
|
|
36
|
-
route_contract_version: "
|
|
69
|
+
route_contract_version: "signed_url";
|
|
37
70
|
urls_expires_at?: string;
|
|
38
|
-
signed_url_flow: "
|
|
71
|
+
signed_url_flow: "signed_url";
|
|
39
72
|
auto_distribute: boolean;
|
|
40
73
|
}
|
|
41
74
|
export interface RouteStreamManifestPayload {
|
|
@@ -88,6 +121,7 @@ export declare class BeamTransferControl {
|
|
|
88
121
|
private authTokenExpiresAt;
|
|
89
122
|
private authResolvePromise;
|
|
90
123
|
private readonly terminalWaitCancellations;
|
|
124
|
+
private readonly routeRecoverySignerCancellations;
|
|
91
125
|
private readonly recoveryLeases;
|
|
92
126
|
private readonly recoveryOperations;
|
|
93
127
|
private readonly recoveryRequestedEpoch;
|
|
@@ -95,6 +129,7 @@ export declare class BeamTransferControl {
|
|
|
95
129
|
private helloTimer;
|
|
96
130
|
constructor(options: TransferControlOptions);
|
|
97
131
|
close(): Promise<void>;
|
|
132
|
+
serveRouteRecoverySigner(transferId: string, handler: (payload: RouteRecoverySignRequestPayload) => Promise<RouteRecoverySignReplyPayload>): Promise<() => void>;
|
|
98
133
|
registerRecoveryLease(lease: TransferRecoveryLease): () => void;
|
|
99
134
|
releaseRecoveryLease(transferId: string): void;
|
|
100
135
|
continueRecoveryLease(transferId: string): void;
|
|
@@ -109,6 +144,8 @@ export declare class BeamTransferControl {
|
|
|
109
144
|
private resolveAuthToken;
|
|
110
145
|
private resolveAuthTokenFresh;
|
|
111
146
|
private authSubject;
|
|
147
|
+
private routeRecoverySignSubject;
|
|
148
|
+
private buildRouteRecoveryReply;
|
|
112
149
|
private requestSubject;
|
|
113
150
|
private terminalSubject;
|
|
114
151
|
}
|
package/dist/nats-control.js
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { encode, decode } from "@msgpack/msgpack";
|
|
2
2
|
import { connect } from "nats";
|
|
3
|
-
export const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/
|
|
3
|
+
export const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/v6";
|
|
4
4
|
export const BEAM_DEFAULT_NATS_URL = "tls://nats.b1m.ai:4222";
|
|
5
5
|
export const BEAM_DEFAULT_NATS_WS_URL = "wss://nats.b1m.ai:443";
|
|
6
6
|
// NATS enforces max_payload per message. This guard splits signed-route control
|
|
7
7
|
// messages before the broker rejects them; transfer bytes never flow through NATS.
|
|
8
8
|
export const BEAM_DEFAULT_MAX_PAYLOAD_BYTES = 8 * 1024 * 1024;
|
|
9
|
+
export const BEAM_ROUTE_TARGET_PAYLOAD_BYTES = 4 * 1024 * 1024;
|
|
9
10
|
const ROUTE_BATCH_AUTH_TOKEN_ESTIMATE_BYTES = 64 * 1024;
|
|
11
|
+
export const ROUTE_RECOVERY_SIGN_MESSAGE_TYPE = "transfer.route_recovery.sign";
|
|
10
12
|
export function buildConnectionOptions(options) {
|
|
11
13
|
const connectionOptions = {
|
|
12
14
|
servers: options.natsUrl,
|
|
@@ -55,6 +57,7 @@ export class BeamTransferControl {
|
|
|
55
57
|
authTokenExpiresAt = 0;
|
|
56
58
|
authResolvePromise = null;
|
|
57
59
|
terminalWaitCancellations = new Set();
|
|
60
|
+
routeRecoverySignerCancellations = new Set();
|
|
58
61
|
recoveryLeases = new Map();
|
|
59
62
|
recoveryOperations = new Map();
|
|
60
63
|
recoveryRequestedEpoch = new Map();
|
|
@@ -80,6 +83,8 @@ export class BeamTransferControl {
|
|
|
80
83
|
this.closed = true;
|
|
81
84
|
for (const cancel of [...this.terminalWaitCancellations])
|
|
82
85
|
cancel();
|
|
86
|
+
for (const cancel of [...this.routeRecoverySignerCancellations])
|
|
87
|
+
cancel();
|
|
83
88
|
if (this.helloTimer)
|
|
84
89
|
clearInterval(this.helloTimer);
|
|
85
90
|
this.helloTimer = null;
|
|
@@ -100,6 +105,53 @@ export class BeamTransferControl {
|
|
|
100
105
|
if (nc && !nc.isClosed())
|
|
101
106
|
await nc.drain();
|
|
102
107
|
}
|
|
108
|
+
async serveRouteRecoverySigner(transferId, handler) {
|
|
109
|
+
const nc = await this.connection();
|
|
110
|
+
const subject = this.routeRecoverySignSubject(transferId);
|
|
111
|
+
let active = true;
|
|
112
|
+
const subscription = nc.subscribe(subject, {
|
|
113
|
+
callback: (_error, message) => {
|
|
114
|
+
if (!active)
|
|
115
|
+
return;
|
|
116
|
+
void (async () => {
|
|
117
|
+
let request = null;
|
|
118
|
+
try {
|
|
119
|
+
request = decode(message.data);
|
|
120
|
+
if (request.schema_version !== TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION
|
|
121
|
+
|| request.environment !== this.environment
|
|
122
|
+
|| request.key_prefix !== this.keyPrefix
|
|
123
|
+
|| request.transfer_id !== transferId
|
|
124
|
+
|| request.message_type !== ROUTE_RECOVERY_SIGN_MESSAGE_TYPE
|
|
125
|
+
|| request.producer !== "transfer-runtime"
|
|
126
|
+
|| request.payload.transfer_id !== transferId
|
|
127
|
+
|| request.payload.route_generation_id.length === 0
|
|
128
|
+
|| !Array.isArray(request.payload.chunks)
|
|
129
|
+
|| request.payload.chunks.length === 0) {
|
|
130
|
+
throw new Error("route recovery request envelope mismatch");
|
|
131
|
+
}
|
|
132
|
+
const payload = await handler(request.payload);
|
|
133
|
+
message.respond(encode(this.buildRouteRecoveryReply(request, true, 200, payload)));
|
|
134
|
+
}
|
|
135
|
+
catch (error) {
|
|
136
|
+
message.respond(encode(this.buildRouteRecoveryReply(request ?? { message_id: "unknown", request_id: "unknown", transfer_id: transferId }, false, 500, undefined, {
|
|
137
|
+
code: "route_recovery_sign_failed",
|
|
138
|
+
message: error instanceof Error ? error.message : String(error)
|
|
139
|
+
})));
|
|
140
|
+
}
|
|
141
|
+
})();
|
|
142
|
+
}
|
|
143
|
+
});
|
|
144
|
+
await nc.flush();
|
|
145
|
+
const stop = () => {
|
|
146
|
+
if (!active)
|
|
147
|
+
return;
|
|
148
|
+
active = false;
|
|
149
|
+
subscription.unsubscribe();
|
|
150
|
+
this.routeRecoverySignerCancellations.delete(stop);
|
|
151
|
+
};
|
|
152
|
+
this.routeRecoverySignerCancellations.add(stop);
|
|
153
|
+
return stop;
|
|
154
|
+
}
|
|
103
155
|
registerRecoveryLease(lease) {
|
|
104
156
|
if (this.closed)
|
|
105
157
|
throw new Error("NATS lifecycle control is closed");
|
|
@@ -285,6 +337,7 @@ export class BeamTransferControl {
|
|
|
285
337
|
}
|
|
286
338
|
}
|
|
287
339
|
splitRoutesForPayload(messageType, basePayload, routes) {
|
|
340
|
+
const targetPayloadBytes = Math.min(this.maxPayloadBytes, BEAM_ROUTE_TARGET_PAYLOAD_BYTES);
|
|
288
341
|
const authTokenEstimateBytes = Math.min(ROUTE_BATCH_AUTH_TOKEN_ESTIMATE_BYTES, Math.max(512, Math.floor(this.maxPayloadBytes / 128)));
|
|
289
342
|
const encodedBytes = (candidate) => {
|
|
290
343
|
const encoded = encode({
|
|
@@ -312,7 +365,7 @@ export class BeamTransferControl {
|
|
|
312
365
|
let accepted = 0;
|
|
313
366
|
while (low <= high) {
|
|
314
367
|
const count = Math.floor((low + high) / 2);
|
|
315
|
-
if (encodedBytes(routes.slice(offset, offset + count)) <=
|
|
368
|
+
if (encodedBytes(routes.slice(offset, offset + count)) <= targetPayloadBytes) {
|
|
316
369
|
accepted = count;
|
|
317
370
|
low = count + 1;
|
|
318
371
|
}
|
|
@@ -321,7 +374,10 @@ export class BeamTransferControl {
|
|
|
321
374
|
}
|
|
322
375
|
if (accepted === 0) {
|
|
323
376
|
const bytes = encodedBytes(routes.slice(offset, offset + 1));
|
|
324
|
-
|
|
377
|
+
if (bytes > this.maxPayloadBytes) {
|
|
378
|
+
throw new Error(`single signed route is ${bytes} bytes, above maxPayloadBytes=${this.maxPayloadBytes}`);
|
|
379
|
+
}
|
|
380
|
+
accepted = 1;
|
|
325
381
|
}
|
|
326
382
|
chunks.push(routes.slice(offset, offset + accepted));
|
|
327
383
|
offset += accepted;
|
|
@@ -493,6 +549,26 @@ export class BeamTransferControl {
|
|
|
493
549
|
authSubject() {
|
|
494
550
|
return `${this.subjectPrefix}.${this.environment}.auth.${this.keyPrefix}.resolve`;
|
|
495
551
|
}
|
|
552
|
+
routeRecoverySignSubject(transferId) {
|
|
553
|
+
return `${this.subjectPrefix}.${this.environment}.sdk.${this.keyPrefix}.transfer.${transferId}.route_recovery_sign`;
|
|
554
|
+
}
|
|
555
|
+
buildRouteRecoveryReply(request, ok, status, payload, error) {
|
|
556
|
+
return {
|
|
557
|
+
message_id: request.message_id,
|
|
558
|
+
schema_version: TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION,
|
|
559
|
+
environment: this.environment,
|
|
560
|
+
key_prefix: this.keyPrefix,
|
|
561
|
+
transfer_id: request.transfer_id,
|
|
562
|
+
message_type: ROUTE_RECOVERY_SIGN_MESSAGE_TYPE,
|
|
563
|
+
request_id: request.request_id,
|
|
564
|
+
occurred_at: new Date().toISOString(),
|
|
565
|
+
producer: "sdk",
|
|
566
|
+
ok,
|
|
567
|
+
status,
|
|
568
|
+
...(payload ? { payload } : {}),
|
|
569
|
+
...(error ? { error } : {})
|
|
570
|
+
};
|
|
571
|
+
}
|
|
496
572
|
requestSubject(messageType, shardId) {
|
|
497
573
|
return `${this.subjectPrefix}.${this.environment}.sdk.${this.keyPrefix}.shard.${shardId}.${messageType.replace(/\./g, "_")}`;
|
|
498
574
|
}
|
|
@@ -517,12 +593,7 @@ const ROUTE_ATTEMPT_METADATA_KEYS = new Set([
|
|
|
517
593
|
"part_number",
|
|
518
594
|
"logical_attempt_index",
|
|
519
595
|
"attempt_slot",
|
|
520
|
-
"route_generation_id"
|
|
521
|
-
"staging_object_key",
|
|
522
|
-
"copy_url",
|
|
523
|
-
"copy_source",
|
|
524
|
-
"copy_source_range",
|
|
525
|
-
"delete_url"
|
|
596
|
+
"route_generation_id"
|
|
526
597
|
]);
|
|
527
598
|
const ROUTE_IDENTITY_METADATA_KEYS = [
|
|
528
599
|
"source_id",
|
|
@@ -22,18 +22,10 @@ export declare function createMultipartUpload(input: {
|
|
|
22
22
|
export declare function signFinalObjectHead(destination: ProviderDestinationConfig, objectKey: string, expiresIn: number): Promise<string>;
|
|
23
23
|
export declare function signCompleteMultipartUpload(destination: ProviderDestinationConfig, objectKey: string, uploadId: string, expiresIn: number): Promise<string>;
|
|
24
24
|
export declare function signAbortMultipartUpload(destination: ProviderDestinationConfig, objectKey: string, uploadId: string, expiresIn: number): Promise<string>;
|
|
25
|
-
export declare function signListObjects(destination: ProviderDestinationConfig, prefix: string, expiresIn: number, options?: {
|
|
26
|
-
startAfter?: string;
|
|
27
|
-
}): Promise<string>;
|
|
28
25
|
export declare function signListMultipartUpload(destination: ProviderDestinationConfig, objectKey: string, uploadId: string, expiresIn: number, options?: {
|
|
29
26
|
maxParts?: number;
|
|
30
27
|
partNumberMarker?: number;
|
|
31
28
|
}): Promise<string>;
|
|
32
|
-
export declare function signUploadPartCopy(destination: ProviderDestinationConfig, finalObjectKey: string, stagingObjectKey: string, uploadId: string, partNumber: number, expiresIn: number, copySourceRange?: string): Promise<{
|
|
33
|
-
copyUrl: string;
|
|
34
|
-
copySource: string;
|
|
35
|
-
}>;
|
|
36
|
-
export declare function signDeleteObject(destination: ProviderDestinationConfig, objectKey: string, expiresIn: number): Promise<string>;
|
|
37
29
|
export declare function abortMultipartUpload(destination: ProviderDestinationConfig, objectKey: string, uploadId: string): Promise<void>;
|
|
38
30
|
export declare function signDestinationRoute(input: {
|
|
39
31
|
chunk: ChunkSigningPlanItem;
|
|
@@ -53,12 +45,6 @@ export declare function signDestinationRoute(input: {
|
|
|
53
45
|
maxPartNumber?: number;
|
|
54
46
|
finalObjectMetadata?: Record<string, string>;
|
|
55
47
|
multipartGroupId?: string;
|
|
56
|
-
commitMethod?: "upload_part_copy";
|
|
57
|
-
stagingObjectKey?: string;
|
|
58
|
-
copyUrl?: string;
|
|
59
|
-
copySource?: string;
|
|
60
|
-
copySourceRange?: string;
|
|
61
|
-
deleteUrl?: string;
|
|
62
48
|
fetchImpl?: typeof fetch;
|
|
63
49
|
}): Promise<SignedChunkRoute>;
|
|
64
50
|
export declare function isS3CompatibleProvider(config: ProviderSourceConfig | ProviderDestinationConfig): config is AnyS3CompatibleProviderConfig;
|
package/dist/provider-signing.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand,
|
|
1
|
+
import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, GetObjectCommand, HeadObjectCommand, ListPartsCommand, PutObjectCommand, S3Client, UploadPartCommand } from "@aws-sdk/client-s3";
|
|
2
2
|
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
|
3
3
|
export function nowIso() {
|
|
4
4
|
return new Date().toISOString();
|
|
@@ -149,18 +149,6 @@ export async function signAbortMultipartUpload(destination, objectKey, uploadId,
|
|
|
149
149
|
}
|
|
150
150
|
throw new Error(`multipart abort is not supported for ${destination.provider}`);
|
|
151
151
|
}
|
|
152
|
-
export async function signListObjects(destination, prefix, expiresIn, options = {}) {
|
|
153
|
-
if (isS3CompatibleProvider(destination)) {
|
|
154
|
-
const client = createS3CompatibleClient(destination);
|
|
155
|
-
return getSignedUrl(client, new ListObjectsV2Command({
|
|
156
|
-
Bucket: destination.bucket,
|
|
157
|
-
Prefix: prefix,
|
|
158
|
-
MaxKeys: 1_000,
|
|
159
|
-
StartAfter: options.startAfter
|
|
160
|
-
}), { expiresIn });
|
|
161
|
-
}
|
|
162
|
-
throw new Error(`multipart staging list is not supported for ${destination.provider}`);
|
|
163
|
-
}
|
|
164
152
|
export async function signListMultipartUpload(destination, objectKey, uploadId, expiresIn, options = {}) {
|
|
165
153
|
if (isS3CompatibleProvider(destination)) {
|
|
166
154
|
const client = createS3CompatibleClient(destination);
|
|
@@ -174,42 +162,6 @@ export async function signListMultipartUpload(destination, objectKey, uploadId,
|
|
|
174
162
|
}
|
|
175
163
|
throw new Error(`multipart list-parts is not supported for ${destination.provider}`);
|
|
176
164
|
}
|
|
177
|
-
export async function signUploadPartCopy(destination, finalObjectKey, stagingObjectKey, uploadId, partNumber, expiresIn, copySourceRange) {
|
|
178
|
-
if (isS3CompatibleProvider(destination)) {
|
|
179
|
-
const client = createS3CompatibleClient(destination);
|
|
180
|
-
const copySource = copySourceHeader(destination, stagingObjectKey);
|
|
181
|
-
// Keep x-amz-copy-source (and x-amz-copy-source-range when present) in
|
|
182
|
-
// X-Amz-SignedHeaders rather than hoisting them to URL query params.
|
|
183
|
-
// The JS AWS SDK default hoists all x-amz-* headers to query params for
|
|
184
|
-
// presigned URLs; R2 ignores copy-source from query params (zero-byte copy)
|
|
185
|
-
// and returns SignatureDoesNotMatch when they are sent as unsigned headers
|
|
186
|
-
// alongside the signed query value. Keeping them as signed headers matches
|
|
187
|
-
// boto3 behavior and the copySourceHeaders replay contract in core-server.
|
|
188
|
-
const unhoistableHeaders = new Set(["x-amz-copy-source"]);
|
|
189
|
-
if (copySourceRange) {
|
|
190
|
-
unhoistableHeaders.add("x-amz-copy-source-range");
|
|
191
|
-
}
|
|
192
|
-
return {
|
|
193
|
-
copySource,
|
|
194
|
-
copyUrl: await getSignedUrl(client, new UploadPartCopyCommand({
|
|
195
|
-
Bucket: destination.bucket,
|
|
196
|
-
Key: finalObjectKey,
|
|
197
|
-
UploadId: uploadId,
|
|
198
|
-
PartNumber: partNumber,
|
|
199
|
-
CopySource: copySource,
|
|
200
|
-
CopySourceRange: copySourceRange
|
|
201
|
-
}), { expiresIn, unhoistableHeaders })
|
|
202
|
-
};
|
|
203
|
-
}
|
|
204
|
-
throw new Error(`multipart copy is not supported for ${destination.provider}`);
|
|
205
|
-
}
|
|
206
|
-
export async function signDeleteObject(destination, objectKey, expiresIn) {
|
|
207
|
-
if (isS3CompatibleProvider(destination)) {
|
|
208
|
-
const client = createS3CompatibleClient(destination);
|
|
209
|
-
return getSignedUrl(client, new DeleteObjectCommand({ Bucket: destination.bucket, Key: objectKey }), { expiresIn });
|
|
210
|
-
}
|
|
211
|
-
throw new Error(`object delete signing is not supported for ${destination.provider}`);
|
|
212
|
-
}
|
|
213
165
|
export async function abortMultipartUpload(destination, objectKey, uploadId) {
|
|
214
166
|
if (isS3CompatibleProvider(destination)) {
|
|
215
167
|
await createS3CompatibleClient(destination).send(new AbortMultipartUploadCommand({ Bucket: destination.bucket, Key: objectKey, UploadId: uploadId }));
|
|
@@ -218,18 +170,14 @@ export async function abortMultipartUpload(destination, objectKey, uploadId) {
|
|
|
218
170
|
throw new Error(`multipart abort is not supported for ${destination.provider}`);
|
|
219
171
|
}
|
|
220
172
|
export async function signDestinationRoute(input) {
|
|
221
|
-
if (input.commitMethod !== undefined && input.commitMethod !== "upload_part_copy") {
|
|
222
|
-
throw new Error(`unsupported signed_url_v2 commit method: ${String(input.commitMethod)}`);
|
|
223
|
-
}
|
|
224
173
|
const targetObjectKey = input.target.object_key;
|
|
225
174
|
if (!targetObjectKey) {
|
|
226
175
|
throw new Error("destination signing target is missing object_key");
|
|
227
176
|
}
|
|
228
|
-
const objectKey = input.stagingObjectKey ?? targetObjectKey;
|
|
229
177
|
const [destUrl, sourceRoute] = await Promise.all([
|
|
230
178
|
signDestinationUrl({
|
|
231
179
|
destination: input.destination,
|
|
232
|
-
objectKey,
|
|
180
|
+
objectKey: targetObjectKey,
|
|
233
181
|
uploadId: input.uploadId,
|
|
234
182
|
partNumber: input.partNumber,
|
|
235
183
|
expiresIn: input.expiresIn,
|
|
@@ -256,12 +204,6 @@ export async function signDestinationRoute(input) {
|
|
|
256
204
|
metadata: {
|
|
257
205
|
...(input.target.metadata ?? {}),
|
|
258
206
|
...(input.multipartGroupId ? { multipart_group_id: input.multipartGroupId } : {}),
|
|
259
|
-
...(input.commitMethod ? { commit_method: input.commitMethod } : {}),
|
|
260
|
-
...(input.stagingObjectKey ? { staging_object_key: input.stagingObjectKey } : {}),
|
|
261
|
-
...(input.copyUrl ? { copy_url: input.copyUrl } : {}),
|
|
262
|
-
...(input.copySource ? { copy_source: input.copySource } : {}),
|
|
263
|
-
...(input.copySourceRange ? { copy_source_range: input.copySourceRange } : {}),
|
|
264
|
-
...(input.deleteUrl ? { delete_url: input.deleteUrl } : {}),
|
|
265
207
|
...(input.uploadId ? { upload_id: input.uploadId } : {}),
|
|
266
208
|
...(input.completeUrl ? { complete_url: input.completeUrl } : {}),
|
|
267
209
|
...(input.abortUrl ? { abort_url: input.abortUrl } : {}),
|