@beam-network/sdk 0.4.4 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/dist/client.d.ts +9 -0
- package/dist/client.js +305 -160
- package/dist/models.d.ts +10 -19
- package/dist/nats-control.d.ts +32 -2
- package/dist/nats-control.js +184 -7
- package/dist/provider-signing.js +1 -2
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -96,10 +96,14 @@ For low-level raw transfer configs, use `createRawTransfer`; lifecycle transport
|
|
|
96
96
|
|
|
97
97
|
## Route Streaming And Payload Size
|
|
98
98
|
|
|
99
|
-
Provider transfers
|
|
99
|
+
Provider transfers use `transfer-client-control/v5`. Every reply carries Runtime and transport epochs; prepare and every route-stream message carry a UUID route generation. S3, R2, and S3-compatible destinations retain direct multipart UploadPart/ListParts/HEAD handling.
|
|
100
100
|
|
|
101
101
|
The client signs up to 64 routes concurrently by default and emits 2,048-route logical batches, split only when encoded MessagePack requests exceed `maxPayloadBytes` (24 MiB by default). The stream ID derives from the transfer, selected flow, and immutable plan identity, and each ordered batch ID includes its route-coordinate checksum. Manual multi-destination attachment requires `delivery_index` on every route. Lifecycle mutations retry transient failures up to three times with the same request identity. `waitForTransfer` subscribes to the API-key-owned, at-most-once terminal signal before its first status read and reconciles every signal through authoritative status; subscription failure degrades to the same jittered 15-to-30-second status fallback. Call `close()` when the client is no longer needed; it also closes outstanding terminal waiters.
|
|
102
102
|
|
|
103
103
|
Hippius uses genuine non-multipart v2 routes, so its manifest is empty and group-level final HEAD verification is not available from that provider flow.
|
|
104
104
|
|
|
105
105
|
NATS requires this guard because the broker rejects messages above its configured `max_payload`. This limit applies only to lifecycle/control metadata; transfer file bytes do not flow through NATS.
|
|
106
|
+
|
|
107
|
+
## Restart Recovery
|
|
108
|
+
|
|
109
|
+
The client keeps one in-memory recovery lease per active transfer and one `runtime.hello` monitor per active shard. The lease is installed before route-stream begin, and a per-transfer lock coalesces initial streaming with replay. Multipart recovery retains the existing upload IDs and compact group state, then re-signs only the expiring route and commit controls. A Runtime epoch change invalidates cached auth, coalesces one `transfer.resume`, and regenerates routes under a fresh generation; a transport-only epoch change replays only when Runtime reports routes missing or expired. Provider credentials and signing inputs are retained only in memory and released on terminal status or `close()`. Low-level `attachSignedUrls` requires `routeGenerationId`, the plan fingerprint/checksum, and a `recoveryFactory`; signed URLs are never journaled to disk. Foreground cancellation, deadline expiry, and Runtime state-loss responses keep the background lease and retained multipart upload alive.
|
package/dist/client.d.ts
CHANGED
|
@@ -37,11 +37,20 @@ export declare class BeamClient {
|
|
|
37
37
|
urlsExpiresAt?: string;
|
|
38
38
|
signedUrlFlow?: SignedUrlFlow;
|
|
39
39
|
idempotencyKey?: string;
|
|
40
|
+
routeGenerationId?: string;
|
|
40
41
|
}): Promise<TransferPrepareResponse>;
|
|
41
42
|
attachSignedUrls(transferId: string, input: {
|
|
42
43
|
chunkRoutes: SignedChunkRoute[];
|
|
43
44
|
multipartGroupManifest: MultipartGroupManifest[];
|
|
44
45
|
transferKey?: string;
|
|
46
|
+
routeGenerationId: string;
|
|
47
|
+
planFingerprint: string;
|
|
48
|
+
coordinateChecksum: string;
|
|
49
|
+
recoveryFactory: (routeGenerationId: string) => Promise<{
|
|
50
|
+
chunkRoutes: SignedChunkRoute[];
|
|
51
|
+
multipartGroupManifest: MultipartGroupManifest[];
|
|
52
|
+
urlsExpiresAt?: string;
|
|
53
|
+
}>;
|
|
45
54
|
urlsExpiresAt?: string;
|
|
46
55
|
autoDistribute?: boolean;
|
|
47
56
|
}): Promise<AttachSignedUrlsResponse>;
|
package/dist/client.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { abortMultipartUpload, createMultipartUpload, expiresAtIso, prepareProviderDestination, prepareProviderSource, signAbortMultipartUpload, signCompleteMultipartUpload, signDeleteObject, signDestinationRoute, signFinalObjectHead, signListMultipartUpload, signListObjects, signUploadPartCopy } from "./provider-signing.js";
|
|
2
|
-
import { BeamTransferControl, BEAM_DEFAULT_NATS_URL, compactSignedRoutes } from "./nats-control.js";
|
|
2
|
+
import { BeamTransferControl, BEAM_DEFAULT_NATS_URL, compactSignedRoutes, isRecoverableRouteStreamError } from "./nats-control.js";
|
|
3
3
|
import { multipartPartNumber } from "./multipart-limits.js";
|
|
4
4
|
export { BEAM_DEFAULT_NATS_URL };
|
|
5
5
|
export class BeamApiError extends Error {
|
|
@@ -73,9 +73,12 @@ export class BeamClient {
|
|
|
73
73
|
createTransfer(input) {
|
|
74
74
|
return this.prepareProviderTransfer(input);
|
|
75
75
|
}
|
|
76
|
-
transferStatus(transferId) {
|
|
76
|
+
async transferStatus(transferId) {
|
|
77
77
|
validateId(transferId, "transferId");
|
|
78
|
-
|
|
78
|
+
const status = await this.control.request("transfer.status", { transfer_id: transferId }, { transferId });
|
|
79
|
+
if (["completed", "failed", "cancelled"].includes(status.status))
|
|
80
|
+
this.control.releaseRecoveryLease(transferId);
|
|
81
|
+
return status;
|
|
79
82
|
}
|
|
80
83
|
distributeTransfer(transferId) {
|
|
81
84
|
validateId(transferId, "transferId");
|
|
@@ -90,6 +93,7 @@ export class BeamClient {
|
|
|
90
93
|
transferId,
|
|
91
94
|
idempotencyKey: `transfer:${transferId}:cancel`
|
|
92
95
|
});
|
|
96
|
+
this.control.releaseRecoveryLease(transferId);
|
|
93
97
|
return result;
|
|
94
98
|
}
|
|
95
99
|
async planTransfer(input) {
|
|
@@ -107,8 +111,10 @@ export class BeamClient {
|
|
|
107
111
|
}
|
|
108
112
|
async prepareTransfer(input) {
|
|
109
113
|
const transferId = await transferIdForIdempotencyKey(input.idempotencyKey);
|
|
114
|
+
const routeGenerationId = input.routeGenerationId ?? randomUuid();
|
|
110
115
|
const result = await this.control.request("transfer.prepare", compact({
|
|
111
116
|
transfer_id: transferId,
|
|
117
|
+
route_generation_id: routeGenerationId,
|
|
112
118
|
sources: input.sources,
|
|
113
119
|
destinations: input.destinations,
|
|
114
120
|
name: input.name,
|
|
@@ -127,56 +133,97 @@ export class BeamClient {
|
|
|
127
133
|
validateId(transferId, "transferId");
|
|
128
134
|
validateMultipartGroupManifest(input.multipartGroupManifest, transferId);
|
|
129
135
|
validateSignedRouteManifestContract(input.chunkRoutes, input.multipartGroupManifest);
|
|
130
|
-
const
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
136
|
+
const streamRoutes = async (chunkRoutes, multipartGroupManifest, routeGenerationId, urlsExpiresAt) => {
|
|
137
|
+
validateMultipartGroupManifest(multipartGroupManifest, transferId);
|
|
138
|
+
validateSignedRouteManifestContract(chunkRoutes, multipartGroupManifest);
|
|
139
|
+
const destinationCount = new Set(chunkRoutes.map((route) => route.destination_id)).size;
|
|
140
|
+
const orderedRoutes = chunkRoutes.map((route) => {
|
|
141
|
+
if (signedRouteDeliveryIndex(route) !== undefined)
|
|
142
|
+
return route;
|
|
143
|
+
if (destinationCount !== 1) {
|
|
144
|
+
throw new Error("delivery_index is required when manually attaching routes for multiple destinations");
|
|
145
|
+
}
|
|
146
|
+
return { ...route, delivery_index: route.chunk_index };
|
|
147
|
+
})
|
|
148
|
+
.sort(compareSignedRoutesByDelivery);
|
|
149
|
+
const streamId = await stableRouteStreamId({
|
|
150
|
+
transferId,
|
|
151
|
+
planIdentity: `${await routeCoordinateChecksum(orderedRoutes)}:${routeGenerationId}`,
|
|
152
|
+
totalRoutes: orderedRoutes.length,
|
|
153
|
+
totalChunks: countDistinctRouteChunks(orderedRoutes),
|
|
154
|
+
signedUrlFlow: multipartGroupManifest.length ? "signed_url_v2" : "signed_url_v1"
|
|
155
|
+
});
|
|
156
|
+
const stream = new RouteStreamSender(this.control, {
|
|
157
|
+
streamId,
|
|
158
|
+
transferId,
|
|
159
|
+
routeGenerationId,
|
|
160
|
+
totalRoutes: orderedRoutes.length,
|
|
161
|
+
totalChunks: countDistinctRouteChunks(orderedRoutes),
|
|
162
|
+
urlsExpiresAt,
|
|
163
|
+
signedUrlFlow: multipartGroupManifest.length ? "signed_url_v2" : "signed_url_v1",
|
|
164
|
+
autoDistribute: input.autoDistribute ?? true
|
|
165
|
+
});
|
|
166
|
+
await stream.begin();
|
|
167
|
+
if (multipartGroupManifest.length)
|
|
168
|
+
await stream.addManifestGroups(multipartGroupManifest);
|
|
169
|
+
for (const route of orderedRoutes)
|
|
170
|
+
await stream.addRoute(route);
|
|
171
|
+
return stream.complete();
|
|
172
|
+
};
|
|
173
|
+
const routeStreamLock = new AsyncMutex();
|
|
174
|
+
const releaseInitialStream = await routeStreamLock.acquire();
|
|
175
|
+
this.control.registerRecoveryLease({
|
|
150
176
|
transferId,
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
177
|
+
planFingerprint: input.planFingerprint,
|
|
178
|
+
coordinateChecksum: input.coordinateChecksum,
|
|
179
|
+
replayRoutes: async (routeGenerationId) => {
|
|
180
|
+
await routeStreamLock.run(async () => {
|
|
181
|
+
const fresh = await input.recoveryFactory(routeGenerationId);
|
|
182
|
+
const attached = await streamRoutes(fresh.chunkRoutes, fresh.multipartGroupManifest, routeGenerationId, fresh.urlsExpiresAt);
|
|
183
|
+
if (!attached.success)
|
|
184
|
+
throw new Error(attached.error ?? attached.message ?? "route replay failed");
|
|
185
|
+
});
|
|
186
|
+
}
|
|
156
187
|
});
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
188
|
+
try {
|
|
189
|
+
const attached = await streamRoutes(input.chunkRoutes, input.multipartGroupManifest, input.routeGenerationId, input.urlsExpiresAt);
|
|
190
|
+
if (!attached.success)
|
|
191
|
+
this.control.releaseRecoveryLease(transferId);
|
|
192
|
+
return attached;
|
|
160
193
|
}
|
|
161
|
-
|
|
162
|
-
|
|
194
|
+
catch (error) {
|
|
195
|
+
if (isRecoverableRouteStreamError(error))
|
|
196
|
+
this.control.continueRecoveryLease(transferId);
|
|
197
|
+
else
|
|
198
|
+
this.control.releaseRecoveryLease(transferId);
|
|
199
|
+
throw error;
|
|
200
|
+
}
|
|
201
|
+
finally {
|
|
202
|
+
releaseInitialStream();
|
|
163
203
|
}
|
|
164
|
-
return stream.complete();
|
|
165
204
|
}
|
|
166
205
|
async prepareProviderTransfer(input) {
|
|
167
206
|
const expiresIn = input.expiresIn ?? 3600;
|
|
168
207
|
const requestedSignedUrlFlow = input.signedUrlFlow ?? "signed_url_v1";
|
|
169
|
-
const
|
|
208
|
+
const retainedRecoveryInput = {
|
|
209
|
+
...input,
|
|
210
|
+
sources: input.sources.map((source) => ({ ...source })),
|
|
211
|
+
destinations: input.destinations.map((destination) => ({ ...destination })),
|
|
212
|
+
onBeforeTransferPrepare: undefined,
|
|
213
|
+
onPrepared: undefined,
|
|
214
|
+
throwIfCancelled: undefined
|
|
215
|
+
};
|
|
216
|
+
const preparedDestinations = retainedRecoveryInput.destinations.map((destination, index) => prepareProviderDestination(destination, { index }));
|
|
170
217
|
const destinationsById = new Map();
|
|
171
218
|
preparedDestinations.forEach((preparedDestination, index) => {
|
|
172
|
-
const destination =
|
|
219
|
+
const destination = retainedRecoveryInput.destinations[index];
|
|
173
220
|
if (!destination) {
|
|
174
221
|
throw new Error(`missing provider destination config for prepared destination ${preparedDestination.destination_id}`);
|
|
175
222
|
}
|
|
176
223
|
destinationsById.set(preparedDestination.destination_id, destination);
|
|
177
224
|
});
|
|
178
225
|
await input.throwIfCancelled?.();
|
|
179
|
-
const preparedSources = await Promise.all(
|
|
226
|
+
const preparedSources = await Promise.all(retainedRecoveryInput.sources.map((source, index) => prepareProviderSource(source, { index, expiresIn, fetchImpl: this.fetchImpl })));
|
|
180
227
|
await input.throwIfCancelled?.();
|
|
181
228
|
await input.onBeforeTransferPrepare?.();
|
|
182
229
|
const prepared = await this.prepareTransfer({
|
|
@@ -185,145 +232,201 @@ export class BeamClient {
|
|
|
185
232
|
name: input.name,
|
|
186
233
|
testMode: input.testMode,
|
|
187
234
|
signedUrlFlow: requestedSignedUrlFlow,
|
|
188
|
-
idempotencyKey: input.idempotencyKey
|
|
235
|
+
idempotencyKey: input.idempotencyKey,
|
|
236
|
+
routeGenerationId: input.routeGenerationId
|
|
189
237
|
});
|
|
190
238
|
if (!prepared.success) {
|
|
191
239
|
return prepared;
|
|
192
240
|
}
|
|
193
|
-
await input.onPrepared?.(prepared);
|
|
194
|
-
await input.throwIfCancelled?.(prepared.transfer_id);
|
|
195
241
|
const sourcesById = new Map();
|
|
196
242
|
preparedSources.forEach((preparedSource, index) => {
|
|
197
|
-
const source =
|
|
243
|
+
const source = retainedRecoveryInput.sources[index];
|
|
198
244
|
if (!source) {
|
|
199
245
|
throw new Error(`missing provider source config for prepared source ${preparedSource.source_id}`);
|
|
200
246
|
}
|
|
201
247
|
sourcesById.set(preparedSource.source_id, source);
|
|
202
248
|
});
|
|
203
249
|
const multipartUploads = new Map();
|
|
204
|
-
const
|
|
205
|
-
let
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
250
|
+
const autoDistribute = input.distribute !== false;
|
|
251
|
+
let initialThrowIfCancelled = input.throwIfCancelled;
|
|
252
|
+
const routeStreamLock = new AsyncMutex();
|
|
253
|
+
const releaseInitialStream = await routeStreamLock.acquire();
|
|
254
|
+
const streamPreparedRoutes = async (routeGenerationId, recoveryReplay) => {
|
|
255
|
+
const pendingRoutes = new Set();
|
|
256
|
+
let signingConcurrency = this.routeSigningConcurrency;
|
|
257
|
+
let signedInWindow = 0;
|
|
258
|
+
let signingWindowStartedAt = performance.now();
|
|
259
|
+
let routeStream = null;
|
|
260
|
+
let routeStreamBeginAttempted = false;
|
|
261
|
+
let multipartManifestTask = null;
|
|
262
|
+
let multipartGroupWaiters = new Map();
|
|
263
|
+
let foregroundCancelled = false;
|
|
264
|
+
const throwIfCancelled = recoveryReplay || !initialThrowIfCancelled ? undefined : async (transferId) => {
|
|
265
|
+
try {
|
|
266
|
+
await initialThrowIfCancelled?.(transferId);
|
|
267
|
+
}
|
|
268
|
+
catch (error) {
|
|
269
|
+
foregroundCancelled = true;
|
|
270
|
+
throw error;
|
|
271
|
+
}
|
|
272
|
+
};
|
|
273
|
+
try {
|
|
274
|
+
const streamId = await stableRouteStreamId({
|
|
275
|
+
transferId: prepared.transfer_id,
|
|
276
|
+
planIdentity: `${prepared.plan_descriptor.plan_nonce}:${routeGenerationId}`,
|
|
277
|
+
totalRoutes: prepared.plan_descriptor.delivery_route_count,
|
|
278
|
+
totalChunks: prepared.plan_descriptor.logical_chunk_count,
|
|
279
|
+
signedUrlFlow: requestedSignedUrlFlow
|
|
280
|
+
});
|
|
281
|
+
routeStream = new RouteStreamSender(this.control, {
|
|
282
|
+
streamId,
|
|
283
|
+
transferId: prepared.transfer_id,
|
|
284
|
+
routeGenerationId,
|
|
285
|
+
totalRoutes: prepared.plan_descriptor.delivery_route_count,
|
|
286
|
+
totalChunks: prepared.plan_descriptor.logical_chunk_count,
|
|
287
|
+
signedUrlFlow: requestedSignedUrlFlow,
|
|
288
|
+
autoDistribute,
|
|
289
|
+
urlsExpiresAt: expiresAtIso(expiresIn)
|
|
290
|
+
});
|
|
291
|
+
routeStreamBeginAttempted = true;
|
|
292
|
+
await routeStream.begin();
|
|
293
|
+
multipartGroupWaiters = createMultipartGroupWaiters(prepared, destinationsById);
|
|
294
|
+
multipartManifestTask = createMultipartGroupManifest({
|
|
295
|
+
prepared,
|
|
296
|
+
destinationsById,
|
|
297
|
+
multipartUploads,
|
|
298
|
+
expiresIn,
|
|
299
|
+
signedUrlFlow: requestedSignedUrlFlow,
|
|
300
|
+
concurrency: this.routeSigningConcurrency,
|
|
301
|
+
preserveUploadsOnFailure: recoveryReplay,
|
|
302
|
+
onGroupReady: async (state) => {
|
|
303
|
+
if (requestedSignedUrlFlow === "signed_url_v2") {
|
|
304
|
+
validateMultipartGroupManifest([state.manifest], prepared.transfer_id);
|
|
305
|
+
await routeStream.addManifestGroups([state.manifest]);
|
|
306
|
+
}
|
|
307
|
+
multipartGroupWaiters.get(state.manifest.multipart_group_id)?.resolve(state);
|
|
308
|
+
},
|
|
309
|
+
onGroupFailed: (groupId, error) => multipartGroupWaiters.get(groupId)?.reject(error)
|
|
310
|
+
});
|
|
311
|
+
const streamNextCompletedRoute = async () => {
|
|
312
|
+
const settled = await Promise.race([...pendingRoutes].map((pending) => pending.then((route) => ({ pending, route }))));
|
|
313
|
+
pendingRoutes.delete(settled.pending);
|
|
314
|
+
await routeStream.addRoute(settled.route);
|
|
315
|
+
signedInWindow += 1;
|
|
316
|
+
if (signedInWindow === ROUTE_STREAM_BATCH_ROUTES) {
|
|
317
|
+
const elapsedMs = performance.now() - signingWindowStartedAt;
|
|
318
|
+
if (!this.routeSigningConcurrencyOverridden && elapsedMs > 4_000 && signingConcurrency < 256) {
|
|
319
|
+
signingConcurrency = Math.min(256, signingConcurrency * 2);
|
|
320
|
+
}
|
|
321
|
+
signedInWindow = 0;
|
|
322
|
+
signingWindowStartedAt = performance.now();
|
|
242
323
|
}
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
324
|
+
};
|
|
325
|
+
for (const chunk of materializePlanChunks(prepared.plan_descriptor, prepared.transfer_id)) {
|
|
326
|
+
await throwIfCancelled?.(prepared.transfer_id);
|
|
327
|
+
for (const target of chunk.destinations) {
|
|
328
|
+
const pendingRoute = (async () => {
|
|
329
|
+
await throwIfCancelled?.(prepared.transfer_id);
|
|
330
|
+
const destination = destinationsById.get(target.destination_id);
|
|
331
|
+
if (!destination)
|
|
332
|
+
throw new Error(`BeamCore returned unknown destination_id: ${target.destination_id}`);
|
|
333
|
+
const source = sourcesById.get(chunk.source_id);
|
|
334
|
+
if (!source)
|
|
335
|
+
throw new Error(`BeamCore returned unknown source_id: ${chunk.source_id}`);
|
|
336
|
+
const metadata = target.metadata ?? {};
|
|
337
|
+
const finalObjectKey = typeof metadata.final_object_key === "string" ? metadata.final_object_key : target.object_key;
|
|
338
|
+
if (!finalObjectKey)
|
|
339
|
+
throw new Error("destination signing target is missing object_key");
|
|
340
|
+
const multipartGroupId = multipartGroupStateKey(prepared.transfer_id, target.destination_id, chunk.source_id, finalObjectKey);
|
|
341
|
+
const upload = isHippiusDestination(destination)
|
|
342
|
+
? undefined
|
|
343
|
+
: await multipartGroupWaiters.get(multipartGroupId)?.promise;
|
|
344
|
+
if (!isHippiusDestination(destination) && !upload)
|
|
345
|
+
throw new Error(`multipart group manifest is missing for ${chunk.source_id}:${target.destination_id}`);
|
|
346
|
+
return this.signProviderRoute({
|
|
347
|
+
chunk,
|
|
348
|
+
target,
|
|
349
|
+
source,
|
|
350
|
+
destination,
|
|
351
|
+
expiresIn,
|
|
352
|
+
...(upload ? { upload } : {}),
|
|
353
|
+
transferId: prepared.transfer_id,
|
|
354
|
+
finalObjectKey,
|
|
355
|
+
signedUrlFlow: requestedSignedUrlFlow,
|
|
356
|
+
partNumber: typeof metadata.part_number === "number"
|
|
357
|
+
? metadata.part_number
|
|
358
|
+
: multipartPartNumber(chunk.source_chunk_index),
|
|
359
|
+
});
|
|
360
|
+
})();
|
|
361
|
+
pendingRoutes.add(pendingRoute);
|
|
362
|
+
if (pendingRoutes.size >= signingConcurrency)
|
|
363
|
+
await streamNextCompletedRoute();
|
|
256
364
|
}
|
|
257
|
-
signedInWindow = 0;
|
|
258
|
-
signingWindowStartedAt = performance.now();
|
|
259
365
|
}
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
await
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
source,
|
|
286
|
-
destination,
|
|
287
|
-
expiresIn,
|
|
288
|
-
...(upload ? { upload } : {}),
|
|
366
|
+
while (pendingRoutes.size)
|
|
367
|
+
await streamNextCompletedRoute();
|
|
368
|
+
const multipartGroupManifest = await multipartManifestTask;
|
|
369
|
+
if (requestedSignedUrlFlow === "signed_url_v2") {
|
|
370
|
+
validateMultipartGroupManifest(multipartGroupManifest, prepared.transfer_id);
|
|
371
|
+
}
|
|
372
|
+
const attached = await routeStream.complete();
|
|
373
|
+
if (!attached.success) {
|
|
374
|
+
throw new Error(attached.error ?? attached.message ?? "route stream failed");
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
catch (error) {
|
|
378
|
+
if (routeStream)
|
|
379
|
+
await routeStream.abort().catch(() => undefined);
|
|
380
|
+
await Promise.allSettled(pendingRoutes);
|
|
381
|
+
if (multipartManifestTask)
|
|
382
|
+
await Promise.allSettled([multipartManifestTask]);
|
|
383
|
+
if (!recoveryReplay) {
|
|
384
|
+
if (foregroundCancelled || isRecoverableRouteStreamError(error)) {
|
|
385
|
+
this.control.continueRecoveryLease(prepared.transfer_id);
|
|
386
|
+
}
|
|
387
|
+
else {
|
|
388
|
+
this.control.releaseRecoveryLease(prepared.transfer_id);
|
|
389
|
+
await cancelAndAbortProviderFailure({
|
|
390
|
+
client: this,
|
|
289
391
|
transferId: prepared.transfer_id,
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
? metadata.part_number
|
|
294
|
-
: multipartPartNumber(chunk.source_chunk_index),
|
|
392
|
+
cause: error,
|
|
393
|
+
uploads: multipartUploads,
|
|
394
|
+
abortBeforeCancel: !routeStreamBeginAttempted
|
|
295
395
|
});
|
|
296
|
-
}
|
|
297
|
-
pendingRoutes.add(pendingRoute);
|
|
298
|
-
if (pendingRoutes.size >= signingConcurrency)
|
|
299
|
-
await streamNextCompletedRoute();
|
|
396
|
+
}
|
|
300
397
|
}
|
|
398
|
+
throw error;
|
|
301
399
|
}
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
400
|
+
};
|
|
401
|
+
this.control.registerRecoveryLease({
|
|
402
|
+
transferId: prepared.transfer_id,
|
|
403
|
+
planFingerprint: prepared.plan_fingerprint,
|
|
404
|
+
coordinateChecksum: prepared.coordinate_checksum,
|
|
405
|
+
replayRoutes: async (routeGenerationId) => {
|
|
406
|
+
await routeStreamLock.run(() => streamPreparedRoutes(routeGenerationId, true));
|
|
407
|
+
},
|
|
408
|
+
disposeSecrets: () => clearRecoverySecrets(retainedRecoveryInput)
|
|
409
|
+
});
|
|
410
|
+
try {
|
|
411
|
+
try {
|
|
412
|
+
await input.onPrepared?.(prepared);
|
|
307
413
|
}
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
throw
|
|
414
|
+
catch (error) {
|
|
415
|
+
this.control.continueRecoveryLease(prepared.transfer_id);
|
|
416
|
+
throw error;
|
|
311
417
|
}
|
|
418
|
+
try {
|
|
419
|
+
await input.throwIfCancelled?.(prepared.transfer_id);
|
|
420
|
+
}
|
|
421
|
+
catch (error) {
|
|
422
|
+
this.control.continueRecoveryLease(prepared.transfer_id);
|
|
423
|
+
throw error;
|
|
424
|
+
}
|
|
425
|
+
await streamPreparedRoutes(prepared.route_generation_id, false);
|
|
426
|
+
initialThrowIfCancelled = undefined;
|
|
312
427
|
}
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
await routeStream.abort();
|
|
316
|
-
await Promise.allSettled(pendingRoutes);
|
|
317
|
-
if (multipartManifestTask)
|
|
318
|
-
await Promise.allSettled([multipartManifestTask]);
|
|
319
|
-
await cancelAndAbortProviderFailure({
|
|
320
|
-
client: this,
|
|
321
|
-
transferId: prepared.transfer_id,
|
|
322
|
-
cause: error,
|
|
323
|
-
uploads: multipartUploads,
|
|
324
|
-
abortBeforeCancel: !routeStreamBeginAttempted
|
|
325
|
-
});
|
|
326
|
-
throw error;
|
|
428
|
+
finally {
|
|
429
|
+
releaseInitialStream();
|
|
327
430
|
}
|
|
328
431
|
return prepared;
|
|
329
432
|
}
|
|
@@ -488,6 +591,27 @@ export class BeamClient {
|
|
|
488
591
|
}
|
|
489
592
|
}
|
|
490
593
|
const ROUTE_STREAM_BATCH_ROUTES = 2_048;
|
|
594
|
+
class AsyncMutex {
|
|
595
|
+
tail = Promise.resolve();
|
|
596
|
+
async acquire() {
|
|
597
|
+
const previous = this.tail;
|
|
598
|
+
let release;
|
|
599
|
+
this.tail = new Promise((resolve) => {
|
|
600
|
+
release = resolve;
|
|
601
|
+
});
|
|
602
|
+
await previous;
|
|
603
|
+
return release;
|
|
604
|
+
}
|
|
605
|
+
async run(operation) {
|
|
606
|
+
const release = await this.acquire();
|
|
607
|
+
try {
|
|
608
|
+
return await operation();
|
|
609
|
+
}
|
|
610
|
+
finally {
|
|
611
|
+
release();
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
}
|
|
491
615
|
class RouteStreamSender {
|
|
492
616
|
control;
|
|
493
617
|
options;
|
|
@@ -507,6 +631,7 @@ class RouteStreamSender {
|
|
|
507
631
|
async begin() {
|
|
508
632
|
await this.control.request("transfer.route_stream.begin", compact({
|
|
509
633
|
transfer_id: this.options.transferId,
|
|
634
|
+
route_generation_id: this.options.routeGenerationId,
|
|
510
635
|
stream_id: this.streamId,
|
|
511
636
|
total_routes: this.options.totalRoutes,
|
|
512
637
|
total_chunks: this.options.totalChunks,
|
|
@@ -526,6 +651,7 @@ class RouteStreamSender {
|
|
|
526
651
|
const identity = await stableManifestBatchIdentity([group]);
|
|
527
652
|
await this.control.request("transfer.route_stream.manifest", {
|
|
528
653
|
transfer_id: this.options.transferId,
|
|
654
|
+
route_generation_id: this.options.routeGenerationId,
|
|
529
655
|
stream_id: this.streamId,
|
|
530
656
|
manifest_batch_id: identity,
|
|
531
657
|
groups: [group]
|
|
@@ -564,6 +690,7 @@ class RouteStreamSender {
|
|
|
564
690
|
throw this.sendError;
|
|
565
691
|
return this.control.request("transfer.route_stream.complete", {
|
|
566
692
|
transfer_id: this.options.transferId,
|
|
693
|
+
route_generation_id: this.options.routeGenerationId,
|
|
567
694
|
stream_id: this.streamId,
|
|
568
695
|
expected_batches: this.batchIndex,
|
|
569
696
|
expected_routes: this.options.totalRoutes,
|
|
@@ -582,6 +709,7 @@ class RouteStreamSender {
|
|
|
582
709
|
const routes = this.batch.splice(0, this.batch.length);
|
|
583
710
|
const chunks = this.control.splitRoutesForPayload("transfer.route_stream.batch", {
|
|
584
711
|
transfer_id: this.options.transferId,
|
|
712
|
+
route_generation_id: this.options.routeGenerationId,
|
|
585
713
|
stream_id: this.streamId,
|
|
586
714
|
batch_id: `${this.streamId}:estimate`,
|
|
587
715
|
batch_index: this.batchIndex
|
|
@@ -601,6 +729,7 @@ class RouteStreamSender {
|
|
|
601
729
|
const batchId = `${this.streamId}:${batchIndex}:${coordinateChecksum}`;
|
|
602
730
|
await this.control.request("transfer.route_stream.batch", {
|
|
603
731
|
transfer_id: this.options.transferId,
|
|
732
|
+
route_generation_id: this.options.routeGenerationId,
|
|
604
733
|
stream_id: this.streamId,
|
|
605
734
|
batch_id: batchId,
|
|
606
735
|
batch_index: batchIndex,
|
|
@@ -880,11 +1009,13 @@ async function createMultipartGroupManifest(input) {
|
|
|
880
1009
|
"beam-transfer-id": input.prepared.transfer_id,
|
|
881
1010
|
...(input.signedUrlFlow === "signed_url_v2" ? { "beam-multipart-group-id": multipartGroupId } : {})
|
|
882
1011
|
};
|
|
883
|
-
const
|
|
1012
|
+
const retainedUpload = input.multipartUploads.get(multipartGroupId);
|
|
1013
|
+
const uploadId = retainedUpload?.uploadId ?? await createMultipartUpload({
|
|
884
1014
|
destination: destinationConfig,
|
|
885
1015
|
objectKey: finalObjectKey,
|
|
886
1016
|
metadata: finalObjectMetadata
|
|
887
1017
|
});
|
|
1018
|
+
const createdUpload = !retainedUpload;
|
|
888
1019
|
try {
|
|
889
1020
|
const urlsExpiresAt = expiresAtIso(input.expiresIn);
|
|
890
1021
|
const stagingObjectPrefix = buildStagingObjectPrefix({
|
|
@@ -942,11 +1073,13 @@ async function createMultipartGroupManifest(input) {
|
|
|
942
1073
|
return { ok: true, state };
|
|
943
1074
|
}
|
|
944
1075
|
catch (error) {
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
1076
|
+
if (createdUpload) {
|
|
1077
|
+
try {
|
|
1078
|
+
await abortMultipartUpload(destinationConfig, finalObjectKey, uploadId);
|
|
1079
|
+
}
|
|
1080
|
+
catch (abortError) {
|
|
1081
|
+
throw new AggregateError([error, abortError], `multipart group setup and cleanup failed for ${multipartGroupId}`);
|
|
1082
|
+
}
|
|
950
1083
|
}
|
|
951
1084
|
throw error;
|
|
952
1085
|
}
|
|
@@ -959,6 +1092,8 @@ async function createMultipartGroupManifest(input) {
|
|
|
959
1092
|
});
|
|
960
1093
|
const failure = results.find((result) => !result.ok);
|
|
961
1094
|
if (failure && !failure.ok) {
|
|
1095
|
+
if (input.preserveUploadsOnFailure)
|
|
1096
|
+
throw failure.error;
|
|
962
1097
|
try {
|
|
963
1098
|
await abortCreatedUploads(input.multipartUploads);
|
|
964
1099
|
}
|
|
@@ -1237,3 +1372,13 @@ function isHippiusDestination(destination) {
|
|
|
1237
1372
|
function sleep(ms) {
|
|
1238
1373
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
1239
1374
|
}
|
|
1375
|
+
function clearRecoverySecrets(input) {
|
|
1376
|
+
const secretKeys = new Set(["access_key_id", "secret_access_key", "session_token", "api_token", "token", "password"]);
|
|
1377
|
+
for (const provider of [...input.sources, ...input.destinations]) {
|
|
1378
|
+
const record = provider;
|
|
1379
|
+
for (const key of secretKeys) {
|
|
1380
|
+
if (key in record)
|
|
1381
|
+
record[key] = "";
|
|
1382
|
+
}
|
|
1383
|
+
}
|
|
1384
|
+
}
|
package/dist/models.d.ts
CHANGED
|
@@ -70,6 +70,8 @@ export interface RawTransferCreateInput {
|
|
|
70
70
|
progressiveMode?: boolean;
|
|
71
71
|
signedUrlFlow?: SignedUrlFlow;
|
|
72
72
|
idempotencyKey?: string;
|
|
73
|
+
/** Internal recovery generation; callers normally leave this unset. */
|
|
74
|
+
routeGenerationId?: string;
|
|
73
75
|
}
|
|
74
76
|
export interface TransferCreateResponse {
|
|
75
77
|
success: boolean;
|
|
@@ -145,7 +147,7 @@ export interface TransferStatusInfo {
|
|
|
145
147
|
};
|
|
146
148
|
}
|
|
147
149
|
export interface TransferTerminalEvent {
|
|
148
|
-
schema_version: "transfer-client-control/
|
|
150
|
+
schema_version: "transfer-client-control/v5";
|
|
149
151
|
producer: "transfer-runtime";
|
|
150
152
|
transfer_id: string;
|
|
151
153
|
status: "completed" | "failed" | "cancelled";
|
|
@@ -176,23 +178,6 @@ export interface R2ProviderConfig {
|
|
|
176
178
|
account_id?: string;
|
|
177
179
|
endpoint_url?: string;
|
|
178
180
|
}
|
|
179
|
-
export interface S3CompatibleProviderCapabilities {
|
|
180
|
-
/**
|
|
181
|
-
* Whether the provider's ListParts / ListObjects response includes a per-part
|
|
182
|
-
* LastModified timestamp. Defaults to true (S3-spec compliant). Providers that
|
|
183
|
-
* omit it (e.g. Hippius) declare false so Beam finalization substitutes a
|
|
184
|
-
* Beam-side timestamp instead of looping into multipart recovery.
|
|
185
|
-
*/
|
|
186
|
-
multipart_part_last_modified?: boolean;
|
|
187
|
-
/**
|
|
188
|
-
* Per-destination override for Beam Core's assignment-recovery deadline
|
|
189
|
-
* (TRANSFER_ASSIGNMENT_TIMEOUT_MS), in milliseconds. Omit to use the global
|
|
190
|
-
* default (15s). Slower providers (e.g. Hippius) declare a larger value so
|
|
191
|
-
* their assignment waves aren't reassigned prematurely. Only integers >= 1000
|
|
192
|
-
* are honoured by Beam Core.
|
|
193
|
-
*/
|
|
194
|
-
assignment_timeout_ms?: number;
|
|
195
|
-
}
|
|
196
181
|
export interface S3CompatibleProviderConfig {
|
|
197
182
|
provider: string;
|
|
198
183
|
driver?: "s3-compatible";
|
|
@@ -206,7 +191,6 @@ export interface S3CompatibleProviderConfig {
|
|
|
206
191
|
session_token?: string;
|
|
207
192
|
force_path_style?: boolean;
|
|
208
193
|
account_id?: string;
|
|
209
|
-
capabilities?: S3CompatibleProviderCapabilities;
|
|
210
194
|
}
|
|
211
195
|
export interface HippiusProviderConfig {
|
|
212
196
|
provider: "hippius";
|
|
@@ -254,6 +238,8 @@ export interface ProviderTransferCreateInput {
|
|
|
254
238
|
signedUrlFlow?: SignedUrlFlow;
|
|
255
239
|
/** Stable caller identity used to derive the transfer and lifecycle request ids. */
|
|
256
240
|
idempotencyKey?: string;
|
|
241
|
+
/** Internal recovery generation; callers normally omit this. */
|
|
242
|
+
routeGenerationId?: string;
|
|
257
243
|
}
|
|
258
244
|
export interface PlanningHttpSource {
|
|
259
245
|
source_id: string;
|
|
@@ -359,6 +345,8 @@ export interface TransferPlanResponse {
|
|
|
359
345
|
total_chunks?: number;
|
|
360
346
|
plan_descriptor: CompactTransferPlanDescriptor;
|
|
361
347
|
signed_url_flow: SignedUrlFlow;
|
|
348
|
+
plan_fingerprint: string;
|
|
349
|
+
coordinate_checksum: string;
|
|
362
350
|
error?: string;
|
|
363
351
|
message?: string;
|
|
364
352
|
}
|
|
@@ -375,6 +363,9 @@ export interface TransferPrepareResponse {
|
|
|
375
363
|
total_chunks?: number;
|
|
376
364
|
plan_descriptor: CompactTransferPlanDescriptor;
|
|
377
365
|
signed_url_flow: SignedUrlFlow;
|
|
366
|
+
plan_fingerprint: string;
|
|
367
|
+
coordinate_checksum: string;
|
|
368
|
+
route_generation_id: string;
|
|
378
369
|
error?: string;
|
|
379
370
|
message?: string;
|
|
380
371
|
}
|
package/dist/nats-control.d.ts
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
|
+
import { type ConnectionOptions } from "nats";
|
|
1
2
|
import type { MultipartGroupManifest, SignedChunkRoute, TransferTerminalSignalWaiter } from "./models.js";
|
|
2
|
-
export declare const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/
|
|
3
|
+
export declare const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/v5";
|
|
3
4
|
export declare const BEAM_DEFAULT_NATS_URL = "tls://nats.b1m.ai:4222";
|
|
4
5
|
export declare const BEAM_DEFAULT_NATS_WS_URL = "wss://nats.b1m.ai:443";
|
|
5
6
|
export declare const BEAM_DEFAULT_MAX_PAYLOAD_BYTES: number;
|
|
6
|
-
export type TransferClientMessageType = "transfer.plan" | "transfer.create" | "transfer.prepare" | "transfer.route_stream.begin" | "transfer.route_stream.manifest" | "transfer.route_stream.batch" | "transfer.route_stream.complete" | "transfer.distribute" | "transfer.status" | "transfer.cancel";
|
|
7
|
+
export type TransferClientMessageType = "runtime.hello" | "transfer.plan" | "transfer.create" | "transfer.prepare" | "transfer.route_stream.begin" | "transfer.route_stream.manifest" | "transfer.route_stream.batch" | "transfer.route_stream.complete" | "transfer.distribute" | "transfer.status" | "transfer.cancel" | "transfer.resume";
|
|
7
8
|
export interface CompactSignedRouteBatch {
|
|
8
9
|
source_chunks: Array<{
|
|
9
10
|
source_ref: number;
|
|
@@ -28,6 +29,7 @@ export interface CompactSignedRouteBatch {
|
|
|
28
29
|
}
|
|
29
30
|
export interface RouteStreamBeginPayload {
|
|
30
31
|
transfer_id: string;
|
|
32
|
+
route_generation_id: string;
|
|
31
33
|
stream_id: string;
|
|
32
34
|
total_routes: number;
|
|
33
35
|
total_chunks: number;
|
|
@@ -38,10 +40,16 @@ export interface RouteStreamBeginPayload {
|
|
|
38
40
|
}
|
|
39
41
|
export interface RouteStreamManifestPayload {
|
|
40
42
|
transfer_id: string;
|
|
43
|
+
route_generation_id: string;
|
|
41
44
|
stream_id: string;
|
|
42
45
|
manifest_batch_id: string;
|
|
43
46
|
groups: MultipartGroupManifest[];
|
|
44
47
|
}
|
|
48
|
+
export declare function buildConnectionOptions(options: {
|
|
49
|
+
natsUrl: string;
|
|
50
|
+
apiKey: string;
|
|
51
|
+
keyPrefix: string;
|
|
52
|
+
}): ConnectionOptions;
|
|
45
53
|
export interface TransferControlOptions {
|
|
46
54
|
apiKey: string;
|
|
47
55
|
natsUrl?: string;
|
|
@@ -55,6 +63,14 @@ export interface TransferControlOptions {
|
|
|
55
63
|
export interface LifecycleRequestOptions {
|
|
56
64
|
transferId?: string;
|
|
57
65
|
idempotencyKey?: string;
|
|
66
|
+
shardId?: number;
|
|
67
|
+
}
|
|
68
|
+
export interface TransferRecoveryLease {
|
|
69
|
+
transferId: string;
|
|
70
|
+
planFingerprint: string;
|
|
71
|
+
coordinateChecksum: string;
|
|
72
|
+
replayRoutes(routeGenerationId: string): Promise<void>;
|
|
73
|
+
disposeSecrets?(): void;
|
|
58
74
|
}
|
|
59
75
|
export declare class BeamTransferControl {
|
|
60
76
|
readonly environment: string;
|
|
@@ -72,12 +88,24 @@ export declare class BeamTransferControl {
|
|
|
72
88
|
private authTokenExpiresAt;
|
|
73
89
|
private authResolvePromise;
|
|
74
90
|
private readonly terminalWaitCancellations;
|
|
91
|
+
private readonly recoveryLeases;
|
|
92
|
+
private readonly recoveryOperations;
|
|
93
|
+
private readonly recoveryRequestedEpoch;
|
|
94
|
+
private readonly runtimeEpochs;
|
|
95
|
+
private helloTimer;
|
|
75
96
|
constructor(options: TransferControlOptions);
|
|
76
97
|
close(): Promise<void>;
|
|
98
|
+
registerRecoveryLease(lease: TransferRecoveryLease): () => void;
|
|
99
|
+
releaseRecoveryLease(transferId: string): void;
|
|
100
|
+
continueRecoveryLease(transferId: string): void;
|
|
77
101
|
request<T>(messageType: TransferClientMessageType, payload: Record<string, unknown>, options?: LifecycleRequestOptions): Promise<T>;
|
|
78
102
|
openTerminalSignalWaiter(transferId: string): Promise<TransferTerminalSignalWaiter>;
|
|
79
103
|
splitRoutesForPayload(messageType: TransferClientMessageType, basePayload: Record<string, unknown>, routes: SignedChunkRoute[]): SignedChunkRoute[][];
|
|
80
104
|
private connection;
|
|
105
|
+
private ensureHelloMonitor;
|
|
106
|
+
private pollRuntimeHello;
|
|
107
|
+
private observeRuntimeEpoch;
|
|
108
|
+
private recoverLease;
|
|
81
109
|
private resolveAuthToken;
|
|
82
110
|
private resolveAuthTokenFresh;
|
|
83
111
|
private authSubject;
|
|
@@ -86,3 +114,5 @@ export declare class BeamTransferControl {
|
|
|
86
114
|
}
|
|
87
115
|
export declare function compactSignedRoutes(routes: SignedChunkRoute[]): CompactSignedRouteBatch;
|
|
88
116
|
export declare function transferShardId(transferId: string, shardCount: number): number;
|
|
117
|
+
export declare function isRetryableLifecycleError(error: unknown): boolean;
|
|
118
|
+
export declare function isRecoverableRouteStreamError(error: unknown): boolean;
|
package/dist/nats-control.js
CHANGED
|
@@ -1,17 +1,22 @@
|
|
|
1
1
|
import { encode, decode } from "@msgpack/msgpack";
|
|
2
2
|
import { connect } from "nats";
|
|
3
|
-
export const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/
|
|
3
|
+
export const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/v5";
|
|
4
4
|
export const BEAM_DEFAULT_NATS_URL = "tls://nats.b1m.ai:4222";
|
|
5
5
|
export const BEAM_DEFAULT_NATS_WS_URL = "wss://nats.b1m.ai:443";
|
|
6
6
|
// NATS enforces max_payload per message. This guard splits signed-route control
|
|
7
7
|
// messages before the broker rejects them; transfer bytes never flow through NATS.
|
|
8
8
|
export const BEAM_DEFAULT_MAX_PAYLOAD_BYTES = 24 * 1024 * 1024;
|
|
9
|
-
function buildConnectionOptions(options) {
|
|
9
|
+
export function buildConnectionOptions(options) {
|
|
10
10
|
const connectionOptions = {
|
|
11
11
|
servers: options.natsUrl,
|
|
12
12
|
user: options.keyPrefix,
|
|
13
13
|
pass: options.apiKey,
|
|
14
|
-
name: `beam-sdk-${options.keyPrefix}
|
|
14
|
+
name: `beam-sdk-${options.keyPrefix}`,
|
|
15
|
+
reconnect: true,
|
|
16
|
+
maxReconnectAttempts: -1,
|
|
17
|
+
reconnectTimeWait: 1_000,
|
|
18
|
+
reconnectJitter: 500,
|
|
19
|
+
reconnectJitterTLS: 1_000
|
|
15
20
|
};
|
|
16
21
|
if (options.natsUrl.startsWith("tls://")) {
|
|
17
22
|
const tlsOptions = { handshakeFirst: true };
|
|
@@ -25,6 +30,14 @@ function buildConnectionOptions(options) {
|
|
|
25
30
|
}
|
|
26
31
|
return connectionOptions;
|
|
27
32
|
}
|
|
33
|
+
class LifecycleRequestError extends Error {
|
|
34
|
+
status;
|
|
35
|
+
constructor(status, message) {
|
|
36
|
+
super(message);
|
|
37
|
+
this.status = status;
|
|
38
|
+
this.name = "LifecycleRequestError";
|
|
39
|
+
}
|
|
40
|
+
}
|
|
28
41
|
export class BeamTransferControl {
|
|
29
42
|
environment;
|
|
30
43
|
keyPrefix;
|
|
@@ -41,6 +54,11 @@ export class BeamTransferControl {
|
|
|
41
54
|
authTokenExpiresAt = 0;
|
|
42
55
|
authResolvePromise = null;
|
|
43
56
|
terminalWaitCancellations = new Set();
|
|
57
|
+
recoveryLeases = new Map();
|
|
58
|
+
recoveryOperations = new Map();
|
|
59
|
+
recoveryRequestedEpoch = new Map();
|
|
60
|
+
runtimeEpochs = new Map();
|
|
61
|
+
helloTimer = null;
|
|
44
62
|
constructor(options) {
|
|
45
63
|
this.apiKey = options.apiKey;
|
|
46
64
|
this.keyPrefix = options.apiKey.slice(0, 12);
|
|
@@ -61,6 +79,13 @@ export class BeamTransferControl {
|
|
|
61
79
|
this.closed = true;
|
|
62
80
|
for (const cancel of [...this.terminalWaitCancellations])
|
|
63
81
|
cancel();
|
|
82
|
+
if (this.helloTimer)
|
|
83
|
+
clearInterval(this.helloTimer);
|
|
84
|
+
this.helloTimer = null;
|
|
85
|
+
for (const lease of this.recoveryLeases.values())
|
|
86
|
+
lease.disposeSecrets?.();
|
|
87
|
+
this.recoveryLeases.clear();
|
|
88
|
+
this.recoveryRequestedEpoch.clear();
|
|
64
89
|
if (this.connectionPromise) {
|
|
65
90
|
try {
|
|
66
91
|
await this.connectionPromise;
|
|
@@ -74,9 +99,53 @@ export class BeamTransferControl {
|
|
|
74
99
|
if (nc && !nc.isClosed())
|
|
75
100
|
await nc.drain();
|
|
76
101
|
}
|
|
102
|
+
registerRecoveryLease(lease) {
|
|
103
|
+
if (this.closed)
|
|
104
|
+
throw new Error("NATS lifecycle control is closed");
|
|
105
|
+
const previous = this.recoveryLeases.get(lease.transferId);
|
|
106
|
+
if (previous && previous !== lease)
|
|
107
|
+
previous.disposeSecrets?.();
|
|
108
|
+
this.recoveryLeases.set(lease.transferId, lease);
|
|
109
|
+
this.ensureHelloMonitor();
|
|
110
|
+
const shardId = transferShardId(lease.transferId, this.shardCount);
|
|
111
|
+
void this.pollRuntimeHello(shardId);
|
|
112
|
+
return () => {
|
|
113
|
+
const current = this.recoveryLeases.get(lease.transferId);
|
|
114
|
+
if (current !== lease)
|
|
115
|
+
return;
|
|
116
|
+
this.recoveryLeases.delete(lease.transferId);
|
|
117
|
+
this.recoveryRequestedEpoch.delete(lease.transferId);
|
|
118
|
+
lease.disposeSecrets?.();
|
|
119
|
+
if (!this.recoveryLeases.size && this.helloTimer) {
|
|
120
|
+
clearInterval(this.helloTimer);
|
|
121
|
+
this.helloTimer = null;
|
|
122
|
+
}
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
releaseRecoveryLease(transferId) {
|
|
126
|
+
const lease = this.recoveryLeases.get(transferId);
|
|
127
|
+
if (!lease)
|
|
128
|
+
return;
|
|
129
|
+
this.recoveryLeases.delete(transferId);
|
|
130
|
+
this.recoveryRequestedEpoch.delete(transferId);
|
|
131
|
+
lease.disposeSecrets?.();
|
|
132
|
+
if (!this.recoveryLeases.size && this.helloTimer) {
|
|
133
|
+
clearInterval(this.helloTimer);
|
|
134
|
+
this.helloTimer = null;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
continueRecoveryLease(transferId) {
|
|
138
|
+
const lease = this.recoveryLeases.get(transferId);
|
|
139
|
+
if (!lease || this.closed)
|
|
140
|
+
return;
|
|
141
|
+
const shardId = transferShardId(transferId, this.shardCount);
|
|
142
|
+
const epoch = this.runtimeEpochs.get(shardId);
|
|
143
|
+
this.recoveryRequestedEpoch.set(transferId, epoch ? `${epoch.runtimeEpoch}:${epoch.transportEpoch}` : `foreground:${randomId()}`);
|
|
144
|
+
void this.recoverLease(lease);
|
|
145
|
+
}
|
|
77
146
|
async request(messageType, payload, options = {}) {
|
|
78
147
|
const authToken = await this.resolveAuthToken();
|
|
79
|
-
const shardId = options.transferId ? transferShardId(options.transferId, this.shardCount) : 0;
|
|
148
|
+
const shardId = options.shardId ?? (options.transferId ? transferShardId(options.transferId, this.shardCount) : 0);
|
|
80
149
|
const requestId = options.idempotencyKey
|
|
81
150
|
? await stableRequestId(`${messageType}:${options.idempotencyKey}`)
|
|
82
151
|
: randomId();
|
|
@@ -104,10 +173,11 @@ export class BeamTransferControl {
|
|
|
104
173
|
const nc = await this.connection();
|
|
105
174
|
const response = await nc.request(subject, bytes, { timeout: this.requestTimeoutMs });
|
|
106
175
|
const decoded = decode(response.data);
|
|
176
|
+
this.observeRuntimeEpoch(shardId, decoded.runtime_epoch, decoded.transport_epoch);
|
|
107
177
|
if (decoded.ok)
|
|
108
178
|
return (decoded.payload ?? {});
|
|
109
179
|
const body = JSON.stringify(decoded.error ?? {});
|
|
110
|
-
const error = new
|
|
180
|
+
const error = new LifecycleRequestError(decoded.status, `Beam lifecycle request failed with ${decoded.status}: ${body}`);
|
|
111
181
|
if (!isRetryableLifecycleStatus(decoded.status) || attempt + 1 >= LIFECYCLE_REQUEST_MAX_ATTEMPTS) {
|
|
112
182
|
throw error;
|
|
113
183
|
}
|
|
@@ -282,6 +352,96 @@ export class BeamTransferControl {
|
|
|
282
352
|
this.connectionPromise = null;
|
|
283
353
|
}
|
|
284
354
|
}
|
|
355
|
+
ensureHelloMonitor() {
|
|
356
|
+
if (this.helloTimer || this.closed)
|
|
357
|
+
return;
|
|
358
|
+
this.helloTimer = setInterval(() => {
|
|
359
|
+
const shards = new Set([...this.recoveryLeases.keys()].map((transferId) => transferShardId(transferId, this.shardCount)));
|
|
360
|
+
for (const shardId of shards)
|
|
361
|
+
void this.pollRuntimeHello(shardId);
|
|
362
|
+
}, 5_000);
|
|
363
|
+
this.helloTimer.unref?.();
|
|
364
|
+
}
|
|
365
|
+
async pollRuntimeHello(shardId) {
|
|
366
|
+
try {
|
|
367
|
+
await this.request("runtime.hello", {}, { idempotencyKey: `runtime:hello:${shardId}`, shardId });
|
|
368
|
+
}
|
|
369
|
+
catch {
|
|
370
|
+
// Infinite NATS reconnect continues in the background; the next hello reconciles epochs.
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
observeRuntimeEpoch(shardId, runtimeEpoch, transportEpoch) {
|
|
374
|
+
if (!runtimeEpoch || !transportEpoch)
|
|
375
|
+
return;
|
|
376
|
+
const previous = this.runtimeEpochs.get(shardId);
|
|
377
|
+
this.runtimeEpochs.set(shardId, { runtimeEpoch, transportEpoch });
|
|
378
|
+
if (!previous)
|
|
379
|
+
return;
|
|
380
|
+
if (previous.runtimeEpoch === runtimeEpoch && previous.transportEpoch === transportEpoch)
|
|
381
|
+
return;
|
|
382
|
+
if (previous.runtimeEpoch !== runtimeEpoch) {
|
|
383
|
+
this.authToken = null;
|
|
384
|
+
this.authTokenExpiresAt = 0;
|
|
385
|
+
}
|
|
386
|
+
for (const lease of this.recoveryLeases.values()) {
|
|
387
|
+
if (transferShardId(lease.transferId, this.shardCount) === shardId) {
|
|
388
|
+
this.recoveryRequestedEpoch.set(lease.transferId, `${runtimeEpoch}:${transportEpoch}`);
|
|
389
|
+
void this.recoverLease(lease);
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
recoverLease(lease) {
|
|
394
|
+
const existing = this.recoveryOperations.get(lease.transferId);
|
|
395
|
+
if (existing)
|
|
396
|
+
return existing;
|
|
397
|
+
let operation;
|
|
398
|
+
operation = (async () => {
|
|
399
|
+
let attempt = 0;
|
|
400
|
+
while (!this.closed && this.recoveryLeases.get(lease.transferId) === lease) {
|
|
401
|
+
const requestedEpoch = this.recoveryRequestedEpoch.get(lease.transferId);
|
|
402
|
+
const routeGenerationId = randomId();
|
|
403
|
+
try {
|
|
404
|
+
const response = await this.request("transfer.resume", {
|
|
405
|
+
transfer_id: lease.transferId,
|
|
406
|
+
plan_fingerprint: lease.planFingerprint,
|
|
407
|
+
coordinate_checksum: lease.coordinateChecksum,
|
|
408
|
+
route_generation_id: routeGenerationId
|
|
409
|
+
}, { transferId: lease.transferId, idempotencyKey: `transfer:${lease.transferId}:resume:${routeGenerationId}` });
|
|
410
|
+
if (response.recovery === "terminal") {
|
|
411
|
+
this.releaseRecoveryLease(lease.transferId);
|
|
412
|
+
if (this.recoveryOperations.get(lease.transferId) === operation) {
|
|
413
|
+
this.recoveryOperations.delete(lease.transferId);
|
|
414
|
+
}
|
|
415
|
+
return;
|
|
416
|
+
}
|
|
417
|
+
if (response.route_replay_required || response.recovery === "route_replay_required") {
|
|
418
|
+
await lease.replayRoutes(routeGenerationId);
|
|
419
|
+
}
|
|
420
|
+
if (this.recoveryRequestedEpoch.get(lease.transferId) !== requestedEpoch) {
|
|
421
|
+
attempt = 0;
|
|
422
|
+
continue;
|
|
423
|
+
}
|
|
424
|
+
if (this.recoveryOperations.get(lease.transferId) === operation) {
|
|
425
|
+
this.recoveryOperations.delete(lease.transferId);
|
|
426
|
+
}
|
|
427
|
+
return;
|
|
428
|
+
}
|
|
429
|
+
catch (error) {
|
|
430
|
+
if (!isRetryableLifecycleError(error)) {
|
|
431
|
+
this.releaseRecoveryLease(lease.transferId);
|
|
432
|
+
return;
|
|
433
|
+
}
|
|
434
|
+
attempt += 1;
|
|
435
|
+
await sleepWithJitter(Math.min(30_000, 500 * (2 ** Math.min(attempt, 6))));
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
})().finally(() => {
|
|
439
|
+
if (this.recoveryOperations.get(lease.transferId) === operation)
|
|
440
|
+
this.recoveryOperations.delete(lease.transferId);
|
|
441
|
+
});
|
|
442
|
+
this.recoveryOperations.set(lease.transferId, operation);
|
|
443
|
+
return operation;
|
|
444
|
+
}
|
|
285
445
|
async resolveAuthToken() {
|
|
286
446
|
const now = Math.floor(Date.now() / 1000);
|
|
287
447
|
if (this.authToken && this.authTokenExpiresAt - 5 > now)
|
|
@@ -456,14 +616,22 @@ function randomId() {
|
|
|
456
616
|
const cryptoApi = globalThis.crypto;
|
|
457
617
|
if (cryptoApi?.randomUUID)
|
|
458
618
|
return cryptoApi.randomUUID();
|
|
459
|
-
|
|
619
|
+
if (!cryptoApi?.getRandomValues)
|
|
620
|
+
throw new Error("secure UUID generation is unavailable");
|
|
621
|
+
const bytes = cryptoApi.getRandomValues(new Uint8Array(16));
|
|
622
|
+
bytes[6] = (bytes[6] & 0x0f) | 0x40;
|
|
623
|
+
bytes[8] = (bytes[8] & 0x3f) | 0x80;
|
|
624
|
+
const hex = [...bytes].map((value) => value.toString(16).padStart(2, "0")).join("");
|
|
625
|
+
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
|
|
460
626
|
}
|
|
461
627
|
const LIFECYCLE_REQUEST_MAX_ATTEMPTS = 3;
|
|
462
628
|
const LIFECYCLE_REQUEST_RETRY_DELAYS_MS = [150, 500];
|
|
463
629
|
function isRetryableLifecycleStatus(status) {
|
|
464
630
|
return status === 408 || status === 425 || status === 429 || status >= 500;
|
|
465
631
|
}
|
|
466
|
-
function isRetryableLifecycleError(error) {
|
|
632
|
+
export function isRetryableLifecycleError(error) {
|
|
633
|
+
if (error instanceof LifecycleRequestError)
|
|
634
|
+
return isRetryableLifecycleStatus(error.status);
|
|
467
635
|
const record = error && typeof error === "object" ? error : null;
|
|
468
636
|
const code = typeof record?.code === "string" ? record.code : "";
|
|
469
637
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -476,10 +644,19 @@ function isRetryableLifecycleError(error) {
|
|
|
476
644
|
"econnreset",
|
|
477
645
|
"econnrefused",
|
|
478
646
|
"etimedout",
|
|
647
|
+
"fetch failed",
|
|
479
648
|
"socket",
|
|
480
649
|
"network"
|
|
481
650
|
].some((token) => combined.includes(token));
|
|
482
651
|
}
|
|
652
|
+
export function isRecoverableRouteStreamError(error) {
|
|
653
|
+
if (isRetryableLifecycleError(error))
|
|
654
|
+
return true;
|
|
655
|
+
if (error instanceof LifecycleRequestError)
|
|
656
|
+
return error.status === 404 || error.status === 409;
|
|
657
|
+
const status = error && typeof error === "object" ? Number(error.status) : NaN;
|
|
658
|
+
return status === 404 || status === 409;
|
|
659
|
+
}
|
|
483
660
|
async function sleepWithJitter(baseMs) {
|
|
484
661
|
const jitter = Math.floor(baseMs * (Math.random() * 0.4 - 0.2));
|
|
485
662
|
await new Promise((resolve) => setTimeout(resolve, Math.max(1, baseMs + jitter)));
|
package/dist/provider-signing.js
CHANGED
|
@@ -405,8 +405,7 @@ function s3CompatibleMetadata(source, endpoint = s3CompatibleEndpoint(source)) {
|
|
|
405
405
|
key: source.key,
|
|
406
406
|
region: s3CompatibleRegion(source),
|
|
407
407
|
endpoint_url: endpoint,
|
|
408
|
-
account_id: "account_id" in source ? source.account_id : undefined
|
|
409
|
-
capabilities: "capabilities" in source ? source.capabilities : undefined
|
|
408
|
+
account_id: "account_id" in source ? source.account_id : undefined
|
|
410
409
|
});
|
|
411
410
|
}
|
|
412
411
|
function isHippiusProvider(config) {
|