@beam-network/sdk 0.5.21 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -4
- package/dist/client.d.ts +6 -1
- package/dist/client.js +183 -71
- package/dist/models.d.ts +46 -3
- package/dist/multipart-limits.d.ts +1 -1
- package/dist/multipart-limits.js +3 -3
- package/dist/nats-control.d.ts +21 -4
- package/dist/nats-control.js +106 -31
- package/dist/performance.d.ts +12 -0
- package/dist/performance.js +25 -0
- package/dist/provider-signing.d.ts +29 -0
- package/dist/provider-signing.js +105 -9
- package/package.json +1 -1
package/dist/models.d.ts
CHANGED
|
@@ -1,5 +1,21 @@
|
|
|
1
1
|
export type SignedUrlFlow = "signed_url";
|
|
2
|
+
export interface SdkPerformanceSummary {
|
|
3
|
+
schema_version: "sdk-performance/v1";
|
|
4
|
+
measurements: Array<{
|
|
5
|
+
name: "sdk.discovery" | "sdk.multipart_create" | "sdk.signing" | "sdk.batch_ack" | "sdk.buffer_wait" | "sdk.preparation";
|
|
6
|
+
count: number;
|
|
7
|
+
work_ms: number;
|
|
8
|
+
max_ms: number;
|
|
9
|
+
}>;
|
|
10
|
+
counters: {
|
|
11
|
+
source_signatures: number;
|
|
12
|
+
source_reuses: number;
|
|
13
|
+
route_batches: number;
|
|
14
|
+
};
|
|
15
|
+
}
|
|
2
16
|
export interface BeamClientOptions {
|
|
17
|
+
/** Best effort, at most one bounded summary per route stream. Never awaited. */
|
|
18
|
+
onDiagnostics?: (summary: SdkPerformanceSummary) => void | Promise<void>;
|
|
3
19
|
apiKey?: string;
|
|
4
20
|
natsUrl?: string;
|
|
5
21
|
natsWsUrl?: string;
|
|
@@ -141,7 +157,34 @@ export interface IntegrityAuditChallenge {
|
|
|
141
157
|
range_bytes: number;
|
|
142
158
|
chunks: IntegrityAuditChallengeChunk[];
|
|
143
159
|
}
|
|
160
|
+
export interface TransferPerformance {
|
|
161
|
+
schema_version: "transfer-performance/v1";
|
|
162
|
+
runtime_epoch: string;
|
|
163
|
+
coverage: "complete" | "restarted";
|
|
164
|
+
elapsed_ms: number;
|
|
165
|
+
/** Work can overlap; these durations must not be summed into elapsed time. */
|
|
166
|
+
measurements: Record<string, {
|
|
167
|
+
count: number;
|
|
168
|
+
work_ms: number;
|
|
169
|
+
max_ms: number;
|
|
170
|
+
}>;
|
|
171
|
+
events: Record<string, {
|
|
172
|
+
count: number;
|
|
173
|
+
first_ms: number;
|
|
174
|
+
last_ms: number;
|
|
175
|
+
}>;
|
|
176
|
+
sdk_counters: {
|
|
177
|
+
source_signatures: number;
|
|
178
|
+
source_reuses: number;
|
|
179
|
+
route_batches: number;
|
|
180
|
+
};
|
|
181
|
+
detail_dropped: number;
|
|
182
|
+
unmeasured?: string[];
|
|
183
|
+
measurement_clocks?: Record<string, "runtime" | "sdk" | "fraud-service">;
|
|
184
|
+
sdk_report_received?: boolean;
|
|
185
|
+
}
|
|
144
186
|
export interface TransferStatusInfo {
|
|
187
|
+
performance?: TransferPerformance | null;
|
|
145
188
|
transfer_id: string;
|
|
146
189
|
name?: string;
|
|
147
190
|
status: string;
|
|
@@ -179,7 +222,7 @@ export interface ProviderTransferResumeInput extends Omit<ProviderTransferCreate
|
|
|
179
222
|
multipartGroups: ProviderMultipartGroupIdentity[];
|
|
180
223
|
}
|
|
181
224
|
export interface TransferTerminalEvent {
|
|
182
|
-
schema_version: "transfer-client-control/
|
|
225
|
+
schema_version: "transfer-client-control/v7";
|
|
183
226
|
producer: "transfer-runtime";
|
|
184
227
|
transfer_id: string;
|
|
185
228
|
status: "completed" | "failed" | "cancelled";
|
|
@@ -377,11 +420,11 @@ export interface CompactTransferPlanDescriptor {
|
|
|
377
420
|
destinations: CompactTransferPlanDestination[];
|
|
378
421
|
logical_chunk_count: number;
|
|
379
422
|
delivery_route_count: number;
|
|
380
|
-
multipart_attempt_slots:
|
|
423
|
+
multipart_attempt_slots: 1;
|
|
381
424
|
formulas: {
|
|
382
425
|
source_offset: "source_chunk_index * chunk_size";
|
|
383
426
|
delivery_index: "chunk_index * destination_count + destination_index";
|
|
384
|
-
part_number: "source_chunk_index
|
|
427
|
+
part_number: "source_chunk_index + 1";
|
|
385
428
|
route_generation_id: "initial-{chunk_index}-{destination_id}";
|
|
386
429
|
};
|
|
387
430
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
export declare const MULTIPART_MAX_PART_NUMBER = 10000;
|
|
2
|
-
export declare const MULTIPART_ATTEMPT_SLOT_COUNT =
|
|
2
|
+
export declare const MULTIPART_ATTEMPT_SLOT_COUNT = 1;
|
|
3
3
|
export declare const MULTIPART_MAX_SOURCE_CHUNKS: number;
|
|
4
4
|
export declare function multipartPartNumber(chunkIndex: number, attemptSlot?: number): number;
|
package/dist/multipart-limits.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export const MULTIPART_MAX_PART_NUMBER = 10_000;
|
|
2
|
-
export const MULTIPART_ATTEMPT_SLOT_COUNT =
|
|
2
|
+
export const MULTIPART_ATTEMPT_SLOT_COUNT = 1;
|
|
3
3
|
export const MULTIPART_MAX_SOURCE_CHUNKS = Math.floor(MULTIPART_MAX_PART_NUMBER / MULTIPART_ATTEMPT_SLOT_COUNT);
|
|
4
4
|
export function multipartPartNumber(chunkIndex, attemptSlot = 0) {
|
|
5
5
|
if (!Number.isInteger(chunkIndex) || chunkIndex < 0) {
|
|
@@ -9,8 +9,8 @@ export function multipartPartNumber(chunkIndex, attemptSlot = 0) {
|
|
|
9
9
|
throw new Error(`source_chunk_index must be less than ${MULTIPART_MAX_SOURCE_CHUNKS}`);
|
|
10
10
|
}
|
|
11
11
|
if (!Number.isInteger(attemptSlot) || attemptSlot < 0 || attemptSlot >= MULTIPART_ATTEMPT_SLOT_COUNT) {
|
|
12
|
-
throw new Error("attempt_slot must be 0
|
|
12
|
+
throw new Error("attempt_slot must be 0 for consecutive multipart uploads");
|
|
13
13
|
}
|
|
14
|
-
const partNumber = chunkIndex
|
|
14
|
+
const partNumber = chunkIndex + 1;
|
|
15
15
|
return partNumber;
|
|
16
16
|
}
|
package/dist/nats-control.d.ts
CHANGED
|
@@ -1,12 +1,21 @@
|
|
|
1
|
+
import type { IntegrityAuditChallenge } from "./models.js";
|
|
1
2
|
import { type ConnectionOptions } from "nats";
|
|
2
3
|
import type { MultipartGroupManifest, SignedChunkRoute, TransferTerminalSignalWaiter } from "./models.js";
|
|
3
|
-
export declare const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/
|
|
4
|
+
export declare const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/v7";
|
|
4
5
|
export declare const BEAM_DEFAULT_NATS_URL = "tls://orch-gateway.b1m.ai:4222";
|
|
5
6
|
export declare const BEAM_DEFAULT_NATS_WS_URL = "wss://orch-gateway.b1m.ai:443";
|
|
6
7
|
export declare const BEAM_DEFAULT_MAX_PAYLOAD_BYTES: number;
|
|
7
8
|
export declare const BEAM_ROUTE_TARGET_PAYLOAD_BYTES: number;
|
|
8
9
|
export declare const ROUTE_RECOVERY_SIGN_MESSAGE_TYPE: "transfer.route_recovery.sign";
|
|
9
10
|
export interface RouteRecoverySignChunk {
|
|
11
|
+
recovery?: {
|
|
12
|
+
operation: "upload" | "controls" | "list" | "delete" | "renew";
|
|
13
|
+
mode: "staged" | "direct";
|
|
14
|
+
attempt_id: string;
|
|
15
|
+
object_key?: string;
|
|
16
|
+
etag?: string;
|
|
17
|
+
continuation_token?: string;
|
|
18
|
+
};
|
|
10
19
|
source_id: string;
|
|
11
20
|
destination_id: string;
|
|
12
21
|
chunk_index: number;
|
|
@@ -14,7 +23,7 @@ export interface RouteRecoverySignChunk {
|
|
|
14
23
|
source_offset: number;
|
|
15
24
|
chunk_size: number;
|
|
16
25
|
logical_attempt_index: number;
|
|
17
|
-
attempt_slot: 0
|
|
26
|
+
attempt_slot: 0;
|
|
18
27
|
part_number: number;
|
|
19
28
|
route_generation_id: string;
|
|
20
29
|
multipart_group_id: string;
|
|
@@ -129,10 +138,18 @@ export declare class BeamTransferControl {
|
|
|
129
138
|
private helloTimer;
|
|
130
139
|
constructor(options: TransferControlOptions);
|
|
131
140
|
close(): Promise<void>;
|
|
141
|
+
serveIntegritySigner(transferId: string, handler: (challenge: IntegrityAuditChallenge) => Promise<Record<string, unknown>>): Promise<() => void>;
|
|
132
142
|
serveRouteRecoverySigner(transferId: string, handler: (payload: RouteRecoverySignRequestPayload) => Promise<RouteRecoverySignReplyPayload>): Promise<() => void>;
|
|
133
143
|
registerRecoveryLease(lease: TransferRecoveryLease): () => void;
|
|
134
|
-
|
|
135
|
-
|
|
144
|
+
/**
|
|
145
|
+
* Release a transfer's recovery lease, disposing its retained secrets.
|
|
146
|
+
*
|
|
147
|
+
* With `owner`, release only when that exact lease is still registered, so a
|
|
148
|
+
* fenced-off owner can never release a replacement owner's lease.
|
|
149
|
+
*/
|
|
150
|
+
releaseRecoveryLease(transferId: string, owner?: TransferRecoveryLease): void;
|
|
151
|
+
/** Request recovery for the registered lease; with `owner`, only if that lease is still registered. */
|
|
152
|
+
continueRecoveryLease(transferId: string, owner?: TransferRecoveryLease): void;
|
|
136
153
|
request<T>(messageType: TransferClientMessageType, payload: Record<string, unknown>, options?: LifecycleRequestOptions): Promise<T>;
|
|
137
154
|
openTerminalSignalWaiter(transferId: string): Promise<TransferTerminalSignalWaiter>;
|
|
138
155
|
splitRoutesForPayload(messageType: TransferClientMessageType, basePayload: Record<string, unknown>, routes: SignedChunkRoute[]): SignedChunkRoute[][];
|
package/dist/nats-control.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { encode, decode } from "@msgpack/msgpack";
|
|
2
2
|
import { connect } from "nats";
|
|
3
|
-
export const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/
|
|
3
|
+
export const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/v7";
|
|
4
4
|
export const BEAM_DEFAULT_NATS_URL = "tls://orch-gateway.b1m.ai:4222";
|
|
5
5
|
export const BEAM_DEFAULT_NATS_WS_URL = "wss://orch-gateway.b1m.ai:443";
|
|
6
6
|
// NATS enforces max_payload per message. This guard splits signed-route control
|
|
@@ -60,6 +60,8 @@ export class BeamTransferControl {
|
|
|
60
60
|
terminalWaitCancellations = new Set();
|
|
61
61
|
routeRecoverySignerCancellations = new Set();
|
|
62
62
|
recoveryLeases = new Map();
|
|
63
|
+
// Keyed by lease identity: a replacement owner's recovery must never be
|
|
64
|
+
// coalesced into (and silently dropped behind) a fenced-off owner's loop.
|
|
63
65
|
recoveryOperations = new Map();
|
|
64
66
|
recoveryRequestedEpoch = new Map();
|
|
65
67
|
runtimeEpochs = new Map();
|
|
@@ -106,6 +108,58 @@ export class BeamTransferControl {
|
|
|
106
108
|
if (nc && !nc.isClosed())
|
|
107
109
|
await nc.drain();
|
|
108
110
|
}
|
|
111
|
+
async serveIntegritySigner(transferId, handler) {
|
|
112
|
+
const nc = await this.connection();
|
|
113
|
+
const subject = this.routeRecoverySignSubject(transferId).replace(/route_recovery_sign$/, "integrity_sign");
|
|
114
|
+
let active = true;
|
|
115
|
+
let pending = null;
|
|
116
|
+
let auditId = "";
|
|
117
|
+
let challengeFingerprint = "";
|
|
118
|
+
let awaiting = 0;
|
|
119
|
+
const subscription = nc.subscribe(subject, { callback: (error, message) => {
|
|
120
|
+
if (!active || error || !message.reply)
|
|
121
|
+
return;
|
|
122
|
+
if (awaiting >= 32) {
|
|
123
|
+
message.respond(new TextEncoder().encode('{"retry":true}'));
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
awaiting++;
|
|
127
|
+
void (async () => {
|
|
128
|
+
try {
|
|
129
|
+
const request = JSON.parse(new TextDecoder().decode(message.data));
|
|
130
|
+
if (request.schema_version !== TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION || request.capability !== "integrity-signing/v1"
|
|
131
|
+
|| request.producer !== "transfer-runtime" || request.environment !== this.environment || request.key_prefix !== this.keyPrefix
|
|
132
|
+
|| request.transfer_id !== transferId || request.challenge?.transfer_id !== transferId || typeof request.request_id !== "string")
|
|
133
|
+
throw new Error("integrity_sign_scope_mismatch");
|
|
134
|
+
const fingerprint = JSON.stringify(request.challenge);
|
|
135
|
+
if (pending && (auditId !== request.challenge.audit_id || challengeFingerprint !== fingerprint))
|
|
136
|
+
throw new Error("integrity_sign_conflicting_audit");
|
|
137
|
+
if (!pending) {
|
|
138
|
+
auditId = request.challenge.audit_id;
|
|
139
|
+
challengeFingerprint = fingerprint;
|
|
140
|
+
pending = handler(request.challenge).finally(() => { pending = null; });
|
|
141
|
+
}
|
|
142
|
+
const payload = await pending;
|
|
143
|
+
if (active)
|
|
144
|
+
message.respond(new TextEncoder().encode(JSON.stringify({ schema_version: TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION,
|
|
145
|
+
capability: "integrity-signing/v1", environment: this.environment, key_prefix: this.keyPrefix,
|
|
146
|
+
transfer_id: transferId, request_id: request.request_id, payload })));
|
|
147
|
+
}
|
|
148
|
+
catch {
|
|
149
|
+
message.respond(new TextEncoder().encode('{"retry":true}'));
|
|
150
|
+
}
|
|
151
|
+
finally {
|
|
152
|
+
awaiting--;
|
|
153
|
+
}
|
|
154
|
+
})();
|
|
155
|
+
} });
|
|
156
|
+
await nc.flush();
|
|
157
|
+
const stop = () => { if (!active)
|
|
158
|
+
return; active = false; subscription.unsubscribe(); this.routeRecoverySignerCancellations.delete(stop); };
|
|
159
|
+
this.routeRecoverySignerCancellations.add(stop);
|
|
160
|
+
void this.pollRuntimeHello(transferShardId(transferId, this.shardCount));
|
|
161
|
+
return stop;
|
|
162
|
+
}
|
|
109
163
|
async serveRouteRecoverySigner(transferId, handler) {
|
|
110
164
|
const nc = await this.connection();
|
|
111
165
|
const subject = this.routeRecoverySignSubject(transferId);
|
|
@@ -165,23 +219,20 @@ export class BeamTransferControl {
|
|
|
165
219
|
this.ensureHelloMonitor();
|
|
166
220
|
const shardId = transferShardId(lease.transferId, this.shardCount);
|
|
167
221
|
void this.pollRuntimeHello(shardId);
|
|
168
|
-
return () =>
|
|
169
|
-
const current = this.recoveryLeases.get(lease.transferId);
|
|
170
|
-
if (current !== lease)
|
|
171
|
-
return;
|
|
172
|
-
this.recoveryLeases.delete(lease.transferId);
|
|
173
|
-
this.recoveryRequestedEpoch.delete(lease.transferId);
|
|
174
|
-
lease.disposeSecrets?.();
|
|
175
|
-
if (!this.recoveryLeases.size && this.helloTimer) {
|
|
176
|
-
clearInterval(this.helloTimer);
|
|
177
|
-
this.helloTimer = null;
|
|
178
|
-
}
|
|
179
|
-
};
|
|
222
|
+
return () => this.releaseRecoveryLease(lease.transferId, lease);
|
|
180
223
|
}
|
|
181
|
-
|
|
224
|
+
/**
|
|
225
|
+
* Release a transfer's recovery lease, disposing its retained secrets.
|
|
226
|
+
*
|
|
227
|
+
* With `owner`, release only when that exact lease is still registered, so a
|
|
228
|
+
* fenced-off owner can never release a replacement owner's lease.
|
|
229
|
+
*/
|
|
230
|
+
releaseRecoveryLease(transferId, owner) {
|
|
182
231
|
const lease = this.recoveryLeases.get(transferId);
|
|
183
232
|
if (!lease)
|
|
184
233
|
return;
|
|
234
|
+
if (owner && lease !== owner)
|
|
235
|
+
return;
|
|
185
236
|
this.recoveryLeases.delete(transferId);
|
|
186
237
|
this.recoveryRequestedEpoch.delete(transferId);
|
|
187
238
|
lease.disposeSecrets?.();
|
|
@@ -190,10 +241,13 @@ export class BeamTransferControl {
|
|
|
190
241
|
this.helloTimer = null;
|
|
191
242
|
}
|
|
192
243
|
}
|
|
193
|
-
|
|
244
|
+
/** Request recovery for the registered lease; with `owner`, only if that lease is still registered. */
|
|
245
|
+
continueRecoveryLease(transferId, owner) {
|
|
194
246
|
const lease = this.recoveryLeases.get(transferId);
|
|
195
247
|
if (!lease || this.closed)
|
|
196
248
|
return;
|
|
249
|
+
if (owner && lease !== owner)
|
|
250
|
+
return;
|
|
197
251
|
const shardId = transferShardId(transferId, this.shardCount);
|
|
198
252
|
const epoch = this.runtimeEpochs.get(shardId);
|
|
199
253
|
this.recoveryRequestedEpoch.set(transferId, epoch ? `${epoch.runtimeEpoch}:${epoch.transportEpoch}` : `foreground:${randomId()}`);
|
|
@@ -436,7 +490,12 @@ export class BeamTransferControl {
|
|
|
436
490
|
}
|
|
437
491
|
async pollRuntimeHello(shardId) {
|
|
438
492
|
try {
|
|
439
|
-
|
|
493
|
+
try {
|
|
494
|
+
await this.request("runtime.hello", { capabilities: ["integrity-signing/v1"] }, { idempotencyKey: `runtime:hello:${shardId}:integrity-signing`, shardId });
|
|
495
|
+
}
|
|
496
|
+
catch {
|
|
497
|
+
await this.request("runtime.hello", {}, { idempotencyKey: `runtime:hello:${shardId}`, shardId });
|
|
498
|
+
}
|
|
440
499
|
}
|
|
441
500
|
catch {
|
|
442
501
|
// Infinite NATS reconnect continues in the background; the next hello reconciles epochs.
|
|
@@ -463,13 +522,14 @@ export class BeamTransferControl {
|
|
|
463
522
|
}
|
|
464
523
|
}
|
|
465
524
|
recoverLease(lease) {
|
|
466
|
-
const existing = this.recoveryOperations.get(lease
|
|
525
|
+
const existing = this.recoveryOperations.get(lease);
|
|
467
526
|
if (existing)
|
|
468
527
|
return existing;
|
|
469
|
-
|
|
470
|
-
|
|
528
|
+
const owns = () => !this.closed && this.recoveryLeases.get(lease.transferId) === lease;
|
|
529
|
+
let replaced = false;
|
|
530
|
+
const operation = (async () => {
|
|
471
531
|
let attempt = 0;
|
|
472
|
-
while (
|
|
532
|
+
while (owns()) {
|
|
473
533
|
const requestedEpoch = this.recoveryRequestedEpoch.get(lease.transferId);
|
|
474
534
|
const routeGenerationId = randomId();
|
|
475
535
|
try {
|
|
@@ -479,39 +539,53 @@ export class BeamTransferControl {
|
|
|
479
539
|
coordinate_checksum: lease.coordinateChecksum,
|
|
480
540
|
route_generation_id: routeGenerationId
|
|
481
541
|
}, { transferId: lease.transferId, idempotencyKey: `transfer:${lease.transferId}:resume:${routeGenerationId}` });
|
|
542
|
+
if (!owns()) {
|
|
543
|
+
// A fenced-off owner must neither act on nor release a replacement's lease.
|
|
544
|
+
replaced = true;
|
|
545
|
+
return;
|
|
546
|
+
}
|
|
482
547
|
if (response.recovery === "terminal") {
|
|
483
|
-
this.releaseRecoveryLease(lease.transferId);
|
|
484
|
-
if (this.recoveryOperations.get(lease.transferId) === operation) {
|
|
485
|
-
this.recoveryOperations.delete(lease.transferId);
|
|
486
|
-
}
|
|
548
|
+
this.releaseRecoveryLease(lease.transferId, lease);
|
|
487
549
|
return;
|
|
488
550
|
}
|
|
489
551
|
if (response.route_replay_required || response.recovery === "route_replay_required") {
|
|
490
552
|
await lease.replayRoutes(routeGenerationId);
|
|
553
|
+
if (!owns()) {
|
|
554
|
+
replaced = true;
|
|
555
|
+
return;
|
|
556
|
+
}
|
|
491
557
|
}
|
|
492
558
|
if (this.recoveryRequestedEpoch.get(lease.transferId) !== requestedEpoch) {
|
|
493
559
|
attempt = 0;
|
|
494
560
|
continue;
|
|
495
561
|
}
|
|
496
|
-
if (this.recoveryOperations.get(lease.transferId) === operation) {
|
|
497
|
-
this.recoveryOperations.delete(lease.transferId);
|
|
498
|
-
}
|
|
499
562
|
return;
|
|
500
563
|
}
|
|
501
564
|
catch (error) {
|
|
565
|
+
if (!owns()) {
|
|
566
|
+
replaced = true;
|
|
567
|
+
return;
|
|
568
|
+
}
|
|
502
569
|
if (!isRetryableLifecycleError(error)) {
|
|
503
|
-
this.releaseRecoveryLease(lease.transferId);
|
|
570
|
+
this.releaseRecoveryLease(lease.transferId, lease);
|
|
504
571
|
return;
|
|
505
572
|
}
|
|
506
573
|
attempt += 1;
|
|
507
574
|
await sleepWithJitter(Math.min(30_000, 500 * (2 ** Math.min(attempt, 6))));
|
|
508
575
|
}
|
|
509
576
|
}
|
|
577
|
+
replaced = !this.closed;
|
|
510
578
|
})().finally(() => {
|
|
511
|
-
if (this.recoveryOperations.get(lease
|
|
512
|
-
this.recoveryOperations.delete(lease
|
|
579
|
+
if (this.recoveryOperations.get(lease) === operation)
|
|
580
|
+
this.recoveryOperations.delete(lease);
|
|
581
|
+
if (!replaced || this.closed)
|
|
582
|
+
return;
|
|
583
|
+
// The recovery this loop was driving still matters: hand it to the replacement owner.
|
|
584
|
+
const current = this.recoveryLeases.get(lease.transferId);
|
|
585
|
+
if (current && current !== lease)
|
|
586
|
+
void this.recoverLease(current);
|
|
513
587
|
});
|
|
514
|
-
this.recoveryOperations.set(lease
|
|
588
|
+
this.recoveryOperations.set(lease, operation);
|
|
515
589
|
return operation;
|
|
516
590
|
}
|
|
517
591
|
async resolveAuthToken() {
|
|
@@ -604,6 +678,7 @@ const ROUTE_ATTEMPT_METADATA_KEYS = new Set([
|
|
|
604
678
|
"part_number",
|
|
605
679
|
"logical_attempt_index",
|
|
606
680
|
"attempt_slot",
|
|
681
|
+
"etag_required",
|
|
607
682
|
"route_generation_id"
|
|
608
683
|
]);
|
|
609
684
|
const ROUTE_IDENTITY_METADATA_KEYS = [
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { SdkPerformanceSummary } from './models.js';
|
|
2
|
+
export declare class SdkPerformanceCollector {
|
|
3
|
+
private readonly measurements;
|
|
4
|
+
readonly counters: {
|
|
5
|
+
source_signatures: number;
|
|
6
|
+
source_reuses: number;
|
|
7
|
+
route_batches: number;
|
|
8
|
+
};
|
|
9
|
+
observe(name: SdkPerformanceSummary['measurements'][number]['name'], milliseconds: number): void;
|
|
10
|
+
measure<T>(name: SdkPerformanceSummary['measurements'][number]['name'], work: () => Promise<T>): Promise<T>;
|
|
11
|
+
snapshot(): SdkPerformanceSummary;
|
|
12
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
export class SdkPerformanceCollector {
|
|
2
|
+
measurements = new Map();
|
|
3
|
+
counters = { source_signatures: 0, source_reuses: 0, route_batches: 0 };
|
|
4
|
+
observe(name, milliseconds) {
|
|
5
|
+
if (!Number.isFinite(milliseconds) || milliseconds < 0)
|
|
6
|
+
return;
|
|
7
|
+
const value = this.measurements.get(name) ?? { count: 0, work_ms: 0, max_ms: 0 };
|
|
8
|
+
value.count++;
|
|
9
|
+
value.work_ms += milliseconds;
|
|
10
|
+
value.max_ms = Math.max(value.max_ms, milliseconds);
|
|
11
|
+
this.measurements.set(name, value);
|
|
12
|
+
}
|
|
13
|
+
async measure(name, work) {
|
|
14
|
+
const start = performance.now();
|
|
15
|
+
try {
|
|
16
|
+
return await work();
|
|
17
|
+
}
|
|
18
|
+
finally {
|
|
19
|
+
this.observe(name, performance.now() - start);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
snapshot() {
|
|
23
|
+
return { schema_version: 'sdk-performance/v1', measurements: [...this.measurements].map(([name, value]) => ({ name: name, ...value })), counters: { ...this.counters } };
|
|
24
|
+
}
|
|
25
|
+
}
|
|
@@ -1,5 +1,14 @@
|
|
|
1
|
+
import type { RouteRecoverySignChunk } from "./nats-control.js";
|
|
1
2
|
import type { ChunkDestinationSigningTarget, ChunkSigningPlanItem, HuggingFaceProviderConfig, PlanningHttpSource, PreparedDestination, PreparedHttpSource, ProviderDestinationConfig, ProviderSourceConfig, R2ProviderConfig, S3CompatibleProviderConfig, S3ProviderConfig, SignedChunkRoute } from "./models.js";
|
|
2
3
|
export type AnyS3CompatibleProviderConfig = S3ProviderConfig | R2ProviderConfig | S3CompatibleProviderConfig;
|
|
4
|
+
/** Core-only grants: the worker receives just the ordinary upload URL. */
|
|
5
|
+
export declare function signMultipartRecovery(input: {
|
|
6
|
+
destination: ProviderDestinationConfig;
|
|
7
|
+
transferId: string;
|
|
8
|
+
requested: RouteRecoverySignChunk;
|
|
9
|
+
route: SignedChunkRoute;
|
|
10
|
+
expiresIn: number;
|
|
11
|
+
}): Promise<SignedChunkRoute>;
|
|
3
12
|
export declare function nowIso(): string;
|
|
4
13
|
export declare function expiresAtIso(expiresIn: number): string;
|
|
5
14
|
export declare function prepareProviderSource(source: ProviderSourceConfig, options?: {
|
|
@@ -54,7 +63,15 @@ export declare function inspectDestinationObject(destination: ProviderDestinatio
|
|
|
54
63
|
versionId: string | undefined;
|
|
55
64
|
metadata: Record<string, string>;
|
|
56
65
|
}>;
|
|
66
|
+
export declare function boundedGrantExpiry(url: string, upperBound: string): string;
|
|
67
|
+
export type SourceRouteGrant = {
|
|
68
|
+
url: string;
|
|
69
|
+
headers: Record<string, string>;
|
|
70
|
+
expiresAt: string;
|
|
71
|
+
};
|
|
72
|
+
export declare function signSourceChunk(input: Parameters<typeof signSourceRoute>[0]): Promise<SourceRouteGrant>;
|
|
57
73
|
export declare function signDestinationRoute(input: {
|
|
74
|
+
sourceGrant?: Promise<SourceRouteGrant>;
|
|
58
75
|
chunk: ChunkSigningPlanItem;
|
|
59
76
|
target: ChunkDestinationSigningTarget;
|
|
60
77
|
source?: ProviderSourceConfig;
|
|
@@ -100,6 +117,16 @@ export declare function signDestinationReadRange(input: {
|
|
|
100
117
|
url: string;
|
|
101
118
|
headers: Record<string, string>;
|
|
102
119
|
}>;
|
|
120
|
+
declare function signSourceRoute(input: {
|
|
121
|
+
source?: ProviderSourceConfig;
|
|
122
|
+
fallbackUrl: string;
|
|
123
|
+
chunk: ChunkSigningPlanItem;
|
|
124
|
+
expiresIn: number;
|
|
125
|
+
fetchImpl?: typeof fetch;
|
|
126
|
+
}): Promise<{
|
|
127
|
+
url: string;
|
|
128
|
+
headers: Record<string, string>;
|
|
129
|
+
}>;
|
|
103
130
|
export declare function signDestinationUrl(input: {
|
|
104
131
|
contentMd5?: string;
|
|
105
132
|
destination: ProviderDestinationConfig;
|
|
@@ -109,8 +136,10 @@ export declare function signDestinationUrl(input: {
|
|
|
109
136
|
expiresIn: number;
|
|
110
137
|
fetchImpl?: typeof fetch;
|
|
111
138
|
}): Promise<string>;
|
|
139
|
+
export declare function releaseProviderClients(configs: Array<ProviderSourceConfig | ProviderDestinationConfig>): void;
|
|
112
140
|
export declare function isS3CompatibleProvider(config: ProviderSourceConfig | ProviderDestinationConfig): config is AnyS3CompatibleProviderConfig;
|
|
113
141
|
export declare function s3CompatibleEndpoint(source: AnyS3CompatibleProviderConfig): string | undefined;
|
|
114
142
|
export declare function s3CompatibleRegion(source: AnyS3CompatibleProviderConfig): string;
|
|
115
143
|
export declare function s3CompatibleForcePathStyle(source: AnyS3CompatibleProviderConfig, endpoint?: string | undefined): boolean | undefined;
|
|
116
144
|
export declare function isHuggingFaceProvider(config: ProviderSourceConfig | ProviderDestinationConfig): config is HuggingFaceProviderConfig;
|
|
145
|
+
export {};
|
package/dist/provider-signing.js
CHANGED
|
@@ -1,11 +1,75 @@
|
|
|
1
1
|
import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, GetObjectCommand, HeadObjectCommand, ListPartsCommand, PutObjectCommand, S3Client, UploadPartCommand } from "@aws-sdk/client-s3";
|
|
2
|
+
import { UploadPartCopyCommand, DeleteObjectCommand, ListObjectsV2Command } from "@aws-sdk/client-s3";
|
|
2
3
|
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
|
3
4
|
import { huggingFaceEndpoint, huggingFaceFileMetadata, huggingFaceRepoType, huggingFaceRevision } from "./huggingface.js";
|
|
5
|
+
/** Core-only grants: the worker receives just the ordinary upload URL. */
|
|
6
|
+
export async function signMultipartRecovery(input) {
|
|
7
|
+
const { requested: r, destination, expiresIn } = input;
|
|
8
|
+
const request = r.recovery;
|
|
9
|
+
if (request?.operation === "renew") {
|
|
10
|
+
const count = Number(input.route.metadata?.expected_part_count);
|
|
11
|
+
if (!Number.isInteger(count) || count < 1 || count > 10_000)
|
|
12
|
+
throw new Error("invalid multipart count");
|
|
13
|
+
const [complete, abort, head, pages] = await Promise.all([
|
|
14
|
+
signCompleteMultipartUpload(destination, r.final_object_key, r.upload_id, expiresIn),
|
|
15
|
+
signAbortMultipartUpload(destination, r.final_object_key, r.upload_id, expiresIn),
|
|
16
|
+
signFinalObjectHead(destination, r.final_object_key, expiresIn),
|
|
17
|
+
Promise.all(Array.from({ length: Math.ceil(count / 1000) }, (_, i) => signListMultipartUpload(destination, r.final_object_key, r.upload_id, expiresIn, { maxParts: 1000, partNumberMarker: i * 1000 })))
|
|
18
|
+
]);
|
|
19
|
+
return { ...input.route, metadata: { ...input.route.metadata, complete_url: complete, abort_url: abort, final_head_url: head,
|
|
20
|
+
list_page_urls: pages, list_page_url: pages[Math.floor((r.part_number - 1) / 1000)], control_urls_expires_at: expiresAtIso(expiresIn) } };
|
|
21
|
+
}
|
|
22
|
+
if (!request || request.mode === "direct") {
|
|
23
|
+
const metadata = { ...input.route.metadata };
|
|
24
|
+
delete metadata.recovery_staging;
|
|
25
|
+
return { ...input.route, metadata };
|
|
26
|
+
}
|
|
27
|
+
if (!isS3CompatibleProvider(destination))
|
|
28
|
+
throw new Error("multipart recovery requires an S3-compatible destination");
|
|
29
|
+
const prefix = `${r.final_object_key}.beam-recovery/${input.transferId}/${encodeURIComponent(r.multipart_group_id).replace(/[!'()*]/g, c => `%${c.charCodeAt(0).toString(16).toUpperCase()}`)}/`;
|
|
30
|
+
const objectKey = `${prefix}${r.part_number}/${request.attempt_id}`;
|
|
31
|
+
if (request.object_key && request.object_key !== objectKey)
|
|
32
|
+
throw new Error("recovery staging identity mismatch");
|
|
33
|
+
const client = createS3CompatibleClient(destination);
|
|
34
|
+
const sign = (command) => {
|
|
35
|
+
if (command instanceof PutObjectCommand)
|
|
36
|
+
return getSignedUrl(client, command, { expiresIn });
|
|
37
|
+
if (command instanceof HeadObjectCommand)
|
|
38
|
+
return getSignedUrl(client, command, { expiresIn });
|
|
39
|
+
if (command instanceof UploadPartCopyCommand)
|
|
40
|
+
return getSignedUrl(client, command, { expiresIn, unhoistableHeaders: new Set(["x-amz-copy-source", "x-amz-copy-source-if-match"]) });
|
|
41
|
+
if (command instanceof DeleteObjectCommand)
|
|
42
|
+
return getSignedUrl(client, command, { expiresIn });
|
|
43
|
+
return getSignedUrl(client, command, { expiresIn });
|
|
44
|
+
};
|
|
45
|
+
const metadata = { ...input.route.metadata };
|
|
46
|
+
delete metadata.recovery_staging;
|
|
47
|
+
if (request.operation === "list") {
|
|
48
|
+
metadata.recovery_listing = { prefix, url: await sign(new ListObjectsV2Command({ Bucket: destination.bucket, Prefix: prefix, ContinuationToken: request.continuation_token, MaxKeys: 1000 })) };
|
|
49
|
+
return { ...input.route, metadata };
|
|
50
|
+
}
|
|
51
|
+
const params = { Bucket: destination.bucket, Key: objectKey };
|
|
52
|
+
if (request.operation === "delete") {
|
|
53
|
+
metadata.recovery_delete = { object_key: objectKey, url: await sign(new DeleteObjectCommand(params)) };
|
|
54
|
+
return { ...input.route, metadata };
|
|
55
|
+
}
|
|
56
|
+
const copySource = `${encodeURIComponent(destination.bucket)}/${objectKey.split("/").map(encodeURIComponent).join("/")}`;
|
|
57
|
+
// R2 does not promise to enforce copy source conditions. Each attempt has its own object.
|
|
58
|
+
const condition = destination.provider === "s3" && request.etag ? `"${request.etag.replace(/^"|"$/g, "")}"` : undefined;
|
|
59
|
+
const [head, copy, remove] = await Promise.all([
|
|
60
|
+
sign(new HeadObjectCommand(params)),
|
|
61
|
+
sign(new UploadPartCopyCommand({ Bucket: destination.bucket, Key: r.final_object_key, UploadId: r.upload_id, PartNumber: r.part_number, CopySource: copySource, CopySourceIfMatch: condition })),
|
|
62
|
+
sign(new DeleteObjectCommand(params))
|
|
63
|
+
]);
|
|
64
|
+
metadata.recovery_staging = { object_key: objectKey, attempt_id: request.attempt_id, head_url: head, copy_url: copy, delete_url: remove,
|
|
65
|
+
copy_headers: { "x-amz-copy-source": copySource, ...(condition ? { "x-amz-copy-source-if-match": condition } : {}) }, expires_at: expiresAtIso(expiresIn) };
|
|
66
|
+
return { ...input.route, metadata, ...(request.operation === "upload" ? { dest_url: await sign(new PutObjectCommand(params)) } : {}) };
|
|
67
|
+
}
|
|
4
68
|
export function nowIso() {
|
|
5
69
|
return new Date().toISOString();
|
|
6
70
|
}
|
|
7
71
|
export function expiresAtIso(expiresIn) {
|
|
8
|
-
return new Date(Date.now() + expiresIn * 1000).toISOString();
|
|
72
|
+
return new Date((Math.floor(Date.now() / 1000) + expiresIn) * 1000).toISOString();
|
|
9
73
|
}
|
|
10
74
|
export async function prepareProviderSource(source, options = {}) {
|
|
11
75
|
const index = options.index ?? 0;
|
|
@@ -243,11 +307,34 @@ export async function inspectDestinationObject(destination, objectKey, signal) {
|
|
|
243
307
|
const result = await createS3CompatibleClient(destination).send(new HeadObjectCommand({ Bucket: destination.bucket, Key: objectKey }), { abortSignal: signal });
|
|
244
308
|
return { size: result.ContentLength, etag: result.ETag, versionId: result.VersionId, metadata: result.Metadata ?? {} };
|
|
245
309
|
}
|
|
310
|
+
// Never advertise a renewed lifetime for a previously issued provider URL.
|
|
311
|
+
export function boundedGrantExpiry(url, upperBound) {
|
|
312
|
+
let expiry = Date.parse(upperBound);
|
|
313
|
+
const query = new URL(url).searchParams;
|
|
314
|
+
for (const prefix of ["X-Amz", "X-Goog"]) {
|
|
315
|
+
const date = query.get(`${prefix}-Date`), duration = Number(query.get(`${prefix}-Expires`));
|
|
316
|
+
if (date && /^\d{8}T\d{6}Z$/.test(date) && Number.isFinite(duration) && duration > 0) {
|
|
317
|
+
const start = Date.parse(`${date.slice(0, 4)}-${date.slice(4, 6)}-${date.slice(6, 8)}T${date.slice(9, 11)}:${date.slice(11, 13)}:${date.slice(13, 15)}Z`);
|
|
318
|
+
if (Number.isFinite(start))
|
|
319
|
+
expiry = Math.min(expiry, start + duration * 1000);
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
const epoch = Number(query.get("Expires"));
|
|
323
|
+
if (Number.isFinite(epoch) && epoch > 0)
|
|
324
|
+
expiry = Math.min(expiry, epoch * 1000);
|
|
325
|
+
return new Date(expiry).toISOString();
|
|
326
|
+
}
|
|
327
|
+
export async function signSourceChunk(input) {
|
|
328
|
+
const expiresAt = expiresAtIso(input.expiresIn);
|
|
329
|
+
const grant = await signSourceRoute(input);
|
|
330
|
+
return { ...grant, expiresAt: boundedGrantExpiry(grant.url, expiresAt) };
|
|
331
|
+
}
|
|
246
332
|
export async function signDestinationRoute(input) {
|
|
247
333
|
const targetObjectKey = input.target.object_key;
|
|
248
334
|
if (!targetObjectKey) {
|
|
249
335
|
throw new Error("destination signing target is missing object_key");
|
|
250
336
|
}
|
|
337
|
+
const destinationExpiresAt = expiresAtIso(input.expiresIn);
|
|
251
338
|
const [destUrl, sourceRoute] = await Promise.all([
|
|
252
339
|
input.destUrl ?? signDestinationUrl({
|
|
253
340
|
destination: input.destination,
|
|
@@ -257,7 +344,7 @@ export async function signDestinationRoute(input) {
|
|
|
257
344
|
expiresIn: input.expiresIn,
|
|
258
345
|
fetchImpl: input.fetchImpl,
|
|
259
346
|
}),
|
|
260
|
-
|
|
347
|
+
input.sourceGrant ?? signSourceChunk({
|
|
261
348
|
source: input.source,
|
|
262
349
|
fallbackUrl: input.chunk.source_url,
|
|
263
350
|
chunk: input.chunk,
|
|
@@ -273,8 +360,8 @@ export async function signDestinationRoute(input) {
|
|
|
273
360
|
dest_url: destUrl,
|
|
274
361
|
source_offset: input.chunk.source_offset,
|
|
275
362
|
chunk_size: input.chunk.chunk_size,
|
|
276
|
-
expires_at:
|
|
277
|
-
headers: sourceRoute.headers,
|
|
363
|
+
expires_at: boundedGrantExpiry(destUrl, new Date(Math.min(Date.parse(destinationExpiresAt), Date.parse(sourceRoute.expiresAt))).toISOString()),
|
|
364
|
+
headers: { ...sourceRoute.headers },
|
|
278
365
|
metadata: {
|
|
279
366
|
...(input.target.metadata ?? {}),
|
|
280
367
|
...(input.multipartGroupId ? { multipart_group_id: input.multipartGroupId } : {}),
|
|
@@ -411,7 +498,16 @@ export async function signDestinationUrl(input) {
|
|
|
411
498
|
}
|
|
412
499
|
return unsupportedProviderConfig(input.destination);
|
|
413
500
|
}
|
|
414
|
-
const s3ClientCache = new
|
|
501
|
+
const s3ClientCache = new WeakMap();
|
|
502
|
+
export function releaseProviderClients(configs) {
|
|
503
|
+
for (const config of configs) {
|
|
504
|
+
if (!isS3CompatibleProvider(config))
|
|
505
|
+
continue;
|
|
506
|
+
const cached = s3ClientCache.get(config);
|
|
507
|
+
s3ClientCache.delete(config);
|
|
508
|
+
cached?.client.destroy();
|
|
509
|
+
}
|
|
510
|
+
}
|
|
415
511
|
function s3ClientCacheKey(source, endpoint) {
|
|
416
512
|
return JSON.stringify({
|
|
417
513
|
provider: source.provider,
|
|
@@ -425,9 +521,9 @@ function s3ClientCacheKey(source, endpoint) {
|
|
|
425
521
|
}
|
|
426
522
|
function createS3CompatibleClient(source, endpoint = s3CompatibleEndpoint(source)) {
|
|
427
523
|
const cacheKey = s3ClientCacheKey(source, endpoint);
|
|
428
|
-
const cached = s3ClientCache.get(
|
|
429
|
-
if (cached)
|
|
430
|
-
return cached;
|
|
524
|
+
const cached = s3ClientCache.get(source);
|
|
525
|
+
if (cached?.identity === cacheKey)
|
|
526
|
+
return cached.client;
|
|
431
527
|
const client = new S3Client({
|
|
432
528
|
region: s3CompatibleRegion(source),
|
|
433
529
|
endpoint,
|
|
@@ -439,7 +535,7 @@ function createS3CompatibleClient(source, endpoint = s3CompatibleEndpoint(source
|
|
|
439
535
|
sessionToken: "session_token" in source ? source.session_token : undefined
|
|
440
536
|
}
|
|
441
537
|
});
|
|
442
|
-
s3ClientCache.set(cacheKey, client);
|
|
538
|
+
s3ClientCache.set(source, { identity: cacheKey, client });
|
|
443
539
|
return client;
|
|
444
540
|
}
|
|
445
541
|
export function isS3CompatibleProvider(config) {
|