@beam-network/sdk 0.2.16 → 0.2.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/dist/client.d.ts +6 -3
- package/dist/client.js +240 -403
- package/dist/models.d.ts +1 -39
- package/dist/nats-control.d.ts +37 -1
- package/dist/nats-control.js +90 -7
- package/dist/provider-signing.d.ts +0 -16
- package/dist/provider-signing.js +0 -301
- package/package.json +36 -36
package/dist/models.d.ts
CHANGED
|
@@ -50,7 +50,6 @@ export interface TransferCreateRequest {
|
|
|
50
50
|
chunk_hashes?: string[];
|
|
51
51
|
callbacks?: CallbackConfig[];
|
|
52
52
|
test_mode?: boolean;
|
|
53
|
-
experimental?: boolean;
|
|
54
53
|
progressive_mode?: boolean;
|
|
55
54
|
signed_url_flow?: SignedUrlFlow;
|
|
56
55
|
}
|
|
@@ -67,10 +66,6 @@ export interface RawTransferCreateInput {
|
|
|
67
66
|
* Maps to BeamCore test_mode.
|
|
68
67
|
*/
|
|
69
68
|
testMode?: boolean;
|
|
70
|
-
/**
|
|
71
|
-
* Experimental transfers keep positive evidence but exclude failures and overseer interventions from PRISM.
|
|
72
|
-
*/
|
|
73
|
-
experimental?: boolean;
|
|
74
69
|
progressiveMode?: boolean;
|
|
75
70
|
signedUrlFlow?: SignedUrlFlow;
|
|
76
71
|
}
|
|
@@ -78,7 +73,6 @@ export interface TransferCreateResponse {
|
|
|
78
73
|
success: boolean;
|
|
79
74
|
transfer_id: string;
|
|
80
75
|
transfer_key?: string;
|
|
81
|
-
experimental?: boolean;
|
|
82
76
|
total_chunks: number;
|
|
83
77
|
total_sources: number;
|
|
84
78
|
total_destinations: number;
|
|
@@ -123,7 +117,6 @@ export interface TransferStatusInfo {
|
|
|
123
117
|
name?: string;
|
|
124
118
|
status: string;
|
|
125
119
|
error_message: string | null;
|
|
126
|
-
experimental?: boolean;
|
|
127
120
|
started_at: string | null;
|
|
128
121
|
completed_at: string | null;
|
|
129
122
|
total_bytes?: number;
|
|
@@ -176,26 +169,6 @@ export interface HippiusProviderConfig {
|
|
|
176
169
|
}
|
|
177
170
|
export type ProviderConfig = S3ProviderConfig | R2ProviderConfig | S3CompatibleProviderConfig | HippiusProviderConfig;
|
|
178
171
|
export type ProviderSourceConfig = ProviderConfig;
|
|
179
|
-
export interface ChunkChecksumProviderGetInput {
|
|
180
|
-
source: ProviderSourceConfig;
|
|
181
|
-
chunk: ChunkSigningPlanItem;
|
|
182
|
-
algorithm: "content-md5";
|
|
183
|
-
}
|
|
184
|
-
export interface ChunkChecksumProviderSetInput extends ChunkChecksumProviderGetInput {
|
|
185
|
-
contentMd5: string;
|
|
186
|
-
}
|
|
187
|
-
export interface ChunkChecksumProvider {
|
|
188
|
-
getContentMd5(input: ChunkChecksumProviderGetInput): Promise<string | null | undefined>;
|
|
189
|
-
setContentMd5?(input: ChunkChecksumProviderSetInput): Promise<void>;
|
|
190
|
-
}
|
|
191
|
-
export interface ChunkChecksumEvent {
|
|
192
|
-
event: "checksum_warmup_started" | "checksum_warmup_completed" | "checksum_cache_hit" | "checksum_cache_miss" | "checksum_cold_chunk_cached" | "checksum_cold_group_completed";
|
|
193
|
-
sourceId: string;
|
|
194
|
-
sourceChunkIndex?: number;
|
|
195
|
-
chunkCount?: number;
|
|
196
|
-
bytes?: number;
|
|
197
|
-
durationMs?: number;
|
|
198
|
-
}
|
|
199
172
|
export type ProviderDestinationConfig = ProviderConfig;
|
|
200
173
|
export type S3ProviderConfigInput = Omit<S3ProviderConfig, "provider">;
|
|
201
174
|
export type R2ProviderConfigInput = Omit<R2ProviderConfig, "provider">;
|
|
@@ -221,25 +194,15 @@ export interface ProviderTransferCreateInput {
|
|
|
221
194
|
* Maps to BeamCore test_mode.
|
|
222
195
|
*/
|
|
223
196
|
testMode?: boolean;
|
|
224
|
-
/**
|
|
225
|
-
* Experimental transfers keep positive evidence but exclude failures and overseer interventions from PRISM.
|
|
226
|
-
*/
|
|
227
|
-
experimental?: boolean;
|
|
228
197
|
expiresIn?: number;
|
|
229
198
|
/**
|
|
230
|
-
* Defaults to true. Set to false to only prepare and
|
|
199
|
+
* Defaults to true. Set to false to only prepare and stream signed routes without distribution.
|
|
231
200
|
*/
|
|
232
201
|
distribute?: boolean;
|
|
233
|
-
onPlanned?: (plan: TransferPlanResponse) => void | Promise<void>;
|
|
234
202
|
onBeforeTransferPrepare?: () => void | Promise<void>;
|
|
235
203
|
onPrepared?: (prepared: TransferPrepareResponse) => void | Promise<void>;
|
|
236
204
|
throwIfCancelled?: (transferId?: string) => void | Promise<void>;
|
|
237
205
|
signedUrlFlow?: SignedUrlFlow;
|
|
238
|
-
checksumProvider?: ChunkChecksumProvider;
|
|
239
|
-
checksumReadConcurrency?: number;
|
|
240
|
-
checksumGroupSizeBytes?: number;
|
|
241
|
-
checksumReadTimeoutMs?: number;
|
|
242
|
-
onChecksumEvent?: (event: ChunkChecksumEvent) => void | Promise<void>;
|
|
243
206
|
}
|
|
244
207
|
export interface PlanningHttpSource {
|
|
245
208
|
source_id: string;
|
|
@@ -308,7 +271,6 @@ export interface TransferPrepareResponse {
|
|
|
308
271
|
transfer_id: string;
|
|
309
272
|
transfer_key?: string;
|
|
310
273
|
test_mode?: boolean;
|
|
311
|
-
experimental?: boolean;
|
|
312
274
|
chunk_size?: number;
|
|
313
275
|
total_size?: number;
|
|
314
276
|
total_sources?: number;
|
package/dist/nats-control.d.ts
CHANGED
|
@@ -1,7 +1,40 @@
|
|
|
1
1
|
export declare const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/v1";
|
|
2
2
|
export declare const BEAM_DEFAULT_NATS_URL = "tls://nats.b1m.ai:4222";
|
|
3
3
|
export declare const BEAM_DEFAULT_NATS_WS_URL = "wss://nats.b1m.ai:443";
|
|
4
|
-
export
|
|
4
|
+
export declare const BEAM_DEFAULT_MAX_PAYLOAD_BYTES: number;
|
|
5
|
+
export declare const ROUTE_RECOVERY_SIGN_MESSAGE_TYPE = "transfer.route_recovery.sign";
|
|
6
|
+
export type TransferClientMessageType = "transfer.plan" | "transfer.create" | "transfer.prepare" | "transfer.route_stream.begin" | "transfer.route_stream.batch" | "transfer.route_stream.complete" | "transfer.distribute" | "transfer.status" | "transfer.cancel";
|
|
7
|
+
export interface RouteRecoverySignChunk {
|
|
8
|
+
source_id: string;
|
|
9
|
+
destination_id: string;
|
|
10
|
+
chunk_index: number;
|
|
11
|
+
delivery_index: number;
|
|
12
|
+
source_offset: number;
|
|
13
|
+
chunk_size: number;
|
|
14
|
+
logical_attempt_index: number;
|
|
15
|
+
attempt_slot: 0 | 1 | 2;
|
|
16
|
+
part_number: number;
|
|
17
|
+
route_generation_id: string;
|
|
18
|
+
multipart_group_id: string;
|
|
19
|
+
final_object_key: string;
|
|
20
|
+
upload_id?: string;
|
|
21
|
+
urls_expires_at?: string;
|
|
22
|
+
multipart_created_at?: string;
|
|
23
|
+
source_metadata?: Record<string, unknown>;
|
|
24
|
+
destination_metadata?: Record<string, unknown>;
|
|
25
|
+
}
|
|
26
|
+
export interface RouteRecoverySignRequestPayload {
|
|
27
|
+
transfer_id: string;
|
|
28
|
+
route_generation_id: string;
|
|
29
|
+
requested_at?: string;
|
|
30
|
+
chunks: RouteRecoverySignChunk[];
|
|
31
|
+
}
|
|
32
|
+
export interface RouteRecoverySignReplyPayload {
|
|
33
|
+
transfer_id: string;
|
|
34
|
+
route_generation_id: string;
|
|
35
|
+
signed_at?: string;
|
|
36
|
+
chunk_routes: unknown[];
|
|
37
|
+
}
|
|
5
38
|
export interface TransferControlOptions {
|
|
6
39
|
apiKey: string;
|
|
7
40
|
natsUrl?: string;
|
|
@@ -25,12 +58,15 @@ export declare class BeamTransferControl {
|
|
|
25
58
|
private authToken;
|
|
26
59
|
private authTokenExpiresAt;
|
|
27
60
|
constructor(options: TransferControlOptions);
|
|
61
|
+
serveRouteRecoverySigner(transferId: string, handler: (payload: RouteRecoverySignRequestPayload) => Promise<RouteRecoverySignReplyPayload>): Promise<() => void>;
|
|
28
62
|
close(): Promise<void>;
|
|
29
63
|
request<T>(messageType: TransferClientMessageType, payload: Record<string, unknown>, transferId?: string): Promise<T>;
|
|
30
64
|
splitRoutesForPayload(messageType: TransferClientMessageType, basePayload: Record<string, unknown>, routes: unknown[]): unknown[][];
|
|
31
65
|
private connection;
|
|
32
66
|
private resolveAuthToken;
|
|
33
67
|
private authSubject;
|
|
68
|
+
private routeRecoverySignSubject;
|
|
69
|
+
private buildRouteRecoveryReply;
|
|
34
70
|
private requestSubject;
|
|
35
71
|
}
|
|
36
72
|
export declare function transferShardId(transferId: string, shardCount: number): number;
|
package/dist/nats-control.js
CHANGED
|
@@ -3,6 +3,29 @@ import { connect } from "nats";
|
|
|
3
3
|
export const TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION = "transfer-client-control/v1";
|
|
4
4
|
export const BEAM_DEFAULT_NATS_URL = "tls://nats.b1m.ai:4222";
|
|
5
5
|
export const BEAM_DEFAULT_NATS_WS_URL = "wss://nats.b1m.ai:443";
|
|
6
|
+
// NATS enforces max_payload per message. This guard splits signed-route control
|
|
7
|
+
// messages before the broker rejects them; transfer bytes never flow through NATS.
|
|
8
|
+
export const BEAM_DEFAULT_MAX_PAYLOAD_BYTES = 24 * 1024 * 1024;
|
|
9
|
+
export const ROUTE_RECOVERY_SIGN_MESSAGE_TYPE = "transfer.route_recovery.sign";
|
|
10
|
+
function buildConnectionOptions(options) {
|
|
11
|
+
const connectionOptions = {
|
|
12
|
+
servers: options.natsUrl,
|
|
13
|
+
user: options.keyPrefix,
|
|
14
|
+
pass: options.apiKey,
|
|
15
|
+
name: `beam-sdk-${options.keyPrefix}`
|
|
16
|
+
};
|
|
17
|
+
if (options.natsUrl.startsWith("tls://")) {
|
|
18
|
+
const tlsOptions = { handshakeFirst: true };
|
|
19
|
+
try {
|
|
20
|
+
tlsOptions.servername = new URL(options.natsUrl).hostname;
|
|
21
|
+
}
|
|
22
|
+
catch {
|
|
23
|
+
// URL validation stays with the NATS client; TLS-first is still required.
|
|
24
|
+
}
|
|
25
|
+
connectionOptions.tls = tlsOptions;
|
|
26
|
+
}
|
|
27
|
+
return connectionOptions;
|
|
28
|
+
}
|
|
6
29
|
export class BeamTransferControl {
|
|
7
30
|
environment;
|
|
8
31
|
keyPrefix;
|
|
@@ -22,7 +45,7 @@ export class BeamTransferControl {
|
|
|
22
45
|
this.subjectPrefix = normalizeSubjectPrefix(options.subjectPrefix ?? "beam.transfer.client");
|
|
23
46
|
this.shardCount = options.transferRuntimeShardCount ?? 1;
|
|
24
47
|
this.requestTimeoutMs = options.requestTimeoutMs ?? 30_000;
|
|
25
|
-
this.maxPayloadBytes = options.maxPayloadBytes ??
|
|
48
|
+
this.maxPayloadBytes = options.maxPayloadBytes ?? BEAM_DEFAULT_MAX_PAYLOAD_BYTES;
|
|
26
49
|
this.natsUrl = options.natsUrl ?? options.natsWsUrl ?? BEAM_DEFAULT_NATS_URL;
|
|
27
50
|
if (/^https?:\/\//i.test(this.natsUrl) || /^wss?:\/\//i.test(this.natsUrl) && !options.natsWsUrl) {
|
|
28
51
|
throw new Error("Beam SDK lifecycle endpoint must be nats://, tls://, or configured as natsWsUrl.");
|
|
@@ -31,6 +54,47 @@ export class BeamTransferControl {
|
|
|
31
54
|
throw new Error("transferRuntimeShardCount must be a positive integer.");
|
|
32
55
|
}
|
|
33
56
|
}
|
|
57
|
+
async serveRouteRecoverySigner(transferId, handler) {
|
|
58
|
+
const nc = await this.connection();
|
|
59
|
+
const subject = this.routeRecoverySignSubject(transferId);
|
|
60
|
+
const sub = nc.subscribe(subject);
|
|
61
|
+
let active = true;
|
|
62
|
+
void (async () => {
|
|
63
|
+
for await (const msg of sub) {
|
|
64
|
+
if (!active)
|
|
65
|
+
break;
|
|
66
|
+
let request = null;
|
|
67
|
+
try {
|
|
68
|
+
request = decode(msg.data);
|
|
69
|
+
if (request.schema_version !== TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION ||
|
|
70
|
+
request.environment !== this.environment ||
|
|
71
|
+
request.key_prefix !== this.keyPrefix ||
|
|
72
|
+
request.transfer_id !== transferId ||
|
|
73
|
+
request.message_type !== ROUTE_RECOVERY_SIGN_MESSAGE_TYPE ||
|
|
74
|
+
request.producer !== "transfer-runtime") {
|
|
75
|
+
throw new Error("route recovery request envelope mismatch");
|
|
76
|
+
}
|
|
77
|
+
const payload = await handler(request.payload);
|
|
78
|
+
msg.respond(encode(this.buildRouteRecoveryReply(request, true, 200, payload)));
|
|
79
|
+
}
|
|
80
|
+
catch (error) {
|
|
81
|
+
const fallback = request ?? {
|
|
82
|
+
message_id: "unknown",
|
|
83
|
+
request_id: "unknown",
|
|
84
|
+
transfer_id: transferId
|
|
85
|
+
};
|
|
86
|
+
msg.respond(encode(this.buildRouteRecoveryReply(fallback, false, 500, undefined, {
|
|
87
|
+
code: "route_recovery_sign_failed",
|
|
88
|
+
message: error instanceof Error ? error.message : String(error)
|
|
89
|
+
})));
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
})();
|
|
93
|
+
return () => {
|
|
94
|
+
active = false;
|
|
95
|
+
sub.unsubscribe();
|
|
96
|
+
};
|
|
97
|
+
}
|
|
34
98
|
async close() {
|
|
35
99
|
if (!this.nc)
|
|
36
100
|
return;
|
|
@@ -100,12 +164,11 @@ export class BeamTransferControl {
|
|
|
100
164
|
async connection() {
|
|
101
165
|
if (this.nc && !this.nc.isClosed())
|
|
102
166
|
return this.nc;
|
|
103
|
-
this.nc = await connect({
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
});
|
|
167
|
+
this.nc = await connect(buildConnectionOptions({
|
|
168
|
+
natsUrl: this.natsUrl,
|
|
169
|
+
apiKey: this.apiKey,
|
|
170
|
+
keyPrefix: this.keyPrefix
|
|
171
|
+
}));
|
|
109
172
|
return this.nc;
|
|
110
173
|
}
|
|
111
174
|
async resolveAuthToken() {
|
|
@@ -126,6 +189,26 @@ export class BeamTransferControl {
|
|
|
126
189
|
authSubject() {
|
|
127
190
|
return `${this.subjectPrefix}.${this.environment}.auth.${this.keyPrefix}.resolve`;
|
|
128
191
|
}
|
|
192
|
+
routeRecoverySignSubject(transferId) {
|
|
193
|
+
return `${this.subjectPrefix}.${this.environment}.sdk.${this.keyPrefix}.transfer.${transferId}.route_recovery_sign`;
|
|
194
|
+
}
|
|
195
|
+
buildRouteRecoveryReply(request, ok, status, payload, error) {
|
|
196
|
+
return {
|
|
197
|
+
message_id: request.message_id,
|
|
198
|
+
schema_version: TRANSFER_CLIENT_CONTROL_SCHEMA_VERSION,
|
|
199
|
+
environment: this.environment,
|
|
200
|
+
key_prefix: this.keyPrefix,
|
|
201
|
+
transfer_id: request.transfer_id,
|
|
202
|
+
message_type: ROUTE_RECOVERY_SIGN_MESSAGE_TYPE,
|
|
203
|
+
request_id: request.request_id,
|
|
204
|
+
occurred_at: new Date().toISOString(),
|
|
205
|
+
producer: "sdk",
|
|
206
|
+
ok,
|
|
207
|
+
status,
|
|
208
|
+
...(payload ? { payload } : {}),
|
|
209
|
+
...(error ? { error } : {})
|
|
210
|
+
};
|
|
211
|
+
}
|
|
129
212
|
requestSubject(messageType, shardId) {
|
|
130
213
|
return `${this.subjectPrefix}.${this.environment}.sdk.${this.keyPrefix}.shard.${shardId}.${messageType.replace(/\./g, "_")}`;
|
|
131
214
|
}
|
|
@@ -46,7 +46,6 @@ export declare function signDestinationRoute(input: {
|
|
|
46
46
|
multipartGroupId?: string;
|
|
47
47
|
multipartCreatedAt?: string;
|
|
48
48
|
commitMethod?: "upload_part" | "upload_part_copy";
|
|
49
|
-
contentMd5?: string;
|
|
50
49
|
partVerifyUrl?: string;
|
|
51
50
|
listPageUrl?: string;
|
|
52
51
|
stagingObjectKey?: string;
|
|
@@ -57,21 +56,6 @@ export declare function signDestinationRoute(input: {
|
|
|
57
56
|
deleteUrl?: string;
|
|
58
57
|
fetchImpl?: typeof fetch;
|
|
59
58
|
}): Promise<SignedChunkRoute>;
|
|
60
|
-
export declare function computeChunksContentMd5(input: {
|
|
61
|
-
source: ProviderSourceConfig;
|
|
62
|
-
chunks: ChunkSigningPlanItem[];
|
|
63
|
-
expiresIn: number;
|
|
64
|
-
fetchImpl?: typeof fetch;
|
|
65
|
-
groupSizeBytes?: number;
|
|
66
|
-
readTimeoutMs?: number;
|
|
67
|
-
}): Promise<Map<number, string>>;
|
|
68
|
-
export declare function computeChunkContentMd5(input: {
|
|
69
|
-
source: ProviderSourceConfig;
|
|
70
|
-
chunk: ChunkSigningPlanItem;
|
|
71
|
-
expiresIn: number;
|
|
72
|
-
fetchImpl?: typeof fetch;
|
|
73
|
-
readTimeoutMs?: number;
|
|
74
|
-
}): Promise<string>;
|
|
75
59
|
export declare function isS3CompatibleProvider(config: ProviderSourceConfig | ProviderDestinationConfig): config is AnyS3CompatibleProviderConfig;
|
|
76
60
|
export declare function s3CompatibleEndpoint(source: AnyS3CompatibleProviderConfig): string | undefined;
|
|
77
61
|
export declare function s3CompatibleRegion(source: AnyS3CompatibleProviderConfig): string;
|
package/dist/provider-signing.js
CHANGED
|
@@ -1,10 +1,8 @@
|
|
|
1
|
-
import { createHash } from "node:crypto";
|
|
2
1
|
import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, ListPartsCommand, PutObjectCommand, S3Client, UploadPartCopyCommand, UploadPartCommand } from "@aws-sdk/client-s3";
|
|
3
2
|
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
|
4
3
|
export function nowIso() {
|
|
5
4
|
return new Date().toISOString();
|
|
6
5
|
}
|
|
7
|
-
const DEFAULT_CHECKSUM_READ_TIMEOUT_MS = 300_000;
|
|
8
6
|
export function expiresAtIso(expiresIn) {
|
|
9
7
|
return new Date(Date.now() + expiresIn * 1000).toISOString();
|
|
10
8
|
}
|
|
@@ -228,7 +226,6 @@ export async function signDestinationRoute(input) {
|
|
|
228
226
|
partNumber: input.partNumber,
|
|
229
227
|
useStagingPut: input.commitMethod === "upload_part_copy",
|
|
230
228
|
fetchImpl: input.fetchImpl,
|
|
231
|
-
contentMd5: input.contentMd5
|
|
232
229
|
});
|
|
233
230
|
const sourceRoute = await signSourceRoute({
|
|
234
231
|
source: input.source,
|
|
@@ -247,7 +244,6 @@ export async function signDestinationRoute(input) {
|
|
|
247
244
|
chunk_size: input.chunk.chunk_size,
|
|
248
245
|
expires_at: expiresAtIso(input.expiresIn),
|
|
249
246
|
headers: sourceRoute.headers,
|
|
250
|
-
...(input.contentMd5 ? { dest_headers: { "Content-MD5": input.contentMd5 } } : {}),
|
|
251
247
|
metadata: {
|
|
252
248
|
...(input.target.metadata ?? {}),
|
|
253
249
|
...(input.transferId ? { transfer_id: input.transferId } : {}),
|
|
@@ -262,7 +258,6 @@ export async function signDestinationRoute(input) {
|
|
|
262
258
|
...(input.deleteUrl ? { delete_url: input.deleteUrl } : {}),
|
|
263
259
|
...(input.uploadId ? { upload_id: input.uploadId } : {}),
|
|
264
260
|
...(input.partNumber ? { part_number: input.partNumber } : {}),
|
|
265
|
-
...(input.contentMd5 ? { checksum_algorithm: "content-md5", checksum_value: input.contentMd5 } : {}),
|
|
266
261
|
...(input.partVerifyUrl ? { part_verify_url: input.partVerifyUrl } : {}),
|
|
267
262
|
...(input.listPageUrl ? { list_page_url: input.listPageUrl } : {}),
|
|
268
263
|
...(input.completeUrl ? { complete_url: input.completeUrl } : {}),
|
|
@@ -303,27 +298,6 @@ async function signSourceRoute(input) {
|
|
|
303
298
|
}
|
|
304
299
|
return unsupportedProviderConfig(input.source);
|
|
305
300
|
}
|
|
306
|
-
async function signSourceRangeRoute(input) {
|
|
307
|
-
const range = rangeHeaderForRange(input.sourceOffset, input.chunkSize);
|
|
308
|
-
if (isS3CompatibleProvider(input.source)) {
|
|
309
|
-
const client = createS3CompatibleClient(input.source);
|
|
310
|
-
return {
|
|
311
|
-
url: await getSignedUrl(client, new GetObjectCommand({
|
|
312
|
-
Bucket: input.source.bucket,
|
|
313
|
-
Key: input.source.key,
|
|
314
|
-
Range: range
|
|
315
|
-
}), { expiresIn: input.expiresIn }),
|
|
316
|
-
headers: { Range: range }
|
|
317
|
-
};
|
|
318
|
-
}
|
|
319
|
-
if (isHippiusProvider(input.source)) {
|
|
320
|
-
return {
|
|
321
|
-
url: await hippiusPresign(input.fetchImpl ?? globalThis.fetch, input.source.base_url ?? "https://api.hippius.com", input.source.api_token, input.source.bucket, input.source.key, "get", input.expiresIn),
|
|
322
|
-
headers: { Range: range }
|
|
323
|
-
};
|
|
324
|
-
}
|
|
325
|
-
return { url: input.fallbackUrl, headers: { Range: range } };
|
|
326
|
-
}
|
|
327
301
|
async function signDestinationUrl(input) {
|
|
328
302
|
if (isS3CompatibleProvider(input.destination)) {
|
|
329
303
|
const client = createS3CompatibleClient(input.destination);
|
|
@@ -333,7 +307,6 @@ async function signDestinationUrl(input) {
|
|
|
333
307
|
Key: input.uploadPartObjectKey,
|
|
334
308
|
UploadId: input.uploadId,
|
|
335
309
|
PartNumber: input.partNumber,
|
|
336
|
-
ContentMD5: input.contentMd5
|
|
337
310
|
}), { expiresIn: input.expiresIn });
|
|
338
311
|
}
|
|
339
312
|
return getSignedUrl(client, new PutObjectCommand({ Bucket: input.destination.bucket, Key: input.objectKey }), { expiresIn: input.expiresIn });
|
|
@@ -343,280 +316,6 @@ async function signDestinationUrl(input) {
|
|
|
343
316
|
}
|
|
344
317
|
return unsupportedProviderConfig(input.destination);
|
|
345
318
|
}
|
|
346
|
-
export async function computeChunksContentMd5(input) {
|
|
347
|
-
const output = new Map();
|
|
348
|
-
const chunks = [...input.chunks].sort((a, b) => a.source_offset - b.source_offset || a.source_chunk_index - b.source_chunk_index);
|
|
349
|
-
const groups = groupContiguousChunks(chunks, input.groupSizeBytes ?? 256 * 1024 * 1024);
|
|
350
|
-
for (const group of groups) {
|
|
351
|
-
const groupOutput = await computeChunkGroupContentMd5Adaptive({
|
|
352
|
-
source: input.source,
|
|
353
|
-
chunks: group,
|
|
354
|
-
expiresIn: input.expiresIn,
|
|
355
|
-
fetchImpl: input.fetchImpl,
|
|
356
|
-
readTimeoutMs: input.readTimeoutMs
|
|
357
|
-
});
|
|
358
|
-
for (const [sourceChunkIndex, checksum] of groupOutput) {
|
|
359
|
-
output.set(sourceChunkIndex, checksum);
|
|
360
|
-
}
|
|
361
|
-
}
|
|
362
|
-
return output;
|
|
363
|
-
}
|
|
364
|
-
export async function computeChunkContentMd5(input) {
|
|
365
|
-
const checksums = await computeChunksContentMd5({
|
|
366
|
-
source: input.source,
|
|
367
|
-
chunks: [input.chunk],
|
|
368
|
-
expiresIn: input.expiresIn,
|
|
369
|
-
fetchImpl: input.fetchImpl,
|
|
370
|
-
groupSizeBytes: input.chunk.chunk_size,
|
|
371
|
-
readTimeoutMs: input.readTimeoutMs
|
|
372
|
-
});
|
|
373
|
-
const checksum = checksums.get(input.chunk.source_chunk_index);
|
|
374
|
-
if (!checksum) {
|
|
375
|
-
throw new Error(`chunk checksum missing for source chunk ${input.chunk.source_chunk_index}`);
|
|
376
|
-
}
|
|
377
|
-
return checksum;
|
|
378
|
-
}
|
|
379
|
-
function groupContiguousChunks(chunks, maxGroupSizeBytes) {
|
|
380
|
-
const groups = [];
|
|
381
|
-
let current = [];
|
|
382
|
-
let groupStart = 0;
|
|
383
|
-
let groupEnd = 0;
|
|
384
|
-
const maxBytes = Math.max(1, Math.floor(maxGroupSizeBytes));
|
|
385
|
-
for (const chunk of chunks) {
|
|
386
|
-
const chunkStart = Math.max(0, Math.floor(chunk.source_offset));
|
|
387
|
-
const chunkEnd = chunkStart + Math.max(1, Math.floor(chunk.chunk_size));
|
|
388
|
-
const startsNewGroup = current.length === 0 ||
|
|
389
|
-
chunkStart !== groupEnd ||
|
|
390
|
-
chunkEnd - groupStart > maxBytes;
|
|
391
|
-
if (startsNewGroup) {
|
|
392
|
-
if (current.length)
|
|
393
|
-
groups.push(current);
|
|
394
|
-
current = [chunk];
|
|
395
|
-
groupStart = chunkStart;
|
|
396
|
-
groupEnd = chunkEnd;
|
|
397
|
-
continue;
|
|
398
|
-
}
|
|
399
|
-
current.push(chunk);
|
|
400
|
-
groupEnd = chunkEnd;
|
|
401
|
-
}
|
|
402
|
-
if (current.length)
|
|
403
|
-
groups.push(current);
|
|
404
|
-
return groups;
|
|
405
|
-
}
|
|
406
|
-
async function computeChunkGroupContentMd5Adaptive(input) {
|
|
407
|
-
try {
|
|
408
|
-
return await retryChecksumRead(() => computeChunkGroupContentMd5(input));
|
|
409
|
-
}
|
|
410
|
-
catch (error) {
|
|
411
|
-
if (input.chunks.length <= 1) {
|
|
412
|
-
throw error;
|
|
413
|
-
}
|
|
414
|
-
const midpoint = Math.ceil(input.chunks.length / 2);
|
|
415
|
-
const output = new Map();
|
|
416
|
-
for (const chunks of [input.chunks.slice(0, midpoint), input.chunks.slice(midpoint)]) {
|
|
417
|
-
const partial = await computeChunkGroupContentMd5Adaptive({ ...input, chunks });
|
|
418
|
-
for (const [sourceChunkIndex, checksum] of partial) {
|
|
419
|
-
output.set(sourceChunkIndex, checksum);
|
|
420
|
-
}
|
|
421
|
-
}
|
|
422
|
-
return output;
|
|
423
|
-
}
|
|
424
|
-
}
|
|
425
|
-
async function computeChunkGroupContentMd5(input) {
|
|
426
|
-
if (!input.chunks.length)
|
|
427
|
-
return new Map();
|
|
428
|
-
const chunks = [...input.chunks].sort((a, b) => a.source_offset - b.source_offset || a.source_chunk_index - b.source_chunk_index);
|
|
429
|
-
const first = chunks[0];
|
|
430
|
-
const last = chunks[chunks.length - 1];
|
|
431
|
-
const sourceOffset = first.source_offset;
|
|
432
|
-
const totalSize = last.source_offset + last.chunk_size - sourceOffset;
|
|
433
|
-
return withChecksumReadTimeout(input.readTimeoutMs, async (abortSignal) => {
|
|
434
|
-
if (isS3CompatibleProvider(input.source)) {
|
|
435
|
-
const client = createS3CompatibleClient(input.source);
|
|
436
|
-
const response = await client.send(new GetObjectCommand({
|
|
437
|
-
Bucket: input.source.bucket,
|
|
438
|
-
Key: input.source.key,
|
|
439
|
-
Range: `bytes=${sourceOffset}-${sourceOffset + totalSize - 1}`
|
|
440
|
-
}), abortSignal ? { abortSignal } : undefined);
|
|
441
|
-
if (!response.Body) {
|
|
442
|
-
throw new Error(`chunk checksum read returned no body for ${input.source.bucket}/${input.source.key}`);
|
|
443
|
-
}
|
|
444
|
-
return hashChunkGroupContentMd5(chunks, totalSize, bodyToAsyncBytes(response.Body));
|
|
445
|
-
}
|
|
446
|
-
const sourceRoute = await signSourceRangeRoute({
|
|
447
|
-
source: input.source,
|
|
448
|
-
fallbackUrl: first.source_url,
|
|
449
|
-
sourceOffset,
|
|
450
|
-
chunkSize: totalSize,
|
|
451
|
-
expiresIn: input.expiresIn,
|
|
452
|
-
fetchImpl: input.fetchImpl
|
|
453
|
-
});
|
|
454
|
-
const fetchImpl = input.fetchImpl ?? globalThis.fetch;
|
|
455
|
-
const response = await fetchImpl(sourceRoute.url, { method: "GET", headers: sourceRoute.headers, signal: abortSignal });
|
|
456
|
-
if (!response.ok) {
|
|
457
|
-
const message = `chunk checksum fetch failed with ${response.status}: ${await response.text()}`;
|
|
458
|
-
throw Object.assign(new Error(message), { status: response.status });
|
|
459
|
-
}
|
|
460
|
-
if (response.body) {
|
|
461
|
-
return hashChunkGroupContentMd5(chunks, totalSize, readableStreamToAsyncBytes(response.body));
|
|
462
|
-
}
|
|
463
|
-
return hashChunkGroupContentMd5(chunks, totalSize, [new Uint8Array(await response.arrayBuffer())]);
|
|
464
|
-
});
|
|
465
|
-
}
|
|
466
|
-
function normalizedChecksumReadTimeoutMs(value) {
|
|
467
|
-
if (value === undefined)
|
|
468
|
-
return DEFAULT_CHECKSUM_READ_TIMEOUT_MS;
|
|
469
|
-
if (!Number.isFinite(value) || value <= 0)
|
|
470
|
-
return 0;
|
|
471
|
-
return Math.floor(value);
|
|
472
|
-
}
|
|
473
|
-
async function withChecksumReadTimeout(readTimeoutMs, task) {
|
|
474
|
-
const timeoutMs = normalizedChecksumReadTimeoutMs(readTimeoutMs);
|
|
475
|
-
if (timeoutMs <= 0) {
|
|
476
|
-
return task(undefined);
|
|
477
|
-
}
|
|
478
|
-
const controller = new AbortController();
|
|
479
|
-
const timeoutError = Object.assign(new Error(`chunk checksum read timed out after ${timeoutMs}ms`), { code: "CHECKSUM_READ_TIMEOUT" });
|
|
480
|
-
let timer;
|
|
481
|
-
const timeout = new Promise((_resolve, reject) => {
|
|
482
|
-
timer = setTimeout(() => {
|
|
483
|
-
controller.abort(timeoutError);
|
|
484
|
-
reject(timeoutError);
|
|
485
|
-
}, timeoutMs);
|
|
486
|
-
});
|
|
487
|
-
try {
|
|
488
|
-
return await Promise.race([task(controller.signal), timeout]);
|
|
489
|
-
}
|
|
490
|
-
catch (error) {
|
|
491
|
-
if (controller.signal.aborted) {
|
|
492
|
-
throw timeoutError;
|
|
493
|
-
}
|
|
494
|
-
throw error;
|
|
495
|
-
}
|
|
496
|
-
finally {
|
|
497
|
-
if (timer)
|
|
498
|
-
clearTimeout(timer);
|
|
499
|
-
}
|
|
500
|
-
}
|
|
501
|
-
async function hashChunkGroupContentMd5(chunks, totalSize, body) {
|
|
502
|
-
const output = new Map();
|
|
503
|
-
let chunkIndex = 0;
|
|
504
|
-
let bytesInChunk = 0;
|
|
505
|
-
let totalRead = 0;
|
|
506
|
-
let hash = createHash("md5");
|
|
507
|
-
const consume = (value) => {
|
|
508
|
-
let offset = 0;
|
|
509
|
-
while (offset < value.byteLength) {
|
|
510
|
-
const chunk = chunks[chunkIndex];
|
|
511
|
-
if (!chunk) {
|
|
512
|
-
throw new Error(`chunk checksum fetch size mismatch: read more than ${totalSize} bytes`);
|
|
513
|
-
}
|
|
514
|
-
const remainingInChunk = chunk.chunk_size - bytesInChunk;
|
|
515
|
-
const take = Math.min(remainingInChunk, value.byteLength - offset);
|
|
516
|
-
hash.update(value.subarray(offset, offset + take));
|
|
517
|
-
offset += take;
|
|
518
|
-
totalRead += take;
|
|
519
|
-
bytesInChunk += take;
|
|
520
|
-
if (bytesInChunk === chunk.chunk_size) {
|
|
521
|
-
output.set(chunk.source_chunk_index, hash.digest("base64"));
|
|
522
|
-
chunkIndex += 1;
|
|
523
|
-
bytesInChunk = 0;
|
|
524
|
-
hash = createHash("md5");
|
|
525
|
-
}
|
|
526
|
-
}
|
|
527
|
-
};
|
|
528
|
-
for await (const value of body) {
|
|
529
|
-
consume(value);
|
|
530
|
-
}
|
|
531
|
-
if (totalRead !== totalSize || output.size !== chunks.length) {
|
|
532
|
-
throw new Error(`chunk checksum fetch size mismatch: expected ${totalSize}, got ${totalRead}`);
|
|
533
|
-
}
|
|
534
|
-
return output;
|
|
535
|
-
}
|
|
536
|
-
async function* readableStreamToAsyncBytes(stream) {
|
|
537
|
-
const reader = stream.getReader();
|
|
538
|
-
try {
|
|
539
|
-
while (true) {
|
|
540
|
-
const { done, value } = await reader.read();
|
|
541
|
-
if (done)
|
|
542
|
-
break;
|
|
543
|
-
if (value)
|
|
544
|
-
yield value;
|
|
545
|
-
}
|
|
546
|
-
}
|
|
547
|
-
finally {
|
|
548
|
-
reader.releaseLock();
|
|
549
|
-
}
|
|
550
|
-
}
|
|
551
|
-
async function* bodyToAsyncBytes(body) {
|
|
552
|
-
if (body instanceof Uint8Array) {
|
|
553
|
-
yield body;
|
|
554
|
-
return;
|
|
555
|
-
}
|
|
556
|
-
if (typeof body === "string") {
|
|
557
|
-
yield new TextEncoder().encode(body);
|
|
558
|
-
return;
|
|
559
|
-
}
|
|
560
|
-
if (body && typeof body.transformToByteArray === "function") {
|
|
561
|
-
yield new Uint8Array(await body.transformToByteArray());
|
|
562
|
-
return;
|
|
563
|
-
}
|
|
564
|
-
if (body && typeof body.arrayBuffer === "function") {
|
|
565
|
-
yield new Uint8Array(await body.arrayBuffer());
|
|
566
|
-
return;
|
|
567
|
-
}
|
|
568
|
-
if (body && typeof body.getReader === "function") {
|
|
569
|
-
yield* readableStreamToAsyncBytes(body);
|
|
570
|
-
return;
|
|
571
|
-
}
|
|
572
|
-
if (body && Symbol.asyncIterator in Object(body)) {
|
|
573
|
-
for await (const value of body) {
|
|
574
|
-
yield toUint8Array(value);
|
|
575
|
-
}
|
|
576
|
-
return;
|
|
577
|
-
}
|
|
578
|
-
throw new Error("provider checksum read returned an unsupported body stream");
|
|
579
|
-
}
|
|
580
|
-
function toUint8Array(value) {
|
|
581
|
-
if (value instanceof Uint8Array)
|
|
582
|
-
return value;
|
|
583
|
-
if (typeof value === "string")
|
|
584
|
-
return new TextEncoder().encode(value);
|
|
585
|
-
if (ArrayBuffer.isView(value)) {
|
|
586
|
-
return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
|
|
587
|
-
}
|
|
588
|
-
if (value instanceof ArrayBuffer)
|
|
589
|
-
return new Uint8Array(value);
|
|
590
|
-
throw new Error("provider checksum stream yielded an unsupported chunk");
|
|
591
|
-
}
|
|
592
|
-
async function retryChecksumRead(task) {
|
|
593
|
-
let lastError;
|
|
594
|
-
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
595
|
-
try {
|
|
596
|
-
return await task();
|
|
597
|
-
}
|
|
598
|
-
catch (error) {
|
|
599
|
-
lastError = error;
|
|
600
|
-
if (isChecksumReadTimeout(error))
|
|
601
|
-
break;
|
|
602
|
-
const status = typeof error === "object" && error && "status" in error ? Number(error.status) : undefined;
|
|
603
|
-
if (status && status >= 400 && status < 500 && status !== 408 && status !== 429)
|
|
604
|
-
break;
|
|
605
|
-
if (attempt < 2)
|
|
606
|
-
await sleep(250 * (attempt + 1));
|
|
607
|
-
}
|
|
608
|
-
}
|
|
609
|
-
throw lastError;
|
|
610
|
-
}
|
|
611
|
-
function isChecksumReadTimeout(error) {
|
|
612
|
-
return Boolean(error &&
|
|
613
|
-
typeof error === "object" &&
|
|
614
|
-
"code" in error &&
|
|
615
|
-
error.code === "CHECKSUM_READ_TIMEOUT");
|
|
616
|
-
}
|
|
617
|
-
function sleep(ms) {
|
|
618
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
619
|
-
}
|
|
620
319
|
const s3ClientCache = new Map();
|
|
621
320
|
function s3ClientCacheKey(source, endpoint) {
|
|
622
321
|
return JSON.stringify({
|