@beam-network/sdk 0.2.11 → 0.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client.d.ts +2 -2
- package/dist/client.js +210 -46
- package/dist/models.d.ts +9 -4
- package/dist/provider-signing.d.ts +5 -1
- package/dist/provider-signing.js +108 -8
- package/package.json +1 -1
package/dist/client.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { AttachSignedUrlsResponse, BeamClientOptions, DistributeResponse, PreparedDestination, PreparedHttpSource, ProviderTransferCreateInput, RawTransferCreateInput, SignedChunkRoute, SignedUrlFlow, TransferCancelResponse, TransferCreateResponse, TransferPlanResponse, TransferPrepareResponse, TransferStatusInfo } from "./models.js";
|
|
1
|
+
import type { AttachSignedUrlsResponse, BeamClientOptions, DistributeResponse, PlanningHttpSource, PreparedDestination, PreparedHttpSource, ProviderTransferCreateInput, RawTransferCreateInput, SignedChunkRoute, SignedUrlFlow, TransferCancelResponse, TransferCreateResponse, TransferPlanResponse, TransferPrepareResponse, TransferStatusInfo } from "./models.js";
|
|
2
2
|
export declare const BEAM_DEFAULT_URL = "https://beamcore.b1m.ai";
|
|
3
3
|
export declare class BeamApiError extends Error {
|
|
4
4
|
readonly status: number;
|
|
@@ -16,7 +16,7 @@ export declare class BeamClient {
|
|
|
16
16
|
distributeTransfer(transferId: string): Promise<DistributeResponse>;
|
|
17
17
|
cancelTransfer(transferId: string): Promise<TransferCancelResponse>;
|
|
18
18
|
planTransfer(input: {
|
|
19
|
-
sources:
|
|
19
|
+
sources: PlanningHttpSource[];
|
|
20
20
|
destinations: PreparedDestination[];
|
|
21
21
|
name?: string;
|
|
22
22
|
testMode?: boolean;
|
package/dist/client.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { abortMultipartUpload, computeChunksContentMd5, createMultipartUpload, nowIso, prepareProviderDestination, prepareProviderSource, isS3CompatibleProvider, listPartsPageMarker, listPartsVerifyMarker, signAbortMultipartUpload, signCompleteMultipartUpload, signDeleteObject, signDestinationRoute, signListMultipartUpload, signUploadPartCopy } from "./provider-signing.js";
|
|
1
|
+
import { abortMultipartUpload, computeChunksContentMd5, createMultipartUpload, nowIso, prepareProviderDestination, prepareProviderSource, prepareProviderSourceForPlan, isS3CompatibleProvider, listPartsPageMarker, listPartsVerifyMarker, signAbortMultipartUpload, signCompleteMultipartUpload, signDeleteObject, signDestinationRoute, signListMultipartUpload, signUploadPartCopy } from "./provider-signing.js";
|
|
2
2
|
export const BEAM_DEFAULT_URL = "https://beamcore.b1m.ai";
|
|
3
3
|
export class BeamApiError extends Error {
|
|
4
4
|
status;
|
|
@@ -92,21 +92,8 @@ export class BeamClient {
|
|
|
92
92
|
}
|
|
93
93
|
async prepareProviderTransfer(input) {
|
|
94
94
|
const expiresIn = input.expiresIn ?? 3600;
|
|
95
|
-
const
|
|
95
|
+
const requestedSignedUrlFlow = input.signedUrlFlow ?? "signed_url_v1";
|
|
96
96
|
const preparedDestinations = input.destinations.map((destination, index) => prepareProviderDestination(destination, { index }));
|
|
97
|
-
const prepared = await this.prepareTransfer({
|
|
98
|
-
sources: preparedSources,
|
|
99
|
-
destinations: preparedDestinations,
|
|
100
|
-
name: input.name,
|
|
101
|
-
testMode: input.testMode,
|
|
102
|
-
experimental: input.experimental,
|
|
103
|
-
signedUrlFlow: input.signedUrlFlow
|
|
104
|
-
});
|
|
105
|
-
if (!prepared.success) {
|
|
106
|
-
return prepared;
|
|
107
|
-
}
|
|
108
|
-
await input.onPrepared?.(prepared);
|
|
109
|
-
await input.throwIfCancelled?.(prepared.transfer_id);
|
|
110
97
|
const destinationsById = new Map();
|
|
111
98
|
preparedDestinations.forEach((preparedDestination, index) => {
|
|
112
99
|
const destination = input.destinations[index];
|
|
@@ -115,41 +102,107 @@ export class BeamClient {
|
|
|
115
102
|
}
|
|
116
103
|
destinationsById.set(preparedDestination.destination_id, destination);
|
|
117
104
|
});
|
|
118
|
-
const sourcesById = new Map();
|
|
119
|
-
preparedSources.forEach((preparedSource, index) => {
|
|
120
|
-
const source = input.sources[index];
|
|
121
|
-
if (!source) {
|
|
122
|
-
throw new Error(`missing provider source config for prepared source ${preparedSource.source_id}`);
|
|
123
|
-
}
|
|
124
|
-
sourcesById.set(preparedSource.source_id, source);
|
|
125
|
-
});
|
|
126
|
-
const signedUrlFlow = prepared.signed_url_flow ?? input.signedUrlFlow ?? "signed_url_v1";
|
|
127
105
|
const checksumCache = new ChunkChecksumCache({
|
|
128
106
|
concurrency: input.checksumReadConcurrency ?? 8,
|
|
129
107
|
expiresIn,
|
|
130
108
|
fetchImpl: this.fetchImpl,
|
|
131
109
|
provider: input.checksumProvider,
|
|
132
110
|
groupSizeBytes: input.checksumGroupSizeBytes ?? 256 * 1024 * 1024,
|
|
133
|
-
onEvent: input.onChecksumEvent
|
|
111
|
+
onEvent: input.onChecksumEvent,
|
|
112
|
+
throwIfCancelled: () => input.throwIfCancelled?.()
|
|
134
113
|
});
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
114
|
+
let preparedSources;
|
|
115
|
+
let prepared;
|
|
116
|
+
if (requestedSignedUrlFlow === "signed_url_v1") {
|
|
117
|
+
await input.throwIfCancelled?.();
|
|
118
|
+
const plannedSources = await Promise.all(input.sources.map((source, index) => prepareProviderSourceForPlan(source, { index, fetchImpl: this.fetchImpl })));
|
|
119
|
+
const planned = await this.planTransfer({
|
|
120
|
+
sources: plannedSources,
|
|
121
|
+
destinations: preparedDestinations,
|
|
122
|
+
name: input.name,
|
|
123
|
+
testMode: input.testMode,
|
|
124
|
+
experimental: input.experimental,
|
|
125
|
+
signedUrlFlow: requestedSignedUrlFlow
|
|
126
|
+
});
|
|
127
|
+
if (!planned.success) {
|
|
128
|
+
return failedPlanAsPrepareResponse(planned);
|
|
129
|
+
}
|
|
130
|
+
await input.onPlanned?.(planned);
|
|
131
|
+
await input.throwIfCancelled?.();
|
|
132
|
+
const plannedSourcesById = mapSourcesByPreparedId(plannedSources, input.sources);
|
|
133
|
+
const chunksBySource = checksumChunksBySource(planned.chunk_plan, destinationsById);
|
|
134
|
+
const totalWarmupChunks = [...chunksBySource.values()].reduce((sum, chunks) => sum + chunks.length, 0);
|
|
135
|
+
const totalWarmupBytes = [...chunksBySource.values()].reduce((sum, chunks) => sum + chunks.reduce((chunkSum, chunk) => chunkSum + chunk.chunk_size, 0), 0);
|
|
136
|
+
if (totalWarmupChunks > 0) {
|
|
137
|
+
const startedAt = Date.now();
|
|
138
|
+
await emitChecksumEvent(input, {
|
|
139
|
+
event: "checksum_warmup_started",
|
|
140
|
+
sourceId: "all",
|
|
141
|
+
chunkCount: totalWarmupChunks,
|
|
142
|
+
bytes: totalWarmupBytes
|
|
143
|
+
});
|
|
144
|
+
for (const [sourceId, chunks] of chunksBySource) {
|
|
145
|
+
await input.throwIfCancelled?.();
|
|
146
|
+
const source = plannedSourcesById.get(sourceId);
|
|
147
|
+
if (!source) {
|
|
148
|
+
throw new Error(`BeamCore planned unknown source_id: ${sourceId}`);
|
|
149
|
+
}
|
|
150
|
+
await checksumCache.ensureMany(source, chunks);
|
|
151
|
+
}
|
|
152
|
+
await emitChecksumEvent(input, {
|
|
153
|
+
event: "checksum_warmup_completed",
|
|
154
|
+
sourceId: "all",
|
|
155
|
+
chunkCount: totalWarmupChunks,
|
|
156
|
+
bytes: totalWarmupBytes,
|
|
157
|
+
durationMs: Date.now() - startedAt
|
|
141
158
|
});
|
|
142
|
-
if (!needsChecksum)
|
|
143
|
-
continue;
|
|
144
|
-
const chunks = chunksBySource.get(chunk.source_id) ?? [];
|
|
145
|
-
chunks.push(chunk);
|
|
146
|
-
chunksBySource.set(chunk.source_id, chunks);
|
|
147
159
|
}
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
160
|
+
await input.throwIfCancelled?.();
|
|
161
|
+
checksumCache.seal();
|
|
162
|
+
preparedSources = await Promise.all(input.sources.map((source, index) => prepareProviderSource(source, { index, expiresIn, fetchImpl: this.fetchImpl })));
|
|
163
|
+
await input.onBeforeTransferPrepare?.();
|
|
164
|
+
prepared = await this.prepareTransfer({
|
|
165
|
+
sources: preparedSources,
|
|
166
|
+
destinations: preparedDestinations,
|
|
167
|
+
name: input.name,
|
|
168
|
+
testMode: input.testMode,
|
|
169
|
+
experimental: input.experimental,
|
|
170
|
+
signedUrlFlow: requestedSignedUrlFlow
|
|
171
|
+
});
|
|
172
|
+
if (!prepared.success) {
|
|
173
|
+
return prepared;
|
|
152
174
|
}
|
|
175
|
+
assertPreparedTransferMatchesPlan(planned, prepared);
|
|
176
|
+
}
|
|
177
|
+
else {
|
|
178
|
+
preparedSources = await Promise.all(input.sources.map((source, index) => prepareProviderSource(source, { index, expiresIn, fetchImpl: this.fetchImpl })));
|
|
179
|
+
await input.onBeforeTransferPrepare?.();
|
|
180
|
+
prepared = await this.prepareTransfer({
|
|
181
|
+
sources: preparedSources,
|
|
182
|
+
destinations: preparedDestinations,
|
|
183
|
+
name: input.name,
|
|
184
|
+
testMode: input.testMode,
|
|
185
|
+
experimental: input.experimental,
|
|
186
|
+
signedUrlFlow: requestedSignedUrlFlow
|
|
187
|
+
});
|
|
188
|
+
if (!prepared.success) {
|
|
189
|
+
return prepared;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
await input.onPrepared?.(prepared);
|
|
193
|
+
await input.throwIfCancelled?.(prepared.transfer_id);
|
|
194
|
+
const sourcesById = new Map();
|
|
195
|
+
preparedSources.forEach((preparedSource, index) => {
|
|
196
|
+
const source = input.sources[index];
|
|
197
|
+
if (!source) {
|
|
198
|
+
throw new Error(`missing provider source config for prepared source ${preparedSource.source_id}`);
|
|
199
|
+
}
|
|
200
|
+
sourcesById.set(preparedSource.source_id, source);
|
|
201
|
+
});
|
|
202
|
+
const signedUrlFlow = prepared.signed_url_flow ?? requestedSignedUrlFlow;
|
|
203
|
+
if (signedUrlFlow === "signed_url_v1") {
|
|
204
|
+
const chunksBySource = checksumChunksBySource(prepared.chunk_plan, destinationsById);
|
|
205
|
+
checksumCache.assertManyWarmed(chunksBySource);
|
|
153
206
|
}
|
|
154
207
|
const routes = [];
|
|
155
208
|
const multipartUploads = new Map();
|
|
@@ -373,15 +426,100 @@ export class BeamClient {
|
|
|
373
426
|
return (text ? JSON.parse(text) : {});
|
|
374
427
|
}
|
|
375
428
|
}
|
|
429
|
+
function checksumChunksBySource(chunkPlan, destinationsById) {
|
|
430
|
+
const chunksBySource = new Map();
|
|
431
|
+
for (const chunk of chunkPlan) {
|
|
432
|
+
const needsChecksum = chunk.destinations.some((target) => {
|
|
433
|
+
const destination = destinationsById.get(target.destination_id);
|
|
434
|
+
return destination ? isS3CompatibleProvider(destination) : false;
|
|
435
|
+
});
|
|
436
|
+
if (!needsChecksum)
|
|
437
|
+
continue;
|
|
438
|
+
const chunks = chunksBySource.get(chunk.source_id) ?? [];
|
|
439
|
+
chunks.push(chunk);
|
|
440
|
+
chunksBySource.set(chunk.source_id, chunks);
|
|
441
|
+
}
|
|
442
|
+
return chunksBySource;
|
|
443
|
+
}
|
|
444
|
+
function mapSourcesByPreparedId(preparedSources, providerSources) {
|
|
445
|
+
const sourcesById = new Map();
|
|
446
|
+
preparedSources.forEach((preparedSource, index) => {
|
|
447
|
+
const source = providerSources[index];
|
|
448
|
+
if (!source) {
|
|
449
|
+
throw new Error(`missing provider source config for planned source ${preparedSource.source_id}`);
|
|
450
|
+
}
|
|
451
|
+
sourcesById.set(preparedSource.source_id, source);
|
|
452
|
+
});
|
|
453
|
+
return sourcesById;
|
|
454
|
+
}
|
|
455
|
+
function failedPlanAsPrepareResponse(plan) {
|
|
456
|
+
return {
|
|
457
|
+
success: false,
|
|
458
|
+
transfer_id: "",
|
|
459
|
+
chunk_size: plan.chunk_size,
|
|
460
|
+
total_size: plan.total_size,
|
|
461
|
+
total_sources: plan.total_sources,
|
|
462
|
+
total_destinations: plan.total_destinations,
|
|
463
|
+
logical_chunks: plan.logical_chunks,
|
|
464
|
+
total_chunks: plan.total_chunks,
|
|
465
|
+
chunk_plan: plan.chunk_plan ?? [],
|
|
466
|
+
signed_url_flow: plan.signed_url_flow,
|
|
467
|
+
error: plan.error,
|
|
468
|
+
message: plan.message
|
|
469
|
+
};
|
|
470
|
+
}
|
|
471
|
+
function assertPreparedTransferMatchesPlan(plan, prepared) {
|
|
472
|
+
const summaryMatches = plan.chunk_size === prepared.chunk_size &&
|
|
473
|
+
plan.total_size === prepared.total_size &&
|
|
474
|
+
plan.total_sources === prepared.total_sources &&
|
|
475
|
+
plan.total_destinations === prepared.total_destinations &&
|
|
476
|
+
plan.logical_chunks === prepared.logical_chunks &&
|
|
477
|
+
plan.total_chunks === prepared.total_chunks;
|
|
478
|
+
if (!summaryMatches || planSignature(plan.chunk_plan) !== planSignature(prepared.chunk_plan)) {
|
|
479
|
+
throw new Error("prepared transfer chunk plan does not match preflight checksum plan");
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
function planSignature(chunks) {
|
|
483
|
+
return JSON.stringify(chunks.map((chunk) => ({
|
|
484
|
+
chunk_index: chunk.chunk_index,
|
|
485
|
+
source_id: chunk.source_id,
|
|
486
|
+
source_chunk_index: chunk.source_chunk_index,
|
|
487
|
+
source_offset: chunk.source_offset,
|
|
488
|
+
chunk_size: chunk.chunk_size,
|
|
489
|
+
destinations: chunk.destinations.map((target) => {
|
|
490
|
+
const metadata = target.metadata ?? {};
|
|
491
|
+
return {
|
|
492
|
+
destination_id: target.destination_id,
|
|
493
|
+
provider: target.provider,
|
|
494
|
+
final_object_key: typeof metadata.final_object_key === "string" ? metadata.final_object_key : target.object_key,
|
|
495
|
+
part_number: typeof metadata.part_number === "number" ? metadata.part_number : undefined,
|
|
496
|
+
delivery_index: typeof metadata.delivery_index === "number" ? metadata.delivery_index : undefined
|
|
497
|
+
};
|
|
498
|
+
})
|
|
499
|
+
})));
|
|
500
|
+
}
|
|
501
|
+
async function emitChecksumEvent(input, event) {
|
|
502
|
+
if (!input.onChecksumEvent)
|
|
503
|
+
return;
|
|
504
|
+
await input.onChecksumEvent(event);
|
|
505
|
+
}
|
|
376
506
|
class ChunkChecksumCache {
|
|
377
507
|
options;
|
|
378
508
|
queue = [];
|
|
379
509
|
cache = new Map();
|
|
380
510
|
active = 0;
|
|
511
|
+
sealed = false;
|
|
381
512
|
constructor(options) {
|
|
382
513
|
this.options = options;
|
|
383
514
|
}
|
|
384
515
|
preloadMany(source, chunks) {
|
|
516
|
+
if (this.sealed) {
|
|
517
|
+
const missing = this.uniqueChunks(chunks).filter((chunk) => !this.cache.has(this.key(chunk)));
|
|
518
|
+
if (missing.length) {
|
|
519
|
+
throw new Error(`checksum cache missing ${missing.length} chunk(s) after preflight warmup; refusing post-prepare source reads`);
|
|
520
|
+
}
|
|
521
|
+
return;
|
|
522
|
+
}
|
|
385
523
|
const pending = this.uniqueChunks(chunks).filter((chunk) => !this.cache.has(this.key(chunk)));
|
|
386
524
|
for (const chunk of pending) {
|
|
387
525
|
this.cache.set(this.key(chunk), deferredChecksum());
|
|
@@ -390,10 +528,18 @@ class ChunkChecksumCache {
|
|
|
390
528
|
void this.run(() => this.fillGroup(source, group)).catch(() => undefined);
|
|
391
529
|
}
|
|
392
530
|
}
|
|
531
|
+
async ensureMany(source, chunks) {
|
|
532
|
+
const unique = this.uniqueChunks(chunks);
|
|
533
|
+
this.preloadMany(source, unique);
|
|
534
|
+
await Promise.all(unique.map((chunk) => this.get(source, chunk)));
|
|
535
|
+
}
|
|
393
536
|
get(source, chunk) {
|
|
394
537
|
const key = this.key(chunk);
|
|
395
538
|
let existing = this.cache.get(key);
|
|
396
539
|
if (!existing) {
|
|
540
|
+
if (this.sealed) {
|
|
541
|
+
return Promise.reject(new Error(`checksum cache missing ${key} after preflight warmup; refusing post-prepare source read`));
|
|
542
|
+
}
|
|
397
543
|
existing = deferredChecksum();
|
|
398
544
|
this.cache.set(key, existing);
|
|
399
545
|
void this.run(() => this.fillGroup(source, [chunk])).catch(() => undefined);
|
|
@@ -403,6 +549,7 @@ class ChunkChecksumCache {
|
|
|
403
549
|
async fillGroup(source, chunks) {
|
|
404
550
|
const unresolved = new Set(chunks.map((chunk) => this.key(chunk)));
|
|
405
551
|
try {
|
|
552
|
+
await this.options.throwIfCancelled?.();
|
|
406
553
|
const misses = [];
|
|
407
554
|
for (const chunk of chunks) {
|
|
408
555
|
const key = this.key(chunk);
|
|
@@ -411,17 +558,18 @@ class ChunkChecksumCache {
|
|
|
411
558
|
continue;
|
|
412
559
|
const cached = await this.options.provider?.getContentMd5({ source, chunk, algorithm: "content-md5" }).catch(() => undefined);
|
|
413
560
|
if (cached) {
|
|
414
|
-
this.emit({ event: "checksum_cache_hit", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
|
|
561
|
+
await this.emit({ event: "checksum_cache_hit", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
|
|
415
562
|
deferred.resolve(cached);
|
|
416
563
|
unresolved.delete(key);
|
|
417
564
|
}
|
|
418
565
|
else {
|
|
419
|
-
this.emit({ event: "checksum_cache_miss", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
|
|
566
|
+
await this.emit({ event: "checksum_cache_miss", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
|
|
420
567
|
misses.push(chunk);
|
|
421
568
|
}
|
|
422
569
|
}
|
|
423
570
|
if (!misses.length)
|
|
424
571
|
return;
|
|
572
|
+
await this.options.throwIfCancelled?.();
|
|
425
573
|
const startedAt = Date.now();
|
|
426
574
|
const checksums = await computeChunksContentMd5({
|
|
427
575
|
source,
|
|
@@ -431,6 +579,7 @@ class ChunkChecksumCache {
|
|
|
431
579
|
groupSizeBytes: this.options.groupSizeBytes
|
|
432
580
|
});
|
|
433
581
|
for (const chunk of misses) {
|
|
582
|
+
await this.options.throwIfCancelled?.();
|
|
434
583
|
const checksum = checksums.get(chunk.source_chunk_index);
|
|
435
584
|
if (!checksum)
|
|
436
585
|
throw new Error(`checksum missing for source chunk ${chunk.source_chunk_index}`);
|
|
@@ -438,9 +587,9 @@ class ChunkChecksumCache {
|
|
|
438
587
|
const deferred = this.cache.get(this.key(chunk));
|
|
439
588
|
deferred?.resolve(checksum);
|
|
440
589
|
unresolved.delete(this.key(chunk));
|
|
441
|
-
this.emit({ event: "checksum_cold_chunk_cached", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
|
|
590
|
+
await this.emit({ event: "checksum_cold_chunk_cached", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
|
|
442
591
|
}
|
|
443
|
-
this.emit({
|
|
592
|
+
await this.emit({
|
|
444
593
|
event: "checksum_cold_group_completed",
|
|
445
594
|
sourceId: misses[0]?.source_id ?? source.id ?? "source",
|
|
446
595
|
chunkCount: misses.length,
|
|
@@ -496,6 +645,21 @@ class ChunkChecksumCache {
|
|
|
496
645
|
key(chunk) {
|
|
497
646
|
return `${chunk.source_id}:${chunk.source_chunk_index}`;
|
|
498
647
|
}
|
|
648
|
+
seal() {
|
|
649
|
+
this.sealed = true;
|
|
650
|
+
}
|
|
651
|
+
assertManyWarmed(chunksBySource) {
|
|
652
|
+
const missing = [];
|
|
653
|
+
for (const chunks of chunksBySource.values()) {
|
|
654
|
+
for (const chunk of this.uniqueChunks(chunks)) {
|
|
655
|
+
if (!this.cache.has(this.key(chunk)))
|
|
656
|
+
missing.push(this.key(chunk));
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
if (missing.length) {
|
|
660
|
+
throw new Error(`checksum cache missing ${missing.length} chunk(s) after preflight warmup; refusing post-prepare source reads`);
|
|
661
|
+
}
|
|
662
|
+
}
|
|
499
663
|
run(task) {
|
|
500
664
|
return new Promise((resolve, reject) => {
|
|
501
665
|
const start = () => {
|
|
@@ -511,10 +675,10 @@ class ChunkChecksumCache {
|
|
|
511
675
|
this.queue.push(start);
|
|
512
676
|
});
|
|
513
677
|
}
|
|
514
|
-
emit(event) {
|
|
678
|
+
async emit(event) {
|
|
515
679
|
if (!this.options.onEvent)
|
|
516
680
|
return;
|
|
517
|
-
|
|
681
|
+
await this.options.onEvent(event);
|
|
518
682
|
}
|
|
519
683
|
}
|
|
520
684
|
function deferredChecksum() {
|
package/dist/models.d.ts
CHANGED
|
@@ -177,7 +177,7 @@ export interface ChunkChecksumProvider {
|
|
|
177
177
|
setContentMd5?(input: ChunkChecksumProviderSetInput): Promise<void>;
|
|
178
178
|
}
|
|
179
179
|
export interface ChunkChecksumEvent {
|
|
180
|
-
event: "checksum_cache_hit" | "checksum_cache_miss" | "checksum_cold_chunk_cached" | "checksum_cold_group_completed";
|
|
180
|
+
event: "checksum_warmup_started" | "checksum_warmup_completed" | "checksum_cache_hit" | "checksum_cache_miss" | "checksum_cold_chunk_cached" | "checksum_cold_group_completed";
|
|
181
181
|
sourceId: string;
|
|
182
182
|
sourceChunkIndex?: number;
|
|
183
183
|
chunkCount?: number;
|
|
@@ -218,25 +218,30 @@ export interface ProviderTransferCreateInput {
|
|
|
218
218
|
* Defaults to true. Set to false to only prepare and attach signed URLs.
|
|
219
219
|
*/
|
|
220
220
|
distribute?: boolean;
|
|
221
|
+
onPlanned?: (plan: TransferPlanResponse) => void | Promise<void>;
|
|
222
|
+
onBeforeTransferPrepare?: () => void | Promise<void>;
|
|
221
223
|
onPrepared?: (prepared: TransferPrepareResponse) => void | Promise<void>;
|
|
222
|
-
throwIfCancelled?: (transferId
|
|
224
|
+
throwIfCancelled?: (transferId?: string) => void | Promise<void>;
|
|
223
225
|
signedUrlFlow?: SignedUrlFlow;
|
|
224
226
|
checksumProvider?: ChunkChecksumProvider;
|
|
225
227
|
checksumReadConcurrency?: number;
|
|
226
228
|
checksumGroupSizeBytes?: number;
|
|
227
229
|
onChecksumEvent?: (event: ChunkChecksumEvent) => void | Promise<void>;
|
|
228
230
|
}
|
|
229
|
-
export interface
|
|
231
|
+
export interface PlanningHttpSource {
|
|
230
232
|
source_id: string;
|
|
231
233
|
type: "http";
|
|
232
234
|
provider?: string;
|
|
233
|
-
url
|
|
235
|
+
url?: string;
|
|
234
236
|
size: number;
|
|
235
237
|
filename?: string;
|
|
236
238
|
headers?: Record<string, string>;
|
|
237
239
|
expires_at?: string;
|
|
238
240
|
metadata?: Record<string, unknown>;
|
|
239
241
|
}
|
|
242
|
+
export interface PreparedHttpSource extends PlanningHttpSource {
|
|
243
|
+
url: string;
|
|
244
|
+
}
|
|
240
245
|
export interface PreparedDestination {
|
|
241
246
|
destination_id: string;
|
|
242
247
|
provider: string;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ChunkDestinationSigningTarget, ChunkSigningPlanItem, PreparedDestination, PreparedHttpSource, ProviderDestinationConfig, ProviderSourceConfig, R2ProviderConfig, S3CompatibleProviderConfig, S3ProviderConfig, SignedChunkRoute } from "./models.js";
|
|
1
|
+
import type { ChunkDestinationSigningTarget, ChunkSigningPlanItem, PlanningHttpSource, PreparedDestination, PreparedHttpSource, ProviderDestinationConfig, ProviderSourceConfig, R2ProviderConfig, S3CompatibleProviderConfig, S3ProviderConfig, SignedChunkRoute } from "./models.js";
|
|
2
2
|
export type AnyS3CompatibleProviderConfig = S3ProviderConfig | R2ProviderConfig | S3CompatibleProviderConfig;
|
|
3
3
|
export declare function nowIso(): string;
|
|
4
4
|
export declare function expiresAtIso(expiresIn: number): string;
|
|
@@ -7,6 +7,10 @@ export declare function prepareProviderSource(source: ProviderSourceConfig, opti
|
|
|
7
7
|
expiresIn?: number;
|
|
8
8
|
fetchImpl?: typeof fetch;
|
|
9
9
|
}): Promise<PreparedHttpSource>;
|
|
10
|
+
export declare function prepareProviderSourceForPlan(source: ProviderSourceConfig, options?: {
|
|
11
|
+
index?: number;
|
|
12
|
+
fetchImpl?: typeof fetch;
|
|
13
|
+
}): Promise<PlanningHttpSource>;
|
|
10
14
|
export declare function prepareProviderDestination(destination: ProviderDestinationConfig, options?: {
|
|
11
15
|
index?: number;
|
|
12
16
|
}): PreparedDestination;
|
package/dist/provider-signing.js
CHANGED
|
@@ -50,6 +50,42 @@ export async function prepareProviderSource(source, options = {}) {
|
|
|
50
50
|
}
|
|
51
51
|
return unsupportedProviderConfig(source);
|
|
52
52
|
}
|
|
53
|
+
export async function prepareProviderSourceForPlan(source, options = {}) {
|
|
54
|
+
const index = options.index ?? 0;
|
|
55
|
+
if (isS3CompatibleProvider(source)) {
|
|
56
|
+
const endpoint = s3CompatibleEndpoint(source);
|
|
57
|
+
const client = createS3CompatibleClient(source, endpoint);
|
|
58
|
+
const head = await client.send(new HeadObjectCommand({ Bucket: source.bucket, Key: source.key }));
|
|
59
|
+
return {
|
|
60
|
+
source_id: source.id ?? `src_${index}`,
|
|
61
|
+
type: "http",
|
|
62
|
+
provider: source.provider,
|
|
63
|
+
size: Number(head.ContentLength ?? 0),
|
|
64
|
+
filename: filename(source.key),
|
|
65
|
+
metadata: {
|
|
66
|
+
...s3CompatibleMetadata(source, endpoint),
|
|
67
|
+
content_length: Number(head.ContentLength ?? 0),
|
|
68
|
+
...(head.ETag ? { etag: head.ETag } : {}),
|
|
69
|
+
...(head.LastModified ? { last_modified: head.LastModified.toISOString() } : {}),
|
|
70
|
+
...(head.VersionId ? { version_id: head.VersionId } : {})
|
|
71
|
+
}
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
if (isHippiusProvider(source)) {
|
|
75
|
+
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
|
76
|
+
const baseUrl = source.base_url ?? "https://api.hippius.com";
|
|
77
|
+
const size = await hippiusObjectSize(fetchImpl, baseUrl, source.api_token, source.bucket, source.key);
|
|
78
|
+
return {
|
|
79
|
+
source_id: source.id ?? `src_${index}`,
|
|
80
|
+
type: "http",
|
|
81
|
+
provider: "hippius",
|
|
82
|
+
size,
|
|
83
|
+
filename: filename(source.key),
|
|
84
|
+
metadata: { bucket: source.bucket, key: source.key, base_url: baseUrl }
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
return unsupportedProviderConfig(source);
|
|
88
|
+
}
|
|
53
89
|
export function prepareProviderDestination(destination, options = {}) {
|
|
54
90
|
const index = options.index ?? 0;
|
|
55
91
|
const destinationId = destination.id ?? `dst_${index}`;
|
|
@@ -364,6 +400,18 @@ async function computeChunkGroupContentMd5(input) {
|
|
|
364
400
|
const last = chunks[chunks.length - 1];
|
|
365
401
|
const sourceOffset = first.source_offset;
|
|
366
402
|
const totalSize = last.source_offset + last.chunk_size - sourceOffset;
|
|
403
|
+
if (isS3CompatibleProvider(input.source)) {
|
|
404
|
+
const client = createS3CompatibleClient(input.source);
|
|
405
|
+
const response = await client.send(new GetObjectCommand({
|
|
406
|
+
Bucket: input.source.bucket,
|
|
407
|
+
Key: input.source.key,
|
|
408
|
+
Range: `bytes=${sourceOffset}-${sourceOffset + totalSize - 1}`
|
|
409
|
+
}));
|
|
410
|
+
if (!response.Body) {
|
|
411
|
+
throw new Error(`chunk checksum read returned no body for ${input.source.bucket}/${input.source.key}`);
|
|
412
|
+
}
|
|
413
|
+
return hashChunkGroupContentMd5(chunks, totalSize, bodyToAsyncBytes(response.Body));
|
|
414
|
+
}
|
|
367
415
|
const sourceRoute = await signSourceRangeRoute({
|
|
368
416
|
source: input.source,
|
|
369
417
|
fallbackUrl: first.source_url,
|
|
@@ -378,6 +426,12 @@ async function computeChunkGroupContentMd5(input) {
|
|
|
378
426
|
const message = `chunk checksum fetch failed with ${response.status}: ${await response.text()}`;
|
|
379
427
|
throw Object.assign(new Error(message), { status: response.status });
|
|
380
428
|
}
|
|
429
|
+
if (response.body) {
|
|
430
|
+
return hashChunkGroupContentMd5(chunks, totalSize, readableStreamToAsyncBytes(response.body));
|
|
431
|
+
}
|
|
432
|
+
return hashChunkGroupContentMd5(chunks, totalSize, [new Uint8Array(await response.arrayBuffer())]);
|
|
433
|
+
}
|
|
434
|
+
async function hashChunkGroupContentMd5(chunks, totalSize, body) {
|
|
381
435
|
const output = new Map();
|
|
382
436
|
let chunkIndex = 0;
|
|
383
437
|
let bytesInChunk = 0;
|
|
@@ -404,23 +458,69 @@ async function computeChunkGroupContentMd5(input) {
|
|
|
404
458
|
}
|
|
405
459
|
}
|
|
406
460
|
};
|
|
407
|
-
|
|
408
|
-
|
|
461
|
+
for await (const value of body) {
|
|
462
|
+
consume(value);
|
|
463
|
+
}
|
|
464
|
+
if (totalRead !== totalSize || output.size !== chunks.length) {
|
|
465
|
+
throw new Error(`chunk checksum fetch size mismatch: expected ${totalSize}, got ${totalRead}`);
|
|
466
|
+
}
|
|
467
|
+
return output;
|
|
468
|
+
}
|
|
469
|
+
async function* readableStreamToAsyncBytes(stream) {
|
|
470
|
+
const reader = stream.getReader();
|
|
471
|
+
try {
|
|
409
472
|
while (true) {
|
|
410
473
|
const { done, value } = await reader.read();
|
|
411
474
|
if (done)
|
|
412
475
|
break;
|
|
413
476
|
if (value)
|
|
414
|
-
|
|
477
|
+
yield value;
|
|
415
478
|
}
|
|
416
479
|
}
|
|
417
|
-
|
|
418
|
-
|
|
480
|
+
finally {
|
|
481
|
+
reader.releaseLock();
|
|
419
482
|
}
|
|
420
|
-
|
|
421
|
-
|
|
483
|
+
}
|
|
484
|
+
async function* bodyToAsyncBytes(body) {
|
|
485
|
+
if (body instanceof Uint8Array) {
|
|
486
|
+
yield body;
|
|
487
|
+
return;
|
|
422
488
|
}
|
|
423
|
-
|
|
489
|
+
if (typeof body === "string") {
|
|
490
|
+
yield new TextEncoder().encode(body);
|
|
491
|
+
return;
|
|
492
|
+
}
|
|
493
|
+
if (body && typeof body.transformToByteArray === "function") {
|
|
494
|
+
yield new Uint8Array(await body.transformToByteArray());
|
|
495
|
+
return;
|
|
496
|
+
}
|
|
497
|
+
if (body && typeof body.arrayBuffer === "function") {
|
|
498
|
+
yield new Uint8Array(await body.arrayBuffer());
|
|
499
|
+
return;
|
|
500
|
+
}
|
|
501
|
+
if (body && typeof body.getReader === "function") {
|
|
502
|
+
yield* readableStreamToAsyncBytes(body);
|
|
503
|
+
return;
|
|
504
|
+
}
|
|
505
|
+
if (body && Symbol.asyncIterator in Object(body)) {
|
|
506
|
+
for await (const value of body) {
|
|
507
|
+
yield toUint8Array(value);
|
|
508
|
+
}
|
|
509
|
+
return;
|
|
510
|
+
}
|
|
511
|
+
throw new Error("provider checksum read returned an unsupported body stream");
|
|
512
|
+
}
|
|
513
|
+
function toUint8Array(value) {
|
|
514
|
+
if (value instanceof Uint8Array)
|
|
515
|
+
return value;
|
|
516
|
+
if (typeof value === "string")
|
|
517
|
+
return new TextEncoder().encode(value);
|
|
518
|
+
if (ArrayBuffer.isView(value)) {
|
|
519
|
+
return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
|
|
520
|
+
}
|
|
521
|
+
if (value instanceof ArrayBuffer)
|
|
522
|
+
return new Uint8Array(value);
|
|
523
|
+
throw new Error("provider checksum stream yielded an unsupported chunk");
|
|
424
524
|
}
|
|
425
525
|
async function retryChecksumRead(task) {
|
|
426
526
|
let lastError;
|