@beam-network/sdk 0.2.12 → 0.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client.js +38 -15
- package/package.json +1 -1
package/dist/client.js
CHANGED
|
@@ -135,7 +135,7 @@ export class BeamClient {
|
|
|
135
135
|
const totalWarmupBytes = [...chunksBySource.values()].reduce((sum, chunks) => sum + chunks.reduce((chunkSum, chunk) => chunkSum + chunk.chunk_size, 0), 0);
|
|
136
136
|
if (totalWarmupChunks > 0) {
|
|
137
137
|
const startedAt = Date.now();
|
|
138
|
-
emitChecksumEvent(input, {
|
|
138
|
+
await emitChecksumEvent(input, {
|
|
139
139
|
event: "checksum_warmup_started",
|
|
140
140
|
sourceId: "all",
|
|
141
141
|
chunkCount: totalWarmupChunks,
|
|
@@ -149,7 +149,7 @@ export class BeamClient {
|
|
|
149
149
|
}
|
|
150
150
|
await checksumCache.ensureMany(source, chunks);
|
|
151
151
|
}
|
|
152
|
-
emitChecksumEvent(input, {
|
|
152
|
+
await emitChecksumEvent(input, {
|
|
153
153
|
event: "checksum_warmup_completed",
|
|
154
154
|
sourceId: "all",
|
|
155
155
|
chunkCount: totalWarmupChunks,
|
|
@@ -158,6 +158,7 @@ export class BeamClient {
|
|
|
158
158
|
});
|
|
159
159
|
}
|
|
160
160
|
await input.throwIfCancelled?.();
|
|
161
|
+
checksumCache.seal();
|
|
161
162
|
preparedSources = await Promise.all(input.sources.map((source, index) => prepareProviderSource(source, { index, expiresIn, fetchImpl: this.fetchImpl })));
|
|
162
163
|
await input.onBeforeTransferPrepare?.();
|
|
163
164
|
prepared = await this.prepareTransfer({
|
|
@@ -201,11 +202,7 @@ export class BeamClient {
|
|
|
201
202
|
const signedUrlFlow = prepared.signed_url_flow ?? requestedSignedUrlFlow;
|
|
202
203
|
if (signedUrlFlow === "signed_url_v1") {
|
|
203
204
|
const chunksBySource = checksumChunksBySource(prepared.chunk_plan, destinationsById);
|
|
204
|
-
|
|
205
|
-
const source = sourcesById.get(sourceId);
|
|
206
|
-
if (source)
|
|
207
|
-
checksumCache.preloadMany(source, chunks);
|
|
208
|
-
}
|
|
205
|
+
checksumCache.assertManyWarmed(chunksBySource);
|
|
209
206
|
}
|
|
210
207
|
const routes = [];
|
|
211
208
|
const multipartUploads = new Map();
|
|
@@ -501,20 +498,28 @@ function planSignature(chunks) {
|
|
|
501
498
|
})
|
|
502
499
|
})));
|
|
503
500
|
}
|
|
504
|
-
function emitChecksumEvent(input, event) {
|
|
501
|
+
async function emitChecksumEvent(input, event) {
|
|
505
502
|
if (!input.onChecksumEvent)
|
|
506
503
|
return;
|
|
507
|
-
|
|
504
|
+
await input.onChecksumEvent(event);
|
|
508
505
|
}
|
|
509
506
|
class ChunkChecksumCache {
|
|
510
507
|
options;
|
|
511
508
|
queue = [];
|
|
512
509
|
cache = new Map();
|
|
513
510
|
active = 0;
|
|
511
|
+
sealed = false;
|
|
514
512
|
constructor(options) {
|
|
515
513
|
this.options = options;
|
|
516
514
|
}
|
|
517
515
|
preloadMany(source, chunks) {
|
|
516
|
+
if (this.sealed) {
|
|
517
|
+
const missing = this.uniqueChunks(chunks).filter((chunk) => !this.cache.has(this.key(chunk)));
|
|
518
|
+
if (missing.length) {
|
|
519
|
+
throw new Error(`checksum cache missing ${missing.length} chunk(s) after preflight warmup; refusing post-prepare source reads`);
|
|
520
|
+
}
|
|
521
|
+
return;
|
|
522
|
+
}
|
|
518
523
|
const pending = this.uniqueChunks(chunks).filter((chunk) => !this.cache.has(this.key(chunk)));
|
|
519
524
|
for (const chunk of pending) {
|
|
520
525
|
this.cache.set(this.key(chunk), deferredChecksum());
|
|
@@ -532,6 +537,9 @@ class ChunkChecksumCache {
|
|
|
532
537
|
const key = this.key(chunk);
|
|
533
538
|
let existing = this.cache.get(key);
|
|
534
539
|
if (!existing) {
|
|
540
|
+
if (this.sealed) {
|
|
541
|
+
return Promise.reject(new Error(`checksum cache missing ${key} after preflight warmup; refusing post-prepare source read`));
|
|
542
|
+
}
|
|
535
543
|
existing = deferredChecksum();
|
|
536
544
|
this.cache.set(key, existing);
|
|
537
545
|
void this.run(() => this.fillGroup(source, [chunk])).catch(() => undefined);
|
|
@@ -550,12 +558,12 @@ class ChunkChecksumCache {
|
|
|
550
558
|
continue;
|
|
551
559
|
const cached = await this.options.provider?.getContentMd5({ source, chunk, algorithm: "content-md5" }).catch(() => undefined);
|
|
552
560
|
if (cached) {
|
|
553
|
-
this.emit({ event: "checksum_cache_hit", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
|
|
561
|
+
await this.emit({ event: "checksum_cache_hit", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
|
|
554
562
|
deferred.resolve(cached);
|
|
555
563
|
unresolved.delete(key);
|
|
556
564
|
}
|
|
557
565
|
else {
|
|
558
|
-
this.emit({ event: "checksum_cache_miss", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
|
|
566
|
+
await this.emit({ event: "checksum_cache_miss", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
|
|
559
567
|
misses.push(chunk);
|
|
560
568
|
}
|
|
561
569
|
}
|
|
@@ -579,9 +587,9 @@ class ChunkChecksumCache {
|
|
|
579
587
|
const deferred = this.cache.get(this.key(chunk));
|
|
580
588
|
deferred?.resolve(checksum);
|
|
581
589
|
unresolved.delete(this.key(chunk));
|
|
582
|
-
this.emit({ event: "checksum_cold_chunk_cached", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
|
|
590
|
+
await this.emit({ event: "checksum_cold_chunk_cached", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
|
|
583
591
|
}
|
|
584
|
-
this.emit({
|
|
592
|
+
await this.emit({
|
|
585
593
|
event: "checksum_cold_group_completed",
|
|
586
594
|
sourceId: misses[0]?.source_id ?? source.id ?? "source",
|
|
587
595
|
chunkCount: misses.length,
|
|
@@ -637,6 +645,21 @@ class ChunkChecksumCache {
|
|
|
637
645
|
key(chunk) {
|
|
638
646
|
return `${chunk.source_id}:${chunk.source_chunk_index}`;
|
|
639
647
|
}
|
|
648
|
+
seal() {
|
|
649
|
+
this.sealed = true;
|
|
650
|
+
}
|
|
651
|
+
assertManyWarmed(chunksBySource) {
|
|
652
|
+
const missing = [];
|
|
653
|
+
for (const chunks of chunksBySource.values()) {
|
|
654
|
+
for (const chunk of this.uniqueChunks(chunks)) {
|
|
655
|
+
if (!this.cache.has(this.key(chunk)))
|
|
656
|
+
missing.push(this.key(chunk));
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
if (missing.length) {
|
|
660
|
+
throw new Error(`checksum cache missing ${missing.length} chunk(s) after preflight warmup; refusing post-prepare source reads`);
|
|
661
|
+
}
|
|
662
|
+
}
|
|
640
663
|
run(task) {
|
|
641
664
|
return new Promise((resolve, reject) => {
|
|
642
665
|
const start = () => {
|
|
@@ -652,10 +675,10 @@ class ChunkChecksumCache {
|
|
|
652
675
|
this.queue.push(start);
|
|
653
676
|
});
|
|
654
677
|
}
|
|
655
|
-
emit(event) {
|
|
678
|
+
async emit(event) {
|
|
656
679
|
if (!this.options.onEvent)
|
|
657
680
|
return;
|
|
658
|
-
|
|
681
|
+
await this.options.onEvent(event);
|
|
659
682
|
}
|
|
660
683
|
}
|
|
661
684
|
function deferredChecksum() {
|