@beam-network/sdk 0.2.11 → 0.2.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/client.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- import type { AttachSignedUrlsResponse, BeamClientOptions, DistributeResponse, PreparedDestination, PreparedHttpSource, ProviderTransferCreateInput, RawTransferCreateInput, SignedChunkRoute, SignedUrlFlow, TransferCancelResponse, TransferCreateResponse, TransferPlanResponse, TransferPrepareResponse, TransferStatusInfo } from "./models.js";
1
+ import type { AttachSignedUrlsResponse, BeamClientOptions, DistributeResponse, PlanningHttpSource, PreparedDestination, PreparedHttpSource, ProviderTransferCreateInput, RawTransferCreateInput, SignedChunkRoute, SignedUrlFlow, TransferCancelResponse, TransferCreateResponse, TransferPlanResponse, TransferPrepareResponse, TransferStatusInfo } from "./models.js";
2
2
  export declare const BEAM_DEFAULT_URL = "https://beamcore.b1m.ai";
3
3
  export declare class BeamApiError extends Error {
4
4
  readonly status: number;
@@ -16,7 +16,7 @@ export declare class BeamClient {
16
16
  distributeTransfer(transferId: string): Promise<DistributeResponse>;
17
17
  cancelTransfer(transferId: string): Promise<TransferCancelResponse>;
18
18
  planTransfer(input: {
19
- sources: PreparedHttpSource[];
19
+ sources: PlanningHttpSource[];
20
20
  destinations: PreparedDestination[];
21
21
  name?: string;
22
22
  testMode?: boolean;
package/dist/client.js CHANGED
@@ -1,4 +1,4 @@
1
- import { abortMultipartUpload, computeChunksContentMd5, createMultipartUpload, nowIso, prepareProviderDestination, prepareProviderSource, isS3CompatibleProvider, listPartsPageMarker, listPartsVerifyMarker, signAbortMultipartUpload, signCompleteMultipartUpload, signDeleteObject, signDestinationRoute, signListMultipartUpload, signUploadPartCopy } from "./provider-signing.js";
1
+ import { abortMultipartUpload, computeChunksContentMd5, createMultipartUpload, nowIso, prepareProviderDestination, prepareProviderSource, prepareProviderSourceForPlan, isS3CompatibleProvider, listPartsPageMarker, listPartsVerifyMarker, signAbortMultipartUpload, signCompleteMultipartUpload, signDeleteObject, signDestinationRoute, signListMultipartUpload, signUploadPartCopy } from "./provider-signing.js";
2
2
  export const BEAM_DEFAULT_URL = "https://beamcore.b1m.ai";
3
3
  export class BeamApiError extends Error {
4
4
  status;
@@ -92,21 +92,8 @@ export class BeamClient {
92
92
  }
93
93
  async prepareProviderTransfer(input) {
94
94
  const expiresIn = input.expiresIn ?? 3600;
95
- const preparedSources = await Promise.all(input.sources.map((source, index) => prepareProviderSource(source, { index, expiresIn, fetchImpl: this.fetchImpl })));
95
+ const requestedSignedUrlFlow = input.signedUrlFlow ?? "signed_url_v1";
96
96
  const preparedDestinations = input.destinations.map((destination, index) => prepareProviderDestination(destination, { index }));
97
- const prepared = await this.prepareTransfer({
98
- sources: preparedSources,
99
- destinations: preparedDestinations,
100
- name: input.name,
101
- testMode: input.testMode,
102
- experimental: input.experimental,
103
- signedUrlFlow: input.signedUrlFlow
104
- });
105
- if (!prepared.success) {
106
- return prepared;
107
- }
108
- await input.onPrepared?.(prepared);
109
- await input.throwIfCancelled?.(prepared.transfer_id);
110
97
  const destinationsById = new Map();
111
98
  preparedDestinations.forEach((preparedDestination, index) => {
112
99
  const destination = input.destinations[index];
@@ -115,41 +102,107 @@ export class BeamClient {
115
102
  }
116
103
  destinationsById.set(preparedDestination.destination_id, destination);
117
104
  });
118
- const sourcesById = new Map();
119
- preparedSources.forEach((preparedSource, index) => {
120
- const source = input.sources[index];
121
- if (!source) {
122
- throw new Error(`missing provider source config for prepared source ${preparedSource.source_id}`);
123
- }
124
- sourcesById.set(preparedSource.source_id, source);
125
- });
126
- const signedUrlFlow = prepared.signed_url_flow ?? input.signedUrlFlow ?? "signed_url_v1";
127
105
  const checksumCache = new ChunkChecksumCache({
128
106
  concurrency: input.checksumReadConcurrency ?? 8,
129
107
  expiresIn,
130
108
  fetchImpl: this.fetchImpl,
131
109
  provider: input.checksumProvider,
132
110
  groupSizeBytes: input.checksumGroupSizeBytes ?? 256 * 1024 * 1024,
133
- onEvent: input.onChecksumEvent
111
+ onEvent: input.onChecksumEvent,
112
+ throwIfCancelled: () => input.throwIfCancelled?.()
134
113
  });
135
- if (signedUrlFlow === "signed_url_v1") {
136
- const chunksBySource = new Map();
137
- for (const chunk of prepared.chunk_plan) {
138
- const needsChecksum = chunk.destinations.some((target) => {
139
- const destination = destinationsById.get(target.destination_id);
140
- return destination ? isS3CompatibleProvider(destination) : false;
114
+ let preparedSources;
115
+ let prepared;
116
+ if (requestedSignedUrlFlow === "signed_url_v1") {
117
+ await input.throwIfCancelled?.();
118
+ const plannedSources = await Promise.all(input.sources.map((source, index) => prepareProviderSourceForPlan(source, { index, fetchImpl: this.fetchImpl })));
119
+ const planned = await this.planTransfer({
120
+ sources: plannedSources,
121
+ destinations: preparedDestinations,
122
+ name: input.name,
123
+ testMode: input.testMode,
124
+ experimental: input.experimental,
125
+ signedUrlFlow: requestedSignedUrlFlow
126
+ });
127
+ if (!planned.success) {
128
+ return failedPlanAsPrepareResponse(planned);
129
+ }
130
+ await input.onPlanned?.(planned);
131
+ await input.throwIfCancelled?.();
132
+ const plannedSourcesById = mapSourcesByPreparedId(plannedSources, input.sources);
133
+ const chunksBySource = checksumChunksBySource(planned.chunk_plan, destinationsById);
134
+ const totalWarmupChunks = [...chunksBySource.values()].reduce((sum, chunks) => sum + chunks.length, 0);
135
+ const totalWarmupBytes = [...chunksBySource.values()].reduce((sum, chunks) => sum + chunks.reduce((chunkSum, chunk) => chunkSum + chunk.chunk_size, 0), 0);
136
+ if (totalWarmupChunks > 0) {
137
+ const startedAt = Date.now();
138
+ await emitChecksumEvent(input, {
139
+ event: "checksum_warmup_started",
140
+ sourceId: "all",
141
+ chunkCount: totalWarmupChunks,
142
+ bytes: totalWarmupBytes
143
+ });
144
+ for (const [sourceId, chunks] of chunksBySource) {
145
+ await input.throwIfCancelled?.();
146
+ const source = plannedSourcesById.get(sourceId);
147
+ if (!source) {
148
+ throw new Error(`BeamCore planned unknown source_id: ${sourceId}`);
149
+ }
150
+ await checksumCache.ensureMany(source, chunks);
151
+ }
152
+ await emitChecksumEvent(input, {
153
+ event: "checksum_warmup_completed",
154
+ sourceId: "all",
155
+ chunkCount: totalWarmupChunks,
156
+ bytes: totalWarmupBytes,
157
+ durationMs: Date.now() - startedAt
141
158
  });
142
- if (!needsChecksum)
143
- continue;
144
- const chunks = chunksBySource.get(chunk.source_id) ?? [];
145
- chunks.push(chunk);
146
- chunksBySource.set(chunk.source_id, chunks);
147
159
  }
148
- for (const [sourceId, chunks] of chunksBySource) {
149
- const source = sourcesById.get(sourceId);
150
- if (source)
151
- checksumCache.preloadMany(source, chunks);
160
+ await input.throwIfCancelled?.();
161
+ checksumCache.seal();
162
+ preparedSources = await Promise.all(input.sources.map((source, index) => prepareProviderSource(source, { index, expiresIn, fetchImpl: this.fetchImpl })));
163
+ await input.onBeforeTransferPrepare?.();
164
+ prepared = await this.prepareTransfer({
165
+ sources: preparedSources,
166
+ destinations: preparedDestinations,
167
+ name: input.name,
168
+ testMode: input.testMode,
169
+ experimental: input.experimental,
170
+ signedUrlFlow: requestedSignedUrlFlow
171
+ });
172
+ if (!prepared.success) {
173
+ return prepared;
152
174
  }
175
+ assertPreparedTransferMatchesPlan(planned, prepared);
176
+ }
177
+ else {
178
+ preparedSources = await Promise.all(input.sources.map((source, index) => prepareProviderSource(source, { index, expiresIn, fetchImpl: this.fetchImpl })));
179
+ await input.onBeforeTransferPrepare?.();
180
+ prepared = await this.prepareTransfer({
181
+ sources: preparedSources,
182
+ destinations: preparedDestinations,
183
+ name: input.name,
184
+ testMode: input.testMode,
185
+ experimental: input.experimental,
186
+ signedUrlFlow: requestedSignedUrlFlow
187
+ });
188
+ if (!prepared.success) {
189
+ return prepared;
190
+ }
191
+ }
192
+ await input.onPrepared?.(prepared);
193
+ await input.throwIfCancelled?.(prepared.transfer_id);
194
+ const sourcesById = new Map();
195
+ preparedSources.forEach((preparedSource, index) => {
196
+ const source = input.sources[index];
197
+ if (!source) {
198
+ throw new Error(`missing provider source config for prepared source ${preparedSource.source_id}`);
199
+ }
200
+ sourcesById.set(preparedSource.source_id, source);
201
+ });
202
+ const signedUrlFlow = prepared.signed_url_flow ?? requestedSignedUrlFlow;
203
+ if (signedUrlFlow === "signed_url_v1") {
204
+ const chunksBySource = checksumChunksBySource(prepared.chunk_plan, destinationsById);
205
+ checksumCache.assertManyWarmed(chunksBySource);
153
206
  }
154
207
  const routes = [];
155
208
  const multipartUploads = new Map();
@@ -373,15 +426,100 @@ export class BeamClient {
373
426
  return (text ? JSON.parse(text) : {});
374
427
  }
375
428
  }
429
+ function checksumChunksBySource(chunkPlan, destinationsById) {
430
+ const chunksBySource = new Map();
431
+ for (const chunk of chunkPlan) {
432
+ const needsChecksum = chunk.destinations.some((target) => {
433
+ const destination = destinationsById.get(target.destination_id);
434
+ return destination ? isS3CompatibleProvider(destination) : false;
435
+ });
436
+ if (!needsChecksum)
437
+ continue;
438
+ const chunks = chunksBySource.get(chunk.source_id) ?? [];
439
+ chunks.push(chunk);
440
+ chunksBySource.set(chunk.source_id, chunks);
441
+ }
442
+ return chunksBySource;
443
+ }
444
+ function mapSourcesByPreparedId(preparedSources, providerSources) {
445
+ const sourcesById = new Map();
446
+ preparedSources.forEach((preparedSource, index) => {
447
+ const source = providerSources[index];
448
+ if (!source) {
449
+ throw new Error(`missing provider source config for planned source ${preparedSource.source_id}`);
450
+ }
451
+ sourcesById.set(preparedSource.source_id, source);
452
+ });
453
+ return sourcesById;
454
+ }
455
+ function failedPlanAsPrepareResponse(plan) {
456
+ return {
457
+ success: false,
458
+ transfer_id: "",
459
+ chunk_size: plan.chunk_size,
460
+ total_size: plan.total_size,
461
+ total_sources: plan.total_sources,
462
+ total_destinations: plan.total_destinations,
463
+ logical_chunks: plan.logical_chunks,
464
+ total_chunks: plan.total_chunks,
465
+ chunk_plan: plan.chunk_plan ?? [],
466
+ signed_url_flow: plan.signed_url_flow,
467
+ error: plan.error,
468
+ message: plan.message
469
+ };
470
+ }
471
+ function assertPreparedTransferMatchesPlan(plan, prepared) {
472
+ const summaryMatches = plan.chunk_size === prepared.chunk_size &&
473
+ plan.total_size === prepared.total_size &&
474
+ plan.total_sources === prepared.total_sources &&
475
+ plan.total_destinations === prepared.total_destinations &&
476
+ plan.logical_chunks === prepared.logical_chunks &&
477
+ plan.total_chunks === prepared.total_chunks;
478
+ if (!summaryMatches || planSignature(plan.chunk_plan) !== planSignature(prepared.chunk_plan)) {
479
+ throw new Error("prepared transfer chunk plan does not match preflight checksum plan");
480
+ }
481
+ }
482
+ function planSignature(chunks) {
483
+ return JSON.stringify(chunks.map((chunk) => ({
484
+ chunk_index: chunk.chunk_index,
485
+ source_id: chunk.source_id,
486
+ source_chunk_index: chunk.source_chunk_index,
487
+ source_offset: chunk.source_offset,
488
+ chunk_size: chunk.chunk_size,
489
+ destinations: chunk.destinations.map((target) => {
490
+ const metadata = target.metadata ?? {};
491
+ return {
492
+ destination_id: target.destination_id,
493
+ provider: target.provider,
494
+ final_object_key: typeof metadata.final_object_key === "string" ? metadata.final_object_key : target.object_key,
495
+ part_number: typeof metadata.part_number === "number" ? metadata.part_number : undefined,
496
+ delivery_index: typeof metadata.delivery_index === "number" ? metadata.delivery_index : undefined
497
+ };
498
+ })
499
+ })));
500
+ }
501
+ async function emitChecksumEvent(input, event) {
502
+ if (!input.onChecksumEvent)
503
+ return;
504
+ await input.onChecksumEvent(event);
505
+ }
376
506
  class ChunkChecksumCache {
377
507
  options;
378
508
  queue = [];
379
509
  cache = new Map();
380
510
  active = 0;
511
+ sealed = false;
381
512
  constructor(options) {
382
513
  this.options = options;
383
514
  }
384
515
  preloadMany(source, chunks) {
516
+ if (this.sealed) {
517
+ const missing = this.uniqueChunks(chunks).filter((chunk) => !this.cache.has(this.key(chunk)));
518
+ if (missing.length) {
519
+ throw new Error(`checksum cache missing ${missing.length} chunk(s) after preflight warmup; refusing post-prepare source reads`);
520
+ }
521
+ return;
522
+ }
385
523
  const pending = this.uniqueChunks(chunks).filter((chunk) => !this.cache.has(this.key(chunk)));
386
524
  for (const chunk of pending) {
387
525
  this.cache.set(this.key(chunk), deferredChecksum());
@@ -390,10 +528,18 @@ class ChunkChecksumCache {
390
528
  void this.run(() => this.fillGroup(source, group)).catch(() => undefined);
391
529
  }
392
530
  }
531
+ async ensureMany(source, chunks) {
532
+ const unique = this.uniqueChunks(chunks);
533
+ this.preloadMany(source, unique);
534
+ await Promise.all(unique.map((chunk) => this.get(source, chunk)));
535
+ }
393
536
  get(source, chunk) {
394
537
  const key = this.key(chunk);
395
538
  let existing = this.cache.get(key);
396
539
  if (!existing) {
540
+ if (this.sealed) {
541
+ return Promise.reject(new Error(`checksum cache missing ${key} after preflight warmup; refusing post-prepare source read`));
542
+ }
397
543
  existing = deferredChecksum();
398
544
  this.cache.set(key, existing);
399
545
  void this.run(() => this.fillGroup(source, [chunk])).catch(() => undefined);
@@ -403,6 +549,7 @@ class ChunkChecksumCache {
403
549
  async fillGroup(source, chunks) {
404
550
  const unresolved = new Set(chunks.map((chunk) => this.key(chunk)));
405
551
  try {
552
+ await this.options.throwIfCancelled?.();
406
553
  const misses = [];
407
554
  for (const chunk of chunks) {
408
555
  const key = this.key(chunk);
@@ -411,17 +558,18 @@ class ChunkChecksumCache {
411
558
  continue;
412
559
  const cached = await this.options.provider?.getContentMd5({ source, chunk, algorithm: "content-md5" }).catch(() => undefined);
413
560
  if (cached) {
414
- this.emit({ event: "checksum_cache_hit", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
561
+ await this.emit({ event: "checksum_cache_hit", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
415
562
  deferred.resolve(cached);
416
563
  unresolved.delete(key);
417
564
  }
418
565
  else {
419
- this.emit({ event: "checksum_cache_miss", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
566
+ await this.emit({ event: "checksum_cache_miss", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
420
567
  misses.push(chunk);
421
568
  }
422
569
  }
423
570
  if (!misses.length)
424
571
  return;
572
+ await this.options.throwIfCancelled?.();
425
573
  const startedAt = Date.now();
426
574
  const checksums = await computeChunksContentMd5({
427
575
  source,
@@ -431,6 +579,7 @@ class ChunkChecksumCache {
431
579
  groupSizeBytes: this.options.groupSizeBytes
432
580
  });
433
581
  for (const chunk of misses) {
582
+ await this.options.throwIfCancelled?.();
434
583
  const checksum = checksums.get(chunk.source_chunk_index);
435
584
  if (!checksum)
436
585
  throw new Error(`checksum missing for source chunk ${chunk.source_chunk_index}`);
@@ -438,9 +587,9 @@ class ChunkChecksumCache {
438
587
  const deferred = this.cache.get(this.key(chunk));
439
588
  deferred?.resolve(checksum);
440
589
  unresolved.delete(this.key(chunk));
441
- this.emit({ event: "checksum_cold_chunk_cached", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
590
+ await this.emit({ event: "checksum_cold_chunk_cached", sourceId: chunk.source_id, sourceChunkIndex: chunk.source_chunk_index, bytes: chunk.chunk_size });
442
591
  }
443
- this.emit({
592
+ await this.emit({
444
593
  event: "checksum_cold_group_completed",
445
594
  sourceId: misses[0]?.source_id ?? source.id ?? "source",
446
595
  chunkCount: misses.length,
@@ -496,6 +645,21 @@ class ChunkChecksumCache {
496
645
  key(chunk) {
497
646
  return `${chunk.source_id}:${chunk.source_chunk_index}`;
498
647
  }
648
+ seal() {
649
+ this.sealed = true;
650
+ }
651
+ assertManyWarmed(chunksBySource) {
652
+ const missing = [];
653
+ for (const chunks of chunksBySource.values()) {
654
+ for (const chunk of this.uniqueChunks(chunks)) {
655
+ if (!this.cache.has(this.key(chunk)))
656
+ missing.push(this.key(chunk));
657
+ }
658
+ }
659
+ if (missing.length) {
660
+ throw new Error(`checksum cache missing ${missing.length} chunk(s) after preflight warmup; refusing post-prepare source reads`);
661
+ }
662
+ }
499
663
  run(task) {
500
664
  return new Promise((resolve, reject) => {
501
665
  const start = () => {
@@ -511,10 +675,10 @@ class ChunkChecksumCache {
511
675
  this.queue.push(start);
512
676
  });
513
677
  }
514
- emit(event) {
678
+ async emit(event) {
515
679
  if (!this.options.onEvent)
516
680
  return;
517
- void Promise.resolve(this.options.onEvent(event)).catch(() => undefined);
681
+ await this.options.onEvent(event);
518
682
  }
519
683
  }
520
684
  function deferredChecksum() {
package/dist/models.d.ts CHANGED
@@ -177,7 +177,7 @@ export interface ChunkChecksumProvider {
177
177
  setContentMd5?(input: ChunkChecksumProviderSetInput): Promise<void>;
178
178
  }
179
179
  export interface ChunkChecksumEvent {
180
- event: "checksum_cache_hit" | "checksum_cache_miss" | "checksum_cold_chunk_cached" | "checksum_cold_group_completed";
180
+ event: "checksum_warmup_started" | "checksum_warmup_completed" | "checksum_cache_hit" | "checksum_cache_miss" | "checksum_cold_chunk_cached" | "checksum_cold_group_completed";
181
181
  sourceId: string;
182
182
  sourceChunkIndex?: number;
183
183
  chunkCount?: number;
@@ -218,25 +218,30 @@ export interface ProviderTransferCreateInput {
218
218
  * Defaults to true. Set to false to only prepare and attach signed URLs.
219
219
  */
220
220
  distribute?: boolean;
221
+ onPlanned?: (plan: TransferPlanResponse) => void | Promise<void>;
222
+ onBeforeTransferPrepare?: () => void | Promise<void>;
221
223
  onPrepared?: (prepared: TransferPrepareResponse) => void | Promise<void>;
222
- throwIfCancelled?: (transferId: string) => void | Promise<void>;
224
+ throwIfCancelled?: (transferId?: string) => void | Promise<void>;
223
225
  signedUrlFlow?: SignedUrlFlow;
224
226
  checksumProvider?: ChunkChecksumProvider;
225
227
  checksumReadConcurrency?: number;
226
228
  checksumGroupSizeBytes?: number;
227
229
  onChecksumEvent?: (event: ChunkChecksumEvent) => void | Promise<void>;
228
230
  }
229
- export interface PreparedHttpSource {
231
+ export interface PlanningHttpSource {
230
232
  source_id: string;
231
233
  type: "http";
232
234
  provider?: string;
233
- url: string;
235
+ url?: string;
234
236
  size: number;
235
237
  filename?: string;
236
238
  headers?: Record<string, string>;
237
239
  expires_at?: string;
238
240
  metadata?: Record<string, unknown>;
239
241
  }
242
+ export interface PreparedHttpSource extends PlanningHttpSource {
243
+ url: string;
244
+ }
240
245
  export interface PreparedDestination {
241
246
  destination_id: string;
242
247
  provider: string;
@@ -1,4 +1,4 @@
1
- import type { ChunkDestinationSigningTarget, ChunkSigningPlanItem, PreparedDestination, PreparedHttpSource, ProviderDestinationConfig, ProviderSourceConfig, R2ProviderConfig, S3CompatibleProviderConfig, S3ProviderConfig, SignedChunkRoute } from "./models.js";
1
+ import type { ChunkDestinationSigningTarget, ChunkSigningPlanItem, PlanningHttpSource, PreparedDestination, PreparedHttpSource, ProviderDestinationConfig, ProviderSourceConfig, R2ProviderConfig, S3CompatibleProviderConfig, S3ProviderConfig, SignedChunkRoute } from "./models.js";
2
2
  export type AnyS3CompatibleProviderConfig = S3ProviderConfig | R2ProviderConfig | S3CompatibleProviderConfig;
3
3
  export declare function nowIso(): string;
4
4
  export declare function expiresAtIso(expiresIn: number): string;
@@ -7,6 +7,10 @@ export declare function prepareProviderSource(source: ProviderSourceConfig, opti
7
7
  expiresIn?: number;
8
8
  fetchImpl?: typeof fetch;
9
9
  }): Promise<PreparedHttpSource>;
10
+ export declare function prepareProviderSourceForPlan(source: ProviderSourceConfig, options?: {
11
+ index?: number;
12
+ fetchImpl?: typeof fetch;
13
+ }): Promise<PlanningHttpSource>;
10
14
  export declare function prepareProviderDestination(destination: ProviderDestinationConfig, options?: {
11
15
  index?: number;
12
16
  }): PreparedDestination;
@@ -50,6 +50,42 @@ export async function prepareProviderSource(source, options = {}) {
50
50
  }
51
51
  return unsupportedProviderConfig(source);
52
52
  }
53
+ export async function prepareProviderSourceForPlan(source, options = {}) {
54
+ const index = options.index ?? 0;
55
+ if (isS3CompatibleProvider(source)) {
56
+ const endpoint = s3CompatibleEndpoint(source);
57
+ const client = createS3CompatibleClient(source, endpoint);
58
+ const head = await client.send(new HeadObjectCommand({ Bucket: source.bucket, Key: source.key }));
59
+ return {
60
+ source_id: source.id ?? `src_${index}`,
61
+ type: "http",
62
+ provider: source.provider,
63
+ size: Number(head.ContentLength ?? 0),
64
+ filename: filename(source.key),
65
+ metadata: {
66
+ ...s3CompatibleMetadata(source, endpoint),
67
+ content_length: Number(head.ContentLength ?? 0),
68
+ ...(head.ETag ? { etag: head.ETag } : {}),
69
+ ...(head.LastModified ? { last_modified: head.LastModified.toISOString() } : {}),
70
+ ...(head.VersionId ? { version_id: head.VersionId } : {})
71
+ }
72
+ };
73
+ }
74
+ if (isHippiusProvider(source)) {
75
+ const fetchImpl = options.fetchImpl ?? globalThis.fetch;
76
+ const baseUrl = source.base_url ?? "https://api.hippius.com";
77
+ const size = await hippiusObjectSize(fetchImpl, baseUrl, source.api_token, source.bucket, source.key);
78
+ return {
79
+ source_id: source.id ?? `src_${index}`,
80
+ type: "http",
81
+ provider: "hippius",
82
+ size,
83
+ filename: filename(source.key),
84
+ metadata: { bucket: source.bucket, key: source.key, base_url: baseUrl }
85
+ };
86
+ }
87
+ return unsupportedProviderConfig(source);
88
+ }
53
89
  export function prepareProviderDestination(destination, options = {}) {
54
90
  const index = options.index ?? 0;
55
91
  const destinationId = destination.id ?? `dst_${index}`;
@@ -364,6 +400,18 @@ async function computeChunkGroupContentMd5(input) {
364
400
  const last = chunks[chunks.length - 1];
365
401
  const sourceOffset = first.source_offset;
366
402
  const totalSize = last.source_offset + last.chunk_size - sourceOffset;
403
+ if (isS3CompatibleProvider(input.source)) {
404
+ const client = createS3CompatibleClient(input.source);
405
+ const response = await client.send(new GetObjectCommand({
406
+ Bucket: input.source.bucket,
407
+ Key: input.source.key,
408
+ Range: `bytes=${sourceOffset}-${sourceOffset + totalSize - 1}`
409
+ }));
410
+ if (!response.Body) {
411
+ throw new Error(`chunk checksum read returned no body for ${input.source.bucket}/${input.source.key}`);
412
+ }
413
+ return hashChunkGroupContentMd5(chunks, totalSize, bodyToAsyncBytes(response.Body));
414
+ }
367
415
  const sourceRoute = await signSourceRangeRoute({
368
416
  source: input.source,
369
417
  fallbackUrl: first.source_url,
@@ -378,6 +426,12 @@ async function computeChunkGroupContentMd5(input) {
378
426
  const message = `chunk checksum fetch failed with ${response.status}: ${await response.text()}`;
379
427
  throw Object.assign(new Error(message), { status: response.status });
380
428
  }
429
+ if (response.body) {
430
+ return hashChunkGroupContentMd5(chunks, totalSize, readableStreamToAsyncBytes(response.body));
431
+ }
432
+ return hashChunkGroupContentMd5(chunks, totalSize, [new Uint8Array(await response.arrayBuffer())]);
433
+ }
434
+ async function hashChunkGroupContentMd5(chunks, totalSize, body) {
381
435
  const output = new Map();
382
436
  let chunkIndex = 0;
383
437
  let bytesInChunk = 0;
@@ -404,23 +458,69 @@ async function computeChunkGroupContentMd5(input) {
404
458
  }
405
459
  }
406
460
  };
407
- if (response.body) {
408
- const reader = response.body.getReader();
461
+ for await (const value of body) {
462
+ consume(value);
463
+ }
464
+ if (totalRead !== totalSize || output.size !== chunks.length) {
465
+ throw new Error(`chunk checksum fetch size mismatch: expected ${totalSize}, got ${totalRead}`);
466
+ }
467
+ return output;
468
+ }
469
+ async function* readableStreamToAsyncBytes(stream) {
470
+ const reader = stream.getReader();
471
+ try {
409
472
  while (true) {
410
473
  const { done, value } = await reader.read();
411
474
  if (done)
412
475
  break;
413
476
  if (value)
414
- consume(value);
477
+ yield value;
415
478
  }
416
479
  }
417
- else {
418
- consume(new Uint8Array(await response.arrayBuffer()));
480
+ finally {
481
+ reader.releaseLock();
419
482
  }
420
- if (totalRead !== totalSize || output.size !== chunks.length) {
421
- throw new Error(`chunk checksum fetch size mismatch: expected ${totalSize}, got ${totalRead}`);
483
+ }
484
+ async function* bodyToAsyncBytes(body) {
485
+ if (body instanceof Uint8Array) {
486
+ yield body;
487
+ return;
422
488
  }
423
- return output;
489
+ if (typeof body === "string") {
490
+ yield new TextEncoder().encode(body);
491
+ return;
492
+ }
493
+ if (body && typeof body.transformToByteArray === "function") {
494
+ yield new Uint8Array(await body.transformToByteArray());
495
+ return;
496
+ }
497
+ if (body && typeof body.arrayBuffer === "function") {
498
+ yield new Uint8Array(await body.arrayBuffer());
499
+ return;
500
+ }
501
+ if (body && typeof body.getReader === "function") {
502
+ yield* readableStreamToAsyncBytes(body);
503
+ return;
504
+ }
505
+ if (body && Symbol.asyncIterator in Object(body)) {
506
+ for await (const value of body) {
507
+ yield toUint8Array(value);
508
+ }
509
+ return;
510
+ }
511
+ throw new Error("provider checksum read returned an unsupported body stream");
512
+ }
513
+ function toUint8Array(value) {
514
+ if (value instanceof Uint8Array)
515
+ return value;
516
+ if (typeof value === "string")
517
+ return new TextEncoder().encode(value);
518
+ if (ArrayBuffer.isView(value)) {
519
+ return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
520
+ }
521
+ if (value instanceof ArrayBuffer)
522
+ return new Uint8Array(value);
523
+ throw new Error("provider checksum stream yielded an unsupported chunk");
424
524
  }
425
525
  async function retryChecksumRead(task) {
426
526
  let lastError;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@beam-network/sdk",
3
- "version": "0.2.11",
3
+ "version": "0.2.13",
4
4
  "description": "TypeScript SDK for BEAM transfer creation and management.",
5
5
  "type": "module",
6
6
  "license": "MIT",