icechunk-js 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.cts CHANGED
@@ -159,12 +159,20 @@ interface ChunkRef {
159
159
  /** Last modified timestamp (seconds since epoch) for virtual chunks */
160
160
  checksumLastModified: number;
161
161
  }
162
- /** Chunk references for a single array */
162
+ /**
163
+ * Chunk references for a single array, resolved lazily from the manifest's
164
+ * FlatBuffer. Keeping refs in the buffer avoids building a JS object per chunk,
165
+ * which would OOM on large virtual shards.
166
+ */
163
167
  interface ArrayManifest {
164
168
  /** Node ID of the array */
165
169
  nodeId: ObjectId8;
166
- /** Chunk references, sorted by index */
167
- refs: ChunkRef[];
170
+ /** Number of chunk refs (sorted by index). */
171
+ readonly numRefs: number;
172
+ /** Coordinate index of ref `i`, for the binary search in `findChunkRef`. */
173
+ refIndex(i: number): number[];
174
+ /** Materialize the full `ChunkRef` at position `i`. */
175
+ refAt(i: number): ChunkRef;
168
176
  }
169
177
  /** Manifest containing chunk references for multiple arrays */
170
178
  interface Manifest {
@@ -329,6 +337,35 @@ type RangeCoalescingFn = (store: AsyncReadable$1, opts?: {
329
337
  coalesceSize?: number;
330
338
  }) => AsyncReadable$1;
331
339
 
340
+ /**
341
+ * Parser for icechunk Repo FlatBuffer format (v2).
342
+ *
343
+ * Uses flatc-generated TypeScript classes for type-safe field access.
344
+ */
345
+
346
+ /**
347
+ * S3 store config carried by a virtual chunk container, mirroring Rust's
348
+ * `S3Options`. Drives endpoint/addressing when translating `s3://` chunk
349
+ * locations to HTTPS — notably `region`, which lets dotted-name buckets
350
+ * address their regional endpoint directly (required for browser CORS, since
351
+ * the global endpoint's region redirect carries no CORS headers).
352
+ */
353
+ interface S3ContainerConfig {
354
+ region?: string;
355
+ endpointUrl?: string;
356
+ forcePathStyle?: boolean;
357
+ }
358
+ /**
359
+ * A virtual chunk container from the repo config, mirroring Rust's
360
+ * `VirtualChunkContainer`. `name` resolves `vcc://name/...` references;
361
+ * `urlPrefix` matches absolute chunk locations (longest prefix wins).
362
+ */
363
+ interface VirtualChunkContainer {
364
+ name: string | null;
365
+ urlPrefix: string;
366
+ s3?: S3ContainerConfig;
367
+ }
368
+
332
369
  /**
333
370
  * ReadSession - Read-only session for accessing icechunk data.
334
371
  */
@@ -380,7 +417,7 @@ declare class ReadSession {
380
417
  private nextFetchClientId;
381
418
  private rangeCoalescerIds?;
382
419
  private nextRangeCoalescerId;
383
- /** VCC name → url_prefix map for resolving `vcc://` chunk locations. */
420
+ /** Virtual chunk containers (from repo config) for resolving chunk locations. */
384
421
  private virtualChunkContainers;
385
422
  private constructor();
386
423
  private getFetchClientKey;
@@ -398,7 +435,7 @@ declare class ReadSession {
398
435
  */
399
436
  static open(storage: Storage, snapshotId: Uint8Array, options?: RequestOptions & {
400
437
  maxManifestCacheSize?: number;
401
- virtualChunkContainers?: Map<string, string>;
438
+ virtualChunkContainers?: VirtualChunkContainer[];
402
439
  }): Promise<ReadSession>;
403
440
  /** Load and parse a snapshot from storage */
404
441
  private static loadSnapshot;
@@ -576,7 +613,15 @@ interface IcechunkStoreOptions {
576
613
  * - Route through a proxy
577
614
  */
578
615
  fetchClient?: FetchClient;
579
- /** Maximum number of manifests to cache in the LRU cache (default: 100) */
616
+ /**
617
+ * Maximum number of manifests to keep in the LRU cache (default: 100).
618
+ *
619
+ * This bounds the *number* of cached manifests, not their total bytes. Each
620
+ * entry retains a whole decompressed manifest shard — hundreds of MB for
621
+ * fine-level virtual stores — so a single oversized shard, or a few held at
622
+ * once, can still dominate memory. Lower this when reading stores with very
623
+ * large manifests.
624
+ */
580
625
  maxManifestCacheSize?: number;
581
626
  /**
582
627
  * Zarrita-backed range coalescing function for chunk payload reads.
package/dist/index.d.ts CHANGED
@@ -159,12 +159,20 @@ interface ChunkRef {
159
159
  /** Last modified timestamp (seconds since epoch) for virtual chunks */
160
160
  checksumLastModified: number;
161
161
  }
162
- /** Chunk references for a single array */
162
+ /**
163
+ * Chunk references for a single array, resolved lazily from the manifest's
164
+ * FlatBuffer. Keeping refs in the buffer avoids building a JS object per chunk,
165
+ * which would OOM on large virtual shards.
166
+ */
163
167
  interface ArrayManifest {
164
168
  /** Node ID of the array */
165
169
  nodeId: ObjectId8;
166
- /** Chunk references, sorted by index */
167
- refs: ChunkRef[];
170
+ /** Number of chunk refs (sorted by index). */
171
+ readonly numRefs: number;
172
+ /** Coordinate index of ref `i`, for the binary search in `findChunkRef`. */
173
+ refIndex(i: number): number[];
174
+ /** Materialize the full `ChunkRef` at position `i`. */
175
+ refAt(i: number): ChunkRef;
168
176
  }
169
177
  /** Manifest containing chunk references for multiple arrays */
170
178
  interface Manifest {
@@ -329,6 +337,35 @@ type RangeCoalescingFn = (store: AsyncReadable$1, opts?: {
329
337
  coalesceSize?: number;
330
338
  }) => AsyncReadable$1;
331
339
 
340
+ /**
341
+ * Parser for icechunk Repo FlatBuffer format (v2).
342
+ *
343
+ * Uses flatc-generated TypeScript classes for type-safe field access.
344
+ */
345
+
346
+ /**
347
+ * S3 store config carried by a virtual chunk container, mirroring Rust's
348
+ * `S3Options`. Drives endpoint/addressing when translating `s3://` chunk
349
+ * locations to HTTPS — notably `region`, which lets dotted-name buckets
350
+ * address their regional endpoint directly (required for browser CORS, since
351
+ * the global endpoint's region redirect carries no CORS headers).
352
+ */
353
+ interface S3ContainerConfig {
354
+ region?: string;
355
+ endpointUrl?: string;
356
+ forcePathStyle?: boolean;
357
+ }
358
+ /**
359
+ * A virtual chunk container from the repo config, mirroring Rust's
360
+ * `VirtualChunkContainer`. `name` resolves `vcc://name/...` references;
361
+ * `urlPrefix` matches absolute chunk locations (longest prefix wins).
362
+ */
363
+ interface VirtualChunkContainer {
364
+ name: string | null;
365
+ urlPrefix: string;
366
+ s3?: S3ContainerConfig;
367
+ }
368
+
332
369
  /**
333
370
  * ReadSession - Read-only session for accessing icechunk data.
334
371
  */
@@ -380,7 +417,7 @@ declare class ReadSession {
380
417
  private nextFetchClientId;
381
418
  private rangeCoalescerIds?;
382
419
  private nextRangeCoalescerId;
383
- /** VCC name → url_prefix map for resolving `vcc://` chunk locations. */
420
+ /** Virtual chunk containers (from repo config) for resolving chunk locations. */
384
421
  private virtualChunkContainers;
385
422
  private constructor();
386
423
  private getFetchClientKey;
@@ -398,7 +435,7 @@ declare class ReadSession {
398
435
  */
399
436
  static open(storage: Storage, snapshotId: Uint8Array, options?: RequestOptions & {
400
437
  maxManifestCacheSize?: number;
401
- virtualChunkContainers?: Map<string, string>;
438
+ virtualChunkContainers?: VirtualChunkContainer[];
402
439
  }): Promise<ReadSession>;
403
440
  /** Load and parse a snapshot from storage */
404
441
  private static loadSnapshot;
@@ -576,7 +613,15 @@ interface IcechunkStoreOptions {
576
613
  * - Route through a proxy
577
614
  */
578
615
  fetchClient?: FetchClient;
579
- /** Maximum number of manifests to cache in the LRU cache (default: 100) */
616
+ /**
617
+ * Maximum number of manifests to keep in the LRU cache (default: 100).
618
+ *
619
+ * This bounds the *number* of cached manifests, not their total bytes. Each
620
+ * entry retains a whole decompressed manifest shard — hundreds of MB for
621
+ * fine-level virtual stores — so a single oversized shard, or a few held at
622
+ * once, can still dominate memory. Lower this when reading stores with very
623
+ * large manifests.
624
+ */
580
625
  maxManifestCacheSize?: number;
581
626
  /**
582
627
  * Zarrita-backed range coalescing function for chunk payload reads.
package/dist/index.js CHANGED
@@ -1226,7 +1226,7 @@ function parseRepo(data) {
1226
1226
  return { repo, specVersion, snapshotsLength, virtualChunkContainers };
1227
1227
  }
1228
1228
  function parseVirtualChunkContainers(repo) {
1229
- const result = /* @__PURE__ */ new Map();
1229
+ const result = [];
1230
1230
  const configBytes = repo.configArray();
1231
1231
  if (!configBytes || configBytes.length === 0) return result;
1232
1232
  let config;
@@ -1244,12 +1244,27 @@ function parseVirtualChunkContainers(repo) {
1244
1244
  if (!isRecord(containers)) return result;
1245
1245
  for (const container of Object.values(containers)) {
1246
1246
  if (!isRecord(container)) continue;
1247
- const { name, url_prefix: urlPrefix } = container;
1248
- if (typeof name !== "string" || typeof urlPrefix !== "string") continue;
1249
- result.set(name, urlPrefix);
1247
+ const urlPrefix = container.url_prefix;
1248
+ if (typeof urlPrefix !== "string") continue;
1249
+ const name = typeof container.name === "string" ? container.name : null;
1250
+ const s3 = parseS3Store(container.store);
1251
+ result.push(s3 ? { name, urlPrefix, s3 } : { name, urlPrefix });
1250
1252
  }
1253
+ result.sort((a, b) => b.urlPrefix.length - a.urlPrefix.length);
1251
1254
  return result;
1252
1255
  }
1256
+ function parseS3Store(store) {
1257
+ if (!isRecord(store)) return void 0;
1258
+ const s3 = store.s3 ?? store.s3_compatible ?? store.tigris;
1259
+ if (!isRecord(s3)) return void 0;
1260
+ const cfg = {};
1261
+ if (typeof s3.region === "string") cfg.region = s3.region;
1262
+ if (typeof s3.endpoint_url === "string") cfg.endpointUrl = s3.endpoint_url;
1263
+ if (typeof s3.force_path_style === "boolean") {
1264
+ cfg.forcePathStyle = s3.force_path_style;
1265
+ }
1266
+ return cfg;
1267
+ }
1253
1268
  function isRecord(value) {
1254
1269
  return typeof value === "object" && value !== null && !Array.isArray(value);
1255
1270
  }
@@ -2388,7 +2403,7 @@ function parseManifest(data) {
2388
2403
  for (let i = 0; i < arraysLength; i++) {
2389
2404
  const fbsArray = fbsManifest.arrays(i);
2390
2405
  if (fbsArray) {
2391
- arrays.push(parseArrayManifest(fbsArray, decodeLocation));
2406
+ arrays.push(makeArrayManifest(fbsArray, decodeLocation));
2392
2407
  }
2393
2408
  }
2394
2409
  return { id, arrays };
@@ -2416,22 +2431,32 @@ function makeLocationDecoder(fbsManifest) {
2416
2431
  return textDecoder.decode(decompressed);
2417
2432
  };
2418
2433
  }
2419
- function parseArrayManifest(fbsArray, decodeLocation) {
2434
+ function makeArrayManifest(fbsArray, decodeLocation) {
2420
2435
  const nodeIdObj = fbsArray.nodeId();
2421
2436
  if (!nodeIdObj)
2422
2437
  throw new Error("ArrayManifest missing required node_id field");
2423
2438
  const nodeId = asObjectId8(
2424
2439
  nodeIdObj.bb.bytes().slice(nodeIdObj.bb_pos, nodeIdObj.bb_pos + 8)
2425
2440
  );
2426
- const refsLength = fbsArray.refsLength();
2427
- const refs = [];
2428
- for (let i = 0; i < refsLength; i++) {
2429
- const fbsRef = fbsArray.refs(i);
2430
- if (fbsRef) {
2431
- refs.push(parseChunkRef(fbsRef, decodeLocation));
2441
+ const numRefs = fbsArray.refsLength();
2442
+ const probe = new ChunkRef();
2443
+ return {
2444
+ nodeId,
2445
+ numRefs,
2446
+ refIndex(i) {
2447
+ const fbsRef = fbsArray.refs(i, probe);
2448
+ if (!fbsRef) throw new Error(`Manifest ref ${i} out of range`);
2449
+ const indexLength = fbsRef.indexLength();
2450
+ const index = new Array(indexLength);
2451
+ for (let d = 0; d < indexLength; d++) index[d] = fbsRef.index(d);
2452
+ return index;
2453
+ },
2454
+ refAt(i) {
2455
+ const fbsRef = fbsArray.refs(i);
2456
+ if (!fbsRef) throw new Error(`Manifest ref ${i} out of range`);
2457
+ return parseChunkRef(fbsRef, decodeLocation);
2432
2458
  }
2433
- }
2434
- return { nodeId, refs };
2459
+ };
2435
2460
  }
2436
2461
  function parseChunkRef(fbsRef, decodeLocation) {
2437
2462
  const indexLength = fbsRef.indexLength();
@@ -2482,7 +2507,7 @@ function parseChunkRef(fbsRef, decodeLocation) {
2482
2507
  function findChunkRef(manifest, nodeId, coords) {
2483
2508
  const arrayManifest = binarySearchArrayManifest(manifest.arrays, nodeId);
2484
2509
  if (!arrayManifest) return null;
2485
- return binarySearchChunkRef(arrayManifest.refs, coords);
2510
+ return binarySearchChunkRef(arrayManifest, coords);
2486
2511
  }
2487
2512
  function binarySearchArrayManifest(arrays, nodeId) {
2488
2513
  let low = 0;
@@ -2514,14 +2539,14 @@ function compareCoords(a, b) {
2514
2539
  }
2515
2540
  return a.length - b.length;
2516
2541
  }
2517
- function binarySearchChunkRef(refs, coords) {
2542
+ function binarySearchChunkRef(array, coords) {
2518
2543
  let low = 0;
2519
- let high = refs.length - 1;
2544
+ let high = array.numRefs - 1;
2520
2545
  while (low <= high) {
2521
2546
  const mid = Math.floor((low + high) / 2);
2522
- const cmp = compareCoords(refs[mid].index, coords);
2547
+ const cmp = compareCoords(array.refIndex(mid), coords);
2523
2548
  if (cmp === 0) {
2524
- return refs[mid];
2549
+ return array.refAt(mid);
2525
2550
  } else if (cmp < 0) {
2526
2551
  low = mid + 1;
2527
2552
  } else {
@@ -2897,29 +2922,53 @@ function parseTransactionLog(data) {
2897
2922
  function expectedRangeLength(range) {
2898
2923
  return "suffixLength" in range ? range.suffixLength : range.length;
2899
2924
  }
2925
+ function regionalS3RedirectUrl(url, response) {
2926
+ if (response.status !== 301 && response.status !== 307) return null;
2927
+ const region = response.headers.get("x-amz-bucket-region");
2928
+ if (!region) return null;
2929
+ const globalPrefix = "https://s3.amazonaws.com/";
2930
+ if (!url.startsWith(globalPrefix)) return null;
2931
+ return `https://s3.${region}.amazonaws.com/${url.slice(globalPrefix.length)}`;
2932
+ }
2900
2933
  function makeUrlStore(opts) {
2901
- const { url, fetchClient, conditionalHeaders } = opts;
2902
- async function doFetch(init) {
2903
- return fetchClient ? await fetchClient.fetch(url, init) : await fetch(url, init);
2934
+ const { fetchClient, conditionalHeaders } = opts;
2935
+ let pinnedUrl = opts.url;
2936
+ async function doFetch(target, init) {
2937
+ return fetchClient ? await fetchClient.fetch(target, init) : await fetch(target, init);
2904
2938
  }
2905
2939
  return {
2906
2940
  async get() {
2907
2941
  throw new Error(
2908
- `Virtual chunk URL store for ${url} only supports ranged reads`
2942
+ `Virtual chunk URL store for ${pinnedUrl} only supports ranged reads`
2909
2943
  );
2910
2944
  },
2911
2945
  async getRange(_key, range, options) {
2912
2946
  const headers = conditionalHeaders ? { ...conditionalHeaders } : {};
2913
2947
  headers.Range = "suffixLength" in range ? `bytes=-${range.suffixLength}` : `bytes=${range.offset}-${range.offset + range.length - 1}`;
2914
- const response = await doFetch({ headers, signal: options?.signal });
2948
+ let requestUrl = pinnedUrl;
2949
+ let response = await doFetch(requestUrl, {
2950
+ headers,
2951
+ signal: options?.signal
2952
+ });
2953
+ const regionalUrl = regionalS3RedirectUrl(requestUrl, response);
2954
+ if (regionalUrl) {
2955
+ response.body?.cancel().catch(() => {
2956
+ });
2957
+ requestUrl = regionalUrl;
2958
+ pinnedUrl = regionalUrl;
2959
+ response = await doFetch(requestUrl, {
2960
+ headers,
2961
+ signal: options?.signal
2962
+ });
2963
+ }
2915
2964
  if (response.status === 412) {
2916
2965
  throw new Error(
2917
- `Virtual chunk at ${url} failed integrity check \u2014 data has been modified since snapshot was created`
2966
+ `Virtual chunk at ${requestUrl} failed integrity check \u2014 data has been modified since snapshot was created`
2918
2967
  );
2919
2968
  }
2920
2969
  if (response.status !== 200 && response.status !== 206) {
2921
2970
  throw new Error(
2922
- `Failed to fetch virtual chunk from ${url}: ${response.status} ${response.statusText}`
2971
+ `Failed to fetch virtual chunk from ${requestUrl}: ${response.status} ${response.statusText}`
2923
2972
  );
2924
2973
  }
2925
2974
  const data = new Uint8Array(await response.arrayBuffer());
@@ -2927,21 +2976,21 @@ function makeUrlStore(opts) {
2927
2976
  const expected = expectedRangeLength(range);
2928
2977
  if (data.length === expected) return data;
2929
2978
  throw new Error(
2930
- `Virtual range response size mismatch for ${url}: expected ${expected} bytes, got ${data.length}`
2979
+ `Virtual range response size mismatch for ${requestUrl}: expected ${expected} bytes, got ${data.length}`
2931
2980
  );
2932
2981
  }
2933
2982
  if ("offset" in range) {
2934
2983
  const end = range.offset + range.length;
2935
2984
  if (data.length >= end) return data.slice(range.offset, end);
2936
2985
  throw new Error(
2937
- `Virtual range request not honored for ${url}: need at least ${end} bytes for fallback slicing, got ${data.length}`
2986
+ `Virtual range request not honored for ${requestUrl}: need at least ${end} bytes for fallback slicing, got ${data.length}`
2938
2987
  );
2939
2988
  }
2940
2989
  if (data.length >= range.suffixLength) {
2941
2990
  return data.slice(data.length - range.suffixLength);
2942
2991
  }
2943
2992
  throw new Error(
2944
- `Virtual suffix range request not honored for ${url}: need at least ${range.suffixLength} bytes for fallback slicing, got ${data.length}`
2993
+ `Virtual suffix range request not honored for ${requestUrl}: need at least ${range.suffixLength} bytes for fallback slicing, got ${data.length}`
2945
2994
  );
2946
2995
  }
2947
2996
  };
@@ -3009,7 +3058,7 @@ var ReadSession = class _ReadSession {
3009
3058
  nextFetchClientId = 1;
3010
3059
  rangeCoalescerIds;
3011
3060
  nextRangeCoalescerId = 1;
3012
- /** VCC name → url_prefix map for resolving `vcc://` chunk locations. */
3061
+ /** Virtual chunk containers (from repo config) for resolving chunk locations. */
3013
3062
  virtualChunkContainers;
3014
3063
  constructor(storage, snapshot, specVersion, maxManifestCacheSize = 100, virtualChunkContainers) {
3015
3064
  this.storage = storage;
@@ -3017,7 +3066,7 @@ var ReadSession = class _ReadSession {
3017
3066
  this.specVersion = specVersion;
3018
3067
  this.manifestCache = new LRUCache(maxManifestCacheSize);
3019
3068
  this.manifestLoader = singleFlight(this.manifestCache);
3020
- this.virtualChunkContainers = virtualChunkContainers ?? /* @__PURE__ */ new Map();
3069
+ this.virtualChunkContainers = virtualChunkContainers ?? [];
3021
3070
  }
3022
3071
  getFetchClientKey(fetchClient) {
3023
3072
  if (!fetchClient) return "default";
@@ -3403,14 +3452,14 @@ var ReadSession = class _ReadSession {
3403
3452
  );
3404
3453
  }
3405
3454
  case "virtual": {
3406
- const absoluteLocation = expandVccUrl(
3455
+ const { url: absoluteLocation, container } = resolveVirtualChunkLocation(
3407
3456
  payload.location,
3408
3457
  this.virtualChunkContainers
3409
3458
  );
3410
- const httpUrl = translateToHttpUrl(
3411
- absoluteLocation,
3412
- options?.azureAccount
3413
- );
3459
+ const httpUrl = translateToHttpUrl(absoluteLocation, {
3460
+ azureAccount: options?.azureAccount,
3461
+ s3: container?.s3
3462
+ });
3414
3463
  const store = await this.getVirtualStoreForPayload(
3415
3464
  httpUrl,
3416
3465
  payload,
@@ -3473,14 +3522,14 @@ var ReadSession = class _ReadSession {
3473
3522
  case "virtual": {
3474
3523
  const absoluteStart = payload.offset + rangeStart;
3475
3524
  const expectedSize = rangeEnd - rangeStart;
3476
- const absoluteLocation = expandVccUrl(
3525
+ const { url: absoluteLocation, container } = resolveVirtualChunkLocation(
3477
3526
  payload.location,
3478
3527
  this.virtualChunkContainers
3479
3528
  );
3480
- const httpUrl = translateToHttpUrl(
3481
- absoluteLocation,
3482
- options?.azureAccount
3483
- );
3529
+ const httpUrl = translateToHttpUrl(absoluteLocation, {
3530
+ azureAccount: options?.azureAccount,
3531
+ s3: container?.s3
3532
+ });
3484
3533
  const store = await this.getVirtualStoreForPayload(
3485
3534
  httpUrl,
3486
3535
  payload,
@@ -3550,44 +3599,62 @@ function compareUtf8Bytes(a, b) {
3550
3599
  return bytesA.length - bytesB.length;
3551
3600
  }
3552
3601
  var VCC_SCHEME = "vcc://";
3553
- function expandVccUrl(location, containers) {
3554
- if (!location.startsWith(VCC_SCHEME)) return location;
3555
- const rest = location.slice(VCC_SCHEME.length);
3556
- const slash = rest.indexOf("/");
3557
- if (slash === -1) {
3558
- throw new Error(
3559
- `Invalid vcc:// URL "${location}": missing "/" after container name`
3560
- );
3561
- }
3562
- const name = rest.slice(0, slash);
3563
- const relativePath = rest.slice(slash + 1);
3564
- const urlPrefix = containers.get(name);
3565
- if (urlPrefix === void 0) {
3566
- if (containers.size === 0) return location;
3567
- throw new Error(
3568
- `Unknown virtual chunk container "${name}" referenced by ${location}`
3569
- );
3602
+ function resolveVirtualChunkLocation(location, containers) {
3603
+ if (location.startsWith(VCC_SCHEME)) {
3604
+ const rest = location.slice(VCC_SCHEME.length);
3605
+ const slash = rest.indexOf("/");
3606
+ if (slash === -1) {
3607
+ throw new Error(
3608
+ `Invalid vcc:// URL "${location}": missing "/" after container name`
3609
+ );
3610
+ }
3611
+ const name = rest.slice(0, slash);
3612
+ const relativePath = rest.slice(slash + 1);
3613
+ const container2 = containers.find((c) => c.name === name);
3614
+ if (!container2) {
3615
+ if (containers.length === 0) return { url: location };
3616
+ throw new Error(
3617
+ `Unknown virtual chunk container "${name}" referenced by ${location}`
3618
+ );
3619
+ }
3620
+ return {
3621
+ url: ensureTrailingSlash(container2.urlPrefix) + relativePath,
3622
+ container: container2
3623
+ };
3570
3624
  }
3571
- const normalizedPrefix = urlPrefix.endsWith("/") ? urlPrefix : `${urlPrefix}/`;
3572
- return normalizedPrefix + relativePath;
3625
+ const container = containers.find(
3626
+ (c) => location.startsWith(ensureTrailingSlash(c.urlPrefix))
3627
+ );
3628
+ return { url: location, container };
3629
+ }
3630
+ function ensureTrailingSlash(prefix) {
3631
+ return prefix.endsWith("/") ? prefix : `${prefix}/`;
3632
+ }
3633
+ function buildS3HttpUrl(bucket, key, s3) {
3634
+ if (s3?.endpointUrl) {
3635
+ const base = s3.endpointUrl.replace(/\/+$/, "");
3636
+ return `${base}/${bucket}/${key}`;
3637
+ }
3638
+ const region = s3?.region;
3639
+ const pathStyle = s3?.forcePathStyle === true || bucket.includes(".");
3640
+ if (pathStyle) {
3641
+ const host2 = region ? `s3.${region}.${awsEndpointSuffix(region)}` : "s3.amazonaws.com";
3642
+ return `https://${host2}/${bucket}/${key}`;
3643
+ }
3644
+ const host = region ? `${bucket}.s3.${region}.${awsEndpointSuffix(region)}` : `${bucket}.s3.amazonaws.com`;
3645
+ return `https://${host}/${key}`;
3573
3646
  }
3574
- function translateToHttpUrl(url, azureAccount) {
3647
+ function awsEndpointSuffix(region) {
3648
+ return region.startsWith("cn-") ? "amazonaws.com.cn" : "amazonaws.com";
3649
+ }
3650
+ function translateToHttpUrl(url, options) {
3651
+ const azureAccount = options?.azureAccount;
3575
3652
  if (url.startsWith("s3://")) {
3576
3653
  const rest = url.slice(5);
3577
3654
  const slashIndex = rest.indexOf("/");
3578
- if (slashIndex === -1) {
3579
- const bucket2 = rest;
3580
- if (bucket2.includes(".")) {
3581
- return `https://s3.amazonaws.com/${bucket2}/`;
3582
- }
3583
- return `https://${bucket2}.s3.amazonaws.com/`;
3584
- }
3585
- const bucket = rest.slice(0, slashIndex);
3586
- const key = rest.slice(slashIndex + 1);
3587
- if (bucket.includes(".")) {
3588
- return `https://s3.amazonaws.com/${bucket}/${key}`;
3589
- }
3590
- return `https://${bucket}.s3.amazonaws.com/${key}`;
3655
+ const bucket = slashIndex === -1 ? rest : rest.slice(0, slashIndex);
3656
+ const key = slashIndex === -1 ? "" : rest.slice(slashIndex + 1);
3657
+ return buildS3HttpUrl(bucket, key, options?.s3);
3591
3658
  }
3592
3659
  if (url.startsWith("gs://") || url.startsWith("gcs://")) {
3593
3660
  const prefixLen = url.startsWith("gs://") ? 5 : 6;