icechunk-js 0.3.2 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -71,12 +71,12 @@ var StorageError = class extends Error {
71
71
  this.name = "StorageError";
72
72
  }
73
73
  };
74
- var AbortError = class extends Error {
75
- constructor(message = "Operation aborted") {
76
- super(message);
77
- this.name = "AbortError";
78
- }
79
- };
74
+ function hasErrorName(error) {
75
+ return typeof error === "object" && error !== null && "name" in error;
76
+ }
77
+ function isAbortError(error) {
78
+ return hasErrorName(error) && error.name === "AbortError";
79
+ }
80
80
 
81
81
  // src/format/constants.ts
82
82
  var PATHS = {
@@ -198,6 +198,7 @@ function decodeObjectId8(str) {
198
198
 
199
199
  // src/format/flatbuffers/repo-parser.ts
200
200
  var import_flatbuffers = require("flatbuffers");
201
+ var flexbuffers = __toESM(require("flatbuffers/js/flexbuffers.js"), 1);
201
202
 
202
203
  // src/format/flatbuffers/generated/metadata-item.ts
203
204
  var MetadataItem = class _MetadataItem {
@@ -684,7 +685,36 @@ function parseRepo(data) {
684
685
  );
685
686
  }
686
687
  const snapshotsLength = repo.snapshotsLength();
687
- return { repo, specVersion, snapshotsLength };
688
+ const virtualChunkContainers = parseVirtualChunkContainers(repo);
689
+ return { repo, specVersion, snapshotsLength, virtualChunkContainers };
690
+ }
691
+ function parseVirtualChunkContainers(repo) {
692
+ const result = /* @__PURE__ */ new Map();
693
+ const configBytes = repo.configArray();
694
+ if (!configBytes || configBytes.length === 0) return result;
695
+ let config;
696
+ try {
697
+ const ab = configBytes.buffer.slice(
698
+ configBytes.byteOffset,
699
+ configBytes.byteOffset + configBytes.byteLength
700
+ );
701
+ config = flexbuffers.toObject(ab);
702
+ } catch {
703
+ return result;
704
+ }
705
+ if (!isRecord(config)) return result;
706
+ const containers = config.virtual_chunk_containers;
707
+ if (!isRecord(containers)) return result;
708
+ for (const container of Object.values(containers)) {
709
+ if (!isRecord(container)) continue;
710
+ const { name, url_prefix: urlPrefix } = container;
711
+ if (typeof name !== "string" || typeof urlPrefix !== "string") continue;
712
+ result.set(name, urlPrefix);
713
+ }
714
+ return result;
715
+ }
716
+ function isRecord(value) {
717
+ return typeof value === "object" && value !== null && !Array.isArray(value);
688
718
  }
689
719
  function compareUtf8ByteOrder(aBytes, bBytes) {
690
720
  const minLen = Math.min(aBytes.length, bBytes.length);
@@ -827,6 +857,85 @@ var LRUCache = class {
827
857
  }
828
858
  };
829
859
 
860
+ // src/cache/single-flight.ts
861
+ function singleFlight(cache) {
862
+ const pending = /* @__PURE__ */ new Map();
863
+ return {
864
+ load(key, fetcher, signal) {
865
+ if (signal?.aborted) return Promise.reject(makeAbortError());
866
+ const hit = cache.get(key);
867
+ if (hit !== void 0) return Promise.resolve(hit);
868
+ let entry = pending.get(key);
869
+ if (!entry) {
870
+ const controller = new AbortController();
871
+ const promise = invokeFetcher(fetcher, controller.signal).then((value) => {
872
+ cache.set(key, value);
873
+ return value;
874
+ }).finally(() => {
875
+ if (pending.get(key)?.promise === promise) pending.delete(key);
876
+ });
877
+ const fresh = { promise, controller, refCount: 0 };
878
+ pending.set(key, fresh);
879
+ entry = fresh;
880
+ }
881
+ const owned = entry;
882
+ owned.refCount++;
883
+ const release = () => {
884
+ owned.refCount--;
885
+ if (owned.refCount > 0) return;
886
+ if (pending.get(key) === owned) {
887
+ pending.delete(key);
888
+ owned.controller.abort();
889
+ }
890
+ };
891
+ if (!signal) {
892
+ return owned.promise.then(
893
+ (value) => {
894
+ release();
895
+ return value;
896
+ },
897
+ (error) => {
898
+ release();
899
+ throw error;
900
+ }
901
+ );
902
+ }
903
+ return new Promise((resolve, reject) => {
904
+ let done = false;
905
+ const finish = () => {
906
+ if (done) return false;
907
+ done = true;
908
+ signal.removeEventListener("abort", handleAbort);
909
+ release();
910
+ return true;
911
+ };
912
+ const handleAbort = () => {
913
+ if (finish()) reject(makeAbortError());
914
+ };
915
+ signal.addEventListener("abort", handleAbort, { once: true });
916
+ owned.promise.then(
917
+ (value) => {
918
+ if (finish()) resolve(value);
919
+ },
920
+ (error) => {
921
+ if (finish()) reject(error);
922
+ }
923
+ );
924
+ });
925
+ }
926
+ };
927
+ }
928
+ function invokeFetcher(fetcher, signal) {
929
+ try {
930
+ return fetcher(signal);
931
+ } catch (error) {
932
+ return Promise.reject(error);
933
+ }
934
+ }
935
+ function makeAbortError() {
936
+ return new DOMException("The operation was aborted.", "AbortError");
937
+ }
938
+
830
939
  // src/format/flatbuffers/types.ts
831
940
  function asObjectId12(bytes) {
832
941
  if (bytes.length !== 12) {
@@ -1818,11 +1927,11 @@ function getChunkPayload(ref) {
1818
1927
 
1819
1928
  // src/format/flatbuffers/metadata.ts
1820
1929
  var import_msgpack = require("@msgpack/msgpack");
1821
- var flexbuffers = __toESM(require("flatbuffers/js/flexbuffers.js"), 1);
1930
+ var flexbuffers2 = __toESM(require("flatbuffers/js/flexbuffers.js"), 1);
1822
1931
  function deserializeMetadata(items, specVersion) {
1823
1932
  const result = /* @__PURE__ */ Object.create(null);
1824
1933
  for (const item of items) {
1825
- result[item.name] = specVersion === 1 /* V1_0 */ ? (0, import_msgpack.decode)(item.value) : flexbuffers.toObject(
1934
+ result[item.name] = specVersion === 1 /* V1_0 */ ? (0, import_msgpack.decode)(item.value) : flexbuffers2.toObject(
1826
1935
  item.value.buffer.slice(
1827
1936
  item.value.byteOffset,
1828
1937
  item.value.byteOffset + item.value.byteLength
@@ -2154,17 +2263,219 @@ function parseTransactionLog(data) {
2154
2263
  };
2155
2264
  }
2156
2265
 
2266
+ // src/reader/range-coalescer.ts
2267
+ function expectedRangeLength(range) {
2268
+ return "suffixLength" in range ? range.suffixLength : range.length;
2269
+ }
2270
+ function makeUrlStore(opts) {
2271
+ const { url, fetchClient, conditionalHeaders } = opts;
2272
+ async function doFetch(init) {
2273
+ return fetchClient ? await fetchClient.fetch(url, init) : await fetch(url, init);
2274
+ }
2275
+ return {
2276
+ async get() {
2277
+ throw new Error(
2278
+ `Virtual chunk URL store for ${url} only supports ranged reads`
2279
+ );
2280
+ },
2281
+ async getRange(_key, range, options) {
2282
+ const headers = conditionalHeaders ? { ...conditionalHeaders } : {};
2283
+ headers.Range = "suffixLength" in range ? `bytes=-${range.suffixLength}` : `bytes=${range.offset}-${range.offset + range.length - 1}`;
2284
+ const response = await doFetch({ headers, signal: options?.signal });
2285
+ if (response.status === 412) {
2286
+ throw new Error(
2287
+ `Virtual chunk at ${url} failed integrity check \u2014 data has been modified since snapshot was created`
2288
+ );
2289
+ }
2290
+ if (response.status !== 200 && response.status !== 206) {
2291
+ throw new Error(
2292
+ `Failed to fetch virtual chunk from ${url}: ${response.status} ${response.statusText}`
2293
+ );
2294
+ }
2295
+ const data = new Uint8Array(await response.arrayBuffer());
2296
+ if (response.status === 206) {
2297
+ const expected = expectedRangeLength(range);
2298
+ if (data.length === expected) return data;
2299
+ throw new Error(
2300
+ `Virtual range response size mismatch for ${url}: expected ${expected} bytes, got ${data.length}`
2301
+ );
2302
+ }
2303
+ if ("offset" in range) {
2304
+ const end = range.offset + range.length;
2305
+ if (data.length >= end) return data.slice(range.offset, end);
2306
+ throw new Error(
2307
+ `Virtual range request not honored for ${url}: need at least ${end} bytes for fallback slicing, got ${data.length}`
2308
+ );
2309
+ }
2310
+ if (data.length >= range.suffixLength) {
2311
+ return data.slice(data.length - range.suffixLength);
2312
+ }
2313
+ throw new Error(
2314
+ `Virtual suffix range request not honored for ${url}: need at least ${range.suffixLength} bytes for fallback slicing, got ${data.length}`
2315
+ );
2316
+ }
2317
+ };
2318
+ }
2319
+ function makeStorageStore(storage) {
2320
+ return {
2321
+ async get(key, options) {
2322
+ const storageOptions = options?.signal ? { signal: options.signal } : void 0;
2323
+ return storage.getObject(key, void 0, storageOptions);
2324
+ },
2325
+ async getRange(key, range, options) {
2326
+ const storageOptions = options?.signal ? { signal: options.signal } : void 0;
2327
+ if ("suffixLength" in range) {
2328
+ throw new Error(
2329
+ `Storage suffix ranges are not supported for ${key}; convert suffixLength to offset/length before reading`
2330
+ );
2331
+ }
2332
+ const storageRange = {
2333
+ start: range.offset,
2334
+ end: range.offset + range.length
2335
+ };
2336
+ const data = await storage.getObject(key, storageRange, storageOptions);
2337
+ if (data.length === range.length) return data;
2338
+ if (data.length >= storageRange.end) {
2339
+ return data.slice(storageRange.start, storageRange.end);
2340
+ }
2341
+ throw new Error(
2342
+ `Storage returned ${data.length} bytes for ${key} range ${storageRange.start}-${storageRange.end - 1}; expected ${range.length} bytes`
2343
+ );
2344
+ }
2345
+ };
2346
+ }
2347
+
2157
2348
  // src/reader/session.ts
2349
+ var RANGE_COALESCE_SIZE = 32 * 1024;
2350
+ var RANGE_STORE_CACHE_SIZE = 256;
2351
+ function makeRangeStoreCacheKey(parts) {
2352
+ return JSON.stringify(parts);
2353
+ }
2158
2354
  var ReadSession = class _ReadSession {
2159
2355
  storage;
2160
2356
  snapshot;
2161
2357
  specVersion;
2162
2358
  manifestCache;
2163
- constructor(storage, snapshot, specVersion, maxManifestCacheSize = 100) {
2359
+ /**
2360
+ * Single-flight loader over `manifestCache`: concurrent misses for the
2361
+ * same manifest share one fetch instead of racing N identical GETs.
2362
+ * Critical for fan-out reads that all need the same manifest (e.g. the
2363
+ * many parallel chunk reads issued when a renderer crosses into a new
2364
+ * pyramid level).
2365
+ */
2366
+ manifestLoader;
2367
+ /**
2368
+ * Bounded cache of `AsyncReadable`s wrapped with zarrita's range coalescer.
2369
+ * The cache key includes request-option identities that must not share one
2370
+ * coalescing queue, notably virtual fetch clients and checksum headers.
2371
+ *
2372
+ * Promise slots are inserted synchronously on first use, so
2373
+ * concurrent requests for the same partition share one coalescing window
2374
+ * instead of racing to create parallel stores.
2375
+ */
2376
+ rangeStores;
2377
+ nativeStore;
2378
+ fetchClientIds;
2379
+ nextFetchClientId = 1;
2380
+ rangeCoalescerIds;
2381
+ nextRangeCoalescerId = 1;
2382
+ /** VCC name → url_prefix map for resolving `vcc://` chunk locations. */
2383
+ virtualChunkContainers;
2384
+ constructor(storage, snapshot, specVersion, maxManifestCacheSize = 100, virtualChunkContainers) {
2164
2385
  this.storage = storage;
2165
2386
  this.snapshot = snapshot;
2166
2387
  this.specVersion = specVersion;
2167
2388
  this.manifestCache = new LRUCache(maxManifestCacheSize);
2389
+ this.manifestLoader = singleFlight(this.manifestCache);
2390
+ this.virtualChunkContainers = virtualChunkContainers ?? /* @__PURE__ */ new Map();
2391
+ }
2392
+ getFetchClientKey(fetchClient) {
2393
+ if (!fetchClient) return "default";
2394
+ if (!this.fetchClientIds) this.fetchClientIds = /* @__PURE__ */ new WeakMap();
2395
+ let id = this.fetchClientIds.get(fetchClient);
2396
+ if (id === void 0) {
2397
+ id = this.nextFetchClientId;
2398
+ this.nextFetchClientId = id + 1;
2399
+ this.fetchClientIds.set(fetchClient, id);
2400
+ }
2401
+ return String(id);
2402
+ }
2403
+ getRangeCoalescerKey(withRangeCoalescing) {
2404
+ if (!this.rangeCoalescerIds) this.rangeCoalescerIds = /* @__PURE__ */ new WeakMap();
2405
+ let id = this.rangeCoalescerIds.get(withRangeCoalescing);
2406
+ if (id === void 0) {
2407
+ id = this.nextRangeCoalescerId;
2408
+ this.nextRangeCoalescerId = id + 1;
2409
+ this.rangeCoalescerIds.set(withRangeCoalescing, id);
2410
+ }
2411
+ return String(id);
2412
+ }
2413
+ getRangeStore(partitionKey, createStore, withRangeCoalescing) {
2414
+ if (!this.rangeStores) {
2415
+ this.rangeStores = new LRUCache(RANGE_STORE_CACHE_SIZE);
2416
+ }
2417
+ const stores = this.rangeStores;
2418
+ const cacheKey = makeRangeStoreCacheKey([
2419
+ ...partitionKey,
2420
+ ["coalescer", this.getRangeCoalescerKey(withRangeCoalescing)]
2421
+ ]);
2422
+ const cached = stores.get(cacheKey);
2423
+ if (cached) return cached;
2424
+ const raw = createStore();
2425
+ const promise = Promise.resolve().then(
2426
+ () => withRangeCoalescing(raw, { coalesceSize: RANGE_COALESCE_SIZE })
2427
+ ).catch((error) => {
2428
+ if (stores.get(cacheKey) === promise) stores.delete(cacheKey);
2429
+ throw error;
2430
+ });
2431
+ stores.set(cacheKey, promise);
2432
+ return promise;
2433
+ }
2434
+ getNativeStore(options) {
2435
+ if (!this.nativeStore) {
2436
+ this.nativeStore = makeStorageStore(this.storage);
2437
+ }
2438
+ const raw = this.nativeStore;
2439
+ const withRangeCoalescing = options?.withRangeCoalescing;
2440
+ if (!withRangeCoalescing) {
2441
+ return Promise.resolve(raw);
2442
+ }
2443
+ return this.getRangeStore(["native"], () => raw, withRangeCoalescing);
2444
+ }
2445
+ getVirtualStoreForPayload(httpUrl, payload, options) {
2446
+ const validate = !!options?.validateChecksums;
2447
+ let conditionalHeaders;
2448
+ if (validate) {
2449
+ conditionalHeaders = {};
2450
+ if (payload.checksumEtag) {
2451
+ conditionalHeaders["If-Match"] = payload.checksumEtag;
2452
+ }
2453
+ if (payload.checksumLastModified > 0) {
2454
+ conditionalHeaders["If-Unmodified-Since"] = new Date(
2455
+ payload.checksumLastModified * 1e3
2456
+ ).toUTCString();
2457
+ }
2458
+ }
2459
+ const checksumKey = validate ? ["checked", payload.checksumEtag ?? "", payload.checksumLastModified] : ["unchecked"];
2460
+ const createStore = () => makeUrlStore({
2461
+ url: httpUrl,
2462
+ fetchClient: options?.fetchClient,
2463
+ conditionalHeaders
2464
+ });
2465
+ const withRangeCoalescing = options?.withRangeCoalescing;
2466
+ if (!withRangeCoalescing) {
2467
+ return Promise.resolve(createStore());
2468
+ }
2469
+ return this.getRangeStore(
2470
+ [
2471
+ "virtual",
2472
+ httpUrl,
2473
+ ["fetch", this.getFetchClientKey(options?.fetchClient)],
2474
+ checksumKey
2475
+ ],
2476
+ createStore,
2477
+ withRangeCoalescing
2478
+ );
2168
2479
  }
2169
2480
  /**
2170
2481
  * Open a read session for a specific snapshot.
@@ -2184,7 +2495,8 @@ var ReadSession = class _ReadSession {
2184
2495
  storage,
2185
2496
  snapshot,
2186
2497
  specVersion,
2187
- options?.maxManifestCacheSize
2498
+ options?.maxManifestCacheSize,
2499
+ options?.virtualChunkContainers
2188
2500
  );
2189
2501
  }
2190
2502
  /** Load and parse a snapshot from storage */
@@ -2202,22 +2514,40 @@ var ReadSession = class _ReadSession {
2202
2514
  specVersion: header.specVersion
2203
2515
  };
2204
2516
  }
2205
- /** Load and parse a manifest from storage */
2206
- async loadManifest(manifestId, options) {
2517
+ /**
2518
+ * Load and parse a manifest from storage.
2519
+ *
2520
+ * Concurrent calls for the same manifest are coalesced through
2521
+ * `manifestLoader` so a fan-out of chunk reads — the typical case
2522
+ * when many tiles cross into a new pyramid level at once — issues one
2523
+ * HTTP GET instead of one per caller. A caller's `signal` rejects only
2524
+ * that caller's await; the shared fetch is aborted only once every
2525
+ * active waiter has aborted.
2526
+ */
2527
+ loadManifest(manifestId, options) {
2207
2528
  const idStr = encodeObjectId12(manifestId);
2208
- const cached = this.manifestCache.get(idStr);
2209
- if (cached) return cached;
2529
+ return this.manifestLoader.load(
2530
+ idStr,
2531
+ (signal) => this.fetchManifest(idStr, signal),
2532
+ options?.signal
2533
+ );
2534
+ }
2535
+ /**
2536
+ * Fetch and parse a single manifest. The `signal` here is the
2537
+ * single-flight loader's own signal — it fires only when every waiter
2538
+ * has aborted, so a hung manifest fetch doesn't trap later callers
2539
+ * behind it.
2540
+ */
2541
+ async fetchManifest(idStr, signal) {
2210
2542
  const path = getManifestPath(idStr);
2211
- const data = await this.storage.getObject(path, void 0, options);
2543
+ const data = await this.storage.getObject(path, void 0, { signal });
2212
2544
  const header = parseHeader(data);
2213
2545
  validateFileType(header, 2 /* Manifest */);
2214
2546
  let flatbufferData = getDataAfterHeader(data);
2215
2547
  if (header.compression === 1 /* Zstd */) {
2216
2548
  flatbufferData = (0, import_fzstd2.decompress)(flatbufferData);
2217
2549
  }
2218
- const manifest = parseManifest(flatbufferData);
2219
- this.manifestCache.set(idStr, manifest);
2220
- return manifest;
2550
+ return parseManifest(flatbufferData);
2221
2551
  }
2222
2552
  /**
2223
2553
  * Get the snapshot ID.
@@ -2348,26 +2678,25 @@ var ReadSession = class _ReadSession {
2348
2678
  * @returns Chunk data bytes or null if not found
2349
2679
  */
2350
2680
  async getChunk(path, coords, options) {
2351
- if (options?.signal?.aborted) return null;
2681
+ options?.signal?.throwIfAborted();
2682
+ const requestOptions = toRequestOptions(options);
2352
2683
  const node = this.getNode(path);
2353
2684
  if (!node || node.nodeData.type !== "array") {
2354
2685
  return null;
2355
2686
  }
2356
2687
  const arrayData = node.nodeData;
2357
- try {
2358
- for (const manifestRef of arrayData.manifests) {
2359
- if (!this.coordsInExtents(coords, manifestRef.extents)) {
2360
- continue;
2361
- }
2362
- const manifest = await this.loadManifest(manifestRef.objectId, options);
2363
- const chunkRef = findChunkRef(manifest, node.id, coords);
2364
- if (!chunkRef) continue;
2365
- const payload = getChunkPayload(chunkRef);
2366
- return this.fetchChunkPayload(payload, options);
2688
+ for (const manifestRef of arrayData.manifests) {
2689
+ if (!this.coordsInExtents(coords, manifestRef.extents)) {
2690
+ continue;
2367
2691
  }
2368
- } catch (error) {
2369
- if (error instanceof AbortError) return null;
2370
- throw error;
2692
+ const manifest = await this.loadManifest(
2693
+ manifestRef.objectId,
2694
+ requestOptions
2695
+ );
2696
+ const chunkRef = findChunkRef(manifest, node.id, coords);
2697
+ if (!chunkRef) continue;
2698
+ const payload = getChunkPayload(chunkRef);
2699
+ return this.fetchChunkPayload(payload, options);
2371
2700
  }
2372
2701
  return null;
2373
2702
  }
@@ -2385,26 +2714,25 @@ var ReadSession = class _ReadSession {
2385
2714
  * @returns Chunk data bytes or null if not found
2386
2715
  */
2387
2716
  async getChunkRange(path, coords, range, options) {
2388
- if (options?.signal?.aborted) return null;
2717
+ options?.signal?.throwIfAborted();
2718
+ const requestOptions = toRequestOptions(options);
2389
2719
  const node = this.getNode(path);
2390
2720
  if (!node || node.nodeData.type !== "array") {
2391
2721
  return null;
2392
2722
  }
2393
2723
  const arrayData = node.nodeData;
2394
- try {
2395
- for (const manifestRef of arrayData.manifests) {
2396
- if (!this.coordsInExtents(coords, manifestRef.extents)) {
2397
- continue;
2398
- }
2399
- const manifest = await this.loadManifest(manifestRef.objectId, options);
2400
- const chunkRef = findChunkRef(manifest, node.id, coords);
2401
- if (!chunkRef) continue;
2402
- const payload = getChunkPayload(chunkRef);
2403
- return this.fetchChunkPayloadRange(payload, range, options);
2724
+ for (const manifestRef of arrayData.manifests) {
2725
+ if (!this.coordsInExtents(coords, manifestRef.extents)) {
2726
+ continue;
2404
2727
  }
2405
- } catch (error) {
2406
- if (error instanceof AbortError) return null;
2407
- throw error;
2728
+ const manifest = await this.loadManifest(
2729
+ manifestRef.objectId,
2730
+ requestOptions
2731
+ );
2732
+ const chunkRef = findChunkRef(manifest, node.id, coords);
2733
+ if (!chunkRef) continue;
2734
+ const payload = getChunkPayload(chunkRef);
2735
+ return this.fetchChunkPayloadRange(payload, range, options);
2408
2736
  }
2409
2737
  return null;
2410
2738
  }
@@ -2426,77 +2754,54 @@ var ReadSession = class _ReadSession {
2426
2754
  return payload.data;
2427
2755
  case "native": {
2428
2756
  const path = getChunkPath(encodeObjectId12(payload.chunkId));
2429
- const range = {
2430
- start: payload.offset,
2431
- end: payload.offset + payload.length
2432
- };
2433
- const data = await this.storage.getObject(path, range, options);
2434
- if (data.length === payload.length) return data;
2435
- if (data.length >= range.end) {
2436
- return data.slice(range.start, range.end);
2757
+ const requestedStart = payload.offset;
2758
+ const requestedEnd = payload.offset + payload.length;
2759
+ const store = await this.getNativeStore(options);
2760
+ const data = await store.getRange(
2761
+ path,
2762
+ { offset: requestedStart, length: payload.length },
2763
+ { signal: options?.signal }
2764
+ );
2765
+ if (!data) {
2766
+ throw new Error(
2767
+ `Failed to fetch native chunk from ${path} range ${requestedStart}-${requestedEnd - 1}: empty response`
2768
+ );
2437
2769
  }
2770
+ if (data.length === payload.length) return data;
2438
2771
  throw new Error(
2439
- `Storage returned ${data.length} bytes for ${path} range ${range.start}-${range.end - 1}; expected ${payload.length} bytes`
2772
+ `Storage returned ${data.length} bytes for ${path} range ${requestedStart}-${requestedEnd - 1}; expected ${payload.length} bytes`
2440
2773
  );
2441
2774
  }
2442
2775
  case "virtual": {
2443
- const httpUrl = translateToHttpUrl(
2776
+ const absoluteLocation = expandVccUrl(
2444
2777
  payload.location,
2778
+ this.virtualChunkContainers
2779
+ );
2780
+ const httpUrl = translateToHttpUrl(
2781
+ absoluteLocation,
2445
2782
  options?.azureAccount
2446
2783
  );
2447
- const headers = {
2448
- Range: `bytes=${payload.offset}-${payload.offset + payload.length - 1}`
2449
- };
2450
- if (options?.validateChecksums) {
2451
- if (payload.checksumEtag) {
2452
- headers["If-Match"] = payload.checksumEtag;
2453
- }
2454
- if (payload.checksumLastModified > 0) {
2455
- headers["If-Unmodified-Since"] = new Date(
2456
- payload.checksumLastModified * 1e3
2457
- ).toUTCString();
2458
- }
2459
- }
2460
- const fetchInit = {
2461
- headers,
2462
- signal: options?.signal
2463
- };
2464
- let response;
2465
- try {
2466
- const client = options?.fetchClient;
2467
- response = client ? await client.fetch(httpUrl, fetchInit) : await fetch(httpUrl, fetchInit);
2468
- } catch (error) {
2469
- if (error instanceof Error && error.name === "AbortError") {
2470
- throw new AbortError();
2471
- }
2472
- throw error;
2473
- }
2474
- if (response.status === 412) {
2784
+ const store = await this.getVirtualStoreForPayload(
2785
+ httpUrl,
2786
+ payload,
2787
+ options
2788
+ );
2789
+ const data = await store.getRange(
2790
+ "/",
2791
+ { offset: payload.offset, length: payload.length },
2792
+ { signal: options?.signal }
2793
+ );
2794
+ if (!data) {
2475
2795
  throw new Error(
2476
- `Virtual chunk at ${httpUrl} failed integrity check \u2014 data has been modified since snapshot was created`
2796
+ `Failed to fetch virtual chunk from ${httpUrl}: empty response`
2477
2797
  );
2478
2798
  }
2479
- if (response.status !== 200 && response.status !== 206) {
2799
+ if (data.length !== payload.length) {
2480
2800
  throw new Error(
2481
- `Failed to fetch virtual chunk from ${httpUrl}: ${response.status} ${response.statusText}`
2801
+ `Virtual range response size mismatch for ${httpUrl}: expected ${payload.length} bytes, got ${data.length}`
2482
2802
  );
2483
2803
  }
2484
- const data = new Uint8Array(await response.arrayBuffer());
2485
- if (response.status === 206) {
2486
- if (data.length !== payload.length) {
2487
- throw new Error(
2488
- `Virtual range response size mismatch for ${httpUrl}: expected ${payload.length} bytes, got ${data.length}`
2489
- );
2490
- }
2491
- return data;
2492
- }
2493
- const absoluteEnd = payload.offset + payload.length;
2494
- if (data.length >= absoluteEnd) {
2495
- return data.slice(payload.offset, absoluteEnd);
2496
- }
2497
- throw new Error(
2498
- `Virtual range request not honored for ${httpUrl}: need at least ${absoluteEnd} bytes for fallback slicing, got ${data.length}`
2499
- );
2804
+ return data;
2500
2805
  }
2501
2806
  }
2502
2807
  }
@@ -2516,80 +2821,57 @@ var ReadSession = class _ReadSession {
2516
2821
  return payload.data.slice(rangeStart, rangeEnd);
2517
2822
  case "native": {
2518
2823
  const path = getChunkPath(encodeObjectId12(payload.chunkId));
2519
- const storageRange = {
2520
- start: payload.offset + rangeStart,
2521
- end: payload.offset + rangeEnd
2522
- };
2824
+ const requestedStart = payload.offset + rangeStart;
2523
2825
  const expectedSize = rangeEnd - rangeStart;
2524
- const data = await this.storage.getObject(path, storageRange, options);
2525
- if (data.length === expectedSize) return data;
2526
- if (data.length >= storageRange.end) {
2527
- return data.slice(storageRange.start, storageRange.end);
2826
+ const requestedEnd = requestedStart + expectedSize;
2827
+ const store = await this.getNativeStore(options);
2828
+ const data = await store.getRange(
2829
+ path,
2830
+ { offset: requestedStart, length: expectedSize },
2831
+ { signal: options?.signal }
2832
+ );
2833
+ if (!data) {
2834
+ throw new Error(
2835
+ `Failed to fetch native chunk from ${path} range ${requestedStart}-${requestedEnd - 1}: empty response`
2836
+ );
2528
2837
  }
2838
+ if (data.length === expectedSize) return data;
2529
2839
  throw new Error(
2530
- `Storage returned ${data.length} bytes for ${path} range ${storageRange.start}-${storageRange.end - 1}; expected ${expectedSize} bytes`
2840
+ `Storage returned ${data.length} bytes for ${path} range ${requestedStart}-${requestedEnd - 1}; expected ${expectedSize} bytes`
2531
2841
  );
2532
2842
  }
2533
2843
  case "virtual": {
2534
2844
  const absoluteStart = payload.offset + rangeStart;
2535
- const absoluteEnd = payload.offset + rangeEnd;
2536
- const httpUrl = translateToHttpUrl(
2845
+ const expectedSize = rangeEnd - rangeStart;
2846
+ const absoluteLocation = expandVccUrl(
2537
2847
  payload.location,
2848
+ this.virtualChunkContainers
2849
+ );
2850
+ const httpUrl = translateToHttpUrl(
2851
+ absoluteLocation,
2538
2852
  options?.azureAccount
2539
2853
  );
2540
- const headers = {
2541
- Range: `bytes=${absoluteStart}-${absoluteEnd - 1}`
2542
- };
2543
- if (options?.validateChecksums) {
2544
- if (payload.checksumEtag) {
2545
- headers["If-Match"] = payload.checksumEtag;
2546
- }
2547
- if (payload.checksumLastModified > 0) {
2548
- headers["If-Unmodified-Since"] = new Date(
2549
- payload.checksumLastModified * 1e3
2550
- ).toUTCString();
2551
- }
2552
- }
2553
- const fetchInit = {
2554
- headers,
2555
- signal: options?.signal
2556
- };
2557
- let response;
2558
- try {
2559
- const client = options?.fetchClient;
2560
- response = client ? await client.fetch(httpUrl, fetchInit) : await fetch(httpUrl, fetchInit);
2561
- } catch (error) {
2562
- if (error instanceof Error && error.name === "AbortError") {
2563
- throw new AbortError();
2564
- }
2565
- throw error;
2566
- }
2567
- if (response.status === 412) {
2854
+ const store = await this.getVirtualStoreForPayload(
2855
+ httpUrl,
2856
+ payload,
2857
+ options
2858
+ );
2859
+ const data = await store.getRange(
2860
+ "/",
2861
+ { offset: absoluteStart, length: expectedSize },
2862
+ { signal: options?.signal }
2863
+ );
2864
+ if (!data) {
2568
2865
  throw new Error(
2569
- `Virtual chunk at ${httpUrl} failed integrity check \u2014 data has been modified since snapshot was created`
2866
+ `Failed to fetch virtual chunk from ${httpUrl}: empty response`
2570
2867
  );
2571
2868
  }
2572
- if (response.status !== 200 && response.status !== 206) {
2869
+ if (data.length !== expectedSize) {
2573
2870
  throw new Error(
2574
- `Failed to fetch virtual chunk from ${httpUrl}: ${response.status} ${response.statusText}`
2871
+ `Virtual range response size mismatch for ${httpUrl}: expected ${expectedSize} bytes, got ${data.length}`
2575
2872
  );
2576
2873
  }
2577
- const expectedSize = rangeEnd - rangeStart;
2578
- const data = new Uint8Array(await response.arrayBuffer());
2579
- if (response.status === 206) {
2580
- if (data.length !== expectedSize) {
2581
- throw new Error(
2582
- `Virtual range response size mismatch for ${httpUrl}: expected ${expectedSize} bytes, got ${data.length}`
2583
- );
2584
- }
2585
- return data;
2586
- }
2587
- if (data.length >= absoluteEnd) {
2588
- return data.slice(absoluteStart, absoluteEnd);
2589
- }
2590
- throw new Error(
2591
- `Virtual range request not honored for ${httpUrl}: need at least ${absoluteEnd} bytes for fallback slicing, got ${data.length}`
2592
- );
2874
+ return data;
2593
2875
  }
2594
2876
  }
2595
2877
  }
@@ -2613,6 +2895,19 @@ var ReadSession = class _ReadSession {
2613
2895
  }
2614
2896
  };
2615
2897
  var utf8Encoder = new TextEncoder();
2898
+ function toRequestOptions(options) {
2899
+ if (!options) return void 0;
2900
+ return {
2901
+ ...options.signal && { signal: options.signal },
2902
+ ...options.fetchClient && { fetchClient: options.fetchClient },
2903
+ ...options.validateChecksums !== void 0 && {
2904
+ validateChecksums: options.validateChecksums
2905
+ },
2906
+ ...options.azureAccount !== void 0 && {
2907
+ azureAccount: options.azureAccount
2908
+ }
2909
+ };
2910
+ }
2616
2911
  function compareUtf8Bytes(a, b) {
2617
2912
  const bytesA = utf8Encoder.encode(a);
2618
2913
  const bytesB = utf8Encoder.encode(b);
@@ -2624,6 +2919,28 @@ function compareUtf8Bytes(a, b) {
2624
2919
  }
2625
2920
  return bytesA.length - bytesB.length;
2626
2921
  }
2922
+ var VCC_SCHEME = "vcc://";
2923
+ function expandVccUrl(location, containers) {
2924
+ if (!location.startsWith(VCC_SCHEME)) return location;
2925
+ const rest = location.slice(VCC_SCHEME.length);
2926
+ const slash = rest.indexOf("/");
2927
+ if (slash === -1) {
2928
+ throw new Error(
2929
+ `Invalid vcc:// URL "${location}": missing "/" after container name`
2930
+ );
2931
+ }
2932
+ const name = rest.slice(0, slash);
2933
+ const relativePath = rest.slice(slash + 1);
2934
+ const urlPrefix = containers.get(name);
2935
+ if (urlPrefix === void 0) {
2936
+ if (containers.size === 0) return location;
2937
+ throw new Error(
2938
+ `Unknown virtual chunk container "${name}" referenced by ${location}`
2939
+ );
2940
+ }
2941
+ const normalizedPrefix = urlPrefix.endsWith("/") ? urlPrefix : `${urlPrefix}/`;
2942
+ return normalizedPrefix + relativePath;
2943
+ }
2627
2944
  function translateToHttpUrl(url, azureAccount) {
2628
2945
  if (url.startsWith("s3://")) {
2629
2946
  const rest = url.slice(5);
@@ -2699,7 +3016,7 @@ var Repository = class _Repository {
2699
3016
  * @param options - Optional request options (signal for cancellation)
2700
3017
  * @returns ParsedRepo if v2 format, null if v1 format
2701
3018
  * @throws Error if repo file exists but fails to parse
2702
- * @throws AbortError if the operation was aborted
3019
+ * @throws An error named "AbortError" if the operation was aborted
2703
3020
  */
2704
3021
  async loadRepoInfo(options) {
2705
3022
  if (this.repoInfoAttempted) {
@@ -2715,7 +3032,7 @@ var Repository = class _Repository {
2715
3032
  this.repoInfo = parseRepo(data);
2716
3033
  return this.repoInfo;
2717
3034
  } catch (error) {
2718
- if (error instanceof AbortError) {
3035
+ if (isAbortError(error)) {
2719
3036
  this.repoInfoAttempted = false;
2720
3037
  throw error;
2721
3038
  }
@@ -2780,7 +3097,7 @@ var Repository = class _Repository {
2780
3097
  }
2781
3098
  }
2782
3099
  } catch (error) {
2783
- if (error instanceof AbortError) {
3100
+ if (isAbortError(error)) {
2784
3101
  throw error;
2785
3102
  }
2786
3103
  if (legacyPath) {
@@ -2813,7 +3130,7 @@ var Repository = class _Repository {
2813
3130
  }
2814
3131
  }
2815
3132
  } catch (error) {
2816
- if (error instanceof AbortError) {
3133
+ if (isAbortError(error)) {
2817
3134
  throw error;
2818
3135
  }
2819
3136
  return legacyPath || null;
@@ -2854,7 +3171,7 @@ var Repository = class _Repository {
2854
3171
  }
2855
3172
  }
2856
3173
  } catch (error) {
2857
- if (error instanceof AbortError) {
3174
+ if (isAbortError(error)) {
2858
3175
  throw error;
2859
3176
  }
2860
3177
  throw new Error("Cannot list branches: storage does not support listing");
@@ -2896,7 +3213,7 @@ var Repository = class _Repository {
2896
3213
  }
2897
3214
  }
2898
3215
  } catch (error) {
2899
- if (error instanceof AbortError) {
3216
+ if (isAbortError(error)) {
2900
3217
  throw error;
2901
3218
  }
2902
3219
  throw new Error("Cannot list tags: storage does not support listing");
@@ -2926,7 +3243,10 @@ var Repository = class _Repository {
2926
3243
  if (!snapshotId2) {
2927
3244
  throw new Error(`Branch not found: ${name}`);
2928
3245
  }
2929
- return ReadSession.open(this.storage, snapshotId2, options);
3246
+ return ReadSession.open(this.storage, snapshotId2, {
3247
+ ...options,
3248
+ virtualChunkContainers: repoInfo.virtualChunkContainers
3249
+ });
2930
3250
  }
2931
3251
  const refDirPath = getBranchRefDirPath(name);
2932
3252
  const refPath = await this.findLatestRefFile(
@@ -2953,7 +3273,10 @@ var Repository = class _Repository {
2953
3273
  if (!snapshotId2) {
2954
3274
  throw new Error(`Tag not found: ${name}`);
2955
3275
  }
2956
- return ReadSession.open(this.storage, snapshotId2, options);
3276
+ return ReadSession.open(this.storage, snapshotId2, {
3277
+ ...options,
3278
+ virtualChunkContainers: repoInfo.virtualChunkContainers
3279
+ });
2957
3280
  }
2958
3281
  const refDirPath = getTagRefDirPath(name);
2959
3282
  const legacyPath = getTagRefPath(name);
@@ -2976,7 +3299,12 @@ var Repository = class _Repository {
2976
3299
  */
2977
3300
  async checkoutSnapshot(snapshotId, options) {
2978
3301
  const id = typeof snapshotId === "string" ? decodeObjectId12(snapshotId) : snapshotId;
2979
- return ReadSession.open(this.storage, id, options);
3302
+ const repoInfo = await this.loadRepoInfo(options);
3303
+ const virtualChunkContainers = repoInfo?.virtualChunkContainers;
3304
+ return ReadSession.open(this.storage, id, {
3305
+ ...options,
3306
+ ...virtualChunkContainers && { virtualChunkContainers }
3307
+ });
2980
3308
  }
2981
3309
  /**
2982
3310
  * Walk the snapshot history chain starting from the given session.
@@ -3055,9 +3383,7 @@ var HttpStorage = class {
3055
3383
  return headers;
3056
3384
  }
3057
3385
  async getObject(path, range, options) {
3058
- if (options?.signal?.aborted) {
3059
- throw new AbortError();
3060
- }
3386
+ options?.signal?.throwIfAborted();
3061
3387
  const url = this.getUrl(path);
3062
3388
  const headers = this.getHeaders(range);
3063
3389
  let response;
@@ -3070,9 +3396,7 @@ var HttpStorage = class {
3070
3396
  signal: options?.signal
3071
3397
  });
3072
3398
  } catch (error) {
3073
- if (error instanceof Error && error.name === "AbortError") {
3074
- throw new AbortError();
3075
- }
3399
+ if (isAbortError(error)) throw error;
3076
3400
  throw new StorageError(
3077
3401
  `Failed to fetch ${url}: ${error instanceof Error ? error.message : String(error)}`,
3078
3402
  error instanceof Error ? error : void 0
@@ -3090,9 +3414,7 @@ var HttpStorage = class {
3090
3414
  return new Uint8Array(arrayBuffer);
3091
3415
  }
3092
3416
  async exists(path, options) {
3093
- if (options?.signal?.aborted) {
3094
- throw new AbortError();
3095
- }
3417
+ options?.signal?.throwIfAborted();
3096
3418
  const url = this.getUrl(path);
3097
3419
  try {
3098
3420
  const response = await fetch(url, {
@@ -3103,9 +3425,7 @@ var HttpStorage = class {
3103
3425
  });
3104
3426
  return response.ok;
3105
3427
  } catch (error) {
3106
- if (error instanceof Error && error.name === "AbortError") {
3107
- throw new AbortError();
3108
- }
3428
+ if (isAbortError(error)) throw error;
3109
3429
  return false;
3110
3430
  }
3111
3431
  }
@@ -3123,8 +3443,9 @@ var IcechunkStore = class _IcechunkStore {
3123
3443
  fetchClient;
3124
3444
  validateChecksums;
3125
3445
  azureAccount;
3446
+ withRangeCoalescing;
3126
3447
  basePath = "";
3127
- constructor(session, fetchClient, validateChecksums, azureAccount) {
3448
+ constructor(session, fetchClient, validateChecksums, azureAccount, withRangeCoalescing) {
3128
3449
  if (!(session instanceof ReadSession)) {
3129
3450
  throw new Error(
3130
3451
  "IcechunkStore constructor is private. Use IcechunkStore.open() instead."
@@ -3134,6 +3455,7 @@ var IcechunkStore = class _IcechunkStore {
3134
3455
  this.fetchClient = fetchClient;
3135
3456
  this.validateChecksums = validateChecksums ?? false;
3136
3457
  this.azureAccount = azureAccount;
3458
+ this.withRangeCoalescing = withRangeCoalescing;
3137
3459
  }
3138
3460
  static async open(arg, options = {}) {
3139
3461
  if (arg instanceof ReadSession) {
@@ -3141,7 +3463,8 @@ var IcechunkStore = class _IcechunkStore {
3141
3463
  arg,
3142
3464
  options.fetchClient,
3143
3465
  options.validateChecksums,
3144
- options.azureAccount
3466
+ options.azureAccount,
3467
+ options.withRangeCoalescing
3145
3468
  );
3146
3469
  }
3147
3470
  const storage = typeof arg === "string" ? new HttpStorage(arg) : arg;
@@ -3169,7 +3492,8 @@ var IcechunkStore = class _IcechunkStore {
3169
3492
  session,
3170
3493
  options.fetchClient,
3171
3494
  options.validateChecksums,
3172
- options.azureAccount
3495
+ options.azureAccount,
3496
+ options.withRangeCoalescing
3173
3497
  );
3174
3498
  }
3175
3499
  /**
@@ -3185,7 +3509,7 @@ var IcechunkStore = class _IcechunkStore {
3185
3509
  * @returns Data bytes or undefined if not found
3186
3510
  */
3187
3511
  async get(key, opts) {
3188
- if (opts?.signal?.aborted) return void 0;
3512
+ opts?.signal?.throwIfAborted();
3189
3513
  const parsed = parseZarrKey(key);
3190
3514
  const resolvedPath = this.resolvePath(parsed.path);
3191
3515
  try {
@@ -3197,11 +3521,15 @@ var IcechunkStore = class _IcechunkStore {
3197
3521
  signal: opts?.signal,
3198
3522
  ...this.fetchClient && { fetchClient: this.fetchClient },
3199
3523
  validateChecksums: this.validateChecksums,
3200
- azureAccount: this.azureAccount
3524
+ azureAccount: this.azureAccount,
3525
+ ...this.withRangeCoalescing && {
3526
+ withRangeCoalescing: this.withRangeCoalescing
3527
+ }
3201
3528
  });
3202
3529
  return chunk ?? void 0;
3203
- } catch {
3204
- return void 0;
3530
+ } catch (err) {
3531
+ if (err instanceof NotFoundError) return void 0;
3532
+ throw err;
3205
3533
  }
3206
3534
  }
3207
3535
  /**
@@ -3217,7 +3545,7 @@ var IcechunkStore = class _IcechunkStore {
3217
3545
  * @returns Data bytes or undefined if not found
3218
3546
  */
3219
3547
  async getRange(key, range, opts) {
3220
- if (opts?.signal?.aborted) return void 0;
3548
+ opts?.signal?.throwIfAborted();
3221
3549
  const parsed = parseZarrKey(key);
3222
3550
  const resolvedPath = this.resolvePath(parsed.path);
3223
3551
  try {
@@ -3230,7 +3558,10 @@ var IcechunkStore = class _IcechunkStore {
3230
3558
  signal: opts?.signal,
3231
3559
  ...this.fetchClient && { fetchClient: this.fetchClient },
3232
3560
  validateChecksums: this.validateChecksums,
3233
- azureAccount: this.azureAccount
3561
+ azureAccount: this.azureAccount,
3562
+ ...this.withRangeCoalescing && {
3563
+ withRangeCoalescing: this.withRangeCoalescing
3564
+ }
3234
3565
  }
3235
3566
  );
3236
3567
  return data2 ?? void 0;
@@ -3241,8 +3572,9 @@ var IcechunkStore = class _IcechunkStore {
3241
3572
  return data.slice(-range.suffixLength);
3242
3573
  }
3243
3574
  return data.slice(range.offset, range.offset + range.length);
3244
- } catch {
3245
- return void 0;
3575
+ } catch (err) {
3576
+ if (err instanceof NotFoundError) return void 0;
3577
+ throw err;
3246
3578
  }
3247
3579
  }
3248
3580
  /** Prepend basePath to a parsed path. */
@@ -3266,7 +3598,8 @@ var IcechunkStore = class _IcechunkStore {
3266
3598
  this.session,
3267
3599
  this.fetchClient,
3268
3600
  this.validateChecksums,
3269
- this.azureAccount
3601
+ this.azureAccount,
3602
+ this.withRangeCoalescing
3270
3603
  );
3271
3604
  const cleanPath = path.replace(/^\/+|\/+$/g, "");
3272
3605
  scoped.basePath = this.basePath ? `${this.basePath}/${cleanPath}` : cleanPath;