icechunk-js 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -137,6 +137,19 @@ Cloud storage URLs in virtual chunk references are automatically translated:
137
137
  - `abfs://container@account.dfs.core.windows.net/path` →
138
138
  `https://account.blob.core.windows.net/container/path`
139
139
 
140
+ For S3, addressing follows the repo's virtual-chunk-container config (region,
141
+ endpoint, path-style), mirroring the Rust implementation. Buckets whose name
142
+ contains a dot must use path-style addressing; when the container config records
143
+ a region (repos written by Icechunk do), reads go straight to that regional
144
+ endpoint — which serves CORS and works in both Node and the browser, with no
145
+ extra configuration.
146
+
147
+ If no region is known, dotted-name buckets fall back to the global endpoint,
148
+ whose region redirect (for buckets outside `us-east-1`) is resolved
149
+ automatically in Node but not in browsers — the cross-origin redirect carries no
150
+ CORS headers. Supply a [`fetchClient`](#virtual-chunk-authentication) that routes
151
+ to the regional endpoint or a CORS-enabled proxy in that case.
152
+
140
153
  ### Repository
141
154
 
142
155
  For direct access to branches, tags, and checkouts.
package/dist/index.cjs CHANGED
@@ -1279,7 +1279,7 @@ function parseRepo(data) {
1279
1279
  return { repo, specVersion, snapshotsLength, virtualChunkContainers };
1280
1280
  }
1281
1281
  function parseVirtualChunkContainers(repo) {
1282
- const result = /* @__PURE__ */ new Map();
1282
+ const result = [];
1283
1283
  const configBytes = repo.configArray();
1284
1284
  if (!configBytes || configBytes.length === 0) return result;
1285
1285
  let config;
@@ -1297,12 +1297,27 @@ function parseVirtualChunkContainers(repo) {
1297
1297
  if (!isRecord(containers)) return result;
1298
1298
  for (const container of Object.values(containers)) {
1299
1299
  if (!isRecord(container)) continue;
1300
- const { name, url_prefix: urlPrefix } = container;
1301
- if (typeof name !== "string" || typeof urlPrefix !== "string") continue;
1302
- result.set(name, urlPrefix);
1300
+ const urlPrefix = container.url_prefix;
1301
+ if (typeof urlPrefix !== "string") continue;
1302
+ const name = typeof container.name === "string" ? container.name : null;
1303
+ const s3 = parseS3Store(container.store);
1304
+ result.push(s3 ? { name, urlPrefix, s3 } : { name, urlPrefix });
1303
1305
  }
1306
+ result.sort((a, b) => b.urlPrefix.length - a.urlPrefix.length);
1304
1307
  return result;
1305
1308
  }
1309
+ function parseS3Store(store) {
1310
+ if (!isRecord(store)) return void 0;
1311
+ const s3 = store.s3 ?? store.s3_compatible ?? store.tigris;
1312
+ if (!isRecord(s3)) return void 0;
1313
+ const cfg = {};
1314
+ if (typeof s3.region === "string") cfg.region = s3.region;
1315
+ if (typeof s3.endpoint_url === "string") cfg.endpointUrl = s3.endpoint_url;
1316
+ if (typeof s3.force_path_style === "boolean") {
1317
+ cfg.forcePathStyle = s3.force_path_style;
1318
+ }
1319
+ return cfg;
1320
+ }
1306
1321
  function isRecord(value) {
1307
1322
  return typeof value === "object" && value !== null && !Array.isArray(value);
1308
1323
  }
@@ -2441,7 +2456,7 @@ function parseManifest(data) {
2441
2456
  for (let i = 0; i < arraysLength; i++) {
2442
2457
  const fbsArray = fbsManifest.arrays(i);
2443
2458
  if (fbsArray) {
2444
- arrays.push(parseArrayManifest(fbsArray, decodeLocation));
2459
+ arrays.push(makeArrayManifest(fbsArray, decodeLocation));
2445
2460
  }
2446
2461
  }
2447
2462
  return { id, arrays };
@@ -2469,22 +2484,32 @@ function makeLocationDecoder(fbsManifest) {
2469
2484
  return textDecoder.decode(decompressed);
2470
2485
  };
2471
2486
  }
2472
- function parseArrayManifest(fbsArray, decodeLocation) {
2487
+ function makeArrayManifest(fbsArray, decodeLocation) {
2473
2488
  const nodeIdObj = fbsArray.nodeId();
2474
2489
  if (!nodeIdObj)
2475
2490
  throw new Error("ArrayManifest missing required node_id field");
2476
2491
  const nodeId = asObjectId8(
2477
2492
  nodeIdObj.bb.bytes().slice(nodeIdObj.bb_pos, nodeIdObj.bb_pos + 8)
2478
2493
  );
2479
- const refsLength = fbsArray.refsLength();
2480
- const refs = [];
2481
- for (let i = 0; i < refsLength; i++) {
2482
- const fbsRef = fbsArray.refs(i);
2483
- if (fbsRef) {
2484
- refs.push(parseChunkRef(fbsRef, decodeLocation));
2494
+ const numRefs = fbsArray.refsLength();
2495
+ const probe = new ChunkRef();
2496
+ return {
2497
+ nodeId,
2498
+ numRefs,
2499
+ refIndex(i) {
2500
+ const fbsRef = fbsArray.refs(i, probe);
2501
+ if (!fbsRef) throw new Error(`Manifest ref ${i} out of range`);
2502
+ const indexLength = fbsRef.indexLength();
2503
+ const index = new Array(indexLength);
2504
+ for (let d = 0; d < indexLength; d++) index[d] = fbsRef.index(d);
2505
+ return index;
2506
+ },
2507
+ refAt(i) {
2508
+ const fbsRef = fbsArray.refs(i);
2509
+ if (!fbsRef) throw new Error(`Manifest ref ${i} out of range`);
2510
+ return parseChunkRef(fbsRef, decodeLocation);
2485
2511
  }
2486
- }
2487
- return { nodeId, refs };
2512
+ };
2488
2513
  }
2489
2514
  function parseChunkRef(fbsRef, decodeLocation) {
2490
2515
  const indexLength = fbsRef.indexLength();
@@ -2535,7 +2560,7 @@ function parseChunkRef(fbsRef, decodeLocation) {
2535
2560
  function findChunkRef(manifest, nodeId, coords) {
2536
2561
  const arrayManifest = binarySearchArrayManifest(manifest.arrays, nodeId);
2537
2562
  if (!arrayManifest) return null;
2538
- return binarySearchChunkRef(arrayManifest.refs, coords);
2563
+ return binarySearchChunkRef(arrayManifest, coords);
2539
2564
  }
2540
2565
  function binarySearchArrayManifest(arrays, nodeId) {
2541
2566
  let low = 0;
@@ -2567,14 +2592,14 @@ function compareCoords(a, b) {
2567
2592
  }
2568
2593
  return a.length - b.length;
2569
2594
  }
2570
- function binarySearchChunkRef(refs, coords) {
2595
+ function binarySearchChunkRef(array, coords) {
2571
2596
  let low = 0;
2572
- let high = refs.length - 1;
2597
+ let high = array.numRefs - 1;
2573
2598
  while (low <= high) {
2574
2599
  const mid = Math.floor((low + high) / 2);
2575
- const cmp = compareCoords(refs[mid].index, coords);
2600
+ const cmp = compareCoords(array.refIndex(mid), coords);
2576
2601
  if (cmp === 0) {
2577
- return refs[mid];
2602
+ return array.refAt(mid);
2578
2603
  } else if (cmp < 0) {
2579
2604
  low = mid + 1;
2580
2605
  } else {
@@ -2950,29 +2975,53 @@ function parseTransactionLog(data) {
2950
2975
  function expectedRangeLength(range) {
2951
2976
  return "suffixLength" in range ? range.suffixLength : range.length;
2952
2977
  }
2978
+ function regionalS3RedirectUrl(url, response) {
2979
+ if (response.status !== 301 && response.status !== 307) return null;
2980
+ const region = response.headers.get("x-amz-bucket-region");
2981
+ if (!region) return null;
2982
+ const globalPrefix = "https://s3.amazonaws.com/";
2983
+ if (!url.startsWith(globalPrefix)) return null;
2984
+ return `https://s3.${region}.amazonaws.com/${url.slice(globalPrefix.length)}`;
2985
+ }
2953
2986
  function makeUrlStore(opts) {
2954
- const { url, fetchClient, conditionalHeaders } = opts;
2955
- async function doFetch(init) {
2956
- return fetchClient ? await fetchClient.fetch(url, init) : await fetch(url, init);
2987
+ const { fetchClient, conditionalHeaders } = opts;
2988
+ let pinnedUrl = opts.url;
2989
+ async function doFetch(target, init) {
2990
+ return fetchClient ? await fetchClient.fetch(target, init) : await fetch(target, init);
2957
2991
  }
2958
2992
  return {
2959
2993
  async get() {
2960
2994
  throw new Error(
2961
- `Virtual chunk URL store for ${url} only supports ranged reads`
2995
+ `Virtual chunk URL store for ${pinnedUrl} only supports ranged reads`
2962
2996
  );
2963
2997
  },
2964
2998
  async getRange(_key, range, options) {
2965
2999
  const headers = conditionalHeaders ? { ...conditionalHeaders } : {};
2966
3000
  headers.Range = "suffixLength" in range ? `bytes=-${range.suffixLength}` : `bytes=${range.offset}-${range.offset + range.length - 1}`;
2967
- const response = await doFetch({ headers, signal: options?.signal });
3001
+ let requestUrl = pinnedUrl;
3002
+ let response = await doFetch(requestUrl, {
3003
+ headers,
3004
+ signal: options?.signal
3005
+ });
3006
+ const regionalUrl = regionalS3RedirectUrl(requestUrl, response);
3007
+ if (regionalUrl) {
3008
+ response.body?.cancel().catch(() => {
3009
+ });
3010
+ requestUrl = regionalUrl;
3011
+ pinnedUrl = regionalUrl;
3012
+ response = await doFetch(requestUrl, {
3013
+ headers,
3014
+ signal: options?.signal
3015
+ });
3016
+ }
2968
3017
  if (response.status === 412) {
2969
3018
  throw new Error(
2970
- `Virtual chunk at ${url} failed integrity check \u2014 data has been modified since snapshot was created`
3019
+ `Virtual chunk at ${requestUrl} failed integrity check \u2014 data has been modified since snapshot was created`
2971
3020
  );
2972
3021
  }
2973
3022
  if (response.status !== 200 && response.status !== 206) {
2974
3023
  throw new Error(
2975
- `Failed to fetch virtual chunk from ${url}: ${response.status} ${response.statusText}`
3024
+ `Failed to fetch virtual chunk from ${requestUrl}: ${response.status} ${response.statusText}`
2976
3025
  );
2977
3026
  }
2978
3027
  const data = new Uint8Array(await response.arrayBuffer());
@@ -2980,21 +3029,21 @@ function makeUrlStore(opts) {
2980
3029
  const expected = expectedRangeLength(range);
2981
3030
  if (data.length === expected) return data;
2982
3031
  throw new Error(
2983
- `Virtual range response size mismatch for ${url}: expected ${expected} bytes, got ${data.length}`
3032
+ `Virtual range response size mismatch for ${requestUrl}: expected ${expected} bytes, got ${data.length}`
2984
3033
  );
2985
3034
  }
2986
3035
  if ("offset" in range) {
2987
3036
  const end = range.offset + range.length;
2988
3037
  if (data.length >= end) return data.slice(range.offset, end);
2989
3038
  throw new Error(
2990
- `Virtual range request not honored for ${url}: need at least ${end} bytes for fallback slicing, got ${data.length}`
3039
+ `Virtual range request not honored for ${requestUrl}: need at least ${end} bytes for fallback slicing, got ${data.length}`
2991
3040
  );
2992
3041
  }
2993
3042
  if (data.length >= range.suffixLength) {
2994
3043
  return data.slice(data.length - range.suffixLength);
2995
3044
  }
2996
3045
  throw new Error(
2997
- `Virtual suffix range request not honored for ${url}: need at least ${range.suffixLength} bytes for fallback slicing, got ${data.length}`
3046
+ `Virtual suffix range request not honored for ${requestUrl}: need at least ${range.suffixLength} bytes for fallback slicing, got ${data.length}`
2998
3047
  );
2999
3048
  }
3000
3049
  };
@@ -3062,7 +3111,7 @@ var ReadSession = class _ReadSession {
3062
3111
  nextFetchClientId = 1;
3063
3112
  rangeCoalescerIds;
3064
3113
  nextRangeCoalescerId = 1;
3065
- /** VCC name → url_prefix map for resolving `vcc://` chunk locations. */
3114
+ /** Virtual chunk containers (from repo config) for resolving chunk locations. */
3066
3115
  virtualChunkContainers;
3067
3116
  constructor(storage, snapshot, specVersion, maxManifestCacheSize = 100, virtualChunkContainers) {
3068
3117
  this.storage = storage;
@@ -3070,7 +3119,7 @@ var ReadSession = class _ReadSession {
3070
3119
  this.specVersion = specVersion;
3071
3120
  this.manifestCache = new LRUCache(maxManifestCacheSize);
3072
3121
  this.manifestLoader = singleFlight(this.manifestCache);
3073
- this.virtualChunkContainers = virtualChunkContainers ?? /* @__PURE__ */ new Map();
3122
+ this.virtualChunkContainers = virtualChunkContainers ?? [];
3074
3123
  }
3075
3124
  getFetchClientKey(fetchClient) {
3076
3125
  if (!fetchClient) return "default";
@@ -3456,14 +3505,14 @@ var ReadSession = class _ReadSession {
3456
3505
  );
3457
3506
  }
3458
3507
  case "virtual": {
3459
- const absoluteLocation = expandVccUrl(
3508
+ const { url: absoluteLocation, container } = resolveVirtualChunkLocation(
3460
3509
  payload.location,
3461
3510
  this.virtualChunkContainers
3462
3511
  );
3463
- const httpUrl = translateToHttpUrl(
3464
- absoluteLocation,
3465
- options?.azureAccount
3466
- );
3512
+ const httpUrl = translateToHttpUrl(absoluteLocation, {
3513
+ azureAccount: options?.azureAccount,
3514
+ s3: container?.s3
3515
+ });
3467
3516
  const store = await this.getVirtualStoreForPayload(
3468
3517
  httpUrl,
3469
3518
  payload,
@@ -3526,14 +3575,14 @@ var ReadSession = class _ReadSession {
3526
3575
  case "virtual": {
3527
3576
  const absoluteStart = payload.offset + rangeStart;
3528
3577
  const expectedSize = rangeEnd - rangeStart;
3529
- const absoluteLocation = expandVccUrl(
3578
+ const { url: absoluteLocation, container } = resolveVirtualChunkLocation(
3530
3579
  payload.location,
3531
3580
  this.virtualChunkContainers
3532
3581
  );
3533
- const httpUrl = translateToHttpUrl(
3534
- absoluteLocation,
3535
- options?.azureAccount
3536
- );
3582
+ const httpUrl = translateToHttpUrl(absoluteLocation, {
3583
+ azureAccount: options?.azureAccount,
3584
+ s3: container?.s3
3585
+ });
3537
3586
  const store = await this.getVirtualStoreForPayload(
3538
3587
  httpUrl,
3539
3588
  payload,
@@ -3603,44 +3652,62 @@ function compareUtf8Bytes(a, b) {
3603
3652
  return bytesA.length - bytesB.length;
3604
3653
  }
3605
3654
  var VCC_SCHEME = "vcc://";
3606
- function expandVccUrl(location, containers) {
3607
- if (!location.startsWith(VCC_SCHEME)) return location;
3608
- const rest = location.slice(VCC_SCHEME.length);
3609
- const slash = rest.indexOf("/");
3610
- if (slash === -1) {
3611
- throw new Error(
3612
- `Invalid vcc:// URL "${location}": missing "/" after container name`
3613
- );
3614
- }
3615
- const name = rest.slice(0, slash);
3616
- const relativePath = rest.slice(slash + 1);
3617
- const urlPrefix = containers.get(name);
3618
- if (urlPrefix === void 0) {
3619
- if (containers.size === 0) return location;
3620
- throw new Error(
3621
- `Unknown virtual chunk container "${name}" referenced by ${location}`
3622
- );
3655
+ function resolveVirtualChunkLocation(location, containers) {
3656
+ if (location.startsWith(VCC_SCHEME)) {
3657
+ const rest = location.slice(VCC_SCHEME.length);
3658
+ const slash = rest.indexOf("/");
3659
+ if (slash === -1) {
3660
+ throw new Error(
3661
+ `Invalid vcc:// URL "${location}": missing "/" after container name`
3662
+ );
3663
+ }
3664
+ const name = rest.slice(0, slash);
3665
+ const relativePath = rest.slice(slash + 1);
3666
+ const container2 = containers.find((c) => c.name === name);
3667
+ if (!container2) {
3668
+ if (containers.length === 0) return { url: location };
3669
+ throw new Error(
3670
+ `Unknown virtual chunk container "${name}" referenced by ${location}`
3671
+ );
3672
+ }
3673
+ return {
3674
+ url: ensureTrailingSlash(container2.urlPrefix) + relativePath,
3675
+ container: container2
3676
+ };
3623
3677
  }
3624
- const normalizedPrefix = urlPrefix.endsWith("/") ? urlPrefix : `${urlPrefix}/`;
3625
- return normalizedPrefix + relativePath;
3678
+ const container = containers.find(
3679
+ (c) => location.startsWith(ensureTrailingSlash(c.urlPrefix))
3680
+ );
3681
+ return { url: location, container };
3682
+ }
3683
+ function ensureTrailingSlash(prefix) {
3684
+ return prefix.endsWith("/") ? prefix : `${prefix}/`;
3685
+ }
3686
+ function buildS3HttpUrl(bucket, key, s3) {
3687
+ if (s3?.endpointUrl) {
3688
+ const base = s3.endpointUrl.replace(/\/+$/, "");
3689
+ return `${base}/${bucket}/${key}`;
3690
+ }
3691
+ const region = s3?.region;
3692
+ const pathStyle = s3?.forcePathStyle === true || bucket.includes(".");
3693
+ if (pathStyle) {
3694
+ const host2 = region ? `s3.${region}.${awsEndpointSuffix(region)}` : "s3.amazonaws.com";
3695
+ return `https://${host2}/${bucket}/${key}`;
3696
+ }
3697
+ const host = region ? `${bucket}.s3.${region}.${awsEndpointSuffix(region)}` : `${bucket}.s3.amazonaws.com`;
3698
+ return `https://${host}/${key}`;
3626
3699
  }
3627
- function translateToHttpUrl(url, azureAccount) {
3700
+ function awsEndpointSuffix(region) {
3701
+ return region.startsWith("cn-") ? "amazonaws.com.cn" : "amazonaws.com";
3702
+ }
3703
+ function translateToHttpUrl(url, options) {
3704
+ const azureAccount = options?.azureAccount;
3628
3705
  if (url.startsWith("s3://")) {
3629
3706
  const rest = url.slice(5);
3630
3707
  const slashIndex = rest.indexOf("/");
3631
- if (slashIndex === -1) {
3632
- const bucket2 = rest;
3633
- if (bucket2.includes(".")) {
3634
- return `https://s3.amazonaws.com/${bucket2}/`;
3635
- }
3636
- return `https://${bucket2}.s3.amazonaws.com/`;
3637
- }
3638
- const bucket = rest.slice(0, slashIndex);
3639
- const key = rest.slice(slashIndex + 1);
3640
- if (bucket.includes(".")) {
3641
- return `https://s3.amazonaws.com/${bucket}/${key}`;
3642
- }
3643
- return `https://${bucket}.s3.amazonaws.com/${key}`;
3708
+ const bucket = slashIndex === -1 ? rest : rest.slice(0, slashIndex);
3709
+ const key = slashIndex === -1 ? "" : rest.slice(slashIndex + 1);
3710
+ return buildS3HttpUrl(bucket, key, options?.s3);
3644
3711
  }
3645
3712
  if (url.startsWith("gs://") || url.startsWith("gcs://")) {
3646
3713
  const prefixLen = url.startsWith("gs://") ? 5 : 6;