icechunk-js 0.5.1 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -137,6 +137,19 @@ Cloud storage URLs in virtual chunk references are automatically translated:
137
137
  - `abfs://container@account.dfs.core.windows.net/path` →
138
138
  `https://account.blob.core.windows.net/container/path`
139
139
 
140
+ For S3, addressing follows the repo's virtual-chunk-container config (region,
141
+ endpoint, path-style), mirroring the Rust implementation. Buckets whose name
142
+ contains a dot must use path-style addressing; when the container config records
143
+ a region (repos written by Icechunk do), reads go straight to that regional
144
+ endpoint — which serves CORS and works in both Node and the browser, with no
145
+ extra configuration.
146
+
147
+ If no region is known, dotted-name buckets fall back to the global endpoint,
148
+ whose region redirect (for buckets outside `us-east-1`) is resolved
149
+ automatically in Node but not in browsers — the cross-origin redirect carries no
150
+ CORS headers. Supply a [`fetchClient`](#virtual-chunk-authentication) that routes
151
+ to the regional endpoint or a CORS-enabled proxy in that case.
152
+
140
153
  ### Repository
141
154
 
142
155
  For direct access to branches, tags, and checkouts.
package/dist/index.cjs CHANGED
@@ -1279,7 +1279,7 @@ function parseRepo(data) {
1279
1279
  return { repo, specVersion, snapshotsLength, virtualChunkContainers };
1280
1280
  }
1281
1281
  function parseVirtualChunkContainers(repo) {
1282
- const result = /* @__PURE__ */ new Map();
1282
+ const result = [];
1283
1283
  const configBytes = repo.configArray();
1284
1284
  if (!configBytes || configBytes.length === 0) return result;
1285
1285
  let config;
@@ -1297,12 +1297,27 @@ function parseVirtualChunkContainers(repo) {
1297
1297
  if (!isRecord(containers)) return result;
1298
1298
  for (const container of Object.values(containers)) {
1299
1299
  if (!isRecord(container)) continue;
1300
- const { name, url_prefix: urlPrefix } = container;
1301
- if (typeof name !== "string" || typeof urlPrefix !== "string") continue;
1302
- result.set(name, urlPrefix);
1300
+ const urlPrefix = container.url_prefix;
1301
+ if (typeof urlPrefix !== "string") continue;
1302
+ const name = typeof container.name === "string" ? container.name : null;
1303
+ const s3 = parseS3Store(container.store);
1304
+ result.push(s3 ? { name, urlPrefix, s3 } : { name, urlPrefix });
1303
1305
  }
1306
+ result.sort((a, b) => b.urlPrefix.length - a.urlPrefix.length);
1304
1307
  return result;
1305
1308
  }
1309
+ function parseS3Store(store) {
1310
+ if (!isRecord(store)) return void 0;
1311
+ const s3 = store.s3 ?? store.s3_compatible ?? store.tigris;
1312
+ if (!isRecord(s3)) return void 0;
1313
+ const cfg = {};
1314
+ if (typeof s3.region === "string") cfg.region = s3.region;
1315
+ if (typeof s3.endpoint_url === "string") cfg.endpointUrl = s3.endpoint_url;
1316
+ if (typeof s3.force_path_style === "boolean") {
1317
+ cfg.forcePathStyle = s3.force_path_style;
1318
+ }
1319
+ return cfg;
1320
+ }
1306
1321
  function isRecord(value) {
1307
1322
  return typeof value === "object" && value !== null && !Array.isArray(value);
1308
1323
  }
@@ -2960,29 +2975,53 @@ function parseTransactionLog(data) {
2960
2975
  function expectedRangeLength(range) {
2961
2976
  return "suffixLength" in range ? range.suffixLength : range.length;
2962
2977
  }
2978
+ function regionalS3RedirectUrl(url, response) {
2979
+ if (response.status !== 301 && response.status !== 307) return null;
2980
+ const region = response.headers.get("x-amz-bucket-region");
2981
+ if (!region) return null;
2982
+ const globalPrefix = "https://s3.amazonaws.com/";
2983
+ if (!url.startsWith(globalPrefix)) return null;
2984
+ return `https://s3.${region}.amazonaws.com/${url.slice(globalPrefix.length)}`;
2985
+ }
2963
2986
  function makeUrlStore(opts) {
2964
- const { url, fetchClient, conditionalHeaders } = opts;
2965
- async function doFetch(init) {
2966
- return fetchClient ? await fetchClient.fetch(url, init) : await fetch(url, init);
2987
+ const { fetchClient, conditionalHeaders } = opts;
2988
+ let pinnedUrl = opts.url;
2989
+ async function doFetch(target, init) {
2990
+ return fetchClient ? await fetchClient.fetch(target, init) : await fetch(target, init);
2967
2991
  }
2968
2992
  return {
2969
2993
  async get() {
2970
2994
  throw new Error(
2971
- `Virtual chunk URL store for ${url} only supports ranged reads`
2995
+ `Virtual chunk URL store for ${pinnedUrl} only supports ranged reads`
2972
2996
  );
2973
2997
  },
2974
2998
  async getRange(_key, range, options) {
2975
2999
  const headers = conditionalHeaders ? { ...conditionalHeaders } : {};
2976
3000
  headers.Range = "suffixLength" in range ? `bytes=-${range.suffixLength}` : `bytes=${range.offset}-${range.offset + range.length - 1}`;
2977
- const response = await doFetch({ headers, signal: options?.signal });
3001
+ let requestUrl = pinnedUrl;
3002
+ let response = await doFetch(requestUrl, {
3003
+ headers,
3004
+ signal: options?.signal
3005
+ });
3006
+ const regionalUrl = regionalS3RedirectUrl(requestUrl, response);
3007
+ if (regionalUrl) {
3008
+ response.body?.cancel().catch(() => {
3009
+ });
3010
+ requestUrl = regionalUrl;
3011
+ pinnedUrl = regionalUrl;
3012
+ response = await doFetch(requestUrl, {
3013
+ headers,
3014
+ signal: options?.signal
3015
+ });
3016
+ }
2978
3017
  if (response.status === 412) {
2979
3018
  throw new Error(
2980
- `Virtual chunk at ${url} failed integrity check \u2014 data has been modified since snapshot was created`
3019
+ `Virtual chunk at ${requestUrl} failed integrity check \u2014 data has been modified since snapshot was created`
2981
3020
  );
2982
3021
  }
2983
3022
  if (response.status !== 200 && response.status !== 206) {
2984
3023
  throw new Error(
2985
- `Failed to fetch virtual chunk from ${url}: ${response.status} ${response.statusText}`
3024
+ `Failed to fetch virtual chunk from ${requestUrl}: ${response.status} ${response.statusText}`
2986
3025
  );
2987
3026
  }
2988
3027
  const data = new Uint8Array(await response.arrayBuffer());
@@ -2990,21 +3029,21 @@ function makeUrlStore(opts) {
2990
3029
  const expected = expectedRangeLength(range);
2991
3030
  if (data.length === expected) return data;
2992
3031
  throw new Error(
2993
- `Virtual range response size mismatch for ${url}: expected ${expected} bytes, got ${data.length}`
3032
+ `Virtual range response size mismatch for ${requestUrl}: expected ${expected} bytes, got ${data.length}`
2994
3033
  );
2995
3034
  }
2996
3035
  if ("offset" in range) {
2997
3036
  const end = range.offset + range.length;
2998
3037
  if (data.length >= end) return data.slice(range.offset, end);
2999
3038
  throw new Error(
3000
- `Virtual range request not honored for ${url}: need at least ${end} bytes for fallback slicing, got ${data.length}`
3039
+ `Virtual range request not honored for ${requestUrl}: need at least ${end} bytes for fallback slicing, got ${data.length}`
3001
3040
  );
3002
3041
  }
3003
3042
  if (data.length >= range.suffixLength) {
3004
3043
  return data.slice(data.length - range.suffixLength);
3005
3044
  }
3006
3045
  throw new Error(
3007
- `Virtual suffix range request not honored for ${url}: need at least ${range.suffixLength} bytes for fallback slicing, got ${data.length}`
3046
+ `Virtual suffix range request not honored for ${requestUrl}: need at least ${range.suffixLength} bytes for fallback slicing, got ${data.length}`
3008
3047
  );
3009
3048
  }
3010
3049
  };
@@ -3072,7 +3111,7 @@ var ReadSession = class _ReadSession {
3072
3111
  nextFetchClientId = 1;
3073
3112
  rangeCoalescerIds;
3074
3113
  nextRangeCoalescerId = 1;
3075
- /** VCC name → url_prefix map for resolving `vcc://` chunk locations. */
3114
+ /** Virtual chunk containers (from repo config) for resolving chunk locations. */
3076
3115
  virtualChunkContainers;
3077
3116
  constructor(storage, snapshot, specVersion, maxManifestCacheSize = 100, virtualChunkContainers) {
3078
3117
  this.storage = storage;
@@ -3080,7 +3119,7 @@ var ReadSession = class _ReadSession {
3080
3119
  this.specVersion = specVersion;
3081
3120
  this.manifestCache = new LRUCache(maxManifestCacheSize);
3082
3121
  this.manifestLoader = singleFlight(this.manifestCache);
3083
- this.virtualChunkContainers = virtualChunkContainers ?? /* @__PURE__ */ new Map();
3122
+ this.virtualChunkContainers = virtualChunkContainers ?? [];
3084
3123
  }
3085
3124
  getFetchClientKey(fetchClient) {
3086
3125
  if (!fetchClient) return "default";
@@ -3466,14 +3505,14 @@ var ReadSession = class _ReadSession {
3466
3505
  );
3467
3506
  }
3468
3507
  case "virtual": {
3469
- const absoluteLocation = expandVccUrl(
3508
+ const { url: absoluteLocation, container } = resolveVirtualChunkLocation(
3470
3509
  payload.location,
3471
3510
  this.virtualChunkContainers
3472
3511
  );
3473
- const httpUrl = translateToHttpUrl(
3474
- absoluteLocation,
3475
- options?.azureAccount
3476
- );
3512
+ const httpUrl = translateToHttpUrl(absoluteLocation, {
3513
+ azureAccount: options?.azureAccount,
3514
+ s3: container?.s3
3515
+ });
3477
3516
  const store = await this.getVirtualStoreForPayload(
3478
3517
  httpUrl,
3479
3518
  payload,
@@ -3536,14 +3575,14 @@ var ReadSession = class _ReadSession {
3536
3575
  case "virtual": {
3537
3576
  const absoluteStart = payload.offset + rangeStart;
3538
3577
  const expectedSize = rangeEnd - rangeStart;
3539
- const absoluteLocation = expandVccUrl(
3578
+ const { url: absoluteLocation, container } = resolveVirtualChunkLocation(
3540
3579
  payload.location,
3541
3580
  this.virtualChunkContainers
3542
3581
  );
3543
- const httpUrl = translateToHttpUrl(
3544
- absoluteLocation,
3545
- options?.azureAccount
3546
- );
3582
+ const httpUrl = translateToHttpUrl(absoluteLocation, {
3583
+ azureAccount: options?.azureAccount,
3584
+ s3: container?.s3
3585
+ });
3547
3586
  const store = await this.getVirtualStoreForPayload(
3548
3587
  httpUrl,
3549
3588
  payload,
@@ -3613,44 +3652,62 @@ function compareUtf8Bytes(a, b) {
3613
3652
  return bytesA.length - bytesB.length;
3614
3653
  }
3615
3654
  var VCC_SCHEME = "vcc://";
3616
- function expandVccUrl(location, containers) {
3617
- if (!location.startsWith(VCC_SCHEME)) return location;
3618
- const rest = location.slice(VCC_SCHEME.length);
3619
- const slash = rest.indexOf("/");
3620
- if (slash === -1) {
3621
- throw new Error(
3622
- `Invalid vcc:// URL "${location}": missing "/" after container name`
3623
- );
3624
- }
3625
- const name = rest.slice(0, slash);
3626
- const relativePath = rest.slice(slash + 1);
3627
- const urlPrefix = containers.get(name);
3628
- if (urlPrefix === void 0) {
3629
- if (containers.size === 0) return location;
3630
- throw new Error(
3631
- `Unknown virtual chunk container "${name}" referenced by ${location}`
3632
- );
3655
+ function resolveVirtualChunkLocation(location, containers) {
3656
+ if (location.startsWith(VCC_SCHEME)) {
3657
+ const rest = location.slice(VCC_SCHEME.length);
3658
+ const slash = rest.indexOf("/");
3659
+ if (slash === -1) {
3660
+ throw new Error(
3661
+ `Invalid vcc:// URL "${location}": missing "/" after container name`
3662
+ );
3663
+ }
3664
+ const name = rest.slice(0, slash);
3665
+ const relativePath = rest.slice(slash + 1);
3666
+ const container2 = containers.find((c) => c.name === name);
3667
+ if (!container2) {
3668
+ if (containers.length === 0) return { url: location };
3669
+ throw new Error(
3670
+ `Unknown virtual chunk container "${name}" referenced by ${location}`
3671
+ );
3672
+ }
3673
+ return {
3674
+ url: ensureTrailingSlash(container2.urlPrefix) + relativePath,
3675
+ container: container2
3676
+ };
3633
3677
  }
3634
- const normalizedPrefix = urlPrefix.endsWith("/") ? urlPrefix : `${urlPrefix}/`;
3635
- return normalizedPrefix + relativePath;
3678
+ const container = containers.find(
3679
+ (c) => location.startsWith(ensureTrailingSlash(c.urlPrefix))
3680
+ );
3681
+ return { url: location, container };
3682
+ }
3683
+ function ensureTrailingSlash(prefix) {
3684
+ return prefix.endsWith("/") ? prefix : `${prefix}/`;
3685
+ }
3686
+ function buildS3HttpUrl(bucket, key, s3) {
3687
+ if (s3?.endpointUrl) {
3688
+ const base = s3.endpointUrl.replace(/\/+$/, "");
3689
+ return `${base}/${bucket}/${key}`;
3690
+ }
3691
+ const region = s3?.region;
3692
+ const pathStyle = s3?.forcePathStyle === true || bucket.includes(".");
3693
+ if (pathStyle) {
3694
+ const host2 = region ? `s3.${region}.${awsEndpointSuffix(region)}` : "s3.amazonaws.com";
3695
+ return `https://${host2}/${bucket}/${key}`;
3696
+ }
3697
+ const host = region ? `${bucket}.s3.${region}.${awsEndpointSuffix(region)}` : `${bucket}.s3.amazonaws.com`;
3698
+ return `https://${host}/${key}`;
3636
3699
  }
3637
- function translateToHttpUrl(url, azureAccount) {
3700
+ function awsEndpointSuffix(region) {
3701
+ return region.startsWith("cn-") ? "amazonaws.com.cn" : "amazonaws.com";
3702
+ }
3703
+ function translateToHttpUrl(url, options) {
3704
+ const azureAccount = options?.azureAccount;
3638
3705
  if (url.startsWith("s3://")) {
3639
3706
  const rest = url.slice(5);
3640
3707
  const slashIndex = rest.indexOf("/");
3641
- if (slashIndex === -1) {
3642
- const bucket2 = rest;
3643
- if (bucket2.includes(".")) {
3644
- return `https://s3.amazonaws.com/${bucket2}/`;
3645
- }
3646
- return `https://${bucket2}.s3.amazonaws.com/`;
3647
- }
3648
- const bucket = rest.slice(0, slashIndex);
3649
- const key = rest.slice(slashIndex + 1);
3650
- if (bucket.includes(".")) {
3651
- return `https://s3.amazonaws.com/${bucket}/${key}`;
3652
- }
3653
- return `https://${bucket}.s3.amazonaws.com/${key}`;
3708
+ const bucket = slashIndex === -1 ? rest : rest.slice(0, slashIndex);
3709
+ const key = slashIndex === -1 ? "" : rest.slice(slashIndex + 1);
3710
+ return buildS3HttpUrl(bucket, key, options?.s3);
3654
3711
  }
3655
3712
  if (url.startsWith("gs://") || url.startsWith("gcs://")) {
3656
3713
  const prefixLen = url.startsWith("gs://") ? 5 : 6;