icechunk-js 0.5.1 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -137,6 +137,19 @@ Cloud storage URLs in virtual chunk references are automatically translated:
137
137
  - `abfs://container@account.dfs.core.windows.net/path` →
138
138
  `https://account.blob.core.windows.net/container/path`
139
139
 
140
+ For S3, addressing follows the repo's virtual-chunk-container config (region,
141
+ endpoint, path-style), mirroring the Rust implementation. Buckets whose name
142
+ contains a dot must use path-style addressing; when the container config records
143
+ a region (repos written by Icechunk do), reads go straight to that regional
144
+ endpoint — which serves CORS and works in both Node and the browser, with no
145
+ extra configuration.
146
+
147
+ If no region is known, dotted-name buckets fall back to the global endpoint,
148
+ whose region redirect (for buckets outside `us-east-1`) is resolved
149
+ automatically in Node but not in browsers — the cross-origin redirect carries no
150
+ CORS headers. Supply a [`fetchClient`](#virtual-chunk-authentication) that routes
151
+ to the regional endpoint or a CORS-enabled proxy in that case.
152
+
140
153
  ### Repository
141
154
 
142
155
  For direct access to branches, tags, and checkouts.
package/dist/index.cjs CHANGED
@@ -1279,7 +1279,7 @@ function parseRepo(data) {
1279
1279
  return { repo, specVersion, snapshotsLength, virtualChunkContainers };
1280
1280
  }
1281
1281
  function parseVirtualChunkContainers(repo) {
1282
- const result = /* @__PURE__ */ new Map();
1282
+ const result = [];
1283
1283
  const configBytes = repo.configArray();
1284
1284
  if (!configBytes || configBytes.length === 0) return result;
1285
1285
  let config;
@@ -1297,12 +1297,27 @@ function parseVirtualChunkContainers(repo) {
1297
1297
  if (!isRecord(containers)) return result;
1298
1298
  for (const container of Object.values(containers)) {
1299
1299
  if (!isRecord(container)) continue;
1300
- const { name, url_prefix: urlPrefix } = container;
1301
- if (typeof name !== "string" || typeof urlPrefix !== "string") continue;
1302
- result.set(name, urlPrefix);
1300
+ const urlPrefix = container.url_prefix;
1301
+ if (typeof urlPrefix !== "string") continue;
1302
+ const name = typeof container.name === "string" ? container.name : null;
1303
+ const s3 = parseS3Store(container.store);
1304
+ result.push(s3 ? { name, urlPrefix, s3 } : { name, urlPrefix });
1303
1305
  }
1306
+ result.sort((a, b) => b.urlPrefix.length - a.urlPrefix.length);
1304
1307
  return result;
1305
1308
  }
1309
+ function parseS3Store(store) {
1310
+ if (!isRecord(store)) return void 0;
1311
+ const s3 = store.s3 ?? store.s3_compatible ?? store.tigris;
1312
+ if (!isRecord(s3)) return void 0;
1313
+ const cfg = {};
1314
+ if (typeof s3.region === "string") cfg.region = s3.region;
1315
+ if (typeof s3.endpoint_url === "string") cfg.endpointUrl = s3.endpoint_url;
1316
+ if (typeof s3.force_path_style === "boolean") {
1317
+ cfg.forcePathStyle = s3.force_path_style;
1318
+ }
1319
+ return cfg;
1320
+ }
1306
1321
  function isRecord(value) {
1307
1322
  return typeof value === "object" && value !== null && !Array.isArray(value);
1308
1323
  }
@@ -1400,6 +1415,33 @@ function listTagsFromRepo(parsedRepo) {
1400
1415
  const { repo } = parsedRepo;
1401
1416
  return listRefs((i) => repo.tags(i), repo.tagsLength());
1402
1417
  }
1418
+ var snapshotIndexCache = /* @__PURE__ */ new WeakMap();
1419
+ function snapshotIndexById(parsedRepo) {
1420
+ let index = snapshotIndexCache.get(parsedRepo);
1421
+ if (!index) {
1422
+ index = /* @__PURE__ */ new Map();
1423
+ for (let i = 0; i < parsedRepo.snapshotsLength; i++) {
1424
+ index.set(bytesKey(getSnapshotIdByIndex(parsedRepo, i)), i);
1425
+ }
1426
+ snapshotIndexCache.set(parsedRepo, index);
1427
+ }
1428
+ return index;
1429
+ }
1430
+ function bytesKey(bytes) {
1431
+ return Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("");
1432
+ }
1433
+ function resolveSnapshotParent(parsedRepo, snapshotId) {
1434
+ const index = snapshotIndexById(parsedRepo).get(bytesKey(snapshotId));
1435
+ if (index === void 0) return void 0;
1436
+ const parentOffset = parsedRepo.repo.snapshots(index).parentOffset();
1437
+ if (parentOffset === -1) return null;
1438
+ if (parentOffset < 0 || parentOffset >= parsedRepo.snapshotsLength) {
1439
+ throw new Error(
1440
+ `Invalid parent offset ${parentOffset} for snapshot at index ${index}, snapshots array has ${parsedRepo.snapshotsLength} entries`
1441
+ );
1442
+ }
1443
+ return getSnapshotIdByIndex(parsedRepo, parentOffset);
1444
+ }
1403
1445
 
1404
1446
  // src/cache/lru.ts
1405
1447
  var LRUCache = class {
@@ -2960,29 +3002,53 @@ function parseTransactionLog(data) {
2960
3002
  function expectedRangeLength(range) {
2961
3003
  return "suffixLength" in range ? range.suffixLength : range.length;
2962
3004
  }
3005
+ function regionalS3RedirectUrl(url, response) {
3006
+ if (response.status !== 301 && response.status !== 307) return null;
3007
+ const region = response.headers.get("x-amz-bucket-region");
3008
+ if (!region) return null;
3009
+ const globalPrefix = "https://s3.amazonaws.com/";
3010
+ if (!url.startsWith(globalPrefix)) return null;
3011
+ return `https://s3.${region}.amazonaws.com/${url.slice(globalPrefix.length)}`;
3012
+ }
2963
3013
  function makeUrlStore(opts) {
2964
- const { url, fetchClient, conditionalHeaders } = opts;
2965
- async function doFetch(init) {
2966
- return fetchClient ? await fetchClient.fetch(url, init) : await fetch(url, init);
3014
+ const { fetchClient, conditionalHeaders } = opts;
3015
+ let pinnedUrl = opts.url;
3016
+ async function doFetch(target, init) {
3017
+ return fetchClient ? await fetchClient.fetch(target, init) : await fetch(target, init);
2967
3018
  }
2968
3019
  return {
2969
3020
  async get() {
2970
3021
  throw new Error(
2971
- `Virtual chunk URL store for ${url} only supports ranged reads`
3022
+ `Virtual chunk URL store for ${pinnedUrl} only supports ranged reads`
2972
3023
  );
2973
3024
  },
2974
3025
  async getRange(_key, range, options) {
2975
3026
  const headers = conditionalHeaders ? { ...conditionalHeaders } : {};
2976
3027
  headers.Range = "suffixLength" in range ? `bytes=-${range.suffixLength}` : `bytes=${range.offset}-${range.offset + range.length - 1}`;
2977
- const response = await doFetch({ headers, signal: options?.signal });
3028
+ let requestUrl = pinnedUrl;
3029
+ let response = await doFetch(requestUrl, {
3030
+ headers,
3031
+ signal: options?.signal
3032
+ });
3033
+ const regionalUrl = regionalS3RedirectUrl(requestUrl, response);
3034
+ if (regionalUrl) {
3035
+ response.body?.cancel().catch(() => {
3036
+ });
3037
+ requestUrl = regionalUrl;
3038
+ pinnedUrl = regionalUrl;
3039
+ response = await doFetch(requestUrl, {
3040
+ headers,
3041
+ signal: options?.signal
3042
+ });
3043
+ }
2978
3044
  if (response.status === 412) {
2979
3045
  throw new Error(
2980
- `Virtual chunk at ${url} failed integrity check \u2014 data has been modified since snapshot was created`
3046
+ `Virtual chunk at ${requestUrl} failed integrity check \u2014 data has been modified since snapshot was created`
2981
3047
  );
2982
3048
  }
2983
3049
  if (response.status !== 200 && response.status !== 206) {
2984
3050
  throw new Error(
2985
- `Failed to fetch virtual chunk from ${url}: ${response.status} ${response.statusText}`
3051
+ `Failed to fetch virtual chunk from ${requestUrl}: ${response.status} ${response.statusText}`
2986
3052
  );
2987
3053
  }
2988
3054
  const data = new Uint8Array(await response.arrayBuffer());
@@ -2990,21 +3056,21 @@ function makeUrlStore(opts) {
2990
3056
  const expected = expectedRangeLength(range);
2991
3057
  if (data.length === expected) return data;
2992
3058
  throw new Error(
2993
- `Virtual range response size mismatch for ${url}: expected ${expected} bytes, got ${data.length}`
3059
+ `Virtual range response size mismatch for ${requestUrl}: expected ${expected} bytes, got ${data.length}`
2994
3060
  );
2995
3061
  }
2996
3062
  if ("offset" in range) {
2997
3063
  const end = range.offset + range.length;
2998
3064
  if (data.length >= end) return data.slice(range.offset, end);
2999
3065
  throw new Error(
3000
- `Virtual range request not honored for ${url}: need at least ${end} bytes for fallback slicing, got ${data.length}`
3066
+ `Virtual range request not honored for ${requestUrl}: need at least ${end} bytes for fallback slicing, got ${data.length}`
3001
3067
  );
3002
3068
  }
3003
3069
  if (data.length >= range.suffixLength) {
3004
3070
  return data.slice(data.length - range.suffixLength);
3005
3071
  }
3006
3072
  throw new Error(
3007
- `Virtual suffix range request not honored for ${url}: need at least ${range.suffixLength} bytes for fallback slicing, got ${data.length}`
3073
+ `Virtual suffix range request not honored for ${requestUrl}: need at least ${range.suffixLength} bytes for fallback slicing, got ${data.length}`
3008
3074
  );
3009
3075
  }
3010
3076
  };
@@ -3072,7 +3138,7 @@ var ReadSession = class _ReadSession {
3072
3138
  nextFetchClientId = 1;
3073
3139
  rangeCoalescerIds;
3074
3140
  nextRangeCoalescerId = 1;
3075
- /** VCC name → url_prefix map for resolving `vcc://` chunk locations. */
3141
+ /** Virtual chunk containers (from repo config) for resolving chunk locations. */
3076
3142
  virtualChunkContainers;
3077
3143
  constructor(storage, snapshot, specVersion, maxManifestCacheSize = 100, virtualChunkContainers) {
3078
3144
  this.storage = storage;
@@ -3080,7 +3146,7 @@ var ReadSession = class _ReadSession {
3080
3146
  this.specVersion = specVersion;
3081
3147
  this.manifestCache = new LRUCache(maxManifestCacheSize);
3082
3148
  this.manifestLoader = singleFlight(this.manifestCache);
3083
- this.virtualChunkContainers = virtualChunkContainers ?? /* @__PURE__ */ new Map();
3149
+ this.virtualChunkContainers = virtualChunkContainers ?? [];
3084
3150
  }
3085
3151
  getFetchClientKey(fetchClient) {
3086
3152
  if (!fetchClient) return "default";
@@ -3466,14 +3532,14 @@ var ReadSession = class _ReadSession {
3466
3532
  );
3467
3533
  }
3468
3534
  case "virtual": {
3469
- const absoluteLocation = expandVccUrl(
3535
+ const { url: absoluteLocation, container } = resolveVirtualChunkLocation(
3470
3536
  payload.location,
3471
3537
  this.virtualChunkContainers
3472
3538
  );
3473
- const httpUrl = translateToHttpUrl(
3474
- absoluteLocation,
3475
- options?.azureAccount
3476
- );
3539
+ const httpUrl = translateToHttpUrl(absoluteLocation, {
3540
+ azureAccount: options?.azureAccount,
3541
+ s3: container?.s3
3542
+ });
3477
3543
  const store = await this.getVirtualStoreForPayload(
3478
3544
  httpUrl,
3479
3545
  payload,
@@ -3536,14 +3602,14 @@ var ReadSession = class _ReadSession {
3536
3602
  case "virtual": {
3537
3603
  const absoluteStart = payload.offset + rangeStart;
3538
3604
  const expectedSize = rangeEnd - rangeStart;
3539
- const absoluteLocation = expandVccUrl(
3605
+ const { url: absoluteLocation, container } = resolveVirtualChunkLocation(
3540
3606
  payload.location,
3541
3607
  this.virtualChunkContainers
3542
3608
  );
3543
- const httpUrl = translateToHttpUrl(
3544
- absoluteLocation,
3545
- options?.azureAccount
3546
- );
3609
+ const httpUrl = translateToHttpUrl(absoluteLocation, {
3610
+ azureAccount: options?.azureAccount,
3611
+ s3: container?.s3
3612
+ });
3547
3613
  const store = await this.getVirtualStoreForPayload(
3548
3614
  httpUrl,
3549
3615
  payload,
@@ -3613,44 +3679,62 @@ function compareUtf8Bytes(a, b) {
3613
3679
  return bytesA.length - bytesB.length;
3614
3680
  }
3615
3681
  var VCC_SCHEME = "vcc://";
3616
- function expandVccUrl(location, containers) {
3617
- if (!location.startsWith(VCC_SCHEME)) return location;
3618
- const rest = location.slice(VCC_SCHEME.length);
3619
- const slash = rest.indexOf("/");
3620
- if (slash === -1) {
3621
- throw new Error(
3622
- `Invalid vcc:// URL "${location}": missing "/" after container name`
3623
- );
3624
- }
3625
- const name = rest.slice(0, slash);
3626
- const relativePath = rest.slice(slash + 1);
3627
- const urlPrefix = containers.get(name);
3628
- if (urlPrefix === void 0) {
3629
- if (containers.size === 0) return location;
3630
- throw new Error(
3631
- `Unknown virtual chunk container "${name}" referenced by ${location}`
3632
- );
3682
+ function resolveVirtualChunkLocation(location, containers) {
3683
+ if (location.startsWith(VCC_SCHEME)) {
3684
+ const rest = location.slice(VCC_SCHEME.length);
3685
+ const slash = rest.indexOf("/");
3686
+ if (slash === -1) {
3687
+ throw new Error(
3688
+ `Invalid vcc:// URL "${location}": missing "/" after container name`
3689
+ );
3690
+ }
3691
+ const name = rest.slice(0, slash);
3692
+ const relativePath = rest.slice(slash + 1);
3693
+ const container2 = containers.find((c) => c.name === name);
3694
+ if (!container2) {
3695
+ if (containers.length === 0) return { url: location };
3696
+ throw new Error(
3697
+ `Unknown virtual chunk container "${name}" referenced by ${location}`
3698
+ );
3699
+ }
3700
+ return {
3701
+ url: ensureTrailingSlash(container2.urlPrefix) + relativePath,
3702
+ container: container2
3703
+ };
3633
3704
  }
3634
- const normalizedPrefix = urlPrefix.endsWith("/") ? urlPrefix : `${urlPrefix}/`;
3635
- return normalizedPrefix + relativePath;
3705
+ const container = containers.find(
3706
+ (c) => location.startsWith(ensureTrailingSlash(c.urlPrefix))
3707
+ );
3708
+ return { url: location, container };
3709
+ }
3710
+ function ensureTrailingSlash(prefix) {
3711
+ return prefix.endsWith("/") ? prefix : `${prefix}/`;
3636
3712
  }
3637
- function translateToHttpUrl(url, azureAccount) {
3713
+ function buildS3HttpUrl(bucket, key, s3) {
3714
+ if (s3?.endpointUrl) {
3715
+ const base = s3.endpointUrl.replace(/\/+$/, "");
3716
+ return `${base}/${bucket}/${key}`;
3717
+ }
3718
+ const region = s3?.region;
3719
+ const pathStyle = s3?.forcePathStyle === true || bucket.includes(".");
3720
+ if (pathStyle) {
3721
+ const host2 = region ? `s3.${region}.${awsEndpointSuffix(region)}` : "s3.amazonaws.com";
3722
+ return `https://${host2}/${bucket}/${key}`;
3723
+ }
3724
+ const host = region ? `${bucket}.s3.${region}.${awsEndpointSuffix(region)}` : `${bucket}.s3.amazonaws.com`;
3725
+ return `https://${host}/${key}`;
3726
+ }
3727
+ function awsEndpointSuffix(region) {
3728
+ return region.startsWith("cn-") ? "amazonaws.com.cn" : "amazonaws.com";
3729
+ }
3730
+ function translateToHttpUrl(url, options) {
3731
+ const azureAccount = options?.azureAccount;
3638
3732
  if (url.startsWith("s3://")) {
3639
3733
  const rest = url.slice(5);
3640
3734
  const slashIndex = rest.indexOf("/");
3641
- if (slashIndex === -1) {
3642
- const bucket2 = rest;
3643
- if (bucket2.includes(".")) {
3644
- return `https://s3.amazonaws.com/${bucket2}/`;
3645
- }
3646
- return `https://${bucket2}.s3.amazonaws.com/`;
3647
- }
3648
- const bucket = rest.slice(0, slashIndex);
3649
- const key = rest.slice(slashIndex + 1);
3650
- if (bucket.includes(".")) {
3651
- return `https://s3.amazonaws.com/${bucket}/${key}`;
3652
- }
3653
- return `https://${bucket}.s3.amazonaws.com/${key}`;
3735
+ const bucket = slashIndex === -1 ? rest : rest.slice(0, slashIndex);
3736
+ const key = slashIndex === -1 ? "" : rest.slice(slashIndex + 1);
3737
+ return buildS3HttpUrl(bucket, key, options?.s3);
3654
3738
  }
3655
3739
  if (url.startsWith("gs://") || url.startsWith("gcs://")) {
3656
3740
  const prefixLen = url.startsWith("gs://") ? 5 : 6;
@@ -4010,18 +4094,37 @@ var Repository = class _Repository {
4010
4094
  */
4011
4095
  async *walkHistory(session, options) {
4012
4096
  let current = session;
4097
+ const seen = /* @__PURE__ */ new Set();
4013
4098
  while (current) {
4099
+ const id = encodeObjectId12(current.getSnapshotId());
4100
+ if (seen.has(id)) {
4101
+ throw new Error(`Cycle in snapshot history at ${id}`);
4102
+ }
4103
+ seen.add(id);
4014
4104
  yield {
4015
- id: encodeObjectId12(current.getSnapshotId()),
4105
+ id,
4016
4106
  message: current.getMessage(),
4017
4107
  flushedAt: current.getFlushedAt(),
4018
4108
  metadata: current.getSnapshotMetadata()
4019
4109
  };
4020
- const parentId = current.getParentSnapshotId();
4110
+ const parentId = current.getParentSnapshotId() ?? await this.resolveParentFromRepoInfo(current, options);
4021
4111
  if (!parentId) break;
4022
4112
  current = await this.checkoutSnapshot(parentId, options);
4023
4113
  }
4024
4114
  }
4115
+ /** v2 snapshots record ancestry only in the repo file. */
4116
+ async resolveParentFromRepoInfo(session, options) {
4117
+ const repoInfo = await this.loadRepoInfo(options);
4118
+ if (!repoInfo) return null;
4119
+ const snapshotId = session.getSnapshotId();
4120
+ const parent = resolveSnapshotParent(repoInfo, snapshotId);
4121
+ if (parent === void 0) {
4122
+ throw new Error(
4123
+ `Snapshot ${encodeObjectId12(snapshotId)} not found in repo info`
4124
+ );
4125
+ }
4126
+ return parent;
4127
+ }
4025
4128
  /**
4026
4129
  * Get the storage backend.
4027
4130
  */