icechunk-js 0.5.1 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.cts CHANGED
@@ -337,6 +337,35 @@ type RangeCoalescingFn = (store: AsyncReadable$1, opts?: {
337
337
  coalesceSize?: number;
338
338
  }) => AsyncReadable$1;
339
339
 
340
+ /**
341
+ * Parser for icechunk Repo FlatBuffer format (v2).
342
+ *
343
+ * Uses flatc-generated TypeScript classes for type-safe field access.
344
+ */
345
+
346
+ /**
347
+ * S3 store config carried by a virtual chunk container, mirroring Rust's
348
+ * `S3Options`. Drives endpoint/addressing when translating `s3://` chunk
349
+ * locations to HTTPS — notably `region`, which lets dotted-name buckets
350
+ * address their regional endpoint directly (required for browser CORS, since
351
+ * the global endpoint's region redirect carries no CORS headers).
352
+ */
353
+ interface S3ContainerConfig {
354
+ region?: string;
355
+ endpointUrl?: string;
356
+ forcePathStyle?: boolean;
357
+ }
358
+ /**
359
+ * A virtual chunk container from the repo config, mirroring Rust's
360
+ * `VirtualChunkContainer`. `name` resolves `vcc://name/...` references;
361
+ * `urlPrefix` matches absolute chunk locations (longest prefix wins).
362
+ */
363
+ interface VirtualChunkContainer {
364
+ name: string | null;
365
+ urlPrefix: string;
366
+ s3?: S3ContainerConfig;
367
+ }
368
+
340
369
  /**
341
370
  * ReadSession - Read-only session for accessing icechunk data.
342
371
  */
@@ -388,7 +417,7 @@ declare class ReadSession {
388
417
  private nextFetchClientId;
389
418
  private rangeCoalescerIds?;
390
419
  private nextRangeCoalescerId;
391
- /** VCC name → url_prefix map for resolving `vcc://` chunk locations. */
420
+ /** Virtual chunk containers (from repo config) for resolving chunk locations. */
392
421
  private virtualChunkContainers;
393
422
  private constructor();
394
423
  private getFetchClientKey;
@@ -406,7 +435,7 @@ declare class ReadSession {
406
435
  */
407
436
  static open(storage: Storage, snapshotId: Uint8Array, options?: RequestOptions & {
408
437
  maxManifestCacheSize?: number;
409
- virtualChunkContainers?: Map<string, string>;
438
+ virtualChunkContainers?: VirtualChunkContainer[];
410
439
  }): Promise<ReadSession>;
411
440
  /** Load and parse a snapshot from storage */
412
441
  private static loadSnapshot;
package/dist/index.d.ts CHANGED
@@ -337,6 +337,35 @@ type RangeCoalescingFn = (store: AsyncReadable$1, opts?: {
337
337
  coalesceSize?: number;
338
338
  }) => AsyncReadable$1;
339
339
 
340
+ /**
341
+ * Parser for icechunk Repo FlatBuffer format (v2).
342
+ *
343
+ * Uses flatc-generated TypeScript classes for type-safe field access.
344
+ */
345
+
346
+ /**
347
+ * S3 store config carried by a virtual chunk container, mirroring Rust's
348
+ * `S3Options`. Drives endpoint/addressing when translating `s3://` chunk
349
+ * locations to HTTPS — notably `region`, which lets dotted-name buckets
350
+ * address their regional endpoint directly (required for browser CORS, since
351
+ * the global endpoint's region redirect carries no CORS headers).
352
+ */
353
+ interface S3ContainerConfig {
354
+ region?: string;
355
+ endpointUrl?: string;
356
+ forcePathStyle?: boolean;
357
+ }
358
+ /**
359
+ * A virtual chunk container from the repo config, mirroring Rust's
360
+ * `VirtualChunkContainer`. `name` resolves `vcc://name/...` references;
361
+ * `urlPrefix` matches absolute chunk locations (longest prefix wins).
362
+ */
363
+ interface VirtualChunkContainer {
364
+ name: string | null;
365
+ urlPrefix: string;
366
+ s3?: S3ContainerConfig;
367
+ }
368
+
340
369
  /**
341
370
  * ReadSession - Read-only session for accessing icechunk data.
342
371
  */
@@ -388,7 +417,7 @@ declare class ReadSession {
388
417
  private nextFetchClientId;
389
418
  private rangeCoalescerIds?;
390
419
  private nextRangeCoalescerId;
391
- /** VCC name → url_prefix map for resolving `vcc://` chunk locations. */
420
+ /** Virtual chunk containers (from repo config) for resolving chunk locations. */
392
421
  private virtualChunkContainers;
393
422
  private constructor();
394
423
  private getFetchClientKey;
@@ -406,7 +435,7 @@ declare class ReadSession {
406
435
  */
407
436
  static open(storage: Storage, snapshotId: Uint8Array, options?: RequestOptions & {
408
437
  maxManifestCacheSize?: number;
409
- virtualChunkContainers?: Map<string, string>;
438
+ virtualChunkContainers?: VirtualChunkContainer[];
410
439
  }): Promise<ReadSession>;
411
440
  /** Load and parse a snapshot from storage */
412
441
  private static loadSnapshot;
package/dist/index.js CHANGED
@@ -1226,7 +1226,7 @@ function parseRepo(data) {
1226
1226
  return { repo, specVersion, snapshotsLength, virtualChunkContainers };
1227
1227
  }
1228
1228
  function parseVirtualChunkContainers(repo) {
1229
- const result = /* @__PURE__ */ new Map();
1229
+ const result = [];
1230
1230
  const configBytes = repo.configArray();
1231
1231
  if (!configBytes || configBytes.length === 0) return result;
1232
1232
  let config;
@@ -1244,12 +1244,27 @@ function parseVirtualChunkContainers(repo) {
1244
1244
  if (!isRecord(containers)) return result;
1245
1245
  for (const container of Object.values(containers)) {
1246
1246
  if (!isRecord(container)) continue;
1247
- const { name, url_prefix: urlPrefix } = container;
1248
- if (typeof name !== "string" || typeof urlPrefix !== "string") continue;
1249
- result.set(name, urlPrefix);
1247
+ const urlPrefix = container.url_prefix;
1248
+ if (typeof urlPrefix !== "string") continue;
1249
+ const name = typeof container.name === "string" ? container.name : null;
1250
+ const s3 = parseS3Store(container.store);
1251
+ result.push(s3 ? { name, urlPrefix, s3 } : { name, urlPrefix });
1250
1252
  }
1253
+ result.sort((a, b) => b.urlPrefix.length - a.urlPrefix.length);
1251
1254
  return result;
1252
1255
  }
1256
+ function parseS3Store(store) {
1257
+ if (!isRecord(store)) return void 0;
1258
+ const s3 = store.s3 ?? store.s3_compatible ?? store.tigris;
1259
+ if (!isRecord(s3)) return void 0;
1260
+ const cfg = {};
1261
+ if (typeof s3.region === "string") cfg.region = s3.region;
1262
+ if (typeof s3.endpoint_url === "string") cfg.endpointUrl = s3.endpoint_url;
1263
+ if (typeof s3.force_path_style === "boolean") {
1264
+ cfg.forcePathStyle = s3.force_path_style;
1265
+ }
1266
+ return cfg;
1267
+ }
1253
1268
  function isRecord(value) {
1254
1269
  return typeof value === "object" && value !== null && !Array.isArray(value);
1255
1270
  }
@@ -2907,29 +2922,53 @@ function parseTransactionLog(data) {
2907
2922
  function expectedRangeLength(range) {
2908
2923
  return "suffixLength" in range ? range.suffixLength : range.length;
2909
2924
  }
2925
+ function regionalS3RedirectUrl(url, response) {
2926
+ if (response.status !== 301 && response.status !== 307) return null;
2927
+ const region = response.headers.get("x-amz-bucket-region");
2928
+ if (!region) return null;
2929
+ const globalPrefix = "https://s3.amazonaws.com/";
2930
+ if (!url.startsWith(globalPrefix)) return null;
2931
+ return `https://s3.${region}.amazonaws.com/${url.slice(globalPrefix.length)}`;
2932
+ }
2910
2933
  function makeUrlStore(opts) {
2911
- const { url, fetchClient, conditionalHeaders } = opts;
2912
- async function doFetch(init) {
2913
- return fetchClient ? await fetchClient.fetch(url, init) : await fetch(url, init);
2934
+ const { fetchClient, conditionalHeaders } = opts;
2935
+ let pinnedUrl = opts.url;
2936
+ async function doFetch(target, init) {
2937
+ return fetchClient ? await fetchClient.fetch(target, init) : await fetch(target, init);
2914
2938
  }
2915
2939
  return {
2916
2940
  async get() {
2917
2941
  throw new Error(
2918
- `Virtual chunk URL store for ${url} only supports ranged reads`
2942
+ `Virtual chunk URL store for ${pinnedUrl} only supports ranged reads`
2919
2943
  );
2920
2944
  },
2921
2945
  async getRange(_key, range, options) {
2922
2946
  const headers = conditionalHeaders ? { ...conditionalHeaders } : {};
2923
2947
  headers.Range = "suffixLength" in range ? `bytes=-${range.suffixLength}` : `bytes=${range.offset}-${range.offset + range.length - 1}`;
2924
- const response = await doFetch({ headers, signal: options?.signal });
2948
+ let requestUrl = pinnedUrl;
2949
+ let response = await doFetch(requestUrl, {
2950
+ headers,
2951
+ signal: options?.signal
2952
+ });
2953
+ const regionalUrl = regionalS3RedirectUrl(requestUrl, response);
2954
+ if (regionalUrl) {
2955
+ response.body?.cancel().catch(() => {
2956
+ });
2957
+ requestUrl = regionalUrl;
2958
+ pinnedUrl = regionalUrl;
2959
+ response = await doFetch(requestUrl, {
2960
+ headers,
2961
+ signal: options?.signal
2962
+ });
2963
+ }
2925
2964
  if (response.status === 412) {
2926
2965
  throw new Error(
2927
- `Virtual chunk at ${url} failed integrity check \u2014 data has been modified since snapshot was created`
2966
+ `Virtual chunk at ${requestUrl} failed integrity check \u2014 data has been modified since snapshot was created`
2928
2967
  );
2929
2968
  }
2930
2969
  if (response.status !== 200 && response.status !== 206) {
2931
2970
  throw new Error(
2932
- `Failed to fetch virtual chunk from ${url}: ${response.status} ${response.statusText}`
2971
+ `Failed to fetch virtual chunk from ${requestUrl}: ${response.status} ${response.statusText}`
2933
2972
  );
2934
2973
  }
2935
2974
  const data = new Uint8Array(await response.arrayBuffer());
@@ -2937,21 +2976,21 @@ function makeUrlStore(opts) {
2937
2976
  const expected = expectedRangeLength(range);
2938
2977
  if (data.length === expected) return data;
2939
2978
  throw new Error(
2940
- `Virtual range response size mismatch for ${url}: expected ${expected} bytes, got ${data.length}`
2979
+ `Virtual range response size mismatch for ${requestUrl}: expected ${expected} bytes, got ${data.length}`
2941
2980
  );
2942
2981
  }
2943
2982
  if ("offset" in range) {
2944
2983
  const end = range.offset + range.length;
2945
2984
  if (data.length >= end) return data.slice(range.offset, end);
2946
2985
  throw new Error(
2947
- `Virtual range request not honored for ${url}: need at least ${end} bytes for fallback slicing, got ${data.length}`
2986
+ `Virtual range request not honored for ${requestUrl}: need at least ${end} bytes for fallback slicing, got ${data.length}`
2948
2987
  );
2949
2988
  }
2950
2989
  if (data.length >= range.suffixLength) {
2951
2990
  return data.slice(data.length - range.suffixLength);
2952
2991
  }
2953
2992
  throw new Error(
2954
- `Virtual suffix range request not honored for ${url}: need at least ${range.suffixLength} bytes for fallback slicing, got ${data.length}`
2993
+ `Virtual suffix range request not honored for ${requestUrl}: need at least ${range.suffixLength} bytes for fallback slicing, got ${data.length}`
2955
2994
  );
2956
2995
  }
2957
2996
  };
@@ -3019,7 +3058,7 @@ var ReadSession = class _ReadSession {
3019
3058
  nextFetchClientId = 1;
3020
3059
  rangeCoalescerIds;
3021
3060
  nextRangeCoalescerId = 1;
3022
- /** VCC name → url_prefix map for resolving `vcc://` chunk locations. */
3061
+ /** Virtual chunk containers (from repo config) for resolving chunk locations. */
3023
3062
  virtualChunkContainers;
3024
3063
  constructor(storage, snapshot, specVersion, maxManifestCacheSize = 100, virtualChunkContainers) {
3025
3064
  this.storage = storage;
@@ -3027,7 +3066,7 @@ var ReadSession = class _ReadSession {
3027
3066
  this.specVersion = specVersion;
3028
3067
  this.manifestCache = new LRUCache(maxManifestCacheSize);
3029
3068
  this.manifestLoader = singleFlight(this.manifestCache);
3030
- this.virtualChunkContainers = virtualChunkContainers ?? /* @__PURE__ */ new Map();
3069
+ this.virtualChunkContainers = virtualChunkContainers ?? [];
3031
3070
  }
3032
3071
  getFetchClientKey(fetchClient) {
3033
3072
  if (!fetchClient) return "default";
@@ -3413,14 +3452,14 @@ var ReadSession = class _ReadSession {
3413
3452
  );
3414
3453
  }
3415
3454
  case "virtual": {
3416
- const absoluteLocation = expandVccUrl(
3455
+ const { url: absoluteLocation, container } = resolveVirtualChunkLocation(
3417
3456
  payload.location,
3418
3457
  this.virtualChunkContainers
3419
3458
  );
3420
- const httpUrl = translateToHttpUrl(
3421
- absoluteLocation,
3422
- options?.azureAccount
3423
- );
3459
+ const httpUrl = translateToHttpUrl(absoluteLocation, {
3460
+ azureAccount: options?.azureAccount,
3461
+ s3: container?.s3
3462
+ });
3424
3463
  const store = await this.getVirtualStoreForPayload(
3425
3464
  httpUrl,
3426
3465
  payload,
@@ -3483,14 +3522,14 @@ var ReadSession = class _ReadSession {
3483
3522
  case "virtual": {
3484
3523
  const absoluteStart = payload.offset + rangeStart;
3485
3524
  const expectedSize = rangeEnd - rangeStart;
3486
- const absoluteLocation = expandVccUrl(
3525
+ const { url: absoluteLocation, container } = resolveVirtualChunkLocation(
3487
3526
  payload.location,
3488
3527
  this.virtualChunkContainers
3489
3528
  );
3490
- const httpUrl = translateToHttpUrl(
3491
- absoluteLocation,
3492
- options?.azureAccount
3493
- );
3529
+ const httpUrl = translateToHttpUrl(absoluteLocation, {
3530
+ azureAccount: options?.azureAccount,
3531
+ s3: container?.s3
3532
+ });
3494
3533
  const store = await this.getVirtualStoreForPayload(
3495
3534
  httpUrl,
3496
3535
  payload,
@@ -3560,44 +3599,62 @@ function compareUtf8Bytes(a, b) {
3560
3599
  return bytesA.length - bytesB.length;
3561
3600
  }
3562
3601
  var VCC_SCHEME = "vcc://";
3563
- function expandVccUrl(location, containers) {
3564
- if (!location.startsWith(VCC_SCHEME)) return location;
3565
- const rest = location.slice(VCC_SCHEME.length);
3566
- const slash = rest.indexOf("/");
3567
- if (slash === -1) {
3568
- throw new Error(
3569
- `Invalid vcc:// URL "${location}": missing "/" after container name`
3570
- );
3571
- }
3572
- const name = rest.slice(0, slash);
3573
- const relativePath = rest.slice(slash + 1);
3574
- const urlPrefix = containers.get(name);
3575
- if (urlPrefix === void 0) {
3576
- if (containers.size === 0) return location;
3577
- throw new Error(
3578
- `Unknown virtual chunk container "${name}" referenced by ${location}`
3579
- );
3602
+ function resolveVirtualChunkLocation(location, containers) {
3603
+ if (location.startsWith(VCC_SCHEME)) {
3604
+ const rest = location.slice(VCC_SCHEME.length);
3605
+ const slash = rest.indexOf("/");
3606
+ if (slash === -1) {
3607
+ throw new Error(
3608
+ `Invalid vcc:// URL "${location}": missing "/" after container name`
3609
+ );
3610
+ }
3611
+ const name = rest.slice(0, slash);
3612
+ const relativePath = rest.slice(slash + 1);
3613
+ const container2 = containers.find((c) => c.name === name);
3614
+ if (!container2) {
3615
+ if (containers.length === 0) return { url: location };
3616
+ throw new Error(
3617
+ `Unknown virtual chunk container "${name}" referenced by ${location}`
3618
+ );
3619
+ }
3620
+ return {
3621
+ url: ensureTrailingSlash(container2.urlPrefix) + relativePath,
3622
+ container: container2
3623
+ };
3580
3624
  }
3581
- const normalizedPrefix = urlPrefix.endsWith("/") ? urlPrefix : `${urlPrefix}/`;
3582
- return normalizedPrefix + relativePath;
3625
+ const container = containers.find(
3626
+ (c) => location.startsWith(ensureTrailingSlash(c.urlPrefix))
3627
+ );
3628
+ return { url: location, container };
3629
+ }
3630
+ function ensureTrailingSlash(prefix) {
3631
+ return prefix.endsWith("/") ? prefix : `${prefix}/`;
3632
+ }
3633
+ function buildS3HttpUrl(bucket, key, s3) {
3634
+ if (s3?.endpointUrl) {
3635
+ const base = s3.endpointUrl.replace(/\/+$/, "");
3636
+ return `${base}/${bucket}/${key}`;
3637
+ }
3638
+ const region = s3?.region;
3639
+ const pathStyle = s3?.forcePathStyle === true || bucket.includes(".");
3640
+ if (pathStyle) {
3641
+ const host2 = region ? `s3.${region}.${awsEndpointSuffix(region)}` : "s3.amazonaws.com";
3642
+ return `https://${host2}/${bucket}/${key}`;
3643
+ }
3644
+ const host = region ? `${bucket}.s3.${region}.${awsEndpointSuffix(region)}` : `${bucket}.s3.amazonaws.com`;
3645
+ return `https://${host}/${key}`;
3583
3646
  }
3584
- function translateToHttpUrl(url, azureAccount) {
3647
+ function awsEndpointSuffix(region) {
3648
+ return region.startsWith("cn-") ? "amazonaws.com.cn" : "amazonaws.com";
3649
+ }
3650
+ function translateToHttpUrl(url, options) {
3651
+ const azureAccount = options?.azureAccount;
3585
3652
  if (url.startsWith("s3://")) {
3586
3653
  const rest = url.slice(5);
3587
3654
  const slashIndex = rest.indexOf("/");
3588
- if (slashIndex === -1) {
3589
- const bucket2 = rest;
3590
- if (bucket2.includes(".")) {
3591
- return `https://s3.amazonaws.com/${bucket2}/`;
3592
- }
3593
- return `https://${bucket2}.s3.amazonaws.com/`;
3594
- }
3595
- const bucket = rest.slice(0, slashIndex);
3596
- const key = rest.slice(slashIndex + 1);
3597
- if (bucket.includes(".")) {
3598
- return `https://s3.amazonaws.com/${bucket}/${key}`;
3599
- }
3600
- return `https://${bucket}.s3.amazonaws.com/${key}`;
3655
+ const bucket = slashIndex === -1 ? rest : rest.slice(0, slashIndex);
3656
+ const key = slashIndex === -1 ? "" : rest.slice(slashIndex + 1);
3657
+ return buildS3HttpUrl(bucket, key, options?.s3);
3601
3658
  }
3602
3659
  if (url.startsWith("gs://") || url.startsWith("gcs://")) {
3603
3660
  const prefixLen = url.startsWith("gs://") ? 5 : 6;