icechunk-js 0.5.1 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.cts CHANGED
@@ -337,6 +337,35 @@ type RangeCoalescingFn = (store: AsyncReadable$1, opts?: {
337
337
  coalesceSize?: number;
338
338
  }) => AsyncReadable$1;
339
339
 
340
+ /**
341
+ * Parser for icechunk Repo FlatBuffer format (v2).
342
+ *
343
+ * Uses flatc-generated TypeScript classes for type-safe field access.
344
+ */
345
+
346
+ /**
347
+ * S3 store config carried by a virtual chunk container, mirroring Rust's
348
+ * `S3Options`. Drives endpoint/addressing when translating `s3://` chunk
349
+ * locations to HTTPS — notably `region`, which lets dotted-name buckets
350
+ * address their regional endpoint directly (required for browser CORS, since
351
+ * the global endpoint's region redirect carries no CORS headers).
352
+ */
353
+ interface S3ContainerConfig {
354
+ region?: string;
355
+ endpointUrl?: string;
356
+ forcePathStyle?: boolean;
357
+ }
358
+ /**
359
+ * A virtual chunk container from the repo config, mirroring Rust's
360
+ * `VirtualChunkContainer`. `name` resolves `vcc://name/...` references;
361
+ * `urlPrefix` matches absolute chunk locations (longest prefix wins).
362
+ */
363
+ interface VirtualChunkContainer {
364
+ name: string | null;
365
+ urlPrefix: string;
366
+ s3?: S3ContainerConfig;
367
+ }
368
+
340
369
  /**
341
370
  * ReadSession - Read-only session for accessing icechunk data.
342
371
  */
@@ -388,7 +417,7 @@ declare class ReadSession {
388
417
  private nextFetchClientId;
389
418
  private rangeCoalescerIds?;
390
419
  private nextRangeCoalescerId;
391
- /** VCC name → url_prefix map for resolving `vcc://` chunk locations. */
420
+ /** Virtual chunk containers (from repo config) for resolving chunk locations. */
392
421
  private virtualChunkContainers;
393
422
  private constructor();
394
423
  private getFetchClientKey;
@@ -406,7 +435,7 @@ declare class ReadSession {
406
435
  */
407
436
  static open(storage: Storage, snapshotId: Uint8Array, options?: RequestOptions & {
408
437
  maxManifestCacheSize?: number;
409
- virtualChunkContainers?: Map<string, string>;
438
+ virtualChunkContainers?: VirtualChunkContainer[];
410
439
  }): Promise<ReadSession>;
411
440
  /** Load and parse a snapshot from storage */
412
441
  private static loadSnapshot;
@@ -866,6 +895,8 @@ declare class Repository {
866
895
  flushedAt: Date;
867
896
  metadata: Record<string, unknown>;
868
897
  }>;
898
+ /** v2 snapshots record ancestry only in the repo file. */
899
+ private resolveParentFromRepoInfo;
869
900
  /**
870
901
  * Get the storage backend.
871
902
  */
package/dist/index.d.ts CHANGED
@@ -337,6 +337,35 @@ type RangeCoalescingFn = (store: AsyncReadable$1, opts?: {
337
337
  coalesceSize?: number;
338
338
  }) => AsyncReadable$1;
339
339
 
340
+ /**
341
+ * Parser for icechunk Repo FlatBuffer format (v2).
342
+ *
343
+ * Uses flatc-generated TypeScript classes for type-safe field access.
344
+ */
345
+
346
+ /**
347
+ * S3 store config carried by a virtual chunk container, mirroring Rust's
348
+ * `S3Options`. Drives endpoint/addressing when translating `s3://` chunk
349
+ * locations to HTTPS — notably `region`, which lets dotted-name buckets
350
+ * address their regional endpoint directly (required for browser CORS, since
351
+ * the global endpoint's region redirect carries no CORS headers).
352
+ */
353
+ interface S3ContainerConfig {
354
+ region?: string;
355
+ endpointUrl?: string;
356
+ forcePathStyle?: boolean;
357
+ }
358
+ /**
359
+ * A virtual chunk container from the repo config, mirroring Rust's
360
+ * `VirtualChunkContainer`. `name` resolves `vcc://name/...` references;
361
+ * `urlPrefix` matches absolute chunk locations (longest prefix wins).
362
+ */
363
+ interface VirtualChunkContainer {
364
+ name: string | null;
365
+ urlPrefix: string;
366
+ s3?: S3ContainerConfig;
367
+ }
368
+
340
369
  /**
341
370
  * ReadSession - Read-only session for accessing icechunk data.
342
371
  */
@@ -388,7 +417,7 @@ declare class ReadSession {
388
417
  private nextFetchClientId;
389
418
  private rangeCoalescerIds?;
390
419
  private nextRangeCoalescerId;
391
- /** VCC name → url_prefix map for resolving `vcc://` chunk locations. */
420
+ /** Virtual chunk containers (from repo config) for resolving chunk locations. */
392
421
  private virtualChunkContainers;
393
422
  private constructor();
394
423
  private getFetchClientKey;
@@ -406,7 +435,7 @@ declare class ReadSession {
406
435
  */
407
436
  static open(storage: Storage, snapshotId: Uint8Array, options?: RequestOptions & {
408
437
  maxManifestCacheSize?: number;
409
- virtualChunkContainers?: Map<string, string>;
438
+ virtualChunkContainers?: VirtualChunkContainer[];
410
439
  }): Promise<ReadSession>;
411
440
  /** Load and parse a snapshot from storage */
412
441
  private static loadSnapshot;
@@ -866,6 +895,8 @@ declare class Repository {
866
895
  flushedAt: Date;
867
896
  metadata: Record<string, unknown>;
868
897
  }>;
898
+ /** v2 snapshots record ancestry only in the repo file. */
899
+ private resolveParentFromRepoInfo;
869
900
  /**
870
901
  * Get the storage backend.
871
902
  */
package/dist/index.js CHANGED
@@ -1226,7 +1226,7 @@ function parseRepo(data) {
1226
1226
  return { repo, specVersion, snapshotsLength, virtualChunkContainers };
1227
1227
  }
1228
1228
  function parseVirtualChunkContainers(repo) {
1229
- const result = /* @__PURE__ */ new Map();
1229
+ const result = [];
1230
1230
  const configBytes = repo.configArray();
1231
1231
  if (!configBytes || configBytes.length === 0) return result;
1232
1232
  let config;
@@ -1244,12 +1244,27 @@ function parseVirtualChunkContainers(repo) {
1244
1244
  if (!isRecord(containers)) return result;
1245
1245
  for (const container of Object.values(containers)) {
1246
1246
  if (!isRecord(container)) continue;
1247
- const { name, url_prefix: urlPrefix } = container;
1248
- if (typeof name !== "string" || typeof urlPrefix !== "string") continue;
1249
- result.set(name, urlPrefix);
1247
+ const urlPrefix = container.url_prefix;
1248
+ if (typeof urlPrefix !== "string") continue;
1249
+ const name = typeof container.name === "string" ? container.name : null;
1250
+ const s3 = parseS3Store(container.store);
1251
+ result.push(s3 ? { name, urlPrefix, s3 } : { name, urlPrefix });
1250
1252
  }
1253
+ result.sort((a, b) => b.urlPrefix.length - a.urlPrefix.length);
1251
1254
  return result;
1252
1255
  }
1256
+ function parseS3Store(store) {
1257
+ if (!isRecord(store)) return void 0;
1258
+ const s3 = store.s3 ?? store.s3_compatible ?? store.tigris;
1259
+ if (!isRecord(s3)) return void 0;
1260
+ const cfg = {};
1261
+ if (typeof s3.region === "string") cfg.region = s3.region;
1262
+ if (typeof s3.endpoint_url === "string") cfg.endpointUrl = s3.endpoint_url;
1263
+ if (typeof s3.force_path_style === "boolean") {
1264
+ cfg.forcePathStyle = s3.force_path_style;
1265
+ }
1266
+ return cfg;
1267
+ }
1253
1268
  function isRecord(value) {
1254
1269
  return typeof value === "object" && value !== null && !Array.isArray(value);
1255
1270
  }
@@ -1347,6 +1362,33 @@ function listTagsFromRepo(parsedRepo) {
1347
1362
  const { repo } = parsedRepo;
1348
1363
  return listRefs((i) => repo.tags(i), repo.tagsLength());
1349
1364
  }
1365
+ var snapshotIndexCache = /* @__PURE__ */ new WeakMap();
1366
+ function snapshotIndexById(parsedRepo) {
1367
+ let index = snapshotIndexCache.get(parsedRepo);
1368
+ if (!index) {
1369
+ index = /* @__PURE__ */ new Map();
1370
+ for (let i = 0; i < parsedRepo.snapshotsLength; i++) {
1371
+ index.set(bytesKey(getSnapshotIdByIndex(parsedRepo, i)), i);
1372
+ }
1373
+ snapshotIndexCache.set(parsedRepo, index);
1374
+ }
1375
+ return index;
1376
+ }
1377
+ function bytesKey(bytes) {
1378
+ return Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("");
1379
+ }
1380
+ function resolveSnapshotParent(parsedRepo, snapshotId) {
1381
+ const index = snapshotIndexById(parsedRepo).get(bytesKey(snapshotId));
1382
+ if (index === void 0) return void 0;
1383
+ const parentOffset = parsedRepo.repo.snapshots(index).parentOffset();
1384
+ if (parentOffset === -1) return null;
1385
+ if (parentOffset < 0 || parentOffset >= parsedRepo.snapshotsLength) {
1386
+ throw new Error(
1387
+ `Invalid parent offset ${parentOffset} for snapshot at index ${index}, snapshots array has ${parsedRepo.snapshotsLength} entries`
1388
+ );
1389
+ }
1390
+ return getSnapshotIdByIndex(parsedRepo, parentOffset);
1391
+ }
1350
1392
 
1351
1393
  // src/cache/lru.ts
1352
1394
  var LRUCache = class {
@@ -2907,29 +2949,53 @@ function parseTransactionLog(data) {
2907
2949
  function expectedRangeLength(range) {
2908
2950
  return "suffixLength" in range ? range.suffixLength : range.length;
2909
2951
  }
2952
+ function regionalS3RedirectUrl(url, response) {
2953
+ if (response.status !== 301 && response.status !== 307) return null;
2954
+ const region = response.headers.get("x-amz-bucket-region");
2955
+ if (!region) return null;
2956
+ const globalPrefix = "https://s3.amazonaws.com/";
2957
+ if (!url.startsWith(globalPrefix)) return null;
2958
+ return `https://s3.${region}.amazonaws.com/${url.slice(globalPrefix.length)}`;
2959
+ }
2910
2960
  function makeUrlStore(opts) {
2911
- const { url, fetchClient, conditionalHeaders } = opts;
2912
- async function doFetch(init) {
2913
- return fetchClient ? await fetchClient.fetch(url, init) : await fetch(url, init);
2961
+ const { fetchClient, conditionalHeaders } = opts;
2962
+ let pinnedUrl = opts.url;
2963
+ async function doFetch(target, init) {
2964
+ return fetchClient ? await fetchClient.fetch(target, init) : await fetch(target, init);
2914
2965
  }
2915
2966
  return {
2916
2967
  async get() {
2917
2968
  throw new Error(
2918
- `Virtual chunk URL store for ${url} only supports ranged reads`
2969
+ `Virtual chunk URL store for ${pinnedUrl} only supports ranged reads`
2919
2970
  );
2920
2971
  },
2921
2972
  async getRange(_key, range, options) {
2922
2973
  const headers = conditionalHeaders ? { ...conditionalHeaders } : {};
2923
2974
  headers.Range = "suffixLength" in range ? `bytes=-${range.suffixLength}` : `bytes=${range.offset}-${range.offset + range.length - 1}`;
2924
- const response = await doFetch({ headers, signal: options?.signal });
2975
+ let requestUrl = pinnedUrl;
2976
+ let response = await doFetch(requestUrl, {
2977
+ headers,
2978
+ signal: options?.signal
2979
+ });
2980
+ const regionalUrl = regionalS3RedirectUrl(requestUrl, response);
2981
+ if (regionalUrl) {
2982
+ response.body?.cancel().catch(() => {
2983
+ });
2984
+ requestUrl = regionalUrl;
2985
+ pinnedUrl = regionalUrl;
2986
+ response = await doFetch(requestUrl, {
2987
+ headers,
2988
+ signal: options?.signal
2989
+ });
2990
+ }
2925
2991
  if (response.status === 412) {
2926
2992
  throw new Error(
2927
- `Virtual chunk at ${url} failed integrity check \u2014 data has been modified since snapshot was created`
2993
+ `Virtual chunk at ${requestUrl} failed integrity check \u2014 data has been modified since snapshot was created`
2928
2994
  );
2929
2995
  }
2930
2996
  if (response.status !== 200 && response.status !== 206) {
2931
2997
  throw new Error(
2932
- `Failed to fetch virtual chunk from ${url}: ${response.status} ${response.statusText}`
2998
+ `Failed to fetch virtual chunk from ${requestUrl}: ${response.status} ${response.statusText}`
2933
2999
  );
2934
3000
  }
2935
3001
  const data = new Uint8Array(await response.arrayBuffer());
@@ -2937,21 +3003,21 @@ function makeUrlStore(opts) {
2937
3003
  const expected = expectedRangeLength(range);
2938
3004
  if (data.length === expected) return data;
2939
3005
  throw new Error(
2940
- `Virtual range response size mismatch for ${url}: expected ${expected} bytes, got ${data.length}`
3006
+ `Virtual range response size mismatch for ${requestUrl}: expected ${expected} bytes, got ${data.length}`
2941
3007
  );
2942
3008
  }
2943
3009
  if ("offset" in range) {
2944
3010
  const end = range.offset + range.length;
2945
3011
  if (data.length >= end) return data.slice(range.offset, end);
2946
3012
  throw new Error(
2947
- `Virtual range request not honored for ${url}: need at least ${end} bytes for fallback slicing, got ${data.length}`
3013
+ `Virtual range request not honored for ${requestUrl}: need at least ${end} bytes for fallback slicing, got ${data.length}`
2948
3014
  );
2949
3015
  }
2950
3016
  if (data.length >= range.suffixLength) {
2951
3017
  return data.slice(data.length - range.suffixLength);
2952
3018
  }
2953
3019
  throw new Error(
2954
- `Virtual suffix range request not honored for ${url}: need at least ${range.suffixLength} bytes for fallback slicing, got ${data.length}`
3020
+ `Virtual suffix range request not honored for ${requestUrl}: need at least ${range.suffixLength} bytes for fallback slicing, got ${data.length}`
2955
3021
  );
2956
3022
  }
2957
3023
  };
@@ -3019,7 +3085,7 @@ var ReadSession = class _ReadSession {
3019
3085
  nextFetchClientId = 1;
3020
3086
  rangeCoalescerIds;
3021
3087
  nextRangeCoalescerId = 1;
3022
- /** VCC name → url_prefix map for resolving `vcc://` chunk locations. */
3088
+ /** Virtual chunk containers (from repo config) for resolving chunk locations. */
3023
3089
  virtualChunkContainers;
3024
3090
  constructor(storage, snapshot, specVersion, maxManifestCacheSize = 100, virtualChunkContainers) {
3025
3091
  this.storage = storage;
@@ -3027,7 +3093,7 @@ var ReadSession = class _ReadSession {
3027
3093
  this.specVersion = specVersion;
3028
3094
  this.manifestCache = new LRUCache(maxManifestCacheSize);
3029
3095
  this.manifestLoader = singleFlight(this.manifestCache);
3030
- this.virtualChunkContainers = virtualChunkContainers ?? /* @__PURE__ */ new Map();
3096
+ this.virtualChunkContainers = virtualChunkContainers ?? [];
3031
3097
  }
3032
3098
  getFetchClientKey(fetchClient) {
3033
3099
  if (!fetchClient) return "default";
@@ -3413,14 +3479,14 @@ var ReadSession = class _ReadSession {
3413
3479
  );
3414
3480
  }
3415
3481
  case "virtual": {
3416
- const absoluteLocation = expandVccUrl(
3482
+ const { url: absoluteLocation, container } = resolveVirtualChunkLocation(
3417
3483
  payload.location,
3418
3484
  this.virtualChunkContainers
3419
3485
  );
3420
- const httpUrl = translateToHttpUrl(
3421
- absoluteLocation,
3422
- options?.azureAccount
3423
- );
3486
+ const httpUrl = translateToHttpUrl(absoluteLocation, {
3487
+ azureAccount: options?.azureAccount,
3488
+ s3: container?.s3
3489
+ });
3424
3490
  const store = await this.getVirtualStoreForPayload(
3425
3491
  httpUrl,
3426
3492
  payload,
@@ -3483,14 +3549,14 @@ var ReadSession = class _ReadSession {
3483
3549
  case "virtual": {
3484
3550
  const absoluteStart = payload.offset + rangeStart;
3485
3551
  const expectedSize = rangeEnd - rangeStart;
3486
- const absoluteLocation = expandVccUrl(
3552
+ const { url: absoluteLocation, container } = resolveVirtualChunkLocation(
3487
3553
  payload.location,
3488
3554
  this.virtualChunkContainers
3489
3555
  );
3490
- const httpUrl = translateToHttpUrl(
3491
- absoluteLocation,
3492
- options?.azureAccount
3493
- );
3556
+ const httpUrl = translateToHttpUrl(absoluteLocation, {
3557
+ azureAccount: options?.azureAccount,
3558
+ s3: container?.s3
3559
+ });
3494
3560
  const store = await this.getVirtualStoreForPayload(
3495
3561
  httpUrl,
3496
3562
  payload,
@@ -3560,44 +3626,62 @@ function compareUtf8Bytes(a, b) {
3560
3626
  return bytesA.length - bytesB.length;
3561
3627
  }
3562
3628
  var VCC_SCHEME = "vcc://";
3563
- function expandVccUrl(location, containers) {
3564
- if (!location.startsWith(VCC_SCHEME)) return location;
3565
- const rest = location.slice(VCC_SCHEME.length);
3566
- const slash = rest.indexOf("/");
3567
- if (slash === -1) {
3568
- throw new Error(
3569
- `Invalid vcc:// URL "${location}": missing "/" after container name`
3570
- );
3571
- }
3572
- const name = rest.slice(0, slash);
3573
- const relativePath = rest.slice(slash + 1);
3574
- const urlPrefix = containers.get(name);
3575
- if (urlPrefix === void 0) {
3576
- if (containers.size === 0) return location;
3577
- throw new Error(
3578
- `Unknown virtual chunk container "${name}" referenced by ${location}`
3579
- );
3629
+ function resolveVirtualChunkLocation(location, containers) {
3630
+ if (location.startsWith(VCC_SCHEME)) {
3631
+ const rest = location.slice(VCC_SCHEME.length);
3632
+ const slash = rest.indexOf("/");
3633
+ if (slash === -1) {
3634
+ throw new Error(
3635
+ `Invalid vcc:// URL "${location}": missing "/" after container name`
3636
+ );
3637
+ }
3638
+ const name = rest.slice(0, slash);
3639
+ const relativePath = rest.slice(slash + 1);
3640
+ const container2 = containers.find((c) => c.name === name);
3641
+ if (!container2) {
3642
+ if (containers.length === 0) return { url: location };
3643
+ throw new Error(
3644
+ `Unknown virtual chunk container "${name}" referenced by ${location}`
3645
+ );
3646
+ }
3647
+ return {
3648
+ url: ensureTrailingSlash(container2.urlPrefix) + relativePath,
3649
+ container: container2
3650
+ };
3580
3651
  }
3581
- const normalizedPrefix = urlPrefix.endsWith("/") ? urlPrefix : `${urlPrefix}/`;
3582
- return normalizedPrefix + relativePath;
3652
+ const container = containers.find(
3653
+ (c) => location.startsWith(ensureTrailingSlash(c.urlPrefix))
3654
+ );
3655
+ return { url: location, container };
3656
+ }
3657
+ function ensureTrailingSlash(prefix) {
3658
+ return prefix.endsWith("/") ? prefix : `${prefix}/`;
3583
3659
  }
3584
- function translateToHttpUrl(url, azureAccount) {
3660
+ function buildS3HttpUrl(bucket, key, s3) {
3661
+ if (s3?.endpointUrl) {
3662
+ const base = s3.endpointUrl.replace(/\/+$/, "");
3663
+ return `${base}/${bucket}/${key}`;
3664
+ }
3665
+ const region = s3?.region;
3666
+ const pathStyle = s3?.forcePathStyle === true || bucket.includes(".");
3667
+ if (pathStyle) {
3668
+ const host2 = region ? `s3.${region}.${awsEndpointSuffix(region)}` : "s3.amazonaws.com";
3669
+ return `https://${host2}/${bucket}/${key}`;
3670
+ }
3671
+ const host = region ? `${bucket}.s3.${region}.${awsEndpointSuffix(region)}` : `${bucket}.s3.amazonaws.com`;
3672
+ return `https://${host}/${key}`;
3673
+ }
3674
+ function awsEndpointSuffix(region) {
3675
+ return region.startsWith("cn-") ? "amazonaws.com.cn" : "amazonaws.com";
3676
+ }
3677
+ function translateToHttpUrl(url, options) {
3678
+ const azureAccount = options?.azureAccount;
3585
3679
  if (url.startsWith("s3://")) {
3586
3680
  const rest = url.slice(5);
3587
3681
  const slashIndex = rest.indexOf("/");
3588
- if (slashIndex === -1) {
3589
- const bucket2 = rest;
3590
- if (bucket2.includes(".")) {
3591
- return `https://s3.amazonaws.com/${bucket2}/`;
3592
- }
3593
- return `https://${bucket2}.s3.amazonaws.com/`;
3594
- }
3595
- const bucket = rest.slice(0, slashIndex);
3596
- const key = rest.slice(slashIndex + 1);
3597
- if (bucket.includes(".")) {
3598
- return `https://s3.amazonaws.com/${bucket}/${key}`;
3599
- }
3600
- return `https://${bucket}.s3.amazonaws.com/${key}`;
3682
+ const bucket = slashIndex === -1 ? rest : rest.slice(0, slashIndex);
3683
+ const key = slashIndex === -1 ? "" : rest.slice(slashIndex + 1);
3684
+ return buildS3HttpUrl(bucket, key, options?.s3);
3601
3685
  }
3602
3686
  if (url.startsWith("gs://") || url.startsWith("gcs://")) {
3603
3687
  const prefixLen = url.startsWith("gs://") ? 5 : 6;
@@ -3957,18 +4041,37 @@ var Repository = class _Repository {
3957
4041
  */
3958
4042
  async *walkHistory(session, options) {
3959
4043
  let current = session;
4044
+ const seen = /* @__PURE__ */ new Set();
3960
4045
  while (current) {
4046
+ const id = encodeObjectId12(current.getSnapshotId());
4047
+ if (seen.has(id)) {
4048
+ throw new Error(`Cycle in snapshot history at ${id}`);
4049
+ }
4050
+ seen.add(id);
3961
4051
  yield {
3962
- id: encodeObjectId12(current.getSnapshotId()),
4052
+ id,
3963
4053
  message: current.getMessage(),
3964
4054
  flushedAt: current.getFlushedAt(),
3965
4055
  metadata: current.getSnapshotMetadata()
3966
4056
  };
3967
- const parentId = current.getParentSnapshotId();
4057
+ const parentId = current.getParentSnapshotId() ?? await this.resolveParentFromRepoInfo(current, options);
3968
4058
  if (!parentId) break;
3969
4059
  current = await this.checkoutSnapshot(parentId, options);
3970
4060
  }
3971
4061
  }
4062
+ /** v2 snapshots record ancestry only in the repo file. */
4063
+ async resolveParentFromRepoInfo(session, options) {
4064
+ const repoInfo = await this.loadRepoInfo(options);
4065
+ if (!repoInfo) return null;
4066
+ const snapshotId = session.getSnapshotId();
4067
+ const parent = resolveSnapshotParent(repoInfo, snapshotId);
4068
+ if (parent === void 0) {
4069
+ throw new Error(
4070
+ `Snapshot ${encodeObjectId12(snapshotId)} not found in repo info`
4071
+ );
4072
+ }
4073
+ return parent;
4074
+ }
3972
4075
  /**
3973
4076
  * Get the storage backend.
3974
4077
  */