icechunk-js 0.5.0 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.cts CHANGED
@@ -159,12 +159,20 @@ interface ChunkRef {
159
159
  /** Last modified timestamp (seconds since epoch) for virtual chunks */
160
160
  checksumLastModified: number;
161
161
  }
162
- /** Chunk references for a single array */
162
+ /**
163
+ * Chunk references for a single array, resolved lazily from the manifest's
164
+ * FlatBuffer. Keeping refs in the buffer avoids building a JS object per chunk,
165
+ * which would OOM on large virtual shards.
166
+ */
163
167
  interface ArrayManifest {
164
168
  /** Node ID of the array */
165
169
  nodeId: ObjectId8;
166
- /** Chunk references, sorted by index */
167
- refs: ChunkRef[];
170
+ /** Number of chunk refs (sorted by index). */
171
+ readonly numRefs: number;
172
+ /** Coordinate index of ref `i`, for the binary search in `findChunkRef`. */
173
+ refIndex(i: number): number[];
174
+ /** Materialize the full `ChunkRef` at position `i`. */
175
+ refAt(i: number): ChunkRef;
168
176
  }
169
177
  /** Manifest containing chunk references for multiple arrays */
170
178
  interface Manifest {
@@ -576,7 +584,15 @@ interface IcechunkStoreOptions {
576
584
  * - Route through a proxy
577
585
  */
578
586
  fetchClient?: FetchClient;
579
- /** Maximum number of manifests to cache in the LRU cache (default: 100) */
587
+ /**
588
+ * Maximum number of manifests to keep in the LRU cache (default: 100).
589
+ *
590
+ * This bounds the *number* of cached manifests, not their total bytes. Each
591
+ * entry retains a whole decompressed manifest shard — hundreds of MB for
592
+ * fine-level virtual stores — so a single oversized shard, or a few held at
593
+ * once, can still dominate memory. Lower this when reading stores with very
594
+ * large manifests.
595
+ */
580
596
  maxManifestCacheSize?: number;
581
597
  /**
582
598
  * Zarrita-backed range coalescing function for chunk payload reads.
package/dist/index.d.ts CHANGED
@@ -159,12 +159,20 @@ interface ChunkRef {
159
159
  /** Last modified timestamp (seconds since epoch) for virtual chunks */
160
160
  checksumLastModified: number;
161
161
  }
162
- /** Chunk references for a single array */
162
+ /**
163
+ * Chunk references for a single array, resolved lazily from the manifest's
164
+ * FlatBuffer. Keeping refs in the buffer avoids building a JS object per chunk,
165
+ * which would OOM on large virtual shards.
166
+ */
163
167
  interface ArrayManifest {
164
168
  /** Node ID of the array */
165
169
  nodeId: ObjectId8;
166
- /** Chunk references, sorted by index */
167
- refs: ChunkRef[];
170
+ /** Number of chunk refs (sorted by index). */
171
+ readonly numRefs: number;
172
+ /** Coordinate index of ref `i`, for the binary search in `findChunkRef`. */
173
+ refIndex(i: number): number[];
174
+ /** Materialize the full `ChunkRef` at position `i`. */
175
+ refAt(i: number): ChunkRef;
168
176
  }
169
177
  /** Manifest containing chunk references for multiple arrays */
170
178
  interface Manifest {
@@ -576,7 +584,15 @@ interface IcechunkStoreOptions {
576
584
  * - Route through a proxy
577
585
  */
578
586
  fetchClient?: FetchClient;
579
- /** Maximum number of manifests to cache in the LRU cache (default: 100) */
587
+ /**
588
+ * Maximum number of manifests to keep in the LRU cache (default: 100).
589
+ *
590
+ * This bounds the *number* of cached manifests, not their total bytes. Each
591
+ * entry retains a whole decompressed manifest shard — hundreds of MB for
592
+ * fine-level virtual stores — so a single oversized shard, or a few held at
593
+ * once, can still dominate memory. Lower this when reading stores with very
594
+ * large manifests.
595
+ */
580
596
  maxManifestCacheSize?: number;
581
597
  /**
582
598
  * Zarrita-backed range coalescing function for chunk payload reads.
package/dist/index.js CHANGED
@@ -2388,7 +2388,7 @@ function parseManifest(data) {
2388
2388
  for (let i = 0; i < arraysLength; i++) {
2389
2389
  const fbsArray = fbsManifest.arrays(i);
2390
2390
  if (fbsArray) {
2391
- arrays.push(parseArrayManifest(fbsArray, decodeLocation));
2391
+ arrays.push(makeArrayManifest(fbsArray, decodeLocation));
2392
2392
  }
2393
2393
  }
2394
2394
  return { id, arrays };
@@ -2416,22 +2416,32 @@ function makeLocationDecoder(fbsManifest) {
2416
2416
  return textDecoder.decode(decompressed);
2417
2417
  };
2418
2418
  }
2419
- function parseArrayManifest(fbsArray, decodeLocation) {
2419
+ function makeArrayManifest(fbsArray, decodeLocation) {
2420
2420
  const nodeIdObj = fbsArray.nodeId();
2421
2421
  if (!nodeIdObj)
2422
2422
  throw new Error("ArrayManifest missing required node_id field");
2423
2423
  const nodeId = asObjectId8(
2424
2424
  nodeIdObj.bb.bytes().slice(nodeIdObj.bb_pos, nodeIdObj.bb_pos + 8)
2425
2425
  );
2426
- const refsLength = fbsArray.refsLength();
2427
- const refs = [];
2428
- for (let i = 0; i < refsLength; i++) {
2429
- const fbsRef = fbsArray.refs(i);
2430
- if (fbsRef) {
2431
- refs.push(parseChunkRef(fbsRef, decodeLocation));
2426
+ const numRefs = fbsArray.refsLength();
2427
+ const probe = new ChunkRef();
2428
+ return {
2429
+ nodeId,
2430
+ numRefs,
2431
+ refIndex(i) {
2432
+ const fbsRef = fbsArray.refs(i, probe);
2433
+ if (!fbsRef) throw new Error(`Manifest ref ${i} out of range`);
2434
+ const indexLength = fbsRef.indexLength();
2435
+ const index = new Array(indexLength);
2436
+ for (let d = 0; d < indexLength; d++) index[d] = fbsRef.index(d);
2437
+ return index;
2438
+ },
2439
+ refAt(i) {
2440
+ const fbsRef = fbsArray.refs(i);
2441
+ if (!fbsRef) throw new Error(`Manifest ref ${i} out of range`);
2442
+ return parseChunkRef(fbsRef, decodeLocation);
2432
2443
  }
2433
- }
2434
- return { nodeId, refs };
2444
+ };
2435
2445
  }
2436
2446
  function parseChunkRef(fbsRef, decodeLocation) {
2437
2447
  const indexLength = fbsRef.indexLength();
@@ -2482,7 +2492,7 @@ function parseChunkRef(fbsRef, decodeLocation) {
2482
2492
  function findChunkRef(manifest, nodeId, coords) {
2483
2493
  const arrayManifest = binarySearchArrayManifest(manifest.arrays, nodeId);
2484
2494
  if (!arrayManifest) return null;
2485
- return binarySearchChunkRef(arrayManifest.refs, coords);
2495
+ return binarySearchChunkRef(arrayManifest, coords);
2486
2496
  }
2487
2497
  function binarySearchArrayManifest(arrays, nodeId) {
2488
2498
  let low = 0;
@@ -2514,14 +2524,14 @@ function compareCoords(a, b) {
2514
2524
  }
2515
2525
  return a.length - b.length;
2516
2526
  }
2517
- function binarySearchChunkRef(refs, coords) {
2527
+ function binarySearchChunkRef(array, coords) {
2518
2528
  let low = 0;
2519
- let high = refs.length - 1;
2529
+ let high = array.numRefs - 1;
2520
2530
  while (low <= high) {
2521
2531
  const mid = Math.floor((low + high) / 2);
2522
- const cmp = compareCoords(refs[mid].index, coords);
2532
+ const cmp = compareCoords(array.refIndex(mid), coords);
2523
2533
  if (cmp === 0) {
2524
- return refs[mid];
2534
+ return array.refAt(mid);
2525
2535
  } else if (cmp < 0) {
2526
2536
  low = mid + 1;
2527
2537
  } else {