icechunk-js 0.5.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +25 -15
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +20 -4
- package/dist/index.d.ts +20 -4
- package/dist/index.js +25 -15
- package/dist/index.js.map +1 -1
- package/package.json +4 -1
package/dist/index.d.cts
CHANGED
|
@@ -159,12 +159,20 @@ interface ChunkRef {
|
|
|
159
159
|
/** Last modified timestamp (seconds since epoch) for virtual chunks */
|
|
160
160
|
checksumLastModified: number;
|
|
161
161
|
}
|
|
162
|
-
/**
|
|
162
|
+
/**
|
|
163
|
+
* Chunk references for a single array, resolved lazily from the manifest's
|
|
164
|
+
* FlatBuffer. Keeping refs in the buffer avoids building a JS object per chunk,
|
|
165
|
+
* which would OOM on large virtual shards.
|
|
166
|
+
*/
|
|
163
167
|
interface ArrayManifest {
|
|
164
168
|
/** Node ID of the array */
|
|
165
169
|
nodeId: ObjectId8;
|
|
166
|
-
/**
|
|
167
|
-
|
|
170
|
+
/** Number of chunk refs (sorted by index). */
|
|
171
|
+
readonly numRefs: number;
|
|
172
|
+
/** Coordinate index of ref `i`, for the binary search in `findChunkRef`. */
|
|
173
|
+
refIndex(i: number): number[];
|
|
174
|
+
/** Materialize the full `ChunkRef` at position `i`. */
|
|
175
|
+
refAt(i: number): ChunkRef;
|
|
168
176
|
}
|
|
169
177
|
/** Manifest containing chunk references for multiple arrays */
|
|
170
178
|
interface Manifest {
|
|
@@ -576,7 +584,15 @@ interface IcechunkStoreOptions {
|
|
|
576
584
|
* - Route through a proxy
|
|
577
585
|
*/
|
|
578
586
|
fetchClient?: FetchClient;
|
|
579
|
-
/**
|
|
587
|
+
/**
|
|
588
|
+
* Maximum number of manifests to keep in the LRU cache (default: 100).
|
|
589
|
+
*
|
|
590
|
+
* This bounds the *number* of cached manifests, not their total bytes. Each
|
|
591
|
+
* entry retains a whole decompressed manifest shard — hundreds of MB for
|
|
592
|
+
* fine-level virtual stores — so a single oversized shard, or a few held at
|
|
593
|
+
* once, can still dominate memory. Lower this when reading stores with very
|
|
594
|
+
* large manifests.
|
|
595
|
+
*/
|
|
580
596
|
maxManifestCacheSize?: number;
|
|
581
597
|
/**
|
|
582
598
|
* Zarrita-backed range coalescing function for chunk payload reads.
|
package/dist/index.d.ts
CHANGED
|
@@ -159,12 +159,20 @@ interface ChunkRef {
|
|
|
159
159
|
/** Last modified timestamp (seconds since epoch) for virtual chunks */
|
|
160
160
|
checksumLastModified: number;
|
|
161
161
|
}
|
|
162
|
-
/**
|
|
162
|
+
/**
|
|
163
|
+
* Chunk references for a single array, resolved lazily from the manifest's
|
|
164
|
+
* FlatBuffer. Keeping refs in the buffer avoids building a JS object per chunk,
|
|
165
|
+
* which would OOM on large virtual shards.
|
|
166
|
+
*/
|
|
163
167
|
interface ArrayManifest {
|
|
164
168
|
/** Node ID of the array */
|
|
165
169
|
nodeId: ObjectId8;
|
|
166
|
-
/**
|
|
167
|
-
|
|
170
|
+
/** Number of chunk refs (sorted by index). */
|
|
171
|
+
readonly numRefs: number;
|
|
172
|
+
/** Coordinate index of ref `i`, for the binary search in `findChunkRef`. */
|
|
173
|
+
refIndex(i: number): number[];
|
|
174
|
+
/** Materialize the full `ChunkRef` at position `i`. */
|
|
175
|
+
refAt(i: number): ChunkRef;
|
|
168
176
|
}
|
|
169
177
|
/** Manifest containing chunk references for multiple arrays */
|
|
170
178
|
interface Manifest {
|
|
@@ -576,7 +584,15 @@ interface IcechunkStoreOptions {
|
|
|
576
584
|
* - Route through a proxy
|
|
577
585
|
*/
|
|
578
586
|
fetchClient?: FetchClient;
|
|
579
|
-
/**
|
|
587
|
+
/**
|
|
588
|
+
* Maximum number of manifests to keep in the LRU cache (default: 100).
|
|
589
|
+
*
|
|
590
|
+
* This bounds the *number* of cached manifests, not their total bytes. Each
|
|
591
|
+
* entry retains a whole decompressed manifest shard — hundreds of MB for
|
|
592
|
+
* fine-level virtual stores — so a single oversized shard, or a few held at
|
|
593
|
+
* once, can still dominate memory. Lower this when reading stores with very
|
|
594
|
+
* large manifests.
|
|
595
|
+
*/
|
|
580
596
|
maxManifestCacheSize?: number;
|
|
581
597
|
/**
|
|
582
598
|
* Zarrita-backed range coalescing function for chunk payload reads.
|
package/dist/index.js
CHANGED
|
@@ -2388,7 +2388,7 @@ function parseManifest(data) {
|
|
|
2388
2388
|
for (let i = 0; i < arraysLength; i++) {
|
|
2389
2389
|
const fbsArray = fbsManifest.arrays(i);
|
|
2390
2390
|
if (fbsArray) {
|
|
2391
|
-
arrays.push(
|
|
2391
|
+
arrays.push(makeArrayManifest(fbsArray, decodeLocation));
|
|
2392
2392
|
}
|
|
2393
2393
|
}
|
|
2394
2394
|
return { id, arrays };
|
|
@@ -2416,22 +2416,32 @@ function makeLocationDecoder(fbsManifest) {
|
|
|
2416
2416
|
return textDecoder.decode(decompressed);
|
|
2417
2417
|
};
|
|
2418
2418
|
}
|
|
2419
|
-
function
|
|
2419
|
+
function makeArrayManifest(fbsArray, decodeLocation) {
|
|
2420
2420
|
const nodeIdObj = fbsArray.nodeId();
|
|
2421
2421
|
if (!nodeIdObj)
|
|
2422
2422
|
throw new Error("ArrayManifest missing required node_id field");
|
|
2423
2423
|
const nodeId = asObjectId8(
|
|
2424
2424
|
nodeIdObj.bb.bytes().slice(nodeIdObj.bb_pos, nodeIdObj.bb_pos + 8)
|
|
2425
2425
|
);
|
|
2426
|
-
const
|
|
2427
|
-
const
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2426
|
+
const numRefs = fbsArray.refsLength();
|
|
2427
|
+
const probe = new ChunkRef();
|
|
2428
|
+
return {
|
|
2429
|
+
nodeId,
|
|
2430
|
+
numRefs,
|
|
2431
|
+
refIndex(i) {
|
|
2432
|
+
const fbsRef = fbsArray.refs(i, probe);
|
|
2433
|
+
if (!fbsRef) throw new Error(`Manifest ref ${i} out of range`);
|
|
2434
|
+
const indexLength = fbsRef.indexLength();
|
|
2435
|
+
const index = new Array(indexLength);
|
|
2436
|
+
for (let d = 0; d < indexLength; d++) index[d] = fbsRef.index(d);
|
|
2437
|
+
return index;
|
|
2438
|
+
},
|
|
2439
|
+
refAt(i) {
|
|
2440
|
+
const fbsRef = fbsArray.refs(i);
|
|
2441
|
+
if (!fbsRef) throw new Error(`Manifest ref ${i} out of range`);
|
|
2442
|
+
return parseChunkRef(fbsRef, decodeLocation);
|
|
2432
2443
|
}
|
|
2433
|
-
}
|
|
2434
|
-
return { nodeId, refs };
|
|
2444
|
+
};
|
|
2435
2445
|
}
|
|
2436
2446
|
function parseChunkRef(fbsRef, decodeLocation) {
|
|
2437
2447
|
const indexLength = fbsRef.indexLength();
|
|
@@ -2482,7 +2492,7 @@ function parseChunkRef(fbsRef, decodeLocation) {
|
|
|
2482
2492
|
function findChunkRef(manifest, nodeId, coords) {
|
|
2483
2493
|
const arrayManifest = binarySearchArrayManifest(manifest.arrays, nodeId);
|
|
2484
2494
|
if (!arrayManifest) return null;
|
|
2485
|
-
return binarySearchChunkRef(arrayManifest
|
|
2495
|
+
return binarySearchChunkRef(arrayManifest, coords);
|
|
2486
2496
|
}
|
|
2487
2497
|
function binarySearchArrayManifest(arrays, nodeId) {
|
|
2488
2498
|
let low = 0;
|
|
@@ -2514,14 +2524,14 @@ function compareCoords(a, b) {
|
|
|
2514
2524
|
}
|
|
2515
2525
|
return a.length - b.length;
|
|
2516
2526
|
}
|
|
2517
|
-
function binarySearchChunkRef(
|
|
2527
|
+
function binarySearchChunkRef(array, coords) {
|
|
2518
2528
|
let low = 0;
|
|
2519
|
-
let high =
|
|
2529
|
+
let high = array.numRefs - 1;
|
|
2520
2530
|
while (low <= high) {
|
|
2521
2531
|
const mid = Math.floor((low + high) / 2);
|
|
2522
|
-
const cmp = compareCoords(
|
|
2532
|
+
const cmp = compareCoords(array.refIndex(mid), coords);
|
|
2523
2533
|
if (cmp === 0) {
|
|
2524
|
-
return
|
|
2534
|
+
return array.refAt(mid);
|
|
2525
2535
|
} else if (cmp < 0) {
|
|
2526
2536
|
low = mid + 1;
|
|
2527
2537
|
} else {
|