icechunk-js 0.3.2 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.cts CHANGED
@@ -80,7 +80,7 @@ interface Storage {
80
80
  * @returns Object data as bytes
81
81
  * @throws NotFoundError if the object doesn't exist
82
82
  * @throws StorageError for other errors
83
- * @throws AbortError if the operation was aborted
83
+ * @throws An error named "AbortError" if the operation was aborted
84
84
  */
85
85
  getObject(path: string, range?: ByteRange, options?: RequestOptions): Promise<Uint8Array>;
86
86
  /**
@@ -89,7 +89,7 @@ interface Storage {
89
89
  * @param path - Path to the object
90
90
  * @param options - Optional request options (signal for cancellation)
91
91
  * @returns True if the object exists
92
- * @throws AbortError if the operation was aborted
92
+ * @throws An error named "AbortError" if the operation was aborted
93
93
  */
94
94
  exists(path: string, options?: RequestOptions): Promise<boolean>;
95
95
  /**
@@ -299,10 +299,51 @@ interface TransactionLogEntry {
299
299
  movedNodes: MoveOperationInfo[];
300
300
  }
301
301
 
302
+ /**
303
+ * Adapters for zarrita's `withRangeCoalescing` (added in zarrita 0.7).
304
+ *
305
+ * The coalescer works over any range-readable store keyed by object path.
306
+ * These adapters expose icechunk's two backing-object cases in that shape:
307
+ *
308
+ * - `makeUrlStore` fetches ranges from one external virtual-chunk URL.
309
+ * - `makeStorageStore` fetches ranges from repository storage objects.
310
+ *
311
+ * Callers pass `zarrita.withRangeCoalescing` into icechunk-js explicitly when
312
+ * they want coalescing, keeping zarrita a true optional dependency.
313
+ */
314
+
315
+ type RangeQuery$1 = {
316
+ offset: number;
317
+ length: number;
318
+ } | {
319
+ suffixLength: number;
320
+ };
321
+ interface GetOptions {
322
+ signal?: AbortSignal;
323
+ }
324
+ interface AsyncReadable$1 {
325
+ get(key: string, options?: GetOptions): Promise<Uint8Array | undefined>;
326
+ getRange(key: string, range: RangeQuery$1, options?: GetOptions): Promise<Uint8Array | undefined>;
327
+ }
328
+ type RangeCoalescingFn = (store: AsyncReadable$1, opts?: {
329
+ coalesceSize?: number;
330
+ }) => AsyncReadable$1;
331
+
302
332
  /**
303
333
  * ReadSession - Read-only session for accessing icechunk data.
304
334
  */
305
335
 
336
+ /** Options for chunk reads from a ReadSession. */
337
+ interface ReadOptions extends RequestOptions {
338
+ /**
339
+ * Zarrita-backed range coalescing function for chunk payload reads.
340
+ *
341
+ * Pass `zarrita.withRangeCoalescing` to opt in. Concurrent range reads
342
+ * against the same backing object may be merged into one larger request,
343
+ * matching zarrita's abort behavior for merged signals.
344
+ */
345
+ withRangeCoalescing?: RangeCoalescingFn;
346
+ }
306
347
  /**
307
348
  * ReadSession provides read access to a specific snapshot.
308
349
  *
@@ -316,7 +357,37 @@ declare class ReadSession {
316
357
  private snapshot;
317
358
  private specVersion;
318
359
  private manifestCache;
360
+ /**
361
+ * Single-flight loader over `manifestCache`: concurrent misses for the
362
+ * same manifest share one fetch instead of racing N identical GETs.
363
+ * Critical for fan-out reads that all need the same manifest (e.g. the
364
+ * many parallel chunk reads issued when a renderer crosses into a new
365
+ * pyramid level).
366
+ */
367
+ private manifestLoader;
368
+ /**
369
+ * Bounded cache of `AsyncReadable`s wrapped with zarrita's range coalescer.
370
+ * The cache key includes request-option identities that must not share one
371
+ * coalescing queue, notably virtual fetch clients and checksum headers.
372
+ *
373
+ * Promise slots are inserted synchronously on first use, so
374
+ * concurrent requests for the same partition share one coalescing window
375
+ * instead of racing to create parallel stores.
376
+ */
377
+ private rangeStores?;
378
+ private nativeStore?;
379
+ private fetchClientIds?;
380
+ private nextFetchClientId;
381
+ private rangeCoalescerIds?;
382
+ private nextRangeCoalescerId;
383
+ /** VCC name → url_prefix map for resolving `vcc://` chunk locations. */
384
+ private virtualChunkContainers;
319
385
  private constructor();
386
+ private getFetchClientKey;
387
+ private getRangeCoalescerKey;
388
+ private getRangeStore;
389
+ private getNativeStore;
390
+ private getVirtualStoreForPayload;
320
391
  /**
321
392
  * Open a read session for a specific snapshot.
322
393
  *
@@ -327,11 +398,28 @@ declare class ReadSession {
327
398
  */
328
399
  static open(storage: Storage, snapshotId: Uint8Array, options?: RequestOptions & {
329
400
  maxManifestCacheSize?: number;
401
+ virtualChunkContainers?: Map<string, string>;
330
402
  }): Promise<ReadSession>;
331
403
  /** Load and parse a snapshot from storage */
332
404
  private static loadSnapshot;
333
- /** Load and parse a manifest from storage */
405
+ /**
406
+ * Load and parse a manifest from storage.
407
+ *
408
+ * Concurrent calls for the same manifest are coalesced through
409
+ * `manifestLoader` so a fan-out of chunk reads — the typical case
410
+ * when many tiles cross into a new pyramid level at once — issues one
411
+ * HTTP GET instead of one per caller. A caller's `signal` rejects only
412
+ * that caller's await; the shared fetch is aborted only once every
413
+ * active waiter has aborted.
414
+ */
334
415
  private loadManifest;
416
+ /**
417
+ * Fetch and parse a single manifest. The `signal` here is the
418
+ * single-flight loader's own signal — it fires only when every waiter
419
+ * has aborted, so a hung manifest fetch doesn't trap later callers
420
+ * behind it.
421
+ */
422
+ private fetchManifest;
335
423
  /**
336
424
  * Get the snapshot ID.
337
425
  */
@@ -410,7 +498,7 @@ declare class ReadSession {
410
498
  * @param options - Optional request options (signal for cancellation)
411
499
  * @returns Chunk data bytes or null if not found
412
500
  */
413
- getChunk(path: string, coords: number[], options?: RequestOptions): Promise<Uint8Array | null>;
501
+ getChunk(path: string, coords: number[], options?: ReadOptions): Promise<Uint8Array | null>;
414
502
  /**
415
503
  * Read a byte range of chunk data for an array.
416
504
  *
@@ -429,7 +517,7 @@ declare class ReadSession {
429
517
  length: number;
430
518
  } | {
431
519
  suffixLength: number;
432
- }, options?: RequestOptions): Promise<Uint8Array | null>;
520
+ }, options?: ReadOptions): Promise<Uint8Array | null>;
433
521
  /** Check if coordinates fall within extent ranges */
434
522
  private coordsInExtents;
435
523
  /** Fetch chunk data based on payload type */
@@ -490,6 +578,13 @@ interface IcechunkStoreOptions {
490
578
  fetchClient?: FetchClient;
491
579
  /** Maximum number of manifests to cache in the LRU cache (default: 100) */
492
580
  maxManifestCacheSize?: number;
581
+ /**
582
+ * Zarrita-backed range coalescing function for chunk payload reads.
583
+ *
584
+ * Pass `zarrita.withRangeCoalescing` to opt in. Concurrent range reads
585
+ * against the same backing object may be merged into one larger request.
586
+ */
587
+ withRangeCoalescing?: RangeCoalescingFn;
493
588
  /**
494
589
  * Send If-Match / If-Unmodified-Since headers on virtual chunk requests.
495
590
  *
@@ -526,6 +621,7 @@ declare class IcechunkStore implements AsyncReadable {
526
621
  private fetchClient?;
527
622
  private validateChecksums;
528
623
  private azureAccount?;
624
+ private withRangeCoalescing?;
529
625
  private basePath;
530
626
  private constructor();
531
627
  /**
@@ -539,9 +635,9 @@ declare class IcechunkStore implements AsyncReadable {
539
635
  * Open an IcechunkStore from an existing ReadSession.
540
636
  *
541
637
  * @param session - Existing ReadSession
542
- * @param options - Store options (only fetchClient is used)
638
+ * @param options - Store options for virtual chunk reads
543
639
  */
544
- static open(session: ReadSession, options?: Pick<IcechunkStoreOptions, "fetchClient">): Promise<IcechunkStore>;
640
+ static open(session: ReadSession, options?: Pick<IcechunkStoreOptions, "fetchClient" | "validateChecksums" | "azureAccount" | "withRangeCoalescing">): Promise<IcechunkStore>;
545
641
  /**
546
642
  * Open an IcechunkStore from a custom Storage backend.
547
643
  *
@@ -660,7 +756,7 @@ declare class Repository {
660
756
  * @param options - Optional request options (signal for cancellation)
661
757
  * @returns ParsedRepo if v2 format, null if v1 format
662
758
  * @throws Error if repo file exists but fails to parse
663
- * @throws AbortError if the operation was aborted
759
+ * @throws An error named "AbortError" if the operation was aborted
664
760
  */
665
761
  private loadRepoInfo;
666
762
  /**
@@ -873,4 +969,4 @@ declare function encodeObjectId8(id: Uint8Array): string;
873
969
  */
874
970
  declare function decodeObjectId8(str: string): Uint8Array;
875
971
 
876
- export { type AbsolutePath, type ArrayManifest, type ArrayNodeData, type ArrayUpdatedChunksInfo, type AsyncReadable, type ByteRange, type ChunkIndexRange, type ChunkPayload, type ChunkRef, CompressionAlgorithm, DefaultFetchClient, type DimensionShape, type FetchClient, FileType, type GroupNodeData, HeaderParseError, HttpStorage, type HttpStorageOptions, IcechunkStore, type IcechunkStoreOptions, type InlineChunkPayload, LRUCache, type Manifest, type ManifestFileInfo, type ManifestRef, type MetadataItem, type MoveOperationInfo, type NativeChunkPayload, type NodeData, type NodeSnapshot, NotFoundError, type ObjectId12, type ObjectId8, type RangeQuery, ReadSession, type RefData, Repository, type RepositoryOptions, type RequestOptions, type Snapshot, SpecVersion, type Storage, StorageError, type TransactionLogEntry, type UpdatedChunkIndices, type VirtualChunkPayload, decodeBase32, decodeObjectId12, decodeObjectId8, encodeBase32, encodeObjectId12, encodeObjectId8 };
972
+ export { type AbsolutePath, type ArrayManifest, type ArrayNodeData, type ArrayUpdatedChunksInfo, type AsyncReadable, type ByteRange, type ChunkIndexRange, type ChunkPayload, type ChunkRef, CompressionAlgorithm, DefaultFetchClient, type DimensionShape, type FetchClient, FileType, type GroupNodeData, HeaderParseError, HttpStorage, type HttpStorageOptions, IcechunkStore, type IcechunkStoreOptions, type InlineChunkPayload, LRUCache, type Manifest, type ManifestFileInfo, type ManifestRef, type MetadataItem, type MoveOperationInfo, type NativeChunkPayload, type NodeData, type NodeSnapshot, NotFoundError, type ObjectId12, type ObjectId8, type RangeCoalescingFn, type RangeQuery, type ReadOptions, ReadSession, type RefData, Repository, type RepositoryOptions, type RequestOptions, type Snapshot, SpecVersion, type Storage, StorageError, type TransactionLogEntry, type UpdatedChunkIndices, type VirtualChunkPayload, decodeBase32, decodeObjectId12, decodeObjectId8, encodeBase32, encodeObjectId12, encodeObjectId8 };
package/dist/index.d.ts CHANGED
@@ -80,7 +80,7 @@ interface Storage {
80
80
  * @returns Object data as bytes
81
81
  * @throws NotFoundError if the object doesn't exist
82
82
  * @throws StorageError for other errors
83
- * @throws AbortError if the operation was aborted
83
+ * @throws An error named "AbortError" if the operation was aborted
84
84
  */
85
85
  getObject(path: string, range?: ByteRange, options?: RequestOptions): Promise<Uint8Array>;
86
86
  /**
@@ -89,7 +89,7 @@ interface Storage {
89
89
  * @param path - Path to the object
90
90
  * @param options - Optional request options (signal for cancellation)
91
91
  * @returns True if the object exists
92
- * @throws AbortError if the operation was aborted
92
+ * @throws An error named "AbortError" if the operation was aborted
93
93
  */
94
94
  exists(path: string, options?: RequestOptions): Promise<boolean>;
95
95
  /**
@@ -299,10 +299,51 @@ interface TransactionLogEntry {
299
299
  movedNodes: MoveOperationInfo[];
300
300
  }
301
301
 
302
+ /**
303
+ * Adapters for zarrita's `withRangeCoalescing` (added in zarrita 0.7).
304
+ *
305
+ * The coalescer works over any range-readable store keyed by object path.
306
+ * These adapters expose icechunk's two backing-object cases in that shape:
307
+ *
308
+ * - `makeUrlStore` fetches ranges from one external virtual-chunk URL.
309
+ * - `makeStorageStore` fetches ranges from repository storage objects.
310
+ *
311
+ * Callers pass `zarrita.withRangeCoalescing` into icechunk-js explicitly when
312
+ * they want coalescing, keeping zarrita a true optional dependency.
313
+ */
314
+
315
+ type RangeQuery$1 = {
316
+ offset: number;
317
+ length: number;
318
+ } | {
319
+ suffixLength: number;
320
+ };
321
+ interface GetOptions {
322
+ signal?: AbortSignal;
323
+ }
324
+ interface AsyncReadable$1 {
325
+ get(key: string, options?: GetOptions): Promise<Uint8Array | undefined>;
326
+ getRange(key: string, range: RangeQuery$1, options?: GetOptions): Promise<Uint8Array | undefined>;
327
+ }
328
+ type RangeCoalescingFn = (store: AsyncReadable$1, opts?: {
329
+ coalesceSize?: number;
330
+ }) => AsyncReadable$1;
331
+
302
332
  /**
303
333
  * ReadSession - Read-only session for accessing icechunk data.
304
334
  */
305
335
 
336
+ /** Options for chunk reads from a ReadSession. */
337
+ interface ReadOptions extends RequestOptions {
338
+ /**
339
+ * Zarrita-backed range coalescing function for chunk payload reads.
340
+ *
341
+ * Pass `zarrita.withRangeCoalescing` to opt in. Concurrent range reads
342
+ * against the same backing object may be merged into one larger request,
343
+ * matching zarrita's abort behavior for merged signals.
344
+ */
345
+ withRangeCoalescing?: RangeCoalescingFn;
346
+ }
306
347
  /**
307
348
  * ReadSession provides read access to a specific snapshot.
308
349
  *
@@ -316,7 +357,37 @@ declare class ReadSession {
316
357
  private snapshot;
317
358
  private specVersion;
318
359
  private manifestCache;
360
+ /**
361
+ * Single-flight loader over `manifestCache`: concurrent misses for the
362
+ * same manifest share one fetch instead of racing N identical GETs.
363
+ * Critical for fan-out reads that all need the same manifest (e.g. the
364
+ * many parallel chunk reads issued when a renderer crosses into a new
365
+ * pyramid level).
366
+ */
367
+ private manifestLoader;
368
+ /**
369
+ * Bounded cache of `AsyncReadable`s wrapped with zarrita's range coalescer.
370
+ * The cache key includes request-option identities that must not share one
371
+ * coalescing queue, notably virtual fetch clients and checksum headers.
372
+ *
373
+ * Promise slots are inserted synchronously on first use, so
374
+ * concurrent requests for the same partition share one coalescing window
375
+ * instead of racing to create parallel stores.
376
+ */
377
+ private rangeStores?;
378
+ private nativeStore?;
379
+ private fetchClientIds?;
380
+ private nextFetchClientId;
381
+ private rangeCoalescerIds?;
382
+ private nextRangeCoalescerId;
383
+ /** VCC name → url_prefix map for resolving `vcc://` chunk locations. */
384
+ private virtualChunkContainers;
319
385
  private constructor();
386
+ private getFetchClientKey;
387
+ private getRangeCoalescerKey;
388
+ private getRangeStore;
389
+ private getNativeStore;
390
+ private getVirtualStoreForPayload;
320
391
  /**
321
392
  * Open a read session for a specific snapshot.
322
393
  *
@@ -327,11 +398,28 @@ declare class ReadSession {
327
398
  */
328
399
  static open(storage: Storage, snapshotId: Uint8Array, options?: RequestOptions & {
329
400
  maxManifestCacheSize?: number;
401
+ virtualChunkContainers?: Map<string, string>;
330
402
  }): Promise<ReadSession>;
331
403
  /** Load and parse a snapshot from storage */
332
404
  private static loadSnapshot;
333
- /** Load and parse a manifest from storage */
405
+ /**
406
+ * Load and parse a manifest from storage.
407
+ *
408
+ * Concurrent calls for the same manifest are coalesced through
409
+ * `manifestLoader` so a fan-out of chunk reads — the typical case
410
+ * when many tiles cross into a new pyramid level at once — issues one
411
+ * HTTP GET instead of one per caller. A caller's `signal` rejects only
412
+ * that caller's await; the shared fetch is aborted only once every
413
+ * active waiter has aborted.
414
+ */
334
415
  private loadManifest;
416
+ /**
417
+ * Fetch and parse a single manifest. The `signal` here is the
418
+ * single-flight loader's own signal — it fires only when every waiter
419
+ * has aborted, so a hung manifest fetch doesn't trap later callers
420
+ * behind it.
421
+ */
422
+ private fetchManifest;
335
423
  /**
336
424
  * Get the snapshot ID.
337
425
  */
@@ -410,7 +498,7 @@ declare class ReadSession {
410
498
  * @param options - Optional request options (signal for cancellation)
411
499
  * @returns Chunk data bytes or null if not found
412
500
  */
413
- getChunk(path: string, coords: number[], options?: RequestOptions): Promise<Uint8Array | null>;
501
+ getChunk(path: string, coords: number[], options?: ReadOptions): Promise<Uint8Array | null>;
414
502
  /**
415
503
  * Read a byte range of chunk data for an array.
416
504
  *
@@ -429,7 +517,7 @@ declare class ReadSession {
429
517
  length: number;
430
518
  } | {
431
519
  suffixLength: number;
432
- }, options?: RequestOptions): Promise<Uint8Array | null>;
520
+ }, options?: ReadOptions): Promise<Uint8Array | null>;
433
521
  /** Check if coordinates fall within extent ranges */
434
522
  private coordsInExtents;
435
523
  /** Fetch chunk data based on payload type */
@@ -490,6 +578,13 @@ interface IcechunkStoreOptions {
490
578
  fetchClient?: FetchClient;
491
579
  /** Maximum number of manifests to cache in the LRU cache (default: 100) */
492
580
  maxManifestCacheSize?: number;
581
+ /**
582
+ * Zarrita-backed range coalescing function for chunk payload reads.
583
+ *
584
+ * Pass `zarrita.withRangeCoalescing` to opt in. Concurrent range reads
585
+ * against the same backing object may be merged into one larger request.
586
+ */
587
+ withRangeCoalescing?: RangeCoalescingFn;
493
588
  /**
494
589
  * Send If-Match / If-Unmodified-Since headers on virtual chunk requests.
495
590
  *
@@ -526,6 +621,7 @@ declare class IcechunkStore implements AsyncReadable {
526
621
  private fetchClient?;
527
622
  private validateChecksums;
528
623
  private azureAccount?;
624
+ private withRangeCoalescing?;
529
625
  private basePath;
530
626
  private constructor();
531
627
  /**
@@ -539,9 +635,9 @@ declare class IcechunkStore implements AsyncReadable {
539
635
  * Open an IcechunkStore from an existing ReadSession.
540
636
  *
541
637
  * @param session - Existing ReadSession
542
- * @param options - Store options (only fetchClient is used)
638
+ * @param options - Store options for virtual chunk reads
543
639
  */
544
- static open(session: ReadSession, options?: Pick<IcechunkStoreOptions, "fetchClient">): Promise<IcechunkStore>;
640
+ static open(session: ReadSession, options?: Pick<IcechunkStoreOptions, "fetchClient" | "validateChecksums" | "azureAccount" | "withRangeCoalescing">): Promise<IcechunkStore>;
545
641
  /**
546
642
  * Open an IcechunkStore from a custom Storage backend.
547
643
  *
@@ -660,7 +756,7 @@ declare class Repository {
660
756
  * @param options - Optional request options (signal for cancellation)
661
757
  * @returns ParsedRepo if v2 format, null if v1 format
662
758
  * @throws Error if repo file exists but fails to parse
663
- * @throws AbortError if the operation was aborted
759
+ * @throws An error named "AbortError" if the operation was aborted
664
760
  */
665
761
  private loadRepoInfo;
666
762
  /**
@@ -873,4 +969,4 @@ declare function encodeObjectId8(id: Uint8Array): string;
873
969
  */
874
970
  declare function decodeObjectId8(str: string): Uint8Array;
875
971
 
876
- export { type AbsolutePath, type ArrayManifest, type ArrayNodeData, type ArrayUpdatedChunksInfo, type AsyncReadable, type ByteRange, type ChunkIndexRange, type ChunkPayload, type ChunkRef, CompressionAlgorithm, DefaultFetchClient, type DimensionShape, type FetchClient, FileType, type GroupNodeData, HeaderParseError, HttpStorage, type HttpStorageOptions, IcechunkStore, type IcechunkStoreOptions, type InlineChunkPayload, LRUCache, type Manifest, type ManifestFileInfo, type ManifestRef, type MetadataItem, type MoveOperationInfo, type NativeChunkPayload, type NodeData, type NodeSnapshot, NotFoundError, type ObjectId12, type ObjectId8, type RangeQuery, ReadSession, type RefData, Repository, type RepositoryOptions, type RequestOptions, type Snapshot, SpecVersion, type Storage, StorageError, type TransactionLogEntry, type UpdatedChunkIndices, type VirtualChunkPayload, decodeBase32, decodeObjectId12, decodeObjectId8, encodeBase32, encodeObjectId12, encodeObjectId8 };
972
+ export { type AbsolutePath, type ArrayManifest, type ArrayNodeData, type ArrayUpdatedChunksInfo, type AsyncReadable, type ByteRange, type ChunkIndexRange, type ChunkPayload, type ChunkRef, CompressionAlgorithm, DefaultFetchClient, type DimensionShape, type FetchClient, FileType, type GroupNodeData, HeaderParseError, HttpStorage, type HttpStorageOptions, IcechunkStore, type IcechunkStoreOptions, type InlineChunkPayload, LRUCache, type Manifest, type ManifestFileInfo, type ManifestRef, type MetadataItem, type MoveOperationInfo, type NativeChunkPayload, type NodeData, type NodeSnapshot, NotFoundError, type ObjectId12, type ObjectId8, type RangeCoalescingFn, type RangeQuery, type ReadOptions, ReadSession, type RefData, Repository, type RepositoryOptions, type RequestOptions, type Snapshot, SpecVersion, type Storage, StorageError, type TransactionLogEntry, type UpdatedChunkIndices, type VirtualChunkPayload, decodeBase32, decodeObjectId12, decodeObjectId8, encodeBase32, encodeObjectId12, encodeObjectId8 };