icechunk-js 0.6.1 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -10
- package/dist/index.cjs +476 -41
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +140 -13
- package/dist/index.d.ts +140 -13
- package/dist/index.js +475 -41
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.d.cts
CHANGED
|
@@ -31,6 +31,17 @@ declare class DefaultFetchClient implements FetchClient {
|
|
|
31
31
|
interface RequestOptions {
|
|
32
32
|
/** AbortSignal for request cancellation */
|
|
33
33
|
signal?: AbortSignal;
|
|
34
|
+
/**
|
|
35
|
+
* Fetch cache mode for this request, overriding the storage default.
|
|
36
|
+
*
|
|
37
|
+
* Content-addressed objects (snapshots, manifests, chunks) never change and
|
|
38
|
+
* may be cached freely. The repo pointer and ref.json files are overwritten
|
|
39
|
+
* in place by every commit; the repository reads them with "no-cache" so a
|
|
40
|
+
* browser revalidates them (an ETag round trip) instead of serving a copy
|
|
41
|
+
* it judged fresh by heuristics, which would pin a client to an old
|
|
42
|
+
* snapshot.
|
|
43
|
+
*/
|
|
44
|
+
cache?: RequestCache;
|
|
34
45
|
/** Pluggable HTTP client for virtual chunk fetching */
|
|
35
46
|
fetchClient?: FetchClient;
|
|
36
47
|
/**
|
|
@@ -63,7 +74,11 @@ declare class NotFoundError extends Error {
|
|
|
63
74
|
/** Error thrown for storage operations */
|
|
64
75
|
declare class StorageError extends Error {
|
|
65
76
|
readonly cause?: Error | undefined;
|
|
66
|
-
|
|
77
|
+
/** HTTP status that produced this error, when there was one. */
|
|
78
|
+
readonly status?: number | undefined;
|
|
79
|
+
constructor(message: string, cause?: Error | undefined,
|
|
80
|
+
/** HTTP status that produced this error, when there was one. */
|
|
81
|
+
status?: number | undefined);
|
|
67
82
|
}
|
|
68
83
|
/**
|
|
69
84
|
* Storage interface for reading icechunk data.
|
|
@@ -308,16 +323,33 @@ interface TransactionLogEntry {
|
|
|
308
323
|
}
|
|
309
324
|
|
|
310
325
|
/**
|
|
311
|
-
*
|
|
326
|
+
* Range coalescing for chunk payload reads.
|
|
312
327
|
*
|
|
313
|
-
*
|
|
314
|
-
* These adapters expose icechunk's two backing-object cases in that shape:
|
|
328
|
+
* Two layers live here:
|
|
315
329
|
*
|
|
316
|
-
*
|
|
317
|
-
*
|
|
330
|
+
* 1. Adapters that expose icechunk's two backing-object cases as a minimal
|
|
331
|
+
* range-readable store keyed by object path:
|
|
332
|
+
* - `makeUrlStore` fetches ranges from one external virtual-chunk URL.
|
|
333
|
+
* - `makeStorageStore` fetches ranges from repository storage objects.
|
|
318
334
|
*
|
|
319
|
-
*
|
|
320
|
-
*
|
|
335
|
+
* 2. `createRangeCoalescer`, the built-in coalescer. It is a
|
|
336
|
+
* `RangeCoalescingFn` — the same shape as zarrita's `withRangeCoalescing`
|
|
337
|
+
* (zarrita >= 0.7) — so it plugs into the same seam in `ReadSession`.
|
|
338
|
+
*
|
|
339
|
+
* Why this seam: `ReadSession` already wraps one `AsyncReadable` per backing
|
|
340
|
+
* object (virtual URL or native chunk object) and partitions those stores by
|
|
341
|
+
* fetch client and conditional headers, so reads that must not share a request
|
|
342
|
+
* never meet in one queue. Coalescing as a wrapper around that store's
|
|
343
|
+
* `getRange` touches neither `Storage` nor `FetchClient`, leaves the session's
|
|
344
|
+
* fetch path unchanged, and lets a caller swap in zarrita's wrapper (or none).
|
|
345
|
+
*
|
|
346
|
+
* The built-in coalescer batches `getRange` calls that arrive in the same
|
|
347
|
+
* window (default: the current microtask turn), sorts them by offset, dedupes
|
|
348
|
+
* identical ranges, and merges neighbours whose gap is at most `maxGap` into
|
|
349
|
+
* one request of at most `maxBytes`. Gap bytes are fetched and discarded. A
|
|
350
|
+
* failed merged request is retried one member at a time so one bad range
|
|
351
|
+
* cannot poison a batch. A waiter's abort rejects only that waiter; the merged
|
|
352
|
+
* request itself is aborted once every waiter has aborted.
|
|
321
353
|
*/
|
|
322
354
|
|
|
323
355
|
type RangeQuery$1 = {
|
|
@@ -336,6 +368,65 @@ interface AsyncReadable$1 {
|
|
|
336
368
|
type RangeCoalescingFn = (store: AsyncReadable$1, opts?: {
|
|
337
369
|
coalesceSize?: number;
|
|
338
370
|
}) => AsyncReadable$1;
|
|
371
|
+
/** Options for the built-in range coalescer. */
|
|
372
|
+
interface RangeCoalesceOptions {
|
|
373
|
+
/**
|
|
374
|
+
* Largest byte gap between two reads that is bridged by one request
|
|
375
|
+
* (default 256 KiB). Gap bytes are fetched and discarded, so this trades
|
|
376
|
+
* bandwidth for request count; `maxWasteRatio` bounds the total.
|
|
377
|
+
*/
|
|
378
|
+
maxGap?: number;
|
|
379
|
+
/**
|
|
380
|
+
* Largest merged request in bytes (default 16 MiB). A single read larger
|
|
381
|
+
* than this is still issued whole; it just never merges with a neighbour.
|
|
382
|
+
*/
|
|
383
|
+
maxBytes?: number;
|
|
384
|
+
/**
|
|
385
|
+
* Largest ratio of gap bytes to delivered bytes one merged request may
|
|
386
|
+
* reach (default 1): a gap is bridged only while the request's discarded
|
|
387
|
+
* bytes stay at or below the bytes its callers asked for. `Infinity`
|
|
388
|
+
* leaves `maxGap` as the only bound; `0` merges adjacent reads only.
|
|
389
|
+
*/
|
|
390
|
+
maxWasteRatio?: number;
|
|
391
|
+
/**
|
|
392
|
+
* Batching window in milliseconds. `0` (default) flushes on the next
|
|
393
|
+
* microtask, so reads issued in the same microtask turn batch together
|
|
394
|
+
* with no added latency. A positive value flushes on a `setTimeout` timer
|
|
395
|
+
* instead and also catches reads that trickle in across turns, at the cost
|
|
396
|
+
* of that much delay per batch — and browsers throttle timers in
|
|
397
|
+
* background tabs.
|
|
398
|
+
*/
|
|
399
|
+
window?: number;
|
|
400
|
+
}
|
|
401
|
+
/** Counters kept by the built-in range coalescer. Cheap; read them any time. */
|
|
402
|
+
interface RangeCoalesceStats {
|
|
403
|
+
/** Range reads received from callers (suffix reads excluded). */
|
|
404
|
+
ranges: number;
|
|
405
|
+
/** Range requests issued to the wrapped store. */
|
|
406
|
+
requests: number;
|
|
407
|
+
/** Bytes callers asked for, duplicates included. */
|
|
408
|
+
bytesRequested: number;
|
|
409
|
+
/** Bytes requested from the wrapped store. */
|
|
410
|
+
bytesFetched: number;
|
|
411
|
+
/** Fetched bytes that no caller asked for (gap bytes). */
|
|
412
|
+
bytesDiscarded: number;
|
|
413
|
+
/** Merged requests that failed and were retried one member at a time. */
|
|
414
|
+
fallbacks: number;
|
|
415
|
+
}
|
|
416
|
+
/** A `RangeCoalescingFn` with shared counters. Built by `createRangeCoalescer`. */
|
|
417
|
+
interface RangeCoalescer extends RangeCoalescingFn {
|
|
418
|
+
readonly stats: RangeCoalesceStats;
|
|
419
|
+
resetStats(): void;
|
|
420
|
+
}
|
|
421
|
+
/**
|
|
422
|
+
* Create the built-in range coalescer.
|
|
423
|
+
*
|
|
424
|
+
* The result wraps one `AsyncReadable` at a time (`ReadSession` calls it once
|
|
425
|
+
* per backing object) and all wrapped stores share the returned `stats`.
|
|
426
|
+
* The `coalesceSize` hint the session passes when wrapping is zarrita's
|
|
427
|
+
* knob; the built-in coalescer uses its own `maxGap` instead.
|
|
428
|
+
*/
|
|
429
|
+
declare function createRangeCoalescer(options?: RangeCoalesceOptions): RangeCoalescer;
|
|
339
430
|
|
|
340
431
|
/**
|
|
341
432
|
* Parser for icechunk Repo FlatBuffer format (v2).
|
|
@@ -624,10 +715,25 @@ interface IcechunkStoreOptions {
|
|
|
624
715
|
*/
|
|
625
716
|
maxManifestCacheSize?: number;
|
|
626
717
|
/**
|
|
627
|
-
*
|
|
718
|
+
* Range coalescing for chunk reads. On by default: reads issued in the
|
|
719
|
+
* same microtask turn for nearby byte ranges of one backing object (a
|
|
720
|
+
* virtual-chunk file or a native chunk object) are merged into one Range
|
|
721
|
+
* request and sliced apart on return. Pass options to tune the gap, request
|
|
722
|
+
* size, waste bound and batching window, or `false` to issue one request
|
|
723
|
+
* per chunk.
|
|
628
724
|
*
|
|
629
|
-
*
|
|
630
|
-
*
|
|
725
|
+
* Each `IcechunkStore.open` call builds its own coalescer, so stores opened
|
|
726
|
+
* separately from one `ReadSession` batch separately and each adds its own
|
|
727
|
+
* entries to the session's bounded cache of wrapped stores; derive
|
|
728
|
+
* sub-stores with `resolve()` to share one. Ignored when
|
|
729
|
+
* `withRangeCoalescing` is set.
|
|
730
|
+
*/
|
|
731
|
+
coalesce?: RangeCoalesceOptions | false;
|
|
732
|
+
/**
|
|
733
|
+
* Custom range-coalescing wrapper for chunk payload reads, replacing the
|
|
734
|
+
* built-in one — for example `zarrita.withRangeCoalescing` (zarrita >= 0.7).
|
|
735
|
+
* Note that zarrita's wrapper merges abort signals with `AbortSignal.any`:
|
|
736
|
+
* aborting one read in a merged batch may reject the others.
|
|
631
737
|
*/
|
|
632
738
|
withRangeCoalescing?: RangeCoalescingFn;
|
|
633
739
|
/**
|
|
@@ -682,7 +788,7 @@ declare class IcechunkStore implements AsyncReadable {
|
|
|
682
788
|
* @param session - Existing ReadSession
|
|
683
789
|
* @param options - Store options for virtual chunk reads
|
|
684
790
|
*/
|
|
685
|
-
static open(session: ReadSession, options?: Pick<IcechunkStoreOptions, "fetchClient" | "validateChecksums" | "azureAccount" | "withRangeCoalescing">): Promise<IcechunkStore>;
|
|
791
|
+
static open(session: ReadSession, options?: Pick<IcechunkStoreOptions, "fetchClient" | "validateChecksums" | "azureAccount" | "coalesce" | "withRangeCoalescing">): Promise<IcechunkStore>;
|
|
686
792
|
/**
|
|
687
793
|
* Open an IcechunkStore from a custom Storage backend.
|
|
688
794
|
*
|
|
@@ -690,6 +796,12 @@ declare class IcechunkStore implements AsyncReadable {
|
|
|
690
796
|
* @param options - Store options (branch, tag, or snapshot to checkout)
|
|
691
797
|
*/
|
|
692
798
|
static open(storage: Storage, options?: IcechunkStoreOptions): Promise<IcechunkStore>;
|
|
799
|
+
/**
|
|
800
|
+
* The built-in range coalescer this store reads chunks through, with its
|
|
801
|
+
* counters (`stats`: requests issued, bytes fetched and discarded, ...).
|
|
802
|
+
* `undefined` when coalescing is off or a custom wrapper was supplied.
|
|
803
|
+
*/
|
|
804
|
+
get rangeCoalescer(): RangeCoalescer | undefined;
|
|
693
805
|
/**
|
|
694
806
|
* Get data for a zarr key.
|
|
695
807
|
*
|
|
@@ -790,6 +902,7 @@ declare class Repository {
|
|
|
790
902
|
private storage;
|
|
791
903
|
private repoInfo;
|
|
792
904
|
private repoInfoAttempted;
|
|
905
|
+
private v1Containers;
|
|
793
906
|
private constructor();
|
|
794
907
|
/**
|
|
795
908
|
* Load and cache the v2 repo info file.
|
|
@@ -804,6 +917,20 @@ declare class Repository {
|
|
|
804
917
|
* @throws An error named "AbortError" if the operation was aborted
|
|
805
918
|
*/
|
|
806
919
|
private loadRepoInfo;
|
|
920
|
+
/**
|
|
921
|
+
* Load and cache the virtual chunk containers of a v1 repository, which
|
|
922
|
+
* keeps its config in `config.yaml` rather than in a repo file.
|
|
923
|
+
*
|
|
924
|
+
* Best-effort, like the v2 config. A repository that never saved a config
|
|
925
|
+
* has no such file (S3 answers 403 rather than 404 for a missing key of a
|
|
926
|
+
* bucket that cannot be listed), and a file that cannot be parsed declares
|
|
927
|
+
* no containers; chunk locations then resolve as they would without a
|
|
928
|
+
* config. Any other failure also yields no containers but is not cached, so
|
|
929
|
+
* the next checkout asks again.
|
|
930
|
+
*
|
|
931
|
+
* @throws An error named "AbortError" if the operation was aborted
|
|
932
|
+
*/
|
|
933
|
+
private loadV1Containers;
|
|
807
934
|
/**
|
|
808
935
|
* Open an icechunk repository.
|
|
809
936
|
*
|
|
@@ -1016,4 +1143,4 @@ declare function encodeObjectId8(id: Uint8Array): string;
|
|
|
1016
1143
|
*/
|
|
1017
1144
|
declare function decodeObjectId8(str: string): Uint8Array;
|
|
1018
1145
|
|
|
1019
|
-
export { type AbsolutePath, type ArrayManifest, type ArrayNodeData, type ArrayUpdatedChunksInfo, type AsyncReadable, type ByteRange, type ChunkIndexRange, type ChunkPayload, type ChunkRef, CompressionAlgorithm, DefaultFetchClient, type DimensionShape, type FetchClient, FileType, type GroupNodeData, HeaderParseError, HttpStorage, type HttpStorageOptions, IcechunkStore, type IcechunkStoreOptions, type InlineChunkPayload, LRUCache, type Manifest, type ManifestFileInfo, type ManifestRef, type MetadataItem, type MoveOperationInfo, type NativeChunkPayload, type NodeData, type NodeSnapshot, NotFoundError, type ObjectId12, type ObjectId8, type RangeCoalescingFn, type RangeQuery, type ReadOptions, ReadSession, type RefData, Repository, type RepositoryOptions, type RequestOptions, type Snapshot, SpecVersion, type Storage, StorageError, type TransactionLogEntry, type UpdatedChunkIndices, type VirtualChunkPayload, decodeBase32, decodeObjectId12, decodeObjectId8, encodeBase32, encodeObjectId12, encodeObjectId8 };
|
|
1146
|
+
export { type AbsolutePath, type ArrayManifest, type ArrayNodeData, type ArrayUpdatedChunksInfo, type AsyncReadable, type ByteRange, type ChunkIndexRange, type ChunkPayload, type ChunkRef, CompressionAlgorithm, DefaultFetchClient, type DimensionShape, type FetchClient, FileType, type GroupNodeData, HeaderParseError, HttpStorage, type HttpStorageOptions, IcechunkStore, type IcechunkStoreOptions, type InlineChunkPayload, LRUCache, type Manifest, type ManifestFileInfo, type ManifestRef, type MetadataItem, type MoveOperationInfo, type NativeChunkPayload, type NodeData, type NodeSnapshot, NotFoundError, type ObjectId12, type ObjectId8, type RangeCoalesceOptions, type RangeCoalesceStats, type RangeCoalescer, type RangeCoalescingFn, type RangeQuery, type ReadOptions, ReadSession, type RefData, Repository, type RepositoryOptions, type RequestOptions, type Snapshot, SpecVersion, type Storage, StorageError, type TransactionLogEntry, type UpdatedChunkIndices, type VirtualChunkPayload, createRangeCoalescer, decodeBase32, decodeObjectId12, decodeObjectId8, encodeBase32, encodeObjectId12, encodeObjectId8 };
|
package/dist/index.d.ts
CHANGED
|
@@ -31,6 +31,17 @@ declare class DefaultFetchClient implements FetchClient {
|
|
|
31
31
|
interface RequestOptions {
|
|
32
32
|
/** AbortSignal for request cancellation */
|
|
33
33
|
signal?: AbortSignal;
|
|
34
|
+
/**
|
|
35
|
+
* Fetch cache mode for this request, overriding the storage default.
|
|
36
|
+
*
|
|
37
|
+
* Content-addressed objects (snapshots, manifests, chunks) never change and
|
|
38
|
+
* may be cached freely. The repo pointer and ref.json files are overwritten
|
|
39
|
+
* in place by every commit; the repository reads them with "no-cache" so a
|
|
40
|
+
* browser revalidates them (an ETag round trip) instead of serving a copy
|
|
41
|
+
* it judged fresh by heuristics, which would pin a client to an old
|
|
42
|
+
* snapshot.
|
|
43
|
+
*/
|
|
44
|
+
cache?: RequestCache;
|
|
34
45
|
/** Pluggable HTTP client for virtual chunk fetching */
|
|
35
46
|
fetchClient?: FetchClient;
|
|
36
47
|
/**
|
|
@@ -63,7 +74,11 @@ declare class NotFoundError extends Error {
|
|
|
63
74
|
/** Error thrown for storage operations */
|
|
64
75
|
declare class StorageError extends Error {
|
|
65
76
|
readonly cause?: Error | undefined;
|
|
66
|
-
|
|
77
|
+
/** HTTP status that produced this error, when there was one. */
|
|
78
|
+
readonly status?: number | undefined;
|
|
79
|
+
constructor(message: string, cause?: Error | undefined,
|
|
80
|
+
/** HTTP status that produced this error, when there was one. */
|
|
81
|
+
status?: number | undefined);
|
|
67
82
|
}
|
|
68
83
|
/**
|
|
69
84
|
* Storage interface for reading icechunk data.
|
|
@@ -308,16 +323,33 @@ interface TransactionLogEntry {
|
|
|
308
323
|
}
|
|
309
324
|
|
|
310
325
|
/**
|
|
311
|
-
*
|
|
326
|
+
* Range coalescing for chunk payload reads.
|
|
312
327
|
*
|
|
313
|
-
*
|
|
314
|
-
* These adapters expose icechunk's two backing-object cases in that shape:
|
|
328
|
+
* Two layers live here:
|
|
315
329
|
*
|
|
316
|
-
*
|
|
317
|
-
*
|
|
330
|
+
* 1. Adapters that expose icechunk's two backing-object cases as a minimal
|
|
331
|
+
* range-readable store keyed by object path:
|
|
332
|
+
* - `makeUrlStore` fetches ranges from one external virtual-chunk URL.
|
|
333
|
+
* - `makeStorageStore` fetches ranges from repository storage objects.
|
|
318
334
|
*
|
|
319
|
-
*
|
|
320
|
-
*
|
|
335
|
+
* 2. `createRangeCoalescer`, the built-in coalescer. It is a
|
|
336
|
+
* `RangeCoalescingFn` — the same shape as zarrita's `withRangeCoalescing`
|
|
337
|
+
* (zarrita >= 0.7) — so it plugs into the same seam in `ReadSession`.
|
|
338
|
+
*
|
|
339
|
+
* Why this seam: `ReadSession` already wraps one `AsyncReadable` per backing
|
|
340
|
+
* object (virtual URL or native chunk object) and partitions those stores by
|
|
341
|
+
* fetch client and conditional headers, so reads that must not share a request
|
|
342
|
+
* never meet in one queue. Coalescing as a wrapper around that store's
|
|
343
|
+
* `getRange` touches neither `Storage` nor `FetchClient`, leaves the session's
|
|
344
|
+
* fetch path unchanged, and lets a caller swap in zarrita's wrapper (or none).
|
|
345
|
+
*
|
|
346
|
+
* The built-in coalescer batches `getRange` calls that arrive in the same
|
|
347
|
+
* window (default: the current microtask turn), sorts them by offset, dedupes
|
|
348
|
+
* identical ranges, and merges neighbours whose gap is at most `maxGap` into
|
|
349
|
+
* one request of at most `maxBytes`. Gap bytes are fetched and discarded. A
|
|
350
|
+
* failed merged request is retried one member at a time so one bad range
|
|
351
|
+
* cannot poison a batch. A waiter's abort rejects only that waiter; the merged
|
|
352
|
+
* request itself is aborted once every waiter has aborted.
|
|
321
353
|
*/
|
|
322
354
|
|
|
323
355
|
type RangeQuery$1 = {
|
|
@@ -336,6 +368,65 @@ interface AsyncReadable$1 {
|
|
|
336
368
|
type RangeCoalescingFn = (store: AsyncReadable$1, opts?: {
|
|
337
369
|
coalesceSize?: number;
|
|
338
370
|
}) => AsyncReadable$1;
|
|
371
|
+
/** Options for the built-in range coalescer. */
|
|
372
|
+
interface RangeCoalesceOptions {
|
|
373
|
+
/**
|
|
374
|
+
* Largest byte gap between two reads that is bridged by one request
|
|
375
|
+
* (default 256 KiB). Gap bytes are fetched and discarded, so this trades
|
|
376
|
+
* bandwidth for request count; `maxWasteRatio` bounds the total.
|
|
377
|
+
*/
|
|
378
|
+
maxGap?: number;
|
|
379
|
+
/**
|
|
380
|
+
* Largest merged request in bytes (default 16 MiB). A single read larger
|
|
381
|
+
* than this is still issued whole; it just never merges with a neighbour.
|
|
382
|
+
*/
|
|
383
|
+
maxBytes?: number;
|
|
384
|
+
/**
|
|
385
|
+
* Largest ratio of gap bytes to delivered bytes one merged request may
|
|
386
|
+
* reach (default 1): a gap is bridged only while the request's discarded
|
|
387
|
+
* bytes stay at or below the bytes its callers asked for. `Infinity`
|
|
388
|
+
* leaves `maxGap` as the only bound; `0` merges adjacent reads only.
|
|
389
|
+
*/
|
|
390
|
+
maxWasteRatio?: number;
|
|
391
|
+
/**
|
|
392
|
+
* Batching window in milliseconds. `0` (default) flushes on the next
|
|
393
|
+
* microtask, so reads issued in the same microtask turn batch together
|
|
394
|
+
* with no added latency. A positive value flushes on a `setTimeout` timer
|
|
395
|
+
* instead and also catches reads that trickle in across turns, at the cost
|
|
396
|
+
* of that much delay per batch — and browsers throttle timers in
|
|
397
|
+
* background tabs.
|
|
398
|
+
*/
|
|
399
|
+
window?: number;
|
|
400
|
+
}
|
|
401
|
+
/** Counters kept by the built-in range coalescer. Cheap; read them any time. */
|
|
402
|
+
interface RangeCoalesceStats {
|
|
403
|
+
/** Range reads received from callers (suffix reads excluded). */
|
|
404
|
+
ranges: number;
|
|
405
|
+
/** Range requests issued to the wrapped store. */
|
|
406
|
+
requests: number;
|
|
407
|
+
/** Bytes callers asked for, duplicates included. */
|
|
408
|
+
bytesRequested: number;
|
|
409
|
+
/** Bytes requested from the wrapped store. */
|
|
410
|
+
bytesFetched: number;
|
|
411
|
+
/** Fetched bytes that no caller asked for (gap bytes). */
|
|
412
|
+
bytesDiscarded: number;
|
|
413
|
+
/** Merged requests that failed and were retried one member at a time. */
|
|
414
|
+
fallbacks: number;
|
|
415
|
+
}
|
|
416
|
+
/** A `RangeCoalescingFn` with shared counters. Built by `createRangeCoalescer`. */
|
|
417
|
+
interface RangeCoalescer extends RangeCoalescingFn {
|
|
418
|
+
readonly stats: RangeCoalesceStats;
|
|
419
|
+
resetStats(): void;
|
|
420
|
+
}
|
|
421
|
+
/**
|
|
422
|
+
* Create the built-in range coalescer.
|
|
423
|
+
*
|
|
424
|
+
* The result wraps one `AsyncReadable` at a time (`ReadSession` calls it once
|
|
425
|
+
* per backing object) and all wrapped stores share the returned `stats`.
|
|
426
|
+
* The `coalesceSize` hint the session passes when wrapping is zarrita's
|
|
427
|
+
* knob; the built-in coalescer uses its own `maxGap` instead.
|
|
428
|
+
*/
|
|
429
|
+
declare function createRangeCoalescer(options?: RangeCoalesceOptions): RangeCoalescer;
|
|
339
430
|
|
|
340
431
|
/**
|
|
341
432
|
* Parser for icechunk Repo FlatBuffer format (v2).
|
|
@@ -624,10 +715,25 @@ interface IcechunkStoreOptions {
|
|
|
624
715
|
*/
|
|
625
716
|
maxManifestCacheSize?: number;
|
|
626
717
|
/**
|
|
627
|
-
*
|
|
718
|
+
* Range coalescing for chunk reads. On by default: reads issued in the
|
|
719
|
+
* same microtask turn for nearby byte ranges of one backing object (a
|
|
720
|
+
* virtual-chunk file or a native chunk object) are merged into one Range
|
|
721
|
+
* request and sliced apart on return. Pass options to tune the gap, request
|
|
722
|
+
* size, waste bound and batching window, or `false` to issue one request
|
|
723
|
+
* per chunk.
|
|
628
724
|
*
|
|
629
|
-
*
|
|
630
|
-
*
|
|
725
|
+
* Each `IcechunkStore.open` call builds its own coalescer, so stores opened
|
|
726
|
+
* separately from one `ReadSession` batch separately and each adds its own
|
|
727
|
+
* entries to the session's bounded cache of wrapped stores; derive
|
|
728
|
+
* sub-stores with `resolve()` to share one. Ignored when
|
|
729
|
+
* `withRangeCoalescing` is set.
|
|
730
|
+
*/
|
|
731
|
+
coalesce?: RangeCoalesceOptions | false;
|
|
732
|
+
/**
|
|
733
|
+
* Custom range-coalescing wrapper for chunk payload reads, replacing the
|
|
734
|
+
* built-in one — for example `zarrita.withRangeCoalescing` (zarrita >= 0.7).
|
|
735
|
+
* Note that zarrita's wrapper merges abort signals with `AbortSignal.any`:
|
|
736
|
+
* aborting one read in a merged batch may reject the others.
|
|
631
737
|
*/
|
|
632
738
|
withRangeCoalescing?: RangeCoalescingFn;
|
|
633
739
|
/**
|
|
@@ -682,7 +788,7 @@ declare class IcechunkStore implements AsyncReadable {
|
|
|
682
788
|
* @param session - Existing ReadSession
|
|
683
789
|
* @param options - Store options for virtual chunk reads
|
|
684
790
|
*/
|
|
685
|
-
static open(session: ReadSession, options?: Pick<IcechunkStoreOptions, "fetchClient" | "validateChecksums" | "azureAccount" | "withRangeCoalescing">): Promise<IcechunkStore>;
|
|
791
|
+
static open(session: ReadSession, options?: Pick<IcechunkStoreOptions, "fetchClient" | "validateChecksums" | "azureAccount" | "coalesce" | "withRangeCoalescing">): Promise<IcechunkStore>;
|
|
686
792
|
/**
|
|
687
793
|
* Open an IcechunkStore from a custom Storage backend.
|
|
688
794
|
*
|
|
@@ -690,6 +796,12 @@ declare class IcechunkStore implements AsyncReadable {
|
|
|
690
796
|
* @param options - Store options (branch, tag, or snapshot to checkout)
|
|
691
797
|
*/
|
|
692
798
|
static open(storage: Storage, options?: IcechunkStoreOptions): Promise<IcechunkStore>;
|
|
799
|
+
/**
|
|
800
|
+
* The built-in range coalescer this store reads chunks through, with its
|
|
801
|
+
* counters (`stats`: requests issued, bytes fetched and discarded, ...).
|
|
802
|
+
* `undefined` when coalescing is off or a custom wrapper was supplied.
|
|
803
|
+
*/
|
|
804
|
+
get rangeCoalescer(): RangeCoalescer | undefined;
|
|
693
805
|
/**
|
|
694
806
|
* Get data for a zarr key.
|
|
695
807
|
*
|
|
@@ -790,6 +902,7 @@ declare class Repository {
|
|
|
790
902
|
private storage;
|
|
791
903
|
private repoInfo;
|
|
792
904
|
private repoInfoAttempted;
|
|
905
|
+
private v1Containers;
|
|
793
906
|
private constructor();
|
|
794
907
|
/**
|
|
795
908
|
* Load and cache the v2 repo info file.
|
|
@@ -804,6 +917,20 @@ declare class Repository {
|
|
|
804
917
|
* @throws An error named "AbortError" if the operation was aborted
|
|
805
918
|
*/
|
|
806
919
|
private loadRepoInfo;
|
|
920
|
+
/**
|
|
921
|
+
* Load and cache the virtual chunk containers of a v1 repository, which
|
|
922
|
+
* keeps its config in `config.yaml` rather than in a repo file.
|
|
923
|
+
*
|
|
924
|
+
* Best-effort, like the v2 config. A repository that never saved a config
|
|
925
|
+
* has no such file (S3 answers 403 rather than 404 for a missing key of a
|
|
926
|
+
* bucket that cannot be listed), and a file that cannot be parsed declares
|
|
927
|
+
* no containers; chunk locations then resolve as they would without a
|
|
928
|
+
* config. Any other failure also yields no containers but is not cached, so
|
|
929
|
+
* the next checkout asks again.
|
|
930
|
+
*
|
|
931
|
+
* @throws An error named "AbortError" if the operation was aborted
|
|
932
|
+
*/
|
|
933
|
+
private loadV1Containers;
|
|
807
934
|
/**
|
|
808
935
|
* Open an icechunk repository.
|
|
809
936
|
*
|
|
@@ -1016,4 +1143,4 @@ declare function encodeObjectId8(id: Uint8Array): string;
|
|
|
1016
1143
|
*/
|
|
1017
1144
|
declare function decodeObjectId8(str: string): Uint8Array;
|
|
1018
1145
|
|
|
1019
|
-
export { type AbsolutePath, type ArrayManifest, type ArrayNodeData, type ArrayUpdatedChunksInfo, type AsyncReadable, type ByteRange, type ChunkIndexRange, type ChunkPayload, type ChunkRef, CompressionAlgorithm, DefaultFetchClient, type DimensionShape, type FetchClient, FileType, type GroupNodeData, HeaderParseError, HttpStorage, type HttpStorageOptions, IcechunkStore, type IcechunkStoreOptions, type InlineChunkPayload, LRUCache, type Manifest, type ManifestFileInfo, type ManifestRef, type MetadataItem, type MoveOperationInfo, type NativeChunkPayload, type NodeData, type NodeSnapshot, NotFoundError, type ObjectId12, type ObjectId8, type RangeCoalescingFn, type RangeQuery, type ReadOptions, ReadSession, type RefData, Repository, type RepositoryOptions, type RequestOptions, type Snapshot, SpecVersion, type Storage, StorageError, type TransactionLogEntry, type UpdatedChunkIndices, type VirtualChunkPayload, decodeBase32, decodeObjectId12, decodeObjectId8, encodeBase32, encodeObjectId12, encodeObjectId8 };
|
|
1146
|
+
export { type AbsolutePath, type ArrayManifest, type ArrayNodeData, type ArrayUpdatedChunksInfo, type AsyncReadable, type ByteRange, type ChunkIndexRange, type ChunkPayload, type ChunkRef, CompressionAlgorithm, DefaultFetchClient, type DimensionShape, type FetchClient, FileType, type GroupNodeData, HeaderParseError, HttpStorage, type HttpStorageOptions, IcechunkStore, type IcechunkStoreOptions, type InlineChunkPayload, LRUCache, type Manifest, type ManifestFileInfo, type ManifestRef, type MetadataItem, type MoveOperationInfo, type NativeChunkPayload, type NodeData, type NodeSnapshot, NotFoundError, type ObjectId12, type ObjectId8, type RangeCoalesceOptions, type RangeCoalesceStats, type RangeCoalescer, type RangeCoalescingFn, type RangeQuery, type ReadOptions, ReadSession, type RefData, Repository, type RepositoryOptions, type RequestOptions, type Snapshot, SpecVersion, type Storage, StorageError, type TransactionLogEntry, type UpdatedChunkIndices, type VirtualChunkPayload, createRangeCoalescer, decodeBase32, decodeObjectId12, decodeObjectId8, encodeBase32, encodeObjectId12, encodeObjectId8 };
|