icechunk-js 0.6.1 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -10
- package/dist/index.cjs +476 -41
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +140 -13
- package/dist/index.d.ts +140 -13
- package/dist/index.js +475 -41
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -53,18 +53,36 @@ if not already available. The version is set in `scripts/ensure-flatc.sh`.
|
|
|
53
53
|
### IcechunkStore
|
|
54
54
|
|
|
55
55
|
The main class for zarrita integration. Implements zarrita's `AsyncReadable`
|
|
56
|
-
interface with both `get()` and `getRange()` (needed for sharded arrays).
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
56
|
+
interface with both `get()` and `getRange()` (needed for sharded arrays).
|
|
57
|
+
|
|
58
|
+
Concurrent chunk reads are range-coalesced by default: reads issued in the same
|
|
59
|
+
microtask turn that target the same backing object — one virtual-chunk file, or
|
|
60
|
+
one native chunk object — are sorted, deduplicated and merged into one Range
|
|
61
|
+
request per contiguous run, then sliced apart so each read gets exactly its
|
|
62
|
+
bytes. (A `zarr.get` normally issues all its chunk reads in one turn; a
|
|
63
|
+
`createQueue` concurrency limit, or chunks waiting on different manifests,
|
|
64
|
+
split them into several batches.) Neighbours up to `maxGap` apart (default
|
|
65
|
+
256 KiB; the gap bytes are fetched and discarded) merge into requests of at
|
|
66
|
+
most `maxBytes` (default 16 MiB) whose gap bytes stay within `maxWasteRatio`
|
|
67
|
+
(default 1) times the bytes delivered. A failed merged request is retried one
|
|
68
|
+
chunk at a time, and aborting one read never cancels the others. Tune it with
|
|
69
|
+
`coalesce: { maxGap, maxBytes, maxWasteRatio, window }` (`window > 0` batches
|
|
70
|
+
on a `setTimeout` timer, which browsers throttle in background tabs), turn it
|
|
71
|
+
off with `coalesce: false`, or read `store.rangeCoalescer.stats` to see
|
|
72
|
+
requests issued and bytes fetched/discarded. Each `IcechunkStore.open` builds
|
|
73
|
+
its own coalescer; use `resolve()` to derive sub-stores that share one.
|
|
74
|
+
|
|
75
|
+
zarrita's own `withRangeCoalescing` (zarrita >= 0.7) can be passed as
|
|
76
|
+
`withRangeCoalescing` to replace the built-in wrapper.
|
|
77
|
+
|
|
78
|
+
> **Note:** zarrita's wrapper merges abort signals with `AbortSignal.any`. If
|
|
79
|
+
> one read in a merged batch is aborted, other reads in the same batch may also
|
|
80
|
+
> reject. With it, avoid sharing an `AbortController` across requests that must
|
|
81
|
+
> cancel independently.
|
|
64
82
|
|
|
65
83
|
```typescript
|
|
66
84
|
import { IcechunkStore } from "icechunk-js";
|
|
67
|
-
import { withRangeCoalescing } from "zarrita";
|
|
85
|
+
import { withRangeCoalescing } from "zarrita"; // optional
|
|
68
86
|
|
|
69
87
|
// Open from a URL (default: branch "main")
|
|
70
88
|
const store = await IcechunkStore.open("https://example.com/repo", {
|
|
@@ -73,7 +91,8 @@ const store = await IcechunkStore.open("https://example.com/repo", {
|
|
|
73
91
|
// snapshot: 'ABC123...',
|
|
74
92
|
// formatVersion: 'v1', // skip format auto-detection for v1 repos
|
|
75
93
|
// maxManifestCacheSize: 50, // LRU cache size (default: 100)
|
|
76
|
-
//
|
|
94
|
+
// coalesce: { maxGap: 256 << 10, maxBytes: 16 << 20, maxWasteRatio: 1, window: 0 }, // or false
|
|
95
|
+
// withRangeCoalescing, // zarrita's wrapper instead of the built-in one
|
|
77
96
|
// signal: abortController.signal, // cancel initialization
|
|
78
97
|
// validateChecksums: true, // integrity headers for virtual chunks
|
|
79
98
|
// azureAccount: 'myaccount', // required for az:// virtual chunks
|
|
@@ -144,6 +163,13 @@ a region (repos written by Icechunk do), reads go straight to that regional
|
|
|
144
163
|
endpoint — which serves CORS and works in both Node and the browser, with no
|
|
145
164
|
extra configuration.
|
|
146
165
|
|
|
166
|
+
The config is read from the repo file of a v2 repository and from `config.yaml`
|
|
167
|
+
of a v1 repository. The `config.yaml` reader covers the layout Icechunk writes
|
|
168
|
+
for ordinary values, not YAML in general: a containers section it cannot read
|
|
169
|
+
(one edited by hand into another style, or holding a value with control
|
|
170
|
+
characters or a line break) is ignored as a whole, as is a missing or
|
|
171
|
+
unreadable file.
|
|
172
|
+
|
|
147
173
|
If no region is known, dotted-name buckets fall back to the global endpoint,
|
|
148
174
|
whose region redirect (for buckets outside `us-east-1`) is resolved
|
|
149
175
|
automatically in Node but not in browsers — the cross-origin redirect carries no
|