icechunk-js 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -14
- package/dist/index.cjs +543 -231
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +110 -12
- package/dist/index.d.ts +110 -12
- package/dist/index.js +543 -231
- package/dist/index.js.map +1 -1
- package/package.json +10 -10
package/README.md
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
#
|
|
1
|
+
# icechunk-js
|
|
2
2
|
|
|
3
3
|
[](https://www.npmjs.com/package/icechunk-js)
|
|
4
4
|
[](https://github.com/earth-mover/icechunk)
|
|
@@ -8,8 +8,7 @@
|
|
|
8
8
|
Read-only JavaScript/TypeScript reader for Icechunk repositories, designed for
|
|
9
9
|
use with [zarrita](https://github.com/manzt/zarrita.js).
|
|
10
10
|
|
|
11
|
-
-
|
|
12
|
-
- Works in browsers, Node.js 18+, Deno, and Bun
|
|
11
|
+
- Pure TypeScript, works in browsers and Node.js 18+
|
|
13
12
|
- Icechunk v1 and v2 format auto-detection
|
|
14
13
|
- All chunk payload types: inline, native, and virtual
|
|
15
14
|
|
|
@@ -29,7 +28,7 @@ import { open, get } from "zarrita";
|
|
|
29
28
|
const store = await IcechunkStore.open("https://bucket.s3.amazonaws.com/repo");
|
|
30
29
|
|
|
31
30
|
// Open an array and read data
|
|
32
|
-
const array = await open(store, { kind: "array"
|
|
31
|
+
const array = await open(store.resolve("/temperature"), { kind: "array" });
|
|
33
32
|
const data = await get(array, [0, 0, null]);
|
|
34
33
|
```
|
|
35
34
|
|
|
@@ -45,19 +44,27 @@ npm run test
|
|
|
45
44
|
npm run typecheck
|
|
46
45
|
```
|
|
47
46
|
|
|
48
|
-
To regenerate FlatBuffers TypeScript after syncing schemas, run
|
|
49
|
-
The pinned `flatc` compiler is downloaded automatically
|
|
50
|
-
The version is set in `scripts/ensure-flatc.sh`.
|
|
47
|
+
To regenerate FlatBuffers TypeScript after syncing schemas, run
|
|
48
|
+
`npm run generate:fbs`. The pinned `flatc` compiler is downloaded automatically
|
|
49
|
+
if not already available. The version is set in `scripts/ensure-flatc.sh`.
|
|
51
50
|
|
|
52
51
|
## API
|
|
53
52
|
|
|
54
53
|
### IcechunkStore
|
|
55
54
|
|
|
56
55
|
The main class for zarrita integration. Implements zarrita's `AsyncReadable`
|
|
57
|
-
interface with both `get()` and `getRange()` (needed for sharded arrays).
|
|
56
|
+
interface with both `get()` and `getRange()` (needed for sharded arrays). Pass
|
|
57
|
+
zarrita's `withRangeCoalescing` to coalesce concurrent reads against the same
|
|
58
|
+
backing object. This requires zarrita >= 0.7.
|
|
59
|
+
|
|
60
|
+
> **Note:** Range coalescing uses zarrita's merged abort-signal behavior. If one
|
|
61
|
+
> read in a merged batch is aborted, other reads in the same batch may also
|
|
62
|
+
> reject. Avoid sharing an `AbortController` across requests that must cancel
|
|
63
|
+
> independently.
|
|
58
64
|
|
|
59
65
|
```typescript
|
|
60
66
|
import { IcechunkStore } from "icechunk-js";
|
|
67
|
+
import { withRangeCoalescing } from "zarrita";
|
|
61
68
|
|
|
62
69
|
// Open from a URL (default: branch "main")
|
|
63
70
|
const store = await IcechunkStore.open("https://example.com/repo", {
|
|
@@ -66,6 +73,10 @@ const store = await IcechunkStore.open("https://example.com/repo", {
|
|
|
66
73
|
// snapshot: 'ABC123...',
|
|
67
74
|
// formatVersion: 'v1', // skip format auto-detection for v1 repos
|
|
68
75
|
// maxManifestCacheSize: 50, // LRU cache size (default: 100)
|
|
76
|
+
// withRangeCoalescing, // opt into merged range reads
|
|
77
|
+
// signal: abortController.signal, // cancel initialization
|
|
78
|
+
// validateChecksums: true, // integrity headers for virtual chunks
|
|
79
|
+
// azureAccount: 'myaccount', // required for az:// virtual chunks
|
|
69
80
|
});
|
|
70
81
|
|
|
71
82
|
// Open from an existing ReadSession
|
|
@@ -103,7 +114,7 @@ const fetchClient: FetchClient = {
|
|
|
103
114
|
async fetch(url, init) {
|
|
104
115
|
// URL rewriting, pre-signing, or header injection happens here.
|
|
105
116
|
// icechunk-js has already translated s3:// → https:// and built
|
|
106
|
-
// Range
|
|
117
|
+
// Range headers in `init` (plus If-Match when validateChecksums is on).
|
|
107
118
|
const signedUrl = await presign(url);
|
|
108
119
|
return globalThis.fetch(signedUrl, {
|
|
109
120
|
...init,
|
|
@@ -120,17 +131,27 @@ const store = await IcechunkStore.open("https://example.com/repo", {
|
|
|
120
131
|
Cloud storage URLs in virtual chunk references are automatically translated:
|
|
121
132
|
|
|
122
133
|
- `s3://bucket/key` → `https://bucket.s3.amazonaws.com/key`
|
|
123
|
-
- `gs://bucket/key` → `https://storage.googleapis.com/bucket/key`
|
|
124
|
-
- `az://
|
|
125
|
-
|
|
134
|
+
- `gs://bucket/key` (or `gcs://`) → `https://storage.googleapis.com/bucket/key`
|
|
135
|
+
- `az://container/path` (or `azure://`) →
|
|
136
|
+
`https://{azureAccount}.blob.core.windows.net/container/path`
|
|
137
|
+
- `abfs://container@account.dfs.core.windows.net/path` →
|
|
138
|
+
`https://account.blob.core.windows.net/container/path`
|
|
126
139
|
|
|
127
140
|
### Repository
|
|
128
141
|
|
|
129
142
|
For direct access to branches, tags, and checkouts.
|
|
130
143
|
|
|
144
|
+
> **Note:** Over plain HTTP, `listBranches()` and `listTags()` only work
|
|
145
|
+
> reliably with v2 repos, which embed refs in the top-level `repo` file. For v1
|
|
146
|
+
> repos, direct `checkoutBranch()` / `checkoutTag()` can work when the target
|
|
147
|
+
> ref still lives at the legacy `ref.json` path, but versioned ref filenames
|
|
148
|
+
> still require `listPrefix()` discovery, which `HttpStorage` does not provide.
|
|
149
|
+
> Use a listing-capable storage backend for full v1 branch/tag support.
|
|
150
|
+
|
|
131
151
|
```typescript
|
|
132
152
|
import { Repository, HttpStorage } from "icechunk-js";
|
|
133
153
|
|
|
154
|
+
// Replace with the root URL of a real Icechunk repository.
|
|
134
155
|
const storage = new HttpStorage("https://example.com/repo");
|
|
135
156
|
|
|
136
157
|
// Auto-detect format (default)
|
|
@@ -139,7 +160,7 @@ const repo = await Repository.open({ storage });
|
|
|
139
160
|
// Or with format version hint (skips /repo request for v1 stores)
|
|
140
161
|
// const repo = await Repository.open({ storage, formatVersion: 'v1' });
|
|
141
162
|
|
|
142
|
-
// List branches and tags
|
|
163
|
+
// List branches and tags (v2 repos, or storage backends that support listing)
|
|
143
164
|
const branches = await repo.listBranches();
|
|
144
165
|
const tags = await repo.listTags();
|
|
145
166
|
|
|
@@ -153,7 +174,7 @@ const session = await repo.checkoutBranch("main");
|
|
|
153
174
|
|
|
154
175
|
```typescript
|
|
155
176
|
for await (const entry of repo.walkHistory(session)) {
|
|
156
|
-
console.log(entry.id, entry.message, entry.flushedAt);
|
|
177
|
+
console.log(entry.id, entry.message, entry.flushedAt, entry.metadata);
|
|
157
178
|
}
|
|
158
179
|
```
|
|
159
180
|
|