icechunk-js 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +211 -0
- package/dist/index.cjs +3057 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +872 -0
- package/dist/index.d.ts +872 -0
- package/dist/index.js +3003 -0
- package/dist/index.js.map +1 -0
- package/package.json +68 -0
package/README.md
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
# Icechunk JavaScript
|
|
2
|
+
|
|
3
|
+
Read-only JavaScript/TypeScript reader for Icechunk repositories, designed for
|
|
4
|
+
use with [zarrita](https://github.com/manzt/zarrita.js).
|
|
5
|
+
|
|
6
|
+
- ~50KB bundle, zero native dependencies
|
|
7
|
+
- Works in browsers, Node.js 18+, Deno, and Bun
|
|
8
|
+
- Icechunk v1 and v2 format auto-detection
|
|
9
|
+
- All chunk payload types: inline, native, and virtual
|
|
10
|
+
|
|
11
|
+
## Getting Started
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
npm install icechunk-js
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
### Basic Usage with zarrita
|
|
18
|
+
|
|
19
|
+
```typescript
|
|
20
|
+
import { IcechunkStore } from "icechunk-js";
|
|
21
|
+
import { open, get } from "zarrita";
|
|
22
|
+
|
|
23
|
+
// Open a store from a URL
|
|
24
|
+
const store = await IcechunkStore.open("https://bucket.s3.amazonaws.com/repo");
|
|
25
|
+
|
|
26
|
+
// Open an array and read data
|
|
27
|
+
const array = await open(store, { kind: "array", path: "/temperature" });
|
|
28
|
+
const data = await get(array, [0, 0, null]);
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
### For Development
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
npm install
|
|
35
|
+
|
|
36
|
+
npm run dev
|
|
37
|
+
|
|
38
|
+
npm run test
|
|
39
|
+
|
|
40
|
+
npm run typecheck
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## API
|
|
44
|
+
|
|
45
|
+
### IcechunkStore
|
|
46
|
+
|
|
47
|
+
The main class for zarrita integration. Implements zarrita's `AsyncReadable`
|
|
48
|
+
interface with both `get()` and `getRange()` (needed for sharded arrays).
|
|
49
|
+
|
|
50
|
+
```typescript
|
|
51
|
+
import { IcechunkStore } from "icechunk-js";
|
|
52
|
+
|
|
53
|
+
// Open from a URL (default: branch "main")
|
|
54
|
+
const store = await IcechunkStore.open("https://example.com/repo", {
|
|
55
|
+
branch: "main",
|
|
56
|
+
// tag: 'v1.0',
|
|
57
|
+
// snapshot: 'ABC123...',
|
|
58
|
+
// formatVersion: 'v1', // skip format auto-detection for v1 repos
|
|
59
|
+
// maxManifestCacheSize: 50, // LRU cache size (default: 100)
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
// Open from an existing ReadSession
|
|
63
|
+
const store = await IcechunkStore.open(session);
|
|
64
|
+
|
|
65
|
+
// Open from a custom Storage backend
|
|
66
|
+
const store = await IcechunkStore.open(myStorage, { branch: "main" });
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
#### Store methods
|
|
70
|
+
|
|
71
|
+
```typescript
|
|
72
|
+
// Scope to a subpath (shares the same session and cache)
|
|
73
|
+
const scoped = store.resolve("group/subgroup");
|
|
74
|
+
|
|
75
|
+
// Browse the hierarchy
|
|
76
|
+
const children = store.listChildren("/"); // direct children of root
|
|
77
|
+
const allNodes = store.listNodes(); // all nodes in the snapshot
|
|
78
|
+
const node = store.getNode("/temperature"); // single node by path
|
|
79
|
+
const meta = store.getMetadata("/temperature"); // parsed zarr.json
|
|
80
|
+
|
|
81
|
+
// Access the underlying session for advanced operations
|
|
82
|
+
const session = store.session;
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
#### Virtual chunk authentication
|
|
86
|
+
|
|
87
|
+
For private datasets with virtual chunks (S3, GCS, Azure), provide a
|
|
88
|
+
`fetchClient` that handles authentication:
|
|
89
|
+
|
|
90
|
+
```typescript
|
|
91
|
+
import type { FetchClient } from "icechunk-js";
|
|
92
|
+
|
|
93
|
+
const fetchClient: FetchClient = {
|
|
94
|
+
async fetch(url, init) {
|
|
95
|
+
// URL rewriting, pre-signing, or header injection happens here.
|
|
96
|
+
// icechunk-js has already translated s3:// → https:// and built
|
|
97
|
+
// Range / If-None-Match headers in `init`.
|
|
98
|
+
const signedUrl = await presign(url);
|
|
99
|
+
return globalThis.fetch(signedUrl, {
|
|
100
|
+
...init,
|
|
101
|
+
headers: { ...init?.headers, Authorization: `Bearer ${token}` },
|
|
102
|
+
});
|
|
103
|
+
},
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
const store = await IcechunkStore.open("https://example.com/repo", {
|
|
107
|
+
fetchClient,
|
|
108
|
+
});
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Cloud storage URLs in virtual chunk references are automatically translated:
|
|
112
|
+
|
|
113
|
+
- `s3://bucket/key` → `https://bucket.s3.amazonaws.com/key`
|
|
114
|
+
- `gs://bucket/key` → `https://storage.googleapis.com/bucket/key`
|
|
115
|
+
- `az://account/container/path` → `https://account.blob.core.windows.net/container/path`
|
|
116
|
+
- `abfs://container@account.dfs.core.windows.net/path` → `https://account.blob.core.windows.net/container/path`
|
|
117
|
+
|
|
118
|
+
### Repository
|
|
119
|
+
|
|
120
|
+
For direct access to branches, tags, and checkouts.
|
|
121
|
+
|
|
122
|
+
```typescript
|
|
123
|
+
import { Repository, HttpStorage } from "icechunk-js";
|
|
124
|
+
|
|
125
|
+
const storage = new HttpStorage("https://example.com/repo");
|
|
126
|
+
|
|
127
|
+
// Auto-detect format (default)
|
|
128
|
+
const repo = await Repository.open({ storage });
|
|
129
|
+
|
|
130
|
+
// Or with format version hint (skips /repo request for v1 stores)
|
|
131
|
+
// const repo = await Repository.open({ storage, formatVersion: 'v1' });
|
|
132
|
+
|
|
133
|
+
// List branches and tags
|
|
134
|
+
const branches = await repo.listBranches();
|
|
135
|
+
const tags = await repo.listTags();
|
|
136
|
+
|
|
137
|
+
// Checkout to get a ReadSession
|
|
138
|
+
const session = await repo.checkoutBranch("main");
|
|
139
|
+
// or: repo.checkoutTag('v1.0')
|
|
140
|
+
// or: repo.checkoutSnapshot('ABCDEFGHIJKLMNOP')
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
#### Walking commit history
|
|
144
|
+
|
|
145
|
+
```typescript
|
|
146
|
+
for await (const entry of repo.walkHistory(session)) {
|
|
147
|
+
console.log(entry.id, entry.message, entry.flushedAt);
|
|
148
|
+
}
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
### ReadSession
|
|
152
|
+
|
|
153
|
+
Low-level access to nodes, chunks, and snapshot metadata.
|
|
154
|
+
|
|
155
|
+
```typescript
|
|
156
|
+
// Snapshot info
|
|
157
|
+
const snapshotId = session.getSnapshotId();
|
|
158
|
+
const parentId = session.getParentSnapshotId(); // null for root
|
|
159
|
+
const message = session.getMessage();
|
|
160
|
+
const timestamp = session.getFlushedAt();
|
|
161
|
+
const metadata = session.getSnapshotMetadata();
|
|
162
|
+
|
|
163
|
+
// Navigate the hierarchy
|
|
164
|
+
const nodes = session.listNodes();
|
|
165
|
+
const children = session.listChildren("/group");
|
|
166
|
+
const node = session.getNode("/array");
|
|
167
|
+
|
|
168
|
+
// Get Zarr metadata and chunks
|
|
169
|
+
const zarrMeta = session.getMetadata("/array");
|
|
170
|
+
const chunk = await session.getChunk("/array", [0, 0, 0]);
|
|
171
|
+
|
|
172
|
+
// Transaction log (what changed in this snapshot)
|
|
173
|
+
const txLog = await session.loadTransactionLog();
|
|
174
|
+
if (txLog) {
|
|
175
|
+
console.log("New arrays:", txLog.newArrays.length);
|
|
176
|
+
console.log("Updated chunks:", txLog.updatedChunks.length);
|
|
177
|
+
}
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
### HttpStorage
|
|
181
|
+
|
|
182
|
+
HTTP/HTTPS storage backend using the Fetch API. Works in Node.js 18+ and
|
|
183
|
+
browsers.
|
|
184
|
+
|
|
185
|
+
```typescript
|
|
186
|
+
import { HttpStorage } from "icechunk-js";
|
|
187
|
+
|
|
188
|
+
const storage = new HttpStorage("https://bucket.s3.amazonaws.com/repo", {
|
|
189
|
+
headers: { Authorization: "Bearer token" },
|
|
190
|
+
credentials: "include",
|
|
191
|
+
cache: "no-store",
|
|
192
|
+
});
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
### Custom Storage
|
|
196
|
+
|
|
197
|
+
Implement the `Storage` interface for other backends:
|
|
198
|
+
|
|
199
|
+
```typescript
|
|
200
|
+
import type { Storage, ByteRange, RequestOptions } from 'icechunk-js';
|
|
201
|
+
|
|
202
|
+
class MyStorage implements Storage {
|
|
203
|
+
async getObject(path: string, range?: ByteRange, options?: RequestOptions): Promise<Uint8Array> { ... }
|
|
204
|
+
async exists(path: string, options?: RequestOptions): Promise<boolean> { ... }
|
|
205
|
+
async *listPrefix(prefix: string): AsyncIterable<string> { ... }
|
|
206
|
+
}
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
## License
|
|
210
|
+
|
|
211
|
+
MIT
|