icechunk-js 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,4 +1,4 @@
1
- # Icechunk JavaScript
1
+ # icechunk-js
2
2
 
3
3
  [![npm version](https://img.shields.io/npm/v/icechunk-js.svg)](https://www.npmjs.com/package/icechunk-js)
4
4
  [![Icechunk](https://img.shields.io/badge/Icechunk-008B8B)](https://github.com/earth-mover/icechunk)
@@ -8,8 +8,7 @@
8
8
  Read-only JavaScript/TypeScript reader for Icechunk repositories, designed for
9
9
  use with [zarrita](https://github.com/manzt/zarrita.js).
10
10
 
11
- - ~50KB bundle, zero native dependencies
12
- - Works in browsers, Node.js 18+, Deno, and Bun
11
+ - Pure TypeScript, works in browsers and Node.js 18+
13
12
  - Icechunk v1 and v2 format auto-detection
14
13
  - All chunk payload types: inline, native, and virtual
15
14
 
@@ -29,7 +28,7 @@ import { open, get } from "zarrita";
29
28
  const store = await IcechunkStore.open("https://bucket.s3.amazonaws.com/repo");
30
29
 
31
30
  // Open an array and read data
32
- const array = await open(store, { kind: "array", path: "/temperature" });
31
+ const array = await open(store.resolve("/temperature"), { kind: "array" });
33
32
  const data = await get(array, [0, 0, null]);
34
33
  ```
35
34
 
@@ -45,19 +44,27 @@ npm run test
45
44
  npm run typecheck
46
45
  ```
47
46
 
48
- To regenerate FlatBuffers TypeScript after syncing schemas, run `npm run generate:fbs`.
49
- The pinned `flatc` compiler is downloaded automatically if not already available.
50
- The version is set in `scripts/ensure-flatc.sh`.
47
+ To regenerate FlatBuffers TypeScript after syncing schemas, run
48
+ `npm run generate:fbs`. The pinned `flatc` compiler is downloaded automatically
49
+ if not already available. The version is set in `scripts/ensure-flatc.sh`.
51
50
 
52
51
  ## API
53
52
 
54
53
  ### IcechunkStore
55
54
 
56
55
  The main class for zarrita integration. Implements zarrita's `AsyncReadable`
57
- interface with both `get()` and `getRange()` (needed for sharded arrays).
56
+ interface with both `get()` and `getRange()` (needed for sharded arrays). Pass
57
+ zarrita's `withRangeCoalescing` to coalesce concurrent reads against the same
58
+ backing object. This requires zarrita >= 0.7.
59
+
60
+ > **Note:** Range coalescing uses zarrita's merged abort-signal behavior. If one
61
+ > read in a merged batch is aborted, other reads in the same batch may also
62
+ > reject. Avoid sharing an `AbortController` across requests that must cancel
63
+ > independently.
58
64
 
59
65
  ```typescript
60
66
  import { IcechunkStore } from "icechunk-js";
67
+ import { withRangeCoalescing } from "zarrita";
61
68
 
62
69
  // Open from a URL (default: branch "main")
63
70
  const store = await IcechunkStore.open("https://example.com/repo", {
@@ -66,6 +73,10 @@ const store = await IcechunkStore.open("https://example.com/repo", {
66
73
  // snapshot: 'ABC123...',
67
74
  // formatVersion: 'v1', // skip format auto-detection for v1 repos
68
75
  // maxManifestCacheSize: 50, // LRU cache size (default: 100)
76
+ // withRangeCoalescing, // opt into merged range reads
77
+ // signal: abortController.signal, // cancel initialization
78
+ // validateChecksums: true, // integrity headers for virtual chunks
79
+ // azureAccount: 'myaccount', // required for az:// virtual chunks
69
80
  });
70
81
 
71
82
  // Open from an existing ReadSession
@@ -103,7 +114,7 @@ const fetchClient: FetchClient = {
103
114
  async fetch(url, init) {
104
115
  // URL rewriting, pre-signing, or header injection happens here.
105
116
  // icechunk-js has already translated s3:// → https:// and built
106
- // Range / If-None-Match headers in `init`.
117
+ // Range headers in `init` (plus If-Match when validateChecksums is on).
107
118
  const signedUrl = await presign(url);
108
119
  return globalThis.fetch(signedUrl, {
109
120
  ...init,
@@ -120,17 +131,27 @@ const store = await IcechunkStore.open("https://example.com/repo", {
120
131
  Cloud storage URLs in virtual chunk references are automatically translated:
121
132
 
122
133
  - `s3://bucket/key` → `https://bucket.s3.amazonaws.com/key`
123
- - `gs://bucket/key` → `https://storage.googleapis.com/bucket/key`
124
- - `az://account/container/path` → `https://account.blob.core.windows.net/container/path`
125
- - `abfs://container@account.dfs.core.windows.net/path` → `https://account.blob.core.windows.net/container/path`
134
+ - `gs://bucket/key` (or `gcs://`) → `https://storage.googleapis.com/bucket/key`
135
+ - `az://container/path` (or `azure://`) →
136
+ `https://{azureAccount}.blob.core.windows.net/container/path`
137
+ - `abfs://container@account.dfs.core.windows.net/path` →
138
+ `https://account.blob.core.windows.net/container/path`
126
139
 
127
140
  ### Repository
128
141
 
129
142
  For direct access to branches, tags, and checkouts.
130
143
 
144
+ > **Note:** Over plain HTTP, `listBranches()` and `listTags()` only work
145
+ > reliably with v2 repos, which embed refs in the top-level `repo` file. For v1
146
+ > repos, direct `checkoutBranch()` / `checkoutTag()` can work when the target
147
+ > ref still lives at the legacy `ref.json` path, but versioned ref filenames
148
+ > still require `listPrefix()` discovery, which `HttpStorage` does not provide.
149
+ > Use a listing-capable storage backend for full v1 branch/tag support.
150
+
131
151
  ```typescript
132
152
  import { Repository, HttpStorage } from "icechunk-js";
133
153
 
154
+ // Replace with the root URL of a real Icechunk repository.
134
155
  const storage = new HttpStorage("https://example.com/repo");
135
156
 
136
157
  // Auto-detect format (default)
@@ -139,7 +160,7 @@ const repo = await Repository.open({ storage });
139
160
  // Or with format version hint (skips /repo request for v1 stores)
140
161
  // const repo = await Repository.open({ storage, formatVersion: 'v1' });
141
162
 
142
- // List branches and tags
163
+ // List branches and tags (v2 repos, or storage backends that support listing)
143
164
  const branches = await repo.listBranches();
144
165
  const tags = await repo.listTags();
145
166
 
@@ -153,7 +174,7 @@ const session = await repo.checkoutBranch("main");
153
174
 
154
175
  ```typescript
155
176
  for await (const entry of repo.walkHistory(session)) {
156
- console.log(entry.id, entry.message, entry.flushedAt);
177
+ console.log(entry.id, entry.message, entry.flushedAt, entry.metadata);
157
178
  }
158
179
  ```
159
180