icechunk-js 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +211 -0
- package/dist/index.cjs +3057 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +872 -0
- package/dist/index.d.ts +872 -0
- package/dist/index.js +3003 -0
- package/dist/index.js.map +1 -0
- package/package.json +68 -0
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,872 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Storage interface for icechunk repositories.
|
|
3
|
+
*/
|
|
4
|
+
/** Byte range for partial reads */
|
|
5
|
+
interface ByteRange {
|
|
6
|
+
/** Start offset (inclusive) */
|
|
7
|
+
start: number;
|
|
8
|
+
/** End offset (exclusive) */
|
|
9
|
+
end: number;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Pluggable HTTP client for virtual chunk fetching.
|
|
13
|
+
*
|
|
14
|
+
* Use this to:
|
|
15
|
+
* - Generate pre-signed S3 URLs
|
|
16
|
+
* - Add authentication headers
|
|
17
|
+
* - Route through a proxy
|
|
18
|
+
*
|
|
19
|
+
* icechunk-js handles URL translation (s3:// → https://) and builds
|
|
20
|
+
* all headers (Range, If-Match, etc.) before calling fetch().
|
|
21
|
+
* The client only needs to execute the HTTP request.
|
|
22
|
+
*/
|
|
23
|
+
interface FetchClient {
|
|
24
|
+
fetch(url: string, init?: RequestInit): Promise<Response>;
|
|
25
|
+
}
|
|
26
|
+
/** Default FetchClient that delegates to globalThis.fetch. */
|
|
27
|
+
declare class DefaultFetchClient implements FetchClient {
|
|
28
|
+
fetch(url: string, init?: RequestInit): Promise<Response>;
|
|
29
|
+
}
|
|
30
|
+
/** Options for storage request operations */
|
|
31
|
+
interface RequestOptions {
|
|
32
|
+
/** AbortSignal for request cancellation */
|
|
33
|
+
signal?: AbortSignal;
|
|
34
|
+
/** Pluggable HTTP client for virtual chunk fetching */
|
|
35
|
+
fetchClient?: FetchClient;
|
|
36
|
+
/**
|
|
37
|
+
* Send If-Match / If-Unmodified-Since headers on virtual chunk requests.
|
|
38
|
+
*
|
|
39
|
+
* When true, the storage server will return 412 Precondition Failed if
|
|
40
|
+
* the underlying file has changed since the snapshot recorded its checksum.
|
|
41
|
+
*
|
|
42
|
+
* Defaults to false because these headers trigger CORS preflight requests
|
|
43
|
+
* in browsers, and most storage servers don't whitelist them by default.
|
|
44
|
+
*/
|
|
45
|
+
validateChecksums?: boolean;
|
|
46
|
+
/**
|
|
47
|
+
* Azure storage account name for translating az:// and azure:// URLs.
|
|
48
|
+
*
|
|
49
|
+
* Required when virtual chunks reference az:// or azure:// URLs, since
|
|
50
|
+
* these schemes encode only the container name (e.g., az://container/path).
|
|
51
|
+
* The account is needed to build the HTTPS endpoint:
|
|
52
|
+
* https://{account}.blob.core.windows.net/{container}/{path}
|
|
53
|
+
*
|
|
54
|
+
* Not needed for abfs:// URLs, which embed the account in the host.
|
|
55
|
+
*/
|
|
56
|
+
azureAccount?: string;
|
|
57
|
+
}
|
|
58
|
+
/** Error thrown when an object is not found */
|
|
59
|
+
declare class NotFoundError extends Error {
|
|
60
|
+
readonly path: string;
|
|
61
|
+
constructor(path: string);
|
|
62
|
+
}
|
|
63
|
+
/** Error thrown for storage operations */
|
|
64
|
+
declare class StorageError extends Error {
|
|
65
|
+
readonly cause?: Error | undefined;
|
|
66
|
+
constructor(message: string, cause?: Error | undefined);
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Storage interface for reading icechunk data.
|
|
70
|
+
*
|
|
71
|
+
* Implementations should handle authentication, caching, etc.
|
|
72
|
+
*/
|
|
73
|
+
interface Storage {
|
|
74
|
+
/**
|
|
75
|
+
* Get an object from storage.
|
|
76
|
+
*
|
|
77
|
+
* @param path - Path to the object (relative to repository root)
|
|
78
|
+
* @param range - Optional byte range for partial reads
|
|
79
|
+
* @param options - Optional request options (signal for cancellation)
|
|
80
|
+
* @returns Object data as bytes
|
|
81
|
+
* @throws NotFoundError if the object doesn't exist
|
|
82
|
+
* @throws StorageError for other errors
|
|
83
|
+
* @throws AbortError if the operation was aborted
|
|
84
|
+
*/
|
|
85
|
+
getObject(path: string, range?: ByteRange, options?: RequestOptions): Promise<Uint8Array>;
|
|
86
|
+
/**
|
|
87
|
+
* Check if an object exists.
|
|
88
|
+
*
|
|
89
|
+
* @param path - Path to the object
|
|
90
|
+
* @param options - Optional request options (signal for cancellation)
|
|
91
|
+
* @returns True if the object exists
|
|
92
|
+
* @throws AbortError if the operation was aborted
|
|
93
|
+
*/
|
|
94
|
+
exists(path: string, options?: RequestOptions): Promise<boolean>;
|
|
95
|
+
/**
|
|
96
|
+
* List objects with a given prefix.
|
|
97
|
+
*
|
|
98
|
+
* @param prefix - Path prefix to filter by
|
|
99
|
+
* @returns Async iterable of object paths
|
|
100
|
+
*/
|
|
101
|
+
listPrefix(prefix: string): AsyncIterable<string>;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** Spec version enum */
|
|
105
|
+
declare enum SpecVersion {
|
|
106
|
+
V1_0 = 1,
|
|
107
|
+
V2_0 = 2
|
|
108
|
+
}
|
|
109
|
+
/** File type enum */
|
|
110
|
+
declare enum FileType {
|
|
111
|
+
Snapshot = 1,
|
|
112
|
+
Manifest = 2,
|
|
113
|
+
Attributes = 3,
|
|
114
|
+
TransactionLog = 4,
|
|
115
|
+
Chunk = 5,
|
|
116
|
+
RepoInfo = 6
|
|
117
|
+
}
|
|
118
|
+
/** Compression algorithm enum */
|
|
119
|
+
declare enum CompressionAlgorithm {
|
|
120
|
+
None = 0,
|
|
121
|
+
Zstd = 1
|
|
122
|
+
}
|
|
123
|
+
/** Error thrown when header parsing fails */
|
|
124
|
+
declare class HeaderParseError extends Error {
|
|
125
|
+
constructor(message: string);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* TypeScript types matching the icechunk FlatBuffer schemas.
|
|
130
|
+
*
|
|
131
|
+
* These types represent the parsed/decoded data, not the binary format.
|
|
132
|
+
*/
|
|
133
|
+
/** 12-byte object ID (SnapshotId, ManifestId, ChunkId) */
|
|
134
|
+
type ObjectId12 = Uint8Array;
|
|
135
|
+
/** 8-byte object ID (NodeId) */
|
|
136
|
+
type ObjectId8 = Uint8Array;
|
|
137
|
+
/** Metadata key-value pair */
|
|
138
|
+
interface MetadataItem {
|
|
139
|
+
name: string;
|
|
140
|
+
/** Value serialized as MessagePack (v1) or FlexBuffers (v2) */
|
|
141
|
+
value: Uint8Array;
|
|
142
|
+
}
|
|
143
|
+
/** Reference to a chunk - can be inline, native, or virtual */
|
|
144
|
+
interface ChunkRef {
|
|
145
|
+
/** Chunk coordinates (N-dimensional) */
|
|
146
|
+
index: number[];
|
|
147
|
+
/** Inline chunk data (if present, this is an inline chunk) */
|
|
148
|
+
inline: Uint8Array | null;
|
|
149
|
+
/** Byte offset within the chunk file or virtual location */
|
|
150
|
+
offset: number;
|
|
151
|
+
/** Byte length of the chunk data */
|
|
152
|
+
length: number;
|
|
153
|
+
/** Chunk ID for native chunks (points to file in chunks/) */
|
|
154
|
+
chunkId: ObjectId12 | null;
|
|
155
|
+
/** URL for virtual chunks */
|
|
156
|
+
location: string | null;
|
|
157
|
+
/** ETag checksum for virtual chunks */
|
|
158
|
+
checksumEtag: string | null;
|
|
159
|
+
/** Last modified timestamp (seconds since epoch) for virtual chunks */
|
|
160
|
+
checksumLastModified: number;
|
|
161
|
+
}
|
|
162
|
+
/** Chunk references for a single array */
|
|
163
|
+
interface ArrayManifest {
|
|
164
|
+
/** Node ID of the array */
|
|
165
|
+
nodeId: ObjectId8;
|
|
166
|
+
/** Chunk references, sorted by index */
|
|
167
|
+
refs: ChunkRef[];
|
|
168
|
+
}
|
|
169
|
+
/** Manifest containing chunk references for multiple arrays */
|
|
170
|
+
interface Manifest {
|
|
171
|
+
/** Manifest ID */
|
|
172
|
+
id: ObjectId12;
|
|
173
|
+
/** Array manifests, sorted by nodeId */
|
|
174
|
+
arrays: ArrayManifest[];
|
|
175
|
+
}
|
|
176
|
+
/** Info about a manifest file */
|
|
177
|
+
interface ManifestFileInfo {
|
|
178
|
+
/** Manifest ID */
|
|
179
|
+
id: ObjectId12;
|
|
180
|
+
/** Size in bytes */
|
|
181
|
+
sizeBytes: number;
|
|
182
|
+
/** Number of chunk refs in the manifest */
|
|
183
|
+
numChunkRefs: number;
|
|
184
|
+
}
|
|
185
|
+
/** Range of chunk indices along one dimension */
|
|
186
|
+
interface ChunkIndexRange {
|
|
187
|
+
/** Inclusive start */
|
|
188
|
+
from: number;
|
|
189
|
+
/** Exclusive end */
|
|
190
|
+
to: number;
|
|
191
|
+
}
|
|
192
|
+
/** Reference to a manifest with extent information */
|
|
193
|
+
interface ManifestRef {
|
|
194
|
+
/** Manifest object ID */
|
|
195
|
+
objectId: ObjectId12;
|
|
196
|
+
/** Chunk index ranges per dimension */
|
|
197
|
+
extents: ChunkIndexRange[];
|
|
198
|
+
}
|
|
199
|
+
/** Shape of one dimension */
|
|
200
|
+
interface DimensionShape {
|
|
201
|
+
/** Length of the array along this dimension */
|
|
202
|
+
arrayLength: number;
|
|
203
|
+
/** Chunk size along this dimension */
|
|
204
|
+
chunkLength: number;
|
|
205
|
+
}
|
|
206
|
+
/** Group node data (empty - just a marker) */
|
|
207
|
+
interface GroupNodeData {
|
|
208
|
+
type: "group";
|
|
209
|
+
}
|
|
210
|
+
/** Array node data */
|
|
211
|
+
interface ArrayNodeData {
|
|
212
|
+
type: "array";
|
|
213
|
+
/** Shape per dimension */
|
|
214
|
+
shape: DimensionShape[];
|
|
215
|
+
/** Optional dimension names */
|
|
216
|
+
dimensionNames: (string | null)[];
|
|
217
|
+
/** Manifest references */
|
|
218
|
+
manifests: ManifestRef[];
|
|
219
|
+
}
|
|
220
|
+
/** Node data - either group or array */
|
|
221
|
+
type NodeData = GroupNodeData | ArrayNodeData;
|
|
222
|
+
/** Snapshot of a single node (array or group) */
|
|
223
|
+
interface NodeSnapshot {
|
|
224
|
+
/** Node ID */
|
|
225
|
+
id: ObjectId8;
|
|
226
|
+
/** Absolute path in the repository */
|
|
227
|
+
path: string;
|
|
228
|
+
/** User data (typically Zarr metadata JSON) */
|
|
229
|
+
userData: Uint8Array;
|
|
230
|
+
/** Node-specific data */
|
|
231
|
+
nodeData: NodeData;
|
|
232
|
+
}
|
|
233
|
+
/** Complete snapshot of a repository state */
|
|
234
|
+
interface Snapshot {
|
|
235
|
+
/** Snapshot ID */
|
|
236
|
+
id: ObjectId12;
|
|
237
|
+
/** Parent snapshot ID (null for root snapshot) */
|
|
238
|
+
parentId: ObjectId12 | null;
|
|
239
|
+
/** Nodes in the snapshot, sorted by path */
|
|
240
|
+
nodes: NodeSnapshot[];
|
|
241
|
+
/** Timestamp when flushed (microseconds since epoch) */
|
|
242
|
+
flushedAt: bigint;
|
|
243
|
+
/** Commit message */
|
|
244
|
+
message: string;
|
|
245
|
+
/** Snapshot metadata */
|
|
246
|
+
metadata: MetadataItem[];
|
|
247
|
+
/** All manifest files referenced by this snapshot */
|
|
248
|
+
manifestFiles: ManifestFileInfo[];
|
|
249
|
+
}
|
|
250
|
+
/** Inline chunk - data embedded in manifest */
|
|
251
|
+
interface InlineChunkPayload {
|
|
252
|
+
type: "inline";
|
|
253
|
+
data: Uint8Array;
|
|
254
|
+
}
|
|
255
|
+
/** Native chunk - stored in repository's chunk storage */
|
|
256
|
+
interface NativeChunkPayload {
|
|
257
|
+
type: "native";
|
|
258
|
+
chunkId: ObjectId12;
|
|
259
|
+
offset: number;
|
|
260
|
+
length: number;
|
|
261
|
+
}
|
|
262
|
+
/** Virtual chunk - stored externally */
|
|
263
|
+
interface VirtualChunkPayload {
|
|
264
|
+
type: "virtual";
|
|
265
|
+
location: string;
|
|
266
|
+
offset: number;
|
|
267
|
+
length: number;
|
|
268
|
+
checksumEtag: string | null;
|
|
269
|
+
checksumLastModified: number;
|
|
270
|
+
}
|
|
271
|
+
/** Union of all chunk payload types */
|
|
272
|
+
type ChunkPayload = InlineChunkPayload | NativeChunkPayload | VirtualChunkPayload;
|
|
273
|
+
/** Chunk coordinates that were updated */
|
|
274
|
+
interface UpdatedChunkIndices {
|
|
275
|
+
coords: number[];
|
|
276
|
+
}
|
|
277
|
+
/** Updated chunks info for a single array */
|
|
278
|
+
interface ArrayUpdatedChunksInfo {
|
|
279
|
+
nodeId: ObjectId8;
|
|
280
|
+
chunks: UpdatedChunkIndices[];
|
|
281
|
+
}
|
|
282
|
+
/** A node move operation */
|
|
283
|
+
interface MoveOperationInfo {
|
|
284
|
+
from: string;
|
|
285
|
+
to: string;
|
|
286
|
+
}
|
|
287
|
+
/** A transaction log entry describing changes in a snapshot */
|
|
288
|
+
interface TransactionLogEntry {
|
|
289
|
+
id: ObjectId12;
|
|
290
|
+
newGroups: ObjectId8[];
|
|
291
|
+
newArrays: ObjectId8[];
|
|
292
|
+
deletedGroups: ObjectId8[];
|
|
293
|
+
deletedArrays: ObjectId8[];
|
|
294
|
+
updatedArrays: ObjectId8[];
|
|
295
|
+
updatedGroups: ObjectId8[];
|
|
296
|
+
updatedChunks: ArrayUpdatedChunksInfo[];
|
|
297
|
+
movedNodes: MoveOperationInfo[];
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* ReadSession - Read-only session for accessing icechunk data.
|
|
302
|
+
*/
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* ReadSession provides read access to a specific snapshot.
|
|
306
|
+
*
|
|
307
|
+
* Use this class to:
|
|
308
|
+
* - Get nodes (arrays/groups) by path
|
|
309
|
+
* - Read chunk data
|
|
310
|
+
* - Access Zarr metadata
|
|
311
|
+
*/
|
|
312
|
+
declare class ReadSession {
|
|
313
|
+
private storage;
|
|
314
|
+
private snapshot;
|
|
315
|
+
private specVersion;
|
|
316
|
+
private manifestCache;
|
|
317
|
+
private constructor();
|
|
318
|
+
/**
|
|
319
|
+
* Open a read session for a specific snapshot.
|
|
320
|
+
*
|
|
321
|
+
* @param storage - Storage backend
|
|
322
|
+
* @param snapshotId - Snapshot ID (12 bytes)
|
|
323
|
+
* @param options - Optional request options (signal for cancellation)
|
|
324
|
+
* @returns ReadSession instance
|
|
325
|
+
*/
|
|
326
|
+
static open(storage: Storage, snapshotId: Uint8Array, options?: RequestOptions & {
|
|
327
|
+
maxManifestCacheSize?: number;
|
|
328
|
+
}): Promise<ReadSession>;
|
|
329
|
+
/** Load and parse a snapshot from storage */
|
|
330
|
+
private static loadSnapshot;
|
|
331
|
+
/** Load and parse a manifest from storage */
|
|
332
|
+
private loadManifest;
|
|
333
|
+
/**
|
|
334
|
+
* Get the snapshot ID.
|
|
335
|
+
*/
|
|
336
|
+
getSnapshotId(): ObjectId12;
|
|
337
|
+
/**
|
|
338
|
+
* Get the spec version of the snapshot.
|
|
339
|
+
*/
|
|
340
|
+
getSpecVersion(): SpecVersion;
|
|
341
|
+
/**
|
|
342
|
+
* Get the parent snapshot ID, or null for root snapshots.
|
|
343
|
+
*/
|
|
344
|
+
getParentSnapshotId(): ObjectId12 | null;
|
|
345
|
+
/**
|
|
346
|
+
* Get the commit message for this snapshot.
|
|
347
|
+
*/
|
|
348
|
+
getMessage(): string;
|
|
349
|
+
/**
|
|
350
|
+
* Get the timestamp when this snapshot was created.
|
|
351
|
+
*/
|
|
352
|
+
getFlushedAt(): Date;
|
|
353
|
+
/**
|
|
354
|
+
* Get deserialized snapshot metadata.
|
|
355
|
+
*
|
|
356
|
+
* Decodes MessagePack (v1) or FlexBuffers (v2) metadata items
|
|
357
|
+
* into a plain key-value object.
|
|
358
|
+
*/
|
|
359
|
+
getSnapshotMetadata(): Record<string, unknown>;
|
|
360
|
+
/**
|
|
361
|
+
* Load and parse the transaction log for this snapshot.
|
|
362
|
+
*
|
|
363
|
+
* Returns null if no transaction log exists (e.g., root snapshot).
|
|
364
|
+
*
|
|
365
|
+
* @param options - Optional request options (signal for cancellation)
|
|
366
|
+
* @returns Parsed transaction log entry or null
|
|
367
|
+
*/
|
|
368
|
+
loadTransactionLog(options?: RequestOptions): Promise<TransactionLogEntry | null>;
|
|
369
|
+
/**
|
|
370
|
+
* Get a node by path.
|
|
371
|
+
*
|
|
372
|
+
* @param path - Absolute path (e.g., "/array" or "/group/nested")
|
|
373
|
+
* @returns NodeSnapshot or null if not found
|
|
374
|
+
*/
|
|
375
|
+
getNode(path: string): NodeSnapshot | null;
|
|
376
|
+
/**
|
|
377
|
+
* List all nodes in the snapshot.
|
|
378
|
+
*
|
|
379
|
+
* @returns Array of all nodes
|
|
380
|
+
*/
|
|
381
|
+
listNodes(): NodeSnapshot[];
|
|
382
|
+
/**
|
|
383
|
+
* List children of a group.
|
|
384
|
+
*
|
|
385
|
+
* @param parentPath - Path to the parent group (use "/" for root)
|
|
386
|
+
* @returns Array of child nodes
|
|
387
|
+
*/
|
|
388
|
+
listChildren(parentPath: string): NodeSnapshot[];
|
|
389
|
+
/**
|
|
390
|
+
* Get the Zarr metadata for a node.
|
|
391
|
+
*
|
|
392
|
+
* @param path - Path to the node
|
|
393
|
+
* @returns Parsed JSON metadata or null if node not found
|
|
394
|
+
*/
|
|
395
|
+
getMetadata(path: string): unknown | null;
|
|
396
|
+
/**
|
|
397
|
+
* Get raw user data (Zarr metadata bytes) for a node.
|
|
398
|
+
*
|
|
399
|
+
* @param path - Path to the node
|
|
400
|
+
* @returns Raw metadata bytes or null if node not found
|
|
401
|
+
*/
|
|
402
|
+
getRawMetadata(path: string): Uint8Array | null;
|
|
403
|
+
/**
|
|
404
|
+
* Read chunk data for an array.
|
|
405
|
+
*
|
|
406
|
+
* @param path - Path to the array
|
|
407
|
+
* @param coords - Chunk coordinates (N-dimensional)
|
|
408
|
+
* @param options - Optional request options (signal for cancellation)
|
|
409
|
+
* @returns Chunk data bytes or null if not found
|
|
410
|
+
*/
|
|
411
|
+
getChunk(path: string, coords: number[], options?: RequestOptions): Promise<Uint8Array | null>;
|
|
412
|
+
/**
|
|
413
|
+
* Read a byte range of chunk data for an array.
|
|
414
|
+
*
|
|
415
|
+
* Like getChunk, but fetches only the requested byte range from storage
|
|
416
|
+
* instead of the full chunk. Used by IcechunkStore.getRange to support
|
|
417
|
+
* zarrita's sharded array reads efficiently.
|
|
418
|
+
*
|
|
419
|
+
* @param path - Path to the array
|
|
420
|
+
* @param coords - Chunk coordinates (N-dimensional)
|
|
421
|
+
* @param range - Byte range within the chunk data
|
|
422
|
+
* @param options - Optional request options (signal for cancellation)
|
|
423
|
+
* @returns Chunk data bytes or null if not found
|
|
424
|
+
*/
|
|
425
|
+
getChunkRange(path: string, coords: number[], range: {
|
|
426
|
+
offset: number;
|
|
427
|
+
length: number;
|
|
428
|
+
} | {
|
|
429
|
+
suffixLength: number;
|
|
430
|
+
}, options?: RequestOptions): Promise<Uint8Array | null>;
|
|
431
|
+
/** Check if coordinates fall within extent ranges */
|
|
432
|
+
private coordsInExtents;
|
|
433
|
+
/** Fetch chunk data based on payload type */
|
|
434
|
+
private fetchChunkPayload;
|
|
435
|
+
/** Fetch a byte range of chunk data based on payload type */
|
|
436
|
+
private fetchChunkPayloadRange;
|
|
437
|
+
/** Binary search for a node by path */
|
|
438
|
+
private binarySearchNode;
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
/**
|
|
442
|
+
* IcechunkStore - zarrita-compatible store adapter.
|
|
443
|
+
*
|
|
444
|
+
* Implements zarrita's AsyncReadable interface to allow using
|
|
445
|
+
* zarrita for array operations on icechunk repositories.
|
|
446
|
+
*/
|
|
447
|
+
|
|
448
|
+
/**
|
|
449
|
+
* zarrita's AbsolutePath type - paths must start with "/"
|
|
450
|
+
*/
|
|
451
|
+
type AbsolutePath<Rest extends string = string> = `/${Rest}`;
|
|
452
|
+
/**
|
|
453
|
+
* zarrita's RangeQuery type for partial reads
|
|
454
|
+
*/
|
|
455
|
+
type RangeQuery = {
|
|
456
|
+
offset: number;
|
|
457
|
+
length: number;
|
|
458
|
+
} | {
|
|
459
|
+
suffixLength: number;
|
|
460
|
+
};
|
|
461
|
+
/**
|
|
462
|
+
* zarrita's AsyncReadable interface
|
|
463
|
+
*/
|
|
464
|
+
interface AsyncReadable<Options = unknown> {
|
|
465
|
+
get(key: AbsolutePath, opts?: Options): Promise<Uint8Array | undefined>;
|
|
466
|
+
getRange?(key: AbsolutePath, range: RangeQuery, opts?: Options): Promise<Uint8Array | undefined>;
|
|
467
|
+
}
|
|
468
|
+
/** Options for IcechunkStore */
|
|
469
|
+
interface IcechunkStoreOptions {
|
|
470
|
+
/** Branch name to checkout (default: "main") */
|
|
471
|
+
branch?: string;
|
|
472
|
+
/** Tag name to checkout (mutually exclusive with branch) */
|
|
473
|
+
tag?: string;
|
|
474
|
+
/** Specific snapshot ID to checkout (Base32 string) */
|
|
475
|
+
snapshot?: string;
|
|
476
|
+
/** AbortSignal for cancelling initialization */
|
|
477
|
+
signal?: AbortSignal;
|
|
478
|
+
/** Format version hint to skip auto-detection. 'v1' skips /repo request. */
|
|
479
|
+
formatVersion?: "v1" | "v2";
|
|
480
|
+
/**
|
|
481
|
+
* Pluggable HTTP client for virtual chunk fetching.
|
|
482
|
+
*
|
|
483
|
+
* Use this to:
|
|
484
|
+
* - Generate pre-signed S3 URLs
|
|
485
|
+
* - Add authentication headers
|
|
486
|
+
* - Route through a proxy
|
|
487
|
+
*/
|
|
488
|
+
fetchClient?: FetchClient;
|
|
489
|
+
/** Maximum number of manifests to cache in the LRU cache (default: 100) */
|
|
490
|
+
maxManifestCacheSize?: number;
|
|
491
|
+
/**
|
|
492
|
+
* Send If-Match / If-Unmodified-Since headers on virtual chunk requests.
|
|
493
|
+
*
|
|
494
|
+
* Defaults to false because these headers trigger CORS preflight in browsers.
|
|
495
|
+
*/
|
|
496
|
+
validateChecksums?: boolean;
|
|
497
|
+
/**
|
|
498
|
+
* Azure storage account name for translating az:// and azure:// URLs.
|
|
499
|
+
*
|
|
500
|
+
* Required when virtual chunks reference az:// or azure:// URLs.
|
|
501
|
+
* Not needed for abfs:// URLs which embed the account in the host.
|
|
502
|
+
*/
|
|
503
|
+
azureAccount?: string;
|
|
504
|
+
}
|
|
505
|
+
/**
|
|
506
|
+
* IcechunkStore - zarrita-compatible store for icechunk repositories.
|
|
507
|
+
*
|
|
508
|
+
* This store implements zarrita's AsyncReadable interface, allowing you
|
|
509
|
+
* to use zarrita's array operations on icechunk data.
|
|
510
|
+
*
|
|
511
|
+
* @example
|
|
512
|
+
* ```typescript
|
|
513
|
+
* import { IcechunkStore } from 'icechunk-js';
|
|
514
|
+
* import { open, get } from 'zarrita';
|
|
515
|
+
*
|
|
516
|
+
* const store = await IcechunkStore.open('https://bucket.s3.amazonaws.com/repo');
|
|
517
|
+
* const array = await open(store, { kind: 'array', path: '/temperature' });
|
|
518
|
+
* const data = await get(array, [0, 0, null]);
|
|
519
|
+
* ```
|
|
520
|
+
*/
|
|
521
|
+
declare class IcechunkStore implements AsyncReadable {
|
|
522
|
+
/** The underlying read session. Exposed for advanced usage. */
|
|
523
|
+
readonly session: ReadSession;
|
|
524
|
+
private fetchClient?;
|
|
525
|
+
private validateChecksums;
|
|
526
|
+
private azureAccount?;
|
|
527
|
+
private basePath;
|
|
528
|
+
private constructor();
|
|
529
|
+
/**
|
|
530
|
+
* Open an IcechunkStore from a URL.
|
|
531
|
+
*
|
|
532
|
+
* @param url - URL to the icechunk repository
|
|
533
|
+
* @param options - Store options (branch, tag, or snapshot to checkout)
|
|
534
|
+
*/
|
|
535
|
+
static open(url: string, options?: IcechunkStoreOptions): Promise<IcechunkStore>;
|
|
536
|
+
/**
|
|
537
|
+
* Open an IcechunkStore from an existing ReadSession.
|
|
538
|
+
*
|
|
539
|
+
* @param session - Existing ReadSession
|
|
540
|
+
* @param options - Store options (only fetchClient is used)
|
|
541
|
+
*/
|
|
542
|
+
static open(session: ReadSession, options?: Pick<IcechunkStoreOptions, "fetchClient">): Promise<IcechunkStore>;
|
|
543
|
+
/**
|
|
544
|
+
* Open an IcechunkStore from a custom Storage backend.
|
|
545
|
+
*
|
|
546
|
+
* @param storage - Custom Storage implementation
|
|
547
|
+
* @param options - Store options (branch, tag, or snapshot to checkout)
|
|
548
|
+
*/
|
|
549
|
+
static open(storage: Storage, options?: IcechunkStoreOptions): Promise<IcechunkStore>;
|
|
550
|
+
/**
|
|
551
|
+
* Get data for a zarr key.
|
|
552
|
+
*
|
|
553
|
+
* zarrita calls this method with keys like:
|
|
554
|
+
* - "/zarr.json" - root metadata
|
|
555
|
+
* - "/array/zarr.json" - array metadata
|
|
556
|
+
* - "/array/c/0/1/2" - chunk at coords [0, 1, 2]
|
|
557
|
+
*
|
|
558
|
+
* @param key - Absolute path (must start with "/")
|
|
559
|
+
* @param opts - Optional request options (supports AbortSignal for cancellation)
|
|
560
|
+
* @returns Data bytes or undefined if not found
|
|
561
|
+
*/
|
|
562
|
+
get(key: AbsolutePath, opts?: {
|
|
563
|
+
signal?: AbortSignal;
|
|
564
|
+
}): Promise<Uint8Array | undefined>;
|
|
565
|
+
/**
|
|
566
|
+
* Get partial data for a zarr key.
|
|
567
|
+
*
|
|
568
|
+
* Required by zarrita for sharded arrays. zarrita uses this to read the
|
|
569
|
+
* shard index (via suffixLength) and extract individual inner chunks
|
|
570
|
+
* (via offset/length) from shard data.
|
|
571
|
+
*
|
|
572
|
+
* @param key - Absolute path
|
|
573
|
+
* @param range - Byte range to fetch
|
|
574
|
+
* @param opts - Optional request options (supports AbortSignal for cancellation)
|
|
575
|
+
* @returns Data bytes or undefined if not found
|
|
576
|
+
*/
|
|
577
|
+
getRange(key: AbsolutePath, range: RangeQuery, opts?: {
|
|
578
|
+
signal?: AbortSignal;
|
|
579
|
+
}): Promise<Uint8Array | undefined>;
|
|
580
|
+
/** Prepend basePath to a parsed path. */
|
|
581
|
+
private resolvePath;
|
|
582
|
+
/**
|
|
583
|
+
* Create a store scoped to a subpath.
|
|
584
|
+
*
|
|
585
|
+
* The returned store shares the same session (and manifest cache)
|
|
586
|
+
* but prepends `path` to all key lookups. This matches zarrita's
|
|
587
|
+
* `root(store).resolve(path)` pattern.
|
|
588
|
+
*
|
|
589
|
+
* @param path - Subpath to scope to (e.g., "group/array")
|
|
590
|
+
* @returns A new IcechunkStore scoped to the subpath
|
|
591
|
+
*/
|
|
592
|
+
resolve(path: string): IcechunkStore;
|
|
593
|
+
/**
|
|
594
|
+
* List direct children of a group by name.
|
|
595
|
+
*
|
|
596
|
+
* @param parentPath - Path to the parent group (use "/" for root).
|
|
597
|
+
* When omitted, uses the store's base path (or root).
|
|
598
|
+
* @returns Array of child names (e.g., ["temperature", "precipitation"])
|
|
599
|
+
*/
|
|
600
|
+
listChildren(parentPath?: string): string[];
|
|
601
|
+
/**
|
|
602
|
+
* List all nodes in the snapshot.
|
|
603
|
+
*
|
|
604
|
+
* @returns Array of all nodes
|
|
605
|
+
*/
|
|
606
|
+
listNodes(): NodeSnapshot[];
|
|
607
|
+
/**
|
|
608
|
+
* Get a node by path.
|
|
609
|
+
*
|
|
610
|
+
* @param path - Absolute path (e.g., "/array" or "/group/nested")
|
|
611
|
+
* @returns NodeSnapshot or null if not found
|
|
612
|
+
*/
|
|
613
|
+
getNode(path: string): NodeSnapshot | null;
|
|
614
|
+
/**
|
|
615
|
+
* Get parsed Zarr metadata for a node.
|
|
616
|
+
*
|
|
617
|
+
* @param path - Path to the node
|
|
618
|
+
* @returns Parsed JSON metadata or null if node not found
|
|
619
|
+
*/
|
|
620
|
+
getMetadata(path: string): unknown | null;
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
/**
|
|
624
|
+
* Repository - Entry point for reading icechunk repositories.
|
|
625
|
+
*/
|
|
626
|
+
|
|
627
|
+
/** Reference data stored in ref.json files */
|
|
628
|
+
interface RefData {
|
|
629
|
+
/** Base32-encoded snapshot ID */
|
|
630
|
+
snapshot: string;
|
|
631
|
+
}
|
|
632
|
+
/** Options for opening a repository */
|
|
633
|
+
interface RepositoryOptions {
|
|
634
|
+
/** Storage backend to use */
|
|
635
|
+
storage: Storage;
|
|
636
|
+
/** Format version hint to skip auto-detection. 'v1' skips /repo request. */
|
|
637
|
+
formatVersion?: "v1" | "v2";
|
|
638
|
+
}
|
|
639
|
+
/**
|
|
640
|
+
* Repository provides access to an icechunk repository.
|
|
641
|
+
*
|
|
642
|
+
* Use this class to:
|
|
643
|
+
* - List branches and tags
|
|
644
|
+
* - Checkout a specific version to get a ReadSession
|
|
645
|
+
*/
|
|
646
|
+
declare class Repository {
|
|
647
|
+
private storage;
|
|
648
|
+
private repoInfo;
|
|
649
|
+
private repoInfoAttempted;
|
|
650
|
+
private constructor();
|
|
651
|
+
/**
|
|
652
|
+
* Load and cache the v2 repo info file.
|
|
653
|
+
*
|
|
654
|
+
* Uses getObject() directly to avoid race conditions with exists().
|
|
655
|
+
* - NotFoundError => v1 format (no repo file)
|
|
656
|
+
* - Any other error => hard error (parse failure, etc.)
|
|
657
|
+
*
|
|
658
|
+
* @param options - Optional request options (signal for cancellation)
|
|
659
|
+
* @returns ParsedRepo if v2 format, null if v1 format
|
|
660
|
+
* @throws Error if repo file exists but fails to parse
|
|
661
|
+
* @throws AbortError if the operation was aborted
|
|
662
|
+
*/
|
|
663
|
+
private loadRepoInfo;
|
|
664
|
+
/**
|
|
665
|
+
* Open an icechunk repository.
|
|
666
|
+
*
|
|
667
|
+
* - Try to load and parse repo info file (v2+ format)
|
|
668
|
+
* - If NotFoundError, check for main branch (v1 format)
|
|
669
|
+
* - Parse failures are hard errors (matching Rust behavior)
|
|
670
|
+
*
|
|
671
|
+
* @param options - Repository options including storage backend
|
|
672
|
+
* @param requestOptions - Optional request options (signal for cancellation)
|
|
673
|
+
* @returns A Repository instance
|
|
674
|
+
*/
|
|
675
|
+
static open(options: RepositoryOptions, requestOptions?: RequestOptions): Promise<Repository>;
|
|
676
|
+
/**
|
|
677
|
+
* Check if any ref file exists in a directory.
|
|
678
|
+
* Ref files are .json files that are not .deleted files.
|
|
679
|
+
*
|
|
680
|
+
* @param dirPrefix - Directory prefix to check
|
|
681
|
+
* @param legacyPath - Optional legacy ref.json path to check if listing fails
|
|
682
|
+
* @param options - Optional request options (signal for cancellation)
|
|
683
|
+
*/
|
|
684
|
+
private hasAnyRefFile;
|
|
685
|
+
/**
|
|
686
|
+
* Find the latest non-deleted ref file in a directory.
|
|
687
|
+
* When storage supports listPrefix(), refs may use versioned filenames
|
|
688
|
+
* (e.g., AAAAAAAA.json) for optimistic concurrency control. The latest
|
|
689
|
+
* version has the highest filename. Deletion tombstones are checked.
|
|
690
|
+
*
|
|
691
|
+
* When listing is not supported (e.g., HTTP storage), only the legacy
|
|
692
|
+
* ref.json path is checked (no tombstone check - assumes read-only).
|
|
693
|
+
*
|
|
694
|
+
* @param dirPrefix - Directory prefix to search
|
|
695
|
+
* @param legacyPath - Optional legacy ref.json path to try if listing fails
|
|
696
|
+
*/
|
|
697
|
+
private findLatestRefFile;
|
|
698
|
+
/**
|
|
699
|
+
* List all branches in the repository.
|
|
700
|
+
*
|
|
701
|
+
* Branches with deletion tombstones on their latest ref file are excluded.
|
|
702
|
+
*
|
|
703
|
+
* @returns Array of branch names
|
|
704
|
+
*/
|
|
705
|
+
listBranches(): Promise<string[]>;
|
|
706
|
+
/**
|
|
707
|
+
* List all tags in the repository.
|
|
708
|
+
*
|
|
709
|
+
* Tags with deletion tombstones on their latest ref file are excluded.
|
|
710
|
+
*
|
|
711
|
+
* @returns Array of tag names
|
|
712
|
+
*/
|
|
713
|
+
listTags(): Promise<string[]>;
|
|
714
|
+
/**
|
|
715
|
+
* Checkout a branch to get a read session.
|
|
716
|
+
*
|
|
717
|
+
* @param name - Branch name
|
|
718
|
+
* @param options - Optional request options (signal for cancellation)
|
|
719
|
+
* @returns Read session at the branch's current snapshot
|
|
720
|
+
*/
|
|
721
|
+
checkoutBranch(name: string, options?: RequestOptions): Promise<ReadSession>;
|
|
722
|
+
/**
|
|
723
|
+
* Checkout a tag to get a read session.
|
|
724
|
+
*
|
|
725
|
+
* @param name - Tag name
|
|
726
|
+
* @param options - Optional request options (signal for cancellation)
|
|
727
|
+
* @returns Read session at the tag's snapshot
|
|
728
|
+
*/
|
|
729
|
+
checkoutTag(name: string, options?: RequestOptions): Promise<ReadSession>;
|
|
730
|
+
/**
|
|
731
|
+
* Checkout a specific snapshot by ID.
|
|
732
|
+
*
|
|
733
|
+
* @param snapshotId - Snapshot ID (12 bytes or Base32 string)
|
|
734
|
+
* @param options - Optional request options (signal for cancellation)
|
|
735
|
+
* @returns Read session at the specified snapshot
|
|
736
|
+
*/
|
|
737
|
+
checkoutSnapshot(snapshotId: Uint8Array | string, options?: RequestOptions): Promise<ReadSession>;
|
|
738
|
+
/**
|
|
739
|
+
* Walk the snapshot history chain starting from the given session.
|
|
740
|
+
*
|
|
741
|
+
* Yields `{ id, message, flushedAt, metadata }` for each snapshot,
|
|
742
|
+
* walking from the current snapshot back to the root.
|
|
743
|
+
*
|
|
744
|
+
* @param session - Read session to start from
|
|
745
|
+
* @param options - Optional request options (signal for cancellation)
|
|
746
|
+
*/
|
|
747
|
+
walkHistory(session: ReadSession, options?: RequestOptions): AsyncGenerator<{
|
|
748
|
+
id: string;
|
|
749
|
+
message: string;
|
|
750
|
+
flushedAt: Date;
|
|
751
|
+
metadata: Record<string, unknown>;
|
|
752
|
+
}>;
|
|
753
|
+
/**
|
|
754
|
+
* Get the storage backend.
|
|
755
|
+
*/
|
|
756
|
+
getStorage(): Storage;
|
|
757
|
+
/** Read and parse a ref file */
|
|
758
|
+
private readRef;
|
|
759
|
+
/** Read snapshot ID from a ref file */
|
|
760
|
+
private readSnapshotIdFromRef;
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
/**
|
|
764
|
+
* HTTP/HTTPS storage backend using the Fetch API.
|
|
765
|
+
*
|
|
766
|
+
* Works in both Node.js 18+ and browsers.
|
|
767
|
+
*/
|
|
768
|
+
|
|
769
|
+
/** Options for HTTP storage */
|
|
770
|
+
interface HttpStorageOptions {
|
|
771
|
+
/** Additional headers to include in requests */
|
|
772
|
+
headers?: Record<string, string>;
|
|
773
|
+
/** Fetch credentials mode */
|
|
774
|
+
credentials?: RequestCredentials;
|
|
775
|
+
/** Fetch cache mode */
|
|
776
|
+
cache?: RequestCache;
|
|
777
|
+
}
|
|
778
|
+
/**
|
|
779
|
+
* HTTP/HTTPS storage backend.
|
|
780
|
+
*
|
|
781
|
+
* Reads objects from a base URL using HTTP GET requests.
|
|
782
|
+
* Supports byte range requests for partial reads.
|
|
783
|
+
*/
|
|
784
|
+
declare class HttpStorage implements Storage {
|
|
785
|
+
private baseUrl;
|
|
786
|
+
private options;
|
|
787
|
+
/**
|
|
788
|
+
* Create an HTTP storage backend.
|
|
789
|
+
*
|
|
790
|
+
* @param baseUrl - Base URL for the repository (e.g., "https://example.com/repo")
|
|
791
|
+
* @param options - Additional options
|
|
792
|
+
*/
|
|
793
|
+
constructor(baseUrl: string, options?: HttpStorageOptions);
|
|
794
|
+
/** Build full URL for a path */
|
|
795
|
+
private getUrl;
|
|
796
|
+
/** Build headers for a request */
|
|
797
|
+
private getHeaders;
|
|
798
|
+
getObject(path: string, range?: ByteRange, options?: RequestOptions): Promise<Uint8Array>;
|
|
799
|
+
exists(path: string, options?: RequestOptions): Promise<boolean>;
|
|
800
|
+
listPrefix(_prefix: string): AsyncIterable<string>;
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
/**
|
|
804
|
+
* Simple LRU cache for manifests and snapshots.
|
|
805
|
+
*/
|
|
806
|
+
declare class LRUCache<K, V> {
|
|
807
|
+
private cache;
|
|
808
|
+
private readonly maxSize;
|
|
809
|
+
constructor(maxSize: number);
|
|
810
|
+
get(key: K): V | undefined;
|
|
811
|
+
set(key: K, value: V): void;
|
|
812
|
+
has(key: K): boolean;
|
|
813
|
+
delete(key: K): boolean;
|
|
814
|
+
clear(): void;
|
|
815
|
+
get size(): number;
|
|
816
|
+
}
|
|
817
|
+
|
|
818
|
+
/**
|
|
819
|
+
* Object ID encoding/decoding using Base32 Crockford.
|
|
820
|
+
*
|
|
821
|
+
* Icechunk uses Base32 Crockford for encoding object IDs:
|
|
822
|
+
* - 12-byte IDs (SnapshotId, ManifestId, ChunkId) → 20 characters
|
|
823
|
+
* - 8-byte IDs (NodeId) → 13 characters
|
|
824
|
+
*
|
|
825
|
+
* Crockford Base32 alphabet: 0123456789ABCDEFGHJKMNPQRSTVWXYZ
|
|
826
|
+
* (excludes I, L, O, U to avoid confusion)
|
|
827
|
+
*/
|
|
828
|
+
/**
|
|
829
|
+
* Encode bytes to Base32 Crockford string.
|
|
830
|
+
*
|
|
831
|
+
* @param bytes - Bytes to encode
|
|
832
|
+
* @returns Base32 Crockford encoded string (uppercase)
|
|
833
|
+
*/
|
|
834
|
+
declare function encodeBase32(bytes: Uint8Array): string;
|
|
835
|
+
/**
|
|
836
|
+
* Decode Base32 Crockford string to bytes.
|
|
837
|
+
*
|
|
838
|
+
* @param str - Base32 Crockford encoded string
|
|
839
|
+
* @returns Decoded bytes
|
|
840
|
+
* @throws Error if the string contains invalid characters
|
|
841
|
+
*/
|
|
842
|
+
declare function decodeBase32(str: string): Uint8Array;
|
|
843
|
+
/**
|
|
844
|
+
* Encode a 12-byte object ID to string.
|
|
845
|
+
*
|
|
846
|
+
* @param id - 12-byte ID
|
|
847
|
+
* @returns Base32 Crockford encoded string (20 characters)
|
|
848
|
+
*/
|
|
849
|
+
declare function encodeObjectId12(id: Uint8Array): string;
|
|
850
|
+
/**
|
|
851
|
+
* Decode a string to 12-byte object ID.
|
|
852
|
+
*
|
|
853
|
+
* @param str - Base32 Crockford encoded string
|
|
854
|
+
* @returns 12-byte ID
|
|
855
|
+
*/
|
|
856
|
+
declare function decodeObjectId12(str: string): Uint8Array;
|
|
857
|
+
/**
|
|
858
|
+
* Encode an 8-byte object ID to string.
|
|
859
|
+
*
|
|
860
|
+
* @param id - 8-byte ID
|
|
861
|
+
* @returns Base32 Crockford encoded string (13 characters)
|
|
862
|
+
*/
|
|
863
|
+
declare function encodeObjectId8(id: Uint8Array): string;
|
|
864
|
+
/**
|
|
865
|
+
* Decode a string to 8-byte object ID.
|
|
866
|
+
*
|
|
867
|
+
* @param str - Base32 Crockford encoded string
|
|
868
|
+
* @returns 8-byte ID
|
|
869
|
+
*/
|
|
870
|
+
declare function decodeObjectId8(str: string): Uint8Array;
|
|
871
|
+
|
|
872
|
+
export { type AbsolutePath, type ArrayManifest, type ArrayNodeData, type ArrayUpdatedChunksInfo, type AsyncReadable, type ByteRange, type ChunkIndexRange, type ChunkPayload, type ChunkRef, CompressionAlgorithm, DefaultFetchClient, type DimensionShape, type FetchClient, FileType, type GroupNodeData, HeaderParseError, HttpStorage, type HttpStorageOptions, IcechunkStore, type IcechunkStoreOptions, type InlineChunkPayload, LRUCache, type Manifest, type ManifestFileInfo, type ManifestRef, type MetadataItem, type MoveOperationInfo, type NativeChunkPayload, type NodeData, type NodeSnapshot, NotFoundError, type ObjectId12, type ObjectId8, type RangeQuery, ReadSession, type RefData, Repository, type RepositoryOptions, type RequestOptions, type Snapshot, SpecVersion, type Storage, StorageError, type TransactionLogEntry, type UpdatedChunkIndices, type VirtualChunkPayload, decodeBase32, decodeObjectId12, decodeObjectId8, encodeBase32, encodeObjectId12, encodeObjectId8 };
|