hdf5-as-virtual-zarr 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -2
- package/dist/bundled/index.js +815 -445
- package/dist/bundled/index.js.map +1 -1
- package/dist/hdf-store.d.ts +33 -15
- package/dist/hdf-store.js +160 -22
- package/dist/index.d.ts +1 -1
- package/dist/jsfive/btree.js +37 -34
- package/dist/jsfive/dataobjects.d.ts +10 -1
- package/dist/jsfive/dataobjects.js +199 -111
- package/dist/jsfive/filters.d.ts +13 -0
- package/dist/jsfive/filters.js +62 -0
- package/dist/jsfive/high-level.js +3 -0
- package/dist/jsfive/misc-low-level.d.ts +7 -1
- package/dist/jsfive/misc-low-level.js +38 -17
- package/dist/single-hdf5-to-zarr.d.ts +33 -4
- package/dist/single-hdf5-to-zarr.js +130 -75
- package/dist/types.d.ts +16 -0
- package/dist/types.js +26 -0
- package/package.json +2 -1
- package/src/hdf-store.ts +182 -26
- package/src/index.ts +1 -1
- package/src/jsfive/btree.ts +49 -37
- package/src/jsfive/dataobjects.ts +218 -97
- package/src/jsfive/filters.ts +64 -0
- package/src/jsfive/high-level.ts +3 -0
- package/src/jsfive/misc-low-level.ts +60 -21
- package/src/single-hdf5-to-zarr.ts +161 -79
- package/src/types.ts +34 -0
package/README.md
CHANGED
|
@@ -13,11 +13,11 @@ This is a port of [SingleHdf5ToZarr](https://github.com/fsspec/kerchunk/blob/mai
|
|
|
13
13
|
## Usage
|
|
14
14
|
|
|
15
15
|
```ts
|
|
16
|
-
import { open, get, root } from "zarrita";
|
|
16
|
+
import { open, get, root, withRangeCoalescing } from "zarrita";
|
|
17
17
|
import { FetchStore } from "@zarrita/storage";
|
|
18
18
|
import { HdfStore } from "hdf5-as-virtual-zarr";
|
|
19
19
|
|
|
20
|
-
const internalStore = new FetchStore("https://example.com/data.h5");
|
|
20
|
+
const internalStore = withRangeCoalescing(new FetchStore("https://example.com/data.h5"));
|
|
21
21
|
const hdfStore = await HdfStore.fromStore(internalStore);
|
|
22
22
|
|
|
23
23
|
const storeRoot = root(hdfStore);
|
|
@@ -25,6 +25,24 @@ const arr = await open(storeRoot.resolve("X"), { kind: "array" });
|
|
|
25
25
|
const arrData = await get(arr);
|
|
26
26
|
```
|
|
27
27
|
|
|
28
|
+
### Assumption: the source is wrapped with `withRangeCoalescing`
|
|
29
|
+
|
|
30
|
+
`HdfStore` assumes the store passed to `HdfStore.fromStore` is wrapped with zarrita's
|
|
31
|
+
[`withRangeCoalescing`](https://github.com/manzt/zarrita.js), as in the example above.
|
|
32
|
+
Reading HDF5 metadata involves many small range reads (object headers, B-tree nodes, heaps).
|
|
33
|
+
`HdfStore` issues independent reads concurrently (sibling groups and datasets, B-tree children,
|
|
34
|
+
global heap collections, chunks) so that `withRangeCoalescing` can merge reads made in the same
|
|
35
|
+
microtask into a few HTTP requests. `HdfStore` keeps no byte cache of its own, so it also reads
|
|
36
|
+
a little ahead (clamped to the end of the file) to avoid follow-up requests for adjacent bytes.
|
|
37
|
+
Without `withRangeCoalescing` the store still works, but it makes many more requests.
|
|
38
|
+
|
|
39
|
+
Translation is lazy:
|
|
40
|
+
|
|
41
|
+
- `HdfStore.fromStore` only reads the superblock and the root group.
|
|
42
|
+
- A group or array's metadata (`zarr.json`) is translated the first time it is requested.
|
|
43
|
+
- Chunk references (B-tree walks, inlined string data) are computed only when a chunk of that array is requested.
|
|
44
|
+
- `await hdfStore.getConsolidatedMetadata()` walks the metadata of the whole file, but reads no chunk data.
|
|
45
|
+
|
|
28
46
|
## Development
|
|
29
47
|
|
|
30
48
|
### Setup
|