hdf5-as-virtual-zarr 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (255) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +39 -0
  3. package/dist/hdf-store.d.ts +9 -0
  4. package/dist/hdf-store.js +18 -0
  5. package/dist/index.d.ts +3 -0
  6. package/dist/index.js +3 -0
  7. package/dist/jsfive/btree.d.ts +52 -0
  8. package/dist/jsfive/btree.js +354 -0
  9. package/dist/jsfive/core.d.ts +35 -0
  10. package/dist/jsfive/core.js +196 -0
  11. package/dist/jsfive/dataobjects.d.ts +64 -0
  12. package/dist/jsfive/dataobjects.js +1001 -0
  13. package/dist/jsfive/datatype-msg.d.ts +15 -0
  14. package/dist/jsfive/datatype-msg.js +110 -0
  15. package/dist/jsfive/high-level.d.ts +42 -0
  16. package/dist/jsfive/high-level.js +150 -0
  17. package/dist/jsfive/misc-low-level.d.ts +67 -0
  18. package/dist/jsfive/misc-low-level.js +506 -0
  19. package/dist/ref-spec-to-consolidated-metadata.d.ts +12 -0
  20. package/dist/ref-spec-to-consolidated-metadata.js +22 -0
  21. package/dist/ref-store.d.ts +10 -0
  22. package/dist/ref-store.js +79 -0
  23. package/dist/single-hdf5-to-zarr.d.ts +76 -0
  24. package/dist/single-hdf5-to-zarr.js +745 -0
  25. package/dist/types.d.ts +34 -0
  26. package/dist/types.js +12 -0
  27. package/package.json +53 -0
  28. package/src/hdf-store.ts +25 -0
  29. package/src/index.ts +3 -0
  30. package/src/jsfive/README.md +1 -0
  31. package/src/jsfive/btree.ts +474 -0
  32. package/src/jsfive/core.ts +228 -0
  33. package/src/jsfive/dataobjects.ts +1262 -0
  34. package/src/jsfive/datatype-msg.ts +111 -0
  35. package/src/jsfive/high-level.ts +182 -0
  36. package/src/jsfive/misc-low-level.ts +641 -0
  37. package/src/ref-spec-to-consolidated-metadata.ts +38 -0
  38. package/src/ref-store.ts +115 -0
  39. package/src/single-hdf5-to-zarr.ts +868 -0
  40. package/src/types.ts +49 -0
  41. package/test/fixtures/dense.adata.zarr/.zattrs +4 -0
  42. package/test/fixtures/dense.adata.zarr/.zgroup +3 -0
  43. package/test/fixtures/dense.adata.zarr/.zmetadata +285 -0
  44. package/test/fixtures/dense.adata.zarr/X/.zarray +22 -0
  45. package/test/fixtures/dense.adata.zarr/X/.zattrs +4 -0
  46. package/test/fixtures/dense.adata.zarr/X/0.0 +0 -0
  47. package/test/fixtures/dense.adata.zarr/layers/.zattrs +4 -0
  48. package/test/fixtures/dense.adata.zarr/layers/.zgroup +3 -0
  49. package/test/fixtures/dense.adata.zarr/obs/.zattrs +9 -0
  50. package/test/fixtures/dense.adata.zarr/obs/.zgroup +3 -0
  51. package/test/fixtures/dense.adata.zarr/obs/_index/.zarray +24 -0
  52. package/test/fixtures/dense.adata.zarr/obs/_index/.zattrs +4 -0
  53. package/test/fixtures/dense.adata.zarr/obs/_index/0 +0 -0
  54. package/test/fixtures/dense.adata.zarr/obs/categorical/.zattrs +5 -0
  55. package/test/fixtures/dense.adata.zarr/obs/categorical/.zgroup +3 -0
  56. package/test/fixtures/dense.adata.zarr/obs/categorical/categories/.zarray +24 -0
  57. package/test/fixtures/dense.adata.zarr/obs/categorical/categories/.zattrs +4 -0
  58. package/test/fixtures/dense.adata.zarr/obs/categorical/categories/0 +0 -0
  59. package/test/fixtures/dense.adata.zarr/obs/categorical/codes/.zarray +20 -0
  60. package/test/fixtures/dense.adata.zarr/obs/categorical/codes/.zattrs +4 -0
  61. package/test/fixtures/dense.adata.zarr/obs/categorical/codes/0 +0 -0
  62. package/test/fixtures/dense.adata.zarr/obs/string/.zarray +24 -0
  63. package/test/fixtures/dense.adata.zarr/obs/string/.zattrs +4 -0
  64. package/test/fixtures/dense.adata.zarr/obs/string/0 +0 -0
  65. package/test/fixtures/dense.adata.zarr/obsm/.zattrs +4 -0
  66. package/test/fixtures/dense.adata.zarr/obsm/.zgroup +3 -0
  67. package/test/fixtures/dense.adata.zarr/obsm/X_umap/.zarray +22 -0
  68. package/test/fixtures/dense.adata.zarr/obsm/X_umap/.zattrs +4 -0
  69. package/test/fixtures/dense.adata.zarr/obsm/X_umap/0.0 +0 -0
  70. package/test/fixtures/dense.adata.zarr/obsp/.zattrs +4 -0
  71. package/test/fixtures/dense.adata.zarr/obsp/.zgroup +3 -0
  72. package/test/fixtures/dense.adata.zarr/raw/.zarray +10 -0
  73. package/test/fixtures/dense.adata.zarr/raw/.zattrs +4 -0
  74. package/test/fixtures/dense.adata.zarr/raw/0 +0 -0
  75. package/test/fixtures/dense.adata.zarr/uns/.zattrs +4 -0
  76. package/test/fixtures/dense.adata.zarr/uns/.zgroup +3 -0
  77. package/test/fixtures/dense.adata.zarr/var/.zattrs +6 -0
  78. package/test/fixtures/dense.adata.zarr/var/.zgroup +3 -0
  79. package/test/fixtures/dense.adata.zarr/var/_index/.zarray +24 -0
  80. package/test/fixtures/dense.adata.zarr/var/_index/.zattrs +4 -0
  81. package/test/fixtures/dense.adata.zarr/var/_index/0 +0 -0
  82. package/test/fixtures/dense.adata.zarr/varm/.zattrs +4 -0
  83. package/test/fixtures/dense.adata.zarr/varm/.zgroup +3 -0
  84. package/test/fixtures/dense.adata.zarr/varp/.zattrs +4 -0
  85. package/test/fixtures/dense.adata.zarr/varp/.zgroup +3 -0
  86. package/test/fixtures/dense.adata.zarr.json +182 -0
  87. package/test/fixtures/dense.adata.zmetadata.json +1 -0
  88. package/test/fixtures/dense.h5ad +0 -0
  89. package/test/fixtures/dense.h5ad.refspec.json +1 -0
  90. package/test/fixtures/dense.v3.adata.zarr/X/c/0/0 +0 -0
  91. package/test/fixtures/dense.v3.adata.zarr/X/zarr.json +45 -0
  92. package/test/fixtures/dense.v3.adata.zarr/layers/zarr.json +8 -0
  93. package/test/fixtures/dense.v3.adata.zarr/obs/_index/c/0 +0 -0
  94. package/test/fixtures/dense.v3.adata.zarr/obs/_index/zarr.json +41 -0
  95. package/test/fixtures/dense.v3.adata.zarr/obs/categorical/categories/c/0 +0 -0
  96. package/test/fixtures/dense.v3.adata.zarr/obs/categorical/categories/zarr.json +41 -0
  97. package/test/fixtures/dense.v3.adata.zarr/obs/categorical/codes/c/0 +0 -0
  98. package/test/fixtures/dense.v3.adata.zarr/obs/categorical/codes/zarr.json +40 -0
  99. package/test/fixtures/dense.v3.adata.zarr/obs/categorical/zarr.json +9 -0
  100. package/test/fixtures/dense.v3.adata.zarr/obs/string/c/0 +0 -0
  101. package/test/fixtures/dense.v3.adata.zarr/obs/string/zarr.json +41 -0
  102. package/test/fixtures/dense.v3.adata.zarr/obs/zarr.json +13 -0
  103. package/test/fixtures/dense.v3.adata.zarr/obsm/X_umap/c/0/0 +0 -0
  104. package/test/fixtures/dense.v3.adata.zarr/obsm/X_umap/zarr.json +45 -0
  105. package/test/fixtures/dense.v3.adata.zarr/obsm/zarr.json +8 -0
  106. package/test/fixtures/dense.v3.adata.zarr/obsp/zarr.json +8 -0
  107. package/test/fixtures/dense.v3.adata.zarr/raw/zarr.json +36 -0
  108. package/test/fixtures/dense.v3.adata.zarr/uns/zarr.json +8 -0
  109. package/test/fixtures/dense.v3.adata.zarr/var/_index/c/0 +0 -0
  110. package/test/fixtures/dense.v3.adata.zarr/var/_index/zarr.json +41 -0
  111. package/test/fixtures/dense.v3.adata.zarr/var/zarr.json +10 -0
  112. package/test/fixtures/dense.v3.adata.zarr/varm/zarr.json +8 -0
  113. package/test/fixtures/dense.v3.adata.zarr/varp/zarr.json +8 -0
  114. package/test/fixtures/dense.v3.adata.zarr/zarr.json +469 -0
  115. package/test/fixtures/dense.v3.adata.zarr.json +102 -0
  116. package/test/fixtures/generate_fixtures.py +157 -0
  117. package/test/fixtures/generate_fixtures_v3.py +126 -0
  118. package/test/fixtures/minimal.adata.zarr/.zattrs +4 -0
  119. package/test/fixtures/minimal.adata.zarr/.zgroup +3 -0
  120. package/test/fixtures/minimal.adata.zarr/.zmetadata +168 -0
  121. package/test/fixtures/minimal.adata.zarr/X/.zarray +22 -0
  122. package/test/fixtures/minimal.adata.zarr/X/.zattrs +4 -0
  123. package/test/fixtures/minimal.adata.zarr/X/0.0 +0 -0
  124. package/test/fixtures/minimal.adata.zarr/layers/.zattrs +4 -0
  125. package/test/fixtures/minimal.adata.zarr/layers/.zgroup +3 -0
  126. package/test/fixtures/minimal.adata.zarr/obs/.zattrs +6 -0
  127. package/test/fixtures/minimal.adata.zarr/obs/.zgroup +3 -0
  128. package/test/fixtures/minimal.adata.zarr/obs/_index/.zarray +24 -0
  129. package/test/fixtures/minimal.adata.zarr/obs/_index/.zattrs +4 -0
  130. package/test/fixtures/minimal.adata.zarr/obs/_index/0 +0 -0
  131. package/test/fixtures/minimal.adata.zarr/obsm/.zattrs +4 -0
  132. package/test/fixtures/minimal.adata.zarr/obsm/.zgroup +3 -0
  133. package/test/fixtures/minimal.adata.zarr/obsp/.zattrs +4 -0
  134. package/test/fixtures/minimal.adata.zarr/obsp/.zgroup +3 -0
  135. package/test/fixtures/minimal.adata.zarr/raw/.zarray +10 -0
  136. package/test/fixtures/minimal.adata.zarr/raw/.zattrs +4 -0
  137. package/test/fixtures/minimal.adata.zarr/raw/0 +0 -0
  138. package/test/fixtures/minimal.adata.zarr/uns/.zattrs +4 -0
  139. package/test/fixtures/minimal.adata.zarr/uns/.zgroup +3 -0
  140. package/test/fixtures/minimal.adata.zarr/var/.zattrs +6 -0
  141. package/test/fixtures/minimal.adata.zarr/var/.zgroup +3 -0
  142. package/test/fixtures/minimal.adata.zarr/var/_index/.zarray +24 -0
  143. package/test/fixtures/minimal.adata.zarr/var/_index/.zattrs +4 -0
  144. package/test/fixtures/minimal.adata.zarr/var/_index/0 +0 -0
  145. package/test/fixtures/minimal.adata.zarr/varm/.zattrs +4 -0
  146. package/test/fixtures/minimal.adata.zarr/varm/.zgroup +3 -0
  147. package/test/fixtures/minimal.adata.zarr/varp/.zattrs +4 -0
  148. package/test/fixtures/minimal.adata.zarr/varp/.zgroup +3 -0
  149. package/test/fixtures/minimal.adata.zarr.json +126 -0
  150. package/test/fixtures/minimal.adata.zmetadata.json +1 -0
  151. package/test/fixtures/minimal.h5ad +0 -0
  152. package/test/fixtures/minimal.h5ad.refspec.json +1 -0
  153. package/test/fixtures/minimal.v3.adata.zarr/X/c/0/0 +0 -0
  154. package/test/fixtures/minimal.v3.adata.zarr/X/zarr.json +45 -0
  155. package/test/fixtures/minimal.v3.adata.zarr/layers/zarr.json +8 -0
  156. package/test/fixtures/minimal.v3.adata.zarr/obs/_index/c/0 +0 -0
  157. package/test/fixtures/minimal.v3.adata.zarr/obs/_index/zarr.json +41 -0
  158. package/test/fixtures/minimal.v3.adata.zarr/obs/zarr.json +10 -0
  159. package/test/fixtures/minimal.v3.adata.zarr/obsm/zarr.json +8 -0
  160. package/test/fixtures/minimal.v3.adata.zarr/obsp/zarr.json +8 -0
  161. package/test/fixtures/minimal.v3.adata.zarr/raw/zarr.json +36 -0
  162. package/test/fixtures/minimal.v3.adata.zarr/uns/zarr.json +8 -0
  163. package/test/fixtures/minimal.v3.adata.zarr/var/_index/c/0 +0 -0
  164. package/test/fixtures/minimal.v3.adata.zarr/var/_index/zarr.json +41 -0
  165. package/test/fixtures/minimal.v3.adata.zarr/var/zarr.json +10 -0
  166. package/test/fixtures/minimal.v3.adata.zarr/varm/zarr.json +8 -0
  167. package/test/fixtures/minimal.v3.adata.zarr/varp/zarr.json +8 -0
  168. package/test/fixtures/minimal.v3.adata.zarr/zarr.json +285 -0
  169. package/test/fixtures/minimal.v3.adata.zarr.json +66 -0
  170. package/test/fixtures/mouse_liver.h5ad +0 -0
  171. package/test/fixtures/mouse_liver.h5ad.refspec.json +1 -0
  172. package/test/fixtures/mouse_liver.py +44 -0
  173. package/test/fixtures/sparse.adata.zarr/.zattrs +4 -0
  174. package/test/fixtures/sparse.adata.zarr/.zgroup +3 -0
  175. package/test/fixtures/sparse.adata.zarr/.zmetadata +304 -0
  176. package/test/fixtures/sparse.adata.zarr/X/.zattrs +8 -0
  177. package/test/fixtures/sparse.adata.zarr/X/.zgroup +3 -0
  178. package/test/fixtures/sparse.adata.zarr/X/data/.zarray +20 -0
  179. package/test/fixtures/sparse.adata.zarr/X/data/0 +0 -0
  180. package/test/fixtures/sparse.adata.zarr/X/indices/.zarray +20 -0
  181. package/test/fixtures/sparse.adata.zarr/X/indices/0 +0 -0
  182. package/test/fixtures/sparse.adata.zarr/X/indptr/.zarray +20 -0
  183. package/test/fixtures/sparse.adata.zarr/X/indptr/0 +0 -0
  184. package/test/fixtures/sparse.adata.zarr/layers/.zattrs +4 -0
  185. package/test/fixtures/sparse.adata.zarr/layers/.zgroup +3 -0
  186. package/test/fixtures/sparse.adata.zarr/obs/.zattrs +9 -0
  187. package/test/fixtures/sparse.adata.zarr/obs/.zgroup +3 -0
  188. package/test/fixtures/sparse.adata.zarr/obs/_index/.zarray +24 -0
  189. package/test/fixtures/sparse.adata.zarr/obs/_index/.zattrs +4 -0
  190. package/test/fixtures/sparse.adata.zarr/obs/_index/0 +0 -0
  191. package/test/fixtures/sparse.adata.zarr/obs/categorical/.zattrs +5 -0
  192. package/test/fixtures/sparse.adata.zarr/obs/categorical/.zgroup +3 -0
  193. package/test/fixtures/sparse.adata.zarr/obs/categorical/categories/.zarray +24 -0
  194. package/test/fixtures/sparse.adata.zarr/obs/categorical/categories/.zattrs +4 -0
  195. package/test/fixtures/sparse.adata.zarr/obs/categorical/categories/0 +0 -0
  196. package/test/fixtures/sparse.adata.zarr/obs/categorical/codes/.zarray +20 -0
  197. package/test/fixtures/sparse.adata.zarr/obs/categorical/codes/.zattrs +4 -0
  198. package/test/fixtures/sparse.adata.zarr/obs/categorical/codes/0 +0 -0
  199. package/test/fixtures/sparse.adata.zarr/obs/string/.zarray +24 -0
  200. package/test/fixtures/sparse.adata.zarr/obs/string/.zattrs +4 -0
  201. package/test/fixtures/sparse.adata.zarr/obs/string/0 +0 -0
  202. package/test/fixtures/sparse.adata.zarr/obsm/.zattrs +4 -0
  203. package/test/fixtures/sparse.adata.zarr/obsm/.zgroup +3 -0
  204. package/test/fixtures/sparse.adata.zarr/obsp/.zattrs +4 -0
  205. package/test/fixtures/sparse.adata.zarr/obsp/.zgroup +3 -0
  206. package/test/fixtures/sparse.adata.zarr/raw/.zarray +10 -0
  207. package/test/fixtures/sparse.adata.zarr/raw/.zattrs +4 -0
  208. package/test/fixtures/sparse.adata.zarr/raw/0 +0 -0
  209. package/test/fixtures/sparse.adata.zarr/uns/.zattrs +4 -0
  210. package/test/fixtures/sparse.adata.zarr/uns/.zgroup +3 -0
  211. package/test/fixtures/sparse.adata.zarr/var/.zattrs +6 -0
  212. package/test/fixtures/sparse.adata.zarr/var/.zgroup +3 -0
  213. package/test/fixtures/sparse.adata.zarr/var/_index/.zarray +24 -0
  214. package/test/fixtures/sparse.adata.zarr/var/_index/.zattrs +4 -0
  215. package/test/fixtures/sparse.adata.zarr/var/_index/0 +0 -0
  216. package/test/fixtures/sparse.adata.zarr/varm/.zattrs +4 -0
  217. package/test/fixtures/sparse.adata.zarr/varm/.zgroup +3 -0
  218. package/test/fixtures/sparse.adata.zarr/varp/.zattrs +4 -0
  219. package/test/fixtures/sparse.adata.zarr/varp/.zgroup +3 -0
  220. package/test/fixtures/sparse.adata.zarr.json +190 -0
  221. package/test/fixtures/sparse.adata.zmetadata.json +1 -0
  222. package/test/fixtures/sparse.h5ad +0 -0
  223. package/test/fixtures/sparse.h5ad.refspec.json +1 -0
  224. package/test/fixtures/sparse.v3.adata.zarr/X/data/c/0 +0 -0
  225. package/test/fixtures/sparse.v3.adata.zarr/X/data/zarr.json +40 -0
  226. package/test/fixtures/sparse.v3.adata.zarr/X/indices/c/0 +0 -0
  227. package/test/fixtures/sparse.v3.adata.zarr/X/indices/zarr.json +40 -0
  228. package/test/fixtures/sparse.v3.adata.zarr/X/indptr/c/0 +0 -0
  229. package/test/fixtures/sparse.v3.adata.zarr/X/indptr/zarr.json +40 -0
  230. package/test/fixtures/sparse.v3.adata.zarr/X/zarr.json +12 -0
  231. package/test/fixtures/sparse.v3.adata.zarr/layers/zarr.json +8 -0
  232. package/test/fixtures/sparse.v3.adata.zarr/obs/_index/c/0 +0 -0
  233. package/test/fixtures/sparse.v3.adata.zarr/obs/_index/zarr.json +41 -0
  234. package/test/fixtures/sparse.v3.adata.zarr/obs/categorical/categories/c/0 +0 -0
  235. package/test/fixtures/sparse.v3.adata.zarr/obs/categorical/categories/zarr.json +41 -0
  236. package/test/fixtures/sparse.v3.adata.zarr/obs/categorical/codes/c/0 +0 -0
  237. package/test/fixtures/sparse.v3.adata.zarr/obs/categorical/codes/zarr.json +40 -0
  238. package/test/fixtures/sparse.v3.adata.zarr/obs/categorical/zarr.json +9 -0
  239. package/test/fixtures/sparse.v3.adata.zarr/obs/string/c/0 +0 -0
  240. package/test/fixtures/sparse.v3.adata.zarr/obs/string/zarr.json +41 -0
  241. package/test/fixtures/sparse.v3.adata.zarr/obs/zarr.json +13 -0
  242. package/test/fixtures/sparse.v3.adata.zarr/obsm/zarr.json +8 -0
  243. package/test/fixtures/sparse.v3.adata.zarr/obsp/zarr.json +8 -0
  244. package/test/fixtures/sparse.v3.adata.zarr/raw/zarr.json +36 -0
  245. package/test/fixtures/sparse.v3.adata.zarr/uns/zarr.json +8 -0
  246. package/test/fixtures/sparse.v3.adata.zarr/var/_index/c/0 +0 -0
  247. package/test/fixtures/sparse.v3.adata.zarr/var/_index/zarr.json +41 -0
  248. package/test/fixtures/sparse.v3.adata.zarr/var/zarr.json +10 -0
  249. package/test/fixtures/sparse.v3.adata.zarr/varm/zarr.json +8 -0
  250. package/test/fixtures/sparse.v3.adata.zarr/varp/zarr.json +8 -0
  251. package/test/fixtures/sparse.v3.adata.zarr/zarr.json +516 -0
  252. package/test/fixtures/sparse.v3.adata.zarr.json +114 -0
  253. package/test/hdf-store.test.ts +268 -0
  254. package/test/index.test.ts +762 -0
  255. package/test/partial-reads.test.ts +150 -0
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Mark Keller
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,39 @@
1
+ # hdf5-as-virtual-zarr
2
+
3
+ Given an input HDF5 file URL, generate a Zarr References Specification JSON output.
4
+
5
+ This TypeScript implementation can run in a web browser, and generates output that is compatible with the [ReferenceStore](https://github.com/manzt/zarrita.js/blob/d511ea44551a62c2af64ba33672672e9286490da/packages/%40zarrita-storage/src/ref.ts#L42) from [zarrita.js](https://github.com/manzt/zarrita.js).
6
+
7
+ It supports large HDF5 files using range requests (fetching only the subset of bytes required). When possible, large arrays should not be accessed; only the minimal metadata should be accessed.
8
+
9
+ This is a port of [SingleHdf5ToZarr](https://github.com/fsspec/kerchunk/blob/main/kerchunk/hdf.py#L52) and its `translate` function from the `kerchunk` Python package.
10
+
11
+
12
+ ## Development
13
+
14
+ ### Setup
15
+
16
+ Install PNPM and UV.
17
+
18
+ ```sh
19
+ pnpm install
20
+ pnpm run generate-fixtures
21
+ ```
22
+
23
+ ### Test
24
+
25
+ ```sh
26
+ pnpm run test
27
+ ```
28
+
29
+ ### Build
30
+
31
+ ```sh
32
+ pnpm run build
33
+ ```
34
+
35
+ ### Publish
36
+
37
+ ```sh
38
+ pnpm publish --filter "hdf5-as-virtual-zarr"
39
+ ```
@@ -0,0 +1,9 @@
1
+ import { ComposableReferenceStore } from "./ref-store.js";
2
+ import type { AsyncReadable } from "./types.js";
3
+ import type { AbsolutePath } from "@zarrita/storage";
4
+ export declare class HdfStore implements AsyncReadable {
5
+ private refStore;
6
+ constructor(refStore: ComposableReferenceStore);
7
+ get(key: AbsolutePath, opts?: RequestInit): Promise<Uint8Array | undefined>;
8
+ static fromStore(hdfSource: AsyncReadable): Promise<HdfStore>;
9
+ }
@@ -0,0 +1,18 @@
1
+ import { ComposableReferenceStore } from "./ref-store.js";
2
+ import { SingleHdf5ToZarr } from "./single-hdf5-to-zarr.js";
3
+ export class HdfStore {
4
+ refStore;
5
+ constructor(refStore) {
6
+ this.refStore = refStore;
7
+ }
8
+ async get(key, opts) {
9
+ return this.refStore.get(key, opts);
10
+ }
11
+ // TODO: does it make sense to support getRange here?
12
+ static async fromStore(hdfSource) {
13
+ const converter = new SingleHdf5ToZarr(hdfSource, { url: null });
14
+ const refSpec = await converter.translate();
15
+ const refStore = ComposableReferenceStore.fromSpec(refSpec, hdfSource);
16
+ return new HdfStore(refStore);
17
+ }
18
+ }
@@ -0,0 +1,3 @@
1
+ export { SingleHdf5ToZarr } from "./single-hdf5-to-zarr.js";
2
+ export { refSpecToConsolidatedMetadata } from "./ref-spec-to-consolidated-metadata.js";
3
+ export { HdfStore } from "./hdf-store.js";
package/dist/index.js ADDED
@@ -0,0 +1,3 @@
1
+ export { SingleHdf5ToZarr } from "./single-hdf5-to-zarr.js";
2
+ export { refSpecToConsolidatedMetadata } from "./ref-spec-to-consolidated-metadata.js";
3
+ export { HdfStore } from "./hdf-store.js";
@@ -0,0 +1,52 @@
1
+ /**
2
+ * Vendored and adapted from jsfive/esm/btree.js.
3
+ * BTreeV1Groups, BTreeV1RawDataChunks, BTreeV2GroupNames, BTreeV2GroupOrders.
4
+ *
5
+ * All classes use async factory methods with AsyncReadable.fetch() for partial reads.
6
+ * construct_data_from_chunks() and _filter_chunk() are omitted –
7
+ * we only need chunk location metadata for the reference spec.
8
+ */
9
+ import type { AsyncReadable } from "../types.js";
10
+ interface BTreeV1Node {
11
+ header: Map<string, any>;
12
+ keys: any[];
13
+ addresses: number[];
14
+ }
15
+ export declare class BTreeV1Groups {
16
+ all_nodes: Map<number, BTreeV1Node[]>;
17
+ depth: number;
18
+ private constructor();
19
+ static create(source: AsyncReadable, offset: number): Promise<BTreeV1Groups>;
20
+ symbol_table_addresses(): number[];
21
+ }
22
+ export interface ChunkKey {
23
+ chunk_size: number;
24
+ filter_mask: number;
25
+ chunk_offset: number[];
26
+ }
27
+ export declare class BTreeV1RawDataChunks {
28
+ all_nodes: Map<number, {
29
+ header: Map<string, any>;
30
+ keys: ChunkKey[];
31
+ addresses: number[];
32
+ }[]>;
33
+ depth: number;
34
+ dims: number;
35
+ private constructor();
36
+ static create(source: AsyncReadable, offset: number, dims: number): Promise<BTreeV1RawDataChunks>;
37
+ }
38
+ export declare class BTreeV2GroupNames {
39
+ all_nodes: Map<number, any[]>;
40
+ header: Map<string, any>;
41
+ private constructor();
42
+ static create(source: AsyncReadable, offset: number): Promise<BTreeV2GroupNames>;
43
+ iter_records(): Generator<Map<string, any>>;
44
+ }
45
+ export declare class BTreeV2GroupOrders {
46
+ all_nodes: Map<number, any[]>;
47
+ header: Map<string, any>;
48
+ private constructor();
49
+ static create(source: AsyncReadable, offset: number): Promise<BTreeV2GroupOrders>;
50
+ iter_records(): Generator<Map<string, any>>;
51
+ }
52
+ export {};
@@ -0,0 +1,354 @@
1
+ /**
2
+ * Vendored and adapted from jsfive/esm/btree.js.
3
+ * BTreeV1Groups, BTreeV1RawDataChunks, BTreeV2GroupNames, BTreeV2GroupOrders.
4
+ *
5
+ * All classes use async factory methods with AsyncReadable.fetch() for partial reads.
6
+ * construct_data_from_chunks() and _filter_chunk() are omitted –
7
+ * we only need chunk location metadata for the reference spec.
8
+ */
9
+ import { fetchRange } from "../types.js";
10
+ import { _unpack_struct_from, _structure_size, struct, bitSize, } from "./core.js";
11
+ // ────────── BTreeV1 common ──────────
12
+ const B_LINK_NODE_V1 = new Map([
13
+ ["signature", "4s"],
14
+ ["node_type", "B"],
15
+ ["node_level", "B"],
16
+ ["entries_used", "H"],
17
+ ["left_sibling", "Q"],
18
+ ["right_sibling", "Q"],
19
+ ]);
20
+ const B_LINK_NODE_V1_SIZE = _structure_size(B_LINK_NODE_V1);
21
+ async function _readV1NodeHeader(source, offset, node_level) {
22
+ // Read header + generous amount for keys/addresses that follow
23
+ // We don't know exact size yet, so read header first
24
+ const hBuf = await fetchRange(source, offset, B_LINK_NODE_V1_SIZE);
25
+ const node = _unpack_struct_from(B_LINK_NODE_V1, hBuf, 0);
26
+ if (node_level != null && node.get("node_level") !== node_level) {
27
+ throw new Error("node level does not match");
28
+ }
29
+ return [node, hBuf];
30
+ }
31
+ // ────────── BTreeV1Groups (type 0) ──────────
32
+ export class BTreeV1Groups {
33
+ all_nodes;
34
+ depth;
35
+ constructor(all_nodes, depth) {
36
+ this.all_nodes = all_nodes;
37
+ this.depth = depth;
38
+ }
39
+ static async create(source, offset) {
40
+ const all_nodes = new Map();
41
+ async function readNode(off, node_level) {
42
+ const [header] = await _readV1NodeHeader(source, off, node_level);
43
+ let readOff = off + B_LINK_NODE_V1_SIZE;
44
+ const entries_used = header.get("entries_used");
45
+ // Each entry: key(Q=8) + address(Q=8) = 16 bytes, plus one extra key
46
+ const dataSize = entries_used * 16 + 8;
47
+ const dataBuf = await fetchRange(source, readOff, dataSize);
48
+ let localOff = 0;
49
+ const keys = [];
50
+ const addresses = [];
51
+ for (let i = 0; i < entries_used; i++) {
52
+ keys.push(struct.unpack_from("<Q", dataBuf, localOff)[0]);
53
+ localOff += 8;
54
+ addresses.push(struct.unpack_from("<Q", dataBuf, localOff)[0]);
55
+ localOff += 8;
56
+ }
57
+ // N+1 key
58
+ keys.push(struct.unpack_from("<Q", dataBuf, localOff)[0]);
59
+ return { header, keys, addresses };
60
+ }
61
+ // Read root
62
+ const root = await readNode(offset, null);
63
+ const depth = root.header.get("node_level");
64
+ addNode(all_nodes, root, depth);
65
+ // Read children
66
+ let node_level = depth;
67
+ while (node_level > 0) {
68
+ const parents = all_nodes.get(node_level) || [];
69
+ for (const parent of parents) {
70
+ for (const child_addr of parent.addresses) {
71
+ const child = await readNode(child_addr, node_level - 1);
72
+ addNode(all_nodes, child, node_level - 1);
73
+ }
74
+ }
75
+ node_level--;
76
+ }
77
+ return new BTreeV1Groups(all_nodes, depth);
78
+ }
79
+ symbol_table_addresses() {
80
+ const all_address = [];
81
+ const root_nodes = this.all_nodes.get(0) || [];
82
+ for (const node of root_nodes) {
83
+ all_address.push(...node.addresses);
84
+ }
85
+ return all_address;
86
+ }
87
+ }
88
+ export class BTreeV1RawDataChunks {
89
+ all_nodes;
90
+ depth;
91
+ dims;
92
+ constructor(all_nodes, depth, dims) {
93
+ this.all_nodes = all_nodes;
94
+ this.depth = depth;
95
+ this.dims = dims;
96
+ }
97
+ static async create(source, offset, dims) {
98
+ const all_nodes = new Map();
99
+ async function readNode(off, node_level) {
100
+ const [header] = await _readV1NodeHeader(source, off, node_level);
101
+ let readOff = off + B_LINK_NODE_V1_SIZE;
102
+ const entries_used = header.get("entries_used");
103
+ // Each entry: chunk_size(I=4) + filter_mask(I=4) + dims*Q + address(Q=8)
104
+ const fmt = "<" + dims.toFixed() + "Q";
105
+ const fmt_size = struct.calcsize(fmt);
106
+ const entrySize = 8 + fmt_size + 8; // II + dims*Q + Q
107
+ const dataBuf = await fetchRange(source, readOff, entries_used * entrySize);
108
+ let localOff = 0;
109
+ const keys = [];
110
+ const addresses = [];
111
+ for (let i = 0; i < entries_used; i++) {
112
+ const [chunk_size, filter_mask] = struct.unpack_from("<II", dataBuf, localOff);
113
+ localOff += 8;
114
+ const chunk_offset = struct.unpack_from(fmt, dataBuf, localOff);
115
+ localOff += fmt_size;
116
+ const chunk_address = struct.unpack_from("<Q", dataBuf, localOff)[0];
117
+ localOff += 8;
118
+ keys.push({ chunk_size, filter_mask, chunk_offset });
119
+ addresses.push(chunk_address);
120
+ }
121
+ return { header, keys, addresses };
122
+ }
123
+ // Read root
124
+ const root = await readNode(offset, null);
125
+ const depth = root.header.get("node_level");
126
+ addNode(all_nodes, root, depth);
127
+ // Read children
128
+ let node_level = depth;
129
+ while (node_level > 0) {
130
+ const parents = all_nodes.get(node_level) || [];
131
+ for (const parent of parents) {
132
+ for (const child_addr of parent.addresses) {
133
+ const child = await readNode(child_addr, node_level - 1);
134
+ addNode(all_nodes, child, node_level - 1);
135
+ }
136
+ }
137
+ node_level--;
138
+ }
139
+ return new BTreeV1RawDataChunks(all_nodes, depth, dims);
140
+ }
141
+ }
142
+ // ────────── BTreeV2 common ──────────
143
+ const B_TREE_HEADER_V2 = new Map([
144
+ ["signature", "4s"],
145
+ ["version", "B"],
146
+ ["node_type", "B"],
147
+ ["node_size", "I"],
148
+ ["record_size", "H"],
149
+ ["depth", "H"],
150
+ ["split_percent", "B"],
151
+ ["merge_percent", "B"],
152
+ ["root_address", "Q"],
153
+ ["root_nrecords", "H"],
154
+ ["total_nrecords", "Q"],
155
+ ]);
156
+ const B_LINK_NODE_V2 = new Map([
157
+ ["signature", "4s"],
158
+ ["version", "B"],
159
+ ["node_type", "B"],
160
+ ]);
161
+ const B_LINK_NODE_V2_SIZE = _structure_size(B_LINK_NODE_V2);
162
+ function _required_bytes(integer) {
163
+ return Math.ceil(bitSize(integer) / 8);
164
+ }
165
+ function _int_format(bytelength) {
166
+ return ["<B", "<H", "<I", "<Q"][bytelength - 1];
167
+ }
168
+ function _nrecords_max(node_size, record_size, addr_size) {
169
+ return Math.floor((node_size - 10 - addr_size) / (record_size + addr_size));
170
+ }
171
+ function _calculate_address_formats(header) {
172
+ const node_size = header.get("node_size");
173
+ const record_size = header.get("record_size");
174
+ let nrecords_max = 0;
175
+ let ntotalrecords_max = 0;
176
+ const address_formats = new Map();
177
+ const max_depth = header.get("depth");
178
+ for (let node_level = 0; node_level <= max_depth; node_level++) {
179
+ let offset_fmt = "";
180
+ let num1_fmt = "";
181
+ let num2_fmt = "";
182
+ let offset_size, num1_size, num2_size;
183
+ if (node_level === 0) {
184
+ offset_size = 0;
185
+ num1_size = 0;
186
+ num2_size = 0;
187
+ }
188
+ else if (node_level === 1) {
189
+ offset_size = 8;
190
+ offset_fmt = "<Q";
191
+ num1_size = _required_bytes(nrecords_max);
192
+ num1_fmt = _int_format(num1_size);
193
+ num2_size = 0;
194
+ }
195
+ else {
196
+ offset_size = 8;
197
+ offset_fmt = "<Q";
198
+ num1_size = _required_bytes(nrecords_max);
199
+ num1_fmt = _int_format(num1_size);
200
+ num2_size = _required_bytes(ntotalrecords_max);
201
+ num2_fmt = _int_format(num2_size);
202
+ }
203
+ address_formats.set(node_level, [
204
+ offset_size, num1_size, num2_size,
205
+ offset_fmt, num1_fmt, num2_fmt,
206
+ ]);
207
+ if (node_level < max_depth) {
208
+ const addr_size = offset_size + num1_size + num2_size;
209
+ nrecords_max = _nrecords_max(node_size, record_size, addr_size);
210
+ if (ntotalrecords_max > 0) {
211
+ ntotalrecords_max *= nrecords_max;
212
+ }
213
+ else {
214
+ ntotalrecords_max = nrecords_max;
215
+ }
216
+ }
217
+ }
218
+ return address_formats;
219
+ }
220
+ async function _readBTreeV2(source, offset, parseRecord) {
221
+ // Read tree header
222
+ const headerSize = _structure_size(B_TREE_HEADER_V2);
223
+ const hBuf = await fetchRange(source, offset, headerSize);
224
+ const header = _unpack_struct_from(B_TREE_HEADER_V2, hBuf, 0);
225
+ const address_formats = _calculate_address_formats(header);
226
+ const depth = header.get("depth");
227
+ const record_size = header.get("record_size");
228
+ const all_nodes = new Map();
229
+ async function readNode(address, node_level) {
230
+ const [off, nrecords] = address;
231
+ // Read node header + records + addresses
232
+ // Estimate max bytes needed
233
+ const addr_fmts = address_formats.get(node_level);
234
+ const [offset_size, num1_size, num2_size] = addr_fmts;
235
+ const addr_entry_size = offset_size + num1_size + num2_size;
236
+ const maxBytes = B_LINK_NODE_V2_SIZE +
237
+ nrecords * record_size +
238
+ (node_level !== 0 ? (nrecords + 1) * addr_entry_size : 0) +
239
+ 4; // checksum
240
+ const nodeBuf = await fetchRange(source, off, maxBytes);
241
+ const node = _unpack_struct_from(B_LINK_NODE_V2, nodeBuf, 0);
242
+ node.set("node_level", node_level);
243
+ let localOff = B_LINK_NODE_V2_SIZE;
244
+ const keys = [];
245
+ for (let i = 0; i < nrecords; i++) {
246
+ keys.push(parseRecord(nodeBuf, localOff, record_size));
247
+ localOff += record_size;
248
+ }
249
+ const addresses = [];
250
+ if (node_level !== 0) {
251
+ const [os, n1s, n2s, offset_fmt, num1_fmt, num2_fmt] = addr_fmts;
252
+ for (let j = 0; j <= nrecords; j++) {
253
+ const address_offset = struct.unpack_from(offset_fmt, nodeBuf, localOff)[0];
254
+ localOff += os;
255
+ const num1 = struct.unpack_from(num1_fmt, nodeBuf, localOff)[0];
256
+ localOff += n1s;
257
+ let num2 = num1;
258
+ if (n2s > 0) {
259
+ num2 = struct.unpack_from(num2_fmt, nodeBuf, localOff)[0];
260
+ localOff += n2s;
261
+ }
262
+ addresses.push([address_offset, num1, num2]);
263
+ }
264
+ }
265
+ return { header: node, keys, addresses };
266
+ }
267
+ // Read root
268
+ const rootAddr = [
269
+ header.get("root_address"),
270
+ header.get("root_nrecords"),
271
+ header.get("total_nrecords"),
272
+ ];
273
+ const root = await readNode(rootAddr, depth);
274
+ addNode(all_nodes, root, depth);
275
+ // Read children
276
+ let node_level = depth;
277
+ while (node_level > 0) {
278
+ const parents = all_nodes.get(node_level) || [];
279
+ for (const parent of parents) {
280
+ for (const child_addr of parent.addresses) {
281
+ const child = await readNode(child_addr, node_level - 1);
282
+ addNode(all_nodes, child, node_level - 1);
283
+ }
284
+ }
285
+ node_level--;
286
+ }
287
+ return { all_nodes, header };
288
+ }
289
+ // ────────── BTreeV2GroupNames (type 5) ──────────
290
+ export class BTreeV2GroupNames {
291
+ all_nodes;
292
+ header;
293
+ constructor(all_nodes, header) {
294
+ this.all_nodes = all_nodes;
295
+ this.header = header;
296
+ }
297
+ static async create(source, offset) {
298
+ const { all_nodes, header } = await _readBTreeV2(source, offset, (buf, off) => {
299
+ const namehash = struct.unpack_from("<I", buf, off)[0];
300
+ return new Map([
301
+ ["namehash", namehash],
302
+ ["heapid", buf.slice(off + 4, off + 4 + 7)],
303
+ ]);
304
+ });
305
+ return new BTreeV2GroupNames(all_nodes, header);
306
+ }
307
+ *iter_records() {
308
+ for (const nodelist of this.all_nodes.values()) {
309
+ for (const node of nodelist) {
310
+ for (const key of node.keys) {
311
+ yield key;
312
+ }
313
+ }
314
+ }
315
+ }
316
+ }
317
+ // ────────── BTreeV2GroupOrders (type 6) ──────────
318
+ export class BTreeV2GroupOrders {
319
+ all_nodes;
320
+ header;
321
+ constructor(all_nodes, header) {
322
+ this.all_nodes = all_nodes;
323
+ this.header = header;
324
+ }
325
+ static async create(source, offset) {
326
+ const { all_nodes, header } = await _readBTreeV2(source, offset, (buf, off) => {
327
+ const creationorder = struct.unpack_from("<Q", buf, off)[0];
328
+ return new Map([
329
+ ["creationorder", creationorder],
330
+ ["heapid", buf.slice(off + 8, off + 8 + 7)],
331
+ ]);
332
+ });
333
+ return new BTreeV2GroupOrders(all_nodes, header);
334
+ }
335
+ *iter_records() {
336
+ for (const nodelist of this.all_nodes.values()) {
337
+ for (const node of nodelist) {
338
+ for (const key of node.keys) {
339
+ yield key;
340
+ }
341
+ }
342
+ }
343
+ }
344
+ }
345
+ // ────────── shared helper ──────────
346
+ function addNode(all_nodes, node, node_level) {
347
+ const level = node.header?.get?.("node_level") ?? node_level;
348
+ if (all_nodes.has(level)) {
349
+ all_nodes.get(level).push(node);
350
+ }
351
+ else {
352
+ all_nodes.set(level, [node]);
353
+ }
354
+ }
@@ -0,0 +1,35 @@
1
+ /**
2
+ * Vendored and adapted from jsfive/esm/core.js.
3
+ * Provides struct unpacking, DataView64, and helper utilities.
4
+ * These are synchronous operations on pre-fetched ArrayBuffers.
5
+ */
6
+ export declare function _unpack_struct_from(structure: Map<string, string>, buf: ArrayBuffer, offset?: number): Map<string, any>;
7
+ export declare function _structure_size(structure: Map<string, string>): number;
8
+ export declare function _padded_size(size: number, padding_multiple?: number): number;
9
+ export declare function dtype_getter(dtype_str: string): [string, boolean, number];
10
+ export declare class Reference {
11
+ address_of_reference: number;
12
+ constructor(address_of_reference: number);
13
+ }
14
+ export declare class DataView64 extends DataView<ArrayBuffer> {
15
+ getFloat16(byteOffset: number, littleEndian: boolean): number;
16
+ getUint64(byteOffset: number, littleEndian: boolean): number;
17
+ getInt64(byteOffset: number, littleEndian: boolean): number;
18
+ getString(byteOffset: number, _littleEndian: boolean, length: number): string;
19
+ getVLENStruct(byteOffset: number, littleEndian: boolean, _length: number): [number, number, number];
20
+ }
21
+ declare class Struct {
22
+ big_endian: boolean;
23
+ getters: Record<string, string>;
24
+ byte_lengths: Record<string, number>;
25
+ fmt_size_regex: string;
26
+ constructor();
27
+ calcsize(fmt: string): number;
28
+ _is_big_endian(fmt: string): boolean;
29
+ unpack_from(fmt: string, buffer: ArrayBuffer, offset?: number): any[];
30
+ }
31
+ export declare const struct: Struct;
32
+ export declare function bitSize(integer: number): number;
33
+ export declare function _unpack_integer(nbytes: number, fh: ArrayBuffer, offset?: number, littleEndian?: boolean): number;
34
+ export declare function assert(thing: any): void;
35
+ export {};