hdf5-as-virtual-zarr 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (229) hide show
  1. package/README.md +17 -1
  2. package/dist/hdf-store.d.ts +9 -0
  3. package/dist/hdf-store.js +9 -0
  4. package/dist/jsfive/dataobjects.js +14 -0
  5. package/dist/jsfive/datatype-msg.d.ts +1 -0
  6. package/dist/jsfive/datatype-msg.js +58 -1
  7. package/dist/ref-store.js +1 -1
  8. package/dist/single-hdf5-to-zarr.js +5 -0
  9. package/package.json +4 -4
  10. package/src/hdf-store.ts +9 -0
  11. package/src/jsfive/dataobjects.ts +12 -0
  12. package/src/jsfive/datatype-msg.ts +67 -1
  13. package/src/ref-store.ts +3 -1
  14. package/src/single-hdf5-to-zarr.ts +6 -0
  15. package/test/fixtures/dense.adata.zarr/.zattrs +0 -4
  16. package/test/fixtures/dense.adata.zarr/.zgroup +0 -3
  17. package/test/fixtures/dense.adata.zarr/.zmetadata +0 -285
  18. package/test/fixtures/dense.adata.zarr/X/.zarray +0 -22
  19. package/test/fixtures/dense.adata.zarr/X/.zattrs +0 -4
  20. package/test/fixtures/dense.adata.zarr/X/0.0 +0 -0
  21. package/test/fixtures/dense.adata.zarr/layers/.zattrs +0 -4
  22. package/test/fixtures/dense.adata.zarr/layers/.zgroup +0 -3
  23. package/test/fixtures/dense.adata.zarr/obs/.zattrs +0 -9
  24. package/test/fixtures/dense.adata.zarr/obs/.zgroup +0 -3
  25. package/test/fixtures/dense.adata.zarr/obs/_index/.zarray +0 -24
  26. package/test/fixtures/dense.adata.zarr/obs/_index/.zattrs +0 -4
  27. package/test/fixtures/dense.adata.zarr/obs/_index/0 +0 -0
  28. package/test/fixtures/dense.adata.zarr/obs/categorical/.zattrs +0 -5
  29. package/test/fixtures/dense.adata.zarr/obs/categorical/.zgroup +0 -3
  30. package/test/fixtures/dense.adata.zarr/obs/categorical/categories/.zarray +0 -24
  31. package/test/fixtures/dense.adata.zarr/obs/categorical/categories/.zattrs +0 -4
  32. package/test/fixtures/dense.adata.zarr/obs/categorical/categories/0 +0 -0
  33. package/test/fixtures/dense.adata.zarr/obs/categorical/codes/.zarray +0 -20
  34. package/test/fixtures/dense.adata.zarr/obs/categorical/codes/.zattrs +0 -4
  35. package/test/fixtures/dense.adata.zarr/obs/categorical/codes/0 +0 -0
  36. package/test/fixtures/dense.adata.zarr/obs/string/.zarray +0 -24
  37. package/test/fixtures/dense.adata.zarr/obs/string/.zattrs +0 -4
  38. package/test/fixtures/dense.adata.zarr/obs/string/0 +0 -0
  39. package/test/fixtures/dense.adata.zarr/obsm/.zattrs +0 -4
  40. package/test/fixtures/dense.adata.zarr/obsm/.zgroup +0 -3
  41. package/test/fixtures/dense.adata.zarr/obsm/X_umap/.zarray +0 -22
  42. package/test/fixtures/dense.adata.zarr/obsm/X_umap/.zattrs +0 -4
  43. package/test/fixtures/dense.adata.zarr/obsm/X_umap/0.0 +0 -0
  44. package/test/fixtures/dense.adata.zarr/obsp/.zattrs +0 -4
  45. package/test/fixtures/dense.adata.zarr/obsp/.zgroup +0 -3
  46. package/test/fixtures/dense.adata.zarr/raw/.zarray +0 -10
  47. package/test/fixtures/dense.adata.zarr/raw/.zattrs +0 -4
  48. package/test/fixtures/dense.adata.zarr/raw/0 +0 -0
  49. package/test/fixtures/dense.adata.zarr/uns/.zattrs +0 -4
  50. package/test/fixtures/dense.adata.zarr/uns/.zgroup +0 -3
  51. package/test/fixtures/dense.adata.zarr/var/.zattrs +0 -6
  52. package/test/fixtures/dense.adata.zarr/var/.zgroup +0 -3
  53. package/test/fixtures/dense.adata.zarr/var/_index/.zarray +0 -24
  54. package/test/fixtures/dense.adata.zarr/var/_index/.zattrs +0 -4
  55. package/test/fixtures/dense.adata.zarr/var/_index/0 +0 -0
  56. package/test/fixtures/dense.adata.zarr/varm/.zattrs +0 -4
  57. package/test/fixtures/dense.adata.zarr/varm/.zgroup +0 -3
  58. package/test/fixtures/dense.adata.zarr/varp/.zattrs +0 -4
  59. package/test/fixtures/dense.adata.zarr/varp/.zgroup +0 -3
  60. package/test/fixtures/dense.adata.zarr.json +0 -182
  61. package/test/fixtures/dense.adata.zmetadata.json +0 -1
  62. package/test/fixtures/dense.h5ad +0 -0
  63. package/test/fixtures/dense.h5ad.refspec.json +0 -1
  64. package/test/fixtures/dense.v3.adata.zarr/X/c/0/0 +0 -0
  65. package/test/fixtures/dense.v3.adata.zarr/X/zarr.json +0 -45
  66. package/test/fixtures/dense.v3.adata.zarr/layers/zarr.json +0 -8
  67. package/test/fixtures/dense.v3.adata.zarr/obs/_index/c/0 +0 -0
  68. package/test/fixtures/dense.v3.adata.zarr/obs/_index/zarr.json +0 -41
  69. package/test/fixtures/dense.v3.adata.zarr/obs/categorical/categories/c/0 +0 -0
  70. package/test/fixtures/dense.v3.adata.zarr/obs/categorical/categories/zarr.json +0 -41
  71. package/test/fixtures/dense.v3.adata.zarr/obs/categorical/codes/c/0 +0 -0
  72. package/test/fixtures/dense.v3.adata.zarr/obs/categorical/codes/zarr.json +0 -40
  73. package/test/fixtures/dense.v3.adata.zarr/obs/categorical/zarr.json +0 -9
  74. package/test/fixtures/dense.v3.adata.zarr/obs/string/c/0 +0 -0
  75. package/test/fixtures/dense.v3.adata.zarr/obs/string/zarr.json +0 -41
  76. package/test/fixtures/dense.v3.adata.zarr/obs/zarr.json +0 -13
  77. package/test/fixtures/dense.v3.adata.zarr/obsm/X_umap/c/0/0 +0 -0
  78. package/test/fixtures/dense.v3.adata.zarr/obsm/X_umap/zarr.json +0 -45
  79. package/test/fixtures/dense.v3.adata.zarr/obsm/zarr.json +0 -8
  80. package/test/fixtures/dense.v3.adata.zarr/obsp/zarr.json +0 -8
  81. package/test/fixtures/dense.v3.adata.zarr/raw/zarr.json +0 -36
  82. package/test/fixtures/dense.v3.adata.zarr/uns/zarr.json +0 -8
  83. package/test/fixtures/dense.v3.adata.zarr/var/_index/c/0 +0 -0
  84. package/test/fixtures/dense.v3.adata.zarr/var/_index/zarr.json +0 -41
  85. package/test/fixtures/dense.v3.adata.zarr/var/zarr.json +0 -10
  86. package/test/fixtures/dense.v3.adata.zarr/varm/zarr.json +0 -8
  87. package/test/fixtures/dense.v3.adata.zarr/varp/zarr.json +0 -8
  88. package/test/fixtures/dense.v3.adata.zarr/zarr.json +0 -469
  89. package/test/fixtures/dense.v3.adata.zarr.json +0 -102
  90. package/test/fixtures/generate_fixtures.py +0 -157
  91. package/test/fixtures/generate_fixtures_v3.py +0 -126
  92. package/test/fixtures/minimal.adata.zarr/.zattrs +0 -4
  93. package/test/fixtures/minimal.adata.zarr/.zgroup +0 -3
  94. package/test/fixtures/minimal.adata.zarr/.zmetadata +0 -168
  95. package/test/fixtures/minimal.adata.zarr/X/.zarray +0 -22
  96. package/test/fixtures/minimal.adata.zarr/X/.zattrs +0 -4
  97. package/test/fixtures/minimal.adata.zarr/X/0.0 +0 -0
  98. package/test/fixtures/minimal.adata.zarr/layers/.zattrs +0 -4
  99. package/test/fixtures/minimal.adata.zarr/layers/.zgroup +0 -3
  100. package/test/fixtures/minimal.adata.zarr/obs/.zattrs +0 -6
  101. package/test/fixtures/minimal.adata.zarr/obs/.zgroup +0 -3
  102. package/test/fixtures/minimal.adata.zarr/obs/_index/.zarray +0 -24
  103. package/test/fixtures/minimal.adata.zarr/obs/_index/.zattrs +0 -4
  104. package/test/fixtures/minimal.adata.zarr/obs/_index/0 +0 -0
  105. package/test/fixtures/minimal.adata.zarr/obsm/.zattrs +0 -4
  106. package/test/fixtures/minimal.adata.zarr/obsm/.zgroup +0 -3
  107. package/test/fixtures/minimal.adata.zarr/obsp/.zattrs +0 -4
  108. package/test/fixtures/minimal.adata.zarr/obsp/.zgroup +0 -3
  109. package/test/fixtures/minimal.adata.zarr/raw/.zarray +0 -10
  110. package/test/fixtures/minimal.adata.zarr/raw/.zattrs +0 -4
  111. package/test/fixtures/minimal.adata.zarr/raw/0 +0 -0
  112. package/test/fixtures/minimal.adata.zarr/uns/.zattrs +0 -4
  113. package/test/fixtures/minimal.adata.zarr/uns/.zgroup +0 -3
  114. package/test/fixtures/minimal.adata.zarr/var/.zattrs +0 -6
  115. package/test/fixtures/minimal.adata.zarr/var/.zgroup +0 -3
  116. package/test/fixtures/minimal.adata.zarr/var/_index/.zarray +0 -24
  117. package/test/fixtures/minimal.adata.zarr/var/_index/.zattrs +0 -4
  118. package/test/fixtures/minimal.adata.zarr/var/_index/0 +0 -0
  119. package/test/fixtures/minimal.adata.zarr/varm/.zattrs +0 -4
  120. package/test/fixtures/minimal.adata.zarr/varm/.zgroup +0 -3
  121. package/test/fixtures/minimal.adata.zarr/varp/.zattrs +0 -4
  122. package/test/fixtures/minimal.adata.zarr/varp/.zgroup +0 -3
  123. package/test/fixtures/minimal.adata.zarr.json +0 -126
  124. package/test/fixtures/minimal.adata.zmetadata.json +0 -1
  125. package/test/fixtures/minimal.h5ad +0 -0
  126. package/test/fixtures/minimal.h5ad.refspec.json +0 -1
  127. package/test/fixtures/minimal.v3.adata.zarr/X/c/0/0 +0 -0
  128. package/test/fixtures/minimal.v3.adata.zarr/X/zarr.json +0 -45
  129. package/test/fixtures/minimal.v3.adata.zarr/layers/zarr.json +0 -8
  130. package/test/fixtures/minimal.v3.adata.zarr/obs/_index/c/0 +0 -0
  131. package/test/fixtures/minimal.v3.adata.zarr/obs/_index/zarr.json +0 -41
  132. package/test/fixtures/minimal.v3.adata.zarr/obs/zarr.json +0 -10
  133. package/test/fixtures/minimal.v3.adata.zarr/obsm/zarr.json +0 -8
  134. package/test/fixtures/minimal.v3.adata.zarr/obsp/zarr.json +0 -8
  135. package/test/fixtures/minimal.v3.adata.zarr/raw/zarr.json +0 -36
  136. package/test/fixtures/minimal.v3.adata.zarr/uns/zarr.json +0 -8
  137. package/test/fixtures/minimal.v3.adata.zarr/var/_index/c/0 +0 -0
  138. package/test/fixtures/minimal.v3.adata.zarr/var/_index/zarr.json +0 -41
  139. package/test/fixtures/minimal.v3.adata.zarr/var/zarr.json +0 -10
  140. package/test/fixtures/minimal.v3.adata.zarr/varm/zarr.json +0 -8
  141. package/test/fixtures/minimal.v3.adata.zarr/varp/zarr.json +0 -8
  142. package/test/fixtures/minimal.v3.adata.zarr/zarr.json +0 -285
  143. package/test/fixtures/minimal.v3.adata.zarr.json +0 -66
  144. package/test/fixtures/mouse_liver.h5ad +0 -0
  145. package/test/fixtures/mouse_liver.h5ad.refspec.json +0 -1
  146. package/test/fixtures/mouse_liver.py +0 -44
  147. package/test/fixtures/sparse.adata.zarr/.zattrs +0 -4
  148. package/test/fixtures/sparse.adata.zarr/.zgroup +0 -3
  149. package/test/fixtures/sparse.adata.zarr/.zmetadata +0 -304
  150. package/test/fixtures/sparse.adata.zarr/X/.zattrs +0 -8
  151. package/test/fixtures/sparse.adata.zarr/X/.zgroup +0 -3
  152. package/test/fixtures/sparse.adata.zarr/X/data/.zarray +0 -20
  153. package/test/fixtures/sparse.adata.zarr/X/data/0 +0 -0
  154. package/test/fixtures/sparse.adata.zarr/X/indices/.zarray +0 -20
  155. package/test/fixtures/sparse.adata.zarr/X/indices/0 +0 -0
  156. package/test/fixtures/sparse.adata.zarr/X/indptr/.zarray +0 -20
  157. package/test/fixtures/sparse.adata.zarr/X/indptr/0 +0 -0
  158. package/test/fixtures/sparse.adata.zarr/layers/.zattrs +0 -4
  159. package/test/fixtures/sparse.adata.zarr/layers/.zgroup +0 -3
  160. package/test/fixtures/sparse.adata.zarr/obs/.zattrs +0 -9
  161. package/test/fixtures/sparse.adata.zarr/obs/.zgroup +0 -3
  162. package/test/fixtures/sparse.adata.zarr/obs/_index/.zarray +0 -24
  163. package/test/fixtures/sparse.adata.zarr/obs/_index/.zattrs +0 -4
  164. package/test/fixtures/sparse.adata.zarr/obs/_index/0 +0 -0
  165. package/test/fixtures/sparse.adata.zarr/obs/categorical/.zattrs +0 -5
  166. package/test/fixtures/sparse.adata.zarr/obs/categorical/.zgroup +0 -3
  167. package/test/fixtures/sparse.adata.zarr/obs/categorical/categories/.zarray +0 -24
  168. package/test/fixtures/sparse.adata.zarr/obs/categorical/categories/.zattrs +0 -4
  169. package/test/fixtures/sparse.adata.zarr/obs/categorical/categories/0 +0 -0
  170. package/test/fixtures/sparse.adata.zarr/obs/categorical/codes/.zarray +0 -20
  171. package/test/fixtures/sparse.adata.zarr/obs/categorical/codes/.zattrs +0 -4
  172. package/test/fixtures/sparse.adata.zarr/obs/categorical/codes/0 +0 -0
  173. package/test/fixtures/sparse.adata.zarr/obs/string/.zarray +0 -24
  174. package/test/fixtures/sparse.adata.zarr/obs/string/.zattrs +0 -4
  175. package/test/fixtures/sparse.adata.zarr/obs/string/0 +0 -0
  176. package/test/fixtures/sparse.adata.zarr/obsm/.zattrs +0 -4
  177. package/test/fixtures/sparse.adata.zarr/obsm/.zgroup +0 -3
  178. package/test/fixtures/sparse.adata.zarr/obsp/.zattrs +0 -4
  179. package/test/fixtures/sparse.adata.zarr/obsp/.zgroup +0 -3
  180. package/test/fixtures/sparse.adata.zarr/raw/.zarray +0 -10
  181. package/test/fixtures/sparse.adata.zarr/raw/.zattrs +0 -4
  182. package/test/fixtures/sparse.adata.zarr/raw/0 +0 -0
  183. package/test/fixtures/sparse.adata.zarr/uns/.zattrs +0 -4
  184. package/test/fixtures/sparse.adata.zarr/uns/.zgroup +0 -3
  185. package/test/fixtures/sparse.adata.zarr/var/.zattrs +0 -6
  186. package/test/fixtures/sparse.adata.zarr/var/.zgroup +0 -3
  187. package/test/fixtures/sparse.adata.zarr/var/_index/.zarray +0 -24
  188. package/test/fixtures/sparse.adata.zarr/var/_index/.zattrs +0 -4
  189. package/test/fixtures/sparse.adata.zarr/var/_index/0 +0 -0
  190. package/test/fixtures/sparse.adata.zarr/varm/.zattrs +0 -4
  191. package/test/fixtures/sparse.adata.zarr/varm/.zgroup +0 -3
  192. package/test/fixtures/sparse.adata.zarr/varp/.zattrs +0 -4
  193. package/test/fixtures/sparse.adata.zarr/varp/.zgroup +0 -3
  194. package/test/fixtures/sparse.adata.zarr.json +0 -190
  195. package/test/fixtures/sparse.adata.zmetadata.json +0 -1
  196. package/test/fixtures/sparse.h5ad +0 -0
  197. package/test/fixtures/sparse.h5ad.refspec.json +0 -1
  198. package/test/fixtures/sparse.v3.adata.zarr/X/data/c/0 +0 -0
  199. package/test/fixtures/sparse.v3.adata.zarr/X/data/zarr.json +0 -40
  200. package/test/fixtures/sparse.v3.adata.zarr/X/indices/c/0 +0 -0
  201. package/test/fixtures/sparse.v3.adata.zarr/X/indices/zarr.json +0 -40
  202. package/test/fixtures/sparse.v3.adata.zarr/X/indptr/c/0 +0 -0
  203. package/test/fixtures/sparse.v3.adata.zarr/X/indptr/zarr.json +0 -40
  204. package/test/fixtures/sparse.v3.adata.zarr/X/zarr.json +0 -12
  205. package/test/fixtures/sparse.v3.adata.zarr/layers/zarr.json +0 -8
  206. package/test/fixtures/sparse.v3.adata.zarr/obs/_index/c/0 +0 -0
  207. package/test/fixtures/sparse.v3.adata.zarr/obs/_index/zarr.json +0 -41
  208. package/test/fixtures/sparse.v3.adata.zarr/obs/categorical/categories/c/0 +0 -0
  209. package/test/fixtures/sparse.v3.adata.zarr/obs/categorical/categories/zarr.json +0 -41
  210. package/test/fixtures/sparse.v3.adata.zarr/obs/categorical/codes/c/0 +0 -0
  211. package/test/fixtures/sparse.v3.adata.zarr/obs/categorical/codes/zarr.json +0 -40
  212. package/test/fixtures/sparse.v3.adata.zarr/obs/categorical/zarr.json +0 -9
  213. package/test/fixtures/sparse.v3.adata.zarr/obs/string/c/0 +0 -0
  214. package/test/fixtures/sparse.v3.adata.zarr/obs/string/zarr.json +0 -41
  215. package/test/fixtures/sparse.v3.adata.zarr/obs/zarr.json +0 -13
  216. package/test/fixtures/sparse.v3.adata.zarr/obsm/zarr.json +0 -8
  217. package/test/fixtures/sparse.v3.adata.zarr/obsp/zarr.json +0 -8
  218. package/test/fixtures/sparse.v3.adata.zarr/raw/zarr.json +0 -36
  219. package/test/fixtures/sparse.v3.adata.zarr/uns/zarr.json +0 -8
  220. package/test/fixtures/sparse.v3.adata.zarr/var/_index/c/0 +0 -0
  221. package/test/fixtures/sparse.v3.adata.zarr/var/_index/zarr.json +0 -41
  222. package/test/fixtures/sparse.v3.adata.zarr/var/zarr.json +0 -10
  223. package/test/fixtures/sparse.v3.adata.zarr/varm/zarr.json +0 -8
  224. package/test/fixtures/sparse.v3.adata.zarr/varp/zarr.json +0 -8
  225. package/test/fixtures/sparse.v3.adata.zarr/zarr.json +0 -516
  226. package/test/fixtures/sparse.v3.adata.zarr.json +0 -114
  227. package/test/hdf-store.test.ts +0 -268
  228. package/test/index.test.ts +0 -762
  229. package/test/partial-reads.test.ts +0 -150
package/README.md CHANGED
@@ -1,4 +1,6 @@
1
- # hdf5-as-virtual-zarr
1
+ # hdf5-as-virtual-zarr.js
2
+
3
+ [![NPM](https://img.shields.io/npm/v/hdf5-as-virtual-zarr.svg?color=black)](https://www.npmjs.com/package/hdf5-as-virtual-zarr)
2
4
 
3
5
  Given an input HDF5 file URL, generate a Zarr References Specification JSON output.
4
6
 
@@ -8,6 +10,20 @@ It supports large HDF5 files using range requests (fetching only the subset of b
8
10
 
9
11
  This is a port of [SingleHdf5ToZarr](https://github.com/fsspec/kerchunk/blob/main/kerchunk/hdf.py#L52) and its `translate` function from the `kerchunk` Python package.
10
12
 
13
+ ## Usage
14
+
15
+ ```ts
16
+ import { open, get, root } from "zarrita";
17
+ import { FetchStore } from "@zarrita/storage";
18
+ import { HdfStore } from "hdf5-as-virtual-zarr";
19
+
20
+ const internalStore = new FetchStore("https://example.com/data.h5");
21
+ const hdfStore = await HdfStore.fromStore(internalStore);
22
+
23
+ const storeRoot = root(hdfStore);
24
+ const arr = await open(storeRoot.resolve("X"), { kind: "array" });
25
+ const arrData = await get(arr);
26
+ ```
11
27
 
12
28
  ## Development
13
29
 
@@ -5,5 +5,14 @@ export declare class HdfStore implements AsyncReadable {
5
5
  private refStore;
6
6
  constructor(refStore: ComposableReferenceStore);
7
7
  get(key: AbsolutePath, opts?: RequestInit): Promise<Uint8Array | undefined>;
8
+ /**
9
+ *
10
+ * @param {AsyncReadable} hdfSource Assumed to be a zarrita Store,
11
+ * with the HDF5 file located at the root key "/". In other words,
12
+ * hdfSource.getRange("/", { offset, length }) should return the
13
+ * corresponding byte range from the HDF5 file.
14
+ * @returns {Promise<HdfStore>} An HdfStore instance which internally
15
+ * uses a ReferenceStore and handles the Reference Specification JSON generation.
16
+ */
8
17
  static fromStore(hdfSource: AsyncReadable): Promise<HdfStore>;
9
18
  }
package/dist/hdf-store.js CHANGED
@@ -9,6 +9,15 @@ export class HdfStore {
9
9
  return this.refStore.get(key, opts);
10
10
  }
11
11
  // TODO: does it make sense to support getRange here?
12
+ /**
13
+ *
14
+ * @param {AsyncReadable} hdfSource Assumed to be a zarrita Store,
15
+ * with the HDF5 file located at the root key "/". In other words,
16
+ * hdfSource.getRange("/", { offset, length }) should return the
17
+ * corresponding byte range from the HDF5 file.
18
+ * @returns {Promise<HdfStore>} An HdfStore instance which internally
19
+ * uses a ReferenceStore and handles the Reference Specification JSON generation.
20
+ */
12
21
  static async fromStore(hdfSource) {
13
22
  const converter = new SingleHdf5ToZarr(hdfSource, { url: null });
14
23
  const refSpec = await converter.translate();
@@ -853,6 +853,11 @@ export class DataObjects {
853
853
  value[i] = await this._attr_value(base_dtype, vlen_data, vlen, 0);
854
854
  offset += 16;
855
855
  }
856
+ else if (dtype_class === "COMPOUND") {
857
+ const total_size = dtype[2];
858
+ value[i] = buf.slice(offset, offset + total_size);
859
+ offset += total_size;
860
+ }
856
861
  else {
857
862
  throw new Error("NotImplementedError: unsupported dtype class " + dtype_class);
858
863
  }
@@ -953,6 +958,15 @@ export class DataObjects {
953
958
  }
954
959
  return output;
955
960
  }
961
+ else if (dtype_class === "COMPOUND") {
962
+ const total_size = dtype[2];
963
+ const dataBuf = await fetchRange(this._source, data_offset, fullsize * total_size);
964
+ const output = new Array(fullsize);
965
+ for (let i = 0; i < fullsize; i++) {
966
+ output[i] = dataBuf.slice(i * total_size, (i + 1) * total_size);
967
+ }
968
+ return output;
969
+ }
956
970
  else {
957
971
  throw new Error("Not implemented datatype class: " + dtype_class);
958
972
  }
@@ -11,5 +11,6 @@ export declare class DatatypeMessage {
11
11
  _determine_dtype_fixed_point(datatype_msg: Map<string, any>): string;
12
12
  _determine_dtype_floating_point(datatype_msg: Map<string, any>): string;
13
13
  _determine_dtype_string(datatype_msg: Map<string, any>): string;
14
+ _determine_dtype_compound(datatype_msg: Map<string, any>): any[];
14
15
  _determine_dtype_vlen(datatype_msg: Map<string, any>): any[];
15
16
  }
@@ -53,7 +53,7 @@ export class DatatypeMessage {
53
53
  throw new Error("Opaque datatype class not supported.");
54
54
  }
55
55
  else if (datatype_class === DATATYPE_COMPOUND) {
56
- throw new Error("Compound datatype class not supported.");
56
+ return this._determine_dtype_compound(datatype_msg);
57
57
  }
58
58
  else if (datatype_class === DATATYPE_REFERENCE) {
59
59
  return ["REFERENCE", datatype_msg.get("size")];
@@ -98,6 +98,63 @@ export class DatatypeMessage {
98
98
  _determine_dtype_string(datatype_msg) {
99
99
  return "S" + datatype_msg.get("size").toFixed();
100
100
  }
101
+ _determine_dtype_compound(datatype_msg) {
102
+ const version = (datatype_msg.get("class_and_version") >> 4) & 0x0f;
103
+ const num_members = datatype_msg.get("class_bit_field_0") |
104
+ (datatype_msg.get("class_bit_field_1") << 8);
105
+ const total_size = datatype_msg.get("size");
106
+ // Return minimal descriptor if we can't parse members — processDataset will skip this anyway
107
+ try {
108
+ const members = [];
109
+ const view = new DataView(this.buf);
110
+ for (let i = 0; i < num_members; i++) {
111
+ // Read null-terminated name
112
+ let nameEnd = this.offset;
113
+ while (nameEnd < this.buf.byteLength && view.getUint8(nameEnd) !== 0) {
114
+ nameEnd++;
115
+ }
116
+ const nameBytes = new Uint8Array(this.buf, this.offset, nameEnd - this.offset);
117
+ const name = new TextDecoder().decode(nameBytes);
118
+ let memberOffset;
119
+ if (version === 1) {
120
+ // Version 1: name padded to 8-byte boundary
121
+ const nameLenWithNull = nameEnd - this.offset + 1;
122
+ const paddedLen = Math.ceil(nameLenWithNull / 8) * 8;
123
+ this.offset += paddedLen;
124
+ // Read byte offset of member (4 bytes)
125
+ memberOffset = view.getUint32(this.offset, true);
126
+ this.offset += 4;
127
+ // Skip: dimensionality (1) + reserved (3) + permutation (4) + reserved (4) + dim sizes (16) = 28 bytes
128
+ this.offset += 28;
129
+ }
130
+ else if (version === 2) {
131
+ // Version 2: null-terminated name, no padding
132
+ this.offset = nameEnd + 1;
133
+ // Read byte offset (4 bytes)
134
+ memberOffset = view.getUint32(this.offset, true);
135
+ this.offset += 4;
136
+ }
137
+ else {
138
+ // Version 3: null-terminated name, variable-width offset
139
+ this.offset = nameEnd + 1;
140
+ const offsetBytes = Math.max(1, Math.ceil(Math.ceil(Math.log2(total_size + 1)) / 8));
141
+ memberOffset = 0;
142
+ for (let j = 0; j < offsetBytes; j++) {
143
+ memberOffset |= view.getUint8(this.offset + j) << (j * 8);
144
+ }
145
+ this.offset += offsetBytes;
146
+ }
147
+ // Parse nested datatype (recursive) — updates this.offset
148
+ const nestedMsg = new DatatypeMessage(this.buf, this.offset);
149
+ this.offset = nestedMsg.offset;
150
+ members.push({ name, dtype: nestedMsg.dtype, offset: memberOffset });
151
+ }
152
+ return ["COMPOUND", members, total_size];
153
+ }
154
+ catch {
155
+ return ["COMPOUND", [], total_size];
156
+ }
157
+ }
101
158
  _determine_dtype_vlen(datatype_msg) {
102
159
  const vlen_type = datatype_msg.get("class_bit_field_0") & 0x01;
103
160
  if (vlen_type !== 1) {
package/dist/ref-store.js CHANGED
@@ -68,7 +68,7 @@ export class ComposableReferenceStore {
68
68
  return this.#internal_store.get(key, opts);
69
69
  }
70
70
  const opts_for_get = this.#overrides ? merge_init(this.#overrides, opts) : opts;
71
- return this.#internal_store.getRange(key, { offset, length: size }, opts_for_get);
71
+ return this.#internal_store.getRange("/", { offset, length: size }, opts_for_get);
72
72
  }
73
73
  // TODO: does it make sense to support getRange here? get already calls getRange on the internal store...
74
74
  static fromSpec(spec, internal_store, overrides) {
@@ -446,6 +446,11 @@ export class SingleHdf5ToZarr {
446
446
  const isVlen = Array.isArray(rawDtype) &&
447
447
  (rawDtype[0] === "VLEN_STRING" || rawDtype[0] === "VLEN_SEQUENCE");
448
448
  const isFixedString = typeof rawDtype === "string" && /^S\d+/.test(rawDtype);
449
+ const isCompound = Array.isArray(rawDtype) && rawDtype[0] === "COMPOUND";
450
+ // Compound datasets cannot be represented in Zarr; skip them
451
+ if (isCompound) {
452
+ return;
453
+ }
449
454
  const dtype = jsfiveDtypeToZarr(rawDtype);
450
455
  const chunks = dataobjects.chunks || [...shape];
451
456
  const filterPipeline = dataobjects.filter_pipeline;
package/package.json CHANGED
@@ -1,14 +1,13 @@
1
1
  {
2
2
  "name": "hdf5-as-virtual-zarr",
3
- "version": "0.1.0",
3
+ "version": "0.1.1",
4
4
  "description": "Given an input HDF5 file URL, generate a Zarr References Specification JSON output.",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",
7
7
  "types": "dist/index.d.ts",
8
8
  "files": [
9
9
  "dist",
10
- "src",
11
- "test"
10
+ "src"
12
11
  ],
13
12
  "exports": {
14
13
  ".": {
@@ -36,7 +35,8 @@
36
35
  "typescript": "^5.7.0",
37
36
  "vite": "^7.3.1",
38
37
  "vitest": "^3.0.0",
39
- "zarrita": "^0.6.1"
38
+ "zarrita": "^0.6.1",
39
+ "anndata.js": "0.0.2"
40
40
  },
41
41
  "scripts": {
42
42
  "build": "tsc",
package/src/hdf-store.ts CHANGED
@@ -16,6 +16,15 @@ export class HdfStore implements AsyncReadable {
16
16
 
17
17
  // TODO: does it make sense to support getRange here?
18
18
 
19
+ /**
20
+ *
21
+ * @param {AsyncReadable} hdfSource Assumed to be a zarrita Store,
22
+ * with the HDF5 file located at the root key "/". In other words,
23
+ * hdfSource.getRange("/", { offset, length }) should return the
24
+ * corresponding byte range from the HDF5 file.
25
+ * @returns {Promise<HdfStore>} An HdfStore instance which internally
26
+ * uses a ReferenceStore and handles the Reference Specification JSON generation.
27
+ */
19
28
  static async fromStore(hdfSource: AsyncReadable): Promise<HdfStore> {
20
29
  const converter = new SingleHdf5ToZarr(hdfSource, { url: null });
21
30
  const refSpec = await converter.translate();
@@ -1091,6 +1091,10 @@ export class DataObjects {
1091
1091
  const [vlen, vlen_data] = await this._vlen_size_and_data(buf, offset);
1092
1092
  value[i] = await this._attr_value(base_dtype, vlen_data, vlen, 0);
1093
1093
  offset += 16;
1094
+ } else if (dtype_class === "COMPOUND") {
1095
+ const total_size = dtype[2] as number;
1096
+ value[i] = buf.slice(offset, offset + total_size);
1097
+ offset += total_size;
1094
1098
  } else {
1095
1099
  throw new Error("NotImplementedError: unsupported dtype class " + dtype_class);
1096
1100
  }
@@ -1201,6 +1205,14 @@ export class DataObjects {
1201
1205
  output[i] = decoder.decode(vlen_data);
1202
1206
  }
1203
1207
  return output;
1208
+ } else if (dtype_class === "COMPOUND") {
1209
+ const total_size = dtype[2] as number;
1210
+ const dataBuf = await fetchRange(this._source, data_offset, fullsize * total_size);
1211
+ const output: ArrayBuffer[] = new Array(fullsize);
1212
+ for (let i = 0; i < fullsize; i++) {
1213
+ output[i] = dataBuf.slice(i * total_size, (i + 1) * total_size);
1214
+ }
1215
+ return output;
1204
1216
  } else {
1205
1217
  throw new Error("Not implemented datatype class: " + dtype_class);
1206
1218
  }
@@ -54,7 +54,7 @@ export class DatatypeMessage {
54
54
  } else if (datatype_class === DATATYPE_OPAQUE) {
55
55
  throw new Error("Opaque datatype class not supported.");
56
56
  } else if (datatype_class === DATATYPE_COMPOUND) {
57
- throw new Error("Compound datatype class not supported.");
57
+ return this._determine_dtype_compound(datatype_msg);
58
58
  } else if (datatype_class === DATATYPE_REFERENCE) {
59
59
  return ["REFERENCE", datatype_msg.get("size")];
60
60
  } else if (datatype_class === DATATYPE_ENUMERATED) {
@@ -99,6 +99,72 @@ export class DatatypeMessage {
99
99
  return "S" + datatype_msg.get("size").toFixed();
100
100
  }
101
101
 
102
+ _determine_dtype_compound(datatype_msg: Map<string, any>): any[] {
103
+ const version = (datatype_msg.get("class_and_version") >> 4) & 0x0f;
104
+ const num_members = datatype_msg.get("class_bit_field_0") |
105
+ (datatype_msg.get("class_bit_field_1") << 8);
106
+ const total_size = datatype_msg.get("size");
107
+
108
+ // Return minimal descriptor if we can't parse members — processDataset will skip this anyway
109
+ try {
110
+ const members: Array<{name: string; dtype: any; offset: number}> = [];
111
+ const view = new DataView(this.buf);
112
+
113
+ for (let i = 0; i < num_members; i++) {
114
+ // Read null-terminated name
115
+ let nameEnd = this.offset;
116
+ while (nameEnd < this.buf.byteLength && view.getUint8(nameEnd) !== 0) {
117
+ nameEnd++;
118
+ }
119
+ const nameBytes = new Uint8Array(this.buf, this.offset, nameEnd - this.offset);
120
+ const name = new TextDecoder().decode(nameBytes);
121
+
122
+ let memberOffset: number;
123
+
124
+ if (version === 1) {
125
+ // Version 1: name padded to 8-byte boundary
126
+ const nameLenWithNull = nameEnd - this.offset + 1;
127
+ const paddedLen = Math.ceil(nameLenWithNull / 8) * 8;
128
+ this.offset += paddedLen;
129
+
130
+ // Read byte offset of member (4 bytes)
131
+ memberOffset = view.getUint32(this.offset, true);
132
+ this.offset += 4;
133
+
134
+ // Skip: dimensionality (1) + reserved (3) + permutation (4) + reserved (4) + dim sizes (16) = 28 bytes
135
+ this.offset += 28;
136
+ } else if (version === 2) {
137
+ // Version 2: null-terminated name, no padding
138
+ this.offset = nameEnd + 1;
139
+
140
+ // Read byte offset (4 bytes)
141
+ memberOffset = view.getUint32(this.offset, true);
142
+ this.offset += 4;
143
+ } else {
144
+ // Version 3: null-terminated name, variable-width offset
145
+ this.offset = nameEnd + 1;
146
+
147
+ const offsetBytes = Math.max(1, Math.ceil(Math.ceil(Math.log2(total_size + 1)) / 8));
148
+ memberOffset = 0;
149
+ for (let j = 0; j < offsetBytes; j++) {
150
+ memberOffset |= view.getUint8(this.offset + j) << (j * 8);
151
+ }
152
+ this.offset += offsetBytes;
153
+ }
154
+
155
+ // Parse nested datatype (recursive) — updates this.offset
156
+ const nestedMsg = new DatatypeMessage(this.buf, this.offset);
157
+ this.offset = nestedMsg.offset;
158
+
159
+ members.push({ name, dtype: nestedMsg.dtype, offset: memberOffset });
160
+ }
161
+
162
+ return ["COMPOUND", members, total_size];
163
+ } catch {
164
+ return ["COMPOUND", [], total_size];
165
+ }
166
+ }
167
+
102
168
  _determine_dtype_vlen(datatype_msg: Map<string, any>): any[] {
103
169
  const vlen_type = datatype_msg.get("class_bit_field_0") & 0x01;
104
170
  if (vlen_type !== 1) {
package/src/ref-store.ts CHANGED
@@ -93,12 +93,14 @@ export class ComposableReferenceStore implements AsyncReadable<RequestInit> {
93
93
  let [urlOrNull, offset, size] = ref;
94
94
 
95
95
  if (offset === undefined || size === undefined || !this.#internal_store.getRange) {
96
+ // TODO: is this correct? should it be .get("/", opts) instead?
97
+ // TODO: throw error in this case?
96
98
  return this.#internal_store.get(key, opts);
97
99
  }
98
100
 
99
101
  const opts_for_get = this.#overrides ? merge_init(this.#overrides, opts) : opts;
100
102
 
101
- return this.#internal_store.getRange(key, { offset, length: size }, opts_for_get);
103
+ return this.#internal_store.getRange("/", { offset, length: size }, opts_for_get);
102
104
  }
103
105
 
104
106
  // TODO: does it make sense to support getRange here? get already calls getRange on the internal store...
@@ -528,6 +528,12 @@ export class SingleHdf5ToZarr {
528
528
  const isVlen = Array.isArray(rawDtype) &&
529
529
  (rawDtype[0] === "VLEN_STRING" || rawDtype[0] === "VLEN_SEQUENCE");
530
530
  const isFixedString = typeof rawDtype === "string" && /^S\d+/.test(rawDtype);
531
+ const isCompound = Array.isArray(rawDtype) && rawDtype[0] === "COMPOUND";
532
+
533
+ // Compound datasets cannot be represented in Zarr; skip them
534
+ if (isCompound) {
535
+ return;
536
+ }
531
537
 
532
538
  const dtype = jsfiveDtypeToZarr(rawDtype);
533
539
  const chunks = dataobjects.chunks || [...shape];
@@ -1,4 +0,0 @@
1
- {
2
- "encoding-type": "anndata",
3
- "encoding-version": "0.1.0"
4
- }
@@ -1,3 +0,0 @@
1
- {
2
- "zarr_format": 2
3
- }
@@ -1,285 +0,0 @@
1
- {
2
- "metadata": {
3
- ".zattrs": {
4
- "encoding-type": "anndata",
5
- "encoding-version": "0.1.0"
6
- },
7
- ".zgroup": {
8
- "zarr_format": 2
9
- },
10
- "X/.zarray": {
11
- "chunks": [
12
- 50,
13
- 25
14
- ],
15
- "compressor": {
16
- "blocksize": 0,
17
- "clevel": 5,
18
- "cname": "lz4",
19
- "id": "blosc",
20
- "shuffle": 1
21
- },
22
- "dtype": "<f4",
23
- "fill_value": 0.0,
24
- "filters": null,
25
- "order": "C",
26
- "shape": [
27
- 50,
28
- 25
29
- ],
30
- "zarr_format": 2
31
- },
32
- "X/.zattrs": {
33
- "encoding-type": "array",
34
- "encoding-version": "0.2.0"
35
- },
36
- "layers/.zattrs": {
37
- "encoding-type": "dict",
38
- "encoding-version": "0.1.0"
39
- },
40
- "layers/.zgroup": {
41
- "zarr_format": 2
42
- },
43
- "obs/.zattrs": {
44
- "_index": "_index",
45
- "column-order": [
46
- "categorical",
47
- "string"
48
- ],
49
- "encoding-type": "dataframe",
50
- "encoding-version": "0.2.0"
51
- },
52
- "obs/.zgroup": {
53
- "zarr_format": 2
54
- },
55
- "obs/_index/.zarray": {
56
- "chunks": [
57
- 50
58
- ],
59
- "compressor": {
60
- "blocksize": 0,
61
- "clevel": 5,
62
- "cname": "lz4",
63
- "id": "blosc",
64
- "shuffle": 1
65
- },
66
- "dtype": "|O",
67
- "fill_value": 0,
68
- "filters": [
69
- {
70
- "id": "vlen-utf8"
71
- }
72
- ],
73
- "order": "C",
74
- "shape": [
75
- 50
76
- ],
77
- "zarr_format": 2
78
- },
79
- "obs/_index/.zattrs": {
80
- "encoding-type": "string-array",
81
- "encoding-version": "0.2.0"
82
- },
83
- "obs/categorical/.zattrs": {
84
- "encoding-type": "categorical",
85
- "encoding-version": "0.2.0",
86
- "ordered": false
87
- },
88
- "obs/categorical/.zgroup": {
89
- "zarr_format": 2
90
- },
91
- "obs/categorical/categories/.zarray": {
92
- "chunks": [
93
- 5
94
- ],
95
- "compressor": {
96
- "blocksize": 0,
97
- "clevel": 5,
98
- "cname": "lz4",
99
- "id": "blosc",
100
- "shuffle": 1
101
- },
102
- "dtype": "|O",
103
- "fill_value": 0,
104
- "filters": [
105
- {
106
- "id": "vlen-utf8"
107
- }
108
- ],
109
- "order": "C",
110
- "shape": [
111
- 5
112
- ],
113
- "zarr_format": 2
114
- },
115
- "obs/categorical/categories/.zattrs": {
116
- "encoding-type": "string-array",
117
- "encoding-version": "0.2.0"
118
- },
119
- "obs/categorical/codes/.zarray": {
120
- "chunks": [
121
- 50
122
- ],
123
- "compressor": {
124
- "blocksize": 0,
125
- "clevel": 5,
126
- "cname": "lz4",
127
- "id": "blosc",
128
- "shuffle": 1
129
- },
130
- "dtype": "|i1",
131
- "fill_value": 0,
132
- "filters": null,
133
- "order": "C",
134
- "shape": [
135
- 50
136
- ],
137
- "zarr_format": 2
138
- },
139
- "obs/categorical/codes/.zattrs": {
140
- "encoding-type": "array",
141
- "encoding-version": "0.2.0"
142
- },
143
- "obs/string/.zarray": {
144
- "chunks": [
145
- 50
146
- ],
147
- "compressor": {
148
- "blocksize": 0,
149
- "clevel": 5,
150
- "cname": "lz4",
151
- "id": "blosc",
152
- "shuffle": 1
153
- },
154
- "dtype": "|O",
155
- "fill_value": 0,
156
- "filters": [
157
- {
158
- "id": "vlen-utf8"
159
- }
160
- ],
161
- "order": "C",
162
- "shape": [
163
- 50
164
- ],
165
- "zarr_format": 2
166
- },
167
- "obs/string/.zattrs": {
168
- "encoding-type": "string-array",
169
- "encoding-version": "0.2.0"
170
- },
171
- "obsm/.zattrs": {
172
- "encoding-type": "dict",
173
- "encoding-version": "0.1.0"
174
- },
175
- "obsm/.zgroup": {
176
- "zarr_format": 2
177
- },
178
- "obsm/X_umap/.zarray": {
179
- "chunks": [
180
- 50,
181
- 2
182
- ],
183
- "compressor": {
184
- "blocksize": 0,
185
- "clevel": 5,
186
- "cname": "lz4",
187
- "id": "blosc",
188
- "shuffle": 1
189
- },
190
- "dtype": "<f4",
191
- "fill_value": 0.0,
192
- "filters": null,
193
- "order": "C",
194
- "shape": [
195
- 50,
196
- 2
197
- ],
198
- "zarr_format": 2
199
- },
200
- "obsm/X_umap/.zattrs": {
201
- "encoding-type": "array",
202
- "encoding-version": "0.2.0"
203
- },
204
- "obsp/.zattrs": {
205
- "encoding-type": "dict",
206
- "encoding-version": "0.1.0"
207
- },
208
- "obsp/.zgroup": {
209
- "zarr_format": 2
210
- },
211
- "raw/.zarray": {
212
- "chunks": [],
213
- "compressor": null,
214
- "dtype": "<f8",
215
- "fill_value": 0.0,
216
- "filters": null,
217
- "order": "C",
218
- "shape": [],
219
- "zarr_format": 2
220
- },
221
- "raw/.zattrs": {
222
- "encoding-type": "null",
223
- "encoding-version": "0.1.0"
224
- },
225
- "uns/.zattrs": {
226
- "encoding-type": "dict",
227
- "encoding-version": "0.1.0"
228
- },
229
- "uns/.zgroup": {
230
- "zarr_format": 2
231
- },
232
- "var/.zattrs": {
233
- "_index": "_index",
234
- "column-order": [],
235
- "encoding-type": "dataframe",
236
- "encoding-version": "0.2.0"
237
- },
238
- "var/.zgroup": {
239
- "zarr_format": 2
240
- },
241
- "var/_index/.zarray": {
242
- "chunks": [
243
- 25
244
- ],
245
- "compressor": {
246
- "blocksize": 0,
247
- "clevel": 5,
248
- "cname": "lz4",
249
- "id": "blosc",
250
- "shuffle": 1
251
- },
252
- "dtype": "|O",
253
- "fill_value": 0,
254
- "filters": [
255
- {
256
- "id": "vlen-utf8"
257
- }
258
- ],
259
- "order": "C",
260
- "shape": [
261
- 25
262
- ],
263
- "zarr_format": 2
264
- },
265
- "var/_index/.zattrs": {
266
- "encoding-type": "string-array",
267
- "encoding-version": "0.2.0"
268
- },
269
- "varm/.zattrs": {
270
- "encoding-type": "dict",
271
- "encoding-version": "0.1.0"
272
- },
273
- "varm/.zgroup": {
274
- "zarr_format": 2
275
- },
276
- "varp/.zattrs": {
277
- "encoding-type": "dict",
278
- "encoding-version": "0.1.0"
279
- },
280
- "varp/.zgroup": {
281
- "zarr_format": 2
282
- }
283
- },
284
- "zarr_consolidated_format": 1
285
- }