hdf5-as-virtual-zarr 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (255) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +39 -0
  3. package/dist/hdf-store.d.ts +9 -0
  4. package/dist/hdf-store.js +18 -0
  5. package/dist/index.d.ts +3 -0
  6. package/dist/index.js +3 -0
  7. package/dist/jsfive/btree.d.ts +52 -0
  8. package/dist/jsfive/btree.js +354 -0
  9. package/dist/jsfive/core.d.ts +35 -0
  10. package/dist/jsfive/core.js +196 -0
  11. package/dist/jsfive/dataobjects.d.ts +64 -0
  12. package/dist/jsfive/dataobjects.js +1001 -0
  13. package/dist/jsfive/datatype-msg.d.ts +15 -0
  14. package/dist/jsfive/datatype-msg.js +110 -0
  15. package/dist/jsfive/high-level.d.ts +42 -0
  16. package/dist/jsfive/high-level.js +150 -0
  17. package/dist/jsfive/misc-low-level.d.ts +67 -0
  18. package/dist/jsfive/misc-low-level.js +506 -0
  19. package/dist/ref-spec-to-consolidated-metadata.d.ts +12 -0
  20. package/dist/ref-spec-to-consolidated-metadata.js +22 -0
  21. package/dist/ref-store.d.ts +10 -0
  22. package/dist/ref-store.js +79 -0
  23. package/dist/single-hdf5-to-zarr.d.ts +76 -0
  24. package/dist/single-hdf5-to-zarr.js +745 -0
  25. package/dist/types.d.ts +34 -0
  26. package/dist/types.js +12 -0
  27. package/package.json +53 -0
  28. package/src/hdf-store.ts +25 -0
  29. package/src/index.ts +3 -0
  30. package/src/jsfive/README.md +1 -0
  31. package/src/jsfive/btree.ts +474 -0
  32. package/src/jsfive/core.ts +228 -0
  33. package/src/jsfive/dataobjects.ts +1262 -0
  34. package/src/jsfive/datatype-msg.ts +111 -0
  35. package/src/jsfive/high-level.ts +182 -0
  36. package/src/jsfive/misc-low-level.ts +641 -0
  37. package/src/ref-spec-to-consolidated-metadata.ts +38 -0
  38. package/src/ref-store.ts +115 -0
  39. package/src/single-hdf5-to-zarr.ts +868 -0
  40. package/src/types.ts +49 -0
  41. package/test/fixtures/dense.adata.zarr/.zattrs +4 -0
  42. package/test/fixtures/dense.adata.zarr/.zgroup +3 -0
  43. package/test/fixtures/dense.adata.zarr/.zmetadata +285 -0
  44. package/test/fixtures/dense.adata.zarr/X/.zarray +22 -0
  45. package/test/fixtures/dense.adata.zarr/X/.zattrs +4 -0
  46. package/test/fixtures/dense.adata.zarr/X/0.0 +0 -0
  47. package/test/fixtures/dense.adata.zarr/layers/.zattrs +4 -0
  48. package/test/fixtures/dense.adata.zarr/layers/.zgroup +3 -0
  49. package/test/fixtures/dense.adata.zarr/obs/.zattrs +9 -0
  50. package/test/fixtures/dense.adata.zarr/obs/.zgroup +3 -0
  51. package/test/fixtures/dense.adata.zarr/obs/_index/.zarray +24 -0
  52. package/test/fixtures/dense.adata.zarr/obs/_index/.zattrs +4 -0
  53. package/test/fixtures/dense.adata.zarr/obs/_index/0 +0 -0
  54. package/test/fixtures/dense.adata.zarr/obs/categorical/.zattrs +5 -0
  55. package/test/fixtures/dense.adata.zarr/obs/categorical/.zgroup +3 -0
  56. package/test/fixtures/dense.adata.zarr/obs/categorical/categories/.zarray +24 -0
  57. package/test/fixtures/dense.adata.zarr/obs/categorical/categories/.zattrs +4 -0
  58. package/test/fixtures/dense.adata.zarr/obs/categorical/categories/0 +0 -0
  59. package/test/fixtures/dense.adata.zarr/obs/categorical/codes/.zarray +20 -0
  60. package/test/fixtures/dense.adata.zarr/obs/categorical/codes/.zattrs +4 -0
  61. package/test/fixtures/dense.adata.zarr/obs/categorical/codes/0 +0 -0
  62. package/test/fixtures/dense.adata.zarr/obs/string/.zarray +24 -0
  63. package/test/fixtures/dense.adata.zarr/obs/string/.zattrs +4 -0
  64. package/test/fixtures/dense.adata.zarr/obs/string/0 +0 -0
  65. package/test/fixtures/dense.adata.zarr/obsm/.zattrs +4 -0
  66. package/test/fixtures/dense.adata.zarr/obsm/.zgroup +3 -0
  67. package/test/fixtures/dense.adata.zarr/obsm/X_umap/.zarray +22 -0
  68. package/test/fixtures/dense.adata.zarr/obsm/X_umap/.zattrs +4 -0
  69. package/test/fixtures/dense.adata.zarr/obsm/X_umap/0.0 +0 -0
  70. package/test/fixtures/dense.adata.zarr/obsp/.zattrs +4 -0
  71. package/test/fixtures/dense.adata.zarr/obsp/.zgroup +3 -0
  72. package/test/fixtures/dense.adata.zarr/raw/.zarray +10 -0
  73. package/test/fixtures/dense.adata.zarr/raw/.zattrs +4 -0
  74. package/test/fixtures/dense.adata.zarr/raw/0 +0 -0
  75. package/test/fixtures/dense.adata.zarr/uns/.zattrs +4 -0
  76. package/test/fixtures/dense.adata.zarr/uns/.zgroup +3 -0
  77. package/test/fixtures/dense.adata.zarr/var/.zattrs +6 -0
  78. package/test/fixtures/dense.adata.zarr/var/.zgroup +3 -0
  79. package/test/fixtures/dense.adata.zarr/var/_index/.zarray +24 -0
  80. package/test/fixtures/dense.adata.zarr/var/_index/.zattrs +4 -0
  81. package/test/fixtures/dense.adata.zarr/var/_index/0 +0 -0
  82. package/test/fixtures/dense.adata.zarr/varm/.zattrs +4 -0
  83. package/test/fixtures/dense.adata.zarr/varm/.zgroup +3 -0
  84. package/test/fixtures/dense.adata.zarr/varp/.zattrs +4 -0
  85. package/test/fixtures/dense.adata.zarr/varp/.zgroup +3 -0
  86. package/test/fixtures/dense.adata.zarr.json +182 -0
  87. package/test/fixtures/dense.adata.zmetadata.json +1 -0
  88. package/test/fixtures/dense.h5ad +0 -0
  89. package/test/fixtures/dense.h5ad.refspec.json +1 -0
  90. package/test/fixtures/dense.v3.adata.zarr/X/c/0/0 +0 -0
  91. package/test/fixtures/dense.v3.adata.zarr/X/zarr.json +45 -0
  92. package/test/fixtures/dense.v3.adata.zarr/layers/zarr.json +8 -0
  93. package/test/fixtures/dense.v3.adata.zarr/obs/_index/c/0 +0 -0
  94. package/test/fixtures/dense.v3.adata.zarr/obs/_index/zarr.json +41 -0
  95. package/test/fixtures/dense.v3.adata.zarr/obs/categorical/categories/c/0 +0 -0
  96. package/test/fixtures/dense.v3.adata.zarr/obs/categorical/categories/zarr.json +41 -0
  97. package/test/fixtures/dense.v3.adata.zarr/obs/categorical/codes/c/0 +0 -0
  98. package/test/fixtures/dense.v3.adata.zarr/obs/categorical/codes/zarr.json +40 -0
  99. package/test/fixtures/dense.v3.adata.zarr/obs/categorical/zarr.json +9 -0
  100. package/test/fixtures/dense.v3.adata.zarr/obs/string/c/0 +0 -0
  101. package/test/fixtures/dense.v3.adata.zarr/obs/string/zarr.json +41 -0
  102. package/test/fixtures/dense.v3.adata.zarr/obs/zarr.json +13 -0
  103. package/test/fixtures/dense.v3.adata.zarr/obsm/X_umap/c/0/0 +0 -0
  104. package/test/fixtures/dense.v3.adata.zarr/obsm/X_umap/zarr.json +45 -0
  105. package/test/fixtures/dense.v3.adata.zarr/obsm/zarr.json +8 -0
  106. package/test/fixtures/dense.v3.adata.zarr/obsp/zarr.json +8 -0
  107. package/test/fixtures/dense.v3.adata.zarr/raw/zarr.json +36 -0
  108. package/test/fixtures/dense.v3.adata.zarr/uns/zarr.json +8 -0
  109. package/test/fixtures/dense.v3.adata.zarr/var/_index/c/0 +0 -0
  110. package/test/fixtures/dense.v3.adata.zarr/var/_index/zarr.json +41 -0
  111. package/test/fixtures/dense.v3.adata.zarr/var/zarr.json +10 -0
  112. package/test/fixtures/dense.v3.adata.zarr/varm/zarr.json +8 -0
  113. package/test/fixtures/dense.v3.adata.zarr/varp/zarr.json +8 -0
  114. package/test/fixtures/dense.v3.adata.zarr/zarr.json +469 -0
  115. package/test/fixtures/dense.v3.adata.zarr.json +102 -0
  116. package/test/fixtures/generate_fixtures.py +157 -0
  117. package/test/fixtures/generate_fixtures_v3.py +126 -0
  118. package/test/fixtures/minimal.adata.zarr/.zattrs +4 -0
  119. package/test/fixtures/minimal.adata.zarr/.zgroup +3 -0
  120. package/test/fixtures/minimal.adata.zarr/.zmetadata +168 -0
  121. package/test/fixtures/minimal.adata.zarr/X/.zarray +22 -0
  122. package/test/fixtures/minimal.adata.zarr/X/.zattrs +4 -0
  123. package/test/fixtures/minimal.adata.zarr/X/0.0 +0 -0
  124. package/test/fixtures/minimal.adata.zarr/layers/.zattrs +4 -0
  125. package/test/fixtures/minimal.adata.zarr/layers/.zgroup +3 -0
  126. package/test/fixtures/minimal.adata.zarr/obs/.zattrs +6 -0
  127. package/test/fixtures/minimal.adata.zarr/obs/.zgroup +3 -0
  128. package/test/fixtures/minimal.adata.zarr/obs/_index/.zarray +24 -0
  129. package/test/fixtures/minimal.adata.zarr/obs/_index/.zattrs +4 -0
  130. package/test/fixtures/minimal.adata.zarr/obs/_index/0 +0 -0
  131. package/test/fixtures/minimal.adata.zarr/obsm/.zattrs +4 -0
  132. package/test/fixtures/minimal.adata.zarr/obsm/.zgroup +3 -0
  133. package/test/fixtures/minimal.adata.zarr/obsp/.zattrs +4 -0
  134. package/test/fixtures/minimal.adata.zarr/obsp/.zgroup +3 -0
  135. package/test/fixtures/minimal.adata.zarr/raw/.zarray +10 -0
  136. package/test/fixtures/minimal.adata.zarr/raw/.zattrs +4 -0
  137. package/test/fixtures/minimal.adata.zarr/raw/0 +0 -0
  138. package/test/fixtures/minimal.adata.zarr/uns/.zattrs +4 -0
  139. package/test/fixtures/minimal.adata.zarr/uns/.zgroup +3 -0
  140. package/test/fixtures/minimal.adata.zarr/var/.zattrs +6 -0
  141. package/test/fixtures/minimal.adata.zarr/var/.zgroup +3 -0
  142. package/test/fixtures/minimal.adata.zarr/var/_index/.zarray +24 -0
  143. package/test/fixtures/minimal.adata.zarr/var/_index/.zattrs +4 -0
  144. package/test/fixtures/minimal.adata.zarr/var/_index/0 +0 -0
  145. package/test/fixtures/minimal.adata.zarr/varm/.zattrs +4 -0
  146. package/test/fixtures/minimal.adata.zarr/varm/.zgroup +3 -0
  147. package/test/fixtures/minimal.adata.zarr/varp/.zattrs +4 -0
  148. package/test/fixtures/minimal.adata.zarr/varp/.zgroup +3 -0
  149. package/test/fixtures/minimal.adata.zarr.json +126 -0
  150. package/test/fixtures/minimal.adata.zmetadata.json +1 -0
  151. package/test/fixtures/minimal.h5ad +0 -0
  152. package/test/fixtures/minimal.h5ad.refspec.json +1 -0
  153. package/test/fixtures/minimal.v3.adata.zarr/X/c/0/0 +0 -0
  154. package/test/fixtures/minimal.v3.adata.zarr/X/zarr.json +45 -0
  155. package/test/fixtures/minimal.v3.adata.zarr/layers/zarr.json +8 -0
  156. package/test/fixtures/minimal.v3.adata.zarr/obs/_index/c/0 +0 -0
  157. package/test/fixtures/minimal.v3.adata.zarr/obs/_index/zarr.json +41 -0
  158. package/test/fixtures/minimal.v3.adata.zarr/obs/zarr.json +10 -0
  159. package/test/fixtures/minimal.v3.adata.zarr/obsm/zarr.json +8 -0
  160. package/test/fixtures/minimal.v3.adata.zarr/obsp/zarr.json +8 -0
  161. package/test/fixtures/minimal.v3.adata.zarr/raw/zarr.json +36 -0
  162. package/test/fixtures/minimal.v3.adata.zarr/uns/zarr.json +8 -0
  163. package/test/fixtures/minimal.v3.adata.zarr/var/_index/c/0 +0 -0
  164. package/test/fixtures/minimal.v3.adata.zarr/var/_index/zarr.json +41 -0
  165. package/test/fixtures/minimal.v3.adata.zarr/var/zarr.json +10 -0
  166. package/test/fixtures/minimal.v3.adata.zarr/varm/zarr.json +8 -0
  167. package/test/fixtures/minimal.v3.adata.zarr/varp/zarr.json +8 -0
  168. package/test/fixtures/minimal.v3.adata.zarr/zarr.json +285 -0
  169. package/test/fixtures/minimal.v3.adata.zarr.json +66 -0
  170. package/test/fixtures/mouse_liver.h5ad +0 -0
  171. package/test/fixtures/mouse_liver.h5ad.refspec.json +1 -0
  172. package/test/fixtures/mouse_liver.py +44 -0
  173. package/test/fixtures/sparse.adata.zarr/.zattrs +4 -0
  174. package/test/fixtures/sparse.adata.zarr/.zgroup +3 -0
  175. package/test/fixtures/sparse.adata.zarr/.zmetadata +304 -0
  176. package/test/fixtures/sparse.adata.zarr/X/.zattrs +8 -0
  177. package/test/fixtures/sparse.adata.zarr/X/.zgroup +3 -0
  178. package/test/fixtures/sparse.adata.zarr/X/data/.zarray +20 -0
  179. package/test/fixtures/sparse.adata.zarr/X/data/0 +0 -0
  180. package/test/fixtures/sparse.adata.zarr/X/indices/.zarray +20 -0
  181. package/test/fixtures/sparse.adata.zarr/X/indices/0 +0 -0
  182. package/test/fixtures/sparse.adata.zarr/X/indptr/.zarray +20 -0
  183. package/test/fixtures/sparse.adata.zarr/X/indptr/0 +0 -0
  184. package/test/fixtures/sparse.adata.zarr/layers/.zattrs +4 -0
  185. package/test/fixtures/sparse.adata.zarr/layers/.zgroup +3 -0
  186. package/test/fixtures/sparse.adata.zarr/obs/.zattrs +9 -0
  187. package/test/fixtures/sparse.adata.zarr/obs/.zgroup +3 -0
  188. package/test/fixtures/sparse.adata.zarr/obs/_index/.zarray +24 -0
  189. package/test/fixtures/sparse.adata.zarr/obs/_index/.zattrs +4 -0
  190. package/test/fixtures/sparse.adata.zarr/obs/_index/0 +0 -0
  191. package/test/fixtures/sparse.adata.zarr/obs/categorical/.zattrs +5 -0
  192. package/test/fixtures/sparse.adata.zarr/obs/categorical/.zgroup +3 -0
  193. package/test/fixtures/sparse.adata.zarr/obs/categorical/categories/.zarray +24 -0
  194. package/test/fixtures/sparse.adata.zarr/obs/categorical/categories/.zattrs +4 -0
  195. package/test/fixtures/sparse.adata.zarr/obs/categorical/categories/0 +0 -0
  196. package/test/fixtures/sparse.adata.zarr/obs/categorical/codes/.zarray +20 -0
  197. package/test/fixtures/sparse.adata.zarr/obs/categorical/codes/.zattrs +4 -0
  198. package/test/fixtures/sparse.adata.zarr/obs/categorical/codes/0 +0 -0
  199. package/test/fixtures/sparse.adata.zarr/obs/string/.zarray +24 -0
  200. package/test/fixtures/sparse.adata.zarr/obs/string/.zattrs +4 -0
  201. package/test/fixtures/sparse.adata.zarr/obs/string/0 +0 -0
  202. package/test/fixtures/sparse.adata.zarr/obsm/.zattrs +4 -0
  203. package/test/fixtures/sparse.adata.zarr/obsm/.zgroup +3 -0
  204. package/test/fixtures/sparse.adata.zarr/obsp/.zattrs +4 -0
  205. package/test/fixtures/sparse.adata.zarr/obsp/.zgroup +3 -0
  206. package/test/fixtures/sparse.adata.zarr/raw/.zarray +10 -0
  207. package/test/fixtures/sparse.adata.zarr/raw/.zattrs +4 -0
  208. package/test/fixtures/sparse.adata.zarr/raw/0 +0 -0
  209. package/test/fixtures/sparse.adata.zarr/uns/.zattrs +4 -0
  210. package/test/fixtures/sparse.adata.zarr/uns/.zgroup +3 -0
  211. package/test/fixtures/sparse.adata.zarr/var/.zattrs +6 -0
  212. package/test/fixtures/sparse.adata.zarr/var/.zgroup +3 -0
  213. package/test/fixtures/sparse.adata.zarr/var/_index/.zarray +24 -0
  214. package/test/fixtures/sparse.adata.zarr/var/_index/.zattrs +4 -0
  215. package/test/fixtures/sparse.adata.zarr/var/_index/0 +0 -0
  216. package/test/fixtures/sparse.adata.zarr/varm/.zattrs +4 -0
  217. package/test/fixtures/sparse.adata.zarr/varm/.zgroup +3 -0
  218. package/test/fixtures/sparse.adata.zarr/varp/.zattrs +4 -0
  219. package/test/fixtures/sparse.adata.zarr/varp/.zgroup +3 -0
  220. package/test/fixtures/sparse.adata.zarr.json +190 -0
  221. package/test/fixtures/sparse.adata.zmetadata.json +1 -0
  222. package/test/fixtures/sparse.h5ad +0 -0
  223. package/test/fixtures/sparse.h5ad.refspec.json +1 -0
  224. package/test/fixtures/sparse.v3.adata.zarr/X/data/c/0 +0 -0
  225. package/test/fixtures/sparse.v3.adata.zarr/X/data/zarr.json +40 -0
  226. package/test/fixtures/sparse.v3.adata.zarr/X/indices/c/0 +0 -0
  227. package/test/fixtures/sparse.v3.adata.zarr/X/indices/zarr.json +40 -0
  228. package/test/fixtures/sparse.v3.adata.zarr/X/indptr/c/0 +0 -0
  229. package/test/fixtures/sparse.v3.adata.zarr/X/indptr/zarr.json +40 -0
  230. package/test/fixtures/sparse.v3.adata.zarr/X/zarr.json +12 -0
  231. package/test/fixtures/sparse.v3.adata.zarr/layers/zarr.json +8 -0
  232. package/test/fixtures/sparse.v3.adata.zarr/obs/_index/c/0 +0 -0
  233. package/test/fixtures/sparse.v3.adata.zarr/obs/_index/zarr.json +41 -0
  234. package/test/fixtures/sparse.v3.adata.zarr/obs/categorical/categories/c/0 +0 -0
  235. package/test/fixtures/sparse.v3.adata.zarr/obs/categorical/categories/zarr.json +41 -0
  236. package/test/fixtures/sparse.v3.adata.zarr/obs/categorical/codes/c/0 +0 -0
  237. package/test/fixtures/sparse.v3.adata.zarr/obs/categorical/codes/zarr.json +40 -0
  238. package/test/fixtures/sparse.v3.adata.zarr/obs/categorical/zarr.json +9 -0
  239. package/test/fixtures/sparse.v3.adata.zarr/obs/string/c/0 +0 -0
  240. package/test/fixtures/sparse.v3.adata.zarr/obs/string/zarr.json +41 -0
  241. package/test/fixtures/sparse.v3.adata.zarr/obs/zarr.json +13 -0
  242. package/test/fixtures/sparse.v3.adata.zarr/obsm/zarr.json +8 -0
  243. package/test/fixtures/sparse.v3.adata.zarr/obsp/zarr.json +8 -0
  244. package/test/fixtures/sparse.v3.adata.zarr/raw/zarr.json +36 -0
  245. package/test/fixtures/sparse.v3.adata.zarr/uns/zarr.json +8 -0
  246. package/test/fixtures/sparse.v3.adata.zarr/var/_index/c/0 +0 -0
  247. package/test/fixtures/sparse.v3.adata.zarr/var/_index/zarr.json +41 -0
  248. package/test/fixtures/sparse.v3.adata.zarr/var/zarr.json +10 -0
  249. package/test/fixtures/sparse.v3.adata.zarr/varm/zarr.json +8 -0
  250. package/test/fixtures/sparse.v3.adata.zarr/varp/zarr.json +8 -0
  251. package/test/fixtures/sparse.v3.adata.zarr/zarr.json +516 -0
  252. package/test/fixtures/sparse.v3.adata.zarr.json +114 -0
  253. package/test/hdf-store.test.ts +268 -0
  254. package/test/index.test.ts +762 -0
  255. package/test/partial-reads.test.ts +150 -0
@@ -0,0 +1,1001 @@
1
+ /**
2
+ * Vendored and adapted from jsfive/esm/dataobjects.js.
3
+ * DataObjects - the HDF5 data object header parser.
4
+ *
5
+ * Key differences from jsfive:
6
+ * - Uses async factory method DataObjects.create(source, offset)
7
+ * - Partial reads via AsyncReadable.fetch() instead of full file buffer
8
+ * - get_links(), get_attributes(), get_data() are async
9
+ * - buf/bufStart design: buf covers all message block ranges,
10
+ * bufStart = absolute offset where buf starts (= object header offset)
11
+ */
12
+ import { fetchRange } from "../types.js";
13
+ import { DatatypeMessage } from "./datatype-msg.js";
14
+ import { _structure_size, _padded_size, _unpack_struct_from, struct, dtype_getter, DataView64, assert, } from "./core.js";
15
+ import { BTreeV1Groups, BTreeV1RawDataChunks, BTreeV2GroupNames, BTreeV2GroupOrders, } from "./btree.js";
16
+ import { Heap, SymbolTable, GlobalHeap, FractalHeap, UNDEFINED_ADDRESS, } from "./misc-low-level.js";
17
+ // ────────── Message type constants ──────────
18
+ const DATASPACE_MSG_TYPE = 0x0001;
19
+ const LINK_INFO_MSG_TYPE = 0x0002;
20
+ const DATATYPE_MSG_TYPE = 0x0003;
21
+ const FILLVALUE_MSG_TYPE = 0x0005;
22
+ const LINK_MSG_TYPE = 0x0006;
23
+ const DATA_STORAGE_MSG_TYPE = 0x0008;
24
+ const DATA_STORAGE_FILTER_PIPELINE_MSG_TYPE = 0x000b;
25
+ const ATTRIBUTE_MSG_TYPE = 0x000c;
26
+ const OBJECT_CONTINUATION_MSG_TYPE = 0x0010;
27
+ const SYMBOL_TABLE_MSG_TYPE = 0x0011;
28
+ const ATTRIBUTE_INFO_MSG_TYPE = 0x0015;
29
+ // ────────── Structures ──────────
30
+ const OBJECT_HEADER_V1 = new Map([
31
+ ["version", "B"],
32
+ ["reserved", "B"],
33
+ ["total_header_messages", "H"],
34
+ ["object_reference_count", "I"],
35
+ ["object_header_size", "I"],
36
+ ["padding", "I"],
37
+ ]);
38
+ const OBJECT_HEADER_V1_SIZE = _structure_size(OBJECT_HEADER_V1);
39
+ const OBJECT_HEADER_V2 = new Map([
40
+ ["signature", "4s"],
41
+ ["version", "B"],
42
+ ["flags", "B"],
43
+ ]);
44
+ const OBJECT_HEADER_V2_SIZE = _structure_size(OBJECT_HEADER_V2);
45
+ const HEADER_MSG_INFO_V1 = new Map([
46
+ ["type", "H"],
47
+ ["size", "H"],
48
+ ["flags", "B"],
49
+ ["reserved", "3s"],
50
+ ]);
51
+ const HEADER_MSG_INFO_V1_SIZE = _structure_size(HEADER_MSG_INFO_V1);
52
+ const HEADER_MSG_INFO_V2 = new Map([
53
+ ["type", "B"],
54
+ ["size", "H"],
55
+ ["flags", "B"],
56
+ ]);
57
+ const HEADER_MSG_INFO_V2_SIZE = _structure_size(HEADER_MSG_INFO_V2);
58
+ const SYMBOL_TABLE_MSG = new Map([
59
+ ["btree_address", "Q"],
60
+ ["heap_address", "Q"],
61
+ ]);
62
+ const ATTR_MSG_HEADER_V1 = new Map([
63
+ ["version", "B"],
64
+ ["reserved", "B"],
65
+ ["name_size", "H"],
66
+ ["datatype_size", "H"],
67
+ ["dataspace_size", "H"],
68
+ ]);
69
+ const ATTR_MSG_HEADER_V1_SIZE = _structure_size(ATTR_MSG_HEADER_V1);
70
+ const ATTR_MSG_HEADER_V3 = new Map([
71
+ ["version", "B"],
72
+ ["flags", "B"],
73
+ ["name_size", "H"],
74
+ ["datatype_size", "H"],
75
+ ["dataspace_size", "H"],
76
+ ["character_set_encoding", "B"],
77
+ ]);
78
+ const ATTR_MSG_HEADER_V3_SIZE = _structure_size(ATTR_MSG_HEADER_V3);
79
+ const DATASPACE_MSG_HEADER_V1 = new Map([
80
+ ["version", "B"],
81
+ ["dimensionality", "B"],
82
+ ["flags", "B"],
83
+ ["reserved_0", "B"],
84
+ ["reserved_1", "I"],
85
+ ]);
86
+ const DATASPACE_MSG_HEADER_V1_SIZE = _structure_size(DATASPACE_MSG_HEADER_V1);
87
+ const DATASPACE_MSG_HEADER_V2 = new Map([
88
+ ["version", "B"],
89
+ ["dimensionality", "B"],
90
+ ["flags", "B"],
91
+ ["type", "B"],
92
+ ]);
93
+ const DATASPACE_MSG_HEADER_V2_SIZE = _structure_size(DATASPACE_MSG_HEADER_V2);
94
+ const FILLVAL_MSG_V1V2 = new Map([
95
+ ["version", "B"],
96
+ ["space_allocation_time", "B"],
97
+ ["fillvalue_write_time", "B"],
98
+ ["fillvalue_defined", "B"],
99
+ ]);
100
+ const FILLVAL_MSG_V1V2_SIZE = _structure_size(FILLVAL_MSG_V1V2);
101
+ const FILLVAL_MSG_V3 = new Map([
102
+ ["version", "B"],
103
+ ["flags", "B"],
104
+ ]);
105
+ const FILLVAL_MSG_V3_SIZE = _structure_size(FILLVAL_MSG_V3);
106
+ const FILTER_PIPELINE_DESCR_V1 = new Map([
107
+ ["filter_id", "H"],
108
+ ["name_length", "H"],
109
+ ["flags", "H"],
110
+ ["client_data_values", "H"],
111
+ ]);
112
+ const FILTER_PIPELINE_DESCR_V1_SIZE = _structure_size(FILTER_PIPELINE_DESCR_V1);
113
+ const GLOBAL_HEAP_ID = new Map([
114
+ ["collection_address", "Q"],
115
+ ["object_index", "I"],
116
+ ]);
117
+ const GLOBAL_HEAP_ID_SIZE = _structure_size(GLOBAL_HEAP_ID);
118
+ const LINK_INFO_MSG1 = new Map([
119
+ ["heap_address", "Q"],
120
+ ["name_btree_address", "Q"],
121
+ ]);
122
+ const LINK_INFO_MSG2 = new Map([
123
+ ["heap_address", "Q"],
124
+ ["name_btree_address", "Q"],
125
+ ["order_btree_address", "Q"],
126
+ ]);
127
+ // ────────── DataObjects ──────────
128
+ export class DataObjects {
129
+ buf;
130
+ bufStart;
131
+ msgs;
132
+ msg_data;
133
+ offset;
134
+ _global_heaps;
135
+ _header;
136
+ _filter_pipeline;
137
+ _chunk_params_set;
138
+ _chunks;
139
+ _chunk_dims;
140
+ _chunk_address;
141
+ _source;
142
+ constructor(source, buf, bufStart, msgs, msg_data, offset, header) {
143
+ this._source = source;
144
+ this.buf = buf;
145
+ this.bufStart = bufStart;
146
+ this.msgs = msgs;
147
+ this.msg_data = msg_data;
148
+ this.offset = offset;
149
+ this._global_heaps = new Map();
150
+ this._header = header;
151
+ this._filter_pipeline = null;
152
+ this._chunk_params_set = false;
153
+ this._chunks = null;
154
+ this._chunk_dims = null;
155
+ this._chunk_address = null;
156
+ }
157
+ static async create(source, offset) {
158
+ const peekBuf = await fetchRange(source, offset, 1);
159
+ const versionHint = struct.unpack_from("<B", peekBuf, 0)[0];
160
+ let result;
161
+ if (versionHint === 1) {
162
+ result = await DataObjects._parseV1(source, offset);
163
+ }
164
+ else if (versionHint === "O".charCodeAt(0)) {
165
+ result = await DataObjects._parseV2(source, offset);
166
+ }
167
+ else {
168
+ throw new Error(`Unknown Data Object Header version hint: ${versionHint}`);
169
+ }
170
+ return new DataObjects(source, result.buf, result.bufStart, result.msgs, result.msg_data, offset, result.header);
171
+ }
172
+ // ────────── V1 parsing ──────────
173
+ static async _parseV1(source, offset) {
174
+ const headerBuf = await fetchRange(source, offset, OBJECT_HEADER_V1_SIZE);
175
+ const header = _unpack_struct_from(OBJECT_HEADER_V1, headerBuf, 0);
176
+ assert(header.get("version") === 1);
177
+ const total_header_messages = header.get("total_header_messages");
178
+ const first_block_size = header.get("object_header_size");
179
+ const first_block_offset = offset + OBJECT_HEADER_V1_SIZE;
180
+ const allBlocks = [];
181
+ const blockQueue = [
182
+ { absOffset: first_block_offset, size: first_block_size },
183
+ ];
184
+ while (blockQueue.length > 0) {
185
+ const { absOffset, size } = blockQueue.shift();
186
+ const data = await fetchRange(source, absOffset, size);
187
+ allBlocks.push({ absOffset, size, data });
188
+ // Scan messages to find continuation blocks
189
+ let local_offset = 0;
190
+ while (local_offset + HEADER_MSG_INFO_V1_SIZE <= size) {
191
+ const msg = _unpack_struct_from(HEADER_MSG_INFO_V1, data, local_offset);
192
+ const msg_type = msg.get("type");
193
+ const msg_size = msg.get("size");
194
+ const offset_in_block = local_offset + HEADER_MSG_INFO_V1_SIZE;
195
+ if (msg_type === OBJECT_CONTINUATION_MSG_TYPE) {
196
+ const [fh_off, cont_size] = struct.unpack_from("<QQ", data, offset_in_block);
197
+ blockQueue.push({ absOffset: fh_off, size: cont_size });
198
+ }
199
+ if (msg_type === 0 && msg_size === 0)
200
+ break; // NIL message, stop
201
+ local_offset += HEADER_MSG_INFO_V1_SIZE + msg_size;
202
+ }
203
+ }
204
+ // Build combined sparse buffer
205
+ // bufStart = minimum of object header offset and all block offsets
206
+ // (continuation blocks can appear before the object header in the file)
207
+ let bufStart = offset;
208
+ let maxEnd = first_block_offset + first_block_size;
209
+ for (const b of allBlocks) {
210
+ bufStart = Math.min(bufStart, b.absOffset);
211
+ maxEnd = Math.max(maxEnd, b.absOffset + b.size);
212
+ }
213
+ const combined = new Uint8Array(maxEnd - bufStart);
214
+ combined.set(new Uint8Array(headerBuf), offset - bufStart);
215
+ for (const b of allBlocks) {
216
+ combined.set(new Uint8Array(b.data), b.absOffset - bufStart);
217
+ }
218
+ const buf = combined.buffer;
219
+ // Parse messages (second pass over combined buffer)
220
+ const msgs = new Array(total_header_messages);
221
+ let object_header_blocks = [[first_block_offset, first_block_size]];
222
+ // Add continuation blocks discovered during first pass
223
+ for (let bi = 1; bi < allBlocks.length; bi++) {
224
+ object_header_blocks.push([allBlocks[bi].absOffset, allBlocks[bi].size]);
225
+ }
226
+ let current_block = 0;
227
+ let block_offset = object_header_blocks[0][0];
228
+ let block_size = object_header_blocks[0][1];
229
+ let local_offset = 0;
230
+ for (let i = 0; i < total_header_messages; i++) {
231
+ if (local_offset >= block_size) {
232
+ current_block++;
233
+ if (current_block >= object_header_blocks.length)
234
+ break;
235
+ [block_offset, block_size] = object_header_blocks[current_block];
236
+ local_offset = 0;
237
+ }
238
+ const absPos = block_offset - bufStart + local_offset;
239
+ const msg = _unpack_struct_from(HEADER_MSG_INFO_V1, buf, absPos);
240
+ const offset_to_message = block_offset + local_offset + HEADER_MSG_INFO_V1_SIZE;
241
+ msg.set("offset_to_message", offset_to_message);
242
+ // Continuation blocks add more blocks (already handled above)
243
+ const msg_size = msg.get("size");
244
+ local_offset += HEADER_MSG_INFO_V1_SIZE + msg_size;
245
+ msgs[i] = msg;
246
+ }
247
+ const msg_data = buf.slice(first_block_offset - bufStart, first_block_offset - bufStart + first_block_size);
248
+ return { buf, bufStart, msgs: msgs.filter(Boolean), msg_data, header };
249
+ }
250
+ // ────────── V2 parsing ──────────
251
+ static async _parseV2(source, offset) {
252
+ // Fetch enough for the full V2 header (max 32 bytes)
253
+ const peekSize = 64;
254
+ const peekBuf = await fetchRange(source, offset, peekSize);
255
+ const { header, creation_order_size, block_offset: first_block_offset } = DataObjects._parseV2Header(peekBuf, 0, offset);
256
+ const first_block_size = header.get("size_of_chunk_0");
257
+ const allBlocks = [];
258
+ const blockQueue = [
259
+ { absOffset: first_block_offset, size: first_block_size },
260
+ ];
261
+ while (blockQueue.length > 0) {
262
+ const { absOffset, size } = blockQueue.shift();
263
+ let data;
264
+ // For small blocks that fit in the peek buffer, reuse it
265
+ if (absOffset >= offset && absOffset + size <= offset + peekSize) {
266
+ data = peekBuf.slice(absOffset - offset, absOffset - offset + size);
267
+ }
268
+ else {
269
+ data = await fetchRange(source, absOffset, size);
270
+ }
271
+ allBlocks.push({ absOffset, size, data });
272
+ // Scan messages to find continuation blocks
273
+ let local_offset = 0;
274
+ while (local_offset + HEADER_MSG_INFO_V2_SIZE <= size) {
275
+ const relOff = local_offset;
276
+ const msg_type = struct.unpack_from("<B", data, relOff)[0];
277
+ const msg_size = struct.unpack_from("<H", data, relOff + 1)[0];
278
+ const offset_in_data = relOff + HEADER_MSG_INFO_V2_SIZE + creation_order_size;
279
+ if (msg_type === OBJECT_CONTINUATION_MSG_TYPE) {
280
+ const [fh_off, cont_size] = struct.unpack_from("<QQ", data, offset_in_data);
281
+ // Skip "OFHC" 4-byte signature in v2 continuation blocks
282
+ blockQueue.push({ absOffset: fh_off + 4, size: cont_size - 4 });
283
+ }
284
+ if (msg_type === 0 && msg_size === 0)
285
+ break;
286
+ local_offset += HEADER_MSG_INFO_V2_SIZE + msg_size + creation_order_size;
287
+ if (local_offset >= size - HEADER_MSG_INFO_V2_SIZE)
288
+ break;
289
+ }
290
+ }
291
+ // Build combined sparse buffer
292
+ const bufStart = offset;
293
+ let maxEnd = first_block_offset + first_block_size;
294
+ for (const b of allBlocks) {
295
+ maxEnd = Math.max(maxEnd, b.absOffset + b.size);
296
+ }
297
+ const combined = new Uint8Array(maxEnd - bufStart);
298
+ // Copy the peek buffer (header)
299
+ const copySize = Math.min(peekSize, combined.length);
300
+ combined.set(new Uint8Array(peekBuf).subarray(0, copySize), 0);
301
+ // Copy all blocks
302
+ for (const b of allBlocks) {
303
+ combined.set(new Uint8Array(b.data), b.absOffset - bufStart);
304
+ }
305
+ const buf = combined.buffer;
306
+ // Parse messages (second pass)
307
+ const msgs = [];
308
+ const object_header_blocks = [];
309
+ for (const b of allBlocks) {
310
+ object_header_blocks.push([b.absOffset, b.size]);
311
+ }
312
+ let current_block = 0;
313
+ let block_offset = object_header_blocks[0][0];
314
+ let block_size = object_header_blocks[0][1];
315
+ let local_offset = 0;
316
+ // eslint-disable-next-line no-constant-condition
317
+ while (true) {
318
+ if (local_offset >= block_size - HEADER_MSG_INFO_V2_SIZE) {
319
+ current_block++;
320
+ if (current_block >= object_header_blocks.length)
321
+ break;
322
+ [block_offset, block_size] = object_header_blocks[current_block];
323
+ local_offset = 0;
324
+ }
325
+ const absPos = block_offset - bufStart + local_offset;
326
+ const msg = _unpack_struct_from(HEADER_MSG_INFO_V2, buf, absPos);
327
+ const offset_to_message = block_offset + local_offset + HEADER_MSG_INFO_V2_SIZE + creation_order_size;
328
+ msg.set("offset_to_message", offset_to_message);
329
+ const msg_size = msg.get("size");
330
+ local_offset += HEADER_MSG_INFO_V2_SIZE + msg_size + creation_order_size;
331
+ msgs.push(msg);
332
+ if (local_offset >= block_size - HEADER_MSG_INFO_V2_SIZE) {
333
+ current_block++;
334
+ if (current_block >= object_header_blocks.length)
335
+ break;
336
+ [block_offset, block_size] = object_header_blocks[current_block];
337
+ local_offset = 0;
338
+ }
339
+ }
340
+ const msg_data = buf.slice(first_block_offset - bufStart, first_block_offset - bufStart + first_block_size);
341
+ return { buf, bufStart, msgs, msg_data, header };
342
+ }
343
+ static _parseV2Header(buf, localOffset, absBase) {
344
+ const header = _unpack_struct_from(OBJECT_HEADER_V2, buf, localOffset);
345
+ assert(header.get("version") === 2);
346
+ localOffset += OBJECT_HEADER_V2_SIZE;
347
+ const flags = header.get("flags");
348
+ const creation_order_size = (flags & 0b00000100) ? 2 : 0;
349
+ assert((flags & 0b00010000) === 0);
350
+ if (flags & 0b00100000) {
351
+ // access/mod/change/birth times (4 uint32s)
352
+ localOffset += 16;
353
+ }
354
+ const chunk_fmt = ["<B", "<H", "<I", "<Q"][flags & 0b00000011];
355
+ header.set("size_of_chunk_0", struct.unpack_from(chunk_fmt, buf, localOffset)[0]);
356
+ localOffset += struct.calcsize(chunk_fmt);
357
+ const block_offset = absBase + localOffset;
358
+ return { header, creation_order_size, block_offset };
359
+ }
360
+ // ────────── Core methods ──────────
361
+ find_msg_type(msg_type) {
362
+ return this.msgs.filter((m) => m.get("type") === msg_type);
363
+ }
364
+ get is_dataset() {
365
+ return this.find_msg_type(DATASPACE_MSG_TYPE).length > 0;
366
+ }
367
+ get dtype() {
368
+ const msg = this.find_msg_type(DATATYPE_MSG_TYPE)[0];
369
+ const msg_offset = msg.get("offset_to_message");
370
+ const relOffset = msg_offset - this.bufStart;
371
+ return new DatatypeMessage(this.buf, relOffset).dtype;
372
+ }
373
+ get shape() {
374
+ const msg = this.find_msg_type(DATASPACE_MSG_TYPE)[0];
375
+ const msg_offset = msg.get("offset_to_message");
376
+ const relOffset = msg_offset - this.bufStart;
377
+ return this._determine_data_shape(this.buf, relOffset);
378
+ }
379
+ get chunks() {
380
+ this._get_chunk_params();
381
+ return this._chunks;
382
+ }
383
+ get filter_pipeline() {
384
+ if (this._filter_pipeline !== null) {
385
+ return this._filter_pipeline;
386
+ }
387
+ const filter_msgs = this.find_msg_type(DATA_STORAGE_FILTER_PIPELINE_MSG_TYPE);
388
+ if (!filter_msgs.length) {
389
+ this._filter_pipeline = null;
390
+ return null;
391
+ }
392
+ const absOffset = filter_msgs[0].get("offset_to_message");
393
+ let offset = absOffset - this.bufStart;
394
+ const buf = this.buf;
395
+ const [version, nfilters] = struct.unpack_from("<BB", buf, offset);
396
+ offset += struct.calcsize("<BB");
397
+ const filters = [];
398
+ if (version === 1) {
399
+ offset += struct.calcsize("<HI"); // reserved fields
400
+ for (let _ = 0; _ < nfilters; _++) {
401
+ const filter_info = _unpack_struct_from(FILTER_PIPELINE_DESCR_V1, buf, offset);
402
+ offset += FILTER_PIPELINE_DESCR_V1_SIZE;
403
+ const padded_name_length = _padded_size(filter_info.get("name_length"), 8);
404
+ const fmt = "<" + padded_name_length.toFixed() + "s";
405
+ const filter_name = struct.unpack_from(fmt, buf, offset)[0];
406
+ filter_info.set("filter_name", filter_name);
407
+ offset += padded_name_length;
408
+ const client_data_values = filter_info.get("client_data_values");
409
+ const client_data_fmt = "<" + client_data_values.toFixed() + "I";
410
+ const client_data = struct.unpack_from(client_data_fmt, buf, offset);
411
+ filter_info.set("client_data", client_data);
412
+ offset += 4 * client_data_values;
413
+ if (client_data_values % 2) {
414
+ offset += 4; // padding for odd number of client data values
415
+ }
416
+ filters.push(filter_info);
417
+ }
418
+ }
419
+ else if (version === 2) {
420
+ for (let nf = 0; nf < nfilters; nf++) {
421
+ const filter_info = new Map();
422
+ const filter_id = struct.unpack_from("<H", buf, offset)[0];
423
+ offset += 2;
424
+ filter_info.set("filter_id", filter_id);
425
+ let name_length = 0;
426
+ if (filter_id > 255) {
427
+ name_length = struct.unpack_from("<H", buf, offset)[0];
428
+ offset += 2;
429
+ }
430
+ const flags_val = struct.unpack_from("<H", buf, offset)[0];
431
+ offset += 2;
432
+ filter_info.set("optional", (flags_val & 1) > 0);
433
+ const num_client_values = struct.unpack_from("<H", buf, offset)[0];
434
+ offset += 2;
435
+ if (name_length > 0) {
436
+ const name = struct.unpack_from(`${name_length}s`, buf, offset)[0];
437
+ filter_info.set("name", name);
438
+ offset += name_length;
439
+ }
440
+ const client_values = struct.unpack_from(`<${num_client_values}i`, buf, offset);
441
+ offset += 4 * num_client_values;
442
+ filter_info.set("client_data", client_values);
443
+ filter_info.set("client_data_values", num_client_values);
444
+ filters.push(filter_info);
445
+ }
446
+ }
447
+ else {
448
+ throw new Error(`Filter pipeline version ${version} not supported`);
449
+ }
450
+ this._filter_pipeline = filters;
451
+ return this._filter_pipeline;
452
+ }
453
+ get fillvalue() {
454
+ const msg = this.find_msg_type(FILLVALUE_MSG_TYPE)[0];
455
+ if (!msg)
456
+ return 0;
457
+ let absOffset = msg.get("offset_to_message");
458
+ let offset = absOffset - this.bufStart;
459
+ const buf = this.buf;
460
+ const version = struct.unpack_from("<B", buf, offset)[0];
461
+ let info;
462
+ let is_defined;
463
+ if (version === 1 || version === 2) {
464
+ info = _unpack_struct_from(FILLVAL_MSG_V1V2, buf, offset);
465
+ offset += FILLVAL_MSG_V1V2_SIZE;
466
+ is_defined = info.get("fillvalue_defined") !== 0;
467
+ }
468
+ else if (version === 3) {
469
+ info = _unpack_struct_from(FILLVAL_MSG_V3, buf, offset);
470
+ offset += FILLVAL_MSG_V3_SIZE;
471
+ is_defined = (info.get("flags") & 0b00100000) !== 0;
472
+ }
473
+ else {
474
+ throw new Error(`Unknown fillvalue message version: ${version}`);
475
+ }
476
+ let size = 0;
477
+ if (is_defined) {
478
+ size = struct.unpack_from("<I", buf, offset)[0];
479
+ offset += 4;
480
+ }
481
+ if (size) {
482
+ const dtype = this.dtype;
483
+ if (typeof dtype === "string") {
484
+ const [getter, big_endian, itemSize] = dtype_getter(dtype);
485
+ const view = new DataView64(buf, 0);
486
+ return view[getter](offset, !big_endian, itemSize);
487
+ }
488
+ }
489
+ return 0;
490
+ }
491
+ _get_data_message_properties(msg_offset) {
492
+ const buf = this.buf;
493
+ const [version, arg1, arg2] = struct.unpack_from("<BBB", buf, msg_offset);
494
+ let dims;
495
+ let layout_class;
496
+ let property_offset;
497
+ if (version === 1 || version === 2) {
498
+ dims = arg1;
499
+ layout_class = arg2;
500
+ property_offset = msg_offset + struct.calcsize("<BBB") + struct.calcsize("<BI");
501
+ }
502
+ else if (version === 3 || version === 4) {
503
+ layout_class = arg1;
504
+ dims = 0;
505
+ property_offset = msg_offset + struct.calcsize("<BB");
506
+ }
507
+ else {
508
+ throw new Error(`Unknown data storage message version: ${version}`);
509
+ }
510
+ return { version, dims, layout_class, property_offset };
511
+ }
512
+ _get_chunk_params() {
513
+ if (this._chunk_params_set)
514
+ return;
515
+ this._chunk_params_set = true;
516
+ const msg = this.find_msg_type(DATA_STORAGE_MSG_TYPE)[0];
517
+ if (!msg)
518
+ return;
519
+ const absOffset = msg.get("offset_to_message");
520
+ const relOffset = absOffset - this.bufStart;
521
+ const { version, dims: msgDims, layout_class, property_offset } = this._get_data_message_properties(relOffset);
522
+ if (layout_class !== 2)
523
+ return; // not chunked
524
+ const buf = this.buf;
525
+ let address;
526
+ let dims;
527
+ let data_offset;
528
+ if (version === 1 || version === 2) {
529
+ address = struct.unpack_from("<Q", buf, property_offset)[0];
530
+ data_offset = property_offset + struct.calcsize("<Q");
531
+ dims = msgDims;
532
+ }
533
+ else if (version === 3) {
534
+ [dims, address] = struct.unpack_from("<BQ", buf, property_offset);
535
+ data_offset = property_offset + struct.calcsize("<BQ");
536
+ }
537
+ else {
538
+ return;
539
+ }
540
+ const fmt = "<" + (dims - 1).toFixed() + "I";
541
+ const chunk_shape = struct.unpack_from(fmt, buf, data_offset);
542
+ this._chunks = chunk_shape;
543
+ this._chunk_dims = dims;
544
+ this._chunk_address = address;
545
+ }
546
+ // ────────── Data shape helper ──────────
547
+ _determine_data_shape(buf, offset) {
548
+ const version = struct.unpack_from("<B", buf, offset)[0];
549
+ let header;
550
+ if (version === 1) {
551
+ header = _unpack_struct_from(DATASPACE_MSG_HEADER_V1, buf, offset);
552
+ assert(header.get("version") === 1);
553
+ offset += DATASPACE_MSG_HEADER_V1_SIZE;
554
+ }
555
+ else if (version === 2) {
556
+ header = _unpack_struct_from(DATASPACE_MSG_HEADER_V2, buf, offset);
557
+ assert(header.get("version") === 2);
558
+ offset += DATASPACE_MSG_HEADER_V2_SIZE;
559
+ }
560
+ else {
561
+ throw new Error(`Unknown dataspace message version: ${version}`);
562
+ }
563
+ const ndims = header.get("dimensionality");
564
+ if (ndims === 0)
565
+ return [];
566
+ // Read dimension sizes as uint64
567
+ const dim_sizes = struct.unpack_from("<" + ndims.toFixed() + "Q", buf, offset);
568
+ return dim_sizes;
569
+ }
570
+ // ────────── Async: get_links ──────────
571
+ async get_links() {
572
+ const links = {};
573
+ for (const msg of this.msgs) {
574
+ const msg_type = msg.get("type");
575
+ if (msg_type === SYMBOL_TABLE_MSG_TYPE) {
576
+ const newLinks = await this._iter_links_from_symbol_tables(msg);
577
+ Object.assign(links, newLinks);
578
+ }
579
+ else if (msg_type === LINK_MSG_TYPE) {
580
+ const absOffset = msg.get("offset_to_message");
581
+ const relOffset = absOffset - this.bufStart;
582
+ const [, [name, address]] = this._decode_link_msg(this.buf, relOffset);
583
+ links[name] = address;
584
+ }
585
+ else if (msg_type === LINK_INFO_MSG_TYPE) {
586
+ const newLinks = await this._iter_link_from_link_info_msg(msg);
587
+ Object.assign(links, newLinks);
588
+ }
589
+ }
590
+ return links;
591
+ }
592
+ async _iter_links_from_symbol_tables(sym_tbl_msg) {
593
+ assert(sym_tbl_msg.get("size") === 16);
594
+ const absOffset = sym_tbl_msg.get("offset_to_message");
595
+ const relOffset = absOffset - this.bufStart;
596
+ const data = _unpack_struct_from(SYMBOL_TABLE_MSG, this.buf, relOffset);
597
+ return this._iter_links_btree_v1(data.get("btree_address"), data.get("heap_address"));
598
+ }
599
+ async _iter_links_btree_v1(btree_address, heap_address) {
600
+ const btree = await BTreeV1Groups.create(this._source, btree_address);
601
+ const heap = await Heap.create(this._source, heap_address);
602
+ const links = {};
603
+ for (const symbol_table_address of btree.symbol_table_addresses()) {
604
+ const table = await SymbolTable.create(this._source, symbol_table_address);
605
+ table.assign_name(heap);
606
+ Object.assign(links, table.get_links(heap));
607
+ }
608
+ return links;
609
+ }
610
+ async _iter_link_from_link_info_msg(info_msg) {
611
+ const absOffset = info_msg.get("offset_to_message");
612
+ const relOffset = absOffset - this.bufStart;
613
+ const data = this._decode_link_info_msg(this.buf, relOffset);
614
+ const heap_address = data.get("heap_address") ?? null;
615
+ const name_btree_address = data.get("name_btree_address") ?? null;
616
+ const order_btree_address = data.get("order_btree_address") ?? null;
617
+ if (name_btree_address !== null) {
618
+ return this._iter_links_btree_v2(name_btree_address, order_btree_address, heap_address);
619
+ }
620
+ return {};
621
+ }
622
+ async _iter_links_btree_v2(name_btree_address, order_btree_address, heap_address) {
623
+ const heap = await FractalHeap.create(this._source, heap_address);
624
+ const ordered = order_btree_address !== null;
625
+ let btree;
626
+ if (ordered) {
627
+ btree = await BTreeV2GroupOrders.create(this._source, order_btree_address);
628
+ }
629
+ else {
630
+ btree = await BTreeV2GroupNames.create(this._source, name_btree_address);
631
+ }
632
+ const items = new Map();
633
+ for (const record of btree.iter_records()) {
634
+ const data = heap.get_data(record.get("heapid"));
635
+ const [creationorder, item] = this._decode_link_msg(data, 0);
636
+ const key = ordered ? creationorder : item[0];
637
+ items.set(key, item);
638
+ }
639
+ const sorted_keys = Array.from(items.keys()).sort();
640
+ const links = {};
641
+ for (const key of sorted_keys) {
642
+ const [name, address] = items.get(key);
643
+ links[name] = address;
644
+ }
645
+ return links;
646
+ }
647
+ _decode_link_msg(data, offset) {
648
+ const [version, flags] = struct.unpack_from("<BB", data, offset);
649
+ offset += 2;
650
+ assert(version === 1);
651
+ const size_of_length_of_link_name = 2 ** (flags & 3);
652
+ const link_type_field_present = (flags & (1 << 3)) > 0;
653
+ const link_name_character_set_field_present = (flags & (1 << 4)) > 0;
654
+ const ordered = (flags & (1 << 2)) > 0;
655
+ let link_type = 0;
656
+ if (link_type_field_present) {
657
+ link_type = struct.unpack_from("<B", data, offset)[0];
658
+ offset += 1;
659
+ }
660
+ assert([0, 1].includes(link_type));
661
+ let creationorder;
662
+ if (ordered) {
663
+ creationorder = struct.unpack_from("<Q", data, offset)[0];
664
+ offset += 8;
665
+ }
666
+ let link_name_character_set = 0;
667
+ if (link_name_character_set_field_present) {
668
+ link_name_character_set = struct.unpack_from("<B", data, offset)[0];
669
+ offset += 1;
670
+ }
671
+ const encoding = link_name_character_set === 0 ? "ascii" : "utf-8";
672
+ const name_size_fmt = ["<B", "<H", "<I", "<Q"][flags & 3];
673
+ const name_size = struct.unpack_from(name_size_fmt, data, offset)[0];
674
+ offset += size_of_length_of_link_name;
675
+ const name = new TextDecoder(encoding).decode(data.slice(offset, offset + name_size));
676
+ offset += name_size;
677
+ let address;
678
+ if (link_type === 0) {
679
+ // hard link
680
+ address = struct.unpack_from("<Q", data, offset)[0];
681
+ }
682
+ else if (link_type === 1) {
683
+ // soft link
684
+ const length_of_soft_link_value = struct.unpack_from("<H", data, offset)[0];
685
+ offset += 2;
686
+ address = new TextDecoder(encoding).decode(data.slice(offset, offset + length_of_soft_link_value));
687
+ }
688
+ return [creationorder, [name, address]];
689
+ }
690
+ _decode_link_info_msg(data, offset) {
691
+ const [version, flags] = struct.unpack_from("<BB", data, offset);
692
+ assert(version === 0);
693
+ offset += 2;
694
+ if ((flags & 1) > 0) {
695
+ // creation order present
696
+ offset += 8;
697
+ }
698
+ const fmt = (flags & 2) > 0 ? LINK_INFO_MSG2 : LINK_INFO_MSG1;
699
+ const link_info = _unpack_struct_from(fmt, data, offset);
700
+ const output = new Map();
701
+ for (const [k, v] of link_info.entries()) {
702
+ output.set(k, v === UNDEFINED_ADDRESS ? null : v);
703
+ }
704
+ // Ensure order_btree_address exists (may be absent in LINK_INFO_MSG1)
705
+ if (!output.has("order_btree_address")) {
706
+ output.set("order_btree_address", null);
707
+ }
708
+ return output;
709
+ }
710
+ // ────────── Async: get_attributes ──────────
711
+ async get_attributes() {
712
+ const attrs = {};
713
+ const attr_msgs = this.find_msg_type(ATTRIBUTE_MSG_TYPE);
714
+ for (const msg of attr_msgs) {
715
+ const absOffset = msg.get("offset_to_message");
716
+ const relOffset = absOffset - this.bufStart;
717
+ try {
718
+ const [name, value] = await this._unpack_attribute(relOffset);
719
+ attrs[name] = value;
720
+ }
721
+ catch {
722
+ // skip unreadable attributes
723
+ }
724
+ }
725
+ // Handle ATTRIBUTE_INFO_MSG_TYPE (dense attribute storage)
726
+ const attr_info_msgs = this.find_msg_type(ATTRIBUTE_INFO_MSG_TYPE);
727
+ for (const info_msg of attr_info_msgs) {
728
+ try {
729
+ const newAttrs = await this._read_dense_attributes(info_msg);
730
+ Object.assign(attrs, newAttrs);
731
+ }
732
+ catch {
733
+ // skip if not supported
734
+ }
735
+ }
736
+ return attrs;
737
+ }
738
+ async _read_dense_attributes(info_msg) {
739
+ const absOffset = info_msg.get("offset_to_message");
740
+ let offset = absOffset - this.bufStart;
741
+ const buf = this.buf;
742
+ const [version, flags] = struct.unpack_from("<BB", buf, offset);
743
+ if (version !== 0)
744
+ return {};
745
+ offset += 2;
746
+ // If flags bit 0: max creation index (2 bytes)
747
+ if (flags & 1)
748
+ offset += 2;
749
+ // fractal_heap_address (8 bytes), name_btree_address (8 bytes)
750
+ const [fractal_heap_address, name_btree_address] = struct.unpack_from("<QQ", buf, offset);
751
+ offset += 16;
752
+ // If flags bit 1: order_btree_address (8 bytes)
753
+ let order_btree_address = null;
754
+ if (flags & 2) {
755
+ order_btree_address = struct.unpack_from("<Q", buf, offset)[0];
756
+ offset += 8;
757
+ }
758
+ if (fractal_heap_address === UNDEFINED_ADDRESS || name_btree_address === UNDEFINED_ADDRESS) {
759
+ return {};
760
+ }
761
+ const heap = await FractalHeap.create(this._source, fractal_heap_address);
762
+ const ordered = order_btree_address !== null && order_btree_address !== UNDEFINED_ADDRESS;
763
+ let btree;
764
+ if (ordered) {
765
+ btree = await BTreeV2GroupOrders.create(this._source, order_btree_address);
766
+ }
767
+ else {
768
+ btree = await BTreeV2GroupNames.create(this._source, name_btree_address);
769
+ }
770
+ const attrs = {};
771
+ for (const record of btree.iter_records()) {
772
+ try {
773
+ const attr_data = heap.get_data(record.get("heapid"));
774
+ const [name, value] = await this._unpack_attribute_from_buf(attr_data, 0);
775
+ attrs[name] = value;
776
+ }
777
+ catch {
778
+ // skip
779
+ }
780
+ }
781
+ return attrs;
782
+ }
783
+ async _unpack_attribute(relOffset) {
784
+ return this._unpack_attribute_from_buf(this.buf, relOffset);
785
+ }
786
+ async _unpack_attribute_from_buf(buf, offset) {
787
+ const version = struct.unpack_from("<B", buf, offset)[0];
788
+ let attr_map;
789
+ let padding_multiple;
790
+ if (version === 1) {
791
+ attr_map = _unpack_struct_from(ATTR_MSG_HEADER_V1, buf, offset);
792
+ assert(attr_map.get("version") === 1);
793
+ offset += ATTR_MSG_HEADER_V1_SIZE;
794
+ padding_multiple = 8;
795
+ }
796
+ else if (version === 3) {
797
+ attr_map = _unpack_struct_from(ATTR_MSG_HEADER_V3, buf, offset);
798
+ assert(attr_map.get("version") === 3);
799
+ offset += ATTR_MSG_HEADER_V3_SIZE;
800
+ padding_multiple = 1;
801
+ }
802
+ else {
803
+ throw new Error(`Unsupported attribute message version: ${version}`);
804
+ }
805
+ // Read attribute name
806
+ const name_size = attr_map.get("name_size");
807
+ let name = struct.unpack_from("<" + name_size.toFixed() + "s", buf, offset)[0];
808
+ name = name.replace(/\x00$/, "");
809
+ offset += _padded_size(name_size, padding_multiple);
810
+ // Read datatype
811
+ let dtype;
812
+ try {
813
+ dtype = new DatatypeMessage(buf, offset).dtype;
814
+ }
815
+ catch {
816
+ console.warn(`Attribute '${name}' type not implemented, set to null.`);
817
+ return [name, null];
818
+ }
819
+ const datatype_size = attr_map.get("datatype_size");
820
+ offset += _padded_size(datatype_size, padding_multiple);
821
+ // Read dataspace (shape)
822
+ const shape = this._determine_data_shape(buf, offset);
823
+ const items = shape.reduce((a, b) => a * b, 1);
824
+ const dataspace_size = attr_map.get("dataspace_size");
825
+ offset += _padded_size(dataspace_size, padding_multiple);
826
+ // Read value
827
+ let value = await this._attr_value(dtype, buf, items, offset);
828
+ if (shape.length === 0) {
829
+ value = value[0];
830
+ }
831
+ return [name, value];
832
+ }
833
+ async _attr_value(dtype, buf, count, offset) {
834
+ const value = new Array(count);
835
+ if (Array.isArray(dtype)) {
836
+ const dtype_class = dtype[0];
837
+ for (let i = 0; i < count; i++) {
838
+ if (dtype_class === "VLEN_STRING") {
839
+ const character_set = dtype[2];
840
+ const [, vlen_data] = await this._vlen_size_and_data(buf, offset);
841
+ const encoding = character_set === 0 ? "ascii" : "utf-8";
842
+ value[i] = new TextDecoder(encoding).decode(vlen_data);
843
+ offset += 16;
844
+ }
845
+ else if (dtype_class === "REFERENCE") {
846
+ const address = struct.unpack_from("<Q", buf, offset);
847
+ value[i] = address;
848
+ offset += 8;
849
+ }
850
+ else if (dtype_class === "VLEN_SEQUENCE") {
851
+ const base_dtype = dtype[1];
852
+ const [vlen, vlen_data] = await this._vlen_size_and_data(buf, offset);
853
+ value[i] = await this._attr_value(base_dtype, vlen_data, vlen, 0);
854
+ offset += 16;
855
+ }
856
+ else {
857
+ throw new Error("NotImplementedError: unsupported dtype class " + dtype_class);
858
+ }
859
+ }
860
+ }
861
+ else {
862
+ const [getter, big_endian, size] = dtype_getter(dtype);
863
+ const view = new DataView64(buf, 0);
864
+ for (let i = 0; i < count; i++) {
865
+ value[i] = view[getter](offset, !big_endian, size);
866
+ offset += size;
867
+ }
868
+ }
869
+ return value;
870
+ }
871
+ async _vlen_size_and_data(buf, offset) {
872
+ const vlen_size = struct.unpack_from("<I", buf, offset)[0];
873
+ const gheap_id = _unpack_struct_from(GLOBAL_HEAP_ID, buf, offset + 4);
874
+ const gheap_address = gheap_id.get("collection_address");
875
+ assert(gheap_address < Number.MAX_SAFE_INTEGER);
876
+ let gheap;
877
+ if (this._global_heaps.has(gheap_address)) {
878
+ gheap = this._global_heaps.get(gheap_address);
879
+ }
880
+ else {
881
+ gheap = await GlobalHeap.create(this._source, gheap_address);
882
+ this._global_heaps.set(gheap_address, gheap);
883
+ }
884
+ const vlen_data = gheap.objects.get(gheap_id.get("object_index"));
885
+ if (!vlen_data) {
886
+ throw new Error(`Global heap object not found at index ${gheap_id.get("object_index")}`);
887
+ }
888
+ return [vlen_size, vlen_data];
889
+ }
890
+ // ────────── Async: get_data ──────────
891
+ async get_data() {
892
+ const msg = this.find_msg_type(DATA_STORAGE_MSG_TYPE)[0];
893
+ if (!msg)
894
+ return [];
895
+ const absOffset = msg.get("offset_to_message");
896
+ const relOffset = absOffset - this.bufStart;
897
+ const { layout_class, property_offset } = this._get_data_message_properties(relOffset);
898
+ if (layout_class === 0) {
899
+ throw new Error("Compact storage of DataObject not implemented");
900
+ }
901
+ else if (layout_class === 1) {
902
+ return this._get_contiguous_data(property_offset);
903
+ }
904
+ else if (layout_class === 2) {
905
+ return this._get_chunked_data(relOffset);
906
+ }
907
+ return [];
908
+ }
909
+ async _get_contiguous_data(property_offset) {
910
+ const buf = this.buf;
911
+ const data_offset = struct.unpack_from("<Q", buf, property_offset)[0];
912
+ const shape = this.shape;
913
+ const fullsize = shape.reduce((a, b) => a * b, 1);
914
+ if (data_offset === UNDEFINED_ADDRESS) {
915
+ return new Array(fullsize);
916
+ }
917
+ const dtype = this.dtype;
918
+ if (!Array.isArray(dtype)) {
919
+ if (/[<>=!@|]?(i|u|f|S)(\d*)/.test(dtype)) {
920
+ const [item_getter, item_is_big_endian, item_size] = dtype_getter(dtype);
921
+ const dataBuf = await fetchRange(this._source, data_offset, fullsize * item_size);
922
+ const view = new DataView64(dataBuf, 0);
923
+ const output = new Array(fullsize);
924
+ for (let i = 0; i < fullsize; i++) {
925
+ output[i] = view[item_getter](i * item_size, !item_is_big_endian, item_size);
926
+ }
927
+ return output;
928
+ }
929
+ else {
930
+ throw new Error("Not implemented - no proper dtype defined: " + dtype);
931
+ }
932
+ }
933
+ else {
934
+ const dtype_class = dtype[0];
935
+ if (dtype_class === "REFERENCE") {
936
+ const size = dtype[1];
937
+ if (size !== 8) {
938
+ throw new Error("Unsupported Reference type size: " + size);
939
+ }
940
+ const dataBuf = await fetchRange(this._source, data_offset, fullsize * 8);
941
+ return [dataBuf];
942
+ }
943
+ else if (dtype_class === "VLEN_STRING") {
944
+ const character_set = dtype[2];
945
+ const encoding = character_set === 0 ? "ascii" : "utf-8";
946
+ const decoder = new TextDecoder(encoding);
947
+ // Fetch all 16-byte vlen refs at once
948
+ const refBuf = await fetchRange(this._source, data_offset, fullsize * 16);
949
+ const output = new Array(fullsize);
950
+ for (let i = 0; i < fullsize; i++) {
951
+ const [, vlen_data] = await this._vlen_size_and_data(refBuf, i * 16);
952
+ output[i] = decoder.decode(vlen_data);
953
+ }
954
+ return output;
955
+ }
956
+ else {
957
+ throw new Error("Not implemented datatype class: " + dtype_class);
958
+ }
959
+ }
960
+ }
961
+ async _get_chunked_data(msg_offset) {
962
+ this._get_chunk_params();
963
+ if (this._chunk_address === null || this._chunk_address === UNDEFINED_ADDRESS) {
964
+ return [];
965
+ }
966
+ const dtype = this.dtype;
967
+ if (!Array.isArray(dtype) || !/^VLEN/.test(dtype[0])) {
968
+ // For non-VLEN chunked data, not needed for our use case
969
+ // (processChunkReferences handles numeric chunked data via BTree directly)
970
+ return [];
971
+ }
972
+ const dtype_class = dtype[0];
973
+ const character_set = dtype[2];
974
+ const encoding = character_set === 0 ? "ascii" : "utf-8";
975
+ const decoder = new TextDecoder(encoding);
976
+ const chunk_btree = await BTreeV1RawDataChunks.create(this._source, this._chunk_address, this._chunk_dims);
977
+ const output = [];
978
+ const leafNodes = chunk_btree.all_nodes.get(0);
979
+ if (!leafNodes)
980
+ return output;
981
+ for (const node of leafNodes) {
982
+ for (let ik = 0; ik < node.keys.length; ik++) {
983
+ const addr = node.addresses[ik];
984
+ const chunk_size = node.keys[ik].chunk_size;
985
+ // Fetch chunk data
986
+ const chunkBuf = await fetchRange(this._source, addr, chunk_size);
987
+ const nItems = chunk_size / 16; // each VLEN ref is 16 bytes
988
+ for (let i = 0; i < nItems; i++) {
989
+ try {
990
+ const [, vlen_data] = await this._vlen_size_and_data(chunkBuf, i * 16);
991
+ output.push(decoder.decode(vlen_data));
992
+ }
993
+ catch {
994
+ // skip bad entries
995
+ }
996
+ }
997
+ }
998
+ }
999
+ return output;
1000
+ }
1001
+ }