hdf5-as-virtual-zarr 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +39 -0
- package/dist/hdf-store.d.ts +9 -0
- package/dist/hdf-store.js +18 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +3 -0
- package/dist/jsfive/btree.d.ts +52 -0
- package/dist/jsfive/btree.js +354 -0
- package/dist/jsfive/core.d.ts +35 -0
- package/dist/jsfive/core.js +196 -0
- package/dist/jsfive/dataobjects.d.ts +64 -0
- package/dist/jsfive/dataobjects.js +1001 -0
- package/dist/jsfive/datatype-msg.d.ts +15 -0
- package/dist/jsfive/datatype-msg.js +110 -0
- package/dist/jsfive/high-level.d.ts +42 -0
- package/dist/jsfive/high-level.js +150 -0
- package/dist/jsfive/misc-low-level.d.ts +67 -0
- package/dist/jsfive/misc-low-level.js +506 -0
- package/dist/ref-spec-to-consolidated-metadata.d.ts +12 -0
- package/dist/ref-spec-to-consolidated-metadata.js +22 -0
- package/dist/ref-store.d.ts +10 -0
- package/dist/ref-store.js +79 -0
- package/dist/single-hdf5-to-zarr.d.ts +76 -0
- package/dist/single-hdf5-to-zarr.js +745 -0
- package/dist/types.d.ts +34 -0
- package/dist/types.js +12 -0
- package/package.json +53 -0
- package/src/hdf-store.ts +25 -0
- package/src/index.ts +3 -0
- package/src/jsfive/README.md +1 -0
- package/src/jsfive/btree.ts +474 -0
- package/src/jsfive/core.ts +228 -0
- package/src/jsfive/dataobjects.ts +1262 -0
- package/src/jsfive/datatype-msg.ts +111 -0
- package/src/jsfive/high-level.ts +182 -0
- package/src/jsfive/misc-low-level.ts +641 -0
- package/src/ref-spec-to-consolidated-metadata.ts +38 -0
- package/src/ref-store.ts +115 -0
- package/src/single-hdf5-to-zarr.ts +868 -0
- package/src/types.ts +49 -0
- package/test/fixtures/dense.adata.zarr/.zattrs +4 -0
- package/test/fixtures/dense.adata.zarr/.zgroup +3 -0
- package/test/fixtures/dense.adata.zarr/.zmetadata +285 -0
- package/test/fixtures/dense.adata.zarr/X/.zarray +22 -0
- package/test/fixtures/dense.adata.zarr/X/.zattrs +4 -0
- package/test/fixtures/dense.adata.zarr/X/0.0 +0 -0
- package/test/fixtures/dense.adata.zarr/layers/.zattrs +4 -0
- package/test/fixtures/dense.adata.zarr/layers/.zgroup +3 -0
- package/test/fixtures/dense.adata.zarr/obs/.zattrs +9 -0
- package/test/fixtures/dense.adata.zarr/obs/.zgroup +3 -0
- package/test/fixtures/dense.adata.zarr/obs/_index/.zarray +24 -0
- package/test/fixtures/dense.adata.zarr/obs/_index/.zattrs +4 -0
- package/test/fixtures/dense.adata.zarr/obs/_index/0 +0 -0
- package/test/fixtures/dense.adata.zarr/obs/categorical/.zattrs +5 -0
- package/test/fixtures/dense.adata.zarr/obs/categorical/.zgroup +3 -0
- package/test/fixtures/dense.adata.zarr/obs/categorical/categories/.zarray +24 -0
- package/test/fixtures/dense.adata.zarr/obs/categorical/categories/.zattrs +4 -0
- package/test/fixtures/dense.adata.zarr/obs/categorical/categories/0 +0 -0
- package/test/fixtures/dense.adata.zarr/obs/categorical/codes/.zarray +20 -0
- package/test/fixtures/dense.adata.zarr/obs/categorical/codes/.zattrs +4 -0
- package/test/fixtures/dense.adata.zarr/obs/categorical/codes/0 +0 -0
- package/test/fixtures/dense.adata.zarr/obs/string/.zarray +24 -0
- package/test/fixtures/dense.adata.zarr/obs/string/.zattrs +4 -0
- package/test/fixtures/dense.adata.zarr/obs/string/0 +0 -0
- package/test/fixtures/dense.adata.zarr/obsm/.zattrs +4 -0
- package/test/fixtures/dense.adata.zarr/obsm/.zgroup +3 -0
- package/test/fixtures/dense.adata.zarr/obsm/X_umap/.zarray +22 -0
- package/test/fixtures/dense.adata.zarr/obsm/X_umap/.zattrs +4 -0
- package/test/fixtures/dense.adata.zarr/obsm/X_umap/0.0 +0 -0
- package/test/fixtures/dense.adata.zarr/obsp/.zattrs +4 -0
- package/test/fixtures/dense.adata.zarr/obsp/.zgroup +3 -0
- package/test/fixtures/dense.adata.zarr/raw/.zarray +10 -0
- package/test/fixtures/dense.adata.zarr/raw/.zattrs +4 -0
- package/test/fixtures/dense.adata.zarr/raw/0 +0 -0
- package/test/fixtures/dense.adata.zarr/uns/.zattrs +4 -0
- package/test/fixtures/dense.adata.zarr/uns/.zgroup +3 -0
- package/test/fixtures/dense.adata.zarr/var/.zattrs +6 -0
- package/test/fixtures/dense.adata.zarr/var/.zgroup +3 -0
- package/test/fixtures/dense.adata.zarr/var/_index/.zarray +24 -0
- package/test/fixtures/dense.adata.zarr/var/_index/.zattrs +4 -0
- package/test/fixtures/dense.adata.zarr/var/_index/0 +0 -0
- package/test/fixtures/dense.adata.zarr/varm/.zattrs +4 -0
- package/test/fixtures/dense.adata.zarr/varm/.zgroup +3 -0
- package/test/fixtures/dense.adata.zarr/varp/.zattrs +4 -0
- package/test/fixtures/dense.adata.zarr/varp/.zgroup +3 -0
- package/test/fixtures/dense.adata.zarr.json +182 -0
- package/test/fixtures/dense.adata.zmetadata.json +1 -0
- package/test/fixtures/dense.h5ad +0 -0
- package/test/fixtures/dense.h5ad.refspec.json +1 -0
- package/test/fixtures/dense.v3.adata.zarr/X/c/0/0 +0 -0
- package/test/fixtures/dense.v3.adata.zarr/X/zarr.json +45 -0
- package/test/fixtures/dense.v3.adata.zarr/layers/zarr.json +8 -0
- package/test/fixtures/dense.v3.adata.zarr/obs/_index/c/0 +0 -0
- package/test/fixtures/dense.v3.adata.zarr/obs/_index/zarr.json +41 -0
- package/test/fixtures/dense.v3.adata.zarr/obs/categorical/categories/c/0 +0 -0
- package/test/fixtures/dense.v3.adata.zarr/obs/categorical/categories/zarr.json +41 -0
- package/test/fixtures/dense.v3.adata.zarr/obs/categorical/codes/c/0 +0 -0
- package/test/fixtures/dense.v3.adata.zarr/obs/categorical/codes/zarr.json +40 -0
- package/test/fixtures/dense.v3.adata.zarr/obs/categorical/zarr.json +9 -0
- package/test/fixtures/dense.v3.adata.zarr/obs/string/c/0 +0 -0
- package/test/fixtures/dense.v3.adata.zarr/obs/string/zarr.json +41 -0
- package/test/fixtures/dense.v3.adata.zarr/obs/zarr.json +13 -0
- package/test/fixtures/dense.v3.adata.zarr/obsm/X_umap/c/0/0 +0 -0
- package/test/fixtures/dense.v3.adata.zarr/obsm/X_umap/zarr.json +45 -0
- package/test/fixtures/dense.v3.adata.zarr/obsm/zarr.json +8 -0
- package/test/fixtures/dense.v3.adata.zarr/obsp/zarr.json +8 -0
- package/test/fixtures/dense.v3.adata.zarr/raw/zarr.json +36 -0
- package/test/fixtures/dense.v3.adata.zarr/uns/zarr.json +8 -0
- package/test/fixtures/dense.v3.adata.zarr/var/_index/c/0 +0 -0
- package/test/fixtures/dense.v3.adata.zarr/var/_index/zarr.json +41 -0
- package/test/fixtures/dense.v3.adata.zarr/var/zarr.json +10 -0
- package/test/fixtures/dense.v3.adata.zarr/varm/zarr.json +8 -0
- package/test/fixtures/dense.v3.adata.zarr/varp/zarr.json +8 -0
- package/test/fixtures/dense.v3.adata.zarr/zarr.json +469 -0
- package/test/fixtures/dense.v3.adata.zarr.json +102 -0
- package/test/fixtures/generate_fixtures.py +157 -0
- package/test/fixtures/generate_fixtures_v3.py +126 -0
- package/test/fixtures/minimal.adata.zarr/.zattrs +4 -0
- package/test/fixtures/minimal.adata.zarr/.zgroup +3 -0
- package/test/fixtures/minimal.adata.zarr/.zmetadata +168 -0
- package/test/fixtures/minimal.adata.zarr/X/.zarray +22 -0
- package/test/fixtures/minimal.adata.zarr/X/.zattrs +4 -0
- package/test/fixtures/minimal.adata.zarr/X/0.0 +0 -0
- package/test/fixtures/minimal.adata.zarr/layers/.zattrs +4 -0
- package/test/fixtures/minimal.adata.zarr/layers/.zgroup +3 -0
- package/test/fixtures/minimal.adata.zarr/obs/.zattrs +6 -0
- package/test/fixtures/minimal.adata.zarr/obs/.zgroup +3 -0
- package/test/fixtures/minimal.adata.zarr/obs/_index/.zarray +24 -0
- package/test/fixtures/minimal.adata.zarr/obs/_index/.zattrs +4 -0
- package/test/fixtures/minimal.adata.zarr/obs/_index/0 +0 -0
- package/test/fixtures/minimal.adata.zarr/obsm/.zattrs +4 -0
- package/test/fixtures/minimal.adata.zarr/obsm/.zgroup +3 -0
- package/test/fixtures/minimal.adata.zarr/obsp/.zattrs +4 -0
- package/test/fixtures/minimal.adata.zarr/obsp/.zgroup +3 -0
- package/test/fixtures/minimal.adata.zarr/raw/.zarray +10 -0
- package/test/fixtures/minimal.adata.zarr/raw/.zattrs +4 -0
- package/test/fixtures/minimal.adata.zarr/raw/0 +0 -0
- package/test/fixtures/minimal.adata.zarr/uns/.zattrs +4 -0
- package/test/fixtures/minimal.adata.zarr/uns/.zgroup +3 -0
- package/test/fixtures/minimal.adata.zarr/var/.zattrs +6 -0
- package/test/fixtures/minimal.adata.zarr/var/.zgroup +3 -0
- package/test/fixtures/minimal.adata.zarr/var/_index/.zarray +24 -0
- package/test/fixtures/minimal.adata.zarr/var/_index/.zattrs +4 -0
- package/test/fixtures/minimal.adata.zarr/var/_index/0 +0 -0
- package/test/fixtures/minimal.adata.zarr/varm/.zattrs +4 -0
- package/test/fixtures/minimal.adata.zarr/varm/.zgroup +3 -0
- package/test/fixtures/minimal.adata.zarr/varp/.zattrs +4 -0
- package/test/fixtures/minimal.adata.zarr/varp/.zgroup +3 -0
- package/test/fixtures/minimal.adata.zarr.json +126 -0
- package/test/fixtures/minimal.adata.zmetadata.json +1 -0
- package/test/fixtures/minimal.h5ad +0 -0
- package/test/fixtures/minimal.h5ad.refspec.json +1 -0
- package/test/fixtures/minimal.v3.adata.zarr/X/c/0/0 +0 -0
- package/test/fixtures/minimal.v3.adata.zarr/X/zarr.json +45 -0
- package/test/fixtures/minimal.v3.adata.zarr/layers/zarr.json +8 -0
- package/test/fixtures/minimal.v3.adata.zarr/obs/_index/c/0 +0 -0
- package/test/fixtures/minimal.v3.adata.zarr/obs/_index/zarr.json +41 -0
- package/test/fixtures/minimal.v3.adata.zarr/obs/zarr.json +10 -0
- package/test/fixtures/minimal.v3.adata.zarr/obsm/zarr.json +8 -0
- package/test/fixtures/minimal.v3.adata.zarr/obsp/zarr.json +8 -0
- package/test/fixtures/minimal.v3.adata.zarr/raw/zarr.json +36 -0
- package/test/fixtures/minimal.v3.adata.zarr/uns/zarr.json +8 -0
- package/test/fixtures/minimal.v3.adata.zarr/var/_index/c/0 +0 -0
- package/test/fixtures/minimal.v3.adata.zarr/var/_index/zarr.json +41 -0
- package/test/fixtures/minimal.v3.adata.zarr/var/zarr.json +10 -0
- package/test/fixtures/minimal.v3.adata.zarr/varm/zarr.json +8 -0
- package/test/fixtures/minimal.v3.adata.zarr/varp/zarr.json +8 -0
- package/test/fixtures/minimal.v3.adata.zarr/zarr.json +285 -0
- package/test/fixtures/minimal.v3.adata.zarr.json +66 -0
- package/test/fixtures/mouse_liver.h5ad +0 -0
- package/test/fixtures/mouse_liver.h5ad.refspec.json +1 -0
- package/test/fixtures/mouse_liver.py +44 -0
- package/test/fixtures/sparse.adata.zarr/.zattrs +4 -0
- package/test/fixtures/sparse.adata.zarr/.zgroup +3 -0
- package/test/fixtures/sparse.adata.zarr/.zmetadata +304 -0
- package/test/fixtures/sparse.adata.zarr/X/.zattrs +8 -0
- package/test/fixtures/sparse.adata.zarr/X/.zgroup +3 -0
- package/test/fixtures/sparse.adata.zarr/X/data/.zarray +20 -0
- package/test/fixtures/sparse.adata.zarr/X/data/0 +0 -0
- package/test/fixtures/sparse.adata.zarr/X/indices/.zarray +20 -0
- package/test/fixtures/sparse.adata.zarr/X/indices/0 +0 -0
- package/test/fixtures/sparse.adata.zarr/X/indptr/.zarray +20 -0
- package/test/fixtures/sparse.adata.zarr/X/indptr/0 +0 -0
- package/test/fixtures/sparse.adata.zarr/layers/.zattrs +4 -0
- package/test/fixtures/sparse.adata.zarr/layers/.zgroup +3 -0
- package/test/fixtures/sparse.adata.zarr/obs/.zattrs +9 -0
- package/test/fixtures/sparse.adata.zarr/obs/.zgroup +3 -0
- package/test/fixtures/sparse.adata.zarr/obs/_index/.zarray +24 -0
- package/test/fixtures/sparse.adata.zarr/obs/_index/.zattrs +4 -0
- package/test/fixtures/sparse.adata.zarr/obs/_index/0 +0 -0
- package/test/fixtures/sparse.adata.zarr/obs/categorical/.zattrs +5 -0
- package/test/fixtures/sparse.adata.zarr/obs/categorical/.zgroup +3 -0
- package/test/fixtures/sparse.adata.zarr/obs/categorical/categories/.zarray +24 -0
- package/test/fixtures/sparse.adata.zarr/obs/categorical/categories/.zattrs +4 -0
- package/test/fixtures/sparse.adata.zarr/obs/categorical/categories/0 +0 -0
- package/test/fixtures/sparse.adata.zarr/obs/categorical/codes/.zarray +20 -0
- package/test/fixtures/sparse.adata.zarr/obs/categorical/codes/.zattrs +4 -0
- package/test/fixtures/sparse.adata.zarr/obs/categorical/codes/0 +0 -0
- package/test/fixtures/sparse.adata.zarr/obs/string/.zarray +24 -0
- package/test/fixtures/sparse.adata.zarr/obs/string/.zattrs +4 -0
- package/test/fixtures/sparse.adata.zarr/obs/string/0 +0 -0
- package/test/fixtures/sparse.adata.zarr/obsm/.zattrs +4 -0
- package/test/fixtures/sparse.adata.zarr/obsm/.zgroup +3 -0
- package/test/fixtures/sparse.adata.zarr/obsp/.zattrs +4 -0
- package/test/fixtures/sparse.adata.zarr/obsp/.zgroup +3 -0
- package/test/fixtures/sparse.adata.zarr/raw/.zarray +10 -0
- package/test/fixtures/sparse.adata.zarr/raw/.zattrs +4 -0
- package/test/fixtures/sparse.adata.zarr/raw/0 +0 -0
- package/test/fixtures/sparse.adata.zarr/uns/.zattrs +4 -0
- package/test/fixtures/sparse.adata.zarr/uns/.zgroup +3 -0
- package/test/fixtures/sparse.adata.zarr/var/.zattrs +6 -0
- package/test/fixtures/sparse.adata.zarr/var/.zgroup +3 -0
- package/test/fixtures/sparse.adata.zarr/var/_index/.zarray +24 -0
- package/test/fixtures/sparse.adata.zarr/var/_index/.zattrs +4 -0
- package/test/fixtures/sparse.adata.zarr/var/_index/0 +0 -0
- package/test/fixtures/sparse.adata.zarr/varm/.zattrs +4 -0
- package/test/fixtures/sparse.adata.zarr/varm/.zgroup +3 -0
- package/test/fixtures/sparse.adata.zarr/varp/.zattrs +4 -0
- package/test/fixtures/sparse.adata.zarr/varp/.zgroup +3 -0
- package/test/fixtures/sparse.adata.zarr.json +190 -0
- package/test/fixtures/sparse.adata.zmetadata.json +1 -0
- package/test/fixtures/sparse.h5ad +0 -0
- package/test/fixtures/sparse.h5ad.refspec.json +1 -0
- package/test/fixtures/sparse.v3.adata.zarr/X/data/c/0 +0 -0
- package/test/fixtures/sparse.v3.adata.zarr/X/data/zarr.json +40 -0
- package/test/fixtures/sparse.v3.adata.zarr/X/indices/c/0 +0 -0
- package/test/fixtures/sparse.v3.adata.zarr/X/indices/zarr.json +40 -0
- package/test/fixtures/sparse.v3.adata.zarr/X/indptr/c/0 +0 -0
- package/test/fixtures/sparse.v3.adata.zarr/X/indptr/zarr.json +40 -0
- package/test/fixtures/sparse.v3.adata.zarr/X/zarr.json +12 -0
- package/test/fixtures/sparse.v3.adata.zarr/layers/zarr.json +8 -0
- package/test/fixtures/sparse.v3.adata.zarr/obs/_index/c/0 +0 -0
- package/test/fixtures/sparse.v3.adata.zarr/obs/_index/zarr.json +41 -0
- package/test/fixtures/sparse.v3.adata.zarr/obs/categorical/categories/c/0 +0 -0
- package/test/fixtures/sparse.v3.adata.zarr/obs/categorical/categories/zarr.json +41 -0
- package/test/fixtures/sparse.v3.adata.zarr/obs/categorical/codes/c/0 +0 -0
- package/test/fixtures/sparse.v3.adata.zarr/obs/categorical/codes/zarr.json +40 -0
- package/test/fixtures/sparse.v3.adata.zarr/obs/categorical/zarr.json +9 -0
- package/test/fixtures/sparse.v3.adata.zarr/obs/string/c/0 +0 -0
- package/test/fixtures/sparse.v3.adata.zarr/obs/string/zarr.json +41 -0
- package/test/fixtures/sparse.v3.adata.zarr/obs/zarr.json +13 -0
- package/test/fixtures/sparse.v3.adata.zarr/obsm/zarr.json +8 -0
- package/test/fixtures/sparse.v3.adata.zarr/obsp/zarr.json +8 -0
- package/test/fixtures/sparse.v3.adata.zarr/raw/zarr.json +36 -0
- package/test/fixtures/sparse.v3.adata.zarr/uns/zarr.json +8 -0
- package/test/fixtures/sparse.v3.adata.zarr/var/_index/c/0 +0 -0
- package/test/fixtures/sparse.v3.adata.zarr/var/_index/zarr.json +41 -0
- package/test/fixtures/sparse.v3.adata.zarr/var/zarr.json +10 -0
- package/test/fixtures/sparse.v3.adata.zarr/varm/zarr.json +8 -0
- package/test/fixtures/sparse.v3.adata.zarr/varp/zarr.json +8 -0
- package/test/fixtures/sparse.v3.adata.zarr/zarr.json +516 -0
- package/test/fixtures/sparse.v3.adata.zarr.json +114 -0
- package/test/hdf-store.test.ts +268 -0
- package/test/index.test.ts +762 -0
- package/test/partial-reads.test.ts +150 -0
|
@@ -0,0 +1,506 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vendored and adapted from jsfive/esm/misc-low-level.js.
|
|
3
|
+
* SuperBlock, Heap, SymbolTable, GlobalHeap, FractalHeap.
|
|
4
|
+
*
|
|
5
|
+
* All classes use async factory methods (static async create())
|
|
6
|
+
* to read byte ranges via AsyncReadable.fetch() instead of synchronous
|
|
7
|
+
* slicing of a monolithic ArrayBuffer.
|
|
8
|
+
*/
|
|
9
|
+
import { fetchRange } from "../types.js";
|
|
10
|
+
import { _structure_size, _padded_size, _unpack_struct_from, struct, assert, _unpack_integer, bitSize, } from "./core.js";
|
|
11
|
+
// ────────── format constants ──────────
|
|
12
|
+
const FORMAT_SIGNATURE = struct.unpack_from("8s", new Uint8Array([137, 72, 68, 70, 13, 10, 26, 10]).buffer)[0];
|
|
13
|
+
export const UNDEFINED_ADDRESS = struct.unpack_from("<Q", new Uint8Array([255, 255, 255, 255, 255, 255, 255, 255]).buffer)[0];
|
|
14
|
+
// Version 0 SUPERBLOCK
|
|
15
|
+
const SUPERBLOCK_V0 = new Map([
|
|
16
|
+
["format_signature", "8s"],
|
|
17
|
+
["superblock_version", "B"],
|
|
18
|
+
["free_storage_version", "B"],
|
|
19
|
+
["root_group_version", "B"],
|
|
20
|
+
["reserved_0", "B"],
|
|
21
|
+
["shared_header_version", "B"],
|
|
22
|
+
["offset_size", "B"],
|
|
23
|
+
["length_size", "B"],
|
|
24
|
+
["reserved_1", "B"],
|
|
25
|
+
["group_leaf_node_k", "H"],
|
|
26
|
+
["group_internal_node_k", "H"],
|
|
27
|
+
["file_consistency_flags", "L"],
|
|
28
|
+
["base_address_lower", "Q"],
|
|
29
|
+
["free_space_address", "Q"],
|
|
30
|
+
["end_of_file_address", "Q"],
|
|
31
|
+
["driver_information_address", "Q"],
|
|
32
|
+
]);
|
|
33
|
+
const SUPERBLOCK_V0_SIZE = _structure_size(SUPERBLOCK_V0);
|
|
34
|
+
const SUPERBLOCK_V2_V3 = new Map([
|
|
35
|
+
["format_signature", "8s"],
|
|
36
|
+
["superblock_version", "B"],
|
|
37
|
+
["offset_size", "B"],
|
|
38
|
+
["length_size", "B"],
|
|
39
|
+
["file_consistency_flags", "B"],
|
|
40
|
+
["base_address", "Q"],
|
|
41
|
+
["superblock_extension_address", "Q"],
|
|
42
|
+
["end_of_file_address", "Q"],
|
|
43
|
+
["root_group_address", "Q"],
|
|
44
|
+
["superblock_checksum", "I"],
|
|
45
|
+
]);
|
|
46
|
+
const SUPERBLOCK_V2_V3_SIZE = _structure_size(SUPERBLOCK_V2_V3);
|
|
47
|
+
const SYMBOL_TABLE_ENTRY = new Map([
|
|
48
|
+
["link_name_offset", "Q"],
|
|
49
|
+
["object_header_address", "Q"],
|
|
50
|
+
["cache_type", "I"],
|
|
51
|
+
["reserved", "I"],
|
|
52
|
+
["scratch", "16s"],
|
|
53
|
+
]);
|
|
54
|
+
const SYMBOL_TABLE_ENTRY_SIZE = _structure_size(SYMBOL_TABLE_ENTRY);
|
|
55
|
+
const SYMBOL_TABLE_NODE = new Map([
|
|
56
|
+
["signature", "4s"],
|
|
57
|
+
["version", "B"],
|
|
58
|
+
["reserved_0", "B"],
|
|
59
|
+
["symbols", "H"],
|
|
60
|
+
]);
|
|
61
|
+
const SYMBOL_TABLE_NODE_SIZE = _structure_size(SYMBOL_TABLE_NODE);
|
|
62
|
+
const LOCAL_HEAP = new Map([
|
|
63
|
+
["signature", "4s"],
|
|
64
|
+
["version", "B"],
|
|
65
|
+
["reserved", "3s"],
|
|
66
|
+
["data_segment_size", "Q"],
|
|
67
|
+
["offset_to_free_list", "Q"],
|
|
68
|
+
["address_of_data_segment", "Q"],
|
|
69
|
+
]);
|
|
70
|
+
const GLOBAL_HEAP_HEADER = new Map([
|
|
71
|
+
["signature", "4s"],
|
|
72
|
+
["version", "B"],
|
|
73
|
+
["reserved", "3s"],
|
|
74
|
+
["collection_size", "Q"],
|
|
75
|
+
]);
|
|
76
|
+
const GLOBAL_HEAP_HEADER_SIZE = _structure_size(GLOBAL_HEAP_HEADER);
|
|
77
|
+
const GLOBAL_HEAP_OBJECT = new Map([
|
|
78
|
+
["object_index", "H"],
|
|
79
|
+
["reference_count", "H"],
|
|
80
|
+
["reserved", "I"],
|
|
81
|
+
["object_size", "Q"],
|
|
82
|
+
]);
|
|
83
|
+
const GLOBAL_HEAP_OBJECT_SIZE = _structure_size(GLOBAL_HEAP_OBJECT);
|
|
84
|
+
const FRACTAL_HEAP_HEADER = new Map([
|
|
85
|
+
["signature", "4s"],
|
|
86
|
+
["version", "B"],
|
|
87
|
+
["object_index_size", "H"],
|
|
88
|
+
["filter_info_size", "H"],
|
|
89
|
+
["flags", "B"],
|
|
90
|
+
["max_managed_object_size", "I"],
|
|
91
|
+
["next_huge_object_index", "Q"],
|
|
92
|
+
["btree_address_huge_objects", "Q"],
|
|
93
|
+
["managed_freespace_size", "Q"],
|
|
94
|
+
["freespace_manager_address", "Q"],
|
|
95
|
+
["managed_space_size", "Q"],
|
|
96
|
+
["managed_alloc_size", "Q"],
|
|
97
|
+
["next_directblock_iterator_address", "Q"],
|
|
98
|
+
["managed_object_count", "Q"],
|
|
99
|
+
["huge_objects_total_size", "Q"],
|
|
100
|
+
["huge_object_count", "Q"],
|
|
101
|
+
["tiny_objects_total_size", "Q"],
|
|
102
|
+
["tiny_object_count", "Q"],
|
|
103
|
+
["table_width", "H"],
|
|
104
|
+
["starting_block_size", "Q"],
|
|
105
|
+
["maximum_direct_block_size", "Q"],
|
|
106
|
+
["log2_maximum_heap_size", "H"],
|
|
107
|
+
["indirect_starting_rows_count", "H"],
|
|
108
|
+
["root_block_address", "Q"],
|
|
109
|
+
["indirect_current_rows_count", "H"],
|
|
110
|
+
]);
|
|
111
|
+
// ────────── SuperBlock ──────────
|
|
112
|
+
export class SuperBlock {
|
|
113
|
+
version;
|
|
114
|
+
_contents;
|
|
115
|
+
_end_of_sblock;
|
|
116
|
+
_root_symbol_table = null;
|
|
117
|
+
_source;
|
|
118
|
+
constructor(source, contents, version, end_of_sblock) {
|
|
119
|
+
this._source = source;
|
|
120
|
+
this._contents = contents;
|
|
121
|
+
this.version = version;
|
|
122
|
+
this._end_of_sblock = end_of_sblock;
|
|
123
|
+
}
|
|
124
|
+
static async create(source, offset) {
|
|
125
|
+
// Read enough bytes for the larger superblock variant
|
|
126
|
+
const needed = Math.max(SUPERBLOCK_V0_SIZE, SUPERBLOCK_V2_V3_SIZE);
|
|
127
|
+
const buf = await fetchRange(source, offset, needed);
|
|
128
|
+
const version_hint = struct.unpack_from("<B", buf, 8)[0];
|
|
129
|
+
let contents;
|
|
130
|
+
let end_of_sblock;
|
|
131
|
+
if (version_hint === 0) {
|
|
132
|
+
contents = _unpack_struct_from(SUPERBLOCK_V0, buf, 0);
|
|
133
|
+
end_of_sblock = offset + SUPERBLOCK_V0_SIZE;
|
|
134
|
+
}
|
|
135
|
+
else if (version_hint === 2 || version_hint === 3) {
|
|
136
|
+
contents = _unpack_struct_from(SUPERBLOCK_V2_V3, buf, 0);
|
|
137
|
+
end_of_sblock = offset + SUPERBLOCK_V2_V3_SIZE;
|
|
138
|
+
}
|
|
139
|
+
else {
|
|
140
|
+
throw new Error("unsupported superblock version: " + version_hint);
|
|
141
|
+
}
|
|
142
|
+
if (contents.get("format_signature") !== FORMAT_SIGNATURE) {
|
|
143
|
+
throw new Error("Incorrect file signature: " + contents.get("format_signature"));
|
|
144
|
+
}
|
|
145
|
+
if (contents.get("offset_size") !== 8 || contents.get("length_size") !== 8) {
|
|
146
|
+
throw new Error("File uses non-64-bit addressing");
|
|
147
|
+
}
|
|
148
|
+
const version = contents.get("superblock_version");
|
|
149
|
+
return new SuperBlock(source, contents, version, end_of_sblock);
|
|
150
|
+
}
|
|
151
|
+
async getOffsetToDataobjects() {
|
|
152
|
+
if (this.version === 0) {
|
|
153
|
+
const sym_table = await SymbolTable.create(this._source, this._end_of_sblock, true);
|
|
154
|
+
this._root_symbol_table = sym_table;
|
|
155
|
+
return sym_table.group_offset;
|
|
156
|
+
}
|
|
157
|
+
else if (this.version === 2 || this.version === 3) {
|
|
158
|
+
return this._contents.get("root_group_address");
|
|
159
|
+
}
|
|
160
|
+
else {
|
|
161
|
+
throw new Error("Not implemented version = " + this.version);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
// ────────── Heap (local) ──────────
|
|
166
|
+
export class Heap {
|
|
167
|
+
data;
|
|
168
|
+
_contents;
|
|
169
|
+
constructor(contents, data) {
|
|
170
|
+
this._contents = contents;
|
|
171
|
+
this.data = data;
|
|
172
|
+
}
|
|
173
|
+
static async create(source, offset) {
|
|
174
|
+
const headerSize = _structure_size(LOCAL_HEAP);
|
|
175
|
+
const hBuf = await fetchRange(source, offset, headerSize);
|
|
176
|
+
const local_heap = _unpack_struct_from(LOCAL_HEAP, hBuf, 0);
|
|
177
|
+
assert(local_heap.get("signature") === "HEAP");
|
|
178
|
+
assert(local_heap.get("version") === 0);
|
|
179
|
+
const data_offset = local_heap.get("address_of_data_segment");
|
|
180
|
+
const seg_size = local_heap.get("data_segment_size");
|
|
181
|
+
const heap_data = await fetchRange(source, data_offset, seg_size);
|
|
182
|
+
local_heap.set("heap_data", heap_data);
|
|
183
|
+
return new Heap(local_heap, heap_data);
|
|
184
|
+
}
|
|
185
|
+
get_object_name(offset) {
|
|
186
|
+
const end = new Uint8Array(this.data).indexOf(0, offset);
|
|
187
|
+
const name_size = end - offset;
|
|
188
|
+
return struct.unpack_from("<" + name_size.toFixed() + "s", this.data, offset)[0];
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
// ────────── SymbolTable ──────────
|
|
192
|
+
export class SymbolTable {
|
|
193
|
+
entries;
|
|
194
|
+
group_offset;
|
|
195
|
+
_contents;
|
|
196
|
+
constructor(contents, entries, group_offset) {
|
|
197
|
+
this._contents = contents;
|
|
198
|
+
this.entries = entries;
|
|
199
|
+
this.group_offset = group_offset;
|
|
200
|
+
}
|
|
201
|
+
static async create(source, offset, root = false) {
|
|
202
|
+
let node;
|
|
203
|
+
let readOffset = offset;
|
|
204
|
+
if (root) {
|
|
205
|
+
node = new Map([["symbols", 1]]);
|
|
206
|
+
}
|
|
207
|
+
else {
|
|
208
|
+
const nodeBuf = await fetchRange(source, offset, SYMBOL_TABLE_NODE_SIZE);
|
|
209
|
+
node = _unpack_struct_from(SYMBOL_TABLE_NODE, nodeBuf, 0);
|
|
210
|
+
if (node.get("signature") !== "SNOD")
|
|
211
|
+
throw new Error("incorrect node type");
|
|
212
|
+
readOffset = offset + SYMBOL_TABLE_NODE_SIZE;
|
|
213
|
+
}
|
|
214
|
+
const n_symbols = node.get("symbols");
|
|
215
|
+
const entryBytes = n_symbols * SYMBOL_TABLE_ENTRY_SIZE;
|
|
216
|
+
const entryBuf = await fetchRange(source, readOffset, entryBytes);
|
|
217
|
+
const entries = [];
|
|
218
|
+
let localOff = 0;
|
|
219
|
+
for (let i = 0; i < n_symbols; i++) {
|
|
220
|
+
entries.push(_unpack_struct_from(SYMBOL_TABLE_ENTRY, entryBuf, localOff));
|
|
221
|
+
localOff += SYMBOL_TABLE_ENTRY_SIZE;
|
|
222
|
+
}
|
|
223
|
+
let group_offset;
|
|
224
|
+
if (root) {
|
|
225
|
+
group_offset = entries[0].get("object_header_address");
|
|
226
|
+
}
|
|
227
|
+
return new SymbolTable(node, entries, group_offset);
|
|
228
|
+
}
|
|
229
|
+
assign_name(heap) {
|
|
230
|
+
for (const entry of this.entries) {
|
|
231
|
+
const offset = entry.get("link_name_offset");
|
|
232
|
+
entry.set("link_name", heap.get_object_name(offset));
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
get_links(heap) {
|
|
236
|
+
const links = {};
|
|
237
|
+
for (const e of this.entries) {
|
|
238
|
+
const cache_type = e.get("cache_type");
|
|
239
|
+
const link_name = e.get("link_name");
|
|
240
|
+
if (cache_type === 0 || cache_type === 1) {
|
|
241
|
+
links[link_name] = e.get("object_header_address");
|
|
242
|
+
}
|
|
243
|
+
else if (cache_type === 2) {
|
|
244
|
+
const scratch = e.get("scratch");
|
|
245
|
+
const buf = new ArrayBuffer(4);
|
|
246
|
+
const bufView = new Uint8Array(buf);
|
|
247
|
+
for (let i = 0; i < 4; i++) {
|
|
248
|
+
bufView[i] = scratch.charCodeAt(i);
|
|
249
|
+
}
|
|
250
|
+
const off = struct.unpack_from("<I", buf, 0)[0];
|
|
251
|
+
links[link_name] = heap.get_object_name(off);
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
return links;
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
// ────────── GlobalHeap ──────────
|
|
258
|
+
export class GlobalHeap {
|
|
259
|
+
heap_data;
|
|
260
|
+
_header;
|
|
261
|
+
_objects = null;
|
|
262
|
+
constructor(header, heap_data) {
|
|
263
|
+
this._header = header;
|
|
264
|
+
this.heap_data = heap_data;
|
|
265
|
+
}
|
|
266
|
+
static async create(source, offset) {
|
|
267
|
+
const hBuf = await fetchRange(source, offset, GLOBAL_HEAP_HEADER_SIZE);
|
|
268
|
+
const header = _unpack_struct_from(GLOBAL_HEAP_HEADER, hBuf, 0);
|
|
269
|
+
const heap_data_size = header.get("collection_size") - GLOBAL_HEAP_HEADER_SIZE;
|
|
270
|
+
const heap_data = await fetchRange(source, offset + GLOBAL_HEAP_HEADER_SIZE, heap_data_size);
|
|
271
|
+
return new GlobalHeap(header, heap_data);
|
|
272
|
+
}
|
|
273
|
+
get objects() {
|
|
274
|
+
if (this._objects === null) {
|
|
275
|
+
this._objects = new Map();
|
|
276
|
+
let offset = 0;
|
|
277
|
+
while (offset <= this.heap_data.byteLength - GLOBAL_HEAP_OBJECT_SIZE) {
|
|
278
|
+
const info = _unpack_struct_from(GLOBAL_HEAP_OBJECT, this.heap_data, offset);
|
|
279
|
+
if (info.get("object_index") === 0)
|
|
280
|
+
break;
|
|
281
|
+
offset += GLOBAL_HEAP_OBJECT_SIZE;
|
|
282
|
+
const obj_data = this.heap_data.slice(offset, offset + info.get("object_size"));
|
|
283
|
+
this._objects.set(info.get("object_index"), obj_data);
|
|
284
|
+
offset += _padded_size(info.get("object_size"));
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
return this._objects;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
// ────────── FractalHeap ──────────
|
|
291
|
+
export class FractalHeap {
|
|
292
|
+
header;
|
|
293
|
+
nobjects;
|
|
294
|
+
managed;
|
|
295
|
+
_managed_object_offset_size;
|
|
296
|
+
_managed_object_length_size;
|
|
297
|
+
_max_direct_nrows;
|
|
298
|
+
_indirect_nrows_sub;
|
|
299
|
+
direct_block_header;
|
|
300
|
+
direct_block_header_size;
|
|
301
|
+
indirect_block_header;
|
|
302
|
+
indirect_block_header_size;
|
|
303
|
+
constructor(header, managed, direct_block_header, direct_block_header_size, indirect_block_header, indirect_block_header_size, managed_object_offset_size, managed_object_length_size, max_direct_nrows, indirect_nrows_sub) {
|
|
304
|
+
this.header = header;
|
|
305
|
+
this.nobjects =
|
|
306
|
+
header.get("managed_object_count") +
|
|
307
|
+
header.get("huge_object_count") +
|
|
308
|
+
header.get("tiny_object_count");
|
|
309
|
+
this.managed = managed;
|
|
310
|
+
this.direct_block_header = direct_block_header;
|
|
311
|
+
this.direct_block_header_size = direct_block_header_size;
|
|
312
|
+
this.indirect_block_header = indirect_block_header;
|
|
313
|
+
this.indirect_block_header_size = indirect_block_header_size;
|
|
314
|
+
this._managed_object_offset_size = managed_object_offset_size;
|
|
315
|
+
this._managed_object_length_size = managed_object_length_size;
|
|
316
|
+
this._max_direct_nrows = max_direct_nrows;
|
|
317
|
+
this._indirect_nrows_sub = indirect_nrows_sub;
|
|
318
|
+
}
|
|
319
|
+
static async create(source, offset) {
|
|
320
|
+
const headerSize = _structure_size(FRACTAL_HEAP_HEADER);
|
|
321
|
+
const hBuf = await fetchRange(source, offset, headerSize);
|
|
322
|
+
const header = _unpack_struct_from(FRACTAL_HEAP_HEADER, hBuf, 0);
|
|
323
|
+
assert(header.get("signature") === "FRHP");
|
|
324
|
+
assert(header.get("version") === 0);
|
|
325
|
+
if (header.get("filter_info_size") > 0) {
|
|
326
|
+
throw new Error("Filter info size not supported on FractalHeap");
|
|
327
|
+
}
|
|
328
|
+
if (header.get("btree_address_huge_objects") === UNDEFINED_ADDRESS) {
|
|
329
|
+
header.set("btree_address_huge_objects", null);
|
|
330
|
+
}
|
|
331
|
+
else {
|
|
332
|
+
throw new Error("Huge objects not implemented in FractalHeap");
|
|
333
|
+
}
|
|
334
|
+
if (header.get("root_block_address") === UNDEFINED_ADDRESS) {
|
|
335
|
+
header.set("root_block_address", null);
|
|
336
|
+
}
|
|
337
|
+
const nbits = header.get("log2_maximum_heap_size");
|
|
338
|
+
const block_offset_size = FractalHeap._min_size_nbits(nbits);
|
|
339
|
+
const h = new Map([
|
|
340
|
+
["signature", "4s"],
|
|
341
|
+
["version", "B"],
|
|
342
|
+
["heap_header_adddress", "Q"],
|
|
343
|
+
["block_offset", `${block_offset_size}B`],
|
|
344
|
+
]);
|
|
345
|
+
const indirect_block_header = new Map(h);
|
|
346
|
+
const indirect_block_header_size = _structure_size(h);
|
|
347
|
+
if ((header.get("flags") & 2) === 2) {
|
|
348
|
+
h.set("checksum", "I");
|
|
349
|
+
}
|
|
350
|
+
const direct_block_header = h;
|
|
351
|
+
const direct_block_header_size = _structure_size(h);
|
|
352
|
+
const maximum_dblock_size = header.get("maximum_direct_block_size");
|
|
353
|
+
const managed_object_offset_size = FractalHeap._min_size_nbits(nbits);
|
|
354
|
+
const value = Math.min(maximum_dblock_size, header.get("max_managed_object_size"));
|
|
355
|
+
const managed_object_length_size = FractalHeap._min_size_integer(value);
|
|
356
|
+
const start_block_size = header.get("starting_block_size");
|
|
357
|
+
const table_width = header.get("table_width");
|
|
358
|
+
if (!(start_block_size > 0)) {
|
|
359
|
+
throw new Error("Starting block size == 0 not implemented");
|
|
360
|
+
}
|
|
361
|
+
const log2_maximum_dblock_size = Math.floor(Math.log2(maximum_dblock_size));
|
|
362
|
+
assert(1n << BigInt(log2_maximum_dblock_size) === BigInt(maximum_dblock_size));
|
|
363
|
+
const log2_start_block_size = Math.floor(Math.log2(start_block_size));
|
|
364
|
+
assert(1n << BigInt(log2_start_block_size) === BigInt(start_block_size));
|
|
365
|
+
const max_direct_nrows = log2_maximum_dblock_size - log2_start_block_size + 2;
|
|
366
|
+
const log2_table_width = Math.floor(Math.log2(table_width));
|
|
367
|
+
assert(1 << log2_table_width === table_width);
|
|
368
|
+
const indirect_nrows_sub = log2_table_width + log2_start_block_size - 1;
|
|
369
|
+
// Read managed data
|
|
370
|
+
const managedParts = [];
|
|
371
|
+
const root_address = header.get("root_block_address");
|
|
372
|
+
let nrows = 0;
|
|
373
|
+
if (root_address != null) {
|
|
374
|
+
nrows = header.get("indirect_current_rows_count");
|
|
375
|
+
}
|
|
376
|
+
// Helper context object for recursive reading
|
|
377
|
+
const ctx = {
|
|
378
|
+
source,
|
|
379
|
+
header,
|
|
380
|
+
direct_block_header,
|
|
381
|
+
direct_block_header_size,
|
|
382
|
+
indirect_block_header,
|
|
383
|
+
indirect_block_header_size,
|
|
384
|
+
max_direct_nrows,
|
|
385
|
+
indirect_nrows_sub,
|
|
386
|
+
table_width,
|
|
387
|
+
};
|
|
388
|
+
if (nrows > 0) {
|
|
389
|
+
for await (const data of FractalHeap._iter_indirect_block(ctx, root_address, nrows)) {
|
|
390
|
+
managedParts.push(data);
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
else if (root_address != null) {
|
|
394
|
+
const data = await FractalHeap._read_direct_block(ctx, root_address, start_block_size);
|
|
395
|
+
managedParts.push(data);
|
|
396
|
+
}
|
|
397
|
+
const data_size = managedParts.reduce((p, c) => p + c.byteLength, 0);
|
|
398
|
+
const combined = new Uint8Array(data_size);
|
|
399
|
+
let moffset = 0;
|
|
400
|
+
for (const m of managedParts) {
|
|
401
|
+
combined.set(new Uint8Array(m), moffset);
|
|
402
|
+
moffset += m.byteLength;
|
|
403
|
+
}
|
|
404
|
+
return new FractalHeap(header, combined.buffer, direct_block_header, direct_block_header_size, indirect_block_header, indirect_block_header_size, managed_object_offset_size, managed_object_length_size, max_direct_nrows, indirect_nrows_sub);
|
|
405
|
+
}
|
|
406
|
+
get_data(heapid) {
|
|
407
|
+
const firstbyte = struct.unpack_from("<B", heapid, 0)[0];
|
|
408
|
+
const idtype = (firstbyte >> 4) & 3;
|
|
409
|
+
const version = firstbyte >> 6;
|
|
410
|
+
let data_offset = 1;
|
|
411
|
+
if (idtype === 0) {
|
|
412
|
+
// managed
|
|
413
|
+
assert(version === 0);
|
|
414
|
+
let nbytes = this._managed_object_offset_size;
|
|
415
|
+
const offset = _unpack_integer(nbytes, heapid, data_offset);
|
|
416
|
+
data_offset += nbytes;
|
|
417
|
+
nbytes = this._managed_object_length_size;
|
|
418
|
+
const size = _unpack_integer(nbytes, heapid, data_offset);
|
|
419
|
+
return this.managed.slice(offset, offset + size);
|
|
420
|
+
}
|
|
421
|
+
else if (idtype === 1) {
|
|
422
|
+
throw new Error("tiny objectID not supported in FractalHeap");
|
|
423
|
+
}
|
|
424
|
+
else if (idtype === 2) {
|
|
425
|
+
throw new Error("huge objectID not supported in FractalHeap");
|
|
426
|
+
}
|
|
427
|
+
else {
|
|
428
|
+
throw new Error("unknown objectID type in FractalHeap");
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
static _min_size_integer(integer) {
|
|
432
|
+
return FractalHeap._min_size_nbits(bitSize(integer));
|
|
433
|
+
}
|
|
434
|
+
static _min_size_nbits(nbits) {
|
|
435
|
+
return Math.ceil(nbits / 8);
|
|
436
|
+
}
|
|
437
|
+
static async _read_direct_block(ctx, offset, block_size) {
|
|
438
|
+
const data = await fetchRange(ctx.source, offset, block_size);
|
|
439
|
+
const header = _unpack_struct_from(ctx.direct_block_header, data, 0);
|
|
440
|
+
assert(header.get("signature") === "FHDB");
|
|
441
|
+
return data;
|
|
442
|
+
}
|
|
443
|
+
static async *_iter_indirect_block(ctx, offset, nrows) {
|
|
444
|
+
const ibHdrBuf = await fetchRange(ctx.source, offset, ctx.indirect_block_header_size);
|
|
445
|
+
const header = _unpack_struct_from(ctx.indirect_block_header, ibHdrBuf, 0);
|
|
446
|
+
let readOff = offset + ctx.indirect_block_header_size;
|
|
447
|
+
assert(header.get("signature") === "FHIB");
|
|
448
|
+
const block_offset_bytes = header.get("block_offset");
|
|
449
|
+
const block_offset = block_offset_bytes.reduce((p, c, i) => p + (c << (i * 8)), 0);
|
|
450
|
+
header.set("block_offset", block_offset);
|
|
451
|
+
const [ndirect, nindirect] = FractalHeap._indirect_info(ctx, nrows);
|
|
452
|
+
// Read addresses for direct blocks
|
|
453
|
+
const addrBytes = (ndirect + nindirect) * 8;
|
|
454
|
+
const addrBuf = await fetchRange(ctx.source, readOff, addrBytes);
|
|
455
|
+
let addrOff = 0;
|
|
456
|
+
const direct_blocks = [];
|
|
457
|
+
for (let i = 0; i < ndirect; i++) {
|
|
458
|
+
const address = struct.unpack_from("<Q", addrBuf, addrOff)[0];
|
|
459
|
+
addrOff += 8;
|
|
460
|
+
if (address === UNDEFINED_ADDRESS)
|
|
461
|
+
break;
|
|
462
|
+
const block_size = FractalHeap._calc_block_size(ctx, i);
|
|
463
|
+
direct_blocks.push([address, block_size]);
|
|
464
|
+
}
|
|
465
|
+
const indirect_blocks = [];
|
|
466
|
+
for (let i = ndirect; i < ndirect + nindirect; i++) {
|
|
467
|
+
const address = struct.unpack_from("<Q", addrBuf, addrOff)[0];
|
|
468
|
+
addrOff += 8;
|
|
469
|
+
if (address === UNDEFINED_ADDRESS)
|
|
470
|
+
break;
|
|
471
|
+
const block_size = FractalHeap._calc_block_size(ctx, i);
|
|
472
|
+
const nr = FractalHeap._iblock_nrows_from_block_size(ctx, block_size);
|
|
473
|
+
indirect_blocks.push([address, nr]);
|
|
474
|
+
}
|
|
475
|
+
for (const [address, block_size] of direct_blocks) {
|
|
476
|
+
yield await FractalHeap._read_direct_block(ctx, address, block_size);
|
|
477
|
+
}
|
|
478
|
+
for (const [address, nr] of indirect_blocks) {
|
|
479
|
+
yield* FractalHeap._iter_indirect_block(ctx, address, nr);
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
static _calc_block_size(ctx, iblock) {
|
|
483
|
+
const row = Math.floor(iblock / ctx.table_width);
|
|
484
|
+
return 2 ** Math.max(row - 1, 0) * ctx.header.get("starting_block_size");
|
|
485
|
+
}
|
|
486
|
+
static _iblock_nrows_from_block_size(ctx, block_size) {
|
|
487
|
+
const log2_block_size = Math.floor(Math.log2(block_size));
|
|
488
|
+
assert(2 ** log2_block_size === block_size);
|
|
489
|
+
return log2_block_size - ctx.indirect_nrows_sub;
|
|
490
|
+
}
|
|
491
|
+
static _indirect_info(ctx, nrows) {
|
|
492
|
+
const table_width = ctx.table_width;
|
|
493
|
+
const nobjects = nrows * table_width;
|
|
494
|
+
const ndirect_max = ctx.max_direct_nrows * table_width;
|
|
495
|
+
let ndirect, nindirect;
|
|
496
|
+
if (nrows <= ndirect_max) {
|
|
497
|
+
ndirect = nobjects;
|
|
498
|
+
nindirect = 0;
|
|
499
|
+
}
|
|
500
|
+
else {
|
|
501
|
+
ndirect = ndirect_max;
|
|
502
|
+
nindirect = nobjects - ndirect_max;
|
|
503
|
+
}
|
|
504
|
+
return [ndirect, nindirect];
|
|
505
|
+
}
|
|
506
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { ReferenceSpec } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Zarr v2 consolidated metadata (.zmetadata).
|
|
4
|
+
*/
|
|
5
|
+
export interface ZarrConsolidatedMetadata {
|
|
6
|
+
zarr_consolidated_format: 1;
|
|
7
|
+
metadata: Record<string, Record<string, unknown>>;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Convert a Reference Spec JSON to Zarr v2 consolidated metadata (.zmetadata).
|
|
11
|
+
*/
|
|
12
|
+
export declare function refSpecToConsolidatedMetadata(refSpec: ReferenceSpec): ZarrConsolidatedMetadata;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Metadata key suffixes included in Zarr consolidated metadata.
|
|
3
|
+
* TODO: support zarr v3 (zarr.json files)
|
|
4
|
+
*/
|
|
5
|
+
const METADATA_SUFFIXES = [".zgroup", ".zarray", ".zattrs"];
|
|
6
|
+
/**
|
|
7
|
+
* Convert a Reference Spec JSON to Zarr v2 consolidated metadata (.zmetadata).
|
|
8
|
+
*/
|
|
9
|
+
export function refSpecToConsolidatedMetadata(refSpec) {
|
|
10
|
+
const metadata = {};
|
|
11
|
+
for (const [key, value] of Object.entries(refSpec.refs)) {
|
|
12
|
+
if (METADATA_SUFFIXES.some((suffix) => key === suffix || key.endsWith(`/${suffix}`))) {
|
|
13
|
+
if (typeof value === "string") {
|
|
14
|
+
metadata[key] = JSON.parse(value);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
return {
|
|
19
|
+
zarr_consolidated_format: 1,
|
|
20
|
+
metadata,
|
|
21
|
+
};
|
|
22
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { AbsolutePath, AsyncReadable } from "@zarrita/storage";
|
|
2
|
+
export declare function to_binary(base64: string): Uint8Array;
|
|
3
|
+
type ReferenceEntry = string | [url: string | null] | [url: string | null, offset: number, length: number];
|
|
4
|
+
export declare class ComposableReferenceStore implements AsyncReadable<RequestInit> {
|
|
5
|
+
#private;
|
|
6
|
+
constructor(refs: Map<string, ReferenceEntry>, internal_store: AsyncReadable, overrides?: RequestInit);
|
|
7
|
+
get(key: AbsolutePath, opts?: RequestInit): Promise<Uint8Array | undefined>;
|
|
8
|
+
static fromSpec(spec: Record<string, unknown>, internal_store: AsyncReadable, overrides?: RequestInit): ComposableReferenceStore;
|
|
9
|
+
}
|
|
10
|
+
export {};
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { parse } from "reference-spec-reader";
|
|
2
|
+
// Copied from @zarrita-storage/src/util.ts
|
|
3
|
+
function strip_prefix(path) {
|
|
4
|
+
// @ts-expect-error - TS can't infer this type correctly
|
|
5
|
+
return path.slice(1);
|
|
6
|
+
}
|
|
7
|
+
// Copied from @zarrita-storage/src/util.ts
|
|
8
|
+
function merge_init(storeOverrides, requestOverrides) {
|
|
9
|
+
// Request overrides take precedence over storeOverrides.
|
|
10
|
+
return {
|
|
11
|
+
...storeOverrides,
|
|
12
|
+
...requestOverrides,
|
|
13
|
+
headers: {
|
|
14
|
+
...storeOverrides.headers,
|
|
15
|
+
...requestOverrides.headers,
|
|
16
|
+
},
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Copied from @zarrita-storage/src/util.ts
|
|
21
|
+
* This is for the "binary" loader (custom code is ~2x faster than "atob") from esbuild.
|
|
22
|
+
* https://github.com/evanw/esbuild/blob/150a01844d47127c007c2b1973158d69c560ca21/internal/runtime/runtime.go#L185
|
|
23
|
+
*/
|
|
24
|
+
let table = new Uint8Array(128);
|
|
25
|
+
for (let i = 0; i < 64; i++) {
|
|
26
|
+
table[i < 26 ? i + 65 : i < 52 ? i + 71 : i < 62 ? i - 4 : i * 4 - 205] = i;
|
|
27
|
+
}
|
|
28
|
+
export function to_binary(base64) {
|
|
29
|
+
const n = base64.length;
|
|
30
|
+
const bytes = new Uint8Array(
|
|
31
|
+
// @ts-ignore
|
|
32
|
+
(((n - (base64[n - 1] === "=") - (base64[n - 2] === "=")) * 3) / 4) | 0);
|
|
33
|
+
for (let i = 0, j = 0; i < n;) {
|
|
34
|
+
const c0 = table[base64.charCodeAt(i++)];
|
|
35
|
+
const c1 = table[base64.charCodeAt(i++)];
|
|
36
|
+
const c2 = table[base64.charCodeAt(i++)];
|
|
37
|
+
const c3 = table[base64.charCodeAt(i++)];
|
|
38
|
+
bytes[j++] = (c0 << 2) | (c1 >> 4);
|
|
39
|
+
bytes[j++] = (c1 << 4) | (c2 >> 2);
|
|
40
|
+
bytes[j++] = (c2 << 6) | c3;
|
|
41
|
+
}
|
|
42
|
+
return bytes;
|
|
43
|
+
}
|
|
44
|
+
// Based on @zarrita-storage/src/ref.ts, but with some differences:
|
|
45
|
+
// - `refs` is not a Promise.
|
|
46
|
+
// - An internal store is passed in directly, avoiding any assumptions about where the data is coming from.
|
|
47
|
+
export class ComposableReferenceStore {
|
|
48
|
+
#refs;
|
|
49
|
+
#internal_store;
|
|
50
|
+
#overrides;
|
|
51
|
+
constructor(refs, internal_store, overrides) {
|
|
52
|
+
this.#refs = refs;
|
|
53
|
+
this.#internal_store = internal_store;
|
|
54
|
+
this.#overrides = overrides;
|
|
55
|
+
}
|
|
56
|
+
async get(key, opts = {}) {
|
|
57
|
+
let ref = this.#refs.get(strip_prefix(key));
|
|
58
|
+
if (!ref)
|
|
59
|
+
return;
|
|
60
|
+
if (typeof ref === "string") {
|
|
61
|
+
if (ref.startsWith("base64:")) {
|
|
62
|
+
return to_binary(ref.slice("base64:".length));
|
|
63
|
+
}
|
|
64
|
+
return new TextEncoder().encode(ref);
|
|
65
|
+
}
|
|
66
|
+
let [urlOrNull, offset, size] = ref;
|
|
67
|
+
if (offset === undefined || size === undefined || !this.#internal_store.getRange) {
|
|
68
|
+
return this.#internal_store.get(key, opts);
|
|
69
|
+
}
|
|
70
|
+
const opts_for_get = this.#overrides ? merge_init(this.#overrides, opts) : opts;
|
|
71
|
+
return this.#internal_store.getRange(key, { offset, length: size }, opts_for_get);
|
|
72
|
+
}
|
|
73
|
+
// TODO: does it make sense to support getRange here? get already calls getRange on the internal store...
|
|
74
|
+
static fromSpec(spec, internal_store, overrides) {
|
|
75
|
+
// @ts-expect-error - TS doesn't like the type of `parse`
|
|
76
|
+
let refs = parse(spec);
|
|
77
|
+
return new ComposableReferenceStore(refs, internal_store, overrides);
|
|
78
|
+
}
|
|
79
|
+
}
|