hdf5-as-virtual-zarr 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -2
- package/dist/bundled/index.js +815 -445
- package/dist/bundled/index.js.map +1 -1
- package/dist/hdf-store.d.ts +33 -15
- package/dist/hdf-store.js +160 -22
- package/dist/index.d.ts +1 -1
- package/dist/jsfive/btree.js +37 -34
- package/dist/jsfive/dataobjects.d.ts +10 -1
- package/dist/jsfive/dataobjects.js +199 -111
- package/dist/jsfive/filters.d.ts +13 -0
- package/dist/jsfive/filters.js +62 -0
- package/dist/jsfive/high-level.js +3 -0
- package/dist/jsfive/misc-low-level.d.ts +7 -1
- package/dist/jsfive/misc-low-level.js +38 -17
- package/dist/single-hdf5-to-zarr.d.ts +33 -4
- package/dist/single-hdf5-to-zarr.js +130 -75
- package/dist/types.d.ts +16 -0
- package/dist/types.js +26 -0
- package/package.json +2 -1
- package/src/hdf-store.ts +182 -26
- package/src/index.ts +1 -1
- package/src/jsfive/btree.ts +49 -37
- package/src/jsfive/dataobjects.ts +218 -97
- package/src/jsfive/filters.ts +64 -0
- package/src/jsfive/high-level.ts +3 -0
- package/src/jsfive/misc-low-level.ts +60 -21
- package/src/single-hdf5-to-zarr.ts +161 -79
- package/src/types.ts +34 -0
package/dist/bundled/index.js
CHANGED
|
@@ -5,6 +5,18 @@ async function fetchRange(source, offset, length) {
|
|
|
5
5
|
copy.set(result);
|
|
6
6
|
return copy.buffer;
|
|
7
7
|
}
|
|
8
|
+
const fileSizes = /* @__PURE__ */ new WeakMap();
|
|
9
|
+
function setFileSize(source, size) {
|
|
10
|
+
fileSizes.set(source, size);
|
|
11
|
+
}
|
|
12
|
+
function fetchRangeWithReadAhead(source, offset, length, readAhead) {
|
|
13
|
+
const size = fileSizes.get(source);
|
|
14
|
+
let want = length;
|
|
15
|
+
if (size !== void 0 && readAhead > length) {
|
|
16
|
+
want = Math.max(length, Math.min(readAhead, size - offset));
|
|
17
|
+
}
|
|
18
|
+
return fetchRange(source, offset, want);
|
|
19
|
+
}
|
|
8
20
|
function _unpack_struct_from(structure, buf, offset = 0) {
|
|
9
21
|
const output = /* @__PURE__ */ new Map();
|
|
10
22
|
for (const [key, fmt] of structure.entries()) {
|
|
@@ -351,13 +363,22 @@ const B_LINK_NODE_V1 = /* @__PURE__ */ new Map([
|
|
|
351
363
|
["right_sibling", "Q"]
|
|
352
364
|
]);
|
|
353
365
|
const B_LINK_NODE_V1_SIZE = _structure_size(B_LINK_NODE_V1);
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
const
|
|
366
|
+
const V1_NODE_READ_AHEAD_ENTRIES = 64;
|
|
367
|
+
async function _readV1Node(source, offset, node_level, entrySize, trailingSize) {
|
|
368
|
+
const buf = await fetchRangeWithReadAhead(
|
|
369
|
+
source,
|
|
370
|
+
offset,
|
|
371
|
+
B_LINK_NODE_V1_SIZE,
|
|
372
|
+
B_LINK_NODE_V1_SIZE + V1_NODE_READ_AHEAD_ENTRIES * entrySize + trailingSize
|
|
373
|
+
);
|
|
374
|
+
const node = _unpack_struct_from(B_LINK_NODE_V1, buf, 0);
|
|
357
375
|
if (node_level != null && node.get("node_level") !== node_level) {
|
|
358
376
|
throw new Error("node level does not match");
|
|
359
377
|
}
|
|
360
|
-
|
|
378
|
+
const dataSize = node.get("entries_used") * entrySize + trailingSize;
|
|
379
|
+
const dataEnd = B_LINK_NODE_V1_SIZE + dataSize;
|
|
380
|
+
const data = dataEnd <= buf.byteLength ? buf.slice(B_LINK_NODE_V1_SIZE, dataEnd) : await fetchRange(source, offset + B_LINK_NODE_V1_SIZE, dataSize);
|
|
381
|
+
return [node, data];
|
|
361
382
|
}
|
|
362
383
|
class BTreeV1Groups {
|
|
363
384
|
all_nodes;
|
|
@@ -369,11 +390,8 @@ class BTreeV1Groups {
|
|
|
369
390
|
static async create(source, offset) {
|
|
370
391
|
const all_nodes = /* @__PURE__ */ new Map();
|
|
371
392
|
async function readNode(off, node_level2) {
|
|
372
|
-
const [header] = await
|
|
373
|
-
let readOff = off + B_LINK_NODE_V1_SIZE;
|
|
393
|
+
const [header, dataBuf] = await _readV1Node(source, off, node_level2, 16, 8);
|
|
374
394
|
const entries_used = header.get("entries_used");
|
|
375
|
-
const dataSize = entries_used * 16 + 8;
|
|
376
|
-
const dataBuf = await fetchRange(source, readOff, dataSize);
|
|
377
395
|
let localOff = 0;
|
|
378
396
|
const keys = [];
|
|
379
397
|
const addresses = [];
|
|
@@ -392,12 +410,11 @@ class BTreeV1Groups {
|
|
|
392
410
|
let node_level = depth;
|
|
393
411
|
while (node_level > 0) {
|
|
394
412
|
const parents = all_nodes.get(node_level) || [];
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
}
|
|
413
|
+
const childAddrs = parents.flatMap((parent) => parent.addresses);
|
|
414
|
+
const children = await Promise.all(
|
|
415
|
+
childAddrs.map((child_addr) => readNode(child_addr, node_level - 1))
|
|
416
|
+
);
|
|
417
|
+
for (const child of children) addNode(all_nodes, child, node_level - 1);
|
|
401
418
|
node_level--;
|
|
402
419
|
}
|
|
403
420
|
return new BTreeV1Groups(all_nodes, depth);
|
|
@@ -423,13 +440,11 @@ class BTreeV1RawDataChunks {
|
|
|
423
440
|
static async create(source, offset, dims) {
|
|
424
441
|
const all_nodes = /* @__PURE__ */ new Map();
|
|
425
442
|
async function readNode(off, node_level2) {
|
|
426
|
-
const [header] = await _readV1NodeHeader(source, off, node_level2);
|
|
427
|
-
let readOff = off + B_LINK_NODE_V1_SIZE;
|
|
428
|
-
const entries_used = header.get("entries_used");
|
|
429
443
|
const fmt = "<" + dims.toFixed() + "Q";
|
|
430
444
|
const fmt_size = struct.calcsize(fmt);
|
|
431
445
|
const entrySize = 8 + fmt_size + 8;
|
|
432
|
-
const dataBuf = await
|
|
446
|
+
const [header, dataBuf] = await _readV1Node(source, off, node_level2, entrySize, 0);
|
|
447
|
+
const entries_used = header.get("entries_used");
|
|
433
448
|
let localOff = 0;
|
|
434
449
|
const keys = [];
|
|
435
450
|
const addresses = [];
|
|
@@ -451,12 +466,11 @@ class BTreeV1RawDataChunks {
|
|
|
451
466
|
let node_level = depth;
|
|
452
467
|
while (node_level > 0) {
|
|
453
468
|
const parents = all_nodes.get(node_level) || [];
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
}
|
|
469
|
+
const childAddrs = parents.flatMap((parent) => parent.addresses);
|
|
470
|
+
const children = await Promise.all(
|
|
471
|
+
childAddrs.map((child_addr) => readNode(child_addr, node_level - 1))
|
|
472
|
+
);
|
|
473
|
+
for (const child of children) addNode(all_nodes, child, node_level - 1);
|
|
460
474
|
node_level--;
|
|
461
475
|
}
|
|
462
476
|
return new BTreeV1RawDataChunks(all_nodes, depth, dims);
|
|
@@ -591,12 +605,11 @@ async function _readBTreeV2(source, offset, parseRecord) {
|
|
|
591
605
|
let node_level = depth;
|
|
592
606
|
while (node_level > 0) {
|
|
593
607
|
const parents = all_nodes.get(node_level) || [];
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
}
|
|
608
|
+
const childAddrs = parents.flatMap((parent) => parent.addresses);
|
|
609
|
+
const children = await Promise.all(
|
|
610
|
+
childAddrs.map((child_addr) => readNode(child_addr, node_level - 1))
|
|
611
|
+
);
|
|
612
|
+
for (const child of children) addNode(all_nodes, child, node_level - 1);
|
|
600
613
|
node_level--;
|
|
601
614
|
}
|
|
602
615
|
return { all_nodes, header };
|
|
@@ -663,6 +676,306 @@ function addNode(all_nodes, node, node_level) {
|
|
|
663
676
|
all_nodes.set(level, [node]);
|
|
664
677
|
}
|
|
665
678
|
}
|
|
679
|
+
const FILTER_DEFLATE = 1;
|
|
680
|
+
const FILTER_SHUFFLE = 2;
|
|
681
|
+
const FILTER_FLETCHER32 = 3;
|
|
682
|
+
async function inflate(data) {
|
|
683
|
+
const stream = new Blob([data]).stream().pipeThrough(new DecompressionStream("deflate"));
|
|
684
|
+
return new Uint8Array(await new Response(stream).arrayBuffer());
|
|
685
|
+
}
|
|
686
|
+
function unshuffle(data, elementSize) {
|
|
687
|
+
if (elementSize <= 1) return data;
|
|
688
|
+
const count = Math.floor(data.length / elementSize);
|
|
689
|
+
const out = new Uint8Array(data.length);
|
|
690
|
+
for (let b = 0; b < elementSize; b++) {
|
|
691
|
+
const src = b * count;
|
|
692
|
+
for (let i = 0; i < count; i++) {
|
|
693
|
+
out[i * elementSize + b] = data[src + i];
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
out.set(data.subarray(count * elementSize), count * elementSize);
|
|
697
|
+
return out;
|
|
698
|
+
}
|
|
699
|
+
async function decodeChunk(data, pipeline, filterMask = 0) {
|
|
700
|
+
if (!pipeline) return data;
|
|
701
|
+
let out = data;
|
|
702
|
+
for (let i = pipeline.length - 1; i >= 0; i--) {
|
|
703
|
+
if (filterMask & 1 << i) continue;
|
|
704
|
+
const filter = pipeline[i];
|
|
705
|
+
const id = filter.get("filter_id");
|
|
706
|
+
const clientData = filter.get("client_data") ?? [];
|
|
707
|
+
if (id === FILTER_DEFLATE) {
|
|
708
|
+
out = await inflate(out);
|
|
709
|
+
} else if (id === FILTER_SHUFFLE) {
|
|
710
|
+
out = unshuffle(out, clientData[0] ?? 1);
|
|
711
|
+
} else if (id === FILTER_FLETCHER32) {
|
|
712
|
+
out = out.subarray(0, out.length - 4);
|
|
713
|
+
} else {
|
|
714
|
+
throw new Error(`Unsupported HDF5 filter id ${id} for chunk decoding`);
|
|
715
|
+
}
|
|
716
|
+
}
|
|
717
|
+
return out;
|
|
718
|
+
}
|
|
719
|
+
const isValidMaxSize = (value) => typeof value === "number" && value > 0 && (Number.isInteger(value) || value === Infinity);
|
|
720
|
+
const hasExpiry = (item) => item.expiry !== void 0 && !Number.isNaN(item.expiry);
|
|
721
|
+
const MODE_KEYS = 0;
|
|
722
|
+
const MODE_VALUES = 1;
|
|
723
|
+
const MODE_ENTRIES = 2;
|
|
724
|
+
const MODE_ITEMS = 3;
|
|
725
|
+
const INVALID_MAX_SIZE_MESSAGE = "`maxSize` must be a whole number greater than 0 or `Infinity`";
|
|
726
|
+
class QuickLRU extends Map {
|
|
727
|
+
#cache = /* @__PURE__ */ new Map();
|
|
728
|
+
#oldCache = /* @__PURE__ */ new Map();
|
|
729
|
+
#maxSize;
|
|
730
|
+
#maxAge;
|
|
731
|
+
#onEviction;
|
|
732
|
+
constructor(options = {}) {
|
|
733
|
+
super();
|
|
734
|
+
if (!isValidMaxSize(options.maxSize)) {
|
|
735
|
+
throw new TypeError(INVALID_MAX_SIZE_MESSAGE);
|
|
736
|
+
}
|
|
737
|
+
if (typeof options.maxAge === "number" && !(options.maxAge > 0)) {
|
|
738
|
+
throw new TypeError("`maxAge` must be a number greater than 0");
|
|
739
|
+
}
|
|
740
|
+
this.#maxSize = options.maxSize;
|
|
741
|
+
this.#maxAge = typeof options.maxAge === "number" ? options.maxAge : Infinity;
|
|
742
|
+
this.#onEviction = options.onEviction;
|
|
743
|
+
}
|
|
744
|
+
#emitEvictions(cache) {
|
|
745
|
+
if (typeof this.#onEviction !== "function") {
|
|
746
|
+
return;
|
|
747
|
+
}
|
|
748
|
+
const evictions = [...cache].filter(([key]) => !this.#cache.has(key) && !this.#oldCache.has(key));
|
|
749
|
+
for (const [key, item] of evictions) {
|
|
750
|
+
this.#onEviction(key, item.value);
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
#deleteIfExpired(key, item) {
|
|
754
|
+
if (hasExpiry(item) && item.expiry <= Date.now()) {
|
|
755
|
+
const deleted = this.delete(key);
|
|
756
|
+
if (deleted && typeof this.#onEviction === "function") {
|
|
757
|
+
this.#onEviction(key, item.value);
|
|
758
|
+
}
|
|
759
|
+
return deleted;
|
|
760
|
+
}
|
|
761
|
+
return false;
|
|
762
|
+
}
|
|
763
|
+
// Returns the value, or `undefined` once the item has been removed as expired.
|
|
764
|
+
#getItemValue(key, item) {
|
|
765
|
+
return this.#deleteIfExpired(key, item) === false ? item.value : void 0;
|
|
766
|
+
}
|
|
767
|
+
#set(key, item) {
|
|
768
|
+
this.#cache.set(key, item);
|
|
769
|
+
if (this.#cache.size < this.#maxSize) {
|
|
770
|
+
return;
|
|
771
|
+
}
|
|
772
|
+
const oldCache = this.#oldCache;
|
|
773
|
+
this.#oldCache = this.#cache;
|
|
774
|
+
this.#cache = /* @__PURE__ */ new Map();
|
|
775
|
+
this.#emitEvictions(oldCache);
|
|
776
|
+
}
|
|
777
|
+
#moveToRecent(key, item) {
|
|
778
|
+
this.#oldCache.delete(key);
|
|
779
|
+
this.#set(key, item);
|
|
780
|
+
}
|
|
781
|
+
// The current item for `key`, or `undefined` if it is gone or has expired. The recent cache wins, since an old cache copy of a key that is in both is stale. Not a generator, since this runs once per item during iteration.
|
|
782
|
+
#liveItem(key) {
|
|
783
|
+
const item = this.#cache.get(key) ?? this.#oldCache.get(key);
|
|
784
|
+
if (item !== void 0 && this.#deleteIfExpired(key, item) === false) {
|
|
785
|
+
return item;
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
// The current item for `key`, promoted to the recent cache when it was in the old one, or `undefined` when the key is missing or has expired. Only for the methods that refresh recency, since promoting is what separates them from `peek()`. One lookup per cache, since where the item is found decides the promotion. Not a generator, since this runs once per lookup.
|
|
789
|
+
#getLiveItem(key) {
|
|
790
|
+
let item = this.#cache.get(key);
|
|
791
|
+
if (item === void 0) {
|
|
792
|
+
item = this.#oldCache.get(key);
|
|
793
|
+
if (item === void 0 || this.#deleteIfExpired(key, item)) {
|
|
794
|
+
return;
|
|
795
|
+
}
|
|
796
|
+
this.#moveToRecent(key, item);
|
|
797
|
+
} else if (this.#deleteIfExpired(key, item)) {
|
|
798
|
+
return;
|
|
799
|
+
}
|
|
800
|
+
return item;
|
|
801
|
+
}
|
|
802
|
+
// The old cache keys that are not also in the recent cache, since an old copy of such a key is stale. A plain loop, since this runs once per item during iteration.
|
|
803
|
+
#olderKeys() {
|
|
804
|
+
const keys = [];
|
|
805
|
+
for (const key of this.#oldCache.keys()) {
|
|
806
|
+
if (!this.#cache.has(key)) {
|
|
807
|
+
keys.push(key);
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
return keys;
|
|
811
|
+
}
|
|
812
|
+
*#walk(mode, descending) {
|
|
813
|
+
const keys = [...this.#olderKeys(), ...this.#cache.keys()];
|
|
814
|
+
if (descending) {
|
|
815
|
+
keys.reverse();
|
|
816
|
+
}
|
|
817
|
+
for (const key of keys) {
|
|
818
|
+
const item = this.#liveItem(key);
|
|
819
|
+
if (item === void 0) {
|
|
820
|
+
continue;
|
|
821
|
+
}
|
|
822
|
+
if (mode === MODE_KEYS) {
|
|
823
|
+
yield key;
|
|
824
|
+
} else if (mode === MODE_VALUES) {
|
|
825
|
+
yield item.value;
|
|
826
|
+
} else if (mode === MODE_ITEMS) {
|
|
827
|
+
yield [key, item];
|
|
828
|
+
} else {
|
|
829
|
+
yield [key, item.value];
|
|
830
|
+
}
|
|
831
|
+
}
|
|
832
|
+
}
|
|
833
|
+
// For tests. Public on purpose, so the test suite can see where an item lives.
|
|
834
|
+
// eslint-disable-next-line unicorn/prefer-private-class-fields
|
|
835
|
+
get __oldCache() {
|
|
836
|
+
return this.#oldCache;
|
|
837
|
+
}
|
|
838
|
+
get(key) {
|
|
839
|
+
const item = this.#getLiveItem(key);
|
|
840
|
+
return item === void 0 ? void 0 : item.value;
|
|
841
|
+
}
|
|
842
|
+
getOrInsert(key, value) {
|
|
843
|
+
const item = this.#getLiveItem(key);
|
|
844
|
+
if (item === void 0) {
|
|
845
|
+
this.set(key, value);
|
|
846
|
+
return value;
|
|
847
|
+
}
|
|
848
|
+
return item.value;
|
|
849
|
+
}
|
|
850
|
+
getOrInsertComputed(key, callback) {
|
|
851
|
+
const item = this.#getLiveItem(key);
|
|
852
|
+
if (item === void 0) {
|
|
853
|
+
const value = callback(key);
|
|
854
|
+
this.set(key, value);
|
|
855
|
+
return value;
|
|
856
|
+
}
|
|
857
|
+
return item.value;
|
|
858
|
+
}
|
|
859
|
+
set(key, value, { maxAge = this.#maxAge } = {}) {
|
|
860
|
+
const expiry = typeof maxAge === "number" && maxAge !== Infinity ? Date.now() + maxAge : void 0;
|
|
861
|
+
this.#set(key, { value, expiry });
|
|
862
|
+
return this;
|
|
863
|
+
}
|
|
864
|
+
has(key) {
|
|
865
|
+
const item = this.#cache.get(key) ?? this.#oldCache.get(key);
|
|
866
|
+
return item !== void 0 && !this.#deleteIfExpired(key, item);
|
|
867
|
+
}
|
|
868
|
+
peek(key) {
|
|
869
|
+
const item = this.#cache.get(key) ?? this.#oldCache.get(key);
|
|
870
|
+
return item === void 0 ? void 0 : this.#getItemValue(key, item);
|
|
871
|
+
}
|
|
872
|
+
expiresIn(key) {
|
|
873
|
+
const item = this.#cache.get(key) ?? this.#oldCache.get(key);
|
|
874
|
+
if (item) {
|
|
875
|
+
return hasExpiry(item) ? item.expiry - Date.now() : Infinity;
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
delete(key) {
|
|
879
|
+
const deleted = this.#cache.delete(key);
|
|
880
|
+
return this.#oldCache.delete(key) || deleted;
|
|
881
|
+
}
|
|
882
|
+
clear() {
|
|
883
|
+
this.#cache.clear();
|
|
884
|
+
this.#oldCache.clear();
|
|
885
|
+
}
|
|
886
|
+
resize(newSize) {
|
|
887
|
+
if (!isValidMaxSize(newSize)) {
|
|
888
|
+
throw new TypeError(INVALID_MAX_SIZE_MESSAGE);
|
|
889
|
+
}
|
|
890
|
+
const items = [...this.#walk(MODE_ITEMS)];
|
|
891
|
+
const removeCount = items.length - newSize;
|
|
892
|
+
if (removeCount < 0) {
|
|
893
|
+
this.#cache = new Map(items);
|
|
894
|
+
this.#oldCache = /* @__PURE__ */ new Map();
|
|
895
|
+
} else {
|
|
896
|
+
this.#oldCache = new Map(items.slice(removeCount));
|
|
897
|
+
this.#cache = /* @__PURE__ */ new Map();
|
|
898
|
+
}
|
|
899
|
+
this.#maxSize = newSize;
|
|
900
|
+
if (removeCount > 0) {
|
|
901
|
+
this.#emitEvictions(items.slice(0, removeCount));
|
|
902
|
+
}
|
|
903
|
+
}
|
|
904
|
+
evict(count = 1) {
|
|
905
|
+
const requested = Number(count);
|
|
906
|
+
if (!requested || requested <= 0) {
|
|
907
|
+
return;
|
|
908
|
+
}
|
|
909
|
+
const live = [...this.#walk(MODE_ITEMS)];
|
|
910
|
+
const evictCount = Math.min(Math.trunc(requested), Math.max(live.length - 1, 0));
|
|
911
|
+
const evicted = [];
|
|
912
|
+
for (const [key] of live.slice(0, evictCount)) {
|
|
913
|
+
const item = this.#cache.get(key) ?? this.#oldCache.get(key);
|
|
914
|
+
if (item !== void 0 && this.delete(key)) {
|
|
915
|
+
evicted.push([key, item]);
|
|
916
|
+
}
|
|
917
|
+
}
|
|
918
|
+
this.#emitEvictions(evicted);
|
|
919
|
+
}
|
|
920
|
+
keys() {
|
|
921
|
+
return this.#walk(MODE_KEYS);
|
|
922
|
+
}
|
|
923
|
+
values() {
|
|
924
|
+
return this.#walk(MODE_VALUES);
|
|
925
|
+
}
|
|
926
|
+
[Symbol.iterator]() {
|
|
927
|
+
return this.entriesAscending();
|
|
928
|
+
}
|
|
929
|
+
entriesDescending() {
|
|
930
|
+
return this.#walk(MODE_ENTRIES, true);
|
|
931
|
+
}
|
|
932
|
+
entriesAscending() {
|
|
933
|
+
return this.#walk(MODE_ENTRIES);
|
|
934
|
+
}
|
|
935
|
+
get size() {
|
|
936
|
+
const recentSize = this.#cache.size;
|
|
937
|
+
const olderSize = this.#oldCache.size;
|
|
938
|
+
if (recentSize === 0 || olderSize === 0) {
|
|
939
|
+
return Math.min(recentSize + olderSize, this.#maxSize);
|
|
940
|
+
}
|
|
941
|
+
let walked = this.#cache;
|
|
942
|
+
let checked = this.#oldCache;
|
|
943
|
+
if (olderSize < recentSize) {
|
|
944
|
+
walked = this.#oldCache;
|
|
945
|
+
checked = this.#cache;
|
|
946
|
+
}
|
|
947
|
+
let duplicates = 0;
|
|
948
|
+
for (const key of walked.keys()) {
|
|
949
|
+
if (checked.has(key)) {
|
|
950
|
+
duplicates++;
|
|
951
|
+
}
|
|
952
|
+
}
|
|
953
|
+
return Math.min(recentSize + olderSize - duplicates, this.#maxSize);
|
|
954
|
+
}
|
|
955
|
+
get maxSize() {
|
|
956
|
+
return this.#maxSize;
|
|
957
|
+
}
|
|
958
|
+
get maxAge() {
|
|
959
|
+
return this.#maxAge;
|
|
960
|
+
}
|
|
961
|
+
entries() {
|
|
962
|
+
return this.entriesAscending();
|
|
963
|
+
}
|
|
964
|
+
forEach(callbackFunction, thisArgument = this) {
|
|
965
|
+
for (const [key, value] of this.entriesAscending()) {
|
|
966
|
+
callbackFunction.call(thisArgument, value, key, this);
|
|
967
|
+
}
|
|
968
|
+
}
|
|
969
|
+
get [Symbol.toStringTag]() {
|
|
970
|
+
return "QuickLRU";
|
|
971
|
+
}
|
|
972
|
+
toString() {
|
|
973
|
+
return `QuickLRU(${this.size}/${this.maxSize})`;
|
|
974
|
+
}
|
|
975
|
+
[/* @__PURE__ */ Symbol.for("nodejs.util.inspect.custom")]() {
|
|
976
|
+
return this.toString();
|
|
977
|
+
}
|
|
978
|
+
}
|
|
666
979
|
const FORMAT_SIGNATURE = struct.unpack_from(
|
|
667
980
|
"8s",
|
|
668
981
|
new Uint8Array([137, 72, 68, 70, 13, 10, 26, 10]).buffer
|
|
@@ -767,6 +1080,8 @@ const FRACTAL_HEAP_HEADER = /* @__PURE__ */ new Map([
|
|
|
767
1080
|
["root_block_address", "Q"],
|
|
768
1081
|
["indirect_current_rows_count", "H"]
|
|
769
1082
|
]);
|
|
1083
|
+
const LOCAL_HEAP_READ_AHEAD = 1024;
|
|
1084
|
+
const GLOBAL_HEAP_READ_AHEAD = 4096;
|
|
770
1085
|
class SuperBlock {
|
|
771
1086
|
version;
|
|
772
1087
|
_contents;
|
|
@@ -803,6 +1118,11 @@ class SuperBlock {
|
|
|
803
1118
|
const version = contents.get("superblock_version");
|
|
804
1119
|
return new SuperBlock(source, contents, version, end_of_sblock);
|
|
805
1120
|
}
|
|
1121
|
+
/** Absolute end-of-file address (i.e. the file size), as recorded in the superblock. */
|
|
1122
|
+
get end_of_file() {
|
|
1123
|
+
const base = this.version === 0 ? this._contents.get("base_address_lower") : this._contents.get("base_address");
|
|
1124
|
+
return base + this._contents.get("end_of_file_address");
|
|
1125
|
+
}
|
|
806
1126
|
async getOffsetToDataobjects() {
|
|
807
1127
|
if (this.version === 0) {
|
|
808
1128
|
const sym_table = await SymbolTable.create(
|
|
@@ -828,13 +1148,14 @@ class Heap {
|
|
|
828
1148
|
}
|
|
829
1149
|
static async create(source, offset) {
|
|
830
1150
|
const headerSize = _structure_size(LOCAL_HEAP);
|
|
831
|
-
const hBuf = await
|
|
1151
|
+
const hBuf = await fetchRangeWithReadAhead(source, offset, headerSize, LOCAL_HEAP_READ_AHEAD);
|
|
832
1152
|
const local_heap = _unpack_struct_from(LOCAL_HEAP, hBuf, 0);
|
|
833
1153
|
assert(local_heap.get("signature") === "HEAP");
|
|
834
1154
|
assert(local_heap.get("version") === 0);
|
|
835
1155
|
const data_offset = local_heap.get("address_of_data_segment");
|
|
836
1156
|
const seg_size = local_heap.get("data_segment_size");
|
|
837
|
-
const
|
|
1157
|
+
const rel = data_offset - offset;
|
|
1158
|
+
const heap_data = rel >= 0 && rel + seg_size <= hBuf.byteLength ? hBuf.slice(rel, rel + seg_size) : await fetchRange(source, data_offset, seg_size);
|
|
838
1159
|
local_heap.set("heap_data", heap_data);
|
|
839
1160
|
return new Heap(local_heap, heap_data);
|
|
840
1161
|
}
|
|
@@ -856,17 +1177,23 @@ class SymbolTable {
|
|
|
856
1177
|
static async create(source, offset, root = false) {
|
|
857
1178
|
let node;
|
|
858
1179
|
let readOffset = offset;
|
|
1180
|
+
let nodeBuf = null;
|
|
859
1181
|
if (root) {
|
|
860
1182
|
node = /* @__PURE__ */ new Map([["symbols", 1]]);
|
|
861
1183
|
} else {
|
|
862
|
-
|
|
1184
|
+
nodeBuf = await fetchRangeWithReadAhead(
|
|
1185
|
+
source,
|
|
1186
|
+
offset,
|
|
1187
|
+
SYMBOL_TABLE_NODE_SIZE,
|
|
1188
|
+
SYMBOL_TABLE_NODE_SIZE + 32 * SYMBOL_TABLE_ENTRY_SIZE
|
|
1189
|
+
);
|
|
863
1190
|
node = _unpack_struct_from(SYMBOL_TABLE_NODE, nodeBuf, 0);
|
|
864
1191
|
if (node.get("signature") !== "SNOD") throw new Error("incorrect node type");
|
|
865
1192
|
readOffset = offset + SYMBOL_TABLE_NODE_SIZE;
|
|
866
1193
|
}
|
|
867
1194
|
const n_symbols = node.get("symbols");
|
|
868
1195
|
const entryBytes = n_symbols * SYMBOL_TABLE_ENTRY_SIZE;
|
|
869
|
-
const entryBuf = await fetchRange(source, readOffset, entryBytes);
|
|
1196
|
+
const entryBuf = nodeBuf && SYMBOL_TABLE_NODE_SIZE + entryBytes <= nodeBuf.byteLength ? nodeBuf.slice(SYMBOL_TABLE_NODE_SIZE, SYMBOL_TABLE_NODE_SIZE + entryBytes) : await fetchRange(source, readOffset, entryBytes);
|
|
870
1197
|
const entries = [];
|
|
871
1198
|
let localOff = 0;
|
|
872
1199
|
for (let i = 0; i < n_symbols; i++) {
|
|
@@ -915,14 +1242,15 @@ class GlobalHeap {
|
|
|
915
1242
|
this.heap_data = heap_data;
|
|
916
1243
|
}
|
|
917
1244
|
static async create(source, offset) {
|
|
918
|
-
const hBuf = await
|
|
919
|
-
const header = _unpack_struct_from(GLOBAL_HEAP_HEADER, hBuf, 0);
|
|
920
|
-
const heap_data_size = header.get("collection_size") - GLOBAL_HEAP_HEADER_SIZE;
|
|
921
|
-
const heap_data = await fetchRange(
|
|
1245
|
+
const hBuf = await fetchRangeWithReadAhead(
|
|
922
1246
|
source,
|
|
923
|
-
offset
|
|
924
|
-
|
|
1247
|
+
offset,
|
|
1248
|
+
GLOBAL_HEAP_HEADER_SIZE,
|
|
1249
|
+
GLOBAL_HEAP_READ_AHEAD
|
|
925
1250
|
);
|
|
1251
|
+
const header = _unpack_struct_from(GLOBAL_HEAP_HEADER, hBuf, 0);
|
|
1252
|
+
const heap_data_size = header.get("collection_size") - GLOBAL_HEAP_HEADER_SIZE;
|
|
1253
|
+
const heap_data = GLOBAL_HEAP_HEADER_SIZE + heap_data_size <= hBuf.byteLength ? hBuf.slice(GLOBAL_HEAP_HEADER_SIZE, GLOBAL_HEAP_HEADER_SIZE + heap_data_size) : await fetchRange(source, offset + GLOBAL_HEAP_HEADER_SIZE, heap_data_size);
|
|
926
1254
|
return new GlobalHeap(header, heap_data);
|
|
927
1255
|
}
|
|
928
1256
|
get objects() {
|
|
@@ -1033,9 +1361,7 @@ class FractalHeap {
|
|
|
1033
1361
|
table_width
|
|
1034
1362
|
};
|
|
1035
1363
|
if (nrows > 0) {
|
|
1036
|
-
|
|
1037
|
-
managedParts.push(data);
|
|
1038
|
-
}
|
|
1364
|
+
managedParts.push(...await FractalHeap._read_indirect_block(ctx, root_address, nrows));
|
|
1039
1365
|
} else if (root_address != null) {
|
|
1040
1366
|
const data = await FractalHeap._read_direct_block(ctx, root_address, start_block_size);
|
|
1041
1367
|
managedParts.push(data);
|
|
@@ -1093,7 +1419,11 @@ class FractalHeap {
|
|
|
1093
1419
|
assert(header.get("signature") === "FHDB");
|
|
1094
1420
|
return data;
|
|
1095
1421
|
}
|
|
1096
|
-
|
|
1422
|
+
/**
|
|
1423
|
+
* Read all direct blocks reachable from an indirect block (in heap order).
|
|
1424
|
+
* Blocks are fetched concurrently.
|
|
1425
|
+
*/
|
|
1426
|
+
static async _read_indirect_block(ctx, offset, nrows) {
|
|
1097
1427
|
const ibHdrBuf = await fetchRange(ctx.source, offset, ctx.indirect_block_header_size);
|
|
1098
1428
|
const header = _unpack_struct_from(ctx.indirect_block_header, ibHdrBuf, 0);
|
|
1099
1429
|
let readOff = offset + ctx.indirect_block_header_size;
|
|
@@ -1125,12 +1455,19 @@ class FractalHeap {
|
|
|
1125
1455
|
const nr = FractalHeap._iblock_nrows_from_block_size(ctx, block_size);
|
|
1126
1456
|
indirect_blocks.push([address, nr]);
|
|
1127
1457
|
}
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1458
|
+
const [direct, indirect] = await Promise.all([
|
|
1459
|
+
Promise.all(
|
|
1460
|
+
direct_blocks.map(
|
|
1461
|
+
([address, block_size]) => FractalHeap._read_direct_block(ctx, address, block_size)
|
|
1462
|
+
)
|
|
1463
|
+
),
|
|
1464
|
+
Promise.all(
|
|
1465
|
+
indirect_blocks.map(
|
|
1466
|
+
([address, nr]) => FractalHeap._read_indirect_block(ctx, address, nr)
|
|
1467
|
+
)
|
|
1468
|
+
)
|
|
1469
|
+
]);
|
|
1470
|
+
return [...direct, ...indirect.flat()];
|
|
1134
1471
|
}
|
|
1135
1472
|
static _calc_block_size(ctx, iblock) {
|
|
1136
1473
|
const row = Math.floor(iblock / ctx.table_width);
|
|
@@ -1264,6 +1601,15 @@ const LINK_INFO_MSG2 = /* @__PURE__ */ new Map([
|
|
|
1264
1601
|
["name_btree_address", "Q"],
|
|
1265
1602
|
["order_btree_address", "Q"]
|
|
1266
1603
|
]);
|
|
1604
|
+
const OBJECT_HEADER_READ_AHEAD = 1024;
|
|
1605
|
+
const GLOBAL_HEAP_CACHE_SIZE = 256;
|
|
1606
|
+
async function sliceOrFetch(source, prefix, prefixOffset, absOffset, size) {
|
|
1607
|
+
const start = absOffset - prefixOffset;
|
|
1608
|
+
if (start >= 0 && start + size <= prefix.byteLength) {
|
|
1609
|
+
return prefix.slice(start, start + size);
|
|
1610
|
+
}
|
|
1611
|
+
return fetchRange(source, absOffset, size);
|
|
1612
|
+
}
|
|
1267
1613
|
class DataObjects {
|
|
1268
1614
|
buf;
|
|
1269
1615
|
bufStart;
|
|
@@ -1285,7 +1631,7 @@ class DataObjects {
|
|
|
1285
1631
|
this.msgs = msgs;
|
|
1286
1632
|
this.msg_data = msg_data;
|
|
1287
1633
|
this.offset = offset;
|
|
1288
|
-
this._global_heaps =
|
|
1634
|
+
this._global_heaps = new QuickLRU({ maxSize: GLOBAL_HEAP_CACHE_SIZE });
|
|
1289
1635
|
this._header = header;
|
|
1290
1636
|
this._filter_pipeline = null;
|
|
1291
1637
|
this._chunk_params_set = false;
|
|
@@ -1294,13 +1640,13 @@ class DataObjects {
|
|
|
1294
1640
|
this._chunk_address = null;
|
|
1295
1641
|
}
|
|
1296
1642
|
static async create(source, offset) {
|
|
1297
|
-
const
|
|
1298
|
-
const versionHint = struct.unpack_from("<B",
|
|
1643
|
+
const prefix = await fetchRangeWithReadAhead(source, offset, 1, OBJECT_HEADER_READ_AHEAD);
|
|
1644
|
+
const versionHint = struct.unpack_from("<B", prefix, 0)[0];
|
|
1299
1645
|
let result;
|
|
1300
1646
|
if (versionHint === 1) {
|
|
1301
|
-
result = await DataObjects._parseV1(source, offset);
|
|
1647
|
+
result = await DataObjects._parseV1(source, offset, prefix);
|
|
1302
1648
|
} else if (versionHint === "O".charCodeAt(0)) {
|
|
1303
|
-
result = await DataObjects._parseV2(source, offset);
|
|
1649
|
+
result = await DataObjects._parseV2(source, offset, prefix);
|
|
1304
1650
|
} else {
|
|
1305
1651
|
throw new Error(`Unknown Data Object Header version hint: ${versionHint}`);
|
|
1306
1652
|
}
|
|
@@ -1315,8 +1661,8 @@ class DataObjects {
|
|
|
1315
1661
|
);
|
|
1316
1662
|
}
|
|
1317
1663
|
// ────────── V1 parsing ──────────
|
|
1318
|
-
static async _parseV1(source, offset) {
|
|
1319
|
-
const headerBuf = await fetchRange(source, offset, OBJECT_HEADER_V1_SIZE);
|
|
1664
|
+
static async _parseV1(source, offset, prefix) {
|
|
1665
|
+
const headerBuf = prefix.byteLength >= OBJECT_HEADER_V1_SIZE ? prefix : await fetchRange(source, offset, OBJECT_HEADER_V1_SIZE);
|
|
1320
1666
|
const header = _unpack_struct_from(OBJECT_HEADER_V1, headerBuf, 0);
|
|
1321
1667
|
assert(header.get("version") === 1);
|
|
1322
1668
|
const total_header_messages = header.get("total_header_messages");
|
|
@@ -1327,21 +1673,29 @@ class DataObjects {
|
|
|
1327
1673
|
{ absOffset: first_block_offset, size: first_block_size }
|
|
1328
1674
|
];
|
|
1329
1675
|
while (blockQueue.length > 0) {
|
|
1330
|
-
const
|
|
1331
|
-
const
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
const
|
|
1338
|
-
const
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1676
|
+
const wave = blockQueue.splice(0, blockQueue.length);
|
|
1677
|
+
const datas = await Promise.all(
|
|
1678
|
+
wave.map(
|
|
1679
|
+
({ absOffset, size }) => sliceOrFetch(source, prefix, offset, absOffset, size)
|
|
1680
|
+
)
|
|
1681
|
+
);
|
|
1682
|
+
for (let w = 0; w < wave.length; w++) {
|
|
1683
|
+
const { absOffset, size } = wave[w];
|
|
1684
|
+
const data = datas[w];
|
|
1685
|
+
allBlocks.push({ absOffset, size, data });
|
|
1686
|
+
let local_offset2 = 0;
|
|
1687
|
+
while (local_offset2 + HEADER_MSG_INFO_V1_SIZE <= size) {
|
|
1688
|
+
const msg = _unpack_struct_from(HEADER_MSG_INFO_V1, data, local_offset2);
|
|
1689
|
+
const msg_type = msg.get("type");
|
|
1690
|
+
const msg_size = msg.get("size");
|
|
1691
|
+
const offset_in_block = local_offset2 + HEADER_MSG_INFO_V1_SIZE;
|
|
1692
|
+
if (msg_type === OBJECT_CONTINUATION_MSG_TYPE) {
|
|
1693
|
+
const [fh_off, cont_size] = struct.unpack_from("<QQ", data, offset_in_block);
|
|
1694
|
+
blockQueue.push({ absOffset: fh_off, size: cont_size });
|
|
1695
|
+
}
|
|
1696
|
+
if (msg_type === 0 && msg_size === 0) break;
|
|
1697
|
+
local_offset2 += HEADER_MSG_INFO_V1_SIZE + msg_size;
|
|
1342
1698
|
}
|
|
1343
|
-
if (msg_type === 0 && msg_size === 0) break;
|
|
1344
|
-
local_offset2 += HEADER_MSG_INFO_V1_SIZE + msg_size;
|
|
1345
1699
|
}
|
|
1346
1700
|
}
|
|
1347
1701
|
let bufStart = offset;
|
|
@@ -1351,7 +1705,7 @@ class DataObjects {
|
|
|
1351
1705
|
maxEnd = Math.max(maxEnd, b.absOffset + b.size);
|
|
1352
1706
|
}
|
|
1353
1707
|
const combined = new Uint8Array(maxEnd - bufStart);
|
|
1354
|
-
combined.set(new Uint8Array(headerBuf), offset - bufStart);
|
|
1708
|
+
combined.set(new Uint8Array(headerBuf, 0, OBJECT_HEADER_V1_SIZE), offset - bufStart);
|
|
1355
1709
|
for (const b of allBlocks) {
|
|
1356
1710
|
combined.set(new Uint8Array(b.data), b.absOffset - bufStart);
|
|
1357
1711
|
}
|
|
@@ -1387,9 +1741,9 @@ class DataObjects {
|
|
|
1387
1741
|
return { buf, bufStart, msgs: msgs.filter(Boolean), msg_data, header };
|
|
1388
1742
|
}
|
|
1389
1743
|
// ────────── V2 parsing ──────────
|
|
1390
|
-
static async _parseV2(source, offset) {
|
|
1391
|
-
const
|
|
1392
|
-
const
|
|
1744
|
+
static async _parseV2(source, offset, prefix) {
|
|
1745
|
+
const peekBuf = prefix.byteLength >= 64 ? prefix : await fetchRange(source, offset, 64);
|
|
1746
|
+
const peekSize = peekBuf.byteLength;
|
|
1393
1747
|
const { header, creation_order_size, block_offset: first_block_offset } = DataObjects._parseV2Header(peekBuf, 0, offset);
|
|
1394
1748
|
const first_block_size = header.get("size_of_chunk_0");
|
|
1395
1749
|
const allBlocks = [];
|
|
@@ -1397,27 +1751,30 @@ class DataObjects {
|
|
|
1397
1751
|
{ absOffset: first_block_offset, size: first_block_size }
|
|
1398
1752
|
];
|
|
1399
1753
|
while (blockQueue.length > 0) {
|
|
1400
|
-
const
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
const
|
|
1416
|
-
|
|
1754
|
+
const wave = blockQueue.splice(0, blockQueue.length);
|
|
1755
|
+
const datas = await Promise.all(
|
|
1756
|
+
wave.map(
|
|
1757
|
+
({ absOffset, size }) => sliceOrFetch(source, peekBuf, offset, absOffset, size)
|
|
1758
|
+
)
|
|
1759
|
+
);
|
|
1760
|
+
for (let w = 0; w < wave.length; w++) {
|
|
1761
|
+
const { absOffset, size } = wave[w];
|
|
1762
|
+
const data = datas[w];
|
|
1763
|
+
allBlocks.push({ absOffset, size, data });
|
|
1764
|
+
let local_offset2 = 0;
|
|
1765
|
+
while (local_offset2 + HEADER_MSG_INFO_V2_SIZE <= size) {
|
|
1766
|
+
const relOff = local_offset2;
|
|
1767
|
+
const msg_type = struct.unpack_from("<B", data, relOff)[0];
|
|
1768
|
+
const msg_size = struct.unpack_from("<H", data, relOff + 1)[0];
|
|
1769
|
+
const offset_in_data = relOff + HEADER_MSG_INFO_V2_SIZE + creation_order_size;
|
|
1770
|
+
if (msg_type === OBJECT_CONTINUATION_MSG_TYPE) {
|
|
1771
|
+
const [fh_off, cont_size] = struct.unpack_from("<QQ", data, offset_in_data);
|
|
1772
|
+
blockQueue.push({ absOffset: fh_off + 4, size: cont_size - 4 });
|
|
1773
|
+
}
|
|
1774
|
+
if (msg_type === 0 && msg_size === 0) break;
|
|
1775
|
+
local_offset2 += HEADER_MSG_INFO_V2_SIZE + msg_size + creation_order_size;
|
|
1776
|
+
if (local_offset2 >= size - HEADER_MSG_INFO_V2_SIZE) break;
|
|
1417
1777
|
}
|
|
1418
|
-
if (msg_type === 0 && msg_size === 0) break;
|
|
1419
|
-
local_offset2 += HEADER_MSG_INFO_V2_SIZE + msg_size + creation_order_size;
|
|
1420
|
-
if (local_offset2 >= size - HEADER_MSG_INFO_V2_SIZE) break;
|
|
1421
1778
|
}
|
|
1422
1779
|
}
|
|
1423
1780
|
const bufStart = offset;
|
|
@@ -1706,11 +2063,15 @@ class DataObjects {
|
|
|
1706
2063
|
return this._iter_links_btree_v1(data.get("btree_address"), data.get("heap_address"));
|
|
1707
2064
|
}
|
|
1708
2065
|
async _iter_links_btree_v1(btree_address, heap_address) {
|
|
1709
|
-
const btree = await
|
|
1710
|
-
|
|
2066
|
+
const [btree, heap] = await Promise.all([
|
|
2067
|
+
BTreeV1Groups.create(this._source, btree_address),
|
|
2068
|
+
Heap.create(this._source, heap_address)
|
|
2069
|
+
]);
|
|
2070
|
+
const tables = await Promise.all(
|
|
2071
|
+
btree.symbol_table_addresses().map((address) => SymbolTable.create(this._source, address))
|
|
2072
|
+
);
|
|
1711
2073
|
const links = {};
|
|
1712
|
-
for (const
|
|
1713
|
-
const table2 = await SymbolTable.create(this._source, symbol_table_address);
|
|
2074
|
+
for (const table2 of tables) {
|
|
1714
2075
|
table2.assign_name(heap);
|
|
1715
2076
|
Object.assign(links, table2.get_links(heap));
|
|
1716
2077
|
}
|
|
@@ -1729,14 +2090,11 @@ class DataObjects {
|
|
|
1729
2090
|
return {};
|
|
1730
2091
|
}
|
|
1731
2092
|
async _iter_links_btree_v2(name_btree_address, order_btree_address, heap_address) {
|
|
1732
|
-
const heap = await FractalHeap.create(this._source, heap_address);
|
|
1733
2093
|
const ordered = order_btree_address !== null;
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
btree = await BTreeV2GroupNames.create(this._source, name_btree_address);
|
|
1739
|
-
}
|
|
2094
|
+
const [heap, btree] = await Promise.all([
|
|
2095
|
+
FractalHeap.create(this._source, heap_address),
|
|
2096
|
+
ordered ? BTreeV2GroupOrders.create(this._source, order_btree_address) : BTreeV2GroupNames.create(this._source, name_btree_address)
|
|
2097
|
+
]);
|
|
1740
2098
|
const items = /* @__PURE__ */ new Map();
|
|
1741
2099
|
for (const record of btree.iter_records()) {
|
|
1742
2100
|
const data = heap.get_data(record.get("heapid"));
|
|
@@ -1855,14 +2213,11 @@ class DataObjects {
|
|
|
1855
2213
|
if (fractal_heap_address === UNDEFINED_ADDRESS || name_btree_address === UNDEFINED_ADDRESS) {
|
|
1856
2214
|
return {};
|
|
1857
2215
|
}
|
|
1858
|
-
const heap = await FractalHeap.create(this._source, fractal_heap_address);
|
|
1859
2216
|
const ordered = order_btree_address !== null && order_btree_address !== UNDEFINED_ADDRESS;
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
btree = await BTreeV2GroupNames.create(this._source, name_btree_address);
|
|
1865
|
-
}
|
|
2217
|
+
const [heap, btree] = await Promise.all([
|
|
2218
|
+
FractalHeap.create(this._source, fractal_heap_address),
|
|
2219
|
+
ordered ? BTreeV2GroupOrders.create(this._source, order_btree_address) : BTreeV2GroupNames.create(this._source, name_btree_address)
|
|
2220
|
+
]);
|
|
1866
2221
|
const attrs = {};
|
|
1867
2222
|
for (const record of btree.iter_records()) {
|
|
1868
2223
|
try {
|
|
@@ -1960,19 +2315,56 @@ class DataObjects {
|
|
|
1960
2315
|
const gheap_id = _unpack_struct_from(GLOBAL_HEAP_ID, buf, offset + 4);
|
|
1961
2316
|
const gheap_address = gheap_id.get("collection_address");
|
|
1962
2317
|
assert(gheap_address < Number.MAX_SAFE_INTEGER);
|
|
1963
|
-
|
|
1964
|
-
if (this._global_heaps.has(gheap_address)) {
|
|
1965
|
-
gheap = this._global_heaps.get(gheap_address);
|
|
1966
|
-
} else {
|
|
1967
|
-
gheap = await GlobalHeap.create(this._source, gheap_address);
|
|
1968
|
-
this._global_heaps.set(gheap_address, gheap);
|
|
1969
|
-
}
|
|
2318
|
+
const gheap = await this._get_global_heap(gheap_address);
|
|
1970
2319
|
const vlen_data = gheap.objects.get(gheap_id.get("object_index"));
|
|
1971
2320
|
if (!vlen_data) {
|
|
1972
2321
|
throw new Error(`Global heap object not found at index ${gheap_id.get("object_index")}`);
|
|
1973
2322
|
}
|
|
1974
2323
|
return [vlen_size, vlen_data];
|
|
1975
2324
|
}
|
|
2325
|
+
_get_global_heap(address) {
|
|
2326
|
+
let gheap = this._global_heaps.get(address);
|
|
2327
|
+
if (!gheap) {
|
|
2328
|
+
const promise = GlobalHeap.create(this._source, address);
|
|
2329
|
+
promise.catch(() => {
|
|
2330
|
+
if (this._global_heaps.peek(address) === promise) this._global_heaps.delete(address);
|
|
2331
|
+
});
|
|
2332
|
+
this._global_heaps.set(address, promise);
|
|
2333
|
+
gheap = promise;
|
|
2334
|
+
}
|
|
2335
|
+
return gheap;
|
|
2336
|
+
}
|
|
2337
|
+
/**
|
|
2338
|
+
* Decode `count` variable-length strings from consecutive 16-byte global heap references
|
|
2339
|
+
* (uint32 length, uint64 collection address, uint32 object index) starting at `offset`.
|
|
2340
|
+
* All referenced global heap collections are fetched concurrently.
|
|
2341
|
+
* Null / missing references decode to "".
|
|
2342
|
+
*/
|
|
2343
|
+
async _read_vlen_strings(buf, offset, count, decoder) {
|
|
2344
|
+
const view = new DataView(buf);
|
|
2345
|
+
const refs = new Array(count);
|
|
2346
|
+
const addresses = /* @__PURE__ */ new Set();
|
|
2347
|
+
for (let i = 0; i < count; i++) {
|
|
2348
|
+
const o = offset + i * 16;
|
|
2349
|
+
const size = view.getUint32(o, true);
|
|
2350
|
+
const address = Number(view.getBigUint64(o + 4, true));
|
|
2351
|
+
const index = view.getUint32(o + 12, true);
|
|
2352
|
+
refs[i] = { size, address, index };
|
|
2353
|
+
if (size > 0 && address !== 0 && address < Number.MAX_SAFE_INTEGER) {
|
|
2354
|
+
addresses.add(address);
|
|
2355
|
+
}
|
|
2356
|
+
}
|
|
2357
|
+
const heaps = /* @__PURE__ */ new Map();
|
|
2358
|
+
await Promise.all(
|
|
2359
|
+
Array.from(addresses, async (address) => {
|
|
2360
|
+
heaps.set(address, await this._get_global_heap(address));
|
|
2361
|
+
})
|
|
2362
|
+
);
|
|
2363
|
+
return refs.map(({ address, index }) => {
|
|
2364
|
+
const data = heaps.get(address)?.objects.get(index);
|
|
2365
|
+
return data ? decoder.decode(data) : "";
|
|
2366
|
+
});
|
|
2367
|
+
}
|
|
1976
2368
|
// ────────── Async: get_data ──────────
|
|
1977
2369
|
async get_data() {
|
|
1978
2370
|
const msg = this.find_msg_type(DATA_STORAGE_MSG_TYPE)[0];
|
|
@@ -2025,12 +2417,7 @@ class DataObjects {
|
|
|
2025
2417
|
const encoding = character_set === 0 ? "ascii" : "utf-8";
|
|
2026
2418
|
const decoder = new TextDecoder(encoding);
|
|
2027
2419
|
const refBuf = await fetchRange(this._source, data_offset, fullsize * 16);
|
|
2028
|
-
|
|
2029
|
-
for (let i = 0; i < fullsize; i++) {
|
|
2030
|
-
const [, vlen_data] = await this._vlen_size_and_data(refBuf, i * 16);
|
|
2031
|
-
output[i] = decoder.decode(vlen_data);
|
|
2032
|
-
}
|
|
2033
|
-
return output;
|
|
2420
|
+
return this._read_vlen_strings(refBuf, 0, fullsize, decoder);
|
|
2034
2421
|
} else if (dtype_class === "COMPOUND") {
|
|
2035
2422
|
const total_size = dtype[2];
|
|
2036
2423
|
const dataBuf = await fetchRange(this._source, data_offset, fullsize * total_size);
|
|
@@ -2044,42 +2431,74 @@ class DataObjects {
|
|
|
2044
2431
|
}
|
|
2045
2432
|
}
|
|
2046
2433
|
}
|
|
2047
|
-
async _get_chunked_data(
|
|
2434
|
+
async _get_chunked_data(_msg_offset) {
|
|
2048
2435
|
this._get_chunk_params();
|
|
2049
2436
|
if (this._chunk_address === null || this._chunk_address === UNDEFINED_ADDRESS) {
|
|
2050
2437
|
return [];
|
|
2051
2438
|
}
|
|
2052
2439
|
const dtype = this.dtype;
|
|
2053
|
-
if (!Array.isArray(dtype) ||
|
|
2440
|
+
if (!Array.isArray(dtype) || dtype[0] !== "VLEN_STRING") {
|
|
2054
2441
|
return [];
|
|
2055
2442
|
}
|
|
2056
|
-
dtype[0];
|
|
2057
2443
|
const character_set = dtype[2];
|
|
2058
2444
|
const encoding = character_set === 0 ? "ascii" : "utf-8";
|
|
2059
2445
|
const decoder = new TextDecoder(encoding);
|
|
2446
|
+
const shape = this.shape;
|
|
2447
|
+
const chunkShape = this._chunks;
|
|
2448
|
+
const ndim = shape.length;
|
|
2449
|
+
const total = shape.reduce((a, b) => a * b, 1);
|
|
2450
|
+
const output = new Array(total).fill("");
|
|
2451
|
+
const pipeline = this.filter_pipeline;
|
|
2060
2452
|
const chunk_btree = await BTreeV1RawDataChunks.create(
|
|
2061
2453
|
this._source,
|
|
2062
2454
|
this._chunk_address,
|
|
2063
2455
|
this._chunk_dims
|
|
2064
2456
|
);
|
|
2065
|
-
const
|
|
2066
|
-
const
|
|
2067
|
-
if (!leafNodes) return output;
|
|
2457
|
+
const leafNodes = chunk_btree.all_nodes.get(0) ?? [];
|
|
2458
|
+
const chunks = [];
|
|
2068
2459
|
for (const node of leafNodes) {
|
|
2069
2460
|
for (let ik = 0; ik < node.keys.length; ik++) {
|
|
2070
|
-
const
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2461
|
+
const key = node.keys[ik];
|
|
2462
|
+
chunks.push({
|
|
2463
|
+
address: node.addresses[ik],
|
|
2464
|
+
size: key.chunk_size,
|
|
2465
|
+
filterMask: key.filter_mask,
|
|
2466
|
+
start: key.chunk_offset.slice(0, ndim)
|
|
2467
|
+
});
|
|
2468
|
+
}
|
|
2469
|
+
}
|
|
2470
|
+
const chunkItems = chunkShape.reduce((a, b) => a * b, 1);
|
|
2471
|
+
const strides = new Array(ndim).fill(1);
|
|
2472
|
+
const chunkStrides = new Array(ndim).fill(1);
|
|
2473
|
+
for (let d = ndim - 2; d >= 0; d--) {
|
|
2474
|
+
strides[d] = strides[d + 1] * shape[d + 1];
|
|
2475
|
+
chunkStrides[d] = chunkStrides[d + 1] * chunkShape[d + 1];
|
|
2476
|
+
}
|
|
2477
|
+
await Promise.all(
|
|
2478
|
+
chunks.map(async ({ address, size, filterMask, start }) => {
|
|
2479
|
+
const raw = new Uint8Array(await fetchRange(this._source, address, size));
|
|
2480
|
+
const decoded = await decodeChunk(raw, pipeline, filterMask);
|
|
2481
|
+
const nItems = Math.min(chunkItems, Math.floor(decoded.byteLength / 16));
|
|
2482
|
+
const buf = decoded.buffer.slice(
|
|
2483
|
+
decoded.byteOffset,
|
|
2484
|
+
decoded.byteOffset + nItems * 16
|
|
2485
|
+
);
|
|
2486
|
+
const strings = await this._read_vlen_strings(buf, 0, nItems, decoder);
|
|
2074
2487
|
for (let i = 0; i < nItems; i++) {
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2488
|
+
let flat = 0;
|
|
2489
|
+
let inBounds = true;
|
|
2490
|
+
for (let d = 0; d < ndim; d++) {
|
|
2491
|
+
const coord = start[d] + Math.floor(i / chunkStrides[d]) % chunkShape[d];
|
|
2492
|
+
if (coord >= shape[d]) {
|
|
2493
|
+
inBounds = false;
|
|
2494
|
+
break;
|
|
2495
|
+
}
|
|
2496
|
+
flat += coord * strides[d];
|
|
2079
2497
|
}
|
|
2498
|
+
if (inBounds) output[flat] = strings[i];
|
|
2080
2499
|
}
|
|
2081
|
-
}
|
|
2082
|
-
|
|
2500
|
+
})
|
|
2501
|
+
);
|
|
2083
2502
|
return output;
|
|
2084
2503
|
}
|
|
2085
2504
|
}
|
|
@@ -2168,6 +2587,7 @@ class File extends Group {
|
|
|
2168
2587
|
}
|
|
2169
2588
|
static async create(source, filename) {
|
|
2170
2589
|
const superblock = await SuperBlock.create(source, 0);
|
|
2590
|
+
setFileSize(source, superblock.end_of_file);
|
|
2171
2591
|
const offset = await superblock.getOffsetToDataobjects();
|
|
2172
2592
|
const dataobjects = await DataObjects.create(source, offset);
|
|
2173
2593
|
const links = await dataobjects.get_links();
|
|
@@ -2672,13 +3092,15 @@ function getAttrDatatypeClasses(dataobjects) {
|
|
|
2672
3092
|
}
|
|
2673
3093
|
return result;
|
|
2674
3094
|
}
|
|
3095
|
+
function isVlenString(rawDtype) {
|
|
3096
|
+
return Array.isArray(rawDtype) && rawDtype[0] === "VLEN_STRING";
|
|
3097
|
+
}
|
|
2675
3098
|
class SingleHdf5ToZarr {
|
|
2676
3099
|
source;
|
|
2677
|
-
h5File;
|
|
3100
|
+
h5File = null;
|
|
2678
3101
|
url;
|
|
2679
3102
|
inlineThreshold;
|
|
2680
3103
|
zarrFormat;
|
|
2681
|
-
refs;
|
|
2682
3104
|
/**
|
|
2683
3105
|
* @param source - An AsyncReadable instance for reading the HDF5 file.
|
|
2684
3106
|
* @param options - Configuration options.
|
|
@@ -2688,62 +3110,103 @@ class SingleHdf5ToZarr {
|
|
|
2688
3110
|
this.url = options.url ?? null;
|
|
2689
3111
|
this.inlineThreshold = options.inlineThreshold ?? 300;
|
|
2690
3112
|
this.zarrFormat = options.zarrFormat ?? 2;
|
|
2691
|
-
|
|
3113
|
+
}
|
|
3114
|
+
/**
|
|
3115
|
+
* Open (and cache) the HDF5 file: reads the superblock and the root group's links.
|
|
3116
|
+
*/
|
|
3117
|
+
open() {
|
|
3118
|
+
if (!this.h5File) {
|
|
3119
|
+
this.h5File = File.create(this.source);
|
|
3120
|
+
this.h5File.catch(() => {
|
|
3121
|
+
this.h5File = null;
|
|
3122
|
+
});
|
|
3123
|
+
}
|
|
3124
|
+
return this.h5File;
|
|
2692
3125
|
}
|
|
2693
3126
|
/**
|
|
2694
3127
|
* Translate the HDF5 file content into a Zarr reference spec.
|
|
3128
|
+
* Independent groups and datasets are processed concurrently.
|
|
2695
3129
|
*/
|
|
2696
3130
|
async translate() {
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
await this.transferAttrs(this.h5File, "");
|
|
2701
|
-
await this.walkGroup(this.h5File, "");
|
|
2702
|
-
const refSpec = {
|
|
2703
|
-
version: 1,
|
|
2704
|
-
refs: this.refs
|
|
2705
|
-
};
|
|
3131
|
+
const h5File = await this.open();
|
|
3132
|
+
const refs = await this.translateTree(h5File, true);
|
|
3133
|
+
const refSpec = { version: 1, refs };
|
|
2706
3134
|
return this.zarrFormat === 3 ? refSpecV2ToV3(refSpec) : refSpec;
|
|
2707
3135
|
}
|
|
2708
3136
|
/**
|
|
2709
|
-
*
|
|
3137
|
+
* Translate the metadata (groups, arrays, attributes) of the whole hierarchy,
|
|
3138
|
+
* without any chunk references. Only HDF5 metadata is read; no data chunks or
|
|
3139
|
+
* variable-length string heaps are touched.
|
|
2710
3140
|
*/
|
|
2711
|
-
async
|
|
2712
|
-
const
|
|
2713
|
-
|
|
2714
|
-
|
|
2715
|
-
|
|
2716
|
-
|
|
2717
|
-
|
|
2718
|
-
|
|
2719
|
-
|
|
2720
|
-
|
|
2721
|
-
|
|
2722
|
-
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
3141
|
+
async translateMetadata() {
|
|
3142
|
+
const h5File = await this.open();
|
|
3143
|
+
const refs = await this.translateTree(h5File, false);
|
|
3144
|
+
const refSpec = { version: 1, refs };
|
|
3145
|
+
return this.zarrFormat === 3 ? refSpecV2ToV3(refSpec) : refSpec;
|
|
3146
|
+
}
|
|
3147
|
+
/**
|
|
3148
|
+
* Translate a single HDF5 group or dataset into Zarr v2 references
|
|
3149
|
+
* (keys are relative to the hierarchy root, e.g. "obs/.zattrs" or "X/data/0").
|
|
3150
|
+
*
|
|
3151
|
+
* @param path Path of the node within the hierarchy ("" for the root).
|
|
3152
|
+
* @param obj The HDF5 group or dataset at that path.
|
|
3153
|
+
* @param options.chunks Whether to include chunk references (and inline data).
|
|
3154
|
+
*/
|
|
3155
|
+
async translateNode(path, obj, options = {}) {
|
|
3156
|
+
const refs = {};
|
|
3157
|
+
if (obj instanceof Dataset) {
|
|
3158
|
+
await this.processDataset(refs, path, obj, options.chunks ?? true);
|
|
3159
|
+
} else {
|
|
3160
|
+
await this.processGroup(refs, path, obj);
|
|
2728
3161
|
}
|
|
3162
|
+
return refs;
|
|
3163
|
+
}
|
|
3164
|
+
/**
|
|
3165
|
+
* Translate a node and all of its descendants. Children are processed
|
|
3166
|
+
* concurrently, but the returned refs are in depth-first order
|
|
3167
|
+
* (the same order a sequential walk would produce).
|
|
3168
|
+
*/
|
|
3169
|
+
async translateTree(root, chunks) {
|
|
3170
|
+
const parts = [];
|
|
3171
|
+
const visit = async (path, obj) => {
|
|
3172
|
+
const own = this.translateNode(path, obj, { chunks });
|
|
3173
|
+
if (!(obj instanceof Group)) return [await own];
|
|
3174
|
+
const children = await Promise.all(
|
|
3175
|
+
obj.keys.map(async (key) => {
|
|
3176
|
+
const childPath = path ? `${path}/${key}` : key;
|
|
3177
|
+
let child;
|
|
3178
|
+
try {
|
|
3179
|
+
child = await obj.get(key);
|
|
3180
|
+
} catch {
|
|
3181
|
+
return [];
|
|
3182
|
+
}
|
|
3183
|
+
if (!child) return [];
|
|
3184
|
+
return visit(childPath, child);
|
|
3185
|
+
})
|
|
3186
|
+
);
|
|
3187
|
+
return [await own, ...children.flat()];
|
|
3188
|
+
};
|
|
3189
|
+
parts.push(...await visit("", root));
|
|
3190
|
+
return Object.assign({}, ...parts);
|
|
2729
3191
|
}
|
|
2730
3192
|
/**
|
|
2731
3193
|
* Process an HDF5 group into Zarr group metadata.
|
|
2732
3194
|
*/
|
|
2733
|
-
async processGroup(path, group) {
|
|
2734
|
-
|
|
3195
|
+
async processGroup(refs, path, group) {
|
|
3196
|
+
const prefix = path ? `${path}/` : "";
|
|
3197
|
+
refs[`${prefix}.zgroup`] = jsonStringifySorted({
|
|
2735
3198
|
zarr_format: 2
|
|
2736
3199
|
});
|
|
2737
|
-
await this.transferAttrs(group, path);
|
|
3200
|
+
await this.transferAttrs(refs, group, path);
|
|
2738
3201
|
}
|
|
2739
3202
|
/**
|
|
2740
|
-
* Process an HDF5 dataset into Zarr array metadata and chunk references.
|
|
3203
|
+
* Process an HDF5 dataset into Zarr array metadata and (optionally) chunk references.
|
|
2741
3204
|
*/
|
|
2742
|
-
async processDataset(path, dataset) {
|
|
3205
|
+
async processDataset(refs, path, dataset, includeChunks) {
|
|
2743
3206
|
const dataobjects = dataset._dataobjects;
|
|
2744
3207
|
const shape = dataobjects.shape;
|
|
2745
3208
|
if (!shape || shape.length === 0) {
|
|
2746
|
-
await this.processScalarDataset(path, dataset, dataobjects);
|
|
3209
|
+
await this.processScalarDataset(refs, path, dataset, dataobjects, includeChunks);
|
|
2747
3210
|
return;
|
|
2748
3211
|
}
|
|
2749
3212
|
const rawDtype = dataobjects.dtype;
|
|
@@ -2778,25 +3241,32 @@ class SingleHdf5ToZarr {
|
|
|
2778
3241
|
order: "C",
|
|
2779
3242
|
filters: zarrFilters
|
|
2780
3243
|
};
|
|
2781
|
-
|
|
2782
|
-
|
|
3244
|
+
if (isVlenString(rawDtype)) {
|
|
3245
|
+
zarrMeta.dtype = "|O";
|
|
3246
|
+
zarrMeta.fill_value = null;
|
|
3247
|
+
zarrMeta.filters = [JSON2_FILTER];
|
|
3248
|
+
zarrMeta.compressor = null;
|
|
3249
|
+
}
|
|
3250
|
+
refs[`${path}/.zarray`] = serializeZarrayMeta(zarrMeta);
|
|
3251
|
+
await this.transferAttrs(refs, dataset, path);
|
|
2783
3252
|
const dims = this.getArrayDims(dataset, path, shape);
|
|
2784
3253
|
const existingAttrsKey = `${path}/.zattrs`;
|
|
2785
|
-
const existingAttrs =
|
|
3254
|
+
const existingAttrs = refs[existingAttrsKey] ? JSON.parse(refs[existingAttrsKey]) : {};
|
|
2786
3255
|
existingAttrs["_ARRAY_DIMENSIONS"] = dims;
|
|
2787
|
-
|
|
3256
|
+
refs[existingAttrsKey] = jsonStringifySorted(existingAttrs);
|
|
3257
|
+
if (!includeChunks) return;
|
|
2788
3258
|
if (isVlen) {
|
|
2789
|
-
await this.inlineVlenData(path, dataset, shape, chunks);
|
|
3259
|
+
await this.inlineVlenData(refs, path, dataset, shape, chunks);
|
|
2790
3260
|
} else if (isFixedString) {
|
|
2791
|
-
await this.inlineFixedStringData(path, dataset, shape, chunks, rawDtype);
|
|
3261
|
+
await this.inlineFixedStringData(refs, path, dataset, shape, chunks, rawDtype);
|
|
2792
3262
|
} else {
|
|
2793
|
-
await this.processChunkReferences(path, dataobjects, shape, chunks, dtype);
|
|
3263
|
+
await this.processChunkReferences(refs, path, dataobjects, shape, chunks, dtype);
|
|
2794
3264
|
}
|
|
2795
3265
|
}
|
|
2796
3266
|
/**
|
|
2797
3267
|
* Process a scalar dataset (shape=[]).
|
|
2798
3268
|
*/
|
|
2799
|
-
async processScalarDataset(path, dataset, dataobjects) {
|
|
3269
|
+
async processScalarDataset(refs, path, dataset, dataobjects, includeChunks) {
|
|
2800
3270
|
const rawDtype = dataobjects.dtype;
|
|
2801
3271
|
const isVlen = Array.isArray(rawDtype) && (rawDtype[0] === "VLEN_STRING" || rawDtype[0] === "VLEN_SEQUENCE");
|
|
2802
3272
|
const dtype = jsfiveDtypeToZarr(rawDtype);
|
|
@@ -2820,17 +3290,18 @@ class SingleHdf5ToZarr {
|
|
|
2820
3290
|
zarrMeta.fill_value = null;
|
|
2821
3291
|
zarrMeta.filters = [JSON2_FILTER];
|
|
2822
3292
|
}
|
|
2823
|
-
|
|
2824
|
-
await this.transferAttrs(dataset, path);
|
|
3293
|
+
refs[`${path}/.zarray`] = serializeZarrayMeta(zarrMeta);
|
|
3294
|
+
await this.transferAttrs(refs, dataset, path);
|
|
2825
3295
|
const existingAttrsKey = `${path}/.zattrs`;
|
|
2826
|
-
const existingAttrs =
|
|
3296
|
+
const existingAttrs = refs[existingAttrsKey] ? JSON.parse(refs[existingAttrsKey]) : {};
|
|
2827
3297
|
existingAttrs["_ARRAY_DIMENSIONS"] = [];
|
|
2828
|
-
|
|
3298
|
+
refs[existingAttrsKey] = jsonStringifySorted(existingAttrs);
|
|
3299
|
+
if (!includeChunks) return;
|
|
2829
3300
|
if (isVlen) {
|
|
2830
3301
|
try {
|
|
2831
3302
|
const data = await dataobjects.get_data();
|
|
2832
3303
|
const strings = Array.isArray(data) ? data.map(String) : [String(data)];
|
|
2833
|
-
|
|
3304
|
+
refs[`${path}/0`] = encodeJson2(strings, []);
|
|
2834
3305
|
} catch {
|
|
2835
3306
|
}
|
|
2836
3307
|
} else {
|
|
@@ -2839,7 +3310,7 @@ class SingleHdf5ToZarr {
|
|
|
2839
3310
|
const value = Array.isArray(data) ? data[0] : data;
|
|
2840
3311
|
const bytes = encodeScalar(value, dtype);
|
|
2841
3312
|
if (bytes) {
|
|
2842
|
-
|
|
3313
|
+
refs[`${path}/0`] = inlineBytes(bytes);
|
|
2843
3314
|
}
|
|
2844
3315
|
} catch {
|
|
2845
3316
|
}
|
|
@@ -2848,26 +3319,26 @@ class SingleHdf5ToZarr {
|
|
|
2848
3319
|
/**
|
|
2849
3320
|
* Inline vlen string data using json2 codec encoding.
|
|
2850
3321
|
*/
|
|
2851
|
-
async inlineVlenData(path, dataset, shape, chunks) {
|
|
3322
|
+
async inlineVlenData(refs, path, dataset, shape, chunks) {
|
|
2852
3323
|
try {
|
|
2853
3324
|
const data = await dataset._dataobjects.get_data();
|
|
2854
3325
|
if (data === null || data === void 0) return;
|
|
2855
3326
|
const strings = Array.isArray(data) ? data.map(String) : [String(data)];
|
|
2856
3327
|
const key = `${path}/${chunks.map(() => "0").join(".")}`;
|
|
2857
|
-
|
|
2858
|
-
const zarrMeta = JSON.parse(
|
|
3328
|
+
refs[key] = encodeJson2(strings, shape);
|
|
3329
|
+
const zarrMeta = JSON.parse(refs[`${path}/.zarray`]);
|
|
2859
3330
|
zarrMeta.dtype = "|O";
|
|
2860
3331
|
zarrMeta.fill_value = null;
|
|
2861
3332
|
zarrMeta.filters = [JSON2_FILTER];
|
|
2862
3333
|
zarrMeta.compressor = null;
|
|
2863
|
-
|
|
3334
|
+
refs[`${path}/.zarray`] = serializeZarrayMeta(zarrMeta);
|
|
2864
3335
|
} catch {
|
|
2865
3336
|
}
|
|
2866
3337
|
}
|
|
2867
3338
|
/**
|
|
2868
3339
|
* Inline fixed-length string data.
|
|
2869
3340
|
*/
|
|
2870
|
-
async inlineFixedStringData(path, dataset, shape, chunks, rawDtype) {
|
|
3341
|
+
async inlineFixedStringData(refs, path, dataset, shape, chunks, rawDtype) {
|
|
2871
3342
|
try {
|
|
2872
3343
|
const data = await dataset._dataobjects.get_data();
|
|
2873
3344
|
if (data === null || data === void 0) return;
|
|
@@ -2882,7 +3353,7 @@ class SingleHdf5ToZarr {
|
|
|
2882
3353
|
const encoded = encoder.encode(strings[i]);
|
|
2883
3354
|
buf.set(encoded.subarray(0, strSize), i * strSize);
|
|
2884
3355
|
}
|
|
2885
|
-
|
|
3356
|
+
refs[key] = inlineBytes(buf);
|
|
2886
3357
|
} catch {
|
|
2887
3358
|
}
|
|
2888
3359
|
}
|
|
@@ -2890,7 +3361,7 @@ class SingleHdf5ToZarr {
|
|
|
2890
3361
|
* Process chunk references for non-string datasets.
|
|
2891
3362
|
* Uses file byte offset references for chunked and contiguous data.
|
|
2892
3363
|
*/
|
|
2893
|
-
async processChunkReferences(path, dataobjects, shape, chunks, dtype) {
|
|
3364
|
+
async processChunkReferences(refs, path, dataobjects, shape, chunks, dtype) {
|
|
2894
3365
|
const storageInfo = getStorageInfo(dataobjects);
|
|
2895
3366
|
if (storageInfo.layoutClass === 1 && storageInfo.contiguousOffset !== void 0) {
|
|
2896
3367
|
const key = `${path}/${shape.map(() => "0").join(".")}`;
|
|
@@ -2898,9 +3369,9 @@ class SingleHdf5ToZarr {
|
|
|
2898
3369
|
const size = storageInfo.contiguousSize;
|
|
2899
3370
|
if (size <= this.inlineThreshold) {
|
|
2900
3371
|
const ab = await fetchRange(this.source, offset, size);
|
|
2901
|
-
|
|
3372
|
+
refs[key] = inlineBytes(new Uint8Array(ab));
|
|
2902
3373
|
} else {
|
|
2903
|
-
|
|
3374
|
+
refs[key] = [this.url, offset, size];
|
|
2904
3375
|
}
|
|
2905
3376
|
} else if (storageInfo.layoutClass === 2 && storageInfo.chunkAddress !== void 0) {
|
|
2906
3377
|
const filterPipeline = dataobjects.filter_pipeline;
|
|
@@ -2926,19 +3397,21 @@ class SingleHdf5ToZarr {
|
|
|
2926
3397
|
shape,
|
|
2927
3398
|
storageInfo.chunkShape
|
|
2928
3399
|
);
|
|
2929
|
-
|
|
2930
|
-
|
|
2931
|
-
|
|
2932
|
-
|
|
2933
|
-
|
|
2934
|
-
|
|
2935
|
-
|
|
2936
|
-
|
|
2937
|
-
|
|
2938
|
-
|
|
2939
|
-
|
|
2940
|
-
|
|
2941
|
-
|
|
3400
|
+
const entries = await Promise.all(
|
|
3401
|
+
chunkLocations.map(async (loc) => {
|
|
3402
|
+
const key = `${path}/${loc.chunkIndex.join(".")}`;
|
|
3403
|
+
let size = hasFilters ? loc.size : uncompressedChunkBytes;
|
|
3404
|
+
if (hasF32) {
|
|
3405
|
+
size -= 4;
|
|
3406
|
+
}
|
|
3407
|
+
if (size <= this.inlineThreshold) {
|
|
3408
|
+
const ab = await fetchRange(this.source, loc.offset, size);
|
|
3409
|
+
return [key, inlineBytes(new Uint8Array(ab))];
|
|
3410
|
+
}
|
|
3411
|
+
return [key, [this.url, loc.offset, size]];
|
|
3412
|
+
})
|
|
3413
|
+
);
|
|
3414
|
+
for (const [key, value] of entries) refs[key] = value;
|
|
2942
3415
|
} else if (storageInfo.layoutClass === 1) ;
|
|
2943
3416
|
}
|
|
2944
3417
|
/**
|
|
@@ -2956,7 +3429,7 @@ class SingleHdf5ToZarr {
|
|
|
2956
3429
|
/**
|
|
2957
3430
|
* Transfer attributes from an HDF5 object to the Zarr .zattrs.
|
|
2958
3431
|
*/
|
|
2959
|
-
async transferAttrs(h5obj, path) {
|
|
3432
|
+
async transferAttrs(refs, h5obj, path) {
|
|
2960
3433
|
const attrs = {};
|
|
2961
3434
|
let rawAttrs;
|
|
2962
3435
|
try {
|
|
@@ -2995,7 +3468,7 @@ class SingleHdf5ToZarr {
|
|
|
2995
3468
|
}
|
|
2996
3469
|
if (Object.keys(attrs).length > 0) {
|
|
2997
3470
|
const key = path ? `${path}/.zattrs` : ".zattrs";
|
|
2998
|
-
|
|
3471
|
+
refs[key] = jsonStringifySorted(attrs);
|
|
2999
3472
|
}
|
|
3000
3473
|
}
|
|
3001
3474
|
}
|
|
@@ -3035,177 +3508,13 @@ function refSpecToConsolidatedMetadataV3(refSpec) {
|
|
|
3035
3508
|
}
|
|
3036
3509
|
};
|
|
3037
3510
|
}
|
|
3038
|
-
function parse$1(template, re = /{{(.*?)}}/) {
|
|
3039
|
-
let result = re.exec(template);
|
|
3040
|
-
const parts = [];
|
|
3041
|
-
let pos;
|
|
3042
|
-
while (result) {
|
|
3043
|
-
pos = result.index;
|
|
3044
|
-
if (pos !== 0) {
|
|
3045
|
-
parts.push({ match: false, str: template.substring(0, pos) });
|
|
3046
|
-
template = template.slice(pos);
|
|
3047
|
-
}
|
|
3048
|
-
parts.push({ match: true, str: result[0] });
|
|
3049
|
-
template = template.slice(result[0].length);
|
|
3050
|
-
result = re.exec(template);
|
|
3051
|
-
}
|
|
3052
|
-
if (template) {
|
|
3053
|
-
parts.push({ match: false, str: template });
|
|
3054
|
-
}
|
|
3055
|
-
return parts;
|
|
3056
|
-
}
|
|
3057
|
-
function matchFn(str) {
|
|
3058
|
-
const match = str.match(/(?<fname>[A-Z_][A-Z_1-9]*)\((?<args>[^)]+)\)/i);
|
|
3059
|
-
if (!match?.groups) return;
|
|
3060
|
-
const { fname, args } = match.groups;
|
|
3061
|
-
const ctx = Object.fromEntries(
|
|
3062
|
-
args.split(",").map((kwarg) => {
|
|
3063
|
-
const { key, num, str: str2 } = kwarg.match(
|
|
3064
|
-
/(?<key>[a-z_0-9]*)\s*=\s*((?<num>[0-9.]+)|('|")(?<str>.*)('|"))/i
|
|
3065
|
-
)?.groups ?? {};
|
|
3066
|
-
if (!key || !(num || str2)) {
|
|
3067
|
-
throw Error(`Failed to match fn kwarg: ${kwarg}`);
|
|
3068
|
-
}
|
|
3069
|
-
return [key, num ? Number(num) : str2];
|
|
3070
|
-
})
|
|
3071
|
-
);
|
|
3072
|
-
return { fname, ctx };
|
|
3073
|
-
}
|
|
3074
3511
|
const alphabet = "abcdefghijklmnopqrstuvwxyz";
|
|
3075
3512
|
const numbers = "01234569789";
|
|
3076
3513
|
const expr = "()*/+-";
|
|
3077
3514
|
const space = " ";
|
|
3078
|
-
|
|
3515
|
+
new Set(
|
|
3079
3516
|
alphabet + alphabet.toUpperCase() + numbers + expr + space
|
|
3080
3517
|
);
|
|
3081
|
-
function matchMathEval(str) {
|
|
3082
|
-
for (let i = 0; i < str.length; i++) {
|
|
3083
|
-
if (!valid.has(str.charAt(i))) return;
|
|
3084
|
-
}
|
|
3085
|
-
return parse$1(str, /[A-Za-z_][A-Za-z0-9_]*/gi);
|
|
3086
|
-
}
|
|
3087
|
-
function render(template, context) {
|
|
3088
|
-
const grps = parse$1(template);
|
|
3089
|
-
return grps.map((grp) => {
|
|
3090
|
-
if (!grp.match) {
|
|
3091
|
-
return grp.str;
|
|
3092
|
-
}
|
|
3093
|
-
let inner = grp.str.split(/{{|}}/).filter(Boolean)[0].trim();
|
|
3094
|
-
if (inner in context) {
|
|
3095
|
-
return context[inner];
|
|
3096
|
-
}
|
|
3097
|
-
const fnMatch = matchFn(inner);
|
|
3098
|
-
if (fnMatch) {
|
|
3099
|
-
const { fname, ctx } = fnMatch;
|
|
3100
|
-
const fn = context[fname];
|
|
3101
|
-
if (typeof fn === "function") {
|
|
3102
|
-
return fn(ctx);
|
|
3103
|
-
}
|
|
3104
|
-
throw Error(
|
|
3105
|
-
`Cannot find function named ${fname} in rendering context.`
|
|
3106
|
-
);
|
|
3107
|
-
}
|
|
3108
|
-
const matches = matchMathEval(inner);
|
|
3109
|
-
if (matches) {
|
|
3110
|
-
const exprParts = matches.map((match) => {
|
|
3111
|
-
if (!match.match) return match.str;
|
|
3112
|
-
const value = context[match.str];
|
|
3113
|
-
if (value == null) {
|
|
3114
|
-
throw Error(
|
|
3115
|
-
`Cannot find number named ${match.str} in rendering context.`
|
|
3116
|
-
);
|
|
3117
|
-
}
|
|
3118
|
-
if (typeof value !== "number") {
|
|
3119
|
-
throw Error(
|
|
3120
|
-
`The provided value for ${match.str} must be a number.`
|
|
3121
|
-
);
|
|
3122
|
-
}
|
|
3123
|
-
return value;
|
|
3124
|
-
});
|
|
3125
|
-
return Function('"use strict";return (' + exprParts.join("") + ")")();
|
|
3126
|
-
}
|
|
3127
|
-
throw new Error(`Unable to match ${grp.str}`);
|
|
3128
|
-
}).join("");
|
|
3129
|
-
}
|
|
3130
|
-
function parse(spec, renderString = render) {
|
|
3131
|
-
return "version" in spec ? parseV1(spec, renderString) : parseV0(spec);
|
|
3132
|
-
}
|
|
3133
|
-
function parseV0(spec) {
|
|
3134
|
-
return new Map(Object.entries(spec));
|
|
3135
|
-
}
|
|
3136
|
-
function parseV1(spec, renderString) {
|
|
3137
|
-
const context = {};
|
|
3138
|
-
for (const [key, template] of Object.entries(spec.templates ?? {})) {
|
|
3139
|
-
if (template.includes("{{")) {
|
|
3140
|
-
context[key] = (ctx) => renderString(template, ctx);
|
|
3141
|
-
} else {
|
|
3142
|
-
context[key] = template;
|
|
3143
|
-
}
|
|
3144
|
-
}
|
|
3145
|
-
const render2 = (t, o) => {
|
|
3146
|
-
return renderString(t, { ...context, ...o });
|
|
3147
|
-
};
|
|
3148
|
-
const refs = /* @__PURE__ */ new Map();
|
|
3149
|
-
for (const [key, ref] of Object.entries(spec.refs ?? {})) {
|
|
3150
|
-
if (typeof ref === "string") {
|
|
3151
|
-
refs.set(key, ref);
|
|
3152
|
-
} else {
|
|
3153
|
-
const url = ref[0]?.includes("{{") ? render2(ref[0]) : ref[0];
|
|
3154
|
-
refs.set(key, ref.length === 1 ? [url] : [url, ref[1], ref[2]]);
|
|
3155
|
-
}
|
|
3156
|
-
}
|
|
3157
|
-
for (const g of spec.gen ?? []) {
|
|
3158
|
-
for (const dims of iterDims(g.dimensions)) {
|
|
3159
|
-
const key = render2(g.key, dims);
|
|
3160
|
-
const url = render2(g.url, dims);
|
|
3161
|
-
if ("offset" in g && "length" in g) {
|
|
3162
|
-
const offset = render2(g.offset, dims);
|
|
3163
|
-
const length = render2(g.length, dims);
|
|
3164
|
-
refs.set(key, [url, parseInt(offset), parseInt(length)]);
|
|
3165
|
-
} else {
|
|
3166
|
-
refs.set(key, [url]);
|
|
3167
|
-
}
|
|
3168
|
-
}
|
|
3169
|
-
}
|
|
3170
|
-
return refs;
|
|
3171
|
-
}
|
|
3172
|
-
function* iterDims(dimensions) {
|
|
3173
|
-
const keys = Object.keys(dimensions);
|
|
3174
|
-
const iterables = Object.values(dimensions).map(
|
|
3175
|
-
(i) => Array.isArray(i) ? i : [...range(i)]
|
|
3176
|
-
);
|
|
3177
|
-
for (const values of product(...iterables)) {
|
|
3178
|
-
yield Object.fromEntries(keys.map((key, i) => [key, values[i]]));
|
|
3179
|
-
}
|
|
3180
|
-
}
|
|
3181
|
-
function* product(...iterables) {
|
|
3182
|
-
if (iterables.length === 0) {
|
|
3183
|
-
return;
|
|
3184
|
-
}
|
|
3185
|
-
const iterators = iterables.map((it) => it[Symbol.iterator]());
|
|
3186
|
-
const results = iterators.map((it) => it.next());
|
|
3187
|
-
if (results.some((r) => r.done)) {
|
|
3188
|
-
throw new Error("Input contains an empty iterator.");
|
|
3189
|
-
}
|
|
3190
|
-
for (let i = 0; ; ) {
|
|
3191
|
-
if (results[i].done) {
|
|
3192
|
-
iterators[i] = iterables[i][Symbol.iterator]();
|
|
3193
|
-
results[i] = iterators[i].next();
|
|
3194
|
-
if (++i >= iterators.length) {
|
|
3195
|
-
return;
|
|
3196
|
-
}
|
|
3197
|
-
} else {
|
|
3198
|
-
yield results.map(({ value }) => value);
|
|
3199
|
-
i = 0;
|
|
3200
|
-
}
|
|
3201
|
-
results[i] = iterators[i].next();
|
|
3202
|
-
}
|
|
3203
|
-
}
|
|
3204
|
-
function* range({ stop, start = 0, step = 1 }) {
|
|
3205
|
-
for (let i = start; i < stop; i += step) {
|
|
3206
|
-
yield i;
|
|
3207
|
-
}
|
|
3208
|
-
}
|
|
3209
3518
|
(() => {
|
|
3210
3519
|
var table2 = new Uint8Array(128);
|
|
3211
3520
|
for (var i = 0; i < 64; i++) {
|
|
@@ -3228,19 +3537,6 @@ function* range({ stop, start = 0, step = 1 }) {
|
|
|
3228
3537
|
};
|
|
3229
3538
|
})();
|
|
3230
3539
|
new TextEncoder();
|
|
3231
|
-
function strip_prefix(path) {
|
|
3232
|
-
return path.slice(1);
|
|
3233
|
-
}
|
|
3234
|
-
function merge_init(storeOverrides, requestOverrides) {
|
|
3235
|
-
return {
|
|
3236
|
-
...storeOverrides,
|
|
3237
|
-
...requestOverrides,
|
|
3238
|
-
headers: {
|
|
3239
|
-
...storeOverrides.headers,
|
|
3240
|
-
...requestOverrides.headers
|
|
3241
|
-
}
|
|
3242
|
-
};
|
|
3243
|
-
}
|
|
3244
3540
|
let table = new Uint8Array(128);
|
|
3245
3541
|
for (let i = 0; i < 64; i++) {
|
|
3246
3542
|
table[i < 26 ? i + 65 : i < 52 ? i + 71 : i < 62 ? i - 4 : i * 4 - 205] = i;
|
|
@@ -3262,70 +3558,144 @@ function to_binary(base64) {
|
|
|
3262
3558
|
}
|
|
3263
3559
|
return bytes;
|
|
3264
3560
|
}
|
|
3265
|
-
|
|
3266
|
-
|
|
3267
|
-
|
|
3268
|
-
|
|
3269
|
-
constructor(refs, internal_store, overrides) {
|
|
3270
|
-
this.#refs = refs;
|
|
3271
|
-
this.#internal_store = internal_store;
|
|
3272
|
-
this.#overrides = overrides;
|
|
3273
|
-
}
|
|
3274
|
-
async get(key, opts = {}) {
|
|
3275
|
-
let ref = this.#refs.get(strip_prefix(key));
|
|
3276
|
-
if (!ref) return;
|
|
3277
|
-
if (typeof ref === "string") {
|
|
3278
|
-
if (ref.startsWith("base64:")) {
|
|
3279
|
-
return to_binary(ref.slice("base64:".length));
|
|
3280
|
-
}
|
|
3281
|
-
return new TextEncoder().encode(ref);
|
|
3282
|
-
}
|
|
3283
|
-
let [urlOrNull, offset, size] = ref;
|
|
3284
|
-
if (offset === void 0 || size === void 0 || !this.#internal_store.getRange) {
|
|
3285
|
-
return this.#internal_store.get(key, opts);
|
|
3286
|
-
}
|
|
3287
|
-
const opts_for_get = this.#overrides ? merge_init(this.#overrides, opts) : opts;
|
|
3288
|
-
return this.#internal_store.getRange("/", { offset, length: size }, opts_for_get);
|
|
3289
|
-
}
|
|
3290
|
-
// TODO: does it make sense to support getRange here? get already calls getRange on the internal store...
|
|
3291
|
-
static fromSpec(spec, internal_store, overrides) {
|
|
3292
|
-
let refs = parse(spec);
|
|
3293
|
-
return new ComposableReferenceStore(refs, internal_store, overrides);
|
|
3294
|
-
}
|
|
3561
|
+
function evictOnReject(cache, key, promise) {
|
|
3562
|
+
promise.catch(() => {
|
|
3563
|
+
if (cache.peek(key) === promise) cache.delete(key);
|
|
3564
|
+
});
|
|
3295
3565
|
}
|
|
3296
3566
|
class HdfStore {
|
|
3297
|
-
|
|
3298
|
-
|
|
3299
|
-
|
|
3300
|
-
|
|
3301
|
-
|
|
3567
|
+
#source;
|
|
3568
|
+
#converter;
|
|
3569
|
+
/** The root group; kept outside the LRU caches so it is never evicted. */
|
|
3570
|
+
#root;
|
|
3571
|
+
#nodes;
|
|
3572
|
+
#nodeRefs;
|
|
3573
|
+
#consolidated = null;
|
|
3574
|
+
constructor(source, converter, file, options) {
|
|
3575
|
+
this.#source = source;
|
|
3576
|
+
this.#converter = converter;
|
|
3577
|
+
this.#root = Promise.resolve(file);
|
|
3578
|
+
this.#nodes = new QuickLRU({ maxSize: options.maxCachedNodes ?? 1e3 });
|
|
3579
|
+
this.#nodeRefs = new QuickLRU({ maxSize: options.maxCachedRefs ?? 500 });
|
|
3580
|
+
}
|
|
3581
|
+
/**
|
|
3582
|
+
* @param {AsyncReadable} hdfSource Assumed to be a zarrita Store,
|
|
3583
|
+
* with the HDF5 file located at the root key "/". In other words,
|
|
3584
|
+
* hdfSource.getRange("/", { offset, length }) should return the
|
|
3585
|
+
* corresponding byte range from the HDF5 file.
|
|
3586
|
+
* @returns {Promise<HdfStore>} An HdfStore instance which exposes the HDF5
|
|
3587
|
+
* file as a Zarr v3 hierarchy.
|
|
3588
|
+
* @param {HdfStoreOptions} options Sizes of the in-memory LRU caches.
|
|
3589
|
+
*/
|
|
3590
|
+
static async fromStore(hdfSource, options = {}) {
|
|
3591
|
+
const converter = new SingleHdf5ToZarr(hdfSource, { url: null, zarrFormat: 2 });
|
|
3592
|
+
const file = await converter.open();
|
|
3593
|
+
return new HdfStore(hdfSource, converter, file, options);
|
|
3302
3594
|
}
|
|
3303
3595
|
async get(key, opts) {
|
|
3304
|
-
|
|
3596
|
+
const path = key.replace(/^\/+/, "");
|
|
3597
|
+
const parts = path === "" ? [] : path.split("/");
|
|
3598
|
+
if (parts[parts.length - 1] === "zarr.json") {
|
|
3599
|
+
const nodePath2 = parts.slice(0, -1).join("/");
|
|
3600
|
+
const refs2 = await this.#getNodeRefs(nodePath2, false);
|
|
3601
|
+
return this.#resolveRef(refs2?.[path], opts);
|
|
3602
|
+
}
|
|
3603
|
+
let nodePath = "";
|
|
3604
|
+
for (const part of parts) {
|
|
3605
|
+
const node = await this.#resolveNode(nodePath);
|
|
3606
|
+
if (!node) return void 0;
|
|
3607
|
+
if (node instanceof Dataset) break;
|
|
3608
|
+
nodePath = nodePath ? `${nodePath}/${part}` : part;
|
|
3609
|
+
}
|
|
3610
|
+
const chunkKey = path.slice(nodePath.length + 1);
|
|
3611
|
+
if (nodePath === path || !(chunkKey === "c" || chunkKey.startsWith("c/")) || !(await this.#resolveNode(nodePath) instanceof Dataset)) {
|
|
3612
|
+
return void 0;
|
|
3613
|
+
}
|
|
3614
|
+
const refs = await this.#getNodeRefs(nodePath, true);
|
|
3615
|
+
return this.#resolveRef(refs?.[path], opts);
|
|
3305
3616
|
}
|
|
3306
|
-
// TODO: does it make sense to support getRange here?
|
|
3307
3617
|
/**
|
|
3308
3618
|
* @returns The root zarr.json document with inline consolidated metadata
|
|
3309
3619
|
* (zarr-python format) describing every group and array in the hierarchy.
|
|
3620
|
+
* Reads the metadata of every node in the file (but no chunk data); nodes
|
|
3621
|
+
* already opened through this store are not read again.
|
|
3310
3622
|
*/
|
|
3311
3623
|
getConsolidatedMetadata() {
|
|
3312
|
-
|
|
3624
|
+
if (!this.#consolidated) {
|
|
3625
|
+
this.#consolidated = this.#collectMetadata("").then(
|
|
3626
|
+
(parts) => refSpecToConsolidatedMetadataV3({ refs: Object.assign({}, ...parts) })
|
|
3627
|
+
);
|
|
3628
|
+
this.#consolidated.catch(() => {
|
|
3629
|
+
this.#consolidated = null;
|
|
3630
|
+
});
|
|
3631
|
+
}
|
|
3632
|
+
return this.#consolidated;
|
|
3313
3633
|
}
|
|
3314
3634
|
/**
|
|
3315
|
-
*
|
|
3316
|
-
*
|
|
3317
|
-
|
|
3318
|
-
|
|
3319
|
-
|
|
3320
|
-
|
|
3321
|
-
|
|
3322
|
-
|
|
3635
|
+
* Metadata refs for a node and all of its descendants, in depth-first order.
|
|
3636
|
+
* Sibling subtrees are visited concurrently.
|
|
3637
|
+
*/
|
|
3638
|
+
async #collectMetadata(path) {
|
|
3639
|
+
const [node, own] = await Promise.all([
|
|
3640
|
+
this.#resolveNode(path),
|
|
3641
|
+
this.#getNodeRefs(path, false)
|
|
3642
|
+
]);
|
|
3643
|
+
if (!(node instanceof Group)) return [own];
|
|
3644
|
+
const children = await Promise.all(
|
|
3645
|
+
node.keys.map((key) => this.#collectMetadata(path ? `${path}/${key}` : key))
|
|
3646
|
+
);
|
|
3647
|
+
return [own, ...children.flat()];
|
|
3648
|
+
}
|
|
3649
|
+
/**
|
|
3650
|
+
* Resolve the HDF5 group or dataset at a hierarchy path (cached).
|
|
3651
|
+
* Returns null if there is no such node.
|
|
3323
3652
|
*/
|
|
3324
|
-
|
|
3325
|
-
|
|
3326
|
-
|
|
3327
|
-
|
|
3328
|
-
|
|
3653
|
+
#resolveNode(path) {
|
|
3654
|
+
if (path === "") return this.#root;
|
|
3655
|
+
let node = this.#nodes.get(path);
|
|
3656
|
+
if (!node) {
|
|
3657
|
+
const idx = path.lastIndexOf("/");
|
|
3658
|
+
const parentPath = idx === -1 ? "" : path.slice(0, idx);
|
|
3659
|
+
const name = path.slice(idx + 1);
|
|
3660
|
+
node = this.#resolveNode(parentPath).then(async (parent) => {
|
|
3661
|
+
if (!(parent instanceof Group) || !(name in parent._links)) return null;
|
|
3662
|
+
try {
|
|
3663
|
+
return await parent.get(name);
|
|
3664
|
+
} catch {
|
|
3665
|
+
return null;
|
|
3666
|
+
}
|
|
3667
|
+
});
|
|
3668
|
+
evictOnReject(this.#nodes, path, node);
|
|
3669
|
+
this.#nodes.set(path, node);
|
|
3670
|
+
}
|
|
3671
|
+
return node;
|
|
3672
|
+
}
|
|
3673
|
+
/**
|
|
3674
|
+
* Zarr v3 references for a single node (cached). Metadata-only refs contain
|
|
3675
|
+
* just the node's zarr.json; with `chunks`, its chunk references too.
|
|
3676
|
+
*/
|
|
3677
|
+
#getNodeRefs(path, chunks) {
|
|
3678
|
+
const cacheKey = `${chunks ? "c" : "m"}:${path}`;
|
|
3679
|
+
let refs = this.#nodeRefs.get(cacheKey);
|
|
3680
|
+
if (!refs) {
|
|
3681
|
+
refs = (async () => {
|
|
3682
|
+
const node = await this.#resolveNode(path);
|
|
3683
|
+
if (!node) return {};
|
|
3684
|
+
const v2Refs = await this.#converter.translateNode(path, node, { chunks });
|
|
3685
|
+
return refSpecV2ToV3({ refs: v2Refs }).refs;
|
|
3686
|
+
})();
|
|
3687
|
+
evictOnReject(this.#nodeRefs, cacheKey, refs);
|
|
3688
|
+
this.#nodeRefs.set(cacheKey, refs);
|
|
3689
|
+
}
|
|
3690
|
+
return refs;
|
|
3691
|
+
}
|
|
3692
|
+
async #resolveRef(ref, opts) {
|
|
3693
|
+
if (ref === void 0) return void 0;
|
|
3694
|
+
if (typeof ref === "string") {
|
|
3695
|
+
return ref.startsWith("base64:") ? to_binary(ref.slice("base64:".length)) : new TextEncoder().encode(ref);
|
|
3696
|
+
}
|
|
3697
|
+
const [, offset, length] = ref;
|
|
3698
|
+
return this.#source.getRange("/", { offset, length }, opts);
|
|
3329
3699
|
}
|
|
3330
3700
|
}
|
|
3331
3701
|
export {
|