hdf5-as-virtual-zarr 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -5,6 +5,18 @@ async function fetchRange(source, offset, length) {
5
5
  copy.set(result);
6
6
  return copy.buffer;
7
7
  }
8
+ const fileSizes = /* @__PURE__ */ new WeakMap();
9
+ function setFileSize(source, size) {
10
+ fileSizes.set(source, size);
11
+ }
12
+ function fetchRangeWithReadAhead(source, offset, length, readAhead) {
13
+ const size = fileSizes.get(source);
14
+ let want = length;
15
+ if (size !== void 0 && readAhead > length) {
16
+ want = Math.max(length, Math.min(readAhead, size - offset));
17
+ }
18
+ return fetchRange(source, offset, want);
19
+ }
8
20
  function _unpack_struct_from(structure, buf, offset = 0) {
9
21
  const output = /* @__PURE__ */ new Map();
10
22
  for (const [key, fmt] of structure.entries()) {
@@ -351,13 +363,22 @@ const B_LINK_NODE_V1 = /* @__PURE__ */ new Map([
351
363
  ["right_sibling", "Q"]
352
364
  ]);
353
365
  const B_LINK_NODE_V1_SIZE = _structure_size(B_LINK_NODE_V1);
354
- async function _readV1NodeHeader(source, offset, node_level) {
355
- const hBuf = await fetchRange(source, offset, B_LINK_NODE_V1_SIZE);
356
- const node = _unpack_struct_from(B_LINK_NODE_V1, hBuf, 0);
366
+ const V1_NODE_READ_AHEAD_ENTRIES = 64;
367
+ async function _readV1Node(source, offset, node_level, entrySize, trailingSize) {
368
+ const buf = await fetchRangeWithReadAhead(
369
+ source,
370
+ offset,
371
+ B_LINK_NODE_V1_SIZE,
372
+ B_LINK_NODE_V1_SIZE + V1_NODE_READ_AHEAD_ENTRIES * entrySize + trailingSize
373
+ );
374
+ const node = _unpack_struct_from(B_LINK_NODE_V1, buf, 0);
357
375
  if (node_level != null && node.get("node_level") !== node_level) {
358
376
  throw new Error("node level does not match");
359
377
  }
360
- return [node, hBuf];
378
+ const dataSize = node.get("entries_used") * entrySize + trailingSize;
379
+ const dataEnd = B_LINK_NODE_V1_SIZE + dataSize;
380
+ const data = dataEnd <= buf.byteLength ? buf.slice(B_LINK_NODE_V1_SIZE, dataEnd) : await fetchRange(source, offset + B_LINK_NODE_V1_SIZE, dataSize);
381
+ return [node, data];
361
382
  }
362
383
  class BTreeV1Groups {
363
384
  all_nodes;
@@ -369,11 +390,8 @@ class BTreeV1Groups {
369
390
  static async create(source, offset) {
370
391
  const all_nodes = /* @__PURE__ */ new Map();
371
392
  async function readNode(off, node_level2) {
372
- const [header] = await _readV1NodeHeader(source, off, node_level2);
373
- let readOff = off + B_LINK_NODE_V1_SIZE;
393
+ const [header, dataBuf] = await _readV1Node(source, off, node_level2, 16, 8);
374
394
  const entries_used = header.get("entries_used");
375
- const dataSize = entries_used * 16 + 8;
376
- const dataBuf = await fetchRange(source, readOff, dataSize);
377
395
  let localOff = 0;
378
396
  const keys = [];
379
397
  const addresses = [];
@@ -392,12 +410,11 @@ class BTreeV1Groups {
392
410
  let node_level = depth;
393
411
  while (node_level > 0) {
394
412
  const parents = all_nodes.get(node_level) || [];
395
- for (const parent of parents) {
396
- for (const child_addr of parent.addresses) {
397
- const child = await readNode(child_addr, node_level - 1);
398
- addNode(all_nodes, child, node_level - 1);
399
- }
400
- }
413
+ const childAddrs = parents.flatMap((parent) => parent.addresses);
414
+ const children = await Promise.all(
415
+ childAddrs.map((child_addr) => readNode(child_addr, node_level - 1))
416
+ );
417
+ for (const child of children) addNode(all_nodes, child, node_level - 1);
401
418
  node_level--;
402
419
  }
403
420
  return new BTreeV1Groups(all_nodes, depth);
@@ -423,13 +440,11 @@ class BTreeV1RawDataChunks {
423
440
  static async create(source, offset, dims) {
424
441
  const all_nodes = /* @__PURE__ */ new Map();
425
442
  async function readNode(off, node_level2) {
426
- const [header] = await _readV1NodeHeader(source, off, node_level2);
427
- let readOff = off + B_LINK_NODE_V1_SIZE;
428
- const entries_used = header.get("entries_used");
429
443
  const fmt = "<" + dims.toFixed() + "Q";
430
444
  const fmt_size = struct.calcsize(fmt);
431
445
  const entrySize = 8 + fmt_size + 8;
432
- const dataBuf = await fetchRange(source, readOff, entries_used * entrySize);
446
+ const [header, dataBuf] = await _readV1Node(source, off, node_level2, entrySize, 0);
447
+ const entries_used = header.get("entries_used");
433
448
  let localOff = 0;
434
449
  const keys = [];
435
450
  const addresses = [];
@@ -451,12 +466,11 @@ class BTreeV1RawDataChunks {
451
466
  let node_level = depth;
452
467
  while (node_level > 0) {
453
468
  const parents = all_nodes.get(node_level) || [];
454
- for (const parent of parents) {
455
- for (const child_addr of parent.addresses) {
456
- const child = await readNode(child_addr, node_level - 1);
457
- addNode(all_nodes, child, node_level - 1);
458
- }
459
- }
469
+ const childAddrs = parents.flatMap((parent) => parent.addresses);
470
+ const children = await Promise.all(
471
+ childAddrs.map((child_addr) => readNode(child_addr, node_level - 1))
472
+ );
473
+ for (const child of children) addNode(all_nodes, child, node_level - 1);
460
474
  node_level--;
461
475
  }
462
476
  return new BTreeV1RawDataChunks(all_nodes, depth, dims);
@@ -591,12 +605,11 @@ async function _readBTreeV2(source, offset, parseRecord) {
591
605
  let node_level = depth;
592
606
  while (node_level > 0) {
593
607
  const parents = all_nodes.get(node_level) || [];
594
- for (const parent of parents) {
595
- for (const child_addr of parent.addresses) {
596
- const child = await readNode(child_addr, node_level - 1);
597
- addNode(all_nodes, child, node_level - 1);
598
- }
599
- }
608
+ const childAddrs = parents.flatMap((parent) => parent.addresses);
609
+ const children = await Promise.all(
610
+ childAddrs.map((child_addr) => readNode(child_addr, node_level - 1))
611
+ );
612
+ for (const child of children) addNode(all_nodes, child, node_level - 1);
600
613
  node_level--;
601
614
  }
602
615
  return { all_nodes, header };
@@ -663,6 +676,306 @@ function addNode(all_nodes, node, node_level) {
663
676
  all_nodes.set(level, [node]);
664
677
  }
665
678
  }
679
+ const FILTER_DEFLATE = 1;
680
+ const FILTER_SHUFFLE = 2;
681
+ const FILTER_FLETCHER32 = 3;
682
+ async function inflate(data) {
683
+ const stream = new Blob([data]).stream().pipeThrough(new DecompressionStream("deflate"));
684
+ return new Uint8Array(await new Response(stream).arrayBuffer());
685
+ }
686
+ function unshuffle(data, elementSize) {
687
+ if (elementSize <= 1) return data;
688
+ const count = Math.floor(data.length / elementSize);
689
+ const out = new Uint8Array(data.length);
690
+ for (let b = 0; b < elementSize; b++) {
691
+ const src = b * count;
692
+ for (let i = 0; i < count; i++) {
693
+ out[i * elementSize + b] = data[src + i];
694
+ }
695
+ }
696
+ out.set(data.subarray(count * elementSize), count * elementSize);
697
+ return out;
698
+ }
699
+ async function decodeChunk(data, pipeline, filterMask = 0) {
700
+ if (!pipeline) return data;
701
+ let out = data;
702
+ for (let i = pipeline.length - 1; i >= 0; i--) {
703
+ if (filterMask & 1 << i) continue;
704
+ const filter = pipeline[i];
705
+ const id = filter.get("filter_id");
706
+ const clientData = filter.get("client_data") ?? [];
707
+ if (id === FILTER_DEFLATE) {
708
+ out = await inflate(out);
709
+ } else if (id === FILTER_SHUFFLE) {
710
+ out = unshuffle(out, clientData[0] ?? 1);
711
+ } else if (id === FILTER_FLETCHER32) {
712
+ out = out.subarray(0, out.length - 4);
713
+ } else {
714
+ throw new Error(`Unsupported HDF5 filter id ${id} for chunk decoding`);
715
+ }
716
+ }
717
+ return out;
718
+ }
719
+ const isValidMaxSize = (value) => typeof value === "number" && value > 0 && (Number.isInteger(value) || value === Infinity);
720
+ const hasExpiry = (item) => item.expiry !== void 0 && !Number.isNaN(item.expiry);
721
+ const MODE_KEYS = 0;
722
+ const MODE_VALUES = 1;
723
+ const MODE_ENTRIES = 2;
724
+ const MODE_ITEMS = 3;
725
+ const INVALID_MAX_SIZE_MESSAGE = "`maxSize` must be a whole number greater than 0 or `Infinity`";
726
+ class QuickLRU extends Map {
727
+ #cache = /* @__PURE__ */ new Map();
728
+ #oldCache = /* @__PURE__ */ new Map();
729
+ #maxSize;
730
+ #maxAge;
731
+ #onEviction;
732
+ constructor(options = {}) {
733
+ super();
734
+ if (!isValidMaxSize(options.maxSize)) {
735
+ throw new TypeError(INVALID_MAX_SIZE_MESSAGE);
736
+ }
737
+ if (typeof options.maxAge === "number" && !(options.maxAge > 0)) {
738
+ throw new TypeError("`maxAge` must be a number greater than 0");
739
+ }
740
+ this.#maxSize = options.maxSize;
741
+ this.#maxAge = typeof options.maxAge === "number" ? options.maxAge : Infinity;
742
+ this.#onEviction = options.onEviction;
743
+ }
744
+ #emitEvictions(cache) {
745
+ if (typeof this.#onEviction !== "function") {
746
+ return;
747
+ }
748
+ const evictions = [...cache].filter(([key]) => !this.#cache.has(key) && !this.#oldCache.has(key));
749
+ for (const [key, item] of evictions) {
750
+ this.#onEviction(key, item.value);
751
+ }
752
+ }
753
+ #deleteIfExpired(key, item) {
754
+ if (hasExpiry(item) && item.expiry <= Date.now()) {
755
+ const deleted = this.delete(key);
756
+ if (deleted && typeof this.#onEviction === "function") {
757
+ this.#onEviction(key, item.value);
758
+ }
759
+ return deleted;
760
+ }
761
+ return false;
762
+ }
763
+ // Returns the value, or `undefined` once the item has been removed as expired.
764
+ #getItemValue(key, item) {
765
+ return this.#deleteIfExpired(key, item) === false ? item.value : void 0;
766
+ }
767
+ #set(key, item) {
768
+ this.#cache.set(key, item);
769
+ if (this.#cache.size < this.#maxSize) {
770
+ return;
771
+ }
772
+ const oldCache = this.#oldCache;
773
+ this.#oldCache = this.#cache;
774
+ this.#cache = /* @__PURE__ */ new Map();
775
+ this.#emitEvictions(oldCache);
776
+ }
777
+ #moveToRecent(key, item) {
778
+ this.#oldCache.delete(key);
779
+ this.#set(key, item);
780
+ }
781
+ // The current item for `key`, or `undefined` if it is gone or has expired. The recent cache wins, since an old cache copy of a key that is in both is stale. Not a generator, since this runs once per item during iteration.
782
+ #liveItem(key) {
783
+ const item = this.#cache.get(key) ?? this.#oldCache.get(key);
784
+ if (item !== void 0 && this.#deleteIfExpired(key, item) === false) {
785
+ return item;
786
+ }
787
+ }
788
+ // The current item for `key`, promoted to the recent cache when it was in the old one, or `undefined` when the key is missing or has expired. Only for the methods that refresh recency, since promoting is what separates them from `peek()`. One lookup per cache, since where the item is found decides the promotion. Not a generator, since this runs once per lookup.
789
+ #getLiveItem(key) {
790
+ let item = this.#cache.get(key);
791
+ if (item === void 0) {
792
+ item = this.#oldCache.get(key);
793
+ if (item === void 0 || this.#deleteIfExpired(key, item)) {
794
+ return;
795
+ }
796
+ this.#moveToRecent(key, item);
797
+ } else if (this.#deleteIfExpired(key, item)) {
798
+ return;
799
+ }
800
+ return item;
801
+ }
802
+ // The old cache keys that are not also in the recent cache, since an old copy of such a key is stale. A plain loop, since this runs once per item during iteration.
803
+ #olderKeys() {
804
+ const keys = [];
805
+ for (const key of this.#oldCache.keys()) {
806
+ if (!this.#cache.has(key)) {
807
+ keys.push(key);
808
+ }
809
+ }
810
+ return keys;
811
+ }
812
+ *#walk(mode, descending) {
813
+ const keys = [...this.#olderKeys(), ...this.#cache.keys()];
814
+ if (descending) {
815
+ keys.reverse();
816
+ }
817
+ for (const key of keys) {
818
+ const item = this.#liveItem(key);
819
+ if (item === void 0) {
820
+ continue;
821
+ }
822
+ if (mode === MODE_KEYS) {
823
+ yield key;
824
+ } else if (mode === MODE_VALUES) {
825
+ yield item.value;
826
+ } else if (mode === MODE_ITEMS) {
827
+ yield [key, item];
828
+ } else {
829
+ yield [key, item.value];
830
+ }
831
+ }
832
+ }
833
+ // For tests. Public on purpose, so the test suite can see where an item lives.
834
+ // eslint-disable-next-line unicorn/prefer-private-class-fields
835
+ get __oldCache() {
836
+ return this.#oldCache;
837
+ }
838
+ get(key) {
839
+ const item = this.#getLiveItem(key);
840
+ return item === void 0 ? void 0 : item.value;
841
+ }
842
+ getOrInsert(key, value) {
843
+ const item = this.#getLiveItem(key);
844
+ if (item === void 0) {
845
+ this.set(key, value);
846
+ return value;
847
+ }
848
+ return item.value;
849
+ }
850
+ getOrInsertComputed(key, callback) {
851
+ const item = this.#getLiveItem(key);
852
+ if (item === void 0) {
853
+ const value = callback(key);
854
+ this.set(key, value);
855
+ return value;
856
+ }
857
+ return item.value;
858
+ }
859
+ set(key, value, { maxAge = this.#maxAge } = {}) {
860
+ const expiry = typeof maxAge === "number" && maxAge !== Infinity ? Date.now() + maxAge : void 0;
861
+ this.#set(key, { value, expiry });
862
+ return this;
863
+ }
864
+ has(key) {
865
+ const item = this.#cache.get(key) ?? this.#oldCache.get(key);
866
+ return item !== void 0 && !this.#deleteIfExpired(key, item);
867
+ }
868
+ peek(key) {
869
+ const item = this.#cache.get(key) ?? this.#oldCache.get(key);
870
+ return item === void 0 ? void 0 : this.#getItemValue(key, item);
871
+ }
872
+ expiresIn(key) {
873
+ const item = this.#cache.get(key) ?? this.#oldCache.get(key);
874
+ if (item) {
875
+ return hasExpiry(item) ? item.expiry - Date.now() : Infinity;
876
+ }
877
+ }
878
+ delete(key) {
879
+ const deleted = this.#cache.delete(key);
880
+ return this.#oldCache.delete(key) || deleted;
881
+ }
882
+ clear() {
883
+ this.#cache.clear();
884
+ this.#oldCache.clear();
885
+ }
886
+ resize(newSize) {
887
+ if (!isValidMaxSize(newSize)) {
888
+ throw new TypeError(INVALID_MAX_SIZE_MESSAGE);
889
+ }
890
+ const items = [...this.#walk(MODE_ITEMS)];
891
+ const removeCount = items.length - newSize;
892
+ if (removeCount < 0) {
893
+ this.#cache = new Map(items);
894
+ this.#oldCache = /* @__PURE__ */ new Map();
895
+ } else {
896
+ this.#oldCache = new Map(items.slice(removeCount));
897
+ this.#cache = /* @__PURE__ */ new Map();
898
+ }
899
+ this.#maxSize = newSize;
900
+ if (removeCount > 0) {
901
+ this.#emitEvictions(items.slice(0, removeCount));
902
+ }
903
+ }
904
+ evict(count = 1) {
905
+ const requested = Number(count);
906
+ if (!requested || requested <= 0) {
907
+ return;
908
+ }
909
+ const live = [...this.#walk(MODE_ITEMS)];
910
+ const evictCount = Math.min(Math.trunc(requested), Math.max(live.length - 1, 0));
911
+ const evicted = [];
912
+ for (const [key] of live.slice(0, evictCount)) {
913
+ const item = this.#cache.get(key) ?? this.#oldCache.get(key);
914
+ if (item !== void 0 && this.delete(key)) {
915
+ evicted.push([key, item]);
916
+ }
917
+ }
918
+ this.#emitEvictions(evicted);
919
+ }
920
+ keys() {
921
+ return this.#walk(MODE_KEYS);
922
+ }
923
+ values() {
924
+ return this.#walk(MODE_VALUES);
925
+ }
926
+ [Symbol.iterator]() {
927
+ return this.entriesAscending();
928
+ }
929
+ entriesDescending() {
930
+ return this.#walk(MODE_ENTRIES, true);
931
+ }
932
+ entriesAscending() {
933
+ return this.#walk(MODE_ENTRIES);
934
+ }
935
+ get size() {
936
+ const recentSize = this.#cache.size;
937
+ const olderSize = this.#oldCache.size;
938
+ if (recentSize === 0 || olderSize === 0) {
939
+ return Math.min(recentSize + olderSize, this.#maxSize);
940
+ }
941
+ let walked = this.#cache;
942
+ let checked = this.#oldCache;
943
+ if (olderSize < recentSize) {
944
+ walked = this.#oldCache;
945
+ checked = this.#cache;
946
+ }
947
+ let duplicates = 0;
948
+ for (const key of walked.keys()) {
949
+ if (checked.has(key)) {
950
+ duplicates++;
951
+ }
952
+ }
953
+ return Math.min(recentSize + olderSize - duplicates, this.#maxSize);
954
+ }
955
+ get maxSize() {
956
+ return this.#maxSize;
957
+ }
958
+ get maxAge() {
959
+ return this.#maxAge;
960
+ }
961
+ entries() {
962
+ return this.entriesAscending();
963
+ }
964
+ forEach(callbackFunction, thisArgument = this) {
965
+ for (const [key, value] of this.entriesAscending()) {
966
+ callbackFunction.call(thisArgument, value, key, this);
967
+ }
968
+ }
969
+ get [Symbol.toStringTag]() {
970
+ return "QuickLRU";
971
+ }
972
+ toString() {
973
+ return `QuickLRU(${this.size}/${this.maxSize})`;
974
+ }
975
+ [/* @__PURE__ */ Symbol.for("nodejs.util.inspect.custom")]() {
976
+ return this.toString();
977
+ }
978
+ }
666
979
  const FORMAT_SIGNATURE = struct.unpack_from(
667
980
  "8s",
668
981
  new Uint8Array([137, 72, 68, 70, 13, 10, 26, 10]).buffer
@@ -767,6 +1080,8 @@ const FRACTAL_HEAP_HEADER = /* @__PURE__ */ new Map([
767
1080
  ["root_block_address", "Q"],
768
1081
  ["indirect_current_rows_count", "H"]
769
1082
  ]);
1083
+ const LOCAL_HEAP_READ_AHEAD = 1024;
1084
+ const GLOBAL_HEAP_READ_AHEAD = 4096;
770
1085
  class SuperBlock {
771
1086
  version;
772
1087
  _contents;
@@ -803,6 +1118,11 @@ class SuperBlock {
803
1118
  const version = contents.get("superblock_version");
804
1119
  return new SuperBlock(source, contents, version, end_of_sblock);
805
1120
  }
1121
+ /** Absolute end-of-file address (i.e. the file size), as recorded in the superblock. */
1122
+ get end_of_file() {
1123
+ const base = this.version === 0 ? this._contents.get("base_address_lower") : this._contents.get("base_address");
1124
+ return base + this._contents.get("end_of_file_address");
1125
+ }
806
1126
  async getOffsetToDataobjects() {
807
1127
  if (this.version === 0) {
808
1128
  const sym_table = await SymbolTable.create(
@@ -828,13 +1148,14 @@ class Heap {
828
1148
  }
829
1149
  static async create(source, offset) {
830
1150
  const headerSize = _structure_size(LOCAL_HEAP);
831
- const hBuf = await fetchRange(source, offset, headerSize);
1151
+ const hBuf = await fetchRangeWithReadAhead(source, offset, headerSize, LOCAL_HEAP_READ_AHEAD);
832
1152
  const local_heap = _unpack_struct_from(LOCAL_HEAP, hBuf, 0);
833
1153
  assert(local_heap.get("signature") === "HEAP");
834
1154
  assert(local_heap.get("version") === 0);
835
1155
  const data_offset = local_heap.get("address_of_data_segment");
836
1156
  const seg_size = local_heap.get("data_segment_size");
837
- const heap_data = await fetchRange(source, data_offset, seg_size);
1157
+ const rel = data_offset - offset;
1158
+ const heap_data = rel >= 0 && rel + seg_size <= hBuf.byteLength ? hBuf.slice(rel, rel + seg_size) : await fetchRange(source, data_offset, seg_size);
838
1159
  local_heap.set("heap_data", heap_data);
839
1160
  return new Heap(local_heap, heap_data);
840
1161
  }
@@ -856,17 +1177,23 @@ class SymbolTable {
856
1177
  static async create(source, offset, root = false) {
857
1178
  let node;
858
1179
  let readOffset = offset;
1180
+ let nodeBuf = null;
859
1181
  if (root) {
860
1182
  node = /* @__PURE__ */ new Map([["symbols", 1]]);
861
1183
  } else {
862
- const nodeBuf = await fetchRange(source, offset, SYMBOL_TABLE_NODE_SIZE);
1184
+ nodeBuf = await fetchRangeWithReadAhead(
1185
+ source,
1186
+ offset,
1187
+ SYMBOL_TABLE_NODE_SIZE,
1188
+ SYMBOL_TABLE_NODE_SIZE + 32 * SYMBOL_TABLE_ENTRY_SIZE
1189
+ );
863
1190
  node = _unpack_struct_from(SYMBOL_TABLE_NODE, nodeBuf, 0);
864
1191
  if (node.get("signature") !== "SNOD") throw new Error("incorrect node type");
865
1192
  readOffset = offset + SYMBOL_TABLE_NODE_SIZE;
866
1193
  }
867
1194
  const n_symbols = node.get("symbols");
868
1195
  const entryBytes = n_symbols * SYMBOL_TABLE_ENTRY_SIZE;
869
- const entryBuf = await fetchRange(source, readOffset, entryBytes);
1196
+ const entryBuf = nodeBuf && SYMBOL_TABLE_NODE_SIZE + entryBytes <= nodeBuf.byteLength ? nodeBuf.slice(SYMBOL_TABLE_NODE_SIZE, SYMBOL_TABLE_NODE_SIZE + entryBytes) : await fetchRange(source, readOffset, entryBytes);
870
1197
  const entries = [];
871
1198
  let localOff = 0;
872
1199
  for (let i = 0; i < n_symbols; i++) {
@@ -915,14 +1242,15 @@ class GlobalHeap {
915
1242
  this.heap_data = heap_data;
916
1243
  }
917
1244
  static async create(source, offset) {
918
- const hBuf = await fetchRange(source, offset, GLOBAL_HEAP_HEADER_SIZE);
919
- const header = _unpack_struct_from(GLOBAL_HEAP_HEADER, hBuf, 0);
920
- const heap_data_size = header.get("collection_size") - GLOBAL_HEAP_HEADER_SIZE;
921
- const heap_data = await fetchRange(
1245
+ const hBuf = await fetchRangeWithReadAhead(
922
1246
  source,
923
- offset + GLOBAL_HEAP_HEADER_SIZE,
924
- heap_data_size
1247
+ offset,
1248
+ GLOBAL_HEAP_HEADER_SIZE,
1249
+ GLOBAL_HEAP_READ_AHEAD
925
1250
  );
1251
+ const header = _unpack_struct_from(GLOBAL_HEAP_HEADER, hBuf, 0);
1252
+ const heap_data_size = header.get("collection_size") - GLOBAL_HEAP_HEADER_SIZE;
1253
+ const heap_data = GLOBAL_HEAP_HEADER_SIZE + heap_data_size <= hBuf.byteLength ? hBuf.slice(GLOBAL_HEAP_HEADER_SIZE, GLOBAL_HEAP_HEADER_SIZE + heap_data_size) : await fetchRange(source, offset + GLOBAL_HEAP_HEADER_SIZE, heap_data_size);
926
1254
  return new GlobalHeap(header, heap_data);
927
1255
  }
928
1256
  get objects() {
@@ -1033,9 +1361,7 @@ class FractalHeap {
1033
1361
  table_width
1034
1362
  };
1035
1363
  if (nrows > 0) {
1036
- for await (const data of FractalHeap._iter_indirect_block(ctx, root_address, nrows)) {
1037
- managedParts.push(data);
1038
- }
1364
+ managedParts.push(...await FractalHeap._read_indirect_block(ctx, root_address, nrows));
1039
1365
  } else if (root_address != null) {
1040
1366
  const data = await FractalHeap._read_direct_block(ctx, root_address, start_block_size);
1041
1367
  managedParts.push(data);
@@ -1093,7 +1419,11 @@ class FractalHeap {
1093
1419
  assert(header.get("signature") === "FHDB");
1094
1420
  return data;
1095
1421
  }
1096
- static async *_iter_indirect_block(ctx, offset, nrows) {
1422
+ /**
1423
+ * Read all direct blocks reachable from an indirect block (in heap order).
1424
+ * Blocks are fetched concurrently.
1425
+ */
1426
+ static async _read_indirect_block(ctx, offset, nrows) {
1097
1427
  const ibHdrBuf = await fetchRange(ctx.source, offset, ctx.indirect_block_header_size);
1098
1428
  const header = _unpack_struct_from(ctx.indirect_block_header, ibHdrBuf, 0);
1099
1429
  let readOff = offset + ctx.indirect_block_header_size;
@@ -1125,12 +1455,19 @@ class FractalHeap {
1125
1455
  const nr = FractalHeap._iblock_nrows_from_block_size(ctx, block_size);
1126
1456
  indirect_blocks.push([address, nr]);
1127
1457
  }
1128
- for (const [address, block_size] of direct_blocks) {
1129
- yield await FractalHeap._read_direct_block(ctx, address, block_size);
1130
- }
1131
- for (const [address, nr] of indirect_blocks) {
1132
- yield* FractalHeap._iter_indirect_block(ctx, address, nr);
1133
- }
1458
+ const [direct, indirect] = await Promise.all([
1459
+ Promise.all(
1460
+ direct_blocks.map(
1461
+ ([address, block_size]) => FractalHeap._read_direct_block(ctx, address, block_size)
1462
+ )
1463
+ ),
1464
+ Promise.all(
1465
+ indirect_blocks.map(
1466
+ ([address, nr]) => FractalHeap._read_indirect_block(ctx, address, nr)
1467
+ )
1468
+ )
1469
+ ]);
1470
+ return [...direct, ...indirect.flat()];
1134
1471
  }
1135
1472
  static _calc_block_size(ctx, iblock) {
1136
1473
  const row = Math.floor(iblock / ctx.table_width);
@@ -1264,6 +1601,15 @@ const LINK_INFO_MSG2 = /* @__PURE__ */ new Map([
1264
1601
  ["name_btree_address", "Q"],
1265
1602
  ["order_btree_address", "Q"]
1266
1603
  ]);
1604
+ const OBJECT_HEADER_READ_AHEAD = 1024;
1605
+ const GLOBAL_HEAP_CACHE_SIZE = 256;
1606
+ async function sliceOrFetch(source, prefix, prefixOffset, absOffset, size) {
1607
+ const start = absOffset - prefixOffset;
1608
+ if (start >= 0 && start + size <= prefix.byteLength) {
1609
+ return prefix.slice(start, start + size);
1610
+ }
1611
+ return fetchRange(source, absOffset, size);
1612
+ }
1267
1613
  class DataObjects {
1268
1614
  buf;
1269
1615
  bufStart;
@@ -1285,7 +1631,7 @@ class DataObjects {
1285
1631
  this.msgs = msgs;
1286
1632
  this.msg_data = msg_data;
1287
1633
  this.offset = offset;
1288
- this._global_heaps = /* @__PURE__ */ new Map();
1634
+ this._global_heaps = new QuickLRU({ maxSize: GLOBAL_HEAP_CACHE_SIZE });
1289
1635
  this._header = header;
1290
1636
  this._filter_pipeline = null;
1291
1637
  this._chunk_params_set = false;
@@ -1294,13 +1640,13 @@ class DataObjects {
1294
1640
  this._chunk_address = null;
1295
1641
  }
1296
1642
  static async create(source, offset) {
1297
- const peekBuf = await fetchRange(source, offset, 1);
1298
- const versionHint = struct.unpack_from("<B", peekBuf, 0)[0];
1643
+ const prefix = await fetchRangeWithReadAhead(source, offset, 1, OBJECT_HEADER_READ_AHEAD);
1644
+ const versionHint = struct.unpack_from("<B", prefix, 0)[0];
1299
1645
  let result;
1300
1646
  if (versionHint === 1) {
1301
- result = await DataObjects._parseV1(source, offset);
1647
+ result = await DataObjects._parseV1(source, offset, prefix);
1302
1648
  } else if (versionHint === "O".charCodeAt(0)) {
1303
- result = await DataObjects._parseV2(source, offset);
1649
+ result = await DataObjects._parseV2(source, offset, prefix);
1304
1650
  } else {
1305
1651
  throw new Error(`Unknown Data Object Header version hint: ${versionHint}`);
1306
1652
  }
@@ -1315,8 +1661,8 @@ class DataObjects {
1315
1661
  );
1316
1662
  }
1317
1663
  // ────────── V1 parsing ──────────
1318
- static async _parseV1(source, offset) {
1319
- const headerBuf = await fetchRange(source, offset, OBJECT_HEADER_V1_SIZE);
1664
+ static async _parseV1(source, offset, prefix) {
1665
+ const headerBuf = prefix.byteLength >= OBJECT_HEADER_V1_SIZE ? prefix : await fetchRange(source, offset, OBJECT_HEADER_V1_SIZE);
1320
1666
  const header = _unpack_struct_from(OBJECT_HEADER_V1, headerBuf, 0);
1321
1667
  assert(header.get("version") === 1);
1322
1668
  const total_header_messages = header.get("total_header_messages");
@@ -1327,21 +1673,29 @@ class DataObjects {
1327
1673
  { absOffset: first_block_offset, size: first_block_size }
1328
1674
  ];
1329
1675
  while (blockQueue.length > 0) {
1330
- const { absOffset, size } = blockQueue.shift();
1331
- const data = await fetchRange(source, absOffset, size);
1332
- allBlocks.push({ absOffset, size, data });
1333
- let local_offset2 = 0;
1334
- while (local_offset2 + HEADER_MSG_INFO_V1_SIZE <= size) {
1335
- const msg = _unpack_struct_from(HEADER_MSG_INFO_V1, data, local_offset2);
1336
- const msg_type = msg.get("type");
1337
- const msg_size = msg.get("size");
1338
- const offset_in_block = local_offset2 + HEADER_MSG_INFO_V1_SIZE;
1339
- if (msg_type === OBJECT_CONTINUATION_MSG_TYPE) {
1340
- const [fh_off, cont_size] = struct.unpack_from("<QQ", data, offset_in_block);
1341
- blockQueue.push({ absOffset: fh_off, size: cont_size });
1676
+ const wave = blockQueue.splice(0, blockQueue.length);
1677
+ const datas = await Promise.all(
1678
+ wave.map(
1679
+ ({ absOffset, size }) => sliceOrFetch(source, prefix, offset, absOffset, size)
1680
+ )
1681
+ );
1682
+ for (let w = 0; w < wave.length; w++) {
1683
+ const { absOffset, size } = wave[w];
1684
+ const data = datas[w];
1685
+ allBlocks.push({ absOffset, size, data });
1686
+ let local_offset2 = 0;
1687
+ while (local_offset2 + HEADER_MSG_INFO_V1_SIZE <= size) {
1688
+ const msg = _unpack_struct_from(HEADER_MSG_INFO_V1, data, local_offset2);
1689
+ const msg_type = msg.get("type");
1690
+ const msg_size = msg.get("size");
1691
+ const offset_in_block = local_offset2 + HEADER_MSG_INFO_V1_SIZE;
1692
+ if (msg_type === OBJECT_CONTINUATION_MSG_TYPE) {
1693
+ const [fh_off, cont_size] = struct.unpack_from("<QQ", data, offset_in_block);
1694
+ blockQueue.push({ absOffset: fh_off, size: cont_size });
1695
+ }
1696
+ if (msg_type === 0 && msg_size === 0) break;
1697
+ local_offset2 += HEADER_MSG_INFO_V1_SIZE + msg_size;
1342
1698
  }
1343
- if (msg_type === 0 && msg_size === 0) break;
1344
- local_offset2 += HEADER_MSG_INFO_V1_SIZE + msg_size;
1345
1699
  }
1346
1700
  }
1347
1701
  let bufStart = offset;
@@ -1351,7 +1705,7 @@ class DataObjects {
1351
1705
  maxEnd = Math.max(maxEnd, b.absOffset + b.size);
1352
1706
  }
1353
1707
  const combined = new Uint8Array(maxEnd - bufStart);
1354
- combined.set(new Uint8Array(headerBuf), offset - bufStart);
1708
+ combined.set(new Uint8Array(headerBuf, 0, OBJECT_HEADER_V1_SIZE), offset - bufStart);
1355
1709
  for (const b of allBlocks) {
1356
1710
  combined.set(new Uint8Array(b.data), b.absOffset - bufStart);
1357
1711
  }
@@ -1387,9 +1741,9 @@ class DataObjects {
1387
1741
  return { buf, bufStart, msgs: msgs.filter(Boolean), msg_data, header };
1388
1742
  }
1389
1743
  // ────────── V2 parsing ──────────
1390
- static async _parseV2(source, offset) {
1391
- const peekSize = 64;
1392
- const peekBuf = await fetchRange(source, offset, peekSize);
1744
+ static async _parseV2(source, offset, prefix) {
1745
+ const peekBuf = prefix.byteLength >= 64 ? prefix : await fetchRange(source, offset, 64);
1746
+ const peekSize = peekBuf.byteLength;
1393
1747
  const { header, creation_order_size, block_offset: first_block_offset } = DataObjects._parseV2Header(peekBuf, 0, offset);
1394
1748
  const first_block_size = header.get("size_of_chunk_0");
1395
1749
  const allBlocks = [];
@@ -1397,27 +1751,30 @@ class DataObjects {
1397
1751
  { absOffset: first_block_offset, size: first_block_size }
1398
1752
  ];
1399
1753
  while (blockQueue.length > 0) {
1400
- const { absOffset, size } = blockQueue.shift();
1401
- let data;
1402
- if (absOffset >= offset && absOffset + size <= offset + peekSize) {
1403
- data = peekBuf.slice(absOffset - offset, absOffset - offset + size);
1404
- } else {
1405
- data = await fetchRange(source, absOffset, size);
1406
- }
1407
- allBlocks.push({ absOffset, size, data });
1408
- let local_offset2 = 0;
1409
- while (local_offset2 + HEADER_MSG_INFO_V2_SIZE <= size) {
1410
- const relOff = local_offset2;
1411
- const msg_type = struct.unpack_from("<B", data, relOff)[0];
1412
- const msg_size = struct.unpack_from("<H", data, relOff + 1)[0];
1413
- const offset_in_data = relOff + HEADER_MSG_INFO_V2_SIZE + creation_order_size;
1414
- if (msg_type === OBJECT_CONTINUATION_MSG_TYPE) {
1415
- const [fh_off, cont_size] = struct.unpack_from("<QQ", data, offset_in_data);
1416
- blockQueue.push({ absOffset: fh_off + 4, size: cont_size - 4 });
1754
+ const wave = blockQueue.splice(0, blockQueue.length);
1755
+ const datas = await Promise.all(
1756
+ wave.map(
1757
+ ({ absOffset, size }) => sliceOrFetch(source, peekBuf, offset, absOffset, size)
1758
+ )
1759
+ );
1760
+ for (let w = 0; w < wave.length; w++) {
1761
+ const { absOffset, size } = wave[w];
1762
+ const data = datas[w];
1763
+ allBlocks.push({ absOffset, size, data });
1764
+ let local_offset2 = 0;
1765
+ while (local_offset2 + HEADER_MSG_INFO_V2_SIZE <= size) {
1766
+ const relOff = local_offset2;
1767
+ const msg_type = struct.unpack_from("<B", data, relOff)[0];
1768
+ const msg_size = struct.unpack_from("<H", data, relOff + 1)[0];
1769
+ const offset_in_data = relOff + HEADER_MSG_INFO_V2_SIZE + creation_order_size;
1770
+ if (msg_type === OBJECT_CONTINUATION_MSG_TYPE) {
1771
+ const [fh_off, cont_size] = struct.unpack_from("<QQ", data, offset_in_data);
1772
+ blockQueue.push({ absOffset: fh_off + 4, size: cont_size - 4 });
1773
+ }
1774
+ if (msg_type === 0 && msg_size === 0) break;
1775
+ local_offset2 += HEADER_MSG_INFO_V2_SIZE + msg_size + creation_order_size;
1776
+ if (local_offset2 >= size - HEADER_MSG_INFO_V2_SIZE) break;
1417
1777
  }
1418
- if (msg_type === 0 && msg_size === 0) break;
1419
- local_offset2 += HEADER_MSG_INFO_V2_SIZE + msg_size + creation_order_size;
1420
- if (local_offset2 >= size - HEADER_MSG_INFO_V2_SIZE) break;
1421
1778
  }
1422
1779
  }
1423
1780
  const bufStart = offset;
@@ -1706,11 +2063,15 @@ class DataObjects {
1706
2063
  return this._iter_links_btree_v1(data.get("btree_address"), data.get("heap_address"));
1707
2064
  }
1708
2065
  async _iter_links_btree_v1(btree_address, heap_address) {
1709
- const btree = await BTreeV1Groups.create(this._source, btree_address);
1710
- const heap = await Heap.create(this._source, heap_address);
2066
+ const [btree, heap] = await Promise.all([
2067
+ BTreeV1Groups.create(this._source, btree_address),
2068
+ Heap.create(this._source, heap_address)
2069
+ ]);
2070
+ const tables = await Promise.all(
2071
+ btree.symbol_table_addresses().map((address) => SymbolTable.create(this._source, address))
2072
+ );
1711
2073
  const links = {};
1712
- for (const symbol_table_address of btree.symbol_table_addresses()) {
1713
- const table2 = await SymbolTable.create(this._source, symbol_table_address);
2074
+ for (const table2 of tables) {
1714
2075
  table2.assign_name(heap);
1715
2076
  Object.assign(links, table2.get_links(heap));
1716
2077
  }
@@ -1729,14 +2090,11 @@ class DataObjects {
1729
2090
  return {};
1730
2091
  }
1731
2092
  async _iter_links_btree_v2(name_btree_address, order_btree_address, heap_address) {
1732
- const heap = await FractalHeap.create(this._source, heap_address);
1733
2093
  const ordered = order_btree_address !== null;
1734
- let btree;
1735
- if (ordered) {
1736
- btree = await BTreeV2GroupOrders.create(this._source, order_btree_address);
1737
- } else {
1738
- btree = await BTreeV2GroupNames.create(this._source, name_btree_address);
1739
- }
2094
+ const [heap, btree] = await Promise.all([
2095
+ FractalHeap.create(this._source, heap_address),
2096
+ ordered ? BTreeV2GroupOrders.create(this._source, order_btree_address) : BTreeV2GroupNames.create(this._source, name_btree_address)
2097
+ ]);
1740
2098
  const items = /* @__PURE__ */ new Map();
1741
2099
  for (const record of btree.iter_records()) {
1742
2100
  const data = heap.get_data(record.get("heapid"));
@@ -1855,14 +2213,11 @@ class DataObjects {
1855
2213
  if (fractal_heap_address === UNDEFINED_ADDRESS || name_btree_address === UNDEFINED_ADDRESS) {
1856
2214
  return {};
1857
2215
  }
1858
- const heap = await FractalHeap.create(this._source, fractal_heap_address);
1859
2216
  const ordered = order_btree_address !== null && order_btree_address !== UNDEFINED_ADDRESS;
1860
- let btree;
1861
- if (ordered) {
1862
- btree = await BTreeV2GroupOrders.create(this._source, order_btree_address);
1863
- } else {
1864
- btree = await BTreeV2GroupNames.create(this._source, name_btree_address);
1865
- }
2217
+ const [heap, btree] = await Promise.all([
2218
+ FractalHeap.create(this._source, fractal_heap_address),
2219
+ ordered ? BTreeV2GroupOrders.create(this._source, order_btree_address) : BTreeV2GroupNames.create(this._source, name_btree_address)
2220
+ ]);
1866
2221
  const attrs = {};
1867
2222
  for (const record of btree.iter_records()) {
1868
2223
  try {
@@ -1960,19 +2315,56 @@ class DataObjects {
1960
2315
  const gheap_id = _unpack_struct_from(GLOBAL_HEAP_ID, buf, offset + 4);
1961
2316
  const gheap_address = gheap_id.get("collection_address");
1962
2317
  assert(gheap_address < Number.MAX_SAFE_INTEGER);
1963
- let gheap;
1964
- if (this._global_heaps.has(gheap_address)) {
1965
- gheap = this._global_heaps.get(gheap_address);
1966
- } else {
1967
- gheap = await GlobalHeap.create(this._source, gheap_address);
1968
- this._global_heaps.set(gheap_address, gheap);
1969
- }
2318
+ const gheap = await this._get_global_heap(gheap_address);
1970
2319
  const vlen_data = gheap.objects.get(gheap_id.get("object_index"));
1971
2320
  if (!vlen_data) {
1972
2321
  throw new Error(`Global heap object not found at index ${gheap_id.get("object_index")}`);
1973
2322
  }
1974
2323
  return [vlen_size, vlen_data];
1975
2324
  }
2325
+ _get_global_heap(address) {
2326
+ let gheap = this._global_heaps.get(address);
2327
+ if (!gheap) {
2328
+ const promise = GlobalHeap.create(this._source, address);
2329
+ promise.catch(() => {
2330
+ if (this._global_heaps.peek(address) === promise) this._global_heaps.delete(address);
2331
+ });
2332
+ this._global_heaps.set(address, promise);
2333
+ gheap = promise;
2334
+ }
2335
+ return gheap;
2336
+ }
2337
+ /**
2338
+ * Decode `count` variable-length strings from consecutive 16-byte global heap references
2339
+ * (uint32 length, uint64 collection address, uint32 object index) starting at `offset`.
2340
+ * All referenced global heap collections are fetched concurrently.
2341
+ * Null / missing references decode to "".
2342
+ */
2343
+ async _read_vlen_strings(buf, offset, count, decoder) {
2344
+ const view = new DataView(buf);
2345
+ const refs = new Array(count);
2346
+ const addresses = /* @__PURE__ */ new Set();
2347
+ for (let i = 0; i < count; i++) {
2348
+ const o = offset + i * 16;
2349
+ const size = view.getUint32(o, true);
2350
+ const address = Number(view.getBigUint64(o + 4, true));
2351
+ const index = view.getUint32(o + 12, true);
2352
+ refs[i] = { size, address, index };
2353
+ if (size > 0 && address !== 0 && address < Number.MAX_SAFE_INTEGER) {
2354
+ addresses.add(address);
2355
+ }
2356
+ }
2357
+ const heaps = /* @__PURE__ */ new Map();
2358
+ await Promise.all(
2359
+ Array.from(addresses, async (address) => {
2360
+ heaps.set(address, await this._get_global_heap(address));
2361
+ })
2362
+ );
2363
+ return refs.map(({ address, index }) => {
2364
+ const data = heaps.get(address)?.objects.get(index);
2365
+ return data ? decoder.decode(data) : "";
2366
+ });
2367
+ }
1976
2368
  // ────────── Async: get_data ──────────
1977
2369
  async get_data() {
1978
2370
  const msg = this.find_msg_type(DATA_STORAGE_MSG_TYPE)[0];
@@ -2025,12 +2417,7 @@ class DataObjects {
2025
2417
  const encoding = character_set === 0 ? "ascii" : "utf-8";
2026
2418
  const decoder = new TextDecoder(encoding);
2027
2419
  const refBuf = await fetchRange(this._source, data_offset, fullsize * 16);
2028
- const output = new Array(fullsize);
2029
- for (let i = 0; i < fullsize; i++) {
2030
- const [, vlen_data] = await this._vlen_size_and_data(refBuf, i * 16);
2031
- output[i] = decoder.decode(vlen_data);
2032
- }
2033
- return output;
2420
+ return this._read_vlen_strings(refBuf, 0, fullsize, decoder);
2034
2421
  } else if (dtype_class === "COMPOUND") {
2035
2422
  const total_size = dtype[2];
2036
2423
  const dataBuf = await fetchRange(this._source, data_offset, fullsize * total_size);
@@ -2044,42 +2431,74 @@ class DataObjects {
2044
2431
  }
2045
2432
  }
2046
2433
  }
2047
- async _get_chunked_data(msg_offset) {
2434
+ async _get_chunked_data(_msg_offset) {
2048
2435
  this._get_chunk_params();
2049
2436
  if (this._chunk_address === null || this._chunk_address === UNDEFINED_ADDRESS) {
2050
2437
  return [];
2051
2438
  }
2052
2439
  const dtype = this.dtype;
2053
- if (!Array.isArray(dtype) || !/^VLEN/.test(dtype[0])) {
2440
+ if (!Array.isArray(dtype) || dtype[0] !== "VLEN_STRING") {
2054
2441
  return [];
2055
2442
  }
2056
- dtype[0];
2057
2443
  const character_set = dtype[2];
2058
2444
  const encoding = character_set === 0 ? "ascii" : "utf-8";
2059
2445
  const decoder = new TextDecoder(encoding);
2446
+ const shape = this.shape;
2447
+ const chunkShape = this._chunks;
2448
+ const ndim = shape.length;
2449
+ const total = shape.reduce((a, b) => a * b, 1);
2450
+ const output = new Array(total).fill("");
2451
+ const pipeline = this.filter_pipeline;
2060
2452
  const chunk_btree = await BTreeV1RawDataChunks.create(
2061
2453
  this._source,
2062
2454
  this._chunk_address,
2063
2455
  this._chunk_dims
2064
2456
  );
2065
- const output = [];
2066
- const leafNodes = chunk_btree.all_nodes.get(0);
2067
- if (!leafNodes) return output;
2457
+ const leafNodes = chunk_btree.all_nodes.get(0) ?? [];
2458
+ const chunks = [];
2068
2459
  for (const node of leafNodes) {
2069
2460
  for (let ik = 0; ik < node.keys.length; ik++) {
2070
- const addr = node.addresses[ik];
2071
- const chunk_size = node.keys[ik].chunk_size;
2072
- const chunkBuf = await fetchRange(this._source, addr, chunk_size);
2073
- const nItems = chunk_size / 16;
2461
+ const key = node.keys[ik];
2462
+ chunks.push({
2463
+ address: node.addresses[ik],
2464
+ size: key.chunk_size,
2465
+ filterMask: key.filter_mask,
2466
+ start: key.chunk_offset.slice(0, ndim)
2467
+ });
2468
+ }
2469
+ }
2470
+ const chunkItems = chunkShape.reduce((a, b) => a * b, 1);
2471
+ const strides = new Array(ndim).fill(1);
2472
+ const chunkStrides = new Array(ndim).fill(1);
2473
+ for (let d = ndim - 2; d >= 0; d--) {
2474
+ strides[d] = strides[d + 1] * shape[d + 1];
2475
+ chunkStrides[d] = chunkStrides[d + 1] * chunkShape[d + 1];
2476
+ }
2477
+ await Promise.all(
2478
+ chunks.map(async ({ address, size, filterMask, start }) => {
2479
+ const raw = new Uint8Array(await fetchRange(this._source, address, size));
2480
+ const decoded = await decodeChunk(raw, pipeline, filterMask);
2481
+ const nItems = Math.min(chunkItems, Math.floor(decoded.byteLength / 16));
2482
+ const buf = decoded.buffer.slice(
2483
+ decoded.byteOffset,
2484
+ decoded.byteOffset + nItems * 16
2485
+ );
2486
+ const strings = await this._read_vlen_strings(buf, 0, nItems, decoder);
2074
2487
  for (let i = 0; i < nItems; i++) {
2075
- try {
2076
- const [, vlen_data] = await this._vlen_size_and_data(chunkBuf, i * 16);
2077
- output.push(decoder.decode(vlen_data));
2078
- } catch {
2488
+ let flat = 0;
2489
+ let inBounds = true;
2490
+ for (let d = 0; d < ndim; d++) {
2491
+ const coord = start[d] + Math.floor(i / chunkStrides[d]) % chunkShape[d];
2492
+ if (coord >= shape[d]) {
2493
+ inBounds = false;
2494
+ break;
2495
+ }
2496
+ flat += coord * strides[d];
2079
2497
  }
2498
+ if (inBounds) output[flat] = strings[i];
2080
2499
  }
2081
- }
2082
- }
2500
+ })
2501
+ );
2083
2502
  return output;
2084
2503
  }
2085
2504
  }
@@ -2168,6 +2587,7 @@ class File extends Group {
2168
2587
  }
2169
2588
  static async create(source, filename) {
2170
2589
  const superblock = await SuperBlock.create(source, 0);
2590
+ setFileSize(source, superblock.end_of_file);
2171
2591
  const offset = await superblock.getOffsetToDataobjects();
2172
2592
  const dataobjects = await DataObjects.create(source, offset);
2173
2593
  const links = await dataobjects.get_links();
@@ -2672,13 +3092,15 @@ function getAttrDatatypeClasses(dataobjects) {
2672
3092
  }
2673
3093
  return result;
2674
3094
  }
3095
+ function isVlenString(rawDtype) {
3096
+ return Array.isArray(rawDtype) && rawDtype[0] === "VLEN_STRING";
3097
+ }
2675
3098
  class SingleHdf5ToZarr {
2676
3099
  source;
2677
- h5File;
3100
+ h5File = null;
2678
3101
  url;
2679
3102
  inlineThreshold;
2680
3103
  zarrFormat;
2681
- refs;
2682
3104
  /**
2683
3105
  * @param source - An AsyncReadable instance for reading the HDF5 file.
2684
3106
  * @param options - Configuration options.
@@ -2688,62 +3110,103 @@ class SingleHdf5ToZarr {
2688
3110
  this.url = options.url ?? null;
2689
3111
  this.inlineThreshold = options.inlineThreshold ?? 300;
2690
3112
  this.zarrFormat = options.zarrFormat ?? 2;
2691
- this.refs = {};
3113
+ }
3114
+ /**
3115
+ * Open (and cache) the HDF5 file: reads the superblock and the root group's links.
3116
+ */
3117
+ open() {
3118
+ if (!this.h5File) {
3119
+ this.h5File = File.create(this.source);
3120
+ this.h5File.catch(() => {
3121
+ this.h5File = null;
3122
+ });
3123
+ }
3124
+ return this.h5File;
2692
3125
  }
2693
3126
  /**
2694
3127
  * Translate the HDF5 file content into a Zarr reference spec.
3128
+ * Independent groups and datasets are processed concurrently.
2695
3129
  */
2696
3130
  async translate() {
2697
- this.h5File = await File.create(this.source);
2698
- this.refs = {};
2699
- this.refs[".zgroup"] = jsonStringifySorted({ zarr_format: 2 });
2700
- await this.transferAttrs(this.h5File, "");
2701
- await this.walkGroup(this.h5File, "");
2702
- const refSpec = {
2703
- version: 1,
2704
- refs: this.refs
2705
- };
3131
+ const h5File = await this.open();
3132
+ const refs = await this.translateTree(h5File, true);
3133
+ const refSpec = { version: 1, refs };
2706
3134
  return this.zarrFormat === 3 ? refSpecV2ToV3(refSpec) : refSpec;
2707
3135
  }
2708
3136
  /**
2709
- * Recursively walk an HDF5 group, processing all children.
3137
+ * Translate the metadata (groups, arrays, attributes) of the whole hierarchy,
3138
+ * without any chunk references. Only HDF5 metadata is read; no data chunks or
3139
+ * variable-length string heaps are touched.
2710
3140
  */
2711
- async walkGroup(group, parentPath) {
2712
- const keys = group.keys;
2713
- for (const key of keys) {
2714
- const childPath = parentPath ? `${parentPath}/${key}` : key;
2715
- let child;
2716
- try {
2717
- child = await group.get(key);
2718
- } catch {
2719
- continue;
2720
- }
2721
- if (!child) continue;
2722
- if (child instanceof Dataset) {
2723
- await this.processDataset(childPath, child);
2724
- } else if (child instanceof Group) {
2725
- await this.processGroup(childPath, child);
2726
- await this.walkGroup(child, childPath);
2727
- }
3141
+ async translateMetadata() {
3142
+ const h5File = await this.open();
3143
+ const refs = await this.translateTree(h5File, false);
3144
+ const refSpec = { version: 1, refs };
3145
+ return this.zarrFormat === 3 ? refSpecV2ToV3(refSpec) : refSpec;
3146
+ }
3147
+ /**
3148
+ * Translate a single HDF5 group or dataset into Zarr v2 references
3149
+ * (keys are relative to the hierarchy root, e.g. "obs/.zattrs" or "X/data/0").
3150
+ *
3151
+ * @param path Path of the node within the hierarchy ("" for the root).
3152
+ * @param obj The HDF5 group or dataset at that path.
3153
+ * @param options.chunks Whether to include chunk references (and inline data).
3154
+ */
3155
+ async translateNode(path, obj, options = {}) {
3156
+ const refs = {};
3157
+ if (obj instanceof Dataset) {
3158
+ await this.processDataset(refs, path, obj, options.chunks ?? true);
3159
+ } else {
3160
+ await this.processGroup(refs, path, obj);
2728
3161
  }
3162
+ return refs;
3163
+ }
3164
+ /**
3165
+ * Translate a node and all of its descendants. Children are processed
3166
+ * concurrently, but the returned refs are in depth-first order
3167
+ * (the same order a sequential walk would produce).
3168
+ */
3169
+ async translateTree(root, chunks) {
3170
+ const parts = [];
3171
+ const visit = async (path, obj) => {
3172
+ const own = this.translateNode(path, obj, { chunks });
3173
+ if (!(obj instanceof Group)) return [await own];
3174
+ const children = await Promise.all(
3175
+ obj.keys.map(async (key) => {
3176
+ const childPath = path ? `${path}/${key}` : key;
3177
+ let child;
3178
+ try {
3179
+ child = await obj.get(key);
3180
+ } catch {
3181
+ return [];
3182
+ }
3183
+ if (!child) return [];
3184
+ return visit(childPath, child);
3185
+ })
3186
+ );
3187
+ return [await own, ...children.flat()];
3188
+ };
3189
+ parts.push(...await visit("", root));
3190
+ return Object.assign({}, ...parts);
2729
3191
  }
2730
3192
  /**
2731
3193
  * Process an HDF5 group into Zarr group metadata.
2732
3194
  */
2733
- async processGroup(path, group) {
2734
- this.refs[`${path}/.zgroup`] = jsonStringifySorted({
3195
+ async processGroup(refs, path, group) {
3196
+ const prefix = path ? `${path}/` : "";
3197
+ refs[`${prefix}.zgroup`] = jsonStringifySorted({
2735
3198
  zarr_format: 2
2736
3199
  });
2737
- await this.transferAttrs(group, path);
3200
+ await this.transferAttrs(refs, group, path);
2738
3201
  }
2739
3202
  /**
2740
- * Process an HDF5 dataset into Zarr array metadata and chunk references.
3203
+ * Process an HDF5 dataset into Zarr array metadata and (optionally) chunk references.
2741
3204
  */
2742
- async processDataset(path, dataset) {
3205
+ async processDataset(refs, path, dataset, includeChunks) {
2743
3206
  const dataobjects = dataset._dataobjects;
2744
3207
  const shape = dataobjects.shape;
2745
3208
  if (!shape || shape.length === 0) {
2746
- await this.processScalarDataset(path, dataset, dataobjects);
3209
+ await this.processScalarDataset(refs, path, dataset, dataobjects, includeChunks);
2747
3210
  return;
2748
3211
  }
2749
3212
  const rawDtype = dataobjects.dtype;
@@ -2778,25 +3241,32 @@ class SingleHdf5ToZarr {
2778
3241
  order: "C",
2779
3242
  filters: zarrFilters
2780
3243
  };
2781
- this.refs[`${path}/.zarray`] = serializeZarrayMeta(zarrMeta);
2782
- await this.transferAttrs(dataset, path);
3244
+ if (isVlenString(rawDtype)) {
3245
+ zarrMeta.dtype = "|O";
3246
+ zarrMeta.fill_value = null;
3247
+ zarrMeta.filters = [JSON2_FILTER];
3248
+ zarrMeta.compressor = null;
3249
+ }
3250
+ refs[`${path}/.zarray`] = serializeZarrayMeta(zarrMeta);
3251
+ await this.transferAttrs(refs, dataset, path);
2783
3252
  const dims = this.getArrayDims(dataset, path, shape);
2784
3253
  const existingAttrsKey = `${path}/.zattrs`;
2785
- const existingAttrs = this.refs[existingAttrsKey] ? JSON.parse(this.refs[existingAttrsKey]) : {};
3254
+ const existingAttrs = refs[existingAttrsKey] ? JSON.parse(refs[existingAttrsKey]) : {};
2786
3255
  existingAttrs["_ARRAY_DIMENSIONS"] = dims;
2787
- this.refs[existingAttrsKey] = jsonStringifySorted(existingAttrs);
3256
+ refs[existingAttrsKey] = jsonStringifySorted(existingAttrs);
3257
+ if (!includeChunks) return;
2788
3258
  if (isVlen) {
2789
- await this.inlineVlenData(path, dataset, shape, chunks);
3259
+ await this.inlineVlenData(refs, path, dataset, shape, chunks);
2790
3260
  } else if (isFixedString) {
2791
- await this.inlineFixedStringData(path, dataset, shape, chunks, rawDtype);
3261
+ await this.inlineFixedStringData(refs, path, dataset, shape, chunks, rawDtype);
2792
3262
  } else {
2793
- await this.processChunkReferences(path, dataobjects, shape, chunks, dtype);
3263
+ await this.processChunkReferences(refs, path, dataobjects, shape, chunks, dtype);
2794
3264
  }
2795
3265
  }
2796
3266
  /**
2797
3267
  * Process a scalar dataset (shape=[]).
2798
3268
  */
2799
- async processScalarDataset(path, dataset, dataobjects) {
3269
+ async processScalarDataset(refs, path, dataset, dataobjects, includeChunks) {
2800
3270
  const rawDtype = dataobjects.dtype;
2801
3271
  const isVlen = Array.isArray(rawDtype) && (rawDtype[0] === "VLEN_STRING" || rawDtype[0] === "VLEN_SEQUENCE");
2802
3272
  const dtype = jsfiveDtypeToZarr(rawDtype);
@@ -2820,17 +3290,18 @@ class SingleHdf5ToZarr {
2820
3290
  zarrMeta.fill_value = null;
2821
3291
  zarrMeta.filters = [JSON2_FILTER];
2822
3292
  }
2823
- this.refs[`${path}/.zarray`] = serializeZarrayMeta(zarrMeta);
2824
- await this.transferAttrs(dataset, path);
3293
+ refs[`${path}/.zarray`] = serializeZarrayMeta(zarrMeta);
3294
+ await this.transferAttrs(refs, dataset, path);
2825
3295
  const existingAttrsKey = `${path}/.zattrs`;
2826
- const existingAttrs = this.refs[existingAttrsKey] ? JSON.parse(this.refs[existingAttrsKey]) : {};
3296
+ const existingAttrs = refs[existingAttrsKey] ? JSON.parse(refs[existingAttrsKey]) : {};
2827
3297
  existingAttrs["_ARRAY_DIMENSIONS"] = [];
2828
- this.refs[existingAttrsKey] = jsonStringifySorted(existingAttrs);
3298
+ refs[existingAttrsKey] = jsonStringifySorted(existingAttrs);
3299
+ if (!includeChunks) return;
2829
3300
  if (isVlen) {
2830
3301
  try {
2831
3302
  const data = await dataobjects.get_data();
2832
3303
  const strings = Array.isArray(data) ? data.map(String) : [String(data)];
2833
- this.refs[`${path}/0`] = encodeJson2(strings, []);
3304
+ refs[`${path}/0`] = encodeJson2(strings, []);
2834
3305
  } catch {
2835
3306
  }
2836
3307
  } else {
@@ -2839,7 +3310,7 @@ class SingleHdf5ToZarr {
2839
3310
  const value = Array.isArray(data) ? data[0] : data;
2840
3311
  const bytes = encodeScalar(value, dtype);
2841
3312
  if (bytes) {
2842
- this.refs[`${path}/0`] = inlineBytes(bytes);
3313
+ refs[`${path}/0`] = inlineBytes(bytes);
2843
3314
  }
2844
3315
  } catch {
2845
3316
  }
@@ -2848,26 +3319,26 @@ class SingleHdf5ToZarr {
2848
3319
  /**
2849
3320
  * Inline vlen string data using json2 codec encoding.
2850
3321
  */
2851
- async inlineVlenData(path, dataset, shape, chunks) {
3322
+ async inlineVlenData(refs, path, dataset, shape, chunks) {
2852
3323
  try {
2853
3324
  const data = await dataset._dataobjects.get_data();
2854
3325
  if (data === null || data === void 0) return;
2855
3326
  const strings = Array.isArray(data) ? data.map(String) : [String(data)];
2856
3327
  const key = `${path}/${chunks.map(() => "0").join(".")}`;
2857
- this.refs[key] = encodeJson2(strings, shape);
2858
- const zarrMeta = JSON.parse(this.refs[`${path}/.zarray`]);
3328
+ refs[key] = encodeJson2(strings, shape);
3329
+ const zarrMeta = JSON.parse(refs[`${path}/.zarray`]);
2859
3330
  zarrMeta.dtype = "|O";
2860
3331
  zarrMeta.fill_value = null;
2861
3332
  zarrMeta.filters = [JSON2_FILTER];
2862
3333
  zarrMeta.compressor = null;
2863
- this.refs[`${path}/.zarray`] = serializeZarrayMeta(zarrMeta);
3334
+ refs[`${path}/.zarray`] = serializeZarrayMeta(zarrMeta);
2864
3335
  } catch {
2865
3336
  }
2866
3337
  }
2867
3338
  /**
2868
3339
  * Inline fixed-length string data.
2869
3340
  */
2870
- async inlineFixedStringData(path, dataset, shape, chunks, rawDtype) {
3341
+ async inlineFixedStringData(refs, path, dataset, shape, chunks, rawDtype) {
2871
3342
  try {
2872
3343
  const data = await dataset._dataobjects.get_data();
2873
3344
  if (data === null || data === void 0) return;
@@ -2882,7 +3353,7 @@ class SingleHdf5ToZarr {
2882
3353
  const encoded = encoder.encode(strings[i]);
2883
3354
  buf.set(encoded.subarray(0, strSize), i * strSize);
2884
3355
  }
2885
- this.refs[key] = inlineBytes(buf);
3356
+ refs[key] = inlineBytes(buf);
2886
3357
  } catch {
2887
3358
  }
2888
3359
  }
@@ -2890,7 +3361,7 @@ class SingleHdf5ToZarr {
2890
3361
  * Process chunk references for non-string datasets.
2891
3362
  * Uses file byte offset references for chunked and contiguous data.
2892
3363
  */
2893
- async processChunkReferences(path, dataobjects, shape, chunks, dtype) {
3364
+ async processChunkReferences(refs, path, dataobjects, shape, chunks, dtype) {
2894
3365
  const storageInfo = getStorageInfo(dataobjects);
2895
3366
  if (storageInfo.layoutClass === 1 && storageInfo.contiguousOffset !== void 0) {
2896
3367
  const key = `${path}/${shape.map(() => "0").join(".")}`;
@@ -2898,9 +3369,9 @@ class SingleHdf5ToZarr {
2898
3369
  const size = storageInfo.contiguousSize;
2899
3370
  if (size <= this.inlineThreshold) {
2900
3371
  const ab = await fetchRange(this.source, offset, size);
2901
- this.refs[key] = inlineBytes(new Uint8Array(ab));
3372
+ refs[key] = inlineBytes(new Uint8Array(ab));
2902
3373
  } else {
2903
- this.refs[key] = [this.url, offset, size];
3374
+ refs[key] = [this.url, offset, size];
2904
3375
  }
2905
3376
  } else if (storageInfo.layoutClass === 2 && storageInfo.chunkAddress !== void 0) {
2906
3377
  const filterPipeline = dataobjects.filter_pipeline;
@@ -2926,19 +3397,21 @@ class SingleHdf5ToZarr {
2926
3397
  shape,
2927
3398
  storageInfo.chunkShape
2928
3399
  );
2929
- for (const loc of chunkLocations) {
2930
- const key = `${path}/${loc.chunkIndex.join(".")}`;
2931
- let size = hasFilters ? loc.size : uncompressedChunkBytes;
2932
- if (hasF32) {
2933
- size -= 4;
2934
- }
2935
- if (size <= this.inlineThreshold) {
2936
- const ab = await fetchRange(this.source, loc.offset, size);
2937
- this.refs[key] = inlineBytes(new Uint8Array(ab));
2938
- } else {
2939
- this.refs[key] = [this.url, loc.offset, size];
2940
- }
2941
- }
3400
+ const entries = await Promise.all(
3401
+ chunkLocations.map(async (loc) => {
3402
+ const key = `${path}/${loc.chunkIndex.join(".")}`;
3403
+ let size = hasFilters ? loc.size : uncompressedChunkBytes;
3404
+ if (hasF32) {
3405
+ size -= 4;
3406
+ }
3407
+ if (size <= this.inlineThreshold) {
3408
+ const ab = await fetchRange(this.source, loc.offset, size);
3409
+ return [key, inlineBytes(new Uint8Array(ab))];
3410
+ }
3411
+ return [key, [this.url, loc.offset, size]];
3412
+ })
3413
+ );
3414
+ for (const [key, value] of entries) refs[key] = value;
2942
3415
  } else if (storageInfo.layoutClass === 1) ;
2943
3416
  }
2944
3417
  /**
@@ -2956,7 +3429,7 @@ class SingleHdf5ToZarr {
2956
3429
  /**
2957
3430
  * Transfer attributes from an HDF5 object to the Zarr .zattrs.
2958
3431
  */
2959
- async transferAttrs(h5obj, path) {
3432
+ async transferAttrs(refs, h5obj, path) {
2960
3433
  const attrs = {};
2961
3434
  let rawAttrs;
2962
3435
  try {
@@ -2995,7 +3468,7 @@ class SingleHdf5ToZarr {
2995
3468
  }
2996
3469
  if (Object.keys(attrs).length > 0) {
2997
3470
  const key = path ? `${path}/.zattrs` : ".zattrs";
2998
- this.refs[key] = jsonStringifySorted(attrs);
3471
+ refs[key] = jsonStringifySorted(attrs);
2999
3472
  }
3000
3473
  }
3001
3474
  }
@@ -3035,177 +3508,13 @@ function refSpecToConsolidatedMetadataV3(refSpec) {
3035
3508
  }
3036
3509
  };
3037
3510
  }
3038
- function parse$1(template, re = /{{(.*?)}}/) {
3039
- let result = re.exec(template);
3040
- const parts = [];
3041
- let pos;
3042
- while (result) {
3043
- pos = result.index;
3044
- if (pos !== 0) {
3045
- parts.push({ match: false, str: template.substring(0, pos) });
3046
- template = template.slice(pos);
3047
- }
3048
- parts.push({ match: true, str: result[0] });
3049
- template = template.slice(result[0].length);
3050
- result = re.exec(template);
3051
- }
3052
- if (template) {
3053
- parts.push({ match: false, str: template });
3054
- }
3055
- return parts;
3056
- }
3057
- function matchFn(str) {
3058
- const match = str.match(/(?<fname>[A-Z_][A-Z_1-9]*)\((?<args>[^)]+)\)/i);
3059
- if (!match?.groups) return;
3060
- const { fname, args } = match.groups;
3061
- const ctx = Object.fromEntries(
3062
- args.split(",").map((kwarg) => {
3063
- const { key, num, str: str2 } = kwarg.match(
3064
- /(?<key>[a-z_0-9]*)\s*=\s*((?<num>[0-9.]+)|('|")(?<str>.*)('|"))/i
3065
- )?.groups ?? {};
3066
- if (!key || !(num || str2)) {
3067
- throw Error(`Failed to match fn kwarg: ${kwarg}`);
3068
- }
3069
- return [key, num ? Number(num) : str2];
3070
- })
3071
- );
3072
- return { fname, ctx };
3073
- }
3074
3511
  const alphabet = "abcdefghijklmnopqrstuvwxyz";
3075
3512
  const numbers = "01234569789";
3076
3513
  const expr = "()*/+-";
3077
3514
  const space = " ";
3078
- const valid = new Set(
3515
+ new Set(
3079
3516
  alphabet + alphabet.toUpperCase() + numbers + expr + space
3080
3517
  );
3081
- function matchMathEval(str) {
3082
- for (let i = 0; i < str.length; i++) {
3083
- if (!valid.has(str.charAt(i))) return;
3084
- }
3085
- return parse$1(str, /[A-Za-z_][A-Za-z0-9_]*/gi);
3086
- }
3087
- function render(template, context) {
3088
- const grps = parse$1(template);
3089
- return grps.map((grp) => {
3090
- if (!grp.match) {
3091
- return grp.str;
3092
- }
3093
- let inner = grp.str.split(/{{|}}/).filter(Boolean)[0].trim();
3094
- if (inner in context) {
3095
- return context[inner];
3096
- }
3097
- const fnMatch = matchFn(inner);
3098
- if (fnMatch) {
3099
- const { fname, ctx } = fnMatch;
3100
- const fn = context[fname];
3101
- if (typeof fn === "function") {
3102
- return fn(ctx);
3103
- }
3104
- throw Error(
3105
- `Cannot find function named ${fname} in rendering context.`
3106
- );
3107
- }
3108
- const matches = matchMathEval(inner);
3109
- if (matches) {
3110
- const exprParts = matches.map((match) => {
3111
- if (!match.match) return match.str;
3112
- const value = context[match.str];
3113
- if (value == null) {
3114
- throw Error(
3115
- `Cannot find number named ${match.str} in rendering context.`
3116
- );
3117
- }
3118
- if (typeof value !== "number") {
3119
- throw Error(
3120
- `The provided value for ${match.str} must be a number.`
3121
- );
3122
- }
3123
- return value;
3124
- });
3125
- return Function('"use strict";return (' + exprParts.join("") + ")")();
3126
- }
3127
- throw new Error(`Unable to match ${grp.str}`);
3128
- }).join("");
3129
- }
3130
- function parse(spec, renderString = render) {
3131
- return "version" in spec ? parseV1(spec, renderString) : parseV0(spec);
3132
- }
3133
- function parseV0(spec) {
3134
- return new Map(Object.entries(spec));
3135
- }
3136
- function parseV1(spec, renderString) {
3137
- const context = {};
3138
- for (const [key, template] of Object.entries(spec.templates ?? {})) {
3139
- if (template.includes("{{")) {
3140
- context[key] = (ctx) => renderString(template, ctx);
3141
- } else {
3142
- context[key] = template;
3143
- }
3144
- }
3145
- const render2 = (t, o) => {
3146
- return renderString(t, { ...context, ...o });
3147
- };
3148
- const refs = /* @__PURE__ */ new Map();
3149
- for (const [key, ref] of Object.entries(spec.refs ?? {})) {
3150
- if (typeof ref === "string") {
3151
- refs.set(key, ref);
3152
- } else {
3153
- const url = ref[0]?.includes("{{") ? render2(ref[0]) : ref[0];
3154
- refs.set(key, ref.length === 1 ? [url] : [url, ref[1], ref[2]]);
3155
- }
3156
- }
3157
- for (const g of spec.gen ?? []) {
3158
- for (const dims of iterDims(g.dimensions)) {
3159
- const key = render2(g.key, dims);
3160
- const url = render2(g.url, dims);
3161
- if ("offset" in g && "length" in g) {
3162
- const offset = render2(g.offset, dims);
3163
- const length = render2(g.length, dims);
3164
- refs.set(key, [url, parseInt(offset), parseInt(length)]);
3165
- } else {
3166
- refs.set(key, [url]);
3167
- }
3168
- }
3169
- }
3170
- return refs;
3171
- }
3172
- function* iterDims(dimensions) {
3173
- const keys = Object.keys(dimensions);
3174
- const iterables = Object.values(dimensions).map(
3175
- (i) => Array.isArray(i) ? i : [...range(i)]
3176
- );
3177
- for (const values of product(...iterables)) {
3178
- yield Object.fromEntries(keys.map((key, i) => [key, values[i]]));
3179
- }
3180
- }
3181
- function* product(...iterables) {
3182
- if (iterables.length === 0) {
3183
- return;
3184
- }
3185
- const iterators = iterables.map((it) => it[Symbol.iterator]());
3186
- const results = iterators.map((it) => it.next());
3187
- if (results.some((r) => r.done)) {
3188
- throw new Error("Input contains an empty iterator.");
3189
- }
3190
- for (let i = 0; ; ) {
3191
- if (results[i].done) {
3192
- iterators[i] = iterables[i][Symbol.iterator]();
3193
- results[i] = iterators[i].next();
3194
- if (++i >= iterators.length) {
3195
- return;
3196
- }
3197
- } else {
3198
- yield results.map(({ value }) => value);
3199
- i = 0;
3200
- }
3201
- results[i] = iterators[i].next();
3202
- }
3203
- }
3204
- function* range({ stop, start = 0, step = 1 }) {
3205
- for (let i = start; i < stop; i += step) {
3206
- yield i;
3207
- }
3208
- }
3209
3518
  (() => {
3210
3519
  var table2 = new Uint8Array(128);
3211
3520
  for (var i = 0; i < 64; i++) {
@@ -3228,19 +3537,6 @@ function* range({ stop, start = 0, step = 1 }) {
3228
3537
  };
3229
3538
  })();
3230
3539
  new TextEncoder();
3231
- function strip_prefix(path) {
3232
- return path.slice(1);
3233
- }
3234
- function merge_init(storeOverrides, requestOverrides) {
3235
- return {
3236
- ...storeOverrides,
3237
- ...requestOverrides,
3238
- headers: {
3239
- ...storeOverrides.headers,
3240
- ...requestOverrides.headers
3241
- }
3242
- };
3243
- }
3244
3540
  let table = new Uint8Array(128);
3245
3541
  for (let i = 0; i < 64; i++) {
3246
3542
  table[i < 26 ? i + 65 : i < 52 ? i + 71 : i < 62 ? i - 4 : i * 4 - 205] = i;
@@ -3262,70 +3558,144 @@ function to_binary(base64) {
3262
3558
  }
3263
3559
  return bytes;
3264
3560
  }
3265
- class ComposableReferenceStore {
3266
- #refs;
3267
- #internal_store;
3268
- #overrides;
3269
- constructor(refs, internal_store, overrides) {
3270
- this.#refs = refs;
3271
- this.#internal_store = internal_store;
3272
- this.#overrides = overrides;
3273
- }
3274
- async get(key, opts = {}) {
3275
- let ref = this.#refs.get(strip_prefix(key));
3276
- if (!ref) return;
3277
- if (typeof ref === "string") {
3278
- if (ref.startsWith("base64:")) {
3279
- return to_binary(ref.slice("base64:".length));
3280
- }
3281
- return new TextEncoder().encode(ref);
3282
- }
3283
- let [urlOrNull, offset, size] = ref;
3284
- if (offset === void 0 || size === void 0 || !this.#internal_store.getRange) {
3285
- return this.#internal_store.get(key, opts);
3286
- }
3287
- const opts_for_get = this.#overrides ? merge_init(this.#overrides, opts) : opts;
3288
- return this.#internal_store.getRange("/", { offset, length: size }, opts_for_get);
3289
- }
3290
- // TODO: does it make sense to support getRange here? get already calls getRange on the internal store...
3291
- static fromSpec(spec, internal_store, overrides) {
3292
- let refs = parse(spec);
3293
- return new ComposableReferenceStore(refs, internal_store, overrides);
3294
- }
3561
+ function evictOnReject(cache, key, promise) {
3562
+ promise.catch(() => {
3563
+ if (cache.peek(key) === promise) cache.delete(key);
3564
+ });
3295
3565
  }
3296
3566
  class HdfStore {
3297
- refStore;
3298
- refSpec;
3299
- constructor(refStore, refSpec) {
3300
- this.refStore = refStore;
3301
- this.refSpec = refSpec;
3567
+ #source;
3568
+ #converter;
3569
+ /** The root group; kept outside the LRU caches so it is never evicted. */
3570
+ #root;
3571
+ #nodes;
3572
+ #nodeRefs;
3573
+ #consolidated = null;
3574
+ constructor(source, converter, file, options) {
3575
+ this.#source = source;
3576
+ this.#converter = converter;
3577
+ this.#root = Promise.resolve(file);
3578
+ this.#nodes = new QuickLRU({ maxSize: options.maxCachedNodes ?? 1e3 });
3579
+ this.#nodeRefs = new QuickLRU({ maxSize: options.maxCachedRefs ?? 500 });
3580
+ }
3581
+ /**
3582
+ * @param {AsyncReadable} hdfSource Assumed to be a zarrita Store,
3583
+ * with the HDF5 file located at the root key "/". In other words,
3584
+ * hdfSource.getRange("/", { offset, length }) should return the
3585
+ * corresponding byte range from the HDF5 file.
3586
+ * @returns {Promise<HdfStore>} An HdfStore instance which exposes the HDF5
3587
+ * file as a Zarr v3 hierarchy.
3588
+ * @param {HdfStoreOptions} options Sizes of the in-memory LRU caches.
3589
+ */
3590
+ static async fromStore(hdfSource, options = {}) {
3591
+ const converter = new SingleHdf5ToZarr(hdfSource, { url: null, zarrFormat: 2 });
3592
+ const file = await converter.open();
3593
+ return new HdfStore(hdfSource, converter, file, options);
3302
3594
  }
3303
3595
  async get(key, opts) {
3304
- return this.refStore.get(key, opts);
3596
+ const path = key.replace(/^\/+/, "");
3597
+ const parts = path === "" ? [] : path.split("/");
3598
+ if (parts[parts.length - 1] === "zarr.json") {
3599
+ const nodePath2 = parts.slice(0, -1).join("/");
3600
+ const refs2 = await this.#getNodeRefs(nodePath2, false);
3601
+ return this.#resolveRef(refs2?.[path], opts);
3602
+ }
3603
+ let nodePath = "";
3604
+ for (const part of parts) {
3605
+ const node = await this.#resolveNode(nodePath);
3606
+ if (!node) return void 0;
3607
+ if (node instanceof Dataset) break;
3608
+ nodePath = nodePath ? `${nodePath}/${part}` : part;
3609
+ }
3610
+ const chunkKey = path.slice(nodePath.length + 1);
3611
+ if (nodePath === path || !(chunkKey === "c" || chunkKey.startsWith("c/")) || !(await this.#resolveNode(nodePath) instanceof Dataset)) {
3612
+ return void 0;
3613
+ }
3614
+ const refs = await this.#getNodeRefs(nodePath, true);
3615
+ return this.#resolveRef(refs?.[path], opts);
3305
3616
  }
3306
- // TODO: does it make sense to support getRange here?
3307
3617
  /**
3308
3618
  * @returns The root zarr.json document with inline consolidated metadata
3309
3619
  * (zarr-python format) describing every group and array in the hierarchy.
3620
+ * Reads the metadata of every node in the file (but no chunk data); nodes
3621
+ * already opened through this store are not read again.
3310
3622
  */
3311
3623
  getConsolidatedMetadata() {
3312
- return refSpecToConsolidatedMetadataV3(this.refSpec);
3624
+ if (!this.#consolidated) {
3625
+ this.#consolidated = this.#collectMetadata("").then(
3626
+ (parts) => refSpecToConsolidatedMetadataV3({ refs: Object.assign({}, ...parts) })
3627
+ );
3628
+ this.#consolidated.catch(() => {
3629
+ this.#consolidated = null;
3630
+ });
3631
+ }
3632
+ return this.#consolidated;
3313
3633
  }
3314
3634
  /**
3315
- *
3316
- * @param {AsyncReadable} hdfSource Assumed to be a zarrita Store,
3317
- * with the HDF5 file located at the root key "/". In other words,
3318
- * hdfSource.getRange("/", { offset, length }) should return the
3319
- * corresponding byte range from the HDF5 file.
3320
- * @returns {Promise<HdfStore>} An HdfStore instance which exposes the HDF5
3321
- * file as a Zarr v3 hierarchy. Internally uses a ReferenceStore and handles
3322
- * the Reference Specification JSON generation.
3635
+ * Metadata refs for a node and all of its descendants, in depth-first order.
3636
+ * Sibling subtrees are visited concurrently.
3637
+ */
3638
+ async #collectMetadata(path) {
3639
+ const [node, own] = await Promise.all([
3640
+ this.#resolveNode(path),
3641
+ this.#getNodeRefs(path, false)
3642
+ ]);
3643
+ if (!(node instanceof Group)) return [own];
3644
+ const children = await Promise.all(
3645
+ node.keys.map((key) => this.#collectMetadata(path ? `${path}/${key}` : key))
3646
+ );
3647
+ return [own, ...children.flat()];
3648
+ }
3649
+ /**
3650
+ * Resolve the HDF5 group or dataset at a hierarchy path (cached).
3651
+ * Returns null if there is no such node.
3323
3652
  */
3324
- static async fromStore(hdfSource) {
3325
- const converter = new SingleHdf5ToZarr(hdfSource, { url: null, zarrFormat: 3 });
3326
- const refSpec = await converter.translate();
3327
- const refStore = ComposableReferenceStore.fromSpec(refSpec, hdfSource);
3328
- return new HdfStore(refStore, refSpec);
3653
+ #resolveNode(path) {
3654
+ if (path === "") return this.#root;
3655
+ let node = this.#nodes.get(path);
3656
+ if (!node) {
3657
+ const idx = path.lastIndexOf("/");
3658
+ const parentPath = idx === -1 ? "" : path.slice(0, idx);
3659
+ const name = path.slice(idx + 1);
3660
+ node = this.#resolveNode(parentPath).then(async (parent) => {
3661
+ if (!(parent instanceof Group) || !(name in parent._links)) return null;
3662
+ try {
3663
+ return await parent.get(name);
3664
+ } catch {
3665
+ return null;
3666
+ }
3667
+ });
3668
+ evictOnReject(this.#nodes, path, node);
3669
+ this.#nodes.set(path, node);
3670
+ }
3671
+ return node;
3672
+ }
3673
+ /**
3674
+ * Zarr v3 references for a single node (cached). Metadata-only refs contain
3675
+ * just the node's zarr.json; with `chunks`, its chunk references too.
3676
+ */
3677
+ #getNodeRefs(path, chunks) {
3678
+ const cacheKey = `${chunks ? "c" : "m"}:${path}`;
3679
+ let refs = this.#nodeRefs.get(cacheKey);
3680
+ if (!refs) {
3681
+ refs = (async () => {
3682
+ const node = await this.#resolveNode(path);
3683
+ if (!node) return {};
3684
+ const v2Refs = await this.#converter.translateNode(path, node, { chunks });
3685
+ return refSpecV2ToV3({ refs: v2Refs }).refs;
3686
+ })();
3687
+ evictOnReject(this.#nodeRefs, cacheKey, refs);
3688
+ this.#nodeRefs.set(cacheKey, refs);
3689
+ }
3690
+ return refs;
3691
+ }
3692
+ async #resolveRef(ref, opts) {
3693
+ if (ref === void 0) return void 0;
3694
+ if (typeof ref === "string") {
3695
+ return ref.startsWith("base64:") ? to_binary(ref.slice("base64:".length)) : new TextEncoder().encode(ref);
3696
+ }
3697
+ const [, offset, length] = ref;
3698
+ return this.#source.getRange("/", { offset, length }, opts);
3329
3699
  }
3330
3700
  }
3331
3701
  export {