@nirs4all/formats-wasm 0.2.12 → 0.2.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/THIRD_PARTY_NOTICES.md +49 -0
  2. package/nirs4all_formats_wasm.d.ts +4 -4
  3. package/nirs4all_formats_wasm.js +170 -149
  4. package/nirs4all_formats_wasm_bg.wasm +0 -0
  5. package/package.json +4 -2
  6. package/reader-backports/hdf5-reader/Cargo.toml +119 -0
  7. package/reader-backports/hdf5-reader/LICENSE-APACHE +164 -0
  8. package/reader-backports/hdf5-reader/LICENSE-MIT +21 -0
  9. package/reader-backports/hdf5-reader/PROVENANCE.json +80 -0
  10. package/reader-backports/hdf5-reader/README.md +308 -0
  11. package/reader-backports/hdf5-reader/UPSTREAM-Cargo.toml +44 -0
  12. package/reader-backports/hdf5-reader/UPSTREAM-VCS.json +6 -0
  13. package/reader-backports/hdf5-reader/src/attribute_api.rs +847 -0
  14. package/reader-backports/hdf5-reader/src/btree_v1.rs +726 -0
  15. package/reader-backports/hdf5-reader/src/btree_v2.rs +2068 -0
  16. package/reader-backports/hdf5-reader/src/cache.rs +354 -0
  17. package/reader-backports/hdf5-reader/src/checksum.rs +56 -0
  18. package/reader-backports/hdf5-reader/src/chunk_index.rs +423 -0
  19. package/reader-backports/hdf5-reader/src/dataset.rs +4733 -0
  20. package/reader-backports/hdf5-reader/src/datatype_api.rs +497 -0
  21. package/reader-backports/hdf5-reader/src/error.rs +167 -0
  22. package/reader-backports/hdf5-reader/src/extensible_array.rs +1886 -0
  23. package/reader-backports/hdf5-reader/src/filters/deflate.rs +93 -0
  24. package/reader-backports/hdf5-reader/src/filters/fletcher32.rs +52 -0
  25. package/reader-backports/hdf5-reader/src/filters/lz4.rs +178 -0
  26. package/reader-backports/hdf5-reader/src/filters/mod.rs +326 -0
  27. package/reader-backports/hdf5-reader/src/filters/nbit.rs +586 -0
  28. package/reader-backports/hdf5-reader/src/filters/scaleoffset.rs +827 -0
  29. package/reader-backports/hdf5-reader/src/filters/shuffle.rs +99 -0
  30. package/reader-backports/hdf5-reader/src/fixed_array.rs +763 -0
  31. package/reader-backports/hdf5-reader/src/fractal_heap.rs +2029 -0
  32. package/reader-backports/hdf5-reader/src/global_heap.rs +284 -0
  33. package/reader-backports/hdf5-reader/src/group.rs +795 -0
  34. package/reader-backports/hdf5-reader/src/io.rs +413 -0
  35. package/reader-backports/hdf5-reader/src/lib.rs +916 -0
  36. package/reader-backports/hdf5-reader/src/local_heap.rs +261 -0
  37. package/reader-backports/hdf5-reader/src/messages/attribute.rs +304 -0
  38. package/reader-backports/hdf5-reader/src/messages/attribute_info.rs +96 -0
  39. package/reader-backports/hdf5-reader/src/messages/btree_k.rs +36 -0
  40. package/reader-backports/hdf5-reader/src/messages/continuation.rs +46 -0
  41. package/reader-backports/hdf5-reader/src/messages/dataspace.rs +259 -0
  42. package/reader-backports/hdf5-reader/src/messages/datatype.rs +938 -0
  43. package/reader-backports/hdf5-reader/src/messages/external_files.rs +50 -0
  44. package/reader-backports/hdf5-reader/src/messages/fill_value.rs +233 -0
  45. package/reader-backports/hdf5-reader/src/messages/filter_pipeline.rs +147 -0
  46. package/reader-backports/hdf5-reader/src/messages/group_info.rs +103 -0
  47. package/reader-backports/hdf5-reader/src/messages/layout.rs +476 -0
  48. package/reader-backports/hdf5-reader/src/messages/link.rs +170 -0
  49. package/reader-backports/hdf5-reader/src/messages/link_info.rs +127 -0
  50. package/reader-backports/hdf5-reader/src/messages/mod.rs +374 -0
  51. package/reader-backports/hdf5-reader/src/messages/modification_time.rs +83 -0
  52. package/reader-backports/hdf5-reader/src/messages/shared.rs +164 -0
  53. package/reader-backports/hdf5-reader/src/messages/shared_table.rs +67 -0
  54. package/reader-backports/hdf5-reader/src/messages/symbol_table_msg.rs +51 -0
  55. package/reader-backports/hdf5-reader/src/object_header.rs +1749 -0
  56. package/reader-backports/hdf5-reader/src/reference.rs +90 -0
  57. package/reader-backports/hdf5-reader/src/shared_message_table.rs +425 -0
  58. package/reader-backports/hdf5-reader/src/storage.rs +538 -0
  59. package/reader-backports/hdf5-reader/src/superblock.rs +285 -0
  60. package/reader-backports/hdf5-reader/src/symbol_table.rs +277 -0
  61. package/reader-backports/netcdf-reader/Cargo.toml +141 -0
  62. package/reader-backports/netcdf-reader/LICENSE-APACHE +164 -0
  63. package/reader-backports/netcdf-reader/LICENSE-MIT +21 -0
  64. package/reader-backports/netcdf-reader/PROVENANCE.json +55 -0
  65. package/reader-backports/netcdf-reader/README.md +308 -0
  66. package/reader-backports/netcdf-reader/UPSTREAM-Cargo.toml +48 -0
  67. package/reader-backports/netcdf-reader/UPSTREAM-VCS.json +6 -0
  68. package/reader-backports/netcdf-reader/src/cf/axes.rs +419 -0
  69. package/reader-backports/netcdf-reader/src/cf/bounds.rs +108 -0
  70. package/reader-backports/netcdf-reader/src/cf/crs.rs +269 -0
  71. package/reader-backports/netcdf-reader/src/cf/mod.rs +30 -0
  72. package/reader-backports/netcdf-reader/src/cf/time.rs +1122 -0
  73. package/reader-backports/netcdf-reader/src/classic/data.rs +1465 -0
  74. package/reader-backports/netcdf-reader/src/classic/header.rs +1097 -0
  75. package/reader-backports/netcdf-reader/src/classic/mod.rs +240 -0
  76. package/reader-backports/netcdf-reader/src/classic/storage.rs +211 -0
  77. package/reader-backports/netcdf-reader/src/classic/types.rs +93 -0
  78. package/reader-backports/netcdf-reader/src/classic/variable.rs +1162 -0
  79. package/reader-backports/netcdf-reader/src/error.rs +59 -0
  80. package/reader-backports/netcdf-reader/src/lib.rs +1531 -0
  81. package/reader-backports/netcdf-reader/src/masked.rs +173 -0
  82. package/reader-backports/netcdf-reader/src/nc4/attributes.rs +203 -0
  83. package/reader-backports/netcdf-reader/src/nc4/dimensions.rs +139 -0
  84. package/reader-backports/netcdf-reader/src/nc4/groups.rs +308 -0
  85. package/reader-backports/netcdf-reader/src/nc4/mod.rs +572 -0
  86. package/reader-backports/netcdf-reader/src/nc4/types.rs +309 -0
  87. package/reader-backports/netcdf-reader/src/nc4/variables.rs +420 -0
  88. package/reader-backports/netcdf-reader/src/types.rs +18 -0
  89. package/reader-backports/netcdf-reader/src/unpack.rs +113 -0
  90. package/reader-backports/netcdf-reader/src/user_defined.rs +569 -0
@@ -0,0 +1,763 @@
1
+ //! HDF5 Fixed Array (FA) chunk index.
2
+ //!
3
+ //! This is the default chunk index for fixed-size chunked datasets created with
4
+ //! `libver='latest'`. It stores chunk entries in a flat array (optionally paged)
5
+ //! with a single-level header → data block structure.
6
+ //!
7
+ //! Structures:
8
+ //! - `FAHD` — Fixed Array Header
9
+ //! - `FADB` — Fixed Array Data Block
10
+
11
+ use crate::checksum::jenkins_lookup3;
12
+ use crate::chunk_index::ChunkEntry;
13
+ use crate::error::{Error, Result};
14
+ use crate::io::Cursor;
15
+ use crate::storage::Storage;
16
+
17
+ const FAHD_SIGNATURE: [u8; 4] = *b"FAHD";
18
+ const FADB_SIGNATURE: [u8; 4] = *b"FADB";
19
+
20
+ /// Parsed Fixed Array Header.
21
+ #[derive(Debug)]
22
+ struct FaHeader {
23
+ client_id: u8,
24
+ entry_size: u8,
25
+ page_bits: u8,
26
+ num_entries: u64,
27
+ data_block_address: u64,
28
+ }
29
+
30
+ /// Parse the Fixed Array Header at the given address.
31
+ ///
32
+ /// On-disk layout (from H5FA_HEADER_SIZE):
33
+ /// sig(4) + ver(1) + client_id(1) + entry_size(1) + page_bits(1)
34
+ /// + nelmts(length_size) + dblk_addr(offset_size) + checksum(4)
35
+ fn parse_header(data: &[u8], address: u64, offset_size: u8, length_size: u8) -> Result<FaHeader> {
36
+ let mut cursor = Cursor::new(data);
37
+ cursor.set_position(address);
38
+
39
+ let sig = cursor.read_bytes(4)?;
40
+ if sig != FAHD_SIGNATURE {
41
+ return Err(Error::InvalidFixedArraySignature {
42
+ context: "header signature mismatch",
43
+ });
44
+ }
45
+
46
+ let version = cursor.read_u8()?;
47
+ if version != 0 {
48
+ return Err(Error::Other(format!(
49
+ "unsupported fixed array header version {}",
50
+ version
51
+ )));
52
+ }
53
+
54
+ let client_id = cursor.read_u8()?;
55
+ let entry_size = cursor.read_u8()?;
56
+ let page_bits = cursor.read_u8()?;
57
+ let num_entries = cursor.read_length(length_size)?;
58
+ let data_block_address = cursor.read_offset(offset_size)?;
59
+
60
+ // Checksum covers everything from signature through data_block_address.
61
+ let header_end = cursor.position();
62
+ let header_bytes = &data[address as usize..header_end as usize];
63
+ let stored_checksum = cursor.read_u32_le()?;
64
+ let computed = jenkins_lookup3(header_bytes);
65
+ if stored_checksum != computed {
66
+ return Err(Error::ChecksumMismatch {
67
+ expected: stored_checksum,
68
+ actual: computed,
69
+ });
70
+ }
71
+
72
+ Ok(FaHeader {
73
+ client_id,
74
+ entry_size,
75
+ page_bits,
76
+ num_entries,
77
+ data_block_address,
78
+ })
79
+ }
80
+
81
+ fn parse_header_storage(
82
+ storage: &dyn Storage,
83
+ address: u64,
84
+ offset_size: u8,
85
+ length_size: u8,
86
+ ) -> Result<FaHeader> {
87
+ let header_len = 4 + 1 + 1 + 1 + 1 + usize::from(length_size) + usize::from(offset_size) + 4;
88
+ let bytes = storage.read_range(address, header_len)?;
89
+ parse_header(bytes.as_ref(), 0, offset_size, length_size)
90
+ }
91
+
92
+ /// A single raw fixed-array entry (before conversion to ChunkEntry).
93
+ #[derive(Debug)]
94
+ struct FaRawEntry {
95
+ address: u64,
96
+ chunk_size: u64,
97
+ filter_mask: u32,
98
+ }
99
+
100
+ /// Read entries from a Fixed Array Data Block.
101
+ fn parse_data_block(
102
+ data: &[u8],
103
+ address: u64,
104
+ header: &FaHeader,
105
+ offset_size: u8,
106
+ ) -> Result<Vec<FaRawEntry>> {
107
+ let mut cursor = Cursor::new(data);
108
+ cursor.set_position(address);
109
+
110
+ let sig = cursor.read_bytes(4)?;
111
+ if sig != FADB_SIGNATURE {
112
+ return Err(Error::InvalidFixedArraySignature {
113
+ context: "data block signature mismatch",
114
+ });
115
+ }
116
+
117
+ let version = cursor.read_u8()?;
118
+ if version != 0 {
119
+ return Err(Error::Other(format!(
120
+ "unsupported fixed array data block version {}",
121
+ version
122
+ )));
123
+ }
124
+
125
+ let _client_id = cursor.read_u8()?;
126
+ let _header_address = cursor.read_offset(offset_size)?;
127
+
128
+ let num_entries = header.num_entries as usize;
129
+ let is_filtered = header.client_id == 1;
130
+
131
+ // Paging is used only when nelmts exceeds 2^page_bits.
132
+ let use_paging = header.page_bits > 0 && num_entries > (1usize << header.page_bits);
133
+
134
+ if !use_paging {
135
+ // Non-paged: all entries inline followed by a single checksum.
136
+ let entries = read_entries(
137
+ &mut cursor,
138
+ num_entries,
139
+ is_filtered,
140
+ offset_size,
141
+ header.entry_size,
142
+ )?;
143
+ // Skip the trailing checksum (already verified structurally).
144
+ let _checksum = cursor.read_u32_le()?;
145
+ Ok(entries)
146
+ } else {
147
+ // Paged: entries are split into pages of `2^page_bits` entries each.
148
+ let entries_per_page = 1usize << header.page_bits;
149
+ let num_pages = num_entries.div_ceil(entries_per_page);
150
+
151
+ // Page init bitmap: ceil(num_pages / 8) bytes — tells which pages
152
+ // have been initialized. We read all pages regardless (uninitialized
153
+ // pages have undefined addresses that we filter out later).
154
+ let bitmap_bytes = num_pages.div_ceil(8);
155
+ let page_bitmap = cursor.read_bytes(bitmap_bytes)?.to_vec();
156
+
157
+ let mut all_entries = Vec::with_capacity(num_entries);
158
+
159
+ for page_idx in 0..num_pages {
160
+ let byte_idx = page_idx / 8;
161
+ let bit_idx = page_idx % 8;
162
+ let page_initialized =
163
+ byte_idx < page_bitmap.len() && (page_bitmap[byte_idx] & (1 << bit_idx)) != 0;
164
+
165
+ let entries_in_this_page = if page_idx == num_pages - 1 {
166
+ let remainder = num_entries % entries_per_page;
167
+ if remainder == 0 {
168
+ entries_per_page
169
+ } else {
170
+ remainder
171
+ }
172
+ } else {
173
+ entries_per_page
174
+ };
175
+
176
+ if page_initialized {
177
+ let page_entries = read_entries(
178
+ &mut cursor,
179
+ entries_in_this_page,
180
+ is_filtered,
181
+ offset_size,
182
+ header.entry_size,
183
+ )?;
184
+ // Each page has its own checksum.
185
+ let _page_checksum = cursor.read_u32_le()?;
186
+ all_entries.extend(page_entries);
187
+ } else {
188
+ // Uninitialized page — fill with undefined entries.
189
+ for _ in 0..entries_in_this_page {
190
+ all_entries.push(FaRawEntry {
191
+ address: u64::MAX,
192
+ chunk_size: 0,
193
+ filter_mask: 0,
194
+ });
195
+ }
196
+ }
197
+ }
198
+
199
+ Ok(all_entries)
200
+ }
201
+ }
202
+
203
+ /// Read `count` entries from the cursor.
204
+ fn read_entries(
205
+ cursor: &mut Cursor<'_>,
206
+ count: usize,
207
+ is_filtered: bool,
208
+ offset_size: u8,
209
+ entry_size: u8,
210
+ ) -> Result<Vec<FaRawEntry>> {
211
+ let mut entries = Vec::with_capacity(count);
212
+ for _ in 0..count {
213
+ let address = cursor.read_offset(offset_size)?;
214
+ let (chunk_size, filter_mask) = if is_filtered {
215
+ let chunk_size_len = entry_size
216
+ .checked_sub(offset_size)
217
+ .and_then(|remaining| remaining.checked_sub(4))
218
+ .ok_or_else(|| Error::InvalidData("invalid fixed array entry size".into()))?;
219
+ // 1–8 bytes wide (sized to the chunk), not a "length" of the superblock size.
220
+ let cs = cursor.read_uvar(chunk_size_len as usize)?;
221
+ let fm = cursor.read_u32_le()?;
222
+ (cs, fm)
223
+ } else {
224
+ (0, 0)
225
+ };
226
+ entries.push(FaRawEntry {
227
+ address,
228
+ chunk_size,
229
+ filter_mask,
230
+ });
231
+ }
232
+ Ok(entries)
233
+ }
234
+
235
+ fn read_entry_at(
236
+ data: &[u8],
237
+ position: u64,
238
+ is_filtered: bool,
239
+ offset_size: u8,
240
+ entry_size: u8,
241
+ ) -> Result<FaRawEntry> {
242
+ let mut cursor = Cursor::new(data);
243
+ cursor.set_position(position);
244
+ let mut entries = read_entries(&mut cursor, 1, is_filtered, offset_size, entry_size)?;
245
+ entries
246
+ .pop()
247
+ .ok_or_else(|| Error::InvalidData("missing fixed array entry".into()))
248
+ }
249
+
250
+ fn read_entry_at_storage(
251
+ storage: &dyn Storage,
252
+ position: u64,
253
+ is_filtered: bool,
254
+ offset_size: u8,
255
+ entry_size: u8,
256
+ ) -> Result<FaRawEntry> {
257
+ let bytes = storage.read_range(position, usize::from(entry_size))?;
258
+ let mut cursor = Cursor::new(bytes.as_ref());
259
+ let mut entries = read_entries(&mut cursor, 1, is_filtered, offset_size, entry_size)?;
260
+ entries
261
+ .pop()
262
+ .ok_or_else(|| Error::InvalidData("missing fixed array entry".into()))
263
+ }
264
+
265
+ fn linear_target_offsets(
266
+ dataset_shape: &[u64],
267
+ chunk_dims: &[u32],
268
+ chunk_bounds: Option<(&[u64], &[u64])>,
269
+ ) -> Vec<(usize, Vec<u64>)> {
270
+ let ndim = dataset_shape.len();
271
+ let chunks_per_dim: Vec<u64> = (0..ndim)
272
+ .map(|i| dataset_shape[i].div_ceil(chunk_dims[i] as u64))
273
+ .collect();
274
+
275
+ if ndim == 0 {
276
+ return vec![(0, Vec::new())];
277
+ }
278
+
279
+ let (first_chunk, last_chunk): (Vec<u64>, Vec<u64>) = match chunk_bounds {
280
+ Some((first, last)) => (first.to_vec(), last.to_vec()),
281
+ None => (
282
+ vec![0u64; ndim],
283
+ chunks_per_dim
284
+ .iter()
285
+ .map(|count| count.saturating_sub(1))
286
+ .collect(),
287
+ ),
288
+ };
289
+
290
+ let mut targets = Vec::new();
291
+ let mut chunk_indices = first_chunk.clone();
292
+ loop {
293
+ let mut linear_idx = 0u64;
294
+ for (dim, chunk_index) in chunk_indices.iter().enumerate() {
295
+ linear_idx = linear_idx * chunks_per_dim[dim] + chunk_index;
296
+ }
297
+ let offsets = chunk_indices
298
+ .iter()
299
+ .enumerate()
300
+ .map(|(dim, chunk_index)| chunk_index * u64::from(chunk_dims[dim]))
301
+ .collect();
302
+ targets.push((linear_idx as usize, offsets));
303
+
304
+ let mut advanced = false;
305
+ for dim in (0..ndim).rev() {
306
+ if chunk_indices[dim] < last_chunk[dim] {
307
+ chunk_indices[dim] += 1;
308
+ if dim + 1 < ndim {
309
+ chunk_indices[(dim + 1)..ndim].copy_from_slice(&first_chunk[(dim + 1)..ndim]);
310
+ }
311
+ advanced = true;
312
+ break;
313
+ }
314
+ }
315
+
316
+ if !advanced {
317
+ break;
318
+ }
319
+ }
320
+
321
+ targets
322
+ }
323
+
324
+ fn collect_fixed_array_chunk_entries_bounded(
325
+ data: &[u8],
326
+ header: &FaHeader,
327
+ offset_size: u8,
328
+ dataset_shape: &[u64],
329
+ chunk_dims: &[u32],
330
+ chunk_bounds: (&[u64], &[u64]),
331
+ ) -> Result<Vec<ChunkEntry>> {
332
+ let targets = linear_target_offsets(dataset_shape, chunk_dims, Some(chunk_bounds));
333
+ let mut cursor = Cursor::new(data);
334
+ cursor.set_position(header.data_block_address);
335
+
336
+ let sig = cursor.read_bytes(4)?;
337
+ if sig != FADB_SIGNATURE {
338
+ return Err(Error::InvalidFixedArraySignature {
339
+ context: "data block signature mismatch",
340
+ });
341
+ }
342
+
343
+ let version = cursor.read_u8()?;
344
+ if version != 0 {
345
+ return Err(Error::Other(format!(
346
+ "unsupported fixed array data block version {}",
347
+ version
348
+ )));
349
+ }
350
+
351
+ let _client_id = cursor.read_u8()?;
352
+ let _header_address = cursor.read_offset(offset_size)?;
353
+
354
+ let num_entries = header.num_entries as usize;
355
+ let is_filtered = header.client_id == 1;
356
+ let entry_bytes = header.entry_size as usize;
357
+ let use_paging = header.page_bits > 0 && num_entries > (1usize << header.page_bits);
358
+
359
+ if !use_paging {
360
+ let entries_start = cursor.position();
361
+ let mut entries = Vec::new();
362
+ for (linear_idx, offsets) in targets {
363
+ let position = entries_start + (linear_idx * entry_bytes) as u64;
364
+ let raw = read_entry_at(data, position, is_filtered, offset_size, header.entry_size)?;
365
+ if Cursor::is_undefined_offset(raw.address, offset_size) {
366
+ continue;
367
+ }
368
+ entries.push(ChunkEntry {
369
+ address: raw.address,
370
+ size: raw.chunk_size,
371
+ filter_mask: raw.filter_mask,
372
+ offsets,
373
+ });
374
+ }
375
+ return Ok(entries);
376
+ }
377
+
378
+ let entries_per_page = 1usize << header.page_bits;
379
+ let num_pages = num_entries.div_ceil(entries_per_page);
380
+ let bitmap_bytes = num_pages.div_ceil(8);
381
+ let page_bitmap = cursor.read_bytes(bitmap_bytes)?.to_vec();
382
+ let pages_start = cursor.position();
383
+
384
+ let mut page_offsets = vec![None; num_pages];
385
+ let mut next_page_start = pages_start;
386
+ for (page_idx, page_offset) in page_offsets.iter_mut().enumerate().take(num_pages) {
387
+ let byte_idx = page_idx / 8;
388
+ let bit_idx = page_idx % 8;
389
+ let page_initialized =
390
+ byte_idx < page_bitmap.len() && (page_bitmap[byte_idx] & (1 << bit_idx)) != 0;
391
+
392
+ let entries_in_page = if page_idx == num_pages - 1 {
393
+ let remainder = num_entries % entries_per_page;
394
+ if remainder == 0 {
395
+ entries_per_page
396
+ } else {
397
+ remainder
398
+ }
399
+ } else {
400
+ entries_per_page
401
+ };
402
+
403
+ if page_initialized {
404
+ *page_offset = Some(next_page_start);
405
+ next_page_start += (entries_in_page * entry_bytes + 4) as u64;
406
+ }
407
+ }
408
+
409
+ let mut entries = Vec::new();
410
+ for (linear_idx, offsets) in targets {
411
+ let page_idx = linear_idx / entries_per_page;
412
+ let within_page = linear_idx % entries_per_page;
413
+ let Some(page_start) = page_offsets[page_idx] else {
414
+ continue;
415
+ };
416
+ let position = page_start + (within_page * entry_bytes) as u64;
417
+ let raw = read_entry_at(data, position, is_filtered, offset_size, header.entry_size)?;
418
+ if Cursor::is_undefined_offset(raw.address, offset_size) {
419
+ continue;
420
+ }
421
+ entries.push(ChunkEntry {
422
+ address: raw.address,
423
+ size: raw.chunk_size,
424
+ filter_mask: raw.filter_mask,
425
+ offsets,
426
+ });
427
+ }
428
+
429
+ Ok(entries)
430
+ }
431
+
432
+ fn collect_fixed_array_chunk_entries_bounded_storage(
433
+ storage: &dyn Storage,
434
+ header: &FaHeader,
435
+ offset_size: u8,
436
+ dataset_shape: &[u64],
437
+ chunk_dims: &[u32],
438
+ chunk_bounds: (&[u64], &[u64]),
439
+ ) -> Result<Vec<ChunkEntry>> {
440
+ let targets = linear_target_offsets(dataset_shape, chunk_dims, Some(chunk_bounds));
441
+ let block_header_len = 4 + 1 + 1 + usize::from(offset_size);
442
+ let header_bytes = storage.read_range(header.data_block_address, block_header_len)?;
443
+ let mut cursor = Cursor::new(header_bytes.as_ref());
444
+
445
+ let sig = cursor.read_bytes(4)?;
446
+ if sig != FADB_SIGNATURE {
447
+ return Err(Error::InvalidFixedArraySignature {
448
+ context: "data block signature mismatch",
449
+ });
450
+ }
451
+
452
+ let version = cursor.read_u8()?;
453
+ if version != 0 {
454
+ return Err(Error::Other(format!(
455
+ "unsupported fixed array data block version {}",
456
+ version
457
+ )));
458
+ }
459
+
460
+ let _client_id = cursor.read_u8()?;
461
+ let _header_address = cursor.read_offset(offset_size)?;
462
+
463
+ let num_entries = usize::try_from(header.num_entries).map_err(|_| {
464
+ Error::InvalidData("fixed array entry count exceeds platform usize capacity".into())
465
+ })?;
466
+ let is_filtered = header.client_id == 1;
467
+ let entry_bytes = usize::from(header.entry_size);
468
+ let use_paging = header.page_bits > 0 && num_entries > (1usize << header.page_bits);
469
+ let entries_start = header.data_block_address
470
+ + u64::try_from(block_header_len)
471
+ .map_err(|_| Error::OffsetOutOfBounds(header.data_block_address))?;
472
+
473
+ if !use_paging {
474
+ let mut entries = Vec::new();
475
+ for (linear_idx, offsets) in targets {
476
+ let position = entries_start
477
+ + u64::try_from(linear_idx * entry_bytes).map_err(|_| {
478
+ Error::InvalidData("fixed array entry offset exceeds u64 capacity".into())
479
+ })?;
480
+ let raw = read_entry_at_storage(
481
+ storage,
482
+ position,
483
+ is_filtered,
484
+ offset_size,
485
+ header.entry_size,
486
+ )?;
487
+ if Cursor::is_undefined_offset(raw.address, offset_size) {
488
+ continue;
489
+ }
490
+ entries.push(ChunkEntry {
491
+ address: raw.address,
492
+ size: raw.chunk_size,
493
+ filter_mask: raw.filter_mask,
494
+ offsets,
495
+ });
496
+ }
497
+ return Ok(entries);
498
+ }
499
+
500
+ let entries_per_page = 1usize << header.page_bits;
501
+ let num_pages = num_entries.div_ceil(entries_per_page);
502
+ let bitmap_bytes = num_pages.div_ceil(8);
503
+ let page_bitmap = storage.read_range(entries_start, bitmap_bytes)?;
504
+ let pages_start = entries_start
505
+ + u64::try_from(bitmap_bytes).map_err(|_| {
506
+ Error::InvalidData("fixed array bitmap size exceeds u64 capacity".into())
507
+ })?;
508
+
509
+ let mut page_offsets = vec![None; num_pages];
510
+ let mut next_page_start = pages_start;
511
+ for (page_idx, page_offset) in page_offsets.iter_mut().enumerate().take(num_pages) {
512
+ let byte_idx = page_idx / 8;
513
+ let bit_idx = page_idx % 8;
514
+ let page_initialized =
515
+ byte_idx < page_bitmap.len() && (page_bitmap[byte_idx] & (1 << bit_idx)) != 0;
516
+
517
+ let entries_in_page = if page_idx == num_pages - 1 {
518
+ let remainder = num_entries % entries_per_page;
519
+ if remainder == 0 {
520
+ entries_per_page
521
+ } else {
522
+ remainder
523
+ }
524
+ } else {
525
+ entries_per_page
526
+ };
527
+
528
+ if page_initialized {
529
+ *page_offset = Some(next_page_start);
530
+ next_page_start += u64::try_from(entries_in_page * entry_bytes + 4).map_err(|_| {
531
+ Error::InvalidData("fixed array page size exceeds u64 capacity".into())
532
+ })?;
533
+ }
534
+ }
535
+
536
+ let mut entries = Vec::new();
537
+ for (linear_idx, offsets) in targets {
538
+ let page_idx = linear_idx / entries_per_page;
539
+ let within_page = linear_idx % entries_per_page;
540
+ let Some(page_start) = page_offsets[page_idx] else {
541
+ continue;
542
+ };
543
+ let position = page_start
544
+ + u64::try_from(within_page * entry_bytes).map_err(|_| {
545
+ Error::InvalidData("fixed array page entry offset exceeds u64 capacity".into())
546
+ })?;
547
+ let raw = read_entry_at_storage(
548
+ storage,
549
+ position,
550
+ is_filtered,
551
+ offset_size,
552
+ header.entry_size,
553
+ )?;
554
+ if Cursor::is_undefined_offset(raw.address, offset_size) {
555
+ continue;
556
+ }
557
+ entries.push(ChunkEntry {
558
+ address: raw.address,
559
+ size: raw.chunk_size,
560
+ filter_mask: raw.filter_mask,
561
+ offsets,
562
+ });
563
+ }
564
+
565
+ Ok(entries)
566
+ }
567
+
568
+ /// Collect chunk entries from a Fixed Array index.
569
+ ///
570
+ /// Reads the FAHD header and FADB data block, then converts linear entry
571
+ /// indices to multi-dimensional chunk offsets.
572
+ pub fn collect_fixed_array_chunk_entries(
573
+ data: &[u8],
574
+ header_address: u64,
575
+ offset_size: u8,
576
+ length_size: u8,
577
+ dataset_shape: &[u64],
578
+ chunk_dims: &[u32],
579
+ chunk_bounds: Option<(&[u64], &[u64])>,
580
+ ) -> Result<Vec<ChunkEntry>> {
581
+ let header = parse_header(data, header_address, offset_size, length_size)?;
582
+
583
+ if Cursor::is_undefined_offset(header.data_block_address, offset_size) {
584
+ return Ok(Vec::new());
585
+ }
586
+
587
+ if let Some(bounds) = chunk_bounds {
588
+ return collect_fixed_array_chunk_entries_bounded(
589
+ data,
590
+ &header,
591
+ offset_size,
592
+ dataset_shape,
593
+ chunk_dims,
594
+ bounds,
595
+ );
596
+ }
597
+
598
+ let raw_entries = parse_data_block(data, header.data_block_address, &header, offset_size)?;
599
+
600
+ let ndim = dataset_shape.len();
601
+ let chunks_per_dim: Vec<u64> = (0..ndim)
602
+ .map(|i| dataset_shape[i].div_ceil(chunk_dims[i] as u64))
603
+ .collect();
604
+
605
+ let mut entries = Vec::new();
606
+ for (linear_idx, raw) in raw_entries.iter().enumerate() {
607
+ // Skip undefined addresses (unallocated chunks).
608
+ if Cursor::is_undefined_offset(raw.address, offset_size) {
609
+ continue;
610
+ }
611
+
612
+ // Convert linear index to multi-dimensional chunk offsets.
613
+ let mut remaining = linear_idx as u64;
614
+ let mut offsets = vec![0u64; ndim];
615
+ for d in (0..ndim).rev() {
616
+ offsets[d] = (remaining % chunks_per_dim[d]) * chunk_dims[d] as u64;
617
+ remaining /= chunks_per_dim[d];
618
+ }
619
+
620
+ if let Some((first_chunk, last_chunk)) = chunk_bounds {
621
+ let overlaps = offsets.iter().enumerate().all(|(dim, offset)| {
622
+ let chunk_index = *offset / u64::from(chunk_dims[dim]);
623
+ chunk_index >= first_chunk[dim] && chunk_index <= last_chunk[dim]
624
+ });
625
+ if !overlaps {
626
+ continue;
627
+ }
628
+ }
629
+
630
+ entries.push(ChunkEntry {
631
+ address: raw.address,
632
+ size: raw.chunk_size,
633
+ filter_mask: raw.filter_mask,
634
+ offsets,
635
+ });
636
+ }
637
+
638
+ Ok(entries)
639
+ }
640
+
641
+ /// Collect chunk entries from a Fixed Array index using random-access storage.
642
+ pub fn collect_fixed_array_chunk_entries_storage(
643
+ storage: &dyn Storage,
644
+ header_address: u64,
645
+ offset_size: u8,
646
+ length_size: u8,
647
+ dataset_shape: &[u64],
648
+ chunk_dims: &[u32],
649
+ chunk_bounds: Option<(&[u64], &[u64])>,
650
+ ) -> Result<Vec<ChunkEntry>> {
651
+ let header = parse_header_storage(storage, header_address, offset_size, length_size)?;
652
+
653
+ if Cursor::is_undefined_offset(header.data_block_address, offset_size) {
654
+ return Ok(Vec::new());
655
+ }
656
+
657
+ if let Some(bounds) = chunk_bounds {
658
+ return collect_fixed_array_chunk_entries_bounded_storage(
659
+ storage,
660
+ &header,
661
+ offset_size,
662
+ dataset_shape,
663
+ chunk_dims,
664
+ bounds,
665
+ );
666
+ }
667
+
668
+ let num_entries = usize::try_from(header.num_entries).map_err(|_| {
669
+ Error::InvalidData("fixed array entry count exceeds platform usize capacity".into())
670
+ })?;
671
+ let is_filtered = header.client_id == 1;
672
+ let header_len = 4 + 1 + 1 + usize::from(offset_size);
673
+ let use_paging = header.page_bits > 0 && num_entries > (1usize << header.page_bits);
674
+ let block_len = if !use_paging {
675
+ header_len + num_entries * usize::from(header.entry_size) + 4
676
+ } else {
677
+ let entries_per_page = 1usize << header.page_bits;
678
+ let num_pages = num_entries.div_ceil(entries_per_page);
679
+ let bitmap_bytes = num_pages.div_ceil(8);
680
+ let mut len = header_len + bitmap_bytes;
681
+ for page_idx in 0..num_pages {
682
+ let entries_in_page = if page_idx == num_pages - 1 {
683
+ let remainder = num_entries % entries_per_page;
684
+ if remainder == 0 {
685
+ entries_per_page
686
+ } else {
687
+ remainder
688
+ }
689
+ } else {
690
+ entries_per_page
691
+ };
692
+ len += entries_in_page * usize::from(header.entry_size) + 4;
693
+ }
694
+ len
695
+ };
696
+ let block = storage.read_range(header.data_block_address, block_len)?;
697
+ let raw_entries = parse_data_block(block.as_ref(), 0, &header, offset_size)?;
698
+
699
+ let ndim = dataset_shape.len();
700
+ let chunks_per_dim: Vec<u64> = (0..ndim)
701
+ .map(|i| dataset_shape[i].div_ceil(chunk_dims[i] as u64))
702
+ .collect();
703
+
704
+ let mut entries = Vec::new();
705
+ for (linear_idx, raw) in raw_entries.iter().enumerate() {
706
+ if Cursor::is_undefined_offset(raw.address, offset_size) {
707
+ continue;
708
+ }
709
+ let mut remaining = linear_idx as u64;
710
+ let mut offsets = vec![0u64; ndim];
711
+ for d in (0..ndim).rev() {
712
+ offsets[d] = (remaining % chunks_per_dim[d]) * chunk_dims[d] as u64;
713
+ remaining /= chunks_per_dim[d];
714
+ }
715
+
716
+ if let Some((first_chunk, last_chunk)) = chunk_bounds {
717
+ let overlaps = offsets.iter().enumerate().all(|(dim, offset)| {
718
+ let chunk_index = *offset / u64::from(chunk_dims[dim]);
719
+ chunk_index >= first_chunk[dim] && chunk_index <= last_chunk[dim]
720
+ });
721
+ if !overlaps {
722
+ continue;
723
+ }
724
+ }
725
+
726
+ entries.push(ChunkEntry {
727
+ address: raw.address,
728
+ size: if is_filtered { raw.chunk_size } else { 0 },
729
+ filter_mask: raw.filter_mask,
730
+ offsets,
731
+ });
732
+ }
733
+
734
+ Ok(entries)
735
+ }
736
+
737
+ #[cfg(test)]
738
+ mod tests {
739
+ use super::*;
740
+
741
+ #[test]
742
+ fn fahd_bad_signature() {
743
+ let mut data = vec![0u8; 64];
744
+ data[0..4].copy_from_slice(b"XXXX");
745
+ let err = parse_header(&data, 0, 8, 8).unwrap_err();
746
+ assert!(matches!(err, Error::InvalidFixedArraySignature { .. }));
747
+ }
748
+
749
+ #[test]
750
+ fn fadb_bad_signature() {
751
+ let header = FaHeader {
752
+ client_id: 0,
753
+ entry_size: 8,
754
+ page_bits: 0,
755
+ num_entries: 1,
756
+ data_block_address: 0,
757
+ };
758
+ let mut data = vec![0u8; 64];
759
+ data[0..4].copy_from_slice(b"XXXX");
760
+ let err = parse_data_block(&data, 0, &header, 8).unwrap_err();
761
+ assert!(matches!(err, Error::InvalidFixedArraySignature { .. }));
762
+ }
763
+ }