@nirs4all/formats-wasm 0.2.12 → 0.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD_PARTY_NOTICES.md +49 -0
- package/nirs4all_formats_wasm.d.ts +4 -4
- package/nirs4all_formats_wasm.js +170 -149
- package/nirs4all_formats_wasm_bg.wasm +0 -0
- package/package.json +4 -2
- package/reader-backports/hdf5-reader/Cargo.toml +119 -0
- package/reader-backports/hdf5-reader/LICENSE-APACHE +164 -0
- package/reader-backports/hdf5-reader/LICENSE-MIT +21 -0
- package/reader-backports/hdf5-reader/PROVENANCE.json +80 -0
- package/reader-backports/hdf5-reader/README.md +308 -0
- package/reader-backports/hdf5-reader/UPSTREAM-Cargo.toml +44 -0
- package/reader-backports/hdf5-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/hdf5-reader/src/attribute_api.rs +847 -0
- package/reader-backports/hdf5-reader/src/btree_v1.rs +726 -0
- package/reader-backports/hdf5-reader/src/btree_v2.rs +2068 -0
- package/reader-backports/hdf5-reader/src/cache.rs +354 -0
- package/reader-backports/hdf5-reader/src/checksum.rs +56 -0
- package/reader-backports/hdf5-reader/src/chunk_index.rs +423 -0
- package/reader-backports/hdf5-reader/src/dataset.rs +4733 -0
- package/reader-backports/hdf5-reader/src/datatype_api.rs +497 -0
- package/reader-backports/hdf5-reader/src/error.rs +167 -0
- package/reader-backports/hdf5-reader/src/extensible_array.rs +1886 -0
- package/reader-backports/hdf5-reader/src/filters/deflate.rs +93 -0
- package/reader-backports/hdf5-reader/src/filters/fletcher32.rs +52 -0
- package/reader-backports/hdf5-reader/src/filters/lz4.rs +178 -0
- package/reader-backports/hdf5-reader/src/filters/mod.rs +326 -0
- package/reader-backports/hdf5-reader/src/filters/nbit.rs +586 -0
- package/reader-backports/hdf5-reader/src/filters/scaleoffset.rs +827 -0
- package/reader-backports/hdf5-reader/src/filters/shuffle.rs +99 -0
- package/reader-backports/hdf5-reader/src/fixed_array.rs +763 -0
- package/reader-backports/hdf5-reader/src/fractal_heap.rs +2029 -0
- package/reader-backports/hdf5-reader/src/global_heap.rs +284 -0
- package/reader-backports/hdf5-reader/src/group.rs +795 -0
- package/reader-backports/hdf5-reader/src/io.rs +413 -0
- package/reader-backports/hdf5-reader/src/lib.rs +916 -0
- package/reader-backports/hdf5-reader/src/local_heap.rs +261 -0
- package/reader-backports/hdf5-reader/src/messages/attribute.rs +304 -0
- package/reader-backports/hdf5-reader/src/messages/attribute_info.rs +96 -0
- package/reader-backports/hdf5-reader/src/messages/btree_k.rs +36 -0
- package/reader-backports/hdf5-reader/src/messages/continuation.rs +46 -0
- package/reader-backports/hdf5-reader/src/messages/dataspace.rs +259 -0
- package/reader-backports/hdf5-reader/src/messages/datatype.rs +938 -0
- package/reader-backports/hdf5-reader/src/messages/external_files.rs +50 -0
- package/reader-backports/hdf5-reader/src/messages/fill_value.rs +233 -0
- package/reader-backports/hdf5-reader/src/messages/filter_pipeline.rs +147 -0
- package/reader-backports/hdf5-reader/src/messages/group_info.rs +103 -0
- package/reader-backports/hdf5-reader/src/messages/layout.rs +476 -0
- package/reader-backports/hdf5-reader/src/messages/link.rs +170 -0
- package/reader-backports/hdf5-reader/src/messages/link_info.rs +127 -0
- package/reader-backports/hdf5-reader/src/messages/mod.rs +374 -0
- package/reader-backports/hdf5-reader/src/messages/modification_time.rs +83 -0
- package/reader-backports/hdf5-reader/src/messages/shared.rs +164 -0
- package/reader-backports/hdf5-reader/src/messages/shared_table.rs +67 -0
- package/reader-backports/hdf5-reader/src/messages/symbol_table_msg.rs +51 -0
- package/reader-backports/hdf5-reader/src/object_header.rs +1749 -0
- package/reader-backports/hdf5-reader/src/reference.rs +90 -0
- package/reader-backports/hdf5-reader/src/shared_message_table.rs +425 -0
- package/reader-backports/hdf5-reader/src/storage.rs +538 -0
- package/reader-backports/hdf5-reader/src/superblock.rs +285 -0
- package/reader-backports/hdf5-reader/src/symbol_table.rs +277 -0
- package/reader-backports/netcdf-reader/Cargo.toml +141 -0
- package/reader-backports/netcdf-reader/LICENSE-APACHE +164 -0
- package/reader-backports/netcdf-reader/LICENSE-MIT +21 -0
- package/reader-backports/netcdf-reader/PROVENANCE.json +55 -0
- package/reader-backports/netcdf-reader/README.md +308 -0
- package/reader-backports/netcdf-reader/UPSTREAM-Cargo.toml +48 -0
- package/reader-backports/netcdf-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/netcdf-reader/src/cf/axes.rs +419 -0
- package/reader-backports/netcdf-reader/src/cf/bounds.rs +108 -0
- package/reader-backports/netcdf-reader/src/cf/crs.rs +269 -0
- package/reader-backports/netcdf-reader/src/cf/mod.rs +30 -0
- package/reader-backports/netcdf-reader/src/cf/time.rs +1122 -0
- package/reader-backports/netcdf-reader/src/classic/data.rs +1465 -0
- package/reader-backports/netcdf-reader/src/classic/header.rs +1097 -0
- package/reader-backports/netcdf-reader/src/classic/mod.rs +240 -0
- package/reader-backports/netcdf-reader/src/classic/storage.rs +211 -0
- package/reader-backports/netcdf-reader/src/classic/types.rs +93 -0
- package/reader-backports/netcdf-reader/src/classic/variable.rs +1162 -0
- package/reader-backports/netcdf-reader/src/error.rs +59 -0
- package/reader-backports/netcdf-reader/src/lib.rs +1531 -0
- package/reader-backports/netcdf-reader/src/masked.rs +173 -0
- package/reader-backports/netcdf-reader/src/nc4/attributes.rs +203 -0
- package/reader-backports/netcdf-reader/src/nc4/dimensions.rs +139 -0
- package/reader-backports/netcdf-reader/src/nc4/groups.rs +308 -0
- package/reader-backports/netcdf-reader/src/nc4/mod.rs +572 -0
- package/reader-backports/netcdf-reader/src/nc4/types.rs +309 -0
- package/reader-backports/netcdf-reader/src/nc4/variables.rs +420 -0
- package/reader-backports/netcdf-reader/src/types.rs +18 -0
- package/reader-backports/netcdf-reader/src/unpack.rs +113 -0
- package/reader-backports/netcdf-reader/src/user_defined.rs +569 -0
|
@@ -0,0 +1,763 @@
|
|
|
1
|
+
//! HDF5 Fixed Array (FA) chunk index.
|
|
2
|
+
//!
|
|
3
|
+
//! This is the default chunk index for fixed-size chunked datasets created with
|
|
4
|
+
//! `libver='latest'`. It stores chunk entries in a flat array (optionally paged)
|
|
5
|
+
//! with a single-level header → data block structure.
|
|
6
|
+
//!
|
|
7
|
+
//! Structures:
|
|
8
|
+
//! - `FAHD` — Fixed Array Header
|
|
9
|
+
//! - `FADB` — Fixed Array Data Block
|
|
10
|
+
|
|
11
|
+
use crate::checksum::jenkins_lookup3;
|
|
12
|
+
use crate::chunk_index::ChunkEntry;
|
|
13
|
+
use crate::error::{Error, Result};
|
|
14
|
+
use crate::io::Cursor;
|
|
15
|
+
use crate::storage::Storage;
|
|
16
|
+
|
|
17
|
+
const FAHD_SIGNATURE: [u8; 4] = *b"FAHD";
|
|
18
|
+
const FADB_SIGNATURE: [u8; 4] = *b"FADB";
|
|
19
|
+
|
|
20
|
+
/// Parsed Fixed Array Header.
|
|
21
|
+
#[derive(Debug)]
|
|
22
|
+
struct FaHeader {
|
|
23
|
+
client_id: u8,
|
|
24
|
+
entry_size: u8,
|
|
25
|
+
page_bits: u8,
|
|
26
|
+
num_entries: u64,
|
|
27
|
+
data_block_address: u64,
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/// Parse the Fixed Array Header at the given address.
|
|
31
|
+
///
|
|
32
|
+
/// On-disk layout (from H5FA_HEADER_SIZE):
|
|
33
|
+
/// sig(4) + ver(1) + client_id(1) + entry_size(1) + page_bits(1)
|
|
34
|
+
/// + nelmts(length_size) + dblk_addr(offset_size) + checksum(4)
|
|
35
|
+
fn parse_header(data: &[u8], address: u64, offset_size: u8, length_size: u8) -> Result<FaHeader> {
|
|
36
|
+
let mut cursor = Cursor::new(data);
|
|
37
|
+
cursor.set_position(address);
|
|
38
|
+
|
|
39
|
+
let sig = cursor.read_bytes(4)?;
|
|
40
|
+
if sig != FAHD_SIGNATURE {
|
|
41
|
+
return Err(Error::InvalidFixedArraySignature {
|
|
42
|
+
context: "header signature mismatch",
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
let version = cursor.read_u8()?;
|
|
47
|
+
if version != 0 {
|
|
48
|
+
return Err(Error::Other(format!(
|
|
49
|
+
"unsupported fixed array header version {}",
|
|
50
|
+
version
|
|
51
|
+
)));
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
let client_id = cursor.read_u8()?;
|
|
55
|
+
let entry_size = cursor.read_u8()?;
|
|
56
|
+
let page_bits = cursor.read_u8()?;
|
|
57
|
+
let num_entries = cursor.read_length(length_size)?;
|
|
58
|
+
let data_block_address = cursor.read_offset(offset_size)?;
|
|
59
|
+
|
|
60
|
+
// Checksum covers everything from signature through data_block_address.
|
|
61
|
+
let header_end = cursor.position();
|
|
62
|
+
let header_bytes = &data[address as usize..header_end as usize];
|
|
63
|
+
let stored_checksum = cursor.read_u32_le()?;
|
|
64
|
+
let computed = jenkins_lookup3(header_bytes);
|
|
65
|
+
if stored_checksum != computed {
|
|
66
|
+
return Err(Error::ChecksumMismatch {
|
|
67
|
+
expected: stored_checksum,
|
|
68
|
+
actual: computed,
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
Ok(FaHeader {
|
|
73
|
+
client_id,
|
|
74
|
+
entry_size,
|
|
75
|
+
page_bits,
|
|
76
|
+
num_entries,
|
|
77
|
+
data_block_address,
|
|
78
|
+
})
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
fn parse_header_storage(
|
|
82
|
+
storage: &dyn Storage,
|
|
83
|
+
address: u64,
|
|
84
|
+
offset_size: u8,
|
|
85
|
+
length_size: u8,
|
|
86
|
+
) -> Result<FaHeader> {
|
|
87
|
+
let header_len = 4 + 1 + 1 + 1 + 1 + usize::from(length_size) + usize::from(offset_size) + 4;
|
|
88
|
+
let bytes = storage.read_range(address, header_len)?;
|
|
89
|
+
parse_header(bytes.as_ref(), 0, offset_size, length_size)
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/// A single raw fixed-array entry (before conversion to ChunkEntry).
|
|
93
|
+
#[derive(Debug)]
|
|
94
|
+
struct FaRawEntry {
|
|
95
|
+
address: u64,
|
|
96
|
+
chunk_size: u64,
|
|
97
|
+
filter_mask: u32,
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/// Read entries from a Fixed Array Data Block.
|
|
101
|
+
fn parse_data_block(
|
|
102
|
+
data: &[u8],
|
|
103
|
+
address: u64,
|
|
104
|
+
header: &FaHeader,
|
|
105
|
+
offset_size: u8,
|
|
106
|
+
) -> Result<Vec<FaRawEntry>> {
|
|
107
|
+
let mut cursor = Cursor::new(data);
|
|
108
|
+
cursor.set_position(address);
|
|
109
|
+
|
|
110
|
+
let sig = cursor.read_bytes(4)?;
|
|
111
|
+
if sig != FADB_SIGNATURE {
|
|
112
|
+
return Err(Error::InvalidFixedArraySignature {
|
|
113
|
+
context: "data block signature mismatch",
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
let version = cursor.read_u8()?;
|
|
118
|
+
if version != 0 {
|
|
119
|
+
return Err(Error::Other(format!(
|
|
120
|
+
"unsupported fixed array data block version {}",
|
|
121
|
+
version
|
|
122
|
+
)));
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
let _client_id = cursor.read_u8()?;
|
|
126
|
+
let _header_address = cursor.read_offset(offset_size)?;
|
|
127
|
+
|
|
128
|
+
let num_entries = header.num_entries as usize;
|
|
129
|
+
let is_filtered = header.client_id == 1;
|
|
130
|
+
|
|
131
|
+
// Paging is used only when nelmts exceeds 2^page_bits.
|
|
132
|
+
let use_paging = header.page_bits > 0 && num_entries > (1usize << header.page_bits);
|
|
133
|
+
|
|
134
|
+
if !use_paging {
|
|
135
|
+
// Non-paged: all entries inline followed by a single checksum.
|
|
136
|
+
let entries = read_entries(
|
|
137
|
+
&mut cursor,
|
|
138
|
+
num_entries,
|
|
139
|
+
is_filtered,
|
|
140
|
+
offset_size,
|
|
141
|
+
header.entry_size,
|
|
142
|
+
)?;
|
|
143
|
+
// Skip the trailing checksum (already verified structurally).
|
|
144
|
+
let _checksum = cursor.read_u32_le()?;
|
|
145
|
+
Ok(entries)
|
|
146
|
+
} else {
|
|
147
|
+
// Paged: entries are split into pages of `2^page_bits` entries each.
|
|
148
|
+
let entries_per_page = 1usize << header.page_bits;
|
|
149
|
+
let num_pages = num_entries.div_ceil(entries_per_page);
|
|
150
|
+
|
|
151
|
+
// Page init bitmap: ceil(num_pages / 8) bytes — tells which pages
|
|
152
|
+
// have been initialized. We read all pages regardless (uninitialized
|
|
153
|
+
// pages have undefined addresses that we filter out later).
|
|
154
|
+
let bitmap_bytes = num_pages.div_ceil(8);
|
|
155
|
+
let page_bitmap = cursor.read_bytes(bitmap_bytes)?.to_vec();
|
|
156
|
+
|
|
157
|
+
let mut all_entries = Vec::with_capacity(num_entries);
|
|
158
|
+
|
|
159
|
+
for page_idx in 0..num_pages {
|
|
160
|
+
let byte_idx = page_idx / 8;
|
|
161
|
+
let bit_idx = page_idx % 8;
|
|
162
|
+
let page_initialized =
|
|
163
|
+
byte_idx < page_bitmap.len() && (page_bitmap[byte_idx] & (1 << bit_idx)) != 0;
|
|
164
|
+
|
|
165
|
+
let entries_in_this_page = if page_idx == num_pages - 1 {
|
|
166
|
+
let remainder = num_entries % entries_per_page;
|
|
167
|
+
if remainder == 0 {
|
|
168
|
+
entries_per_page
|
|
169
|
+
} else {
|
|
170
|
+
remainder
|
|
171
|
+
}
|
|
172
|
+
} else {
|
|
173
|
+
entries_per_page
|
|
174
|
+
};
|
|
175
|
+
|
|
176
|
+
if page_initialized {
|
|
177
|
+
let page_entries = read_entries(
|
|
178
|
+
&mut cursor,
|
|
179
|
+
entries_in_this_page,
|
|
180
|
+
is_filtered,
|
|
181
|
+
offset_size,
|
|
182
|
+
header.entry_size,
|
|
183
|
+
)?;
|
|
184
|
+
// Each page has its own checksum.
|
|
185
|
+
let _page_checksum = cursor.read_u32_le()?;
|
|
186
|
+
all_entries.extend(page_entries);
|
|
187
|
+
} else {
|
|
188
|
+
// Uninitialized page — fill with undefined entries.
|
|
189
|
+
for _ in 0..entries_in_this_page {
|
|
190
|
+
all_entries.push(FaRawEntry {
|
|
191
|
+
address: u64::MAX,
|
|
192
|
+
chunk_size: 0,
|
|
193
|
+
filter_mask: 0,
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
Ok(all_entries)
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/// Read `count` entries from the cursor.
|
|
204
|
+
fn read_entries(
|
|
205
|
+
cursor: &mut Cursor<'_>,
|
|
206
|
+
count: usize,
|
|
207
|
+
is_filtered: bool,
|
|
208
|
+
offset_size: u8,
|
|
209
|
+
entry_size: u8,
|
|
210
|
+
) -> Result<Vec<FaRawEntry>> {
|
|
211
|
+
let mut entries = Vec::with_capacity(count);
|
|
212
|
+
for _ in 0..count {
|
|
213
|
+
let address = cursor.read_offset(offset_size)?;
|
|
214
|
+
let (chunk_size, filter_mask) = if is_filtered {
|
|
215
|
+
let chunk_size_len = entry_size
|
|
216
|
+
.checked_sub(offset_size)
|
|
217
|
+
.and_then(|remaining| remaining.checked_sub(4))
|
|
218
|
+
.ok_or_else(|| Error::InvalidData("invalid fixed array entry size".into()))?;
|
|
219
|
+
// 1–8 bytes wide (sized to the chunk), not a "length" of the superblock size.
|
|
220
|
+
let cs = cursor.read_uvar(chunk_size_len as usize)?;
|
|
221
|
+
let fm = cursor.read_u32_le()?;
|
|
222
|
+
(cs, fm)
|
|
223
|
+
} else {
|
|
224
|
+
(0, 0)
|
|
225
|
+
};
|
|
226
|
+
entries.push(FaRawEntry {
|
|
227
|
+
address,
|
|
228
|
+
chunk_size,
|
|
229
|
+
filter_mask,
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
Ok(entries)
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
fn read_entry_at(
|
|
236
|
+
data: &[u8],
|
|
237
|
+
position: u64,
|
|
238
|
+
is_filtered: bool,
|
|
239
|
+
offset_size: u8,
|
|
240
|
+
entry_size: u8,
|
|
241
|
+
) -> Result<FaRawEntry> {
|
|
242
|
+
let mut cursor = Cursor::new(data);
|
|
243
|
+
cursor.set_position(position);
|
|
244
|
+
let mut entries = read_entries(&mut cursor, 1, is_filtered, offset_size, entry_size)?;
|
|
245
|
+
entries
|
|
246
|
+
.pop()
|
|
247
|
+
.ok_or_else(|| Error::InvalidData("missing fixed array entry".into()))
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
fn read_entry_at_storage(
|
|
251
|
+
storage: &dyn Storage,
|
|
252
|
+
position: u64,
|
|
253
|
+
is_filtered: bool,
|
|
254
|
+
offset_size: u8,
|
|
255
|
+
entry_size: u8,
|
|
256
|
+
) -> Result<FaRawEntry> {
|
|
257
|
+
let bytes = storage.read_range(position, usize::from(entry_size))?;
|
|
258
|
+
let mut cursor = Cursor::new(bytes.as_ref());
|
|
259
|
+
let mut entries = read_entries(&mut cursor, 1, is_filtered, offset_size, entry_size)?;
|
|
260
|
+
entries
|
|
261
|
+
.pop()
|
|
262
|
+
.ok_or_else(|| Error::InvalidData("missing fixed array entry".into()))
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
fn linear_target_offsets(
|
|
266
|
+
dataset_shape: &[u64],
|
|
267
|
+
chunk_dims: &[u32],
|
|
268
|
+
chunk_bounds: Option<(&[u64], &[u64])>,
|
|
269
|
+
) -> Vec<(usize, Vec<u64>)> {
|
|
270
|
+
let ndim = dataset_shape.len();
|
|
271
|
+
let chunks_per_dim: Vec<u64> = (0..ndim)
|
|
272
|
+
.map(|i| dataset_shape[i].div_ceil(chunk_dims[i] as u64))
|
|
273
|
+
.collect();
|
|
274
|
+
|
|
275
|
+
if ndim == 0 {
|
|
276
|
+
return vec![(0, Vec::new())];
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
let (first_chunk, last_chunk): (Vec<u64>, Vec<u64>) = match chunk_bounds {
|
|
280
|
+
Some((first, last)) => (first.to_vec(), last.to_vec()),
|
|
281
|
+
None => (
|
|
282
|
+
vec![0u64; ndim],
|
|
283
|
+
chunks_per_dim
|
|
284
|
+
.iter()
|
|
285
|
+
.map(|count| count.saturating_sub(1))
|
|
286
|
+
.collect(),
|
|
287
|
+
),
|
|
288
|
+
};
|
|
289
|
+
|
|
290
|
+
let mut targets = Vec::new();
|
|
291
|
+
let mut chunk_indices = first_chunk.clone();
|
|
292
|
+
loop {
|
|
293
|
+
let mut linear_idx = 0u64;
|
|
294
|
+
for (dim, chunk_index) in chunk_indices.iter().enumerate() {
|
|
295
|
+
linear_idx = linear_idx * chunks_per_dim[dim] + chunk_index;
|
|
296
|
+
}
|
|
297
|
+
let offsets = chunk_indices
|
|
298
|
+
.iter()
|
|
299
|
+
.enumerate()
|
|
300
|
+
.map(|(dim, chunk_index)| chunk_index * u64::from(chunk_dims[dim]))
|
|
301
|
+
.collect();
|
|
302
|
+
targets.push((linear_idx as usize, offsets));
|
|
303
|
+
|
|
304
|
+
let mut advanced = false;
|
|
305
|
+
for dim in (0..ndim).rev() {
|
|
306
|
+
if chunk_indices[dim] < last_chunk[dim] {
|
|
307
|
+
chunk_indices[dim] += 1;
|
|
308
|
+
if dim + 1 < ndim {
|
|
309
|
+
chunk_indices[(dim + 1)..ndim].copy_from_slice(&first_chunk[(dim + 1)..ndim]);
|
|
310
|
+
}
|
|
311
|
+
advanced = true;
|
|
312
|
+
break;
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
if !advanced {
|
|
317
|
+
break;
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
targets
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
fn collect_fixed_array_chunk_entries_bounded(
|
|
325
|
+
data: &[u8],
|
|
326
|
+
header: &FaHeader,
|
|
327
|
+
offset_size: u8,
|
|
328
|
+
dataset_shape: &[u64],
|
|
329
|
+
chunk_dims: &[u32],
|
|
330
|
+
chunk_bounds: (&[u64], &[u64]),
|
|
331
|
+
) -> Result<Vec<ChunkEntry>> {
|
|
332
|
+
let targets = linear_target_offsets(dataset_shape, chunk_dims, Some(chunk_bounds));
|
|
333
|
+
let mut cursor = Cursor::new(data);
|
|
334
|
+
cursor.set_position(header.data_block_address);
|
|
335
|
+
|
|
336
|
+
let sig = cursor.read_bytes(4)?;
|
|
337
|
+
if sig != FADB_SIGNATURE {
|
|
338
|
+
return Err(Error::InvalidFixedArraySignature {
|
|
339
|
+
context: "data block signature mismatch",
|
|
340
|
+
});
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
let version = cursor.read_u8()?;
|
|
344
|
+
if version != 0 {
|
|
345
|
+
return Err(Error::Other(format!(
|
|
346
|
+
"unsupported fixed array data block version {}",
|
|
347
|
+
version
|
|
348
|
+
)));
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
let _client_id = cursor.read_u8()?;
|
|
352
|
+
let _header_address = cursor.read_offset(offset_size)?;
|
|
353
|
+
|
|
354
|
+
let num_entries = header.num_entries as usize;
|
|
355
|
+
let is_filtered = header.client_id == 1;
|
|
356
|
+
let entry_bytes = header.entry_size as usize;
|
|
357
|
+
let use_paging = header.page_bits > 0 && num_entries > (1usize << header.page_bits);
|
|
358
|
+
|
|
359
|
+
if !use_paging {
|
|
360
|
+
let entries_start = cursor.position();
|
|
361
|
+
let mut entries = Vec::new();
|
|
362
|
+
for (linear_idx, offsets) in targets {
|
|
363
|
+
let position = entries_start + (linear_idx * entry_bytes) as u64;
|
|
364
|
+
let raw = read_entry_at(data, position, is_filtered, offset_size, header.entry_size)?;
|
|
365
|
+
if Cursor::is_undefined_offset(raw.address, offset_size) {
|
|
366
|
+
continue;
|
|
367
|
+
}
|
|
368
|
+
entries.push(ChunkEntry {
|
|
369
|
+
address: raw.address,
|
|
370
|
+
size: raw.chunk_size,
|
|
371
|
+
filter_mask: raw.filter_mask,
|
|
372
|
+
offsets,
|
|
373
|
+
});
|
|
374
|
+
}
|
|
375
|
+
return Ok(entries);
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
let entries_per_page = 1usize << header.page_bits;
|
|
379
|
+
let num_pages = num_entries.div_ceil(entries_per_page);
|
|
380
|
+
let bitmap_bytes = num_pages.div_ceil(8);
|
|
381
|
+
let page_bitmap = cursor.read_bytes(bitmap_bytes)?.to_vec();
|
|
382
|
+
let pages_start = cursor.position();
|
|
383
|
+
|
|
384
|
+
let mut page_offsets = vec![None; num_pages];
|
|
385
|
+
let mut next_page_start = pages_start;
|
|
386
|
+
for (page_idx, page_offset) in page_offsets.iter_mut().enumerate().take(num_pages) {
|
|
387
|
+
let byte_idx = page_idx / 8;
|
|
388
|
+
let bit_idx = page_idx % 8;
|
|
389
|
+
let page_initialized =
|
|
390
|
+
byte_idx < page_bitmap.len() && (page_bitmap[byte_idx] & (1 << bit_idx)) != 0;
|
|
391
|
+
|
|
392
|
+
let entries_in_page = if page_idx == num_pages - 1 {
|
|
393
|
+
let remainder = num_entries % entries_per_page;
|
|
394
|
+
if remainder == 0 {
|
|
395
|
+
entries_per_page
|
|
396
|
+
} else {
|
|
397
|
+
remainder
|
|
398
|
+
}
|
|
399
|
+
} else {
|
|
400
|
+
entries_per_page
|
|
401
|
+
};
|
|
402
|
+
|
|
403
|
+
if page_initialized {
|
|
404
|
+
*page_offset = Some(next_page_start);
|
|
405
|
+
next_page_start += (entries_in_page * entry_bytes + 4) as u64;
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
let mut entries = Vec::new();
|
|
410
|
+
for (linear_idx, offsets) in targets {
|
|
411
|
+
let page_idx = linear_idx / entries_per_page;
|
|
412
|
+
let within_page = linear_idx % entries_per_page;
|
|
413
|
+
let Some(page_start) = page_offsets[page_idx] else {
|
|
414
|
+
continue;
|
|
415
|
+
};
|
|
416
|
+
let position = page_start + (within_page * entry_bytes) as u64;
|
|
417
|
+
let raw = read_entry_at(data, position, is_filtered, offset_size, header.entry_size)?;
|
|
418
|
+
if Cursor::is_undefined_offset(raw.address, offset_size) {
|
|
419
|
+
continue;
|
|
420
|
+
}
|
|
421
|
+
entries.push(ChunkEntry {
|
|
422
|
+
address: raw.address,
|
|
423
|
+
size: raw.chunk_size,
|
|
424
|
+
filter_mask: raw.filter_mask,
|
|
425
|
+
offsets,
|
|
426
|
+
});
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
Ok(entries)
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
fn collect_fixed_array_chunk_entries_bounded_storage(
|
|
433
|
+
storage: &dyn Storage,
|
|
434
|
+
header: &FaHeader,
|
|
435
|
+
offset_size: u8,
|
|
436
|
+
dataset_shape: &[u64],
|
|
437
|
+
chunk_dims: &[u32],
|
|
438
|
+
chunk_bounds: (&[u64], &[u64]),
|
|
439
|
+
) -> Result<Vec<ChunkEntry>> {
|
|
440
|
+
let targets = linear_target_offsets(dataset_shape, chunk_dims, Some(chunk_bounds));
|
|
441
|
+
let block_header_len = 4 + 1 + 1 + usize::from(offset_size);
|
|
442
|
+
let header_bytes = storage.read_range(header.data_block_address, block_header_len)?;
|
|
443
|
+
let mut cursor = Cursor::new(header_bytes.as_ref());
|
|
444
|
+
|
|
445
|
+
let sig = cursor.read_bytes(4)?;
|
|
446
|
+
if sig != FADB_SIGNATURE {
|
|
447
|
+
return Err(Error::InvalidFixedArraySignature {
|
|
448
|
+
context: "data block signature mismatch",
|
|
449
|
+
});
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
let version = cursor.read_u8()?;
|
|
453
|
+
if version != 0 {
|
|
454
|
+
return Err(Error::Other(format!(
|
|
455
|
+
"unsupported fixed array data block version {}",
|
|
456
|
+
version
|
|
457
|
+
)));
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
let _client_id = cursor.read_u8()?;
|
|
461
|
+
let _header_address = cursor.read_offset(offset_size)?;
|
|
462
|
+
|
|
463
|
+
let num_entries = usize::try_from(header.num_entries).map_err(|_| {
|
|
464
|
+
Error::InvalidData("fixed array entry count exceeds platform usize capacity".into())
|
|
465
|
+
})?;
|
|
466
|
+
let is_filtered = header.client_id == 1;
|
|
467
|
+
let entry_bytes = usize::from(header.entry_size);
|
|
468
|
+
let use_paging = header.page_bits > 0 && num_entries > (1usize << header.page_bits);
|
|
469
|
+
let entries_start = header.data_block_address
|
|
470
|
+
+ u64::try_from(block_header_len)
|
|
471
|
+
.map_err(|_| Error::OffsetOutOfBounds(header.data_block_address))?;
|
|
472
|
+
|
|
473
|
+
if !use_paging {
|
|
474
|
+
let mut entries = Vec::new();
|
|
475
|
+
for (linear_idx, offsets) in targets {
|
|
476
|
+
let position = entries_start
|
|
477
|
+
+ u64::try_from(linear_idx * entry_bytes).map_err(|_| {
|
|
478
|
+
Error::InvalidData("fixed array entry offset exceeds u64 capacity".into())
|
|
479
|
+
})?;
|
|
480
|
+
let raw = read_entry_at_storage(
|
|
481
|
+
storage,
|
|
482
|
+
position,
|
|
483
|
+
is_filtered,
|
|
484
|
+
offset_size,
|
|
485
|
+
header.entry_size,
|
|
486
|
+
)?;
|
|
487
|
+
if Cursor::is_undefined_offset(raw.address, offset_size) {
|
|
488
|
+
continue;
|
|
489
|
+
}
|
|
490
|
+
entries.push(ChunkEntry {
|
|
491
|
+
address: raw.address,
|
|
492
|
+
size: raw.chunk_size,
|
|
493
|
+
filter_mask: raw.filter_mask,
|
|
494
|
+
offsets,
|
|
495
|
+
});
|
|
496
|
+
}
|
|
497
|
+
return Ok(entries);
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
let entries_per_page = 1usize << header.page_bits;
|
|
501
|
+
let num_pages = num_entries.div_ceil(entries_per_page);
|
|
502
|
+
let bitmap_bytes = num_pages.div_ceil(8);
|
|
503
|
+
let page_bitmap = storage.read_range(entries_start, bitmap_bytes)?;
|
|
504
|
+
let pages_start = entries_start
|
|
505
|
+
+ u64::try_from(bitmap_bytes).map_err(|_| {
|
|
506
|
+
Error::InvalidData("fixed array bitmap size exceeds u64 capacity".into())
|
|
507
|
+
})?;
|
|
508
|
+
|
|
509
|
+
let mut page_offsets = vec![None; num_pages];
|
|
510
|
+
let mut next_page_start = pages_start;
|
|
511
|
+
for (page_idx, page_offset) in page_offsets.iter_mut().enumerate().take(num_pages) {
|
|
512
|
+
let byte_idx = page_idx / 8;
|
|
513
|
+
let bit_idx = page_idx % 8;
|
|
514
|
+
let page_initialized =
|
|
515
|
+
byte_idx < page_bitmap.len() && (page_bitmap[byte_idx] & (1 << bit_idx)) != 0;
|
|
516
|
+
|
|
517
|
+
let entries_in_page = if page_idx == num_pages - 1 {
|
|
518
|
+
let remainder = num_entries % entries_per_page;
|
|
519
|
+
if remainder == 0 {
|
|
520
|
+
entries_per_page
|
|
521
|
+
} else {
|
|
522
|
+
remainder
|
|
523
|
+
}
|
|
524
|
+
} else {
|
|
525
|
+
entries_per_page
|
|
526
|
+
};
|
|
527
|
+
|
|
528
|
+
if page_initialized {
|
|
529
|
+
*page_offset = Some(next_page_start);
|
|
530
|
+
next_page_start += u64::try_from(entries_in_page * entry_bytes + 4).map_err(|_| {
|
|
531
|
+
Error::InvalidData("fixed array page size exceeds u64 capacity".into())
|
|
532
|
+
})?;
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
let mut entries = Vec::new();
|
|
537
|
+
for (linear_idx, offsets) in targets {
|
|
538
|
+
let page_idx = linear_idx / entries_per_page;
|
|
539
|
+
let within_page = linear_idx % entries_per_page;
|
|
540
|
+
let Some(page_start) = page_offsets[page_idx] else {
|
|
541
|
+
continue;
|
|
542
|
+
};
|
|
543
|
+
let position = page_start
|
|
544
|
+
+ u64::try_from(within_page * entry_bytes).map_err(|_| {
|
|
545
|
+
Error::InvalidData("fixed array page entry offset exceeds u64 capacity".into())
|
|
546
|
+
})?;
|
|
547
|
+
let raw = read_entry_at_storage(
|
|
548
|
+
storage,
|
|
549
|
+
position,
|
|
550
|
+
is_filtered,
|
|
551
|
+
offset_size,
|
|
552
|
+
header.entry_size,
|
|
553
|
+
)?;
|
|
554
|
+
if Cursor::is_undefined_offset(raw.address, offset_size) {
|
|
555
|
+
continue;
|
|
556
|
+
}
|
|
557
|
+
entries.push(ChunkEntry {
|
|
558
|
+
address: raw.address,
|
|
559
|
+
size: raw.chunk_size,
|
|
560
|
+
filter_mask: raw.filter_mask,
|
|
561
|
+
offsets,
|
|
562
|
+
});
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
Ok(entries)
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
/// Collect chunk entries from a Fixed Array index.
|
|
569
|
+
///
|
|
570
|
+
/// Reads the FAHD header and FADB data block, then converts linear entry
|
|
571
|
+
/// indices to multi-dimensional chunk offsets.
|
|
572
|
+
pub fn collect_fixed_array_chunk_entries(
|
|
573
|
+
data: &[u8],
|
|
574
|
+
header_address: u64,
|
|
575
|
+
offset_size: u8,
|
|
576
|
+
length_size: u8,
|
|
577
|
+
dataset_shape: &[u64],
|
|
578
|
+
chunk_dims: &[u32],
|
|
579
|
+
chunk_bounds: Option<(&[u64], &[u64])>,
|
|
580
|
+
) -> Result<Vec<ChunkEntry>> {
|
|
581
|
+
let header = parse_header(data, header_address, offset_size, length_size)?;
|
|
582
|
+
|
|
583
|
+
if Cursor::is_undefined_offset(header.data_block_address, offset_size) {
|
|
584
|
+
return Ok(Vec::new());
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
if let Some(bounds) = chunk_bounds {
|
|
588
|
+
return collect_fixed_array_chunk_entries_bounded(
|
|
589
|
+
data,
|
|
590
|
+
&header,
|
|
591
|
+
offset_size,
|
|
592
|
+
dataset_shape,
|
|
593
|
+
chunk_dims,
|
|
594
|
+
bounds,
|
|
595
|
+
);
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
let raw_entries = parse_data_block(data, header.data_block_address, &header, offset_size)?;
|
|
599
|
+
|
|
600
|
+
let ndim = dataset_shape.len();
|
|
601
|
+
let chunks_per_dim: Vec<u64> = (0..ndim)
|
|
602
|
+
.map(|i| dataset_shape[i].div_ceil(chunk_dims[i] as u64))
|
|
603
|
+
.collect();
|
|
604
|
+
|
|
605
|
+
let mut entries = Vec::new();
|
|
606
|
+
for (linear_idx, raw) in raw_entries.iter().enumerate() {
|
|
607
|
+
// Skip undefined addresses (unallocated chunks).
|
|
608
|
+
if Cursor::is_undefined_offset(raw.address, offset_size) {
|
|
609
|
+
continue;
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
// Convert linear index to multi-dimensional chunk offsets.
|
|
613
|
+
let mut remaining = linear_idx as u64;
|
|
614
|
+
let mut offsets = vec![0u64; ndim];
|
|
615
|
+
for d in (0..ndim).rev() {
|
|
616
|
+
offsets[d] = (remaining % chunks_per_dim[d]) * chunk_dims[d] as u64;
|
|
617
|
+
remaining /= chunks_per_dim[d];
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
if let Some((first_chunk, last_chunk)) = chunk_bounds {
|
|
621
|
+
let overlaps = offsets.iter().enumerate().all(|(dim, offset)| {
|
|
622
|
+
let chunk_index = *offset / u64::from(chunk_dims[dim]);
|
|
623
|
+
chunk_index >= first_chunk[dim] && chunk_index <= last_chunk[dim]
|
|
624
|
+
});
|
|
625
|
+
if !overlaps {
|
|
626
|
+
continue;
|
|
627
|
+
}
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
entries.push(ChunkEntry {
|
|
631
|
+
address: raw.address,
|
|
632
|
+
size: raw.chunk_size,
|
|
633
|
+
filter_mask: raw.filter_mask,
|
|
634
|
+
offsets,
|
|
635
|
+
});
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
Ok(entries)
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
/// Collect chunk entries from a Fixed Array index using random-access storage.
|
|
642
|
+
pub fn collect_fixed_array_chunk_entries_storage(
|
|
643
|
+
storage: &dyn Storage,
|
|
644
|
+
header_address: u64,
|
|
645
|
+
offset_size: u8,
|
|
646
|
+
length_size: u8,
|
|
647
|
+
dataset_shape: &[u64],
|
|
648
|
+
chunk_dims: &[u32],
|
|
649
|
+
chunk_bounds: Option<(&[u64], &[u64])>,
|
|
650
|
+
) -> Result<Vec<ChunkEntry>> {
|
|
651
|
+
let header = parse_header_storage(storage, header_address, offset_size, length_size)?;
|
|
652
|
+
|
|
653
|
+
if Cursor::is_undefined_offset(header.data_block_address, offset_size) {
|
|
654
|
+
return Ok(Vec::new());
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
if let Some(bounds) = chunk_bounds {
|
|
658
|
+
return collect_fixed_array_chunk_entries_bounded_storage(
|
|
659
|
+
storage,
|
|
660
|
+
&header,
|
|
661
|
+
offset_size,
|
|
662
|
+
dataset_shape,
|
|
663
|
+
chunk_dims,
|
|
664
|
+
bounds,
|
|
665
|
+
);
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
let num_entries = usize::try_from(header.num_entries).map_err(|_| {
|
|
669
|
+
Error::InvalidData("fixed array entry count exceeds platform usize capacity".into())
|
|
670
|
+
})?;
|
|
671
|
+
let is_filtered = header.client_id == 1;
|
|
672
|
+
let header_len = 4 + 1 + 1 + usize::from(offset_size);
|
|
673
|
+
let use_paging = header.page_bits > 0 && num_entries > (1usize << header.page_bits);
|
|
674
|
+
let block_len = if !use_paging {
|
|
675
|
+
header_len + num_entries * usize::from(header.entry_size) + 4
|
|
676
|
+
} else {
|
|
677
|
+
let entries_per_page = 1usize << header.page_bits;
|
|
678
|
+
let num_pages = num_entries.div_ceil(entries_per_page);
|
|
679
|
+
let bitmap_bytes = num_pages.div_ceil(8);
|
|
680
|
+
let mut len = header_len + bitmap_bytes;
|
|
681
|
+
for page_idx in 0..num_pages {
|
|
682
|
+
let entries_in_page = if page_idx == num_pages - 1 {
|
|
683
|
+
let remainder = num_entries % entries_per_page;
|
|
684
|
+
if remainder == 0 {
|
|
685
|
+
entries_per_page
|
|
686
|
+
} else {
|
|
687
|
+
remainder
|
|
688
|
+
}
|
|
689
|
+
} else {
|
|
690
|
+
entries_per_page
|
|
691
|
+
};
|
|
692
|
+
len += entries_in_page * usize::from(header.entry_size) + 4;
|
|
693
|
+
}
|
|
694
|
+
len
|
|
695
|
+
};
|
|
696
|
+
let block = storage.read_range(header.data_block_address, block_len)?;
|
|
697
|
+
let raw_entries = parse_data_block(block.as_ref(), 0, &header, offset_size)?;
|
|
698
|
+
|
|
699
|
+
let ndim = dataset_shape.len();
|
|
700
|
+
let chunks_per_dim: Vec<u64> = (0..ndim)
|
|
701
|
+
.map(|i| dataset_shape[i].div_ceil(chunk_dims[i] as u64))
|
|
702
|
+
.collect();
|
|
703
|
+
|
|
704
|
+
let mut entries = Vec::new();
|
|
705
|
+
for (linear_idx, raw) in raw_entries.iter().enumerate() {
|
|
706
|
+
if Cursor::is_undefined_offset(raw.address, offset_size) {
|
|
707
|
+
continue;
|
|
708
|
+
}
|
|
709
|
+
let mut remaining = linear_idx as u64;
|
|
710
|
+
let mut offsets = vec![0u64; ndim];
|
|
711
|
+
for d in (0..ndim).rev() {
|
|
712
|
+
offsets[d] = (remaining % chunks_per_dim[d]) * chunk_dims[d] as u64;
|
|
713
|
+
remaining /= chunks_per_dim[d];
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
if let Some((first_chunk, last_chunk)) = chunk_bounds {
|
|
717
|
+
let overlaps = offsets.iter().enumerate().all(|(dim, offset)| {
|
|
718
|
+
let chunk_index = *offset / u64::from(chunk_dims[dim]);
|
|
719
|
+
chunk_index >= first_chunk[dim] && chunk_index <= last_chunk[dim]
|
|
720
|
+
});
|
|
721
|
+
if !overlaps {
|
|
722
|
+
continue;
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
entries.push(ChunkEntry {
|
|
727
|
+
address: raw.address,
|
|
728
|
+
size: if is_filtered { raw.chunk_size } else { 0 },
|
|
729
|
+
filter_mask: raw.filter_mask,
|
|
730
|
+
offsets,
|
|
731
|
+
});
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
Ok(entries)
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
#[cfg(test)]
|
|
738
|
+
mod tests {
|
|
739
|
+
use super::*;
|
|
740
|
+
|
|
741
|
+
#[test]
|
|
742
|
+
fn fahd_bad_signature() {
|
|
743
|
+
let mut data = vec![0u8; 64];
|
|
744
|
+
data[0..4].copy_from_slice(b"XXXX");
|
|
745
|
+
let err = parse_header(&data, 0, 8, 8).unwrap_err();
|
|
746
|
+
assert!(matches!(err, Error::InvalidFixedArraySignature { .. }));
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
#[test]
|
|
750
|
+
fn fadb_bad_signature() {
|
|
751
|
+
let header = FaHeader {
|
|
752
|
+
client_id: 0,
|
|
753
|
+
entry_size: 8,
|
|
754
|
+
page_bits: 0,
|
|
755
|
+
num_entries: 1,
|
|
756
|
+
data_block_address: 0,
|
|
757
|
+
};
|
|
758
|
+
let mut data = vec![0u8; 64];
|
|
759
|
+
data[0..4].copy_from_slice(b"XXXX");
|
|
760
|
+
let err = parse_data_block(&data, 0, &header, 8).unwrap_err();
|
|
761
|
+
assert!(matches!(err, Error::InvalidFixedArraySignature { .. }));
|
|
762
|
+
}
|
|
763
|
+
}
|