@nirs4all/formats-wasm 0.2.12 → 0.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD_PARTY_NOTICES.md +49 -0
- package/nirs4all_formats_wasm_bg.wasm +0 -0
- package/package.json +4 -2
- package/reader-backports/hdf5-reader/Cargo.toml +119 -0
- package/reader-backports/hdf5-reader/LICENSE-APACHE +164 -0
- package/reader-backports/hdf5-reader/LICENSE-MIT +21 -0
- package/reader-backports/hdf5-reader/PROVENANCE.json +80 -0
- package/reader-backports/hdf5-reader/README.md +308 -0
- package/reader-backports/hdf5-reader/UPSTREAM-Cargo.toml +44 -0
- package/reader-backports/hdf5-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/hdf5-reader/src/attribute_api.rs +847 -0
- package/reader-backports/hdf5-reader/src/btree_v1.rs +726 -0
- package/reader-backports/hdf5-reader/src/btree_v2.rs +2068 -0
- package/reader-backports/hdf5-reader/src/cache.rs +354 -0
- package/reader-backports/hdf5-reader/src/checksum.rs +56 -0
- package/reader-backports/hdf5-reader/src/chunk_index.rs +423 -0
- package/reader-backports/hdf5-reader/src/dataset.rs +4733 -0
- package/reader-backports/hdf5-reader/src/datatype_api.rs +497 -0
- package/reader-backports/hdf5-reader/src/error.rs +167 -0
- package/reader-backports/hdf5-reader/src/extensible_array.rs +1886 -0
- package/reader-backports/hdf5-reader/src/filters/deflate.rs +93 -0
- package/reader-backports/hdf5-reader/src/filters/fletcher32.rs +52 -0
- package/reader-backports/hdf5-reader/src/filters/lz4.rs +178 -0
- package/reader-backports/hdf5-reader/src/filters/mod.rs +326 -0
- package/reader-backports/hdf5-reader/src/filters/nbit.rs +586 -0
- package/reader-backports/hdf5-reader/src/filters/scaleoffset.rs +827 -0
- package/reader-backports/hdf5-reader/src/filters/shuffle.rs +99 -0
- package/reader-backports/hdf5-reader/src/fixed_array.rs +763 -0
- package/reader-backports/hdf5-reader/src/fractal_heap.rs +2029 -0
- package/reader-backports/hdf5-reader/src/global_heap.rs +284 -0
- package/reader-backports/hdf5-reader/src/group.rs +795 -0
- package/reader-backports/hdf5-reader/src/io.rs +413 -0
- package/reader-backports/hdf5-reader/src/lib.rs +916 -0
- package/reader-backports/hdf5-reader/src/local_heap.rs +261 -0
- package/reader-backports/hdf5-reader/src/messages/attribute.rs +304 -0
- package/reader-backports/hdf5-reader/src/messages/attribute_info.rs +96 -0
- package/reader-backports/hdf5-reader/src/messages/btree_k.rs +36 -0
- package/reader-backports/hdf5-reader/src/messages/continuation.rs +46 -0
- package/reader-backports/hdf5-reader/src/messages/dataspace.rs +259 -0
- package/reader-backports/hdf5-reader/src/messages/datatype.rs +938 -0
- package/reader-backports/hdf5-reader/src/messages/external_files.rs +50 -0
- package/reader-backports/hdf5-reader/src/messages/fill_value.rs +233 -0
- package/reader-backports/hdf5-reader/src/messages/filter_pipeline.rs +147 -0
- package/reader-backports/hdf5-reader/src/messages/group_info.rs +103 -0
- package/reader-backports/hdf5-reader/src/messages/layout.rs +476 -0
- package/reader-backports/hdf5-reader/src/messages/link.rs +170 -0
- package/reader-backports/hdf5-reader/src/messages/link_info.rs +127 -0
- package/reader-backports/hdf5-reader/src/messages/mod.rs +374 -0
- package/reader-backports/hdf5-reader/src/messages/modification_time.rs +83 -0
- package/reader-backports/hdf5-reader/src/messages/shared.rs +164 -0
- package/reader-backports/hdf5-reader/src/messages/shared_table.rs +67 -0
- package/reader-backports/hdf5-reader/src/messages/symbol_table_msg.rs +51 -0
- package/reader-backports/hdf5-reader/src/object_header.rs +1749 -0
- package/reader-backports/hdf5-reader/src/reference.rs +90 -0
- package/reader-backports/hdf5-reader/src/shared_message_table.rs +425 -0
- package/reader-backports/hdf5-reader/src/storage.rs +538 -0
- package/reader-backports/hdf5-reader/src/superblock.rs +285 -0
- package/reader-backports/hdf5-reader/src/symbol_table.rs +277 -0
- package/reader-backports/netcdf-reader/Cargo.toml +141 -0
- package/reader-backports/netcdf-reader/LICENSE-APACHE +164 -0
- package/reader-backports/netcdf-reader/LICENSE-MIT +21 -0
- package/reader-backports/netcdf-reader/PROVENANCE.json +55 -0
- package/reader-backports/netcdf-reader/README.md +308 -0
- package/reader-backports/netcdf-reader/UPSTREAM-Cargo.toml +48 -0
- package/reader-backports/netcdf-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/netcdf-reader/src/cf/axes.rs +419 -0
- package/reader-backports/netcdf-reader/src/cf/bounds.rs +108 -0
- package/reader-backports/netcdf-reader/src/cf/crs.rs +269 -0
- package/reader-backports/netcdf-reader/src/cf/mod.rs +30 -0
- package/reader-backports/netcdf-reader/src/cf/time.rs +1122 -0
- package/reader-backports/netcdf-reader/src/classic/data.rs +1465 -0
- package/reader-backports/netcdf-reader/src/classic/header.rs +1097 -0
- package/reader-backports/netcdf-reader/src/classic/mod.rs +240 -0
- package/reader-backports/netcdf-reader/src/classic/storage.rs +211 -0
- package/reader-backports/netcdf-reader/src/classic/types.rs +93 -0
- package/reader-backports/netcdf-reader/src/classic/variable.rs +1162 -0
- package/reader-backports/netcdf-reader/src/error.rs +59 -0
- package/reader-backports/netcdf-reader/src/lib.rs +1531 -0
- package/reader-backports/netcdf-reader/src/masked.rs +173 -0
- package/reader-backports/netcdf-reader/src/nc4/attributes.rs +203 -0
- package/reader-backports/netcdf-reader/src/nc4/dimensions.rs +139 -0
- package/reader-backports/netcdf-reader/src/nc4/groups.rs +308 -0
- package/reader-backports/netcdf-reader/src/nc4/mod.rs +572 -0
- package/reader-backports/netcdf-reader/src/nc4/types.rs +309 -0
- package/reader-backports/netcdf-reader/src/nc4/variables.rs +420 -0
- package/reader-backports/netcdf-reader/src/types.rs +18 -0
- package/reader-backports/netcdf-reader/src/unpack.rs +113 -0
- package/reader-backports/netcdf-reader/src/user_defined.rs +569 -0
|
@@ -0,0 +1,1886 @@
|
|
|
1
|
+
//! HDF5 Extensible Array (EA) chunk index.
|
|
2
|
+
//!
|
|
3
|
+
//! This is the default chunk index for datasets with one unlimited dimension
|
|
4
|
+
//! and `libver='latest'`. It uses a three-level hierarchy:
|
|
5
|
+
//!
|
|
6
|
+
//! - `EAHD` — Extensible Array Header
|
|
7
|
+
//! - `EAIB` — Extensible Array Index Block
|
|
8
|
+
//! - `EADB` — Extensible Array Data Block
|
|
9
|
+
//! - `EASB` — Extensible Array Secondary Block
|
|
10
|
+
|
|
11
|
+
use crate::checksum::jenkins_lookup3;
|
|
12
|
+
use crate::chunk_index::ChunkEntry;
|
|
13
|
+
use crate::error::{Error, Result};
|
|
14
|
+
use crate::io::Cursor;
|
|
15
|
+
use crate::storage::Storage;
|
|
16
|
+
|
|
17
|
+
const EAHD_SIGNATURE: [u8; 4] = *b"EAHD";
|
|
18
|
+
const EAIB_SIGNATURE: [u8; 4] = *b"EAIB";
|
|
19
|
+
const EADB_SIGNATURE: [u8; 4] = *b"EADB";
|
|
20
|
+
const EASB_SIGNATURE: [u8; 4] = *b"EASB";
|
|
21
|
+
|
|
22
|
+
/// Parsed Extensible Array Header.
|
|
23
|
+
#[derive(Debug)]
|
|
24
|
+
struct EaHeader {
|
|
25
|
+
client_id: u8,
|
|
26
|
+
element_size: u8,
|
|
27
|
+
_max_nelmts_bits: u8,
|
|
28
|
+
idx_blk_elmts: u8,
|
|
29
|
+
data_blk_min_elmts: u8,
|
|
30
|
+
sec_blk_min_data_ptrs: u8,
|
|
31
|
+
max_dblk_page_nelmts_bits: u8,
|
|
32
|
+
_nelmts: u64,
|
|
33
|
+
index_block_address: u64,
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/// Parse the Extensible Array Header.
|
|
37
|
+
///
|
|
38
|
+
/// On-disk layout (from H5EA_HEADER_SIZE):
|
|
39
|
+
/// sig(4) + ver(1) + client_id(1) +
|
|
40
|
+
/// element_size(1) + max_nelmts_bits(1) + idx_blk_elmts(1) +
|
|
41
|
+
/// data_blk_min_elmts(1) + sec_blk_min_data_ptrs(1) + max_dblk_page_nelmts_bits(1) +
|
|
42
|
+
/// 6 statistics fields (each length_size) +
|
|
43
|
+
/// index_block_address(offset_size) + checksum(4)
|
|
44
|
+
fn parse_header(data: &[u8], address: u64, offset_size: u8, length_size: u8) -> Result<EaHeader> {
|
|
45
|
+
let mut cursor = Cursor::new(data);
|
|
46
|
+
cursor.set_position(address);
|
|
47
|
+
|
|
48
|
+
let sig = cursor.read_bytes(4)?;
|
|
49
|
+
if sig != EAHD_SIGNATURE {
|
|
50
|
+
return Err(Error::InvalidExtensibleArraySignature {
|
|
51
|
+
context: "header signature mismatch",
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
let version = cursor.read_u8()?;
|
|
56
|
+
if version != 0 {
|
|
57
|
+
return Err(Error::Other(format!(
|
|
58
|
+
"unsupported extensible array header version {}",
|
|
59
|
+
version
|
|
60
|
+
)));
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
let client_id = cursor.read_u8()?;
|
|
64
|
+
let element_size = cursor.read_u8()?;
|
|
65
|
+
let max_nelmts_bits = cursor.read_u8()?;
|
|
66
|
+
let idx_blk_elmts = cursor.read_u8()?;
|
|
67
|
+
let data_blk_min_elmts = cursor.read_u8()?;
|
|
68
|
+
let sec_blk_min_data_ptrs = cursor.read_u8()?;
|
|
69
|
+
let max_dblk_page_nelmts_bits = cursor.read_u8()?;
|
|
70
|
+
|
|
71
|
+
// Statistics (6 fields, each length_size bytes)
|
|
72
|
+
let _nsuper_blks = cursor.read_length(length_size)?;
|
|
73
|
+
let _super_blk_size = cursor.read_length(length_size)?;
|
|
74
|
+
let _ndata_blks = cursor.read_length(length_size)?;
|
|
75
|
+
let _data_blk_size = cursor.read_length(length_size)?;
|
|
76
|
+
let _max_idx_set = cursor.read_length(length_size)?;
|
|
77
|
+
let nelmts = cursor.read_length(length_size)?;
|
|
78
|
+
|
|
79
|
+
let index_block_address = cursor.read_offset(offset_size)?;
|
|
80
|
+
|
|
81
|
+
// Checksum
|
|
82
|
+
let header_end = cursor.position();
|
|
83
|
+
let header_bytes = &data[address as usize..header_end as usize];
|
|
84
|
+
let stored_checksum = cursor.read_u32_le()?;
|
|
85
|
+
let computed = jenkins_lookup3(header_bytes);
|
|
86
|
+
if stored_checksum != computed {
|
|
87
|
+
return Err(Error::ChecksumMismatch {
|
|
88
|
+
expected: stored_checksum,
|
|
89
|
+
actual: computed,
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
Ok(EaHeader {
|
|
94
|
+
client_id,
|
|
95
|
+
element_size,
|
|
96
|
+
_max_nelmts_bits: max_nelmts_bits,
|
|
97
|
+
idx_blk_elmts,
|
|
98
|
+
data_blk_min_elmts,
|
|
99
|
+
sec_blk_min_data_ptrs,
|
|
100
|
+
max_dblk_page_nelmts_bits,
|
|
101
|
+
_nelmts: nelmts,
|
|
102
|
+
index_block_address,
|
|
103
|
+
})
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
fn parse_header_storage(
|
|
107
|
+
storage: &dyn Storage,
|
|
108
|
+
address: u64,
|
|
109
|
+
offset_size: u8,
|
|
110
|
+
length_size: u8,
|
|
111
|
+
) -> Result<EaHeader> {
|
|
112
|
+
let header_len = 4
|
|
113
|
+
+ 1
|
|
114
|
+
+ 1
|
|
115
|
+
+ 1
|
|
116
|
+
+ 1
|
|
117
|
+
+ 1
|
|
118
|
+
+ 1
|
|
119
|
+
+ 1
|
|
120
|
+
+ 1
|
|
121
|
+
+ 6 * usize::from(length_size)
|
|
122
|
+
+ usize::from(offset_size)
|
|
123
|
+
+ 4;
|
|
124
|
+
let bytes = storage.read_range(address, header_len)?;
|
|
125
|
+
parse_header(bytes.as_ref(), 0, offset_size, length_size)
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/// Compute the super block layout.
|
|
129
|
+
///
|
|
130
|
+
/// Returns a vec of (elements_per_data_block, num_data_blocks) for each super block.
|
|
131
|
+
/// Stops generating entries once cumulative capacity exceeds `nelmts`.
|
|
132
|
+
fn compute_super_block_layout(header: &EaHeader) -> Vec<(u64, u64)> {
|
|
133
|
+
let mut layout = Vec::new();
|
|
134
|
+
let dblk_min = header.data_blk_min_elmts as u64;
|
|
135
|
+
let sblk_min = header.sec_blk_min_data_ptrs as u64;
|
|
136
|
+
let nelmts = header._nelmts;
|
|
137
|
+
let mut cumulative = header.idx_blk_elmts as u64;
|
|
138
|
+
|
|
139
|
+
for sb_idx in 0u32..64 {
|
|
140
|
+
if cumulative >= nelmts {
|
|
141
|
+
break;
|
|
142
|
+
}
|
|
143
|
+
let elmts_per_dblk = dblk_min * (1u64 << (sb_idx / 2));
|
|
144
|
+
let num_dblks = sblk_min * (1u64 << (sb_idx.div_ceil(2)));
|
|
145
|
+
layout.push((elmts_per_dblk, num_dblks));
|
|
146
|
+
cumulative += elmts_per_dblk * num_dblks;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
layout
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
fn checked_add_usize(left: usize, right: usize, context: &str) -> Result<usize> {
|
|
153
|
+
left.checked_add(right)
|
|
154
|
+
.ok_or_else(|| Error::InvalidData(format!("{context} exceeds platform usize capacity")))
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
fn checked_mul_usize(left: usize, right: usize, context: &str) -> Result<usize> {
|
|
158
|
+
left.checked_mul(right)
|
|
159
|
+
.ok_or_else(|| Error::InvalidData(format!("{context} exceeds platform usize capacity")))
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
fn checked_add_u64(left: u64, right: u64, context: &str) -> Result<u64> {
|
|
163
|
+
left.checked_add(right)
|
|
164
|
+
.ok_or_else(|| Error::InvalidData(format!("{context} exceeds u64 capacity")))
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
fn checked_usize_from_u64(value: u64, context: &str) -> Result<usize> {
|
|
168
|
+
usize::try_from(value).map_err(|_| {
|
|
169
|
+
Error::InvalidData(format!(
|
|
170
|
+
"{context} value {value} exceeds platform usize capacity"
|
|
171
|
+
))
|
|
172
|
+
})
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
fn checked_u64_from_usize(value: usize, context: &str) -> Result<u64> {
|
|
176
|
+
u64::try_from(value).map_err(|_| Error::InvalidData(format!("{context} exceeds u64 capacity")))
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
fn ea_block_header_len(offset_size: u8, sizeof_nelmts: usize) -> Result<usize> {
|
|
180
|
+
let len = checked_add_usize(4, 1, "extensible array block header length")?;
|
|
181
|
+
let len = checked_add_usize(len, 1, "extensible array block header length")?;
|
|
182
|
+
let len = checked_add_usize(
|
|
183
|
+
len,
|
|
184
|
+
usize::from(offset_size),
|
|
185
|
+
"extensible array block header length",
|
|
186
|
+
)?;
|
|
187
|
+
checked_add_usize(len, sizeof_nelmts, "extensible array block header length")
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
fn ea_page_nelmts(max_page_bits: u8) -> Result<usize> {
|
|
191
|
+
if max_page_bits == 0 {
|
|
192
|
+
return Ok(0);
|
|
193
|
+
}
|
|
194
|
+
1usize.checked_shl(u32::from(max_page_bits)).ok_or_else(|| {
|
|
195
|
+
Error::InvalidData("extensible array page element count exceeds usize capacity".into())
|
|
196
|
+
})
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
fn index_block_len(header: &EaHeader, sb_layout: &[(u64, u64)], offset_size: u8) -> Result<usize> {
|
|
200
|
+
let offset_size = usize::from(offset_size);
|
|
201
|
+
let inline_bytes = checked_mul_usize(
|
|
202
|
+
usize::from(header.idx_blk_elmts),
|
|
203
|
+
usize::from(header.element_size),
|
|
204
|
+
"extensible array index block inline entry bytes",
|
|
205
|
+
)?;
|
|
206
|
+
let direct_ptrs = checked_mul_usize(
|
|
207
|
+
2,
|
|
208
|
+
usize::from(header.sec_blk_min_data_ptrs),
|
|
209
|
+
"extensible array index block direct data block pointer count",
|
|
210
|
+
)?;
|
|
211
|
+
let direct_bytes = checked_mul_usize(
|
|
212
|
+
direct_ptrs,
|
|
213
|
+
offset_size,
|
|
214
|
+
"extensible array index block direct data block address bytes",
|
|
215
|
+
)?;
|
|
216
|
+
let secondary_ptrs = sb_layout.len().saturating_sub(direct_ptrs);
|
|
217
|
+
let secondary_bytes = checked_mul_usize(
|
|
218
|
+
secondary_ptrs,
|
|
219
|
+
offset_size,
|
|
220
|
+
"extensible array index block secondary block address bytes",
|
|
221
|
+
)?;
|
|
222
|
+
|
|
223
|
+
let mut len = 4usize;
|
|
224
|
+
for part in [
|
|
225
|
+
1,
|
|
226
|
+
1,
|
|
227
|
+
offset_size,
|
|
228
|
+
inline_bytes,
|
|
229
|
+
direct_bytes,
|
|
230
|
+
secondary_bytes,
|
|
231
|
+
4,
|
|
232
|
+
] {
|
|
233
|
+
len = checked_add_usize(len, part, "extensible array index block length")?;
|
|
234
|
+
}
|
|
235
|
+
Ok(len)
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
fn secondary_page_bitmap_bytes(
|
|
239
|
+
max_page_bits: u8,
|
|
240
|
+
elmts_per_dblk: u64,
|
|
241
|
+
num_dblks: u64,
|
|
242
|
+
) -> Result<usize> {
|
|
243
|
+
let page_nelmts = ea_page_nelmts(max_page_bits)?;
|
|
244
|
+
if page_nelmts == 0 {
|
|
245
|
+
return Ok(0);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
let elmts_per_dblk =
|
|
249
|
+
checked_usize_from_u64(elmts_per_dblk, "extensible array data block element count")?;
|
|
250
|
+
if elmts_per_dblk <= page_nelmts {
|
|
251
|
+
return Ok(0);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
let num_dblks = checked_usize_from_u64(
|
|
255
|
+
num_dblks,
|
|
256
|
+
"extensible array secondary block data block count",
|
|
257
|
+
)?;
|
|
258
|
+
let pages_per_dblk = elmts_per_dblk.div_ceil(page_nelmts);
|
|
259
|
+
let total_pages = checked_mul_usize(
|
|
260
|
+
num_dblks,
|
|
261
|
+
pages_per_dblk,
|
|
262
|
+
"extensible array secondary block page bitmap bit count",
|
|
263
|
+
)?;
|
|
264
|
+
Ok(total_pages.div_ceil(8))
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
fn secondary_block_len(
|
|
268
|
+
num_dblk_addrs: usize,
|
|
269
|
+
offset_size: u8,
|
|
270
|
+
sizeof_nelmts: usize,
|
|
271
|
+
page_bitmap_bytes: usize,
|
|
272
|
+
) -> Result<usize> {
|
|
273
|
+
let offset_bytes = usize::from(offset_size);
|
|
274
|
+
let addr_bytes = checked_mul_usize(
|
|
275
|
+
num_dblk_addrs,
|
|
276
|
+
offset_bytes,
|
|
277
|
+
"extensible array secondary block data block address bytes",
|
|
278
|
+
)?;
|
|
279
|
+
|
|
280
|
+
let mut len = ea_block_header_len(offset_size, sizeof_nelmts)?;
|
|
281
|
+
for part in [page_bitmap_bytes, addr_bytes, 4] {
|
|
282
|
+
len = checked_add_usize(len, part, "extensible array secondary block length")?;
|
|
283
|
+
}
|
|
284
|
+
Ok(len)
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
fn data_block_len_from_bitmap(
|
|
288
|
+
num_entries: usize,
|
|
289
|
+
page_nelmts: usize,
|
|
290
|
+
entry_size: u8,
|
|
291
|
+
offset_size: u8,
|
|
292
|
+
sizeof_nelmts: usize,
|
|
293
|
+
page_bitmap: &[u8],
|
|
294
|
+
) -> Result<usize> {
|
|
295
|
+
let num_pages = num_entries.div_ceil(page_nelmts);
|
|
296
|
+
let bitmap_bytes = num_pages.div_ceil(8);
|
|
297
|
+
let mut len = checked_add_usize(
|
|
298
|
+
ea_block_header_len(offset_size, sizeof_nelmts)?,
|
|
299
|
+
bitmap_bytes,
|
|
300
|
+
"extensible array data block length",
|
|
301
|
+
)?;
|
|
302
|
+
|
|
303
|
+
for page_idx in 0..num_pages {
|
|
304
|
+
let byte_idx = page_idx / 8;
|
|
305
|
+
let bit_idx = page_idx % 8;
|
|
306
|
+
let page_initialized =
|
|
307
|
+
byte_idx < page_bitmap.len() && (page_bitmap[byte_idx] & (1 << bit_idx)) != 0;
|
|
308
|
+
if !page_initialized {
|
|
309
|
+
continue;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
let entries_in_page = if page_idx == num_pages - 1 {
|
|
313
|
+
let remainder = num_entries % page_nelmts;
|
|
314
|
+
if remainder == 0 {
|
|
315
|
+
page_nelmts
|
|
316
|
+
} else {
|
|
317
|
+
remainder
|
|
318
|
+
}
|
|
319
|
+
} else {
|
|
320
|
+
page_nelmts
|
|
321
|
+
};
|
|
322
|
+
let entry_bytes = checked_mul_usize(
|
|
323
|
+
entries_in_page,
|
|
324
|
+
usize::from(entry_size),
|
|
325
|
+
"extensible array data block page entry bytes",
|
|
326
|
+
)?;
|
|
327
|
+
let page_bytes = checked_add_usize(
|
|
328
|
+
entry_bytes,
|
|
329
|
+
4,
|
|
330
|
+
"extensible array data block page byte length",
|
|
331
|
+
)?;
|
|
332
|
+
len = checked_add_usize(len, page_bytes, "extensible array data block length")?;
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
Ok(len)
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
fn unpaged_data_block_len(
|
|
339
|
+
num_entries: usize,
|
|
340
|
+
offset_size: u8,
|
|
341
|
+
entry_size: u8,
|
|
342
|
+
sizeof_nelmts: usize,
|
|
343
|
+
) -> Result<usize> {
|
|
344
|
+
let entry_bytes = checked_mul_usize(
|
|
345
|
+
num_entries,
|
|
346
|
+
usize::from(entry_size),
|
|
347
|
+
"extensible array data block entry bytes",
|
|
348
|
+
)?;
|
|
349
|
+
let len = checked_add_usize(
|
|
350
|
+
ea_block_header_len(offset_size, sizeof_nelmts)?,
|
|
351
|
+
entry_bytes,
|
|
352
|
+
"extensible array data block length",
|
|
353
|
+
)?;
|
|
354
|
+
checked_add_usize(len, 4, "extensible array data block length")
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
/// A single raw entry.
|
|
358
|
+
struct EaRawEntry {
|
|
359
|
+
address: u64,
|
|
360
|
+
chunk_size: u64,
|
|
361
|
+
filter_mask: u32,
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
/// Read `count` entries from the cursor.
|
|
365
|
+
fn read_entries(
|
|
366
|
+
cursor: &mut Cursor<'_>,
|
|
367
|
+
count: usize,
|
|
368
|
+
is_filtered: bool,
|
|
369
|
+
offset_size: u8,
|
|
370
|
+
entry_size: u8,
|
|
371
|
+
) -> Result<Vec<EaRawEntry>> {
|
|
372
|
+
let mut entries = Vec::with_capacity(count);
|
|
373
|
+
for _ in 0..count {
|
|
374
|
+
let address = cursor.read_offset(offset_size)?;
|
|
375
|
+
let (chunk_size, filter_mask) = if is_filtered {
|
|
376
|
+
let chunk_size_len = entry_size
|
|
377
|
+
.checked_sub(offset_size)
|
|
378
|
+
.and_then(|remaining| remaining.checked_sub(4))
|
|
379
|
+
.ok_or_else(|| Error::InvalidData("invalid extensible array entry size".into()))?;
|
|
380
|
+
// 1–8 bytes wide (sized to the chunk), not a "length" of the superblock size.
|
|
381
|
+
let cs = cursor.read_uvar(chunk_size_len as usize)?;
|
|
382
|
+
let fm = cursor.read_u32_le()?;
|
|
383
|
+
(cs, fm)
|
|
384
|
+
} else {
|
|
385
|
+
(0, 0)
|
|
386
|
+
};
|
|
387
|
+
entries.push(EaRawEntry {
|
|
388
|
+
address,
|
|
389
|
+
chunk_size,
|
|
390
|
+
filter_mask,
|
|
391
|
+
});
|
|
392
|
+
}
|
|
393
|
+
Ok(entries)
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
/// Parse a data block and return its entries.
|
|
397
|
+
///
|
|
398
|
+
/// `sizeof_nelmts` is `ceil(max_nelmts_bits / 8)` — used for the block_off field.
|
|
399
|
+
#[allow(clippy::too_many_arguments)]
|
|
400
|
+
fn parse_data_block(
|
|
401
|
+
data: &[u8],
|
|
402
|
+
address: u64,
|
|
403
|
+
num_entries: usize,
|
|
404
|
+
is_filtered: bool,
|
|
405
|
+
max_page_bits: u8,
|
|
406
|
+
offset_size: u8,
|
|
407
|
+
entry_size: u8,
|
|
408
|
+
sizeof_nelmts: usize,
|
|
409
|
+
) -> Result<Vec<EaRawEntry>> {
|
|
410
|
+
let mut cursor = Cursor::new(data);
|
|
411
|
+
cursor.set_position(address);
|
|
412
|
+
|
|
413
|
+
let sig = cursor.read_bytes(4)?;
|
|
414
|
+
if sig != EADB_SIGNATURE {
|
|
415
|
+
return Err(Error::InvalidExtensibleArraySignature {
|
|
416
|
+
context: "data block signature mismatch",
|
|
417
|
+
});
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
let version = cursor.read_u8()?;
|
|
421
|
+
if version != 0 {
|
|
422
|
+
return Err(Error::Other(format!(
|
|
423
|
+
"unsupported extensible array data block version {}",
|
|
424
|
+
version
|
|
425
|
+
)));
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
let _client_id = cursor.read_u8()?;
|
|
429
|
+
let _header_address = cursor.read_offset(offset_size)?;
|
|
430
|
+
|
|
431
|
+
// Block offset: sizeof_nelmts bytes indicating this block's element index offset.
|
|
432
|
+
cursor.skip(sizeof_nelmts)?;
|
|
433
|
+
|
|
434
|
+
// Paging is used only when nelmts exceeds 2^page_bits.
|
|
435
|
+
let page_nelmts = ea_page_nelmts(max_page_bits)?;
|
|
436
|
+
|
|
437
|
+
if page_nelmts > 0 && num_entries > page_nelmts {
|
|
438
|
+
// Paged data block
|
|
439
|
+
let num_pages = num_entries.div_ceil(page_nelmts);
|
|
440
|
+
let bitmap_bytes = num_pages.div_ceil(8);
|
|
441
|
+
let page_bitmap = cursor.read_bytes(bitmap_bytes)?.to_vec();
|
|
442
|
+
|
|
443
|
+
let mut all_entries = Vec::with_capacity(num_entries);
|
|
444
|
+
for page_idx in 0..num_pages {
|
|
445
|
+
let byte_idx = page_idx / 8;
|
|
446
|
+
let bit_idx = page_idx % 8;
|
|
447
|
+
let page_initialized =
|
|
448
|
+
byte_idx < page_bitmap.len() && (page_bitmap[byte_idx] & (1 << bit_idx)) != 0;
|
|
449
|
+
|
|
450
|
+
let entries_in_page = if page_idx == num_pages - 1 {
|
|
451
|
+
let remainder = num_entries % page_nelmts;
|
|
452
|
+
if remainder == 0 {
|
|
453
|
+
page_nelmts
|
|
454
|
+
} else {
|
|
455
|
+
remainder
|
|
456
|
+
}
|
|
457
|
+
} else {
|
|
458
|
+
page_nelmts
|
|
459
|
+
};
|
|
460
|
+
|
|
461
|
+
if page_initialized {
|
|
462
|
+
let page_entries = read_entries(
|
|
463
|
+
&mut cursor,
|
|
464
|
+
entries_in_page,
|
|
465
|
+
is_filtered,
|
|
466
|
+
offset_size,
|
|
467
|
+
entry_size,
|
|
468
|
+
)?;
|
|
469
|
+
let _page_checksum = cursor.read_u32_le()?;
|
|
470
|
+
all_entries.extend(page_entries);
|
|
471
|
+
} else {
|
|
472
|
+
for _ in 0..entries_in_page {
|
|
473
|
+
all_entries.push(EaRawEntry {
|
|
474
|
+
address: u64::MAX,
|
|
475
|
+
chunk_size: 0,
|
|
476
|
+
filter_mask: 0,
|
|
477
|
+
});
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
Ok(all_entries)
|
|
482
|
+
} else {
|
|
483
|
+
// Non-paged data block
|
|
484
|
+
let entries = read_entries(
|
|
485
|
+
&mut cursor,
|
|
486
|
+
num_entries,
|
|
487
|
+
is_filtered,
|
|
488
|
+
offset_size,
|
|
489
|
+
entry_size,
|
|
490
|
+
)?;
|
|
491
|
+
let _checksum = cursor.read_u32_le()?;
|
|
492
|
+
Ok(entries)
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
/// Parse a secondary block and return its data block addresses.
|
|
497
|
+
fn parse_secondary_block(
|
|
498
|
+
data: &[u8],
|
|
499
|
+
address: u64,
|
|
500
|
+
num_dblk_addrs: usize,
|
|
501
|
+
offset_size: u8,
|
|
502
|
+
sizeof_nelmts: usize,
|
|
503
|
+
page_bitmap_bytes: usize,
|
|
504
|
+
) -> Result<Vec<u64>> {
|
|
505
|
+
let mut cursor = Cursor::new(data);
|
|
506
|
+
cursor.set_position(address);
|
|
507
|
+
|
|
508
|
+
let sig = cursor.read_bytes(4)?;
|
|
509
|
+
if sig != EASB_SIGNATURE {
|
|
510
|
+
return Err(Error::InvalidExtensibleArraySignature {
|
|
511
|
+
context: "secondary block signature mismatch",
|
|
512
|
+
});
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
let version = cursor.read_u8()?;
|
|
516
|
+
if version != 0 {
|
|
517
|
+
return Err(Error::Other(format!(
|
|
518
|
+
"unsupported extensible array secondary block version {}",
|
|
519
|
+
version
|
|
520
|
+
)));
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
let _client_id = cursor.read_u8()?;
|
|
524
|
+
let _header_address = cursor.read_offset(offset_size)?;
|
|
525
|
+
cursor.skip(sizeof_nelmts)?;
|
|
526
|
+
|
|
527
|
+
if page_bitmap_bytes > 0 {
|
|
528
|
+
cursor.skip(page_bitmap_bytes)?;
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
let mut addrs = Vec::with_capacity(num_dblk_addrs);
|
|
532
|
+
for _ in 0..num_dblk_addrs {
|
|
533
|
+
addrs.push(cursor.read_offset(offset_size)?);
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
// Skip checksum
|
|
537
|
+
let _checksum = cursor.read_u32_le()?;
|
|
538
|
+
|
|
539
|
+
Ok(addrs)
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
fn parse_secondary_block_storage(
|
|
543
|
+
storage: &dyn Storage,
|
|
544
|
+
address: u64,
|
|
545
|
+
num_dblk_addrs: usize,
|
|
546
|
+
offset_size: u8,
|
|
547
|
+
sizeof_nelmts: usize,
|
|
548
|
+
page_bitmap_bytes: usize,
|
|
549
|
+
) -> Result<Vec<u64>> {
|
|
550
|
+
let read_len = secondary_block_len(
|
|
551
|
+
num_dblk_addrs,
|
|
552
|
+
offset_size,
|
|
553
|
+
sizeof_nelmts,
|
|
554
|
+
page_bitmap_bytes,
|
|
555
|
+
)?;
|
|
556
|
+
let bytes = storage.read_range(address, read_len)?;
|
|
557
|
+
parse_secondary_block(
|
|
558
|
+
bytes.as_ref(),
|
|
559
|
+
0,
|
|
560
|
+
num_dblk_addrs,
|
|
561
|
+
offset_size,
|
|
562
|
+
sizeof_nelmts,
|
|
563
|
+
page_bitmap_bytes,
|
|
564
|
+
)
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
fn read_entry_at(
|
|
568
|
+
data: &[u8],
|
|
569
|
+
position: u64,
|
|
570
|
+
is_filtered: bool,
|
|
571
|
+
offset_size: u8,
|
|
572
|
+
entry_size: u8,
|
|
573
|
+
) -> Result<EaRawEntry> {
|
|
574
|
+
let mut cursor = Cursor::new(data);
|
|
575
|
+
cursor.set_position(position);
|
|
576
|
+
let mut entries = read_entries(&mut cursor, 1, is_filtered, offset_size, entry_size)?;
|
|
577
|
+
entries
|
|
578
|
+
.pop()
|
|
579
|
+
.ok_or_else(|| Error::InvalidData("missing extensible array entry".into()))
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
fn read_entry_at_storage(
|
|
583
|
+
storage: &dyn Storage,
|
|
584
|
+
position: u64,
|
|
585
|
+
is_filtered: bool,
|
|
586
|
+
offset_size: u8,
|
|
587
|
+
entry_size: u8,
|
|
588
|
+
) -> Result<EaRawEntry> {
|
|
589
|
+
let bytes = storage.read_range(position, usize::from(entry_size))?;
|
|
590
|
+
let mut cursor = Cursor::new(bytes.as_ref());
|
|
591
|
+
let mut entries = read_entries(&mut cursor, 1, is_filtered, offset_size, entry_size)?;
|
|
592
|
+
entries
|
|
593
|
+
.pop()
|
|
594
|
+
.ok_or_else(|| Error::InvalidData("missing extensible array entry".into()))
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
fn linear_target_offsets(
|
|
598
|
+
dataset_shape: &[u64],
|
|
599
|
+
chunk_dims: &[u32],
|
|
600
|
+
chunk_bounds: Option<(&[u64], &[u64])>,
|
|
601
|
+
) -> Vec<(usize, Vec<u64>)> {
|
|
602
|
+
let ndim = dataset_shape.len();
|
|
603
|
+
let chunks_per_dim: Vec<u64> = (0..ndim)
|
|
604
|
+
.map(|i| dataset_shape[i].div_ceil(chunk_dims[i] as u64))
|
|
605
|
+
.collect();
|
|
606
|
+
|
|
607
|
+
if ndim == 0 {
|
|
608
|
+
return vec![(0, Vec::new())];
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
let (first_chunk, last_chunk): (Vec<u64>, Vec<u64>) = match chunk_bounds {
|
|
612
|
+
Some((first, last)) => (first.to_vec(), last.to_vec()),
|
|
613
|
+
None => (
|
|
614
|
+
vec![0u64; ndim],
|
|
615
|
+
chunks_per_dim
|
|
616
|
+
.iter()
|
|
617
|
+
.map(|count| count.saturating_sub(1))
|
|
618
|
+
.collect(),
|
|
619
|
+
),
|
|
620
|
+
};
|
|
621
|
+
|
|
622
|
+
let mut targets = Vec::new();
|
|
623
|
+
let mut chunk_indices = first_chunk.clone();
|
|
624
|
+
loop {
|
|
625
|
+
let mut linear_idx = 0u64;
|
|
626
|
+
for (dim, chunk_index) in chunk_indices.iter().enumerate() {
|
|
627
|
+
linear_idx = linear_idx * chunks_per_dim[dim] + chunk_index;
|
|
628
|
+
}
|
|
629
|
+
let offsets = chunk_indices
|
|
630
|
+
.iter()
|
|
631
|
+
.enumerate()
|
|
632
|
+
.map(|(dim, chunk_index)| chunk_index * u64::from(chunk_dims[dim]))
|
|
633
|
+
.collect();
|
|
634
|
+
targets.push((linear_idx as usize, offsets));
|
|
635
|
+
|
|
636
|
+
let mut advanced = false;
|
|
637
|
+
for dim in (0..ndim).rev() {
|
|
638
|
+
if chunk_indices[dim] < last_chunk[dim] {
|
|
639
|
+
chunk_indices[dim] += 1;
|
|
640
|
+
if dim + 1 < ndim {
|
|
641
|
+
chunk_indices[(dim + 1)..ndim].copy_from_slice(&first_chunk[(dim + 1)..ndim]);
|
|
642
|
+
}
|
|
643
|
+
advanced = true;
|
|
644
|
+
break;
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
if !advanced {
|
|
649
|
+
break;
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
targets
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
#[allow(clippy::too_many_arguments)]
|
|
657
|
+
fn read_data_block_entry(
|
|
658
|
+
data: &[u8],
|
|
659
|
+
address: u64,
|
|
660
|
+
num_entries: usize,
|
|
661
|
+
local_idx: usize,
|
|
662
|
+
is_filtered: bool,
|
|
663
|
+
max_page_bits: u8,
|
|
664
|
+
offset_size: u8,
|
|
665
|
+
entry_size: u8,
|
|
666
|
+
sizeof_nelmts: usize,
|
|
667
|
+
) -> Result<EaRawEntry> {
|
|
668
|
+
let mut cursor = Cursor::new(data);
|
|
669
|
+
cursor.set_position(address);
|
|
670
|
+
|
|
671
|
+
let sig = cursor.read_bytes(4)?;
|
|
672
|
+
if sig != EADB_SIGNATURE {
|
|
673
|
+
return Err(Error::InvalidExtensibleArraySignature {
|
|
674
|
+
context: "data block signature mismatch",
|
|
675
|
+
});
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
let version = cursor.read_u8()?;
|
|
679
|
+
if version != 0 {
|
|
680
|
+
return Err(Error::Other(format!(
|
|
681
|
+
"unsupported extensible array data block version {}",
|
|
682
|
+
version
|
|
683
|
+
)));
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
let _client_id = cursor.read_u8()?;
|
|
687
|
+
let _header_address = cursor.read_offset(offset_size)?;
|
|
688
|
+
cursor.skip(sizeof_nelmts)?;
|
|
689
|
+
|
|
690
|
+
let page_nelmts = ea_page_nelmts(max_page_bits)?;
|
|
691
|
+
|
|
692
|
+
if page_nelmts > 0 && num_entries > page_nelmts {
|
|
693
|
+
let num_pages = num_entries.div_ceil(page_nelmts);
|
|
694
|
+
let bitmap_bytes = num_pages.div_ceil(8);
|
|
695
|
+
let page_bitmap = cursor.read_bytes(bitmap_bytes)?.to_vec();
|
|
696
|
+
let data_start = cursor.position();
|
|
697
|
+
|
|
698
|
+
let target_page = local_idx / page_nelmts;
|
|
699
|
+
let within_page = local_idx % page_nelmts;
|
|
700
|
+
let byte_idx = target_page / 8;
|
|
701
|
+
let bit_idx = target_page % 8;
|
|
702
|
+
let page_initialized =
|
|
703
|
+
byte_idx < page_bitmap.len() && (page_bitmap[byte_idx] & (1 << bit_idx)) != 0;
|
|
704
|
+
if !page_initialized {
|
|
705
|
+
return Ok(EaRawEntry {
|
|
706
|
+
address: u64::MAX,
|
|
707
|
+
chunk_size: 0,
|
|
708
|
+
filter_mask: 0,
|
|
709
|
+
});
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
let mut page_start = data_start;
|
|
713
|
+
for page_idx in 0..target_page {
|
|
714
|
+
let entries_in_page = if page_idx == num_pages - 1 {
|
|
715
|
+
let remainder = num_entries % page_nelmts;
|
|
716
|
+
if remainder == 0 {
|
|
717
|
+
page_nelmts
|
|
718
|
+
} else {
|
|
719
|
+
remainder
|
|
720
|
+
}
|
|
721
|
+
} else {
|
|
722
|
+
page_nelmts
|
|
723
|
+
};
|
|
724
|
+
let page_byte_idx = page_idx / 8;
|
|
725
|
+
let page_bit_idx = page_idx % 8;
|
|
726
|
+
let initialized = page_byte_idx < page_bitmap.len()
|
|
727
|
+
&& (page_bitmap[page_byte_idx] & (1 << page_bit_idx)) != 0;
|
|
728
|
+
if initialized {
|
|
729
|
+
page_start += (entries_in_page * entry_size as usize + 4) as u64;
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
|
|
733
|
+
let position = page_start + (within_page * entry_size as usize) as u64;
|
|
734
|
+
return read_entry_at(data, position, is_filtered, offset_size, entry_size);
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
let position = cursor.position() + (local_idx * entry_size as usize) as u64;
|
|
738
|
+
read_entry_at(data, position, is_filtered, offset_size, entry_size)
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
#[allow(clippy::too_many_arguments)]
|
|
742
|
+
fn read_data_block_entry_storage(
|
|
743
|
+
storage: &dyn Storage,
|
|
744
|
+
address: u64,
|
|
745
|
+
num_entries: usize,
|
|
746
|
+
local_idx: usize,
|
|
747
|
+
is_filtered: bool,
|
|
748
|
+
max_page_bits: u8,
|
|
749
|
+
offset_size: u8,
|
|
750
|
+
entry_size: u8,
|
|
751
|
+
sizeof_nelmts: usize,
|
|
752
|
+
) -> Result<EaRawEntry> {
|
|
753
|
+
let header_len = 4 + 1 + 1 + usize::from(offset_size) + sizeof_nelmts;
|
|
754
|
+
let header = storage.read_range(address, header_len)?;
|
|
755
|
+
let mut cursor = Cursor::new(header.as_ref());
|
|
756
|
+
|
|
757
|
+
let sig = cursor.read_bytes(4)?;
|
|
758
|
+
if sig != EADB_SIGNATURE {
|
|
759
|
+
return Err(Error::InvalidExtensibleArraySignature {
|
|
760
|
+
context: "data block signature mismatch",
|
|
761
|
+
});
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
let version = cursor.read_u8()?;
|
|
765
|
+
if version != 0 {
|
|
766
|
+
return Err(Error::Other(format!(
|
|
767
|
+
"unsupported extensible array data block version {}",
|
|
768
|
+
version
|
|
769
|
+
)));
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
let _client_id = cursor.read_u8()?;
|
|
773
|
+
let _header_address = cursor.read_offset(offset_size)?;
|
|
774
|
+
cursor.skip(sizeof_nelmts)?;
|
|
775
|
+
|
|
776
|
+
let base = checked_add_u64(
|
|
777
|
+
address,
|
|
778
|
+
checked_u64_from_usize(header_len, "EA data block header length")?,
|
|
779
|
+
"EA data block entry base",
|
|
780
|
+
)?;
|
|
781
|
+
let page_nelmts = ea_page_nelmts(max_page_bits)?;
|
|
782
|
+
|
|
783
|
+
if page_nelmts > 0 && num_entries > page_nelmts {
|
|
784
|
+
let num_pages = num_entries.div_ceil(page_nelmts);
|
|
785
|
+
let bitmap_bytes = num_pages.div_ceil(8);
|
|
786
|
+
let page_bitmap = storage.read_range(base, bitmap_bytes)?;
|
|
787
|
+
let data_start = checked_add_u64(
|
|
788
|
+
base,
|
|
789
|
+
checked_u64_from_usize(bitmap_bytes, "EA bitmap size")?,
|
|
790
|
+
"EA data block page data start",
|
|
791
|
+
)?;
|
|
792
|
+
|
|
793
|
+
let target_page = local_idx / page_nelmts;
|
|
794
|
+
let within_page = local_idx % page_nelmts;
|
|
795
|
+
let byte_idx = target_page / 8;
|
|
796
|
+
let bit_idx = target_page % 8;
|
|
797
|
+
let page_initialized =
|
|
798
|
+
byte_idx < page_bitmap.len() && (page_bitmap[byte_idx] & (1 << bit_idx)) != 0;
|
|
799
|
+
if !page_initialized {
|
|
800
|
+
return Ok(EaRawEntry {
|
|
801
|
+
address: u64::MAX,
|
|
802
|
+
chunk_size: 0,
|
|
803
|
+
filter_mask: 0,
|
|
804
|
+
});
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
let mut page_start = data_start;
|
|
808
|
+
for page_idx in 0..target_page {
|
|
809
|
+
let entries_in_page = if page_idx == num_pages - 1 {
|
|
810
|
+
let remainder = num_entries % page_nelmts;
|
|
811
|
+
if remainder == 0 {
|
|
812
|
+
page_nelmts
|
|
813
|
+
} else {
|
|
814
|
+
remainder
|
|
815
|
+
}
|
|
816
|
+
} else {
|
|
817
|
+
page_nelmts
|
|
818
|
+
};
|
|
819
|
+
let page_byte_idx = page_idx / 8;
|
|
820
|
+
let page_bit_idx = page_idx % 8;
|
|
821
|
+
let initialized = page_byte_idx < page_bitmap.len()
|
|
822
|
+
&& (page_bitmap[page_byte_idx] & (1 << page_bit_idx)) != 0;
|
|
823
|
+
if initialized {
|
|
824
|
+
let entry_bytes = checked_mul_usize(
|
|
825
|
+
entries_in_page,
|
|
826
|
+
usize::from(entry_size),
|
|
827
|
+
"EA page entry bytes",
|
|
828
|
+
)?;
|
|
829
|
+
let page_size = checked_add_usize(entry_bytes, 4, "EA page size")?;
|
|
830
|
+
page_start = checked_add_u64(
|
|
831
|
+
page_start,
|
|
832
|
+
checked_u64_from_usize(page_size, "EA page size")?,
|
|
833
|
+
"EA page start",
|
|
834
|
+
)?;
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
|
|
838
|
+
let within_page_offset =
|
|
839
|
+
checked_mul_usize(within_page, usize::from(entry_size), "EA page entry offset")?;
|
|
840
|
+
let position = checked_add_u64(
|
|
841
|
+
page_start,
|
|
842
|
+
checked_u64_from_usize(within_page_offset, "EA page entry offset")?,
|
|
843
|
+
"EA page entry position",
|
|
844
|
+
)?;
|
|
845
|
+
return read_entry_at_storage(storage, position, is_filtered, offset_size, entry_size);
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
let local_offset = checked_mul_usize(local_idx, usize::from(entry_size), "EA entry offset")?;
|
|
849
|
+
let position = checked_add_u64(
|
|
850
|
+
base,
|
|
851
|
+
checked_u64_from_usize(local_offset, "EA entry offset")?,
|
|
852
|
+
"EA entry position",
|
|
853
|
+
)?;
|
|
854
|
+
read_entry_at_storage(storage, position, is_filtered, offset_size, entry_size)
|
|
855
|
+
}
|
|
856
|
+
|
|
857
|
+
#[allow(clippy::too_many_arguments)]
|
|
858
|
+
fn read_data_block_storage(
|
|
859
|
+
storage: &dyn Storage,
|
|
860
|
+
address: u64,
|
|
861
|
+
num_entries: usize,
|
|
862
|
+
is_filtered: bool,
|
|
863
|
+
max_page_bits: u8,
|
|
864
|
+
offset_size: u8,
|
|
865
|
+
entry_size: u8,
|
|
866
|
+
sizeof_nelmts: usize,
|
|
867
|
+
) -> Result<Vec<EaRawEntry>> {
|
|
868
|
+
let page_nelmts = ea_page_nelmts(max_page_bits)?;
|
|
869
|
+
let read_len = if page_nelmts > 0 && num_entries > page_nelmts {
|
|
870
|
+
let header_len = ea_block_header_len(offset_size, sizeof_nelmts)?;
|
|
871
|
+
let num_pages = num_entries.div_ceil(page_nelmts);
|
|
872
|
+
let bitmap_bytes = num_pages.div_ceil(8);
|
|
873
|
+
let bitmap_end = checked_add_usize(
|
|
874
|
+
header_len,
|
|
875
|
+
bitmap_bytes,
|
|
876
|
+
"extensible array data block bitmap range",
|
|
877
|
+
)?;
|
|
878
|
+
let prefix = storage.read_range(address, bitmap_end)?;
|
|
879
|
+
let page_bitmap = &prefix.as_ref()[header_len..bitmap_end];
|
|
880
|
+
data_block_len_from_bitmap(
|
|
881
|
+
num_entries,
|
|
882
|
+
page_nelmts,
|
|
883
|
+
entry_size,
|
|
884
|
+
offset_size,
|
|
885
|
+
sizeof_nelmts,
|
|
886
|
+
page_bitmap,
|
|
887
|
+
)?
|
|
888
|
+
} else {
|
|
889
|
+
unpaged_data_block_len(num_entries, offset_size, entry_size, sizeof_nelmts)?
|
|
890
|
+
};
|
|
891
|
+
|
|
892
|
+
let block = storage.read_range(address, read_len)?;
|
|
893
|
+
parse_data_block(
|
|
894
|
+
block.as_ref(),
|
|
895
|
+
0,
|
|
896
|
+
num_entries,
|
|
897
|
+
is_filtered,
|
|
898
|
+
max_page_bits,
|
|
899
|
+
offset_size,
|
|
900
|
+
entry_size,
|
|
901
|
+
sizeof_nelmts,
|
|
902
|
+
)
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
#[allow(clippy::too_many_arguments)]
|
|
906
|
+
fn collect_extensible_array_chunk_entries_bounded(
|
|
907
|
+
data: &[u8],
|
|
908
|
+
header: &EaHeader,
|
|
909
|
+
offset_size: u8,
|
|
910
|
+
dataset_shape: &[u64],
|
|
911
|
+
chunk_dims: &[u32],
|
|
912
|
+
chunk_bounds: (&[u64], &[u64]),
|
|
913
|
+
sb_layout: &[(u64, u64)],
|
|
914
|
+
sizeof_nelmts: usize,
|
|
915
|
+
) -> Result<Vec<ChunkEntry>> {
|
|
916
|
+
let is_filtered = header.client_id == 1;
|
|
917
|
+
let targets = linear_target_offsets(dataset_shape, chunk_dims, Some(chunk_bounds));
|
|
918
|
+
|
|
919
|
+
let mut cursor = Cursor::new(data);
|
|
920
|
+
cursor.set_position(header.index_block_address);
|
|
921
|
+
|
|
922
|
+
let sig = cursor.read_bytes(4)?;
|
|
923
|
+
if sig != EAIB_SIGNATURE {
|
|
924
|
+
return Err(Error::InvalidExtensibleArraySignature {
|
|
925
|
+
context: "index block signature mismatch",
|
|
926
|
+
});
|
|
927
|
+
}
|
|
928
|
+
|
|
929
|
+
let version = cursor.read_u8()?;
|
|
930
|
+
if version != 0 {
|
|
931
|
+
return Err(Error::Other(format!(
|
|
932
|
+
"unsupported extensible array index block version {}",
|
|
933
|
+
version
|
|
934
|
+
)));
|
|
935
|
+
}
|
|
936
|
+
|
|
937
|
+
let _client_id = cursor.read_u8()?;
|
|
938
|
+
let _header_address = cursor.read_offset(offset_size)?;
|
|
939
|
+
|
|
940
|
+
let num_inline = header.idx_blk_elmts as usize;
|
|
941
|
+
let inline_start = cursor.position();
|
|
942
|
+
cursor.skip(num_inline * header.element_size as usize)?;
|
|
943
|
+
|
|
944
|
+
let ndblk_addrs = 2 * header.sec_blk_min_data_ptrs as usize;
|
|
945
|
+
let mut direct_dblk_addrs = Vec::with_capacity(ndblk_addrs);
|
|
946
|
+
for _ in 0..ndblk_addrs {
|
|
947
|
+
direct_dblk_addrs.push(cursor.read_offset(offset_size)?);
|
|
948
|
+
}
|
|
949
|
+
|
|
950
|
+
let nsblks = sb_layout.len();
|
|
951
|
+
let nsblk_addrs = nsblks.saturating_sub(ndblk_addrs);
|
|
952
|
+
let mut sec_block_addrs = Vec::with_capacity(nsblk_addrs);
|
|
953
|
+
for _ in 0..nsblk_addrs {
|
|
954
|
+
sec_block_addrs.push(cursor.read_offset(offset_size)?);
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
let mut secondary_block_cache: Vec<Option<Vec<u64>>> = vec![None; sec_block_addrs.len()];
|
|
958
|
+
let mut entries = Vec::new();
|
|
959
|
+
|
|
960
|
+
for (linear_idx, offsets) in targets {
|
|
961
|
+
let raw = if linear_idx < num_inline {
|
|
962
|
+
read_entry_at(
|
|
963
|
+
data,
|
|
964
|
+
inline_start + (linear_idx * header.element_size as usize) as u64,
|
|
965
|
+
is_filtered,
|
|
966
|
+
offset_size,
|
|
967
|
+
header.element_size,
|
|
968
|
+
)?
|
|
969
|
+
} else {
|
|
970
|
+
let mut relative_idx = (linear_idx - num_inline) as u64;
|
|
971
|
+
let mut sb_idx = None;
|
|
972
|
+
for (candidate_idx, (elmts_per_dblk, num_dblks)) in sb_layout.iter().enumerate() {
|
|
973
|
+
let capacity = elmts_per_dblk * num_dblks;
|
|
974
|
+
if relative_idx < capacity {
|
|
975
|
+
sb_idx = Some(candidate_idx);
|
|
976
|
+
break;
|
|
977
|
+
}
|
|
978
|
+
relative_idx -= capacity;
|
|
979
|
+
}
|
|
980
|
+
|
|
981
|
+
let Some(sb_idx) = sb_idx else {
|
|
982
|
+
continue;
|
|
983
|
+
};
|
|
984
|
+
let (elmts_per_dblk, _) = sb_layout[sb_idx];
|
|
985
|
+
let dblk_idx = (relative_idx / elmts_per_dblk) as usize;
|
|
986
|
+
let local_idx = (relative_idx % elmts_per_dblk) as usize;
|
|
987
|
+
|
|
988
|
+
let dblk_addr = if sb_idx < 2 {
|
|
989
|
+
let base = sb_layout[..sb_idx]
|
|
990
|
+
.iter()
|
|
991
|
+
.map(|(_, num_dblks)| *num_dblks as usize)
|
|
992
|
+
.sum::<usize>();
|
|
993
|
+
*direct_dblk_addrs.get(base + dblk_idx).unwrap_or(&u64::MAX)
|
|
994
|
+
} else {
|
|
995
|
+
let sec_cache_idx = sb_idx - 2;
|
|
996
|
+
if secondary_block_cache[sec_cache_idx].is_none() {
|
|
997
|
+
let sec_addr = sec_block_addrs
|
|
998
|
+
.get(sec_cache_idx)
|
|
999
|
+
.copied()
|
|
1000
|
+
.unwrap_or(u64::MAX);
|
|
1001
|
+
if Cursor::is_undefined_offset(sec_addr, offset_size) {
|
|
1002
|
+
secondary_block_cache[sec_cache_idx] = Some(Vec::new());
|
|
1003
|
+
} else {
|
|
1004
|
+
let (_, num_dblks) = sb_layout[sb_idx];
|
|
1005
|
+
let page_bitmap_bytes = secondary_page_bitmap_bytes(
|
|
1006
|
+
header.max_dblk_page_nelmts_bits,
|
|
1007
|
+
elmts_per_dblk,
|
|
1008
|
+
num_dblks,
|
|
1009
|
+
)?;
|
|
1010
|
+
secondary_block_cache[sec_cache_idx] = Some(parse_secondary_block(
|
|
1011
|
+
data,
|
|
1012
|
+
sec_addr,
|
|
1013
|
+
checked_usize_from_u64(
|
|
1014
|
+
num_dblks,
|
|
1015
|
+
"extensible array secondary block data block count",
|
|
1016
|
+
)?,
|
|
1017
|
+
offset_size,
|
|
1018
|
+
sizeof_nelmts,
|
|
1019
|
+
page_bitmap_bytes,
|
|
1020
|
+
)?);
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
|
|
1024
|
+
secondary_block_cache[sec_cache_idx]
|
|
1025
|
+
.as_ref()
|
|
1026
|
+
.and_then(|addrs| addrs.get(dblk_idx))
|
|
1027
|
+
.copied()
|
|
1028
|
+
.unwrap_or(u64::MAX)
|
|
1029
|
+
};
|
|
1030
|
+
|
|
1031
|
+
if Cursor::is_undefined_offset(dblk_addr, offset_size) {
|
|
1032
|
+
continue;
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
read_data_block_entry(
|
|
1036
|
+
data,
|
|
1037
|
+
dblk_addr,
|
|
1038
|
+
elmts_per_dblk as usize,
|
|
1039
|
+
local_idx,
|
|
1040
|
+
is_filtered,
|
|
1041
|
+
header.max_dblk_page_nelmts_bits,
|
|
1042
|
+
offset_size,
|
|
1043
|
+
header.element_size,
|
|
1044
|
+
sizeof_nelmts,
|
|
1045
|
+
)?
|
|
1046
|
+
};
|
|
1047
|
+
|
|
1048
|
+
if Cursor::is_undefined_offset(raw.address, offset_size) {
|
|
1049
|
+
continue;
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
entries.push(ChunkEntry {
|
|
1053
|
+
address: raw.address,
|
|
1054
|
+
size: raw.chunk_size,
|
|
1055
|
+
filter_mask: raw.filter_mask,
|
|
1056
|
+
offsets,
|
|
1057
|
+
});
|
|
1058
|
+
}
|
|
1059
|
+
|
|
1060
|
+
Ok(entries)
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
/// Collect chunk entries from an Extensible Array index.
|
|
1064
|
+
///
|
|
1065
|
+
/// Walks the EAHD → EAIB → (EADB / EASB → EADB) hierarchy and converts
|
|
1066
|
+
/// linear entry indices to multi-dimensional chunk offsets.
|
|
1067
|
+
pub fn collect_extensible_array_chunk_entries(
|
|
1068
|
+
data: &[u8],
|
|
1069
|
+
header_address: u64,
|
|
1070
|
+
offset_size: u8,
|
|
1071
|
+
length_size: u8,
|
|
1072
|
+
dataset_shape: &[u64],
|
|
1073
|
+
chunk_dims: &[u32],
|
|
1074
|
+
chunk_bounds: Option<(&[u64], &[u64])>,
|
|
1075
|
+
) -> Result<Vec<ChunkEntry>> {
|
|
1076
|
+
let header = parse_header(data, header_address, offset_size, length_size)?;
|
|
1077
|
+
|
|
1078
|
+
if Cursor::is_undefined_offset(header.index_block_address, offset_size) {
|
|
1079
|
+
return Ok(Vec::new());
|
|
1080
|
+
}
|
|
1081
|
+
|
|
1082
|
+
let is_filtered = header.client_id == 1;
|
|
1083
|
+
let sb_layout = compute_super_block_layout(&header);
|
|
1084
|
+
let sizeof_nelmts = (header._max_nelmts_bits as usize).div_ceil(8);
|
|
1085
|
+
|
|
1086
|
+
if let Some(bounds) = chunk_bounds {
|
|
1087
|
+
return collect_extensible_array_chunk_entries_bounded(
|
|
1088
|
+
data,
|
|
1089
|
+
&header,
|
|
1090
|
+
offset_size,
|
|
1091
|
+
dataset_shape,
|
|
1092
|
+
chunk_dims,
|
|
1093
|
+
bounds,
|
|
1094
|
+
&sb_layout,
|
|
1095
|
+
sizeof_nelmts,
|
|
1096
|
+
);
|
|
1097
|
+
}
|
|
1098
|
+
|
|
1099
|
+
// Parse the index block.
|
|
1100
|
+
let mut cursor = Cursor::new(data);
|
|
1101
|
+
cursor.set_position(header.index_block_address);
|
|
1102
|
+
|
|
1103
|
+
let sig = cursor.read_bytes(4)?;
|
|
1104
|
+
if sig != EAIB_SIGNATURE {
|
|
1105
|
+
return Err(Error::InvalidExtensibleArraySignature {
|
|
1106
|
+
context: "index block signature mismatch",
|
|
1107
|
+
});
|
|
1108
|
+
}
|
|
1109
|
+
|
|
1110
|
+
let version = cursor.read_u8()?;
|
|
1111
|
+
if version != 0 {
|
|
1112
|
+
return Err(Error::Other(format!(
|
|
1113
|
+
"unsupported extensible array index block version {}",
|
|
1114
|
+
version
|
|
1115
|
+
)));
|
|
1116
|
+
}
|
|
1117
|
+
|
|
1118
|
+
let _client_id = cursor.read_u8()?;
|
|
1119
|
+
let _header_address = cursor.read_offset(offset_size)?;
|
|
1120
|
+
|
|
1121
|
+
// 1. Inline elements (idx_blk_elmts entries stored directly).
|
|
1122
|
+
let num_inline = header.idx_blk_elmts as usize;
|
|
1123
|
+
let inline_entries = read_entries(
|
|
1124
|
+
&mut cursor,
|
|
1125
|
+
num_inline,
|
|
1126
|
+
is_filtered,
|
|
1127
|
+
offset_size,
|
|
1128
|
+
header.element_size,
|
|
1129
|
+
)?;
|
|
1130
|
+
|
|
1131
|
+
// 2. Data block addresses stored directly in the index block.
|
|
1132
|
+
// The number is 2 * sec_blk_min_data_ptrs (from HDF5: EA_IBLOCK_NDBLK_ADDRS).
|
|
1133
|
+
let ndblk_addrs = 2 * header.sec_blk_min_data_ptrs as usize;
|
|
1134
|
+
let mut direct_dblk_addrs = Vec::with_capacity(ndblk_addrs);
|
|
1135
|
+
for _ in 0..ndblk_addrs {
|
|
1136
|
+
direct_dblk_addrs.push(cursor.read_offset(offset_size)?);
|
|
1137
|
+
}
|
|
1138
|
+
|
|
1139
|
+
// 3. Secondary block addresses for super blocks 2+.
|
|
1140
|
+
// nsblk_addrs = max(0, nsblks - ndblk_addrs) where nsblks is the total
|
|
1141
|
+
// number of super blocks needed to cover nelmts.
|
|
1142
|
+
// compute_super_block_layout already stops once capacity >= nelmts,
|
|
1143
|
+
// so sb_layout.len() is the total number of super blocks needed.
|
|
1144
|
+
let nsblks = sb_layout.len();
|
|
1145
|
+
|
|
1146
|
+
let nsblk_addrs = nsblks.saturating_sub(ndblk_addrs);
|
|
1147
|
+
let mut sec_block_addrs = Vec::with_capacity(nsblk_addrs);
|
|
1148
|
+
for _ in 0..nsblk_addrs {
|
|
1149
|
+
sec_block_addrs.push(cursor.read_offset(offset_size)?);
|
|
1150
|
+
}
|
|
1151
|
+
|
|
1152
|
+
// Skip checksum at end of index block
|
|
1153
|
+
let _checksum = cursor.read_u32_le()?;
|
|
1154
|
+
|
|
1155
|
+
// Now collect all entries.
|
|
1156
|
+
let mut all_entries: Vec<EaRawEntry> = Vec::new();
|
|
1157
|
+
|
|
1158
|
+
// Inline entries
|
|
1159
|
+
all_entries.extend(inline_entries);
|
|
1160
|
+
|
|
1161
|
+
// Data blocks from direct addresses (super blocks 0-1)
|
|
1162
|
+
let mut dblk_addr_idx = 0;
|
|
1163
|
+
for sb_idx_iter in 0..2usize.min(nsblks) {
|
|
1164
|
+
if sb_idx_iter >= sb_layout.len() {
|
|
1165
|
+
break;
|
|
1166
|
+
}
|
|
1167
|
+
let (elmts_per_dblk, num_dblks) = sb_layout[sb_idx_iter];
|
|
1168
|
+
for _ in 0..num_dblks {
|
|
1169
|
+
if dblk_addr_idx >= direct_dblk_addrs.len() {
|
|
1170
|
+
break;
|
|
1171
|
+
}
|
|
1172
|
+
let dblk_addr = direct_dblk_addrs[dblk_addr_idx];
|
|
1173
|
+
dblk_addr_idx += 1;
|
|
1174
|
+
|
|
1175
|
+
if Cursor::is_undefined_offset(dblk_addr, offset_size) {
|
|
1176
|
+
for _ in 0..elmts_per_dblk {
|
|
1177
|
+
all_entries.push(EaRawEntry {
|
|
1178
|
+
address: u64::MAX,
|
|
1179
|
+
chunk_size: 0,
|
|
1180
|
+
filter_mask: 0,
|
|
1181
|
+
});
|
|
1182
|
+
}
|
|
1183
|
+
} else {
|
|
1184
|
+
let dblk_entries = parse_data_block(
|
|
1185
|
+
data,
|
|
1186
|
+
dblk_addr,
|
|
1187
|
+
elmts_per_dblk as usize,
|
|
1188
|
+
is_filtered,
|
|
1189
|
+
header.max_dblk_page_nelmts_bits,
|
|
1190
|
+
offset_size,
|
|
1191
|
+
header.element_size,
|
|
1192
|
+
sizeof_nelmts,
|
|
1193
|
+
)?;
|
|
1194
|
+
all_entries.extend(dblk_entries);
|
|
1195
|
+
}
|
|
1196
|
+
}
|
|
1197
|
+
}
|
|
1198
|
+
|
|
1199
|
+
// Data blocks from super blocks 2+ (via secondary blocks)
|
|
1200
|
+
for (sec_idx, &sec_addr) in sec_block_addrs.iter().enumerate() {
|
|
1201
|
+
let sb_idx_iter = sec_idx + 2;
|
|
1202
|
+
if sb_idx_iter >= sb_layout.len() {
|
|
1203
|
+
break;
|
|
1204
|
+
}
|
|
1205
|
+
let (elmts_per_dblk, num_dblks) = sb_layout[sb_idx_iter];
|
|
1206
|
+
|
|
1207
|
+
if Cursor::is_undefined_offset(sec_addr, offset_size) {
|
|
1208
|
+
for _ in 0..(elmts_per_dblk * num_dblks) {
|
|
1209
|
+
all_entries.push(EaRawEntry {
|
|
1210
|
+
address: u64::MAX,
|
|
1211
|
+
chunk_size: 0,
|
|
1212
|
+
filter_mask: 0,
|
|
1213
|
+
});
|
|
1214
|
+
}
|
|
1215
|
+
continue;
|
|
1216
|
+
}
|
|
1217
|
+
|
|
1218
|
+
// Per HDF5 spec III.H "Extensible Array Secondary Block", the secondary
|
|
1219
|
+
// block contains a page initialization bitmap when data blocks are paged.
|
|
1220
|
+
// Bitmap size = ceil(num_dblks * pages_per_dblk / 8).
|
|
1221
|
+
let page_bitmap_bytes = secondary_page_bitmap_bytes(
|
|
1222
|
+
header.max_dblk_page_nelmts_bits,
|
|
1223
|
+
elmts_per_dblk,
|
|
1224
|
+
num_dblks,
|
|
1225
|
+
)?;
|
|
1226
|
+
let dblk_addrs = parse_secondary_block(
|
|
1227
|
+
data,
|
|
1228
|
+
sec_addr,
|
|
1229
|
+
checked_usize_from_u64(
|
|
1230
|
+
num_dblks,
|
|
1231
|
+
"extensible array secondary block data block count",
|
|
1232
|
+
)?,
|
|
1233
|
+
offset_size,
|
|
1234
|
+
sizeof_nelmts,
|
|
1235
|
+
page_bitmap_bytes,
|
|
1236
|
+
)?;
|
|
1237
|
+
|
|
1238
|
+
for &dblk_addr in &dblk_addrs {
|
|
1239
|
+
if Cursor::is_undefined_offset(dblk_addr, offset_size) {
|
|
1240
|
+
for _ in 0..elmts_per_dblk {
|
|
1241
|
+
all_entries.push(EaRawEntry {
|
|
1242
|
+
address: u64::MAX,
|
|
1243
|
+
chunk_size: 0,
|
|
1244
|
+
filter_mask: 0,
|
|
1245
|
+
});
|
|
1246
|
+
}
|
|
1247
|
+
} else {
|
|
1248
|
+
let dblk_entries = parse_data_block(
|
|
1249
|
+
data,
|
|
1250
|
+
dblk_addr,
|
|
1251
|
+
elmts_per_dblk as usize,
|
|
1252
|
+
is_filtered,
|
|
1253
|
+
header.max_dblk_page_nelmts_bits,
|
|
1254
|
+
offset_size,
|
|
1255
|
+
header.element_size,
|
|
1256
|
+
sizeof_nelmts,
|
|
1257
|
+
)?;
|
|
1258
|
+
all_entries.extend(dblk_entries);
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1261
|
+
}
|
|
1262
|
+
|
|
1263
|
+
// Convert linear indices to chunk offsets.
|
|
1264
|
+
let ndim = dataset_shape.len();
|
|
1265
|
+
let chunks_per_dim: Vec<u64> = (0..ndim)
|
|
1266
|
+
.map(|i| dataset_shape[i].div_ceil(chunk_dims[i] as u64))
|
|
1267
|
+
.collect();
|
|
1268
|
+
|
|
1269
|
+
let mut entries = Vec::new();
|
|
1270
|
+
for (linear_idx, raw) in all_entries.iter().enumerate() {
|
|
1271
|
+
if Cursor::is_undefined_offset(raw.address, offset_size) {
|
|
1272
|
+
continue;
|
|
1273
|
+
}
|
|
1274
|
+
|
|
1275
|
+
let mut remaining = linear_idx as u64;
|
|
1276
|
+
let mut offsets = vec![0u64; ndim];
|
|
1277
|
+
for d in (0..ndim).rev() {
|
|
1278
|
+
offsets[d] = (remaining % chunks_per_dim[d]) * chunk_dims[d] as u64;
|
|
1279
|
+
remaining /= chunks_per_dim[d];
|
|
1280
|
+
}
|
|
1281
|
+
|
|
1282
|
+
if let Some((first_chunk, last_chunk)) = chunk_bounds {
|
|
1283
|
+
let overlaps = offsets.iter().enumerate().all(|(dim, offset)| {
|
|
1284
|
+
let chunk_index = *offset / u64::from(chunk_dims[dim]);
|
|
1285
|
+
chunk_index >= first_chunk[dim] && chunk_index <= last_chunk[dim]
|
|
1286
|
+
});
|
|
1287
|
+
if !overlaps {
|
|
1288
|
+
continue;
|
|
1289
|
+
}
|
|
1290
|
+
}
|
|
1291
|
+
|
|
1292
|
+
entries.push(ChunkEntry {
|
|
1293
|
+
address: raw.address,
|
|
1294
|
+
size: raw.chunk_size,
|
|
1295
|
+
filter_mask: raw.filter_mask,
|
|
1296
|
+
offsets,
|
|
1297
|
+
});
|
|
1298
|
+
}
|
|
1299
|
+
|
|
1300
|
+
Ok(entries)
|
|
1301
|
+
}
|
|
1302
|
+
|
|
1303
|
+
/// Collect chunk entries from an Extensible Array index using random-access storage.
|
|
1304
|
+
pub fn collect_extensible_array_chunk_entries_storage(
|
|
1305
|
+
storage: &dyn Storage,
|
|
1306
|
+
header_address: u64,
|
|
1307
|
+
offset_size: u8,
|
|
1308
|
+
length_size: u8,
|
|
1309
|
+
dataset_shape: &[u64],
|
|
1310
|
+
chunk_dims: &[u32],
|
|
1311
|
+
chunk_bounds: Option<(&[u64], &[u64])>,
|
|
1312
|
+
) -> Result<Vec<ChunkEntry>> {
|
|
1313
|
+
let header = parse_header_storage(storage, header_address, offset_size, length_size)?;
|
|
1314
|
+
|
|
1315
|
+
if Cursor::is_undefined_offset(header.index_block_address, offset_size) {
|
|
1316
|
+
return Ok(Vec::new());
|
|
1317
|
+
}
|
|
1318
|
+
|
|
1319
|
+
let is_filtered = header.client_id == 1;
|
|
1320
|
+
let sb_layout = compute_super_block_layout(&header);
|
|
1321
|
+
let sizeof_nelmts = (header._max_nelmts_bits as usize).div_ceil(8);
|
|
1322
|
+
|
|
1323
|
+
if let Some(bounds) = chunk_bounds {
|
|
1324
|
+
let targets = linear_target_offsets(dataset_shape, chunk_dims, Some(bounds));
|
|
1325
|
+
let index_block = storage.read_range(
|
|
1326
|
+
header.index_block_address,
|
|
1327
|
+
index_block_len(&header, &sb_layout, offset_size)?,
|
|
1328
|
+
)?;
|
|
1329
|
+
let mut cursor = Cursor::new(index_block.as_ref());
|
|
1330
|
+
let sig = cursor.read_bytes(4)?;
|
|
1331
|
+
if sig != EAIB_SIGNATURE {
|
|
1332
|
+
return Err(Error::InvalidExtensibleArraySignature {
|
|
1333
|
+
context: "index block signature mismatch",
|
|
1334
|
+
});
|
|
1335
|
+
}
|
|
1336
|
+
let version = cursor.read_u8()?;
|
|
1337
|
+
if version != 0 {
|
|
1338
|
+
return Err(Error::Other(format!(
|
|
1339
|
+
"unsupported extensible array index block version {}",
|
|
1340
|
+
version
|
|
1341
|
+
)));
|
|
1342
|
+
}
|
|
1343
|
+
let _client_id = cursor.read_u8()?;
|
|
1344
|
+
let _header_address = cursor.read_offset(offset_size)?;
|
|
1345
|
+
let num_inline = header.idx_blk_elmts as usize;
|
|
1346
|
+
let inline_start = cursor.position();
|
|
1347
|
+
cursor.skip(num_inline * header.element_size as usize)?;
|
|
1348
|
+
|
|
1349
|
+
let ndblk_addrs = 2 * header.sec_blk_min_data_ptrs as usize;
|
|
1350
|
+
let mut direct_dblk_addrs = Vec::with_capacity(ndblk_addrs);
|
|
1351
|
+
for _ in 0..ndblk_addrs {
|
|
1352
|
+
direct_dblk_addrs.push(cursor.read_offset(offset_size)?);
|
|
1353
|
+
}
|
|
1354
|
+
|
|
1355
|
+
let nsblks = sb_layout.len();
|
|
1356
|
+
let nsblk_addrs = nsblks.saturating_sub(ndblk_addrs);
|
|
1357
|
+
let mut sec_block_addrs = Vec::with_capacity(nsblk_addrs);
|
|
1358
|
+
for _ in 0..nsblk_addrs {
|
|
1359
|
+
sec_block_addrs.push(cursor.read_offset(offset_size)?);
|
|
1360
|
+
}
|
|
1361
|
+
|
|
1362
|
+
let mut secondary_block_cache: Vec<Option<Vec<u64>>> = vec![None; sec_block_addrs.len()];
|
|
1363
|
+
let mut entries = Vec::new();
|
|
1364
|
+
|
|
1365
|
+
for (linear_idx, offsets) in targets {
|
|
1366
|
+
let raw = if linear_idx < num_inline {
|
|
1367
|
+
let inline_offset = inline_start
|
|
1368
|
+
+ u64::try_from(linear_idx * usize::from(header.element_size)).map_err(
|
|
1369
|
+
|_| {
|
|
1370
|
+
Error::InvalidData("EA inline entry offset exceeds u64 capacity".into())
|
|
1371
|
+
},
|
|
1372
|
+
)?;
|
|
1373
|
+
let position = header.index_block_address + inline_offset;
|
|
1374
|
+
read_entry_at_storage(
|
|
1375
|
+
storage,
|
|
1376
|
+
position,
|
|
1377
|
+
is_filtered,
|
|
1378
|
+
offset_size,
|
|
1379
|
+
header.element_size,
|
|
1380
|
+
)?
|
|
1381
|
+
} else {
|
|
1382
|
+
let mut relative_idx = (linear_idx - num_inline) as u64;
|
|
1383
|
+
let mut sb_idx = None;
|
|
1384
|
+
for (candidate_idx, (elmts_per_dblk, num_dblks)) in sb_layout.iter().enumerate() {
|
|
1385
|
+
let capacity = elmts_per_dblk * num_dblks;
|
|
1386
|
+
if relative_idx < capacity {
|
|
1387
|
+
sb_idx = Some(candidate_idx);
|
|
1388
|
+
break;
|
|
1389
|
+
}
|
|
1390
|
+
relative_idx -= capacity;
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1393
|
+
let Some(sb_idx) = sb_idx else {
|
|
1394
|
+
continue;
|
|
1395
|
+
};
|
|
1396
|
+
let (elmts_per_dblk, _) = sb_layout[sb_idx];
|
|
1397
|
+
let dblk_idx = (relative_idx / elmts_per_dblk) as usize;
|
|
1398
|
+
let local_idx = (relative_idx % elmts_per_dblk) as usize;
|
|
1399
|
+
|
|
1400
|
+
let dblk_addr = if sb_idx < 2 {
|
|
1401
|
+
let base = sb_layout[..sb_idx]
|
|
1402
|
+
.iter()
|
|
1403
|
+
.map(|(_, num_dblks)| *num_dblks as usize)
|
|
1404
|
+
.sum::<usize>();
|
|
1405
|
+
*direct_dblk_addrs.get(base + dblk_idx).unwrap_or(&u64::MAX)
|
|
1406
|
+
} else {
|
|
1407
|
+
let sec_cache_idx = sb_idx - 2;
|
|
1408
|
+
if secondary_block_cache[sec_cache_idx].is_none() {
|
|
1409
|
+
let sec_addr = sec_block_addrs
|
|
1410
|
+
.get(sec_cache_idx)
|
|
1411
|
+
.copied()
|
|
1412
|
+
.unwrap_or(u64::MAX);
|
|
1413
|
+
if Cursor::is_undefined_offset(sec_addr, offset_size) {
|
|
1414
|
+
secondary_block_cache[sec_cache_idx] = Some(Vec::new());
|
|
1415
|
+
} else {
|
|
1416
|
+
let (_, num_dblks) = sb_layout[sb_idx];
|
|
1417
|
+
let page_bitmap_bytes = secondary_page_bitmap_bytes(
|
|
1418
|
+
header.max_dblk_page_nelmts_bits,
|
|
1419
|
+
elmts_per_dblk,
|
|
1420
|
+
num_dblks,
|
|
1421
|
+
)?;
|
|
1422
|
+
secondary_block_cache[sec_cache_idx] =
|
|
1423
|
+
Some(parse_secondary_block_storage(
|
|
1424
|
+
storage,
|
|
1425
|
+
sec_addr,
|
|
1426
|
+
checked_usize_from_u64(
|
|
1427
|
+
num_dblks,
|
|
1428
|
+
"extensible array secondary block data block count",
|
|
1429
|
+
)?,
|
|
1430
|
+
offset_size,
|
|
1431
|
+
sizeof_nelmts,
|
|
1432
|
+
page_bitmap_bytes,
|
|
1433
|
+
)?);
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
|
|
1437
|
+
secondary_block_cache[sec_cache_idx]
|
|
1438
|
+
.as_ref()
|
|
1439
|
+
.and_then(|addrs| addrs.get(dblk_idx))
|
|
1440
|
+
.copied()
|
|
1441
|
+
.unwrap_or(u64::MAX)
|
|
1442
|
+
};
|
|
1443
|
+
|
|
1444
|
+
if Cursor::is_undefined_offset(dblk_addr, offset_size) {
|
|
1445
|
+
continue;
|
|
1446
|
+
}
|
|
1447
|
+
|
|
1448
|
+
read_data_block_entry_storage(
|
|
1449
|
+
storage,
|
|
1450
|
+
dblk_addr,
|
|
1451
|
+
checked_usize_from_u64(
|
|
1452
|
+
elmts_per_dblk,
|
|
1453
|
+
"extensible array data block element count",
|
|
1454
|
+
)?,
|
|
1455
|
+
local_idx,
|
|
1456
|
+
is_filtered,
|
|
1457
|
+
header.max_dblk_page_nelmts_bits,
|
|
1458
|
+
offset_size,
|
|
1459
|
+
header.element_size,
|
|
1460
|
+
sizeof_nelmts,
|
|
1461
|
+
)?
|
|
1462
|
+
};
|
|
1463
|
+
|
|
1464
|
+
if Cursor::is_undefined_offset(raw.address, offset_size) {
|
|
1465
|
+
continue;
|
|
1466
|
+
}
|
|
1467
|
+
|
|
1468
|
+
entries.push(ChunkEntry {
|
|
1469
|
+
address: raw.address,
|
|
1470
|
+
size: raw.chunk_size,
|
|
1471
|
+
filter_mask: raw.filter_mask,
|
|
1472
|
+
offsets,
|
|
1473
|
+
});
|
|
1474
|
+
}
|
|
1475
|
+
|
|
1476
|
+
return Ok(entries);
|
|
1477
|
+
}
|
|
1478
|
+
|
|
1479
|
+
let index_block_len = index_block_len(&header, &sb_layout, offset_size)?;
|
|
1480
|
+
let data = storage.read_range(header.index_block_address, index_block_len)?;
|
|
1481
|
+
let mut cursor = Cursor::new(data.as_ref());
|
|
1482
|
+
cursor.set_position(0);
|
|
1483
|
+
|
|
1484
|
+
let sig = cursor.read_bytes(4)?;
|
|
1485
|
+
if sig != EAIB_SIGNATURE {
|
|
1486
|
+
return Err(Error::InvalidExtensibleArraySignature {
|
|
1487
|
+
context: "index block signature mismatch",
|
|
1488
|
+
});
|
|
1489
|
+
}
|
|
1490
|
+
|
|
1491
|
+
let version = cursor.read_u8()?;
|
|
1492
|
+
if version != 0 {
|
|
1493
|
+
return Err(Error::Other(format!(
|
|
1494
|
+
"unsupported extensible array index block version {}",
|
|
1495
|
+
version
|
|
1496
|
+
)));
|
|
1497
|
+
}
|
|
1498
|
+
|
|
1499
|
+
let _client_id = cursor.read_u8()?;
|
|
1500
|
+
let _header_address = cursor.read_offset(offset_size)?;
|
|
1501
|
+
|
|
1502
|
+
let num_inline = header.idx_blk_elmts as usize;
|
|
1503
|
+
let inline_entries = read_entries(
|
|
1504
|
+
&mut cursor,
|
|
1505
|
+
num_inline,
|
|
1506
|
+
is_filtered,
|
|
1507
|
+
offset_size,
|
|
1508
|
+
header.element_size,
|
|
1509
|
+
)?;
|
|
1510
|
+
|
|
1511
|
+
let ndblk_addrs = 2 * header.sec_blk_min_data_ptrs as usize;
|
|
1512
|
+
let mut direct_dblk_addrs = Vec::with_capacity(ndblk_addrs);
|
|
1513
|
+
for _ in 0..ndblk_addrs {
|
|
1514
|
+
direct_dblk_addrs.push(cursor.read_offset(offset_size)?);
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1517
|
+
let nsblks = sb_layout.len();
|
|
1518
|
+
let nsblk_addrs = nsblks.saturating_sub(ndblk_addrs);
|
|
1519
|
+
let mut sec_block_addrs = Vec::with_capacity(nsblk_addrs);
|
|
1520
|
+
for _ in 0..nsblk_addrs {
|
|
1521
|
+
sec_block_addrs.push(cursor.read_offset(offset_size)?);
|
|
1522
|
+
}
|
|
1523
|
+
let _checksum = cursor.read_u32_le()?;
|
|
1524
|
+
|
|
1525
|
+
let mut all_entries: Vec<EaRawEntry> = Vec::new();
|
|
1526
|
+
all_entries.extend(inline_entries);
|
|
1527
|
+
|
|
1528
|
+
let mut dblk_addr_idx = 0;
|
|
1529
|
+
for sb_idx_iter in 0..2usize.min(nsblks) {
|
|
1530
|
+
if sb_idx_iter >= sb_layout.len() {
|
|
1531
|
+
break;
|
|
1532
|
+
}
|
|
1533
|
+
let (elmts_per_dblk, num_dblks) = sb_layout[sb_idx_iter];
|
|
1534
|
+
for _ in 0..num_dblks {
|
|
1535
|
+
if dblk_addr_idx >= direct_dblk_addrs.len() {
|
|
1536
|
+
break;
|
|
1537
|
+
}
|
|
1538
|
+
let dblk_addr = direct_dblk_addrs[dblk_addr_idx];
|
|
1539
|
+
dblk_addr_idx += 1;
|
|
1540
|
+
|
|
1541
|
+
if Cursor::is_undefined_offset(dblk_addr, offset_size) {
|
|
1542
|
+
for _ in 0..elmts_per_dblk {
|
|
1543
|
+
all_entries.push(EaRawEntry {
|
|
1544
|
+
address: u64::MAX,
|
|
1545
|
+
chunk_size: 0,
|
|
1546
|
+
filter_mask: 0,
|
|
1547
|
+
});
|
|
1548
|
+
}
|
|
1549
|
+
} else {
|
|
1550
|
+
let dblk_entries = read_data_block_storage(
|
|
1551
|
+
storage,
|
|
1552
|
+
dblk_addr,
|
|
1553
|
+
checked_usize_from_u64(
|
|
1554
|
+
elmts_per_dblk,
|
|
1555
|
+
"extensible array data block element count",
|
|
1556
|
+
)?,
|
|
1557
|
+
is_filtered,
|
|
1558
|
+
header.max_dblk_page_nelmts_bits,
|
|
1559
|
+
offset_size,
|
|
1560
|
+
header.element_size,
|
|
1561
|
+
sizeof_nelmts,
|
|
1562
|
+
)?;
|
|
1563
|
+
all_entries.extend(dblk_entries);
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
}
|
|
1567
|
+
|
|
1568
|
+
for (sb_idx_iter, &(elmts_per_dblk, num_dblks)) in sb_layout.iter().enumerate().skip(2) {
|
|
1569
|
+
let sec_idx = sb_idx_iter - 2;
|
|
1570
|
+
let sec_addr = *sec_block_addrs.get(sec_idx).unwrap_or(&u64::MAX);
|
|
1571
|
+
if Cursor::is_undefined_offset(sec_addr, offset_size) {
|
|
1572
|
+
for _ in 0..(elmts_per_dblk * num_dblks) {
|
|
1573
|
+
all_entries.push(EaRawEntry {
|
|
1574
|
+
address: u64::MAX,
|
|
1575
|
+
chunk_size: 0,
|
|
1576
|
+
filter_mask: 0,
|
|
1577
|
+
});
|
|
1578
|
+
}
|
|
1579
|
+
continue;
|
|
1580
|
+
}
|
|
1581
|
+
|
|
1582
|
+
let page_bitmap_bytes = secondary_page_bitmap_bytes(
|
|
1583
|
+
header.max_dblk_page_nelmts_bits,
|
|
1584
|
+
elmts_per_dblk,
|
|
1585
|
+
num_dblks,
|
|
1586
|
+
)?;
|
|
1587
|
+
let dblk_addrs = parse_secondary_block_storage(
|
|
1588
|
+
storage,
|
|
1589
|
+
sec_addr,
|
|
1590
|
+
checked_usize_from_u64(
|
|
1591
|
+
num_dblks,
|
|
1592
|
+
"extensible array secondary block data block count",
|
|
1593
|
+
)?,
|
|
1594
|
+
offset_size,
|
|
1595
|
+
sizeof_nelmts,
|
|
1596
|
+
page_bitmap_bytes,
|
|
1597
|
+
)?;
|
|
1598
|
+
|
|
1599
|
+
for dblk_addr in dblk_addrs {
|
|
1600
|
+
if Cursor::is_undefined_offset(dblk_addr, offset_size) {
|
|
1601
|
+
for _ in 0..elmts_per_dblk {
|
|
1602
|
+
all_entries.push(EaRawEntry {
|
|
1603
|
+
address: u64::MAX,
|
|
1604
|
+
chunk_size: 0,
|
|
1605
|
+
filter_mask: 0,
|
|
1606
|
+
});
|
|
1607
|
+
}
|
|
1608
|
+
} else {
|
|
1609
|
+
let dblk_entries = read_data_block_storage(
|
|
1610
|
+
storage,
|
|
1611
|
+
dblk_addr,
|
|
1612
|
+
checked_usize_from_u64(
|
|
1613
|
+
elmts_per_dblk,
|
|
1614
|
+
"extensible array data block element count",
|
|
1615
|
+
)?,
|
|
1616
|
+
is_filtered,
|
|
1617
|
+
header.max_dblk_page_nelmts_bits,
|
|
1618
|
+
offset_size,
|
|
1619
|
+
header.element_size,
|
|
1620
|
+
sizeof_nelmts,
|
|
1621
|
+
)?;
|
|
1622
|
+
all_entries.extend(dblk_entries);
|
|
1623
|
+
}
|
|
1624
|
+
}
|
|
1625
|
+
}
|
|
1626
|
+
|
|
1627
|
+
let ndim = dataset_shape.len();
|
|
1628
|
+
let chunks_per_dim: Vec<u64> = (0..ndim)
|
|
1629
|
+
.map(|i| dataset_shape[i].div_ceil(chunk_dims[i] as u64))
|
|
1630
|
+
.collect();
|
|
1631
|
+
|
|
1632
|
+
let mut entries = Vec::new();
|
|
1633
|
+
for (linear_idx, raw) in all_entries.iter().enumerate() {
|
|
1634
|
+
if Cursor::is_undefined_offset(raw.address, offset_size) {
|
|
1635
|
+
continue;
|
|
1636
|
+
}
|
|
1637
|
+
|
|
1638
|
+
let mut remaining = linear_idx as u64;
|
|
1639
|
+
let mut offsets = vec![0u64; ndim];
|
|
1640
|
+
for d in (0..ndim).rev() {
|
|
1641
|
+
offsets[d] = (remaining % chunks_per_dim[d]) * chunk_dims[d] as u64;
|
|
1642
|
+
remaining /= chunks_per_dim[d];
|
|
1643
|
+
}
|
|
1644
|
+
|
|
1645
|
+
if let Some((first_chunk, last_chunk)) = chunk_bounds {
|
|
1646
|
+
let overlaps = offsets.iter().enumerate().all(|(dim, offset)| {
|
|
1647
|
+
let chunk_index = *offset / u64::from(chunk_dims[dim]);
|
|
1648
|
+
chunk_index >= first_chunk[dim] && chunk_index <= last_chunk[dim]
|
|
1649
|
+
});
|
|
1650
|
+
if !overlaps {
|
|
1651
|
+
continue;
|
|
1652
|
+
}
|
|
1653
|
+
}
|
|
1654
|
+
|
|
1655
|
+
entries.push(ChunkEntry {
|
|
1656
|
+
address: raw.address,
|
|
1657
|
+
size: raw.chunk_size,
|
|
1658
|
+
filter_mask: raw.filter_mask,
|
|
1659
|
+
offsets,
|
|
1660
|
+
});
|
|
1661
|
+
}
|
|
1662
|
+
|
|
1663
|
+
Ok(entries)
|
|
1664
|
+
}
|
|
1665
|
+
|
|
1666
|
+
#[cfg(test)]
|
|
1667
|
+
mod tests {
|
|
1668
|
+
use super::*;
|
|
1669
|
+
use crate::storage::{Storage, StorageBuffer};
|
|
1670
|
+
use std::sync::Mutex;
|
|
1671
|
+
|
|
1672
|
+
const TEST_OFFSET_SIZE: u8 = 8;
|
|
1673
|
+
const TEST_LENGTH_SIZE: u8 = 8;
|
|
1674
|
+
const TEST_HEADER_ADDR: u64 = 0;
|
|
1675
|
+
const TEST_INDEX_ADDR: u64 = 128;
|
|
1676
|
+
const TEST_SECONDARY_ADDR: u64 = 512;
|
|
1677
|
+
const TEST_DATA_BLOCK_ADDR: u64 = 768;
|
|
1678
|
+
|
|
1679
|
+
struct RecordingStorage {
|
|
1680
|
+
data: Vec<u8>,
|
|
1681
|
+
ranges: Mutex<Vec<(u64, usize)>>,
|
|
1682
|
+
}
|
|
1683
|
+
|
|
1684
|
+
impl RecordingStorage {
|
|
1685
|
+
fn new(data: Vec<u8>) -> Self {
|
|
1686
|
+
Self {
|
|
1687
|
+
data,
|
|
1688
|
+
ranges: Mutex::new(Vec::new()),
|
|
1689
|
+
}
|
|
1690
|
+
}
|
|
1691
|
+
|
|
1692
|
+
fn ranges(&self) -> Vec<(u64, usize)> {
|
|
1693
|
+
self.ranges.lock().unwrap().clone()
|
|
1694
|
+
}
|
|
1695
|
+
|
|
1696
|
+
fn clear_ranges(&self) {
|
|
1697
|
+
self.ranges.lock().unwrap().clear();
|
|
1698
|
+
}
|
|
1699
|
+
}
|
|
1700
|
+
|
|
1701
|
+
impl Storage for RecordingStorage {
|
|
1702
|
+
fn len(&self) -> u64 {
|
|
1703
|
+
self.data.len() as u64
|
|
1704
|
+
}
|
|
1705
|
+
|
|
1706
|
+
fn read_range(&self, offset: u64, len: usize) -> Result<StorageBuffer> {
|
|
1707
|
+
let start = usize::try_from(offset).map_err(|_| Error::OffsetOutOfBounds(offset))?;
|
|
1708
|
+
let end = start.checked_add(len).ok_or(Error::UnexpectedEof {
|
|
1709
|
+
offset,
|
|
1710
|
+
needed: len as u64,
|
|
1711
|
+
available: self.data.len().saturating_sub(start) as u64,
|
|
1712
|
+
})?;
|
|
1713
|
+
if end > self.data.len() {
|
|
1714
|
+
return Err(Error::UnexpectedEof {
|
|
1715
|
+
offset,
|
|
1716
|
+
needed: len as u64,
|
|
1717
|
+
available: self.data.len().saturating_sub(start) as u64,
|
|
1718
|
+
});
|
|
1719
|
+
}
|
|
1720
|
+
self.ranges.lock().unwrap().push((offset, len));
|
|
1721
|
+
Ok(StorageBuffer::from_vec(self.data[start..end].to_vec()))
|
|
1722
|
+
}
|
|
1723
|
+
}
|
|
1724
|
+
|
|
1725
|
+
fn put(data: &mut [u8], offset: u64, bytes: &[u8]) {
|
|
1726
|
+
let start = offset as usize;
|
|
1727
|
+
data[start..start + bytes.len()].copy_from_slice(bytes);
|
|
1728
|
+
}
|
|
1729
|
+
|
|
1730
|
+
fn push_u64(buf: &mut Vec<u8>, value: u64) {
|
|
1731
|
+
buf.extend_from_slice(&value.to_le_bytes());
|
|
1732
|
+
}
|
|
1733
|
+
|
|
1734
|
+
fn push_offset(buf: &mut Vec<u8>, value: u64) {
|
|
1735
|
+
push_u64(buf, value);
|
|
1736
|
+
}
|
|
1737
|
+
|
|
1738
|
+
fn ea_header_bytes(nelmts: u64) -> Vec<u8> {
|
|
1739
|
+
let mut buf = Vec::new();
|
|
1740
|
+
buf.extend_from_slice(&EAHD_SIGNATURE);
|
|
1741
|
+
buf.push(0); // version
|
|
1742
|
+
buf.push(0); // unfiltered raw chunk client
|
|
1743
|
+
buf.push(8); // element size: address only
|
|
1744
|
+
buf.push(8); // max_nelmts_bits, so sizeof_nelmts is 1
|
|
1745
|
+
buf.push(0); // inline elements
|
|
1746
|
+
buf.push(1); // minimum data block elements
|
|
1747
|
+
buf.push(1); // minimum secondary block data pointers
|
|
1748
|
+
buf.push(0); // unpaged data blocks
|
|
1749
|
+
for value in [0, 0, 0, 0, 0, nelmts] {
|
|
1750
|
+
push_u64(&mut buf, value);
|
|
1751
|
+
}
|
|
1752
|
+
push_offset(&mut buf, TEST_INDEX_ADDR);
|
|
1753
|
+
let checksum = jenkins_lookup3(&buf);
|
|
1754
|
+
buf.extend_from_slice(&checksum.to_le_bytes());
|
|
1755
|
+
buf
|
|
1756
|
+
}
|
|
1757
|
+
|
|
1758
|
+
fn index_block_bytes() -> Vec<u8> {
|
|
1759
|
+
let mut buf = Vec::new();
|
|
1760
|
+
buf.extend_from_slice(&EAIB_SIGNATURE);
|
|
1761
|
+
buf.push(0); // version
|
|
1762
|
+
buf.push(0); // client id
|
|
1763
|
+
push_offset(&mut buf, TEST_HEADER_ADDR);
|
|
1764
|
+
push_offset(&mut buf, u64::MAX); // first direct data block
|
|
1765
|
+
push_offset(&mut buf, u64::MAX); // second direct data block
|
|
1766
|
+
push_offset(&mut buf, TEST_SECONDARY_ADDR);
|
|
1767
|
+
buf.extend_from_slice(&0u32.to_le_bytes());
|
|
1768
|
+
buf
|
|
1769
|
+
}
|
|
1770
|
+
|
|
1771
|
+
fn secondary_block_bytes() -> Vec<u8> {
|
|
1772
|
+
let mut buf = Vec::new();
|
|
1773
|
+
buf.extend_from_slice(&EASB_SIGNATURE);
|
|
1774
|
+
buf.push(0); // version
|
|
1775
|
+
buf.push(0); // client id
|
|
1776
|
+
push_offset(&mut buf, TEST_HEADER_ADDR);
|
|
1777
|
+
buf.push(0); // block offset, sizeof_nelmts = 1
|
|
1778
|
+
push_offset(&mut buf, TEST_DATA_BLOCK_ADDR);
|
|
1779
|
+
push_offset(&mut buf, u64::MAX);
|
|
1780
|
+
buf.extend_from_slice(&0u32.to_le_bytes());
|
|
1781
|
+
buf
|
|
1782
|
+
}
|
|
1783
|
+
|
|
1784
|
+
fn data_block_bytes() -> Vec<u8> {
|
|
1785
|
+
let mut buf = Vec::new();
|
|
1786
|
+
buf.extend_from_slice(&EADB_SIGNATURE);
|
|
1787
|
+
buf.push(0); // version
|
|
1788
|
+
buf.push(0); // client id
|
|
1789
|
+
push_offset(&mut buf, TEST_HEADER_ADDR);
|
|
1790
|
+
buf.push(0); // block offset, sizeof_nelmts = 1
|
|
1791
|
+
push_offset(&mut buf, 0x2000);
|
|
1792
|
+
push_offset(&mut buf, 0x2010);
|
|
1793
|
+
buf.extend_from_slice(&0u32.to_le_bytes());
|
|
1794
|
+
buf
|
|
1795
|
+
}
|
|
1796
|
+
|
|
1797
|
+
fn storage_fixture() -> RecordingStorage {
|
|
1798
|
+
let mut data = vec![0u8; 4096];
|
|
1799
|
+
put(&mut data, TEST_HEADER_ADDR, &ea_header_bytes(7));
|
|
1800
|
+
put(&mut data, TEST_INDEX_ADDR, &index_block_bytes());
|
|
1801
|
+
put(&mut data, TEST_SECONDARY_ADDR, &secondary_block_bytes());
|
|
1802
|
+
put(&mut data, TEST_DATA_BLOCK_ADDR, &data_block_bytes());
|
|
1803
|
+
RecordingStorage::new(data)
|
|
1804
|
+
}
|
|
1805
|
+
|
|
1806
|
+
#[test]
|
|
1807
|
+
fn eahd_bad_signature() {
|
|
1808
|
+
let mut data = vec![0u8; 64];
|
|
1809
|
+
data[0..4].copy_from_slice(b"XXXX");
|
|
1810
|
+
let err = parse_header(&data, 0, 8, 8).unwrap_err();
|
|
1811
|
+
assert!(matches!(err, Error::InvalidExtensibleArraySignature { .. }));
|
|
1812
|
+
}
|
|
1813
|
+
|
|
1814
|
+
#[test]
|
|
1815
|
+
fn super_block_layout_matches_spec_rows() {
|
|
1816
|
+
let header = EaHeader {
|
|
1817
|
+
client_id: 0,
|
|
1818
|
+
element_size: 8,
|
|
1819
|
+
_max_nelmts_bits: 32,
|
|
1820
|
+
idx_blk_elmts: 2,
|
|
1821
|
+
data_blk_min_elmts: 2,
|
|
1822
|
+
sec_blk_min_data_ptrs: 2,
|
|
1823
|
+
max_dblk_page_nelmts_bits: 0,
|
|
1824
|
+
_nelmts: 100,
|
|
1825
|
+
index_block_address: 0,
|
|
1826
|
+
};
|
|
1827
|
+
let layout = compute_super_block_layout(&header);
|
|
1828
|
+
// sb 0: elmts_per_dblk = 2 * 2^0 = 2, num_dblks = 2 * 2^0 = 2 (cap = 4 elements)
|
|
1829
|
+
assert_eq!(layout[0], (2, 2));
|
|
1830
|
+
// sb 1: elmts_per_dblk = 2 * 2^0 = 2, num_dblks = 2 * 2^1 = 4 (cap = 8 elements)
|
|
1831
|
+
assert_eq!(layout[1], (2, 4));
|
|
1832
|
+
// sb 2: elmts_per_dblk = 2 * 2^1 = 4, num_dblks = 2 * 2^1 = 4 (cap = 16 elements)
|
|
1833
|
+
assert_eq!(layout[2], (4, 4));
|
|
1834
|
+
// sb 3: elmts_per_dblk = 2 * 2^1 = 4, num_dblks = 2 * 2^2 = 8 (cap = 32 elements)
|
|
1835
|
+
assert_eq!(layout[3], (4, 8));
|
|
1836
|
+
}
|
|
1837
|
+
|
|
1838
|
+
#[test]
|
|
1839
|
+
fn storage_full_scan_reads_exact_secondary_and_data_block_lengths() {
|
|
1840
|
+
let storage = storage_fixture();
|
|
1841
|
+
|
|
1842
|
+
let entries = collect_extensible_array_chunk_entries_storage(
|
|
1843
|
+
&storage,
|
|
1844
|
+
TEST_HEADER_ADDR,
|
|
1845
|
+
TEST_OFFSET_SIZE,
|
|
1846
|
+
TEST_LENGTH_SIZE,
|
|
1847
|
+
&[7],
|
|
1848
|
+
&[1],
|
|
1849
|
+
None,
|
|
1850
|
+
)
|
|
1851
|
+
.unwrap();
|
|
1852
|
+
|
|
1853
|
+
assert!(!entries.is_empty());
|
|
1854
|
+
let ranges = storage.ranges();
|
|
1855
|
+
assert!(ranges.contains(&(TEST_HEADER_ADDR, 72)));
|
|
1856
|
+
assert!(ranges.contains(&(TEST_INDEX_ADDR, 42)));
|
|
1857
|
+
assert!(ranges.contains(&(TEST_SECONDARY_ADDR, 35)));
|
|
1858
|
+
assert!(ranges.contains(&(TEST_DATA_BLOCK_ADDR, 35)));
|
|
1859
|
+
assert!(ranges.iter().all(|(_, len)| *len <= 72), "{ranges:?}");
|
|
1860
|
+
}
|
|
1861
|
+
|
|
1862
|
+
#[test]
|
|
1863
|
+
fn storage_bounded_scan_reads_exact_index_block_length() {
|
|
1864
|
+
let storage = storage_fixture();
|
|
1865
|
+
storage.clear_ranges();
|
|
1866
|
+
|
|
1867
|
+
let entries = collect_extensible_array_chunk_entries_storage(
|
|
1868
|
+
&storage,
|
|
1869
|
+
TEST_HEADER_ADDR,
|
|
1870
|
+
TEST_OFFSET_SIZE,
|
|
1871
|
+
TEST_LENGTH_SIZE,
|
|
1872
|
+
&[7],
|
|
1873
|
+
&[1],
|
|
1874
|
+
Some((&[3], &[3])),
|
|
1875
|
+
)
|
|
1876
|
+
.unwrap();
|
|
1877
|
+
|
|
1878
|
+
assert_eq!(entries.len(), 1);
|
|
1879
|
+
assert_eq!(entries[0].address, 0x2000);
|
|
1880
|
+
let ranges = storage.ranges();
|
|
1881
|
+
assert!(ranges.contains(&(TEST_HEADER_ADDR, 72)));
|
|
1882
|
+
assert!(ranges.contains(&(TEST_INDEX_ADDR, 42)));
|
|
1883
|
+
assert!(ranges.contains(&(TEST_SECONDARY_ADDR, 35)));
|
|
1884
|
+
assert!(ranges.iter().all(|(_, len)| *len <= 72), "{ranges:?}");
|
|
1885
|
+
}
|
|
1886
|
+
}
|