@nirs4all/formats-wasm 0.2.12 → 0.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD_PARTY_NOTICES.md +49 -0
- package/nirs4all_formats_wasm_bg.wasm +0 -0
- package/package.json +4 -2
- package/reader-backports/hdf5-reader/Cargo.toml +119 -0
- package/reader-backports/hdf5-reader/LICENSE-APACHE +164 -0
- package/reader-backports/hdf5-reader/LICENSE-MIT +21 -0
- package/reader-backports/hdf5-reader/PROVENANCE.json +80 -0
- package/reader-backports/hdf5-reader/README.md +308 -0
- package/reader-backports/hdf5-reader/UPSTREAM-Cargo.toml +44 -0
- package/reader-backports/hdf5-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/hdf5-reader/src/attribute_api.rs +847 -0
- package/reader-backports/hdf5-reader/src/btree_v1.rs +726 -0
- package/reader-backports/hdf5-reader/src/btree_v2.rs +2068 -0
- package/reader-backports/hdf5-reader/src/cache.rs +354 -0
- package/reader-backports/hdf5-reader/src/checksum.rs +56 -0
- package/reader-backports/hdf5-reader/src/chunk_index.rs +423 -0
- package/reader-backports/hdf5-reader/src/dataset.rs +4733 -0
- package/reader-backports/hdf5-reader/src/datatype_api.rs +497 -0
- package/reader-backports/hdf5-reader/src/error.rs +167 -0
- package/reader-backports/hdf5-reader/src/extensible_array.rs +1886 -0
- package/reader-backports/hdf5-reader/src/filters/deflate.rs +93 -0
- package/reader-backports/hdf5-reader/src/filters/fletcher32.rs +52 -0
- package/reader-backports/hdf5-reader/src/filters/lz4.rs +178 -0
- package/reader-backports/hdf5-reader/src/filters/mod.rs +326 -0
- package/reader-backports/hdf5-reader/src/filters/nbit.rs +586 -0
- package/reader-backports/hdf5-reader/src/filters/scaleoffset.rs +827 -0
- package/reader-backports/hdf5-reader/src/filters/shuffle.rs +99 -0
- package/reader-backports/hdf5-reader/src/fixed_array.rs +763 -0
- package/reader-backports/hdf5-reader/src/fractal_heap.rs +2029 -0
- package/reader-backports/hdf5-reader/src/global_heap.rs +284 -0
- package/reader-backports/hdf5-reader/src/group.rs +795 -0
- package/reader-backports/hdf5-reader/src/io.rs +413 -0
- package/reader-backports/hdf5-reader/src/lib.rs +916 -0
- package/reader-backports/hdf5-reader/src/local_heap.rs +261 -0
- package/reader-backports/hdf5-reader/src/messages/attribute.rs +304 -0
- package/reader-backports/hdf5-reader/src/messages/attribute_info.rs +96 -0
- package/reader-backports/hdf5-reader/src/messages/btree_k.rs +36 -0
- package/reader-backports/hdf5-reader/src/messages/continuation.rs +46 -0
- package/reader-backports/hdf5-reader/src/messages/dataspace.rs +259 -0
- package/reader-backports/hdf5-reader/src/messages/datatype.rs +938 -0
- package/reader-backports/hdf5-reader/src/messages/external_files.rs +50 -0
- package/reader-backports/hdf5-reader/src/messages/fill_value.rs +233 -0
- package/reader-backports/hdf5-reader/src/messages/filter_pipeline.rs +147 -0
- package/reader-backports/hdf5-reader/src/messages/group_info.rs +103 -0
- package/reader-backports/hdf5-reader/src/messages/layout.rs +476 -0
- package/reader-backports/hdf5-reader/src/messages/link.rs +170 -0
- package/reader-backports/hdf5-reader/src/messages/link_info.rs +127 -0
- package/reader-backports/hdf5-reader/src/messages/mod.rs +374 -0
- package/reader-backports/hdf5-reader/src/messages/modification_time.rs +83 -0
- package/reader-backports/hdf5-reader/src/messages/shared.rs +164 -0
- package/reader-backports/hdf5-reader/src/messages/shared_table.rs +67 -0
- package/reader-backports/hdf5-reader/src/messages/symbol_table_msg.rs +51 -0
- package/reader-backports/hdf5-reader/src/object_header.rs +1749 -0
- package/reader-backports/hdf5-reader/src/reference.rs +90 -0
- package/reader-backports/hdf5-reader/src/shared_message_table.rs +425 -0
- package/reader-backports/hdf5-reader/src/storage.rs +538 -0
- package/reader-backports/hdf5-reader/src/superblock.rs +285 -0
- package/reader-backports/hdf5-reader/src/symbol_table.rs +277 -0
- package/reader-backports/netcdf-reader/Cargo.toml +141 -0
- package/reader-backports/netcdf-reader/LICENSE-APACHE +164 -0
- package/reader-backports/netcdf-reader/LICENSE-MIT +21 -0
- package/reader-backports/netcdf-reader/PROVENANCE.json +55 -0
- package/reader-backports/netcdf-reader/README.md +308 -0
- package/reader-backports/netcdf-reader/UPSTREAM-Cargo.toml +48 -0
- package/reader-backports/netcdf-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/netcdf-reader/src/cf/axes.rs +419 -0
- package/reader-backports/netcdf-reader/src/cf/bounds.rs +108 -0
- package/reader-backports/netcdf-reader/src/cf/crs.rs +269 -0
- package/reader-backports/netcdf-reader/src/cf/mod.rs +30 -0
- package/reader-backports/netcdf-reader/src/cf/time.rs +1122 -0
- package/reader-backports/netcdf-reader/src/classic/data.rs +1465 -0
- package/reader-backports/netcdf-reader/src/classic/header.rs +1097 -0
- package/reader-backports/netcdf-reader/src/classic/mod.rs +240 -0
- package/reader-backports/netcdf-reader/src/classic/storage.rs +211 -0
- package/reader-backports/netcdf-reader/src/classic/types.rs +93 -0
- package/reader-backports/netcdf-reader/src/classic/variable.rs +1162 -0
- package/reader-backports/netcdf-reader/src/error.rs +59 -0
- package/reader-backports/netcdf-reader/src/lib.rs +1531 -0
- package/reader-backports/netcdf-reader/src/masked.rs +173 -0
- package/reader-backports/netcdf-reader/src/nc4/attributes.rs +203 -0
- package/reader-backports/netcdf-reader/src/nc4/dimensions.rs +139 -0
- package/reader-backports/netcdf-reader/src/nc4/groups.rs +308 -0
- package/reader-backports/netcdf-reader/src/nc4/mod.rs +572 -0
- package/reader-backports/netcdf-reader/src/nc4/types.rs +309 -0
- package/reader-backports/netcdf-reader/src/nc4/variables.rs +420 -0
- package/reader-backports/netcdf-reader/src/types.rs +18 -0
- package/reader-backports/netcdf-reader/src/unpack.rs +113 -0
- package/reader-backports/netcdf-reader/src/user_defined.rs +569 -0
|
@@ -0,0 +1,2068 @@
|
|
|
1
|
+
//! HDF5 B-tree Version 2.
|
|
2
|
+
//!
|
|
3
|
+
//! V2 B-trees are used by newer-style groups and datasets for indexed link
|
|
4
|
+
//! storage, attribute storage, and chunked dataset indexing. The header
|
|
5
|
+
//! (`BTHD`) describes the tree parameters. Internal nodes (`BTIN`) and leaf
|
|
6
|
+
//! nodes (`BTLF`) contain the actual records.
|
|
7
|
+
//!
|
|
8
|
+
//! This module provides the header parse, record types, and a traversal
|
|
9
|
+
//! function that collects all records from a tree.
|
|
10
|
+
|
|
11
|
+
use std::collections::HashSet;
|
|
12
|
+
|
|
13
|
+
use crate::checksum::jenkins_lookup3;
|
|
14
|
+
use crate::error::{Error, Result};
|
|
15
|
+
use crate::io::Cursor;
|
|
16
|
+
use crate::storage::Storage;
|
|
17
|
+
|
|
18
|
+
// ---------------------------------------------------------------------------
|
|
19
|
+
// Signatures
|
|
20
|
+
// ---------------------------------------------------------------------------
|
|
21
|
+
|
|
22
|
+
const BTHD_SIGNATURE: [u8; 4] = *b"BTHD";
|
|
23
|
+
const BTIN_SIGNATURE: [u8; 4] = *b"BTIN";
|
|
24
|
+
const BTLF_SIGNATURE: [u8; 4] = *b"BTLF";
|
|
25
|
+
const MAX_BTREE_V2_DEPTH: u16 = 64;
|
|
26
|
+
|
|
27
|
+
// ---------------------------------------------------------------------------
|
|
28
|
+
// Header
|
|
29
|
+
// ---------------------------------------------------------------------------
|
|
30
|
+
|
|
31
|
+
/// Parsed B-tree v2 header.
|
|
32
|
+
#[derive(Debug, Clone)]
|
|
33
|
+
pub struct BTreeV2Header {
|
|
34
|
+
/// B-tree type (determines the record format).
|
|
35
|
+
pub btree_type: u8,
|
|
36
|
+
/// Size in bytes of each B-tree node (both internal and leaf).
|
|
37
|
+
pub node_size: u32,
|
|
38
|
+
/// Size in bytes of each record.
|
|
39
|
+
pub record_size: u16,
|
|
40
|
+
/// Depth of the tree (0 = root is a leaf).
|
|
41
|
+
pub depth: u16,
|
|
42
|
+
/// Percent full at which to split a node.
|
|
43
|
+
pub split_percent: u8,
|
|
44
|
+
/// Percent full at which to merge a node.
|
|
45
|
+
pub merge_percent: u8,
|
|
46
|
+
/// Address of the root node.
|
|
47
|
+
pub root_node_address: u64,
|
|
48
|
+
/// Number of records in the root node.
|
|
49
|
+
pub num_records_in_root: u16,
|
|
50
|
+
/// Total number of records in the entire tree.
|
|
51
|
+
pub total_records: u64,
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
impl BTreeV2Header {
|
|
55
|
+
/// Parse a B-tree v2 header at the current cursor position.
|
|
56
|
+
///
|
|
57
|
+
/// Format:
|
|
58
|
+
/// - Signature: `BTHD` (4 bytes)
|
|
59
|
+
/// - Version: 0 (1 byte)
|
|
60
|
+
/// - B-tree type (u8)
|
|
61
|
+
/// - Node size (u32 LE)
|
|
62
|
+
/// - Record size (u16 LE)
|
|
63
|
+
/// - Depth (u16 LE)
|
|
64
|
+
/// - Split percent (u8)
|
|
65
|
+
/// - Merge percent (u8)
|
|
66
|
+
/// - Root node address (`offset_size` bytes)
|
|
67
|
+
/// - Number of records in root node (u16 LE)
|
|
68
|
+
/// - Total number of records in tree (`length_size` bytes)
|
|
69
|
+
/// - Checksum (u32 LE)
|
|
70
|
+
pub fn parse(cursor: &mut Cursor, offset_size: u8, length_size: u8) -> Result<Self> {
|
|
71
|
+
let start = cursor.position();
|
|
72
|
+
|
|
73
|
+
let sig = cursor.read_bytes(4)?;
|
|
74
|
+
if sig != BTHD_SIGNATURE {
|
|
75
|
+
return Err(Error::InvalidBTreeV2Signature { context: "header" });
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
let version = cursor.read_u8()?;
|
|
79
|
+
if version != 0 {
|
|
80
|
+
return Err(Error::UnsupportedBTreeVersion(version));
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
let btree_type = cursor.read_u8()?;
|
|
84
|
+
let node_size = cursor.read_u32_le()?;
|
|
85
|
+
let record_size = cursor.read_u16_le()?;
|
|
86
|
+
let depth = cursor.read_u16_le()?;
|
|
87
|
+
let split_percent = cursor.read_u8()?;
|
|
88
|
+
let merge_percent = cursor.read_u8()?;
|
|
89
|
+
let root_node_address = cursor.read_offset(offset_size)?;
|
|
90
|
+
let num_records_in_root = cursor.read_u16_le()?;
|
|
91
|
+
let total_records = cursor.read_length(length_size)?;
|
|
92
|
+
|
|
93
|
+
// Checksum covers everything from signature through total_records.
|
|
94
|
+
let checksum_end = cursor.position();
|
|
95
|
+
let stored_checksum = cursor.read_u32_le()?;
|
|
96
|
+
|
|
97
|
+
let computed = jenkins_lookup3(&cursor.data()[start as usize..checksum_end as usize]);
|
|
98
|
+
if computed != stored_checksum {
|
|
99
|
+
return Err(Error::ChecksumMismatch {
|
|
100
|
+
expected: stored_checksum,
|
|
101
|
+
actual: computed,
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
Ok(BTreeV2Header {
|
|
106
|
+
btree_type,
|
|
107
|
+
node_size,
|
|
108
|
+
record_size,
|
|
109
|
+
depth,
|
|
110
|
+
split_percent,
|
|
111
|
+
merge_percent,
|
|
112
|
+
root_node_address,
|
|
113
|
+
num_records_in_root,
|
|
114
|
+
total_records,
|
|
115
|
+
})
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/// Parse a B-tree v2 header from random-access storage.
|
|
119
|
+
pub fn parse_at_storage(
|
|
120
|
+
storage: &dyn Storage,
|
|
121
|
+
address: u64,
|
|
122
|
+
offset_size: u8,
|
|
123
|
+
length_size: u8,
|
|
124
|
+
) -> Result<Self> {
|
|
125
|
+
let header_len = 4
|
|
126
|
+
+ 1
|
|
127
|
+
+ 1
|
|
128
|
+
+ 4
|
|
129
|
+
+ 2
|
|
130
|
+
+ 2
|
|
131
|
+
+ 1
|
|
132
|
+
+ 1
|
|
133
|
+
+ usize::from(offset_size)
|
|
134
|
+
+ 2
|
|
135
|
+
+ usize::from(length_size)
|
|
136
|
+
+ 4;
|
|
137
|
+
let bytes = storage.read_range(address, header_len)?;
|
|
138
|
+
let mut cursor = Cursor::new(bytes.as_ref());
|
|
139
|
+
Self::parse(&mut cursor, offset_size, length_size)
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// ---------------------------------------------------------------------------
|
|
144
|
+
// Records
|
|
145
|
+
// ---------------------------------------------------------------------------
|
|
146
|
+
|
|
147
|
+
/// A record from a B-tree v2.
|
|
148
|
+
///
|
|
149
|
+
/// The record format depends on the B-tree type field in the header.
|
|
150
|
+
#[derive(Debug, Clone)]
|
|
151
|
+
pub enum BTreeV2Record {
|
|
152
|
+
/// Type 1: indirectly accessed, non-filtered huge fractal heap object.
|
|
153
|
+
HugeIndirectNonFiltered {
|
|
154
|
+
address: u64,
|
|
155
|
+
length: u64,
|
|
156
|
+
object_id: u64,
|
|
157
|
+
},
|
|
158
|
+
/// Type 2: indirectly accessed, filtered huge fractal heap object.
|
|
159
|
+
HugeIndirectFiltered {
|
|
160
|
+
address: u64,
|
|
161
|
+
filtered_length: u64,
|
|
162
|
+
filter_mask: u32,
|
|
163
|
+
memory_length: u64,
|
|
164
|
+
object_id: u64,
|
|
165
|
+
},
|
|
166
|
+
/// Type 3: directly accessed, non-filtered huge fractal heap object.
|
|
167
|
+
HugeDirectNonFiltered { address: u64, length: u64 },
|
|
168
|
+
/// Type 4: directly accessed, filtered huge fractal heap object.
|
|
169
|
+
HugeDirectFiltered {
|
|
170
|
+
address: u64,
|
|
171
|
+
filtered_length: u64,
|
|
172
|
+
filter_mask: u32,
|
|
173
|
+
memory_length: u64,
|
|
174
|
+
},
|
|
175
|
+
/// Type 5: Link name for indexed group (hashed).
|
|
176
|
+
LinkNameHash { hash: u32, heap_id: Vec<u8> },
|
|
177
|
+
/// Type 6: Creation order for indexed group.
|
|
178
|
+
CreationOrder { order: u64, heap_id: Vec<u8> },
|
|
179
|
+
/// Type 8: Attribute name for indexed group (hashed).
|
|
180
|
+
AttributeNameHash {
|
|
181
|
+
hash: u32,
|
|
182
|
+
flags: u8,
|
|
183
|
+
creation_order: u32,
|
|
184
|
+
heap_id: Vec<u8>,
|
|
185
|
+
},
|
|
186
|
+
/// Type 9: Attribute creation order.
|
|
187
|
+
AttributeCreationOrder { order: u32, heap_id: Vec<u8> },
|
|
188
|
+
/// Type 10: Non-filtered chunked dataset record (v2 chunk index).
|
|
189
|
+
ChunkedNonFiltered { address: u64, offsets: Vec<u64> },
|
|
190
|
+
/// Type 11: Filtered chunked dataset record (v2 chunk index).
|
|
191
|
+
ChunkedFiltered {
|
|
192
|
+
address: u64,
|
|
193
|
+
chunk_size: u64,
|
|
194
|
+
filter_mask: u32,
|
|
195
|
+
offsets: Vec<u64>,
|
|
196
|
+
},
|
|
197
|
+
/// Type 7: shared object-header message stored in the SOHM heap.
|
|
198
|
+
SharedMessageHeap {
|
|
199
|
+
hash: u32,
|
|
200
|
+
reference_count: u32,
|
|
201
|
+
heap_id: Vec<u8>,
|
|
202
|
+
},
|
|
203
|
+
/// Type 7: shared object-header message stored in an object header.
|
|
204
|
+
SharedMessageObjectHeader {
|
|
205
|
+
hash: u32,
|
|
206
|
+
message_type: u16,
|
|
207
|
+
object_header_index: u16,
|
|
208
|
+
object_header_address: u64,
|
|
209
|
+
},
|
|
210
|
+
/// Unknown/unsupported record type — raw bytes preserved.
|
|
211
|
+
Unknown { record_type: u8, data: Vec<u8> },
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
// ---------------------------------------------------------------------------
|
|
215
|
+
// Record parsing
|
|
216
|
+
// ---------------------------------------------------------------------------
|
|
217
|
+
|
|
218
|
+
/// Parse a single record of the given B-tree type.
|
|
219
|
+
#[allow(clippy::too_many_arguments)]
|
|
220
|
+
fn parse_record(
|
|
221
|
+
cursor: &mut Cursor,
|
|
222
|
+
btree_type: u8,
|
|
223
|
+
record_size: u16,
|
|
224
|
+
offset_size: u8,
|
|
225
|
+
length_size: u8,
|
|
226
|
+
ndims: Option<u32>,
|
|
227
|
+
chunk_dims: &[u32],
|
|
228
|
+
heap_id_len: usize,
|
|
229
|
+
) -> Result<BTreeV2Record> {
|
|
230
|
+
let record_start = cursor.position();
|
|
231
|
+
|
|
232
|
+
let record = match btree_type {
|
|
233
|
+
// Type 1: indirectly accessed, non-filtered huge fractal heap object.
|
|
234
|
+
1 => BTreeV2Record::HugeIndirectNonFiltered {
|
|
235
|
+
address: cursor.read_offset(offset_size)?,
|
|
236
|
+
length: cursor.read_length(length_size)?,
|
|
237
|
+
object_id: cursor.read_length(length_size)?,
|
|
238
|
+
},
|
|
239
|
+
|
|
240
|
+
// Type 2: indirectly accessed, filtered huge fractal heap object.
|
|
241
|
+
2 => BTreeV2Record::HugeIndirectFiltered {
|
|
242
|
+
address: cursor.read_offset(offset_size)?,
|
|
243
|
+
filtered_length: cursor.read_length(length_size)?,
|
|
244
|
+
filter_mask: cursor.read_u32_le()?,
|
|
245
|
+
memory_length: cursor.read_length(length_size)?,
|
|
246
|
+
object_id: cursor.read_length(length_size)?,
|
|
247
|
+
},
|
|
248
|
+
|
|
249
|
+
// Type 3: directly accessed, non-filtered huge fractal heap object.
|
|
250
|
+
3 => BTreeV2Record::HugeDirectNonFiltered {
|
|
251
|
+
address: cursor.read_offset(offset_size)?,
|
|
252
|
+
length: cursor.read_length(length_size)?,
|
|
253
|
+
},
|
|
254
|
+
|
|
255
|
+
// Type 4: directly accessed, filtered huge fractal heap object.
|
|
256
|
+
4 => BTreeV2Record::HugeDirectFiltered {
|
|
257
|
+
address: cursor.read_offset(offset_size)?,
|
|
258
|
+
filtered_length: cursor.read_length(length_size)?,
|
|
259
|
+
filter_mask: cursor.read_u32_le()?,
|
|
260
|
+
memory_length: cursor.read_length(length_size)?,
|
|
261
|
+
},
|
|
262
|
+
|
|
263
|
+
// Type 5: link name hash
|
|
264
|
+
5 => {
|
|
265
|
+
let hash = cursor.read_u32_le()?;
|
|
266
|
+
let heap_id = cursor.read_bytes(heap_id_len)?.to_vec();
|
|
267
|
+
BTreeV2Record::LinkNameHash { hash, heap_id }
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
// Type 6: creation order
|
|
271
|
+
6 => {
|
|
272
|
+
let order = cursor.read_u64_le()?;
|
|
273
|
+
let heap_id = cursor.read_bytes(heap_id_len)?.to_vec();
|
|
274
|
+
BTreeV2Record::CreationOrder { order, heap_id }
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
// Type 7: shared object-header messages.
|
|
278
|
+
7 => {
|
|
279
|
+
let location = cursor.read_u8()?;
|
|
280
|
+
cursor.skip(3)?;
|
|
281
|
+
let hash = cursor.read_u32_le()?;
|
|
282
|
+
match location {
|
|
283
|
+
0 => {
|
|
284
|
+
let reference_count = cursor.read_u32_le()?;
|
|
285
|
+
let heap_id = cursor.read_bytes(8)?.to_vec();
|
|
286
|
+
BTreeV2Record::SharedMessageHeap {
|
|
287
|
+
hash,
|
|
288
|
+
reference_count,
|
|
289
|
+
heap_id,
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
1 => {
|
|
293
|
+
let _reserved = cursor.read_u8()?;
|
|
294
|
+
let message_type = u16::from(cursor.read_u8()?);
|
|
295
|
+
let object_header_index = cursor.read_u16_le()?;
|
|
296
|
+
let object_header_address = cursor.read_offset(offset_size)?;
|
|
297
|
+
BTreeV2Record::SharedMessageObjectHeader {
|
|
298
|
+
hash,
|
|
299
|
+
message_type,
|
|
300
|
+
object_header_index,
|
|
301
|
+
object_header_address,
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
other => {
|
|
305
|
+
return Err(Error::InvalidData(format!(
|
|
306
|
+
"unknown SOHM B-tree record location: {other}"
|
|
307
|
+
)));
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
// Type 8: attribute name hash
|
|
313
|
+
8 => {
|
|
314
|
+
let heap_id = cursor.read_bytes(heap_id_len)?.to_vec();
|
|
315
|
+
let flags = cursor.read_u8()?;
|
|
316
|
+
let creation_order = cursor.read_u32_le()?;
|
|
317
|
+
let hash = cursor.read_u32_le()?;
|
|
318
|
+
BTreeV2Record::AttributeNameHash {
|
|
319
|
+
hash,
|
|
320
|
+
flags,
|
|
321
|
+
creation_order,
|
|
322
|
+
heap_id,
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
// Type 9: attribute creation order
|
|
327
|
+
9 => {
|
|
328
|
+
let heap_id = cursor.read_bytes(heap_id_len)?.to_vec();
|
|
329
|
+
let _flags = cursor.read_u8()?;
|
|
330
|
+
let order = cursor.read_u32_le()?;
|
|
331
|
+
BTreeV2Record::AttributeCreationOrder { order, heap_id }
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
// Type 10: non-filtered chunk
|
|
335
|
+
10 => {
|
|
336
|
+
// Chunk offsets are encoded as scaled 64-bit values.
|
|
337
|
+
// The number of offset dimensions is calculated from the record size.
|
|
338
|
+
// Each offset is 8 bytes in a type-10 record.
|
|
339
|
+
let offset_bytes_available = record_payload_len(
|
|
340
|
+
record_size,
|
|
341
|
+
offset_size as usize,
|
|
342
|
+
"B-tree v2 type-10 chunk record is shorter than its address",
|
|
343
|
+
)?;
|
|
344
|
+
let address = cursor.read_offset(offset_size)?;
|
|
345
|
+
let num_offsets = offset_bytes_available / 8;
|
|
346
|
+
let offsets = read_scaled_chunk_offsets(cursor, num_offsets, ndims, chunk_dims)?;
|
|
347
|
+
BTreeV2Record::ChunkedNonFiltered { address, offsets }
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
// Type 11: filtered chunk. The on-disk chunk-size field width is not
|
|
351
|
+
// the file's global length size — libhdf5 sizes it to the nominal
|
|
352
|
+
// chunk byte count. We recover that width from the record size:
|
|
353
|
+
// record_size = address + chunk_size_field + filter_mask(4) + offsets
|
|
354
|
+
// where the number of scaled 8-byte offsets equals the dimensionality.
|
|
355
|
+
11 => {
|
|
356
|
+
let ndims_val = ndims
|
|
357
|
+
.map(|n| n as usize)
|
|
358
|
+
.or_else(|| (!chunk_dims.is_empty()).then_some(chunk_dims.len()))
|
|
359
|
+
.ok_or_else(|| {
|
|
360
|
+
Error::InvalidData(
|
|
361
|
+
"B-tree v2 type-11 chunk record needs a known dimensionality".to_string(),
|
|
362
|
+
)
|
|
363
|
+
})?;
|
|
364
|
+
let offsets_bytes = ndims_val
|
|
365
|
+
.checked_mul(8)
|
|
366
|
+
.ok_or_else(|| Error::InvalidData("chunk offset bytes overflow".to_string()))?;
|
|
367
|
+
let fixed_size = offset_size as usize + 4 + offsets_bytes;
|
|
368
|
+
let chunk_size_len =
|
|
369
|
+
(record_size as usize)
|
|
370
|
+
.checked_sub(fixed_size)
|
|
371
|
+
.ok_or_else(|| {
|
|
372
|
+
Error::InvalidData(
|
|
373
|
+
"B-tree v2 type-11 chunk record is shorter than its fixed fields"
|
|
374
|
+
.to_string(),
|
|
375
|
+
)
|
|
376
|
+
})?;
|
|
377
|
+
if chunk_size_len == 0 || chunk_size_len > 8 {
|
|
378
|
+
return Err(Error::InvalidData(format!(
|
|
379
|
+
"B-tree v2 type-11 chunk-size field width {chunk_size_len} is out of range"
|
|
380
|
+
)));
|
|
381
|
+
}
|
|
382
|
+
let address = cursor.read_offset(offset_size)?;
|
|
383
|
+
// The chunk-size field is 1–8 bytes wide (sized to the chunk), not a "length".
|
|
384
|
+
let chunk_size = cursor.read_uvar(chunk_size_len)?;
|
|
385
|
+
let filter_mask = cursor.read_u32_le()?;
|
|
386
|
+
let offsets = read_scaled_chunk_offsets(cursor, ndims_val, ndims, chunk_dims)?;
|
|
387
|
+
BTreeV2Record::ChunkedFiltered {
|
|
388
|
+
address,
|
|
389
|
+
chunk_size,
|
|
390
|
+
filter_mask,
|
|
391
|
+
offsets,
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
// Unknown type — read raw bytes.
|
|
396
|
+
_ => {
|
|
397
|
+
let data = cursor.read_bytes(record_size as usize)?.to_vec();
|
|
398
|
+
return Ok(BTreeV2Record::Unknown {
|
|
399
|
+
record_type: btree_type,
|
|
400
|
+
data,
|
|
401
|
+
});
|
|
402
|
+
}
|
|
403
|
+
};
|
|
404
|
+
|
|
405
|
+
// Ensure we consumed no more than record_size bytes, then skip padding.
|
|
406
|
+
let consumed = cursor.position() - record_start;
|
|
407
|
+
let record_size = u64::from(record_size);
|
|
408
|
+
if consumed > record_size {
|
|
409
|
+
return Err(Error::InvalidData(format!(
|
|
410
|
+
"B-tree v2 type-{btree_type} record consumed {consumed} bytes but record size is {record_size}"
|
|
411
|
+
)));
|
|
412
|
+
}
|
|
413
|
+
if consumed < record_size {
|
|
414
|
+
cursor.skip((record_size - consumed) as usize)?;
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
Ok(record)
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
fn record_payload_len(record_size: u16, fixed_size: usize, error_message: &str) -> Result<usize> {
|
|
421
|
+
(record_size as usize)
|
|
422
|
+
.checked_sub(fixed_size)
|
|
423
|
+
.ok_or_else(|| Error::InvalidData(error_message.into()))
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
fn read_scaled_chunk_offsets(
|
|
427
|
+
cursor: &mut Cursor,
|
|
428
|
+
num_offsets: usize,
|
|
429
|
+
ndims: Option<u32>,
|
|
430
|
+
chunk_dims: &[u32],
|
|
431
|
+
) -> Result<Vec<u64>> {
|
|
432
|
+
if let Some(ndims) = ndims {
|
|
433
|
+
let ndims = usize::try_from(ndims)
|
|
434
|
+
.map_err(|_| Error::InvalidData("B-tree v2 chunk rank exceeds usize".into()))?;
|
|
435
|
+
if num_offsets != ndims {
|
|
436
|
+
return Err(Error::InvalidData(format!(
|
|
437
|
+
"B-tree v2 chunk record has {num_offsets} offsets for {ndims} dimensions"
|
|
438
|
+
)));
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
if !chunk_dims.is_empty() && num_offsets != chunk_dims.len() {
|
|
443
|
+
return Err(Error::InvalidData(format!(
|
|
444
|
+
"B-tree v2 chunk record has {num_offsets} offsets but {} chunk dimensions",
|
|
445
|
+
chunk_dims.len()
|
|
446
|
+
)));
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
let mut offsets = Vec::with_capacity(num_offsets);
|
|
450
|
+
for dim in 0..num_offsets {
|
|
451
|
+
let scaled = cursor.read_u64_le()?;
|
|
452
|
+
let chunk_extent = chunk_dims.get(dim).copied().unwrap_or(1);
|
|
453
|
+
let offset = scaled
|
|
454
|
+
.checked_mul(u64::from(chunk_extent))
|
|
455
|
+
.ok_or_else(|| Error::InvalidData("B-tree v2 chunk offset overflows u64".into()))?;
|
|
456
|
+
offsets.push(offset);
|
|
457
|
+
}
|
|
458
|
+
Ok(offsets)
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
fn record_matches_chunk_bounds(
|
|
462
|
+
record: &BTreeV2Record,
|
|
463
|
+
chunk_dims: &[u32],
|
|
464
|
+
chunk_bounds: Option<(&[u64], &[u64])>,
|
|
465
|
+
) -> bool {
|
|
466
|
+
let Some((first_chunk, last_chunk)) = chunk_bounds else {
|
|
467
|
+
return true;
|
|
468
|
+
};
|
|
469
|
+
|
|
470
|
+
let offsets = match record {
|
|
471
|
+
BTreeV2Record::ChunkedNonFiltered { offsets, .. }
|
|
472
|
+
| BTreeV2Record::ChunkedFiltered { offsets, .. } => offsets,
|
|
473
|
+
_ => return true,
|
|
474
|
+
};
|
|
475
|
+
|
|
476
|
+
offsets.iter().enumerate().all(|(dim, offset)| {
|
|
477
|
+
let chunk_index = *offset / u64::from(chunk_dims[dim]);
|
|
478
|
+
chunk_index >= first_chunk[dim] && chunk_index <= last_chunk[dim]
|
|
479
|
+
})
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
fn record_matches_link_name_hash(record: &BTreeV2Record, target_hash: Option<u32>) -> bool {
|
|
483
|
+
match target_hash {
|
|
484
|
+
Some(target_hash) => {
|
|
485
|
+
matches!(record, BTreeV2Record::LinkNameHash { hash, .. } if *hash == target_hash)
|
|
486
|
+
}
|
|
487
|
+
None => true,
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
fn record_matches_query(
|
|
492
|
+
record: &BTreeV2Record,
|
|
493
|
+
chunk_dims: &[u32],
|
|
494
|
+
chunk_bounds: Option<(&[u64], &[u64])>,
|
|
495
|
+
link_name_hash: Option<u32>,
|
|
496
|
+
) -> bool {
|
|
497
|
+
record_matches_link_name_hash(record, link_name_hash)
|
|
498
|
+
&& record_matches_chunk_bounds(record, chunk_dims, chunk_bounds)
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
fn link_name_hash(record: &BTreeV2Record) -> Option<u32> {
|
|
502
|
+
match record {
|
|
503
|
+
BTreeV2Record::LinkNameHash { hash, .. } => Some(*hash),
|
|
504
|
+
_ => None,
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
fn child_may_match_link_name_hash(
|
|
509
|
+
records: &[BTreeV2Record],
|
|
510
|
+
child_index: usize,
|
|
511
|
+
target_hash: Option<u32>,
|
|
512
|
+
) -> bool {
|
|
513
|
+
let Some(target_hash) = target_hash else {
|
|
514
|
+
return true;
|
|
515
|
+
};
|
|
516
|
+
|
|
517
|
+
let lower_matches = child_index == 0
|
|
518
|
+
|| link_name_hash(&records[child_index - 1]).map_or(true, |hash| target_hash >= hash);
|
|
519
|
+
let upper_matches = child_index == records.len()
|
|
520
|
+
|| link_name_hash(&records[child_index]).map_or(true, |hash| target_hash <= hash);
|
|
521
|
+
lower_matches && upper_matches
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
fn validate_btree_v2_depth(depth: u16) -> Result<()> {
|
|
525
|
+
if depth > MAX_BTREE_V2_DEPTH {
|
|
526
|
+
return Err(Error::InvalidData(format!(
|
|
527
|
+
"B-tree v2 depth {depth} exceeds traversal limit {MAX_BTREE_V2_DEPTH}"
|
|
528
|
+
)));
|
|
529
|
+
}
|
|
530
|
+
Ok(())
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
fn enter_btree_v2_node(visited: &mut HashSet<u64>, address: u64) -> Result<()> {
|
|
534
|
+
if !visited.insert(address) {
|
|
535
|
+
return Err(Error::InvalidData(format!(
|
|
536
|
+
"B-tree v2 traversal revisits node at offset {address:#x}"
|
|
537
|
+
)));
|
|
538
|
+
}
|
|
539
|
+
Ok(())
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
// ---------------------------------------------------------------------------
|
|
543
|
+
// Node parsing
|
|
544
|
+
// ---------------------------------------------------------------------------
|
|
545
|
+
|
|
546
|
+
/// Compute the number of bytes needed to represent `max_records` as an
|
|
547
|
+
/// unsigned integer (used for child-node record counts in internal nodes).
|
|
548
|
+
fn num_records_size(max_records: u64) -> usize {
|
|
549
|
+
if max_records <= 0xFF {
|
|
550
|
+
1
|
|
551
|
+
} else if max_records <= 0xFFFF {
|
|
552
|
+
2
|
|
553
|
+
} else if max_records <= 0xFFFF_FFFF {
|
|
554
|
+
4
|
|
555
|
+
} else {
|
|
556
|
+
8
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
/// Compute the maximum number of records that fit in a leaf node.
|
|
561
|
+
fn max_leaf_records(node_size: u32, record_size: u16) -> u64 {
|
|
562
|
+
// Leaf node overhead: signature(4) + version(1) + type(1) + checksum(4) = 10
|
|
563
|
+
let overhead = 10u32;
|
|
564
|
+
if node_size <= overhead || record_size == 0 {
|
|
565
|
+
return 0;
|
|
566
|
+
}
|
|
567
|
+
((node_size - overhead) / record_size as u32) as u64
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
/// Compute the maximum number of records that fit in an internal node.
|
|
571
|
+
/// This depends on the pointer size (offset_size) and the number-of-records
|
|
572
|
+
/// encoding for child nodes, which makes it recursive in principle. We use
|
|
573
|
+
/// an iterative approach.
|
|
574
|
+
fn max_internal_records(
|
|
575
|
+
node_size: u32,
|
|
576
|
+
record_size: u16,
|
|
577
|
+
offset_size: u8,
|
|
578
|
+
max_child_records: u64,
|
|
579
|
+
) -> u64 {
|
|
580
|
+
// Internal node overhead: signature(4) + version(1) + type(1) + checksum(4) = 10
|
|
581
|
+
let overhead = 10u32;
|
|
582
|
+
if node_size <= overhead || record_size == 0 {
|
|
583
|
+
return 0;
|
|
584
|
+
}
|
|
585
|
+
let available = (node_size - overhead) as u64;
|
|
586
|
+
// Each entry in an internal node is: record(record_size) + child_pointer(offset_size) + num_records(var)
|
|
587
|
+
let nrec_size = num_records_size(max_child_records) as u64;
|
|
588
|
+
let entry_size = record_size as u64 + offset_size as u64 + nrec_size;
|
|
589
|
+
// There is one more child pointer + num_records than records.
|
|
590
|
+
// So: n * record_size + (n+1) * (offset_size + nrec_size) <= available
|
|
591
|
+
// => n * (record_size + offset_size + nrec_size) + offset_size + nrec_size <= available
|
|
592
|
+
let extra = offset_size as u64 + nrec_size;
|
|
593
|
+
if available <= extra {
|
|
594
|
+
return 0;
|
|
595
|
+
}
|
|
596
|
+
(available - extra) / entry_size
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
fn node_checksum_bounds(
|
|
600
|
+
cursor: &Cursor<'_>,
|
|
601
|
+
start: u64,
|
|
602
|
+
node_size: u32,
|
|
603
|
+
compact_end: u64,
|
|
604
|
+
context: &'static str,
|
|
605
|
+
) -> Result<usize> {
|
|
606
|
+
let start = usize::try_from(start)
|
|
607
|
+
.map_err(|_| Error::InvalidData(format!("B-tree v2 {context} offset is too large")))?;
|
|
608
|
+
let node_size = usize::try_from(node_size)
|
|
609
|
+
.map_err(|_| Error::InvalidData(format!("B-tree v2 {context} size is too large")))?;
|
|
610
|
+
if node_size < 10 {
|
|
611
|
+
return Err(Error::InvalidData(format!(
|
|
612
|
+
"B-tree v2 {context} node is too small: {node_size} bytes"
|
|
613
|
+
)));
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
let checksum_pos = usize::try_from(compact_end)
|
|
617
|
+
.map_err(|_| Error::InvalidData(format!("B-tree v2 {context} checksum is too large")))?;
|
|
618
|
+
if checksum_pos < start {
|
|
619
|
+
return Err(Error::InvalidData(format!(
|
|
620
|
+
"B-tree v2 {context} payload starts before node"
|
|
621
|
+
)));
|
|
622
|
+
}
|
|
623
|
+
let node_end = start
|
|
624
|
+
.checked_add(node_size)
|
|
625
|
+
.ok_or_else(|| Error::InvalidData(format!("B-tree v2 {context} size overflow")))?;
|
|
626
|
+
|
|
627
|
+
if node_end > cursor.data().len() {
|
|
628
|
+
return Err(Error::UnexpectedEof {
|
|
629
|
+
offset: start as u64,
|
|
630
|
+
needed: node_size as u64,
|
|
631
|
+
available: cursor.data().len().saturating_sub(start) as u64,
|
|
632
|
+
});
|
|
633
|
+
}
|
|
634
|
+
let checksum_end = checksum_pos
|
|
635
|
+
.checked_add(4)
|
|
636
|
+
.ok_or_else(|| Error::InvalidData(format!("B-tree v2 {context} checksum overflow")))?;
|
|
637
|
+
if checksum_end > node_end {
|
|
638
|
+
return Err(Error::InvalidData(format!(
|
|
639
|
+
"B-tree v2 {context} contents exceed node size"
|
|
640
|
+
)));
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
Ok(checksum_pos)
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
fn verify_node_checksum(
|
|
647
|
+
cursor: &mut Cursor<'_>,
|
|
648
|
+
start: u64,
|
|
649
|
+
node_size: u32,
|
|
650
|
+
compact_end: u64,
|
|
651
|
+
context: &'static str,
|
|
652
|
+
) -> Result<()> {
|
|
653
|
+
let checksum_pos = node_checksum_bounds(cursor, start, node_size, compact_end, context)?;
|
|
654
|
+
cursor.set_position(checksum_pos as u64);
|
|
655
|
+
let stored_checksum = cursor.read_u32_le()?;
|
|
656
|
+
let computed = jenkins_lookup3(&cursor.data()[start as usize..checksum_pos]);
|
|
657
|
+
if computed != stored_checksum {
|
|
658
|
+
return Err(Error::ChecksumMismatch {
|
|
659
|
+
expected: stored_checksum,
|
|
660
|
+
actual: computed,
|
|
661
|
+
});
|
|
662
|
+
}
|
|
663
|
+
Ok(())
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
/// Parse a leaf node and collect its records.
|
|
667
|
+
#[allow(clippy::too_many_arguments)]
|
|
668
|
+
fn parse_leaf_node(
|
|
669
|
+
cursor: &mut Cursor,
|
|
670
|
+
header: &BTreeV2Header,
|
|
671
|
+
offset_size: u8,
|
|
672
|
+
length_size: u8,
|
|
673
|
+
ndims: Option<u32>,
|
|
674
|
+
chunk_dims: &[u32],
|
|
675
|
+
chunk_bounds: Option<(&[u64], &[u64])>,
|
|
676
|
+
link_name_hash: Option<u32>,
|
|
677
|
+
num_records: u16,
|
|
678
|
+
heap_id_len: usize,
|
|
679
|
+
records: &mut Vec<BTreeV2Record>,
|
|
680
|
+
) -> Result<()> {
|
|
681
|
+
let start = cursor.position();
|
|
682
|
+
|
|
683
|
+
let sig = cursor.read_bytes(4)?;
|
|
684
|
+
if sig != BTLF_SIGNATURE {
|
|
685
|
+
return Err(Error::InvalidBTreeV2Signature {
|
|
686
|
+
context: "leaf node",
|
|
687
|
+
});
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
let version = cursor.read_u8()?;
|
|
691
|
+
if version != 0 {
|
|
692
|
+
return Err(Error::UnsupportedBTreeVersion(version));
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
let node_type = cursor.read_u8()?;
|
|
696
|
+
if node_type != header.btree_type {
|
|
697
|
+
return Err(Error::InvalidData(format!(
|
|
698
|
+
"B-tree v2 leaf node type mismatch: header says {}, node says {}",
|
|
699
|
+
header.btree_type, node_type
|
|
700
|
+
)));
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
for _ in 0..num_records {
|
|
704
|
+
let record = parse_record(
|
|
705
|
+
cursor,
|
|
706
|
+
header.btree_type,
|
|
707
|
+
header.record_size,
|
|
708
|
+
offset_size,
|
|
709
|
+
length_size,
|
|
710
|
+
ndims,
|
|
711
|
+
chunk_dims,
|
|
712
|
+
heap_id_len,
|
|
713
|
+
)?;
|
|
714
|
+
if record_matches_query(&record, chunk_dims, chunk_bounds, link_name_hash) {
|
|
715
|
+
records.push(record);
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
verify_node_checksum(
|
|
720
|
+
cursor,
|
|
721
|
+
start,
|
|
722
|
+
header.node_size,
|
|
723
|
+
cursor.position(),
|
|
724
|
+
"leaf node",
|
|
725
|
+
)?;
|
|
726
|
+
|
|
727
|
+
Ok(())
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
/// Parse an internal node, collecting child addresses and recursing.
|
|
731
|
+
#[allow(clippy::too_many_arguments)]
|
|
732
|
+
fn parse_internal_node(
|
|
733
|
+
data: &[u8],
|
|
734
|
+
address: u64,
|
|
735
|
+
header: &BTreeV2Header,
|
|
736
|
+
offset_size: u8,
|
|
737
|
+
length_size: u8,
|
|
738
|
+
ndims: Option<u32>,
|
|
739
|
+
chunk_dims: &[u32],
|
|
740
|
+
chunk_bounds: Option<(&[u64], &[u64])>,
|
|
741
|
+
link_name_hash: Option<u32>,
|
|
742
|
+
num_records: u16,
|
|
743
|
+
depth: u16,
|
|
744
|
+
heap_id_len: usize,
|
|
745
|
+
visited: &mut HashSet<u64>,
|
|
746
|
+
records: &mut Vec<BTreeV2Record>,
|
|
747
|
+
) -> Result<()> {
|
|
748
|
+
if depth == 0 {
|
|
749
|
+
return Err(Error::InvalidData(
|
|
750
|
+
"B-tree v2 internal node traversal reached depth zero".into(),
|
|
751
|
+
));
|
|
752
|
+
}
|
|
753
|
+
enter_btree_v2_node(visited, address)?;
|
|
754
|
+
|
|
755
|
+
let mut cursor = Cursor::new(data);
|
|
756
|
+
cursor.set_position(address);
|
|
757
|
+
|
|
758
|
+
let start = cursor.position();
|
|
759
|
+
|
|
760
|
+
let sig = cursor.read_bytes(4)?;
|
|
761
|
+
if sig != BTIN_SIGNATURE {
|
|
762
|
+
return Err(Error::InvalidBTreeV2Signature {
|
|
763
|
+
context: "internal node",
|
|
764
|
+
});
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
let version = cursor.read_u8()?;
|
|
768
|
+
if version != 0 {
|
|
769
|
+
return Err(Error::UnsupportedBTreeVersion(version));
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
let node_type = cursor.read_u8()?;
|
|
773
|
+
if node_type != header.btree_type {
|
|
774
|
+
return Err(Error::InvalidData(format!(
|
|
775
|
+
"B-tree v2 internal node type mismatch: header says {}, node says {}",
|
|
776
|
+
header.btree_type, node_type
|
|
777
|
+
)));
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
// Compute max records for children to know the encoding size for
|
|
781
|
+
// child record counts.
|
|
782
|
+
let max_child_records = if depth == 1 {
|
|
783
|
+
max_leaf_records(header.node_size, header.record_size)
|
|
784
|
+
} else {
|
|
785
|
+
// For deeper trees, compute iteratively.
|
|
786
|
+
let leaf_max = max_leaf_records(header.node_size, header.record_size);
|
|
787
|
+
let mut prev_max = leaf_max;
|
|
788
|
+
for _ in 1..depth {
|
|
789
|
+
prev_max =
|
|
790
|
+
max_internal_records(header.node_size, header.record_size, offset_size, prev_max);
|
|
791
|
+
}
|
|
792
|
+
prev_max
|
|
793
|
+
};
|
|
794
|
+
let nrec_bytes = num_records_size(max_child_records);
|
|
795
|
+
|
|
796
|
+
// Read records and child pointers interleaved:
|
|
797
|
+
// child[0], record[0], child[1], record[1], ..., record[n-1], child[n]
|
|
798
|
+
// Plus total_records counts for each child.
|
|
799
|
+
//
|
|
800
|
+
// Actually per the HDF5 spec the layout is:
|
|
801
|
+
// record[0], record[1], ..., record[n-1],
|
|
802
|
+
// child_ptr[0], nrec[0], total[0], child_ptr[1], nrec[1], total[1], ..., child_ptr[n], nrec[n], total[n]
|
|
803
|
+
//
|
|
804
|
+
// Records first, then child pointers with their metadata.
|
|
805
|
+
|
|
806
|
+
// Read all records.
|
|
807
|
+
let mut node_records = Vec::with_capacity(num_records as usize);
|
|
808
|
+
for _ in 0..num_records {
|
|
809
|
+
let record = parse_record(
|
|
810
|
+
&mut cursor,
|
|
811
|
+
header.btree_type,
|
|
812
|
+
header.record_size,
|
|
813
|
+
offset_size,
|
|
814
|
+
length_size,
|
|
815
|
+
ndims,
|
|
816
|
+
chunk_dims,
|
|
817
|
+
heap_id_len,
|
|
818
|
+
)?;
|
|
819
|
+
node_records.push(record);
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
// Read child node pointers (num_records + 1 of them).
|
|
823
|
+
let num_children = num_records as usize + 1;
|
|
824
|
+
|
|
825
|
+
// Whether to include a "total records" field for each child.
|
|
826
|
+
// This is present when depth > 1.
|
|
827
|
+
let has_total_records = depth > 1;
|
|
828
|
+
// total_records encoding size for deeper nodes
|
|
829
|
+
let total_nrec_bytes = if has_total_records {
|
|
830
|
+
// Total records in a sub-tree — need enough bytes to hold the
|
|
831
|
+
// maximum total records. We use length_size as an upper bound.
|
|
832
|
+
length_size as usize
|
|
833
|
+
} else {
|
|
834
|
+
0
|
|
835
|
+
};
|
|
836
|
+
|
|
837
|
+
let mut child_addresses = Vec::with_capacity(num_children);
|
|
838
|
+
let mut child_nrecords = Vec::with_capacity(num_children);
|
|
839
|
+
|
|
840
|
+
for _ in 0..num_children {
|
|
841
|
+
let child_addr = cursor.read_offset(offset_size)?;
|
|
842
|
+
child_addresses.push(child_addr);
|
|
843
|
+
let nrec = cursor.read_uvar(nrec_bytes)?;
|
|
844
|
+
child_nrecords.push(nrec as u16);
|
|
845
|
+
if has_total_records {
|
|
846
|
+
// Skip total records count
|
|
847
|
+
cursor.read_uvar(total_nrec_bytes)?;
|
|
848
|
+
}
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
let compact_end = cursor.position();
|
|
852
|
+
verify_node_checksum(
|
|
853
|
+
&mut cursor,
|
|
854
|
+
start,
|
|
855
|
+
header.node_size,
|
|
856
|
+
compact_end,
|
|
857
|
+
"internal node",
|
|
858
|
+
)?;
|
|
859
|
+
|
|
860
|
+
// HDF5's v2 B-tree iterator visits child[i], then record[i], then the
|
|
861
|
+
// final child. Internal-node records are real records, not only separators.
|
|
862
|
+
let child_depth = depth - 1;
|
|
863
|
+
for (i, record) in node_records.iter().enumerate() {
|
|
864
|
+
let child_addr = child_addresses[i];
|
|
865
|
+
if !Cursor::is_undefined_offset(child_addr, offset_size)
|
|
866
|
+
&& child_may_match_link_name_hash(&node_records, i, link_name_hash)
|
|
867
|
+
{
|
|
868
|
+
let child_nrec = child_nrecords[i];
|
|
869
|
+
if child_depth == 0 {
|
|
870
|
+
enter_btree_v2_node(visited, child_addr)?;
|
|
871
|
+
let mut child_cursor = Cursor::new(data);
|
|
872
|
+
child_cursor.set_position(child_addr);
|
|
873
|
+
parse_leaf_node(
|
|
874
|
+
&mut child_cursor,
|
|
875
|
+
header,
|
|
876
|
+
offset_size,
|
|
877
|
+
length_size,
|
|
878
|
+
ndims,
|
|
879
|
+
chunk_dims,
|
|
880
|
+
chunk_bounds,
|
|
881
|
+
link_name_hash,
|
|
882
|
+
child_nrec,
|
|
883
|
+
heap_id_len,
|
|
884
|
+
records,
|
|
885
|
+
)?;
|
|
886
|
+
} else {
|
|
887
|
+
parse_internal_node(
|
|
888
|
+
data,
|
|
889
|
+
child_addr,
|
|
890
|
+
header,
|
|
891
|
+
offset_size,
|
|
892
|
+
length_size,
|
|
893
|
+
ndims,
|
|
894
|
+
chunk_dims,
|
|
895
|
+
chunk_bounds,
|
|
896
|
+
link_name_hash,
|
|
897
|
+
child_nrec,
|
|
898
|
+
child_depth,
|
|
899
|
+
heap_id_len,
|
|
900
|
+
visited,
|
|
901
|
+
records,
|
|
902
|
+
)?;
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
if record_matches_query(record, chunk_dims, chunk_bounds, link_name_hash) {
|
|
907
|
+
records.push(record.clone());
|
|
908
|
+
}
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
let final_child_index = node_records.len();
|
|
912
|
+
let final_child_addr = child_addresses[final_child_index];
|
|
913
|
+
if !Cursor::is_undefined_offset(final_child_addr, offset_size)
|
|
914
|
+
&& child_may_match_link_name_hash(&node_records, final_child_index, link_name_hash)
|
|
915
|
+
{
|
|
916
|
+
let child_nrec = child_nrecords[final_child_index];
|
|
917
|
+
if child_depth == 0 {
|
|
918
|
+
enter_btree_v2_node(visited, final_child_addr)?;
|
|
919
|
+
let mut child_cursor = Cursor::new(data);
|
|
920
|
+
child_cursor.set_position(final_child_addr);
|
|
921
|
+
parse_leaf_node(
|
|
922
|
+
&mut child_cursor,
|
|
923
|
+
header,
|
|
924
|
+
offset_size,
|
|
925
|
+
length_size,
|
|
926
|
+
ndims,
|
|
927
|
+
chunk_dims,
|
|
928
|
+
chunk_bounds,
|
|
929
|
+
link_name_hash,
|
|
930
|
+
child_nrec,
|
|
931
|
+
heap_id_len,
|
|
932
|
+
records,
|
|
933
|
+
)?;
|
|
934
|
+
} else {
|
|
935
|
+
parse_internal_node(
|
|
936
|
+
data,
|
|
937
|
+
final_child_addr,
|
|
938
|
+
header,
|
|
939
|
+
offset_size,
|
|
940
|
+
length_size,
|
|
941
|
+
ndims,
|
|
942
|
+
chunk_dims,
|
|
943
|
+
chunk_bounds,
|
|
944
|
+
link_name_hash,
|
|
945
|
+
child_nrec,
|
|
946
|
+
child_depth,
|
|
947
|
+
heap_id_len,
|
|
948
|
+
visited,
|
|
949
|
+
records,
|
|
950
|
+
)?;
|
|
951
|
+
}
|
|
952
|
+
}
|
|
953
|
+
|
|
954
|
+
Ok(())
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
// ---------------------------------------------------------------------------
|
|
958
|
+
// Public traversal
|
|
959
|
+
// ---------------------------------------------------------------------------
|
|
960
|
+
|
|
961
|
+
/// Collect all records from a B-tree v2 by traversing from the root.
|
|
962
|
+
///
|
|
963
|
+
/// `header` must be a previously parsed `BTreeV2Header`. `data` is the full
|
|
964
|
+
/// file buffer. `ndims` is needed for chunk index record types (10 and 11).
|
|
965
|
+
pub fn collect_btree_v2_records(
|
|
966
|
+
data: &[u8],
|
|
967
|
+
header: &BTreeV2Header,
|
|
968
|
+
offset_size: u8,
|
|
969
|
+
length_size: u8,
|
|
970
|
+
ndims: Option<u32>,
|
|
971
|
+
chunk_dims: &[u32],
|
|
972
|
+
chunk_bounds: Option<(&[u64], &[u64])>,
|
|
973
|
+
) -> Result<Vec<BTreeV2Record>> {
|
|
974
|
+
collect_btree_v2_records_with_link_hash(
|
|
975
|
+
data,
|
|
976
|
+
header,
|
|
977
|
+
offset_size,
|
|
978
|
+
length_size,
|
|
979
|
+
ndims,
|
|
980
|
+
chunk_dims,
|
|
981
|
+
chunk_bounds,
|
|
982
|
+
None,
|
|
983
|
+
)
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
#[allow(clippy::too_many_arguments)]
|
|
987
|
+
fn collect_btree_v2_records_with_link_hash(
|
|
988
|
+
data: &[u8],
|
|
989
|
+
header: &BTreeV2Header,
|
|
990
|
+
offset_size: u8,
|
|
991
|
+
length_size: u8,
|
|
992
|
+
ndims: Option<u32>,
|
|
993
|
+
chunk_dims: &[u32],
|
|
994
|
+
chunk_bounds: Option<(&[u64], &[u64])>,
|
|
995
|
+
link_name_hash: Option<u32>,
|
|
996
|
+
) -> Result<Vec<BTreeV2Record>> {
|
|
997
|
+
if Cursor::is_undefined_offset(header.root_node_address, offset_size) {
|
|
998
|
+
return Ok(Vec::new());
|
|
999
|
+
}
|
|
1000
|
+
|
|
1001
|
+
if header.total_records == 0 || header.num_records_in_root == 0 {
|
|
1002
|
+
return Ok(Vec::new());
|
|
1003
|
+
}
|
|
1004
|
+
validate_btree_v2_depth(header.depth)?;
|
|
1005
|
+
|
|
1006
|
+
// Determine heap_id_len from the record_size and btree_type.
|
|
1007
|
+
let heap_id_len = compute_heap_id_len(header);
|
|
1008
|
+
|
|
1009
|
+
let mut records = Vec::new();
|
|
1010
|
+
let mut visited = HashSet::new();
|
|
1011
|
+
|
|
1012
|
+
if header.depth == 0 {
|
|
1013
|
+
// Root is a leaf node.
|
|
1014
|
+
enter_btree_v2_node(&mut visited, header.root_node_address)?;
|
|
1015
|
+
let mut cursor = Cursor::new(data);
|
|
1016
|
+
cursor.set_position(header.root_node_address);
|
|
1017
|
+
parse_leaf_node(
|
|
1018
|
+
&mut cursor,
|
|
1019
|
+
header,
|
|
1020
|
+
offset_size,
|
|
1021
|
+
length_size,
|
|
1022
|
+
ndims,
|
|
1023
|
+
chunk_dims,
|
|
1024
|
+
chunk_bounds,
|
|
1025
|
+
link_name_hash,
|
|
1026
|
+
header.num_records_in_root,
|
|
1027
|
+
heap_id_len,
|
|
1028
|
+
&mut records,
|
|
1029
|
+
)?;
|
|
1030
|
+
} else {
|
|
1031
|
+
// Root is an internal node.
|
|
1032
|
+
parse_internal_node(
|
|
1033
|
+
data,
|
|
1034
|
+
header.root_node_address,
|
|
1035
|
+
header,
|
|
1036
|
+
offset_size,
|
|
1037
|
+
length_size,
|
|
1038
|
+
ndims,
|
|
1039
|
+
chunk_dims,
|
|
1040
|
+
chunk_bounds,
|
|
1041
|
+
link_name_hash,
|
|
1042
|
+
header.num_records_in_root,
|
|
1043
|
+
header.depth,
|
|
1044
|
+
heap_id_len,
|
|
1045
|
+
&mut visited,
|
|
1046
|
+
&mut records,
|
|
1047
|
+
)?;
|
|
1048
|
+
}
|
|
1049
|
+
|
|
1050
|
+
Ok(records)
|
|
1051
|
+
}
|
|
1052
|
+
|
|
1053
|
+
/// Collect link-name records whose stored hash matches `target_hash`.
|
|
1054
|
+
#[cfg(test)]
|
|
1055
|
+
pub(crate) fn collect_btree_v2_link_name_hash_records(
|
|
1056
|
+
data: &[u8],
|
|
1057
|
+
header: &BTreeV2Header,
|
|
1058
|
+
offset_size: u8,
|
|
1059
|
+
length_size: u8,
|
|
1060
|
+
target_hash: u32,
|
|
1061
|
+
) -> Result<Vec<BTreeV2Record>> {
|
|
1062
|
+
collect_btree_v2_records_with_link_hash(
|
|
1063
|
+
data,
|
|
1064
|
+
header,
|
|
1065
|
+
offset_size,
|
|
1066
|
+
length_size,
|
|
1067
|
+
None,
|
|
1068
|
+
&[],
|
|
1069
|
+
None,
|
|
1070
|
+
Some(target_hash),
|
|
1071
|
+
)
|
|
1072
|
+
}
|
|
1073
|
+
|
|
1074
|
+
/// Collect all records from a B-tree v2 using random-access storage.
|
|
1075
|
+
pub fn collect_btree_v2_records_storage(
|
|
1076
|
+
storage: &dyn Storage,
|
|
1077
|
+
header: &BTreeV2Header,
|
|
1078
|
+
offset_size: u8,
|
|
1079
|
+
length_size: u8,
|
|
1080
|
+
ndims: Option<u32>,
|
|
1081
|
+
chunk_dims: &[u32],
|
|
1082
|
+
chunk_bounds: Option<(&[u64], &[u64])>,
|
|
1083
|
+
) -> Result<Vec<BTreeV2Record>> {
|
|
1084
|
+
collect_btree_v2_records_storage_with_link_hash(
|
|
1085
|
+
storage,
|
|
1086
|
+
header,
|
|
1087
|
+
offset_size,
|
|
1088
|
+
length_size,
|
|
1089
|
+
ndims,
|
|
1090
|
+
chunk_dims,
|
|
1091
|
+
chunk_bounds,
|
|
1092
|
+
None,
|
|
1093
|
+
)
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
#[allow(clippy::too_many_arguments)]
|
|
1097
|
+
fn collect_btree_v2_records_storage_with_link_hash(
|
|
1098
|
+
storage: &dyn Storage,
|
|
1099
|
+
header: &BTreeV2Header,
|
|
1100
|
+
offset_size: u8,
|
|
1101
|
+
length_size: u8,
|
|
1102
|
+
ndims: Option<u32>,
|
|
1103
|
+
chunk_dims: &[u32],
|
|
1104
|
+
chunk_bounds: Option<(&[u64], &[u64])>,
|
|
1105
|
+
link_name_hash: Option<u32>,
|
|
1106
|
+
) -> Result<Vec<BTreeV2Record>> {
|
|
1107
|
+
if Cursor::is_undefined_offset(header.root_node_address, offset_size) {
|
|
1108
|
+
return Ok(Vec::new());
|
|
1109
|
+
}
|
|
1110
|
+
|
|
1111
|
+
if header.total_records == 0 || header.num_records_in_root == 0 {
|
|
1112
|
+
return Ok(Vec::new());
|
|
1113
|
+
}
|
|
1114
|
+
validate_btree_v2_depth(header.depth)?;
|
|
1115
|
+
|
|
1116
|
+
let heap_id_len = compute_heap_id_len(header);
|
|
1117
|
+
let mut records = Vec::new();
|
|
1118
|
+
let mut visited = HashSet::new();
|
|
1119
|
+
|
|
1120
|
+
if header.depth == 0 {
|
|
1121
|
+
enter_btree_v2_node(&mut visited, header.root_node_address)?;
|
|
1122
|
+
parse_leaf_node_storage(
|
|
1123
|
+
storage,
|
|
1124
|
+
header.root_node_address,
|
|
1125
|
+
header,
|
|
1126
|
+
offset_size,
|
|
1127
|
+
length_size,
|
|
1128
|
+
ndims,
|
|
1129
|
+
chunk_dims,
|
|
1130
|
+
chunk_bounds,
|
|
1131
|
+
link_name_hash,
|
|
1132
|
+
header.num_records_in_root,
|
|
1133
|
+
heap_id_len,
|
|
1134
|
+
&mut records,
|
|
1135
|
+
)?;
|
|
1136
|
+
} else {
|
|
1137
|
+
parse_internal_node_storage(
|
|
1138
|
+
storage,
|
|
1139
|
+
header.root_node_address,
|
|
1140
|
+
header,
|
|
1141
|
+
offset_size,
|
|
1142
|
+
length_size,
|
|
1143
|
+
ndims,
|
|
1144
|
+
chunk_dims,
|
|
1145
|
+
chunk_bounds,
|
|
1146
|
+
link_name_hash,
|
|
1147
|
+
header.num_records_in_root,
|
|
1148
|
+
header.depth,
|
|
1149
|
+
heap_id_len,
|
|
1150
|
+
&mut visited,
|
|
1151
|
+
&mut records,
|
|
1152
|
+
)?;
|
|
1153
|
+
}
|
|
1154
|
+
|
|
1155
|
+
Ok(records)
|
|
1156
|
+
}
|
|
1157
|
+
|
|
1158
|
+
/// Collect link-name records whose stored hash matches `target_hash`.
|
|
1159
|
+
pub(crate) fn collect_btree_v2_link_name_hash_records_storage(
|
|
1160
|
+
storage: &dyn Storage,
|
|
1161
|
+
header: &BTreeV2Header,
|
|
1162
|
+
offset_size: u8,
|
|
1163
|
+
length_size: u8,
|
|
1164
|
+
target_hash: u32,
|
|
1165
|
+
) -> Result<Vec<BTreeV2Record>> {
|
|
1166
|
+
collect_btree_v2_records_storage_with_link_hash(
|
|
1167
|
+
storage,
|
|
1168
|
+
header,
|
|
1169
|
+
offset_size,
|
|
1170
|
+
length_size,
|
|
1171
|
+
None,
|
|
1172
|
+
&[],
|
|
1173
|
+
None,
|
|
1174
|
+
Some(target_hash),
|
|
1175
|
+
)
|
|
1176
|
+
}
|
|
1177
|
+
|
|
1178
|
+
#[allow(clippy::too_many_arguments)]
|
|
1179
|
+
fn parse_leaf_node_storage(
|
|
1180
|
+
storage: &dyn Storage,
|
|
1181
|
+
address: u64,
|
|
1182
|
+
header: &BTreeV2Header,
|
|
1183
|
+
offset_size: u8,
|
|
1184
|
+
length_size: u8,
|
|
1185
|
+
ndims: Option<u32>,
|
|
1186
|
+
chunk_dims: &[u32],
|
|
1187
|
+
chunk_bounds: Option<(&[u64], &[u64])>,
|
|
1188
|
+
link_name_hash: Option<u32>,
|
|
1189
|
+
num_records: u16,
|
|
1190
|
+
heap_id_len: usize,
|
|
1191
|
+
records: &mut Vec<BTreeV2Record>,
|
|
1192
|
+
) -> Result<()> {
|
|
1193
|
+
let node_len = usize::try_from(header.node_size).map_err(|_| {
|
|
1194
|
+
Error::InvalidData("B-tree v2 node size exceeds platform usize capacity".into())
|
|
1195
|
+
})?;
|
|
1196
|
+
let node_bytes = storage.read_range(address, node_len)?;
|
|
1197
|
+
let mut cursor = Cursor::new(node_bytes.as_ref());
|
|
1198
|
+
parse_leaf_node(
|
|
1199
|
+
&mut cursor,
|
|
1200
|
+
header,
|
|
1201
|
+
offset_size,
|
|
1202
|
+
length_size,
|
|
1203
|
+
ndims,
|
|
1204
|
+
chunk_dims,
|
|
1205
|
+
chunk_bounds,
|
|
1206
|
+
link_name_hash,
|
|
1207
|
+
num_records,
|
|
1208
|
+
heap_id_len,
|
|
1209
|
+
records,
|
|
1210
|
+
)
|
|
1211
|
+
}
|
|
1212
|
+
|
|
1213
|
+
#[allow(clippy::too_many_arguments)]
|
|
1214
|
+
fn parse_internal_node_storage(
|
|
1215
|
+
storage: &dyn Storage,
|
|
1216
|
+
address: u64,
|
|
1217
|
+
header: &BTreeV2Header,
|
|
1218
|
+
offset_size: u8,
|
|
1219
|
+
length_size: u8,
|
|
1220
|
+
ndims: Option<u32>,
|
|
1221
|
+
chunk_dims: &[u32],
|
|
1222
|
+
chunk_bounds: Option<(&[u64], &[u64])>,
|
|
1223
|
+
link_name_hash: Option<u32>,
|
|
1224
|
+
num_records: u16,
|
|
1225
|
+
depth: u16,
|
|
1226
|
+
heap_id_len: usize,
|
|
1227
|
+
visited: &mut HashSet<u64>,
|
|
1228
|
+
records: &mut Vec<BTreeV2Record>,
|
|
1229
|
+
) -> Result<()> {
|
|
1230
|
+
if depth == 0 {
|
|
1231
|
+
return Err(Error::InvalidData(
|
|
1232
|
+
"B-tree v2 internal node traversal reached depth zero".into(),
|
|
1233
|
+
));
|
|
1234
|
+
}
|
|
1235
|
+
enter_btree_v2_node(visited, address)?;
|
|
1236
|
+
|
|
1237
|
+
let node_len = usize::try_from(header.node_size).map_err(|_| {
|
|
1238
|
+
Error::InvalidData("B-tree v2 node size exceeds platform usize capacity".into())
|
|
1239
|
+
})?;
|
|
1240
|
+
let node_bytes = storage.read_range(address, node_len)?;
|
|
1241
|
+
let mut cursor = Cursor::new(node_bytes.as_ref());
|
|
1242
|
+
let start = cursor.position();
|
|
1243
|
+
|
|
1244
|
+
let sig = cursor.read_bytes(4)?;
|
|
1245
|
+
if sig != BTIN_SIGNATURE {
|
|
1246
|
+
return Err(Error::InvalidBTreeV2Signature {
|
|
1247
|
+
context: "internal node",
|
|
1248
|
+
});
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
let version = cursor.read_u8()?;
|
|
1252
|
+
if version != 0 {
|
|
1253
|
+
return Err(Error::UnsupportedBTreeVersion(version));
|
|
1254
|
+
}
|
|
1255
|
+
|
|
1256
|
+
let node_type = cursor.read_u8()?;
|
|
1257
|
+
if node_type != header.btree_type {
|
|
1258
|
+
return Err(Error::InvalidData(format!(
|
|
1259
|
+
"B-tree v2 internal node type mismatch: header says {}, node says {}",
|
|
1260
|
+
header.btree_type, node_type
|
|
1261
|
+
)));
|
|
1262
|
+
}
|
|
1263
|
+
|
|
1264
|
+
let max_child_records = if depth == 1 {
|
|
1265
|
+
max_leaf_records(header.node_size, header.record_size)
|
|
1266
|
+
} else {
|
|
1267
|
+
let leaf_max = max_leaf_records(header.node_size, header.record_size);
|
|
1268
|
+
let mut prev_max = leaf_max;
|
|
1269
|
+
for _ in 1..depth {
|
|
1270
|
+
prev_max =
|
|
1271
|
+
max_internal_records(header.node_size, header.record_size, offset_size, prev_max);
|
|
1272
|
+
}
|
|
1273
|
+
prev_max
|
|
1274
|
+
};
|
|
1275
|
+
let nrec_bytes = num_records_size(max_child_records);
|
|
1276
|
+
|
|
1277
|
+
let mut node_records = Vec::with_capacity(num_records as usize);
|
|
1278
|
+
for _ in 0..num_records {
|
|
1279
|
+
let record = parse_record(
|
|
1280
|
+
&mut cursor,
|
|
1281
|
+
header.btree_type,
|
|
1282
|
+
header.record_size,
|
|
1283
|
+
offset_size,
|
|
1284
|
+
length_size,
|
|
1285
|
+
ndims,
|
|
1286
|
+
chunk_dims,
|
|
1287
|
+
heap_id_len,
|
|
1288
|
+
)?;
|
|
1289
|
+
node_records.push(record);
|
|
1290
|
+
}
|
|
1291
|
+
|
|
1292
|
+
let num_children = usize::from(num_records) + 1;
|
|
1293
|
+
let has_total_records = depth > 1;
|
|
1294
|
+
let total_nrec_bytes = if has_total_records {
|
|
1295
|
+
usize::from(length_size)
|
|
1296
|
+
} else {
|
|
1297
|
+
0
|
|
1298
|
+
};
|
|
1299
|
+
|
|
1300
|
+
let mut child_addresses = Vec::with_capacity(num_children);
|
|
1301
|
+
let mut child_nrecords = Vec::with_capacity(num_children);
|
|
1302
|
+
for _ in 0..num_children {
|
|
1303
|
+
child_addresses.push(cursor.read_offset(offset_size)?);
|
|
1304
|
+
child_nrecords.push(cursor.read_uvar(nrec_bytes)? as u16);
|
|
1305
|
+
if has_total_records {
|
|
1306
|
+
cursor.read_uvar(total_nrec_bytes)?;
|
|
1307
|
+
}
|
|
1308
|
+
}
|
|
1309
|
+
|
|
1310
|
+
let compact_end = cursor.position();
|
|
1311
|
+
verify_node_checksum(
|
|
1312
|
+
&mut cursor,
|
|
1313
|
+
start,
|
|
1314
|
+
header.node_size,
|
|
1315
|
+
compact_end,
|
|
1316
|
+
"internal node",
|
|
1317
|
+
)?;
|
|
1318
|
+
|
|
1319
|
+
let child_depth = depth - 1;
|
|
1320
|
+
for (i, record) in node_records.iter().enumerate() {
|
|
1321
|
+
let child_addr = child_addresses[i];
|
|
1322
|
+
if !Cursor::is_undefined_offset(child_addr, offset_size)
|
|
1323
|
+
&& child_may_match_link_name_hash(&node_records, i, link_name_hash)
|
|
1324
|
+
{
|
|
1325
|
+
let child_nrec = child_nrecords[i];
|
|
1326
|
+
if child_depth == 0 {
|
|
1327
|
+
enter_btree_v2_node(visited, child_addr)?;
|
|
1328
|
+
parse_leaf_node_storage(
|
|
1329
|
+
storage,
|
|
1330
|
+
child_addr,
|
|
1331
|
+
header,
|
|
1332
|
+
offset_size,
|
|
1333
|
+
length_size,
|
|
1334
|
+
ndims,
|
|
1335
|
+
chunk_dims,
|
|
1336
|
+
chunk_bounds,
|
|
1337
|
+
link_name_hash,
|
|
1338
|
+
child_nrec,
|
|
1339
|
+
heap_id_len,
|
|
1340
|
+
records,
|
|
1341
|
+
)?;
|
|
1342
|
+
} else {
|
|
1343
|
+
parse_internal_node_storage(
|
|
1344
|
+
storage,
|
|
1345
|
+
child_addr,
|
|
1346
|
+
header,
|
|
1347
|
+
offset_size,
|
|
1348
|
+
length_size,
|
|
1349
|
+
ndims,
|
|
1350
|
+
chunk_dims,
|
|
1351
|
+
chunk_bounds,
|
|
1352
|
+
link_name_hash,
|
|
1353
|
+
child_nrec,
|
|
1354
|
+
child_depth,
|
|
1355
|
+
heap_id_len,
|
|
1356
|
+
visited,
|
|
1357
|
+
records,
|
|
1358
|
+
)?;
|
|
1359
|
+
}
|
|
1360
|
+
}
|
|
1361
|
+
|
|
1362
|
+
if record_matches_query(record, chunk_dims, chunk_bounds, link_name_hash) {
|
|
1363
|
+
records.push(record.clone());
|
|
1364
|
+
}
|
|
1365
|
+
}
|
|
1366
|
+
|
|
1367
|
+
let final_child_index = node_records.len();
|
|
1368
|
+
let final_child_addr = child_addresses[final_child_index];
|
|
1369
|
+
if !Cursor::is_undefined_offset(final_child_addr, offset_size)
|
|
1370
|
+
&& child_may_match_link_name_hash(&node_records, final_child_index, link_name_hash)
|
|
1371
|
+
{
|
|
1372
|
+
let child_nrec = child_nrecords[final_child_index];
|
|
1373
|
+
if child_depth == 0 {
|
|
1374
|
+
enter_btree_v2_node(visited, final_child_addr)?;
|
|
1375
|
+
parse_leaf_node_storage(
|
|
1376
|
+
storage,
|
|
1377
|
+
final_child_addr,
|
|
1378
|
+
header,
|
|
1379
|
+
offset_size,
|
|
1380
|
+
length_size,
|
|
1381
|
+
ndims,
|
|
1382
|
+
chunk_dims,
|
|
1383
|
+
chunk_bounds,
|
|
1384
|
+
link_name_hash,
|
|
1385
|
+
child_nrec,
|
|
1386
|
+
heap_id_len,
|
|
1387
|
+
records,
|
|
1388
|
+
)?;
|
|
1389
|
+
} else {
|
|
1390
|
+
parse_internal_node_storage(
|
|
1391
|
+
storage,
|
|
1392
|
+
final_child_addr,
|
|
1393
|
+
header,
|
|
1394
|
+
offset_size,
|
|
1395
|
+
length_size,
|
|
1396
|
+
ndims,
|
|
1397
|
+
chunk_dims,
|
|
1398
|
+
chunk_bounds,
|
|
1399
|
+
link_name_hash,
|
|
1400
|
+
child_nrec,
|
|
1401
|
+
child_depth,
|
|
1402
|
+
heap_id_len,
|
|
1403
|
+
visited,
|
|
1404
|
+
records,
|
|
1405
|
+
)?;
|
|
1406
|
+
}
|
|
1407
|
+
}
|
|
1408
|
+
|
|
1409
|
+
Ok(())
|
|
1410
|
+
}
|
|
1411
|
+
|
|
1412
|
+
/// Compute the heap ID length from the record size and tree type.
|
|
1413
|
+
///
|
|
1414
|
+
/// For link/attribute B-trees (types 5, 6, 8, 9), the heap ID occupies
|
|
1415
|
+
/// the bytes not used by the fixed fields. For chunk types (10, 11) or
|
|
1416
|
+
/// unknown types, return 0 (heap_id is not used).
|
|
1417
|
+
fn compute_heap_id_len(header: &BTreeV2Header) -> usize {
|
|
1418
|
+
let rs = header.record_size as usize;
|
|
1419
|
+
match header.btree_type {
|
|
1420
|
+
5 => rs.saturating_sub(4), // hash(4)
|
|
1421
|
+
6 => rs.saturating_sub(8), // order(8)
|
|
1422
|
+
8 => rs.saturating_sub(4 + 1 + 4), // hash(4) + flags(1) + creation_order(4)
|
|
1423
|
+
9 => rs.saturating_sub(1 + 4), // flags(1) + order(4)
|
|
1424
|
+
_ => 0,
|
|
1425
|
+
}
|
|
1426
|
+
}
|
|
1427
|
+
|
|
1428
|
+
#[cfg(test)]
|
|
1429
|
+
mod tests {
|
|
1430
|
+
use super::*;
|
|
1431
|
+
|
|
1432
|
+
/// Build a minimal BTHD with the given parameters.
|
|
1433
|
+
#[allow(clippy::too_many_arguments)]
|
|
1434
|
+
fn build_header(
|
|
1435
|
+
btree_type: u8,
|
|
1436
|
+
node_size: u32,
|
|
1437
|
+
record_size: u16,
|
|
1438
|
+
depth: u16,
|
|
1439
|
+
root_node_address: u64,
|
|
1440
|
+
num_records_in_root: u16,
|
|
1441
|
+
total_records: u64,
|
|
1442
|
+
offset_size: u8,
|
|
1443
|
+
length_size: u8,
|
|
1444
|
+
) -> Vec<u8> {
|
|
1445
|
+
let mut buf = Vec::new();
|
|
1446
|
+
buf.extend_from_slice(b"BTHD");
|
|
1447
|
+
buf.push(0); // version
|
|
1448
|
+
buf.push(btree_type);
|
|
1449
|
+
buf.extend_from_slice(&node_size.to_le_bytes());
|
|
1450
|
+
buf.extend_from_slice(&record_size.to_le_bytes());
|
|
1451
|
+
buf.extend_from_slice(&depth.to_le_bytes());
|
|
1452
|
+
buf.push(75); // split percent
|
|
1453
|
+
buf.push(40); // merge percent
|
|
1454
|
+
match offset_size {
|
|
1455
|
+
4 => buf.extend_from_slice(&(root_node_address as u32).to_le_bytes()),
|
|
1456
|
+
8 => buf.extend_from_slice(&root_node_address.to_le_bytes()),
|
|
1457
|
+
_ => panic!("unsupported"),
|
|
1458
|
+
}
|
|
1459
|
+
buf.extend_from_slice(&num_records_in_root.to_le_bytes());
|
|
1460
|
+
match length_size {
|
|
1461
|
+
4 => buf.extend_from_slice(&(total_records as u32).to_le_bytes()),
|
|
1462
|
+
8 => buf.extend_from_slice(&total_records.to_le_bytes()),
|
|
1463
|
+
_ => panic!("unsupported"),
|
|
1464
|
+
}
|
|
1465
|
+
// Compute and append checksum.
|
|
1466
|
+
let checksum = jenkins_lookup3(&buf);
|
|
1467
|
+
buf.extend_from_slice(&checksum.to_le_bytes());
|
|
1468
|
+
buf
|
|
1469
|
+
}
|
|
1470
|
+
|
|
1471
|
+
#[test]
|
|
1472
|
+
fn parse_header() {
|
|
1473
|
+
let data = build_header(5, 4096, 12, 0, 0x1000, 3, 3, 8, 8);
|
|
1474
|
+
let mut cursor = Cursor::new(&data);
|
|
1475
|
+
let hdr = BTreeV2Header::parse(&mut cursor, 8, 8).unwrap();
|
|
1476
|
+
|
|
1477
|
+
assert_eq!(hdr.btree_type, 5);
|
|
1478
|
+
assert_eq!(hdr.node_size, 4096);
|
|
1479
|
+
assert_eq!(hdr.record_size, 12);
|
|
1480
|
+
assert_eq!(hdr.depth, 0);
|
|
1481
|
+
assert_eq!(hdr.split_percent, 75);
|
|
1482
|
+
assert_eq!(hdr.merge_percent, 40);
|
|
1483
|
+
assert_eq!(hdr.root_node_address, 0x1000);
|
|
1484
|
+
assert_eq!(hdr.num_records_in_root, 3);
|
|
1485
|
+
assert_eq!(hdr.total_records, 3);
|
|
1486
|
+
}
|
|
1487
|
+
|
|
1488
|
+
#[test]
|
|
1489
|
+
fn bad_signature() {
|
|
1490
|
+
let mut data = build_header(5, 4096, 12, 0, 0x1000, 0, 0, 8, 8);
|
|
1491
|
+
data[0] = b'X';
|
|
1492
|
+
let mut cursor = Cursor::new(&data);
|
|
1493
|
+
assert!(matches!(
|
|
1494
|
+
BTreeV2Header::parse(&mut cursor, 8, 8),
|
|
1495
|
+
Err(Error::InvalidBTreeV2Signature { .. })
|
|
1496
|
+
));
|
|
1497
|
+
}
|
|
1498
|
+
|
|
1499
|
+
#[test]
|
|
1500
|
+
fn bad_checksum() {
|
|
1501
|
+
let mut data = build_header(5, 4096, 12, 0, 0x1000, 0, 0, 8, 8);
|
|
1502
|
+
// Corrupt a byte in the middle.
|
|
1503
|
+
data[6] = 0xFF;
|
|
1504
|
+
let mut cursor = Cursor::new(&data);
|
|
1505
|
+
assert!(matches!(
|
|
1506
|
+
BTreeV2Header::parse(&mut cursor, 8, 8),
|
|
1507
|
+
Err(Error::ChecksumMismatch { .. })
|
|
1508
|
+
));
|
|
1509
|
+
}
|
|
1510
|
+
|
|
1511
|
+
#[test]
|
|
1512
|
+
fn collect_empty_tree() {
|
|
1513
|
+
let header = BTreeV2Header {
|
|
1514
|
+
btree_type: 5,
|
|
1515
|
+
node_size: 4096,
|
|
1516
|
+
record_size: 12,
|
|
1517
|
+
depth: 0,
|
|
1518
|
+
split_percent: 75,
|
|
1519
|
+
merge_percent: 40,
|
|
1520
|
+
root_node_address: u64::MAX,
|
|
1521
|
+
num_records_in_root: 0,
|
|
1522
|
+
total_records: 0,
|
|
1523
|
+
};
|
|
1524
|
+
let data = vec![0u8; 100];
|
|
1525
|
+
let records = collect_btree_v2_records(&data, &header, 8, 8, None, &[], None).unwrap();
|
|
1526
|
+
assert!(records.is_empty());
|
|
1527
|
+
}
|
|
1528
|
+
|
|
1529
|
+
#[test]
|
|
1530
|
+
fn heap_id_len_uses_record_type_sizes() {
|
|
1531
|
+
// Type 5: record_size - 4
|
|
1532
|
+
let h5 = BTreeV2Header {
|
|
1533
|
+
btree_type: 5,
|
|
1534
|
+
record_size: 12,
|
|
1535
|
+
node_size: 0,
|
|
1536
|
+
depth: 0,
|
|
1537
|
+
split_percent: 0,
|
|
1538
|
+
merge_percent: 0,
|
|
1539
|
+
root_node_address: 0,
|
|
1540
|
+
num_records_in_root: 0,
|
|
1541
|
+
total_records: 0,
|
|
1542
|
+
};
|
|
1543
|
+
assert_eq!(compute_heap_id_len(&h5), 8);
|
|
1544
|
+
|
|
1545
|
+
// Type 8: record_size - 9
|
|
1546
|
+
let h8 = BTreeV2Header {
|
|
1547
|
+
btree_type: 8,
|
|
1548
|
+
record_size: 17,
|
|
1549
|
+
..h5
|
|
1550
|
+
};
|
|
1551
|
+
assert_eq!(compute_heap_id_len(&h8), 8);
|
|
1552
|
+
|
|
1553
|
+
// Type 9: record_size - flags(1) - order(4)
|
|
1554
|
+
let h9 = BTreeV2Header {
|
|
1555
|
+
btree_type: 9,
|
|
1556
|
+
record_size: 13,
|
|
1557
|
+
..h5
|
|
1558
|
+
};
|
|
1559
|
+
assert_eq!(compute_heap_id_len(&h9), 8);
|
|
1560
|
+
}
|
|
1561
|
+
|
|
1562
|
+
#[test]
|
|
1563
|
+
fn parse_attribute_name_record_uses_on_disk_field_order() {
|
|
1564
|
+
let heap_id = [0x00, 0xbe, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x00];
|
|
1565
|
+
let mut data = heap_id.to_vec();
|
|
1566
|
+
data.push(0x02);
|
|
1567
|
+
data.extend_from_slice(&6u32.to_le_bytes());
|
|
1568
|
+
data.extend_from_slice(&0x0396_dda5u32.to_le_bytes());
|
|
1569
|
+
let mut cursor = Cursor::new(&data);
|
|
1570
|
+
|
|
1571
|
+
let record = parse_record(&mut cursor, 8, data.len() as u16, 8, 8, None, &[], 8).unwrap();
|
|
1572
|
+
match record {
|
|
1573
|
+
BTreeV2Record::AttributeNameHash {
|
|
1574
|
+
hash,
|
|
1575
|
+
flags,
|
|
1576
|
+
creation_order,
|
|
1577
|
+
heap_id: parsed_heap_id,
|
|
1578
|
+
} => {
|
|
1579
|
+
assert_eq!(parsed_heap_id, heap_id);
|
|
1580
|
+
assert_eq!(flags, 0x02);
|
|
1581
|
+
assert_eq!(creation_order, 6);
|
|
1582
|
+
assert_eq!(hash, 0x0396_dda5);
|
|
1583
|
+
}
|
|
1584
|
+
other => panic!("expected attribute-name record, got {other:?}"),
|
|
1585
|
+
}
|
|
1586
|
+
}
|
|
1587
|
+
|
|
1588
|
+
#[test]
|
|
1589
|
+
fn parse_attribute_creation_order_record_uses_on_disk_field_order() {
|
|
1590
|
+
let heap_id = [0x00, 0x16, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x00];
|
|
1591
|
+
let mut data = heap_id.to_vec();
|
|
1592
|
+
data.push(0x02);
|
|
1593
|
+
data.extend_from_slice(&7u32.to_le_bytes());
|
|
1594
|
+
let mut cursor = Cursor::new(&data);
|
|
1595
|
+
|
|
1596
|
+
let record = parse_record(&mut cursor, 9, data.len() as u16, 8, 8, None, &[], 8).unwrap();
|
|
1597
|
+
match record {
|
|
1598
|
+
BTreeV2Record::AttributeCreationOrder {
|
|
1599
|
+
order,
|
|
1600
|
+
heap_id: parsed_heap_id,
|
|
1601
|
+
} => {
|
|
1602
|
+
assert_eq!(parsed_heap_id, heap_id);
|
|
1603
|
+
assert_eq!(order, 7);
|
|
1604
|
+
}
|
|
1605
|
+
other => panic!("expected attribute-creation-order record, got {other:?}"),
|
|
1606
|
+
}
|
|
1607
|
+
}
|
|
1608
|
+
|
|
1609
|
+
#[test]
|
|
1610
|
+
fn max_leaf_records_accounts_for_node_overhead() {
|
|
1611
|
+
// node_size=4096, record_size=12, overhead=10
|
|
1612
|
+
// => (4096 - 10) / 12 = 340
|
|
1613
|
+
assert_eq!(max_leaf_records(4096, 12), 340);
|
|
1614
|
+
}
|
|
1615
|
+
|
|
1616
|
+
#[test]
|
|
1617
|
+
fn record_count_size_scales_at_integer_boundaries() {
|
|
1618
|
+
assert_eq!(num_records_size(0), 1);
|
|
1619
|
+
assert_eq!(num_records_size(255), 1);
|
|
1620
|
+
assert_eq!(num_records_size(256), 2);
|
|
1621
|
+
assert_eq!(num_records_size(65535), 2);
|
|
1622
|
+
assert_eq!(num_records_size(65536), 4);
|
|
1623
|
+
}
|
|
1624
|
+
|
|
1625
|
+
#[test]
|
|
1626
|
+
fn parse_huge_indirect_record() {
|
|
1627
|
+
let mut data = Vec::new();
|
|
1628
|
+
data.extend_from_slice(&0x1234u64.to_le_bytes());
|
|
1629
|
+
data.extend_from_slice(&99u64.to_le_bytes());
|
|
1630
|
+
data.extend_from_slice(&7u64.to_le_bytes());
|
|
1631
|
+
let mut cursor = Cursor::new(&data);
|
|
1632
|
+
|
|
1633
|
+
let record = parse_record(&mut cursor, 1, data.len() as u16, 8, 8, None, &[], 0).unwrap();
|
|
1634
|
+
match record {
|
|
1635
|
+
BTreeV2Record::HugeIndirectNonFiltered {
|
|
1636
|
+
address,
|
|
1637
|
+
length,
|
|
1638
|
+
object_id,
|
|
1639
|
+
} => {
|
|
1640
|
+
assert_eq!(address, 0x1234);
|
|
1641
|
+
assert_eq!(length, 99);
|
|
1642
|
+
assert_eq!(object_id, 7);
|
|
1643
|
+
}
|
|
1644
|
+
other => panic!("expected huge record, got {:?}", other),
|
|
1645
|
+
}
|
|
1646
|
+
}
|
|
1647
|
+
|
|
1648
|
+
#[test]
|
|
1649
|
+
fn parse_shared_heap_record() {
|
|
1650
|
+
let mut data = Vec::new();
|
|
1651
|
+
data.push(0);
|
|
1652
|
+
data.extend_from_slice(&[0, 0, 0]);
|
|
1653
|
+
data.extend_from_slice(&0xAABB_CCDDu32.to_le_bytes());
|
|
1654
|
+
data.extend_from_slice(&3u32.to_le_bytes());
|
|
1655
|
+
data.extend_from_slice(&[1, 2, 3, 4, 5, 6, 7, 8]);
|
|
1656
|
+
let mut cursor = Cursor::new(&data);
|
|
1657
|
+
|
|
1658
|
+
let record = parse_record(&mut cursor, 7, data.len() as u16, 8, 8, None, &[], 0).unwrap();
|
|
1659
|
+
match record {
|
|
1660
|
+
BTreeV2Record::SharedMessageHeap {
|
|
1661
|
+
hash,
|
|
1662
|
+
reference_count,
|
|
1663
|
+
heap_id,
|
|
1664
|
+
} => {
|
|
1665
|
+
assert_eq!(hash, 0xAABB_CCDD);
|
|
1666
|
+
assert_eq!(reference_count, 3);
|
|
1667
|
+
assert_eq!(heap_id, vec![1, 2, 3, 4, 5, 6, 7, 8]);
|
|
1668
|
+
}
|
|
1669
|
+
other => panic!("expected shared heap record, got {:?}", other),
|
|
1670
|
+
}
|
|
1671
|
+
}
|
|
1672
|
+
|
|
1673
|
+
#[test]
|
|
1674
|
+
fn parse_record_rejects_known_record_that_exceeds_record_size() {
|
|
1675
|
+
let mut data = Vec::new();
|
|
1676
|
+
data.extend_from_slice(&0x1234u64.to_le_bytes());
|
|
1677
|
+
data.extend_from_slice(&99u64.to_le_bytes());
|
|
1678
|
+
data.extend_from_slice(&7u64.to_le_bytes());
|
|
1679
|
+
let mut cursor = Cursor::new(&data);
|
|
1680
|
+
|
|
1681
|
+
let err = parse_record(&mut cursor, 1, 16, 8, 8, None, &[], 0).unwrap_err();
|
|
1682
|
+
assert!(
|
|
1683
|
+
matches!(err, Error::InvalidData(message) if message.contains("consumed 24 bytes but record size is 16"))
|
|
1684
|
+
);
|
|
1685
|
+
}
|
|
1686
|
+
|
|
1687
|
+
#[test]
|
|
1688
|
+
fn parse_chunk_record_rejects_size_shorter_than_address() {
|
|
1689
|
+
let mut data = Vec::new();
|
|
1690
|
+
data.extend_from_slice(&0x1234u64.to_le_bytes());
|
|
1691
|
+
let mut cursor = Cursor::new(&data);
|
|
1692
|
+
|
|
1693
|
+
let err = parse_record(&mut cursor, 10, 4, 8, 8, None, &[], 0).unwrap_err();
|
|
1694
|
+
assert!(
|
|
1695
|
+
matches!(err, Error::InvalidData(message) if message.contains("shorter than its address"))
|
|
1696
|
+
);
|
|
1697
|
+
assert_eq!(cursor.position(), 0);
|
|
1698
|
+
}
|
|
1699
|
+
|
|
1700
|
+
#[test]
|
|
1701
|
+
fn parse_filtered_chunk_record_rejects_size_shorter_than_fixed_fields() {
|
|
1702
|
+
let mut data = Vec::new();
|
|
1703
|
+
data.extend_from_slice(&0x1234u64.to_le_bytes());
|
|
1704
|
+
data.extend_from_slice(&99u64.to_le_bytes());
|
|
1705
|
+
data.extend_from_slice(&0xAu32.to_le_bytes());
|
|
1706
|
+
let mut cursor = Cursor::new(&data);
|
|
1707
|
+
|
|
1708
|
+
// One dimension: fixed fields are address(8) + chunk_size(>=1) +
|
|
1709
|
+
// filter_mask(4) + offset(8) = 21+, so a 16-byte record is too short.
|
|
1710
|
+
let err = parse_record(&mut cursor, 11, 16, 8, 8, Some(1), &[], 0).unwrap_err();
|
|
1711
|
+
assert!(
|
|
1712
|
+
matches!(err, Error::InvalidData(message) if message.contains("shorter than its fixed fields"))
|
|
1713
|
+
);
|
|
1714
|
+
assert_eq!(cursor.position(), 0);
|
|
1715
|
+
}
|
|
1716
|
+
|
|
1717
|
+
#[test]
|
|
1718
|
+
fn parse_chunk_record_scales_offsets_by_chunk_dimensions() {
|
|
1719
|
+
let mut data = Vec::new();
|
|
1720
|
+
data.extend_from_slice(&0x1234u64.to_le_bytes());
|
|
1721
|
+
data.extend_from_slice(&2u64.to_le_bytes());
|
|
1722
|
+
data.extend_from_slice(&1u64.to_le_bytes());
|
|
1723
|
+
let mut cursor = Cursor::new(&data);
|
|
1724
|
+
|
|
1725
|
+
let record = parse_record(
|
|
1726
|
+
&mut cursor,
|
|
1727
|
+
10,
|
|
1728
|
+
data.len() as u16,
|
|
1729
|
+
8,
|
|
1730
|
+
8,
|
|
1731
|
+
Some(2),
|
|
1732
|
+
&[5, 7],
|
|
1733
|
+
0,
|
|
1734
|
+
)
|
|
1735
|
+
.unwrap();
|
|
1736
|
+
match record {
|
|
1737
|
+
BTreeV2Record::ChunkedNonFiltered { address, offsets } => {
|
|
1738
|
+
assert_eq!(address, 0x1234);
|
|
1739
|
+
assert_eq!(offsets, vec![10, 7]);
|
|
1740
|
+
}
|
|
1741
|
+
other => panic!("expected non-filtered chunk record, got {:?}", other),
|
|
1742
|
+
}
|
|
1743
|
+
}
|
|
1744
|
+
|
|
1745
|
+
#[test]
|
|
1746
|
+
fn parse_filtered_chunk_record_scales_offsets_by_chunk_dimensions() {
|
|
1747
|
+
let mut data = Vec::new();
|
|
1748
|
+
data.extend_from_slice(&0x1234u64.to_le_bytes());
|
|
1749
|
+
data.extend_from_slice(&99u64.to_le_bytes());
|
|
1750
|
+
data.extend_from_slice(&0xAu32.to_le_bytes());
|
|
1751
|
+
data.extend_from_slice(&3u64.to_le_bytes());
|
|
1752
|
+
data.extend_from_slice(&4u64.to_le_bytes());
|
|
1753
|
+
let mut cursor = Cursor::new(&data);
|
|
1754
|
+
|
|
1755
|
+
let record = parse_record(
|
|
1756
|
+
&mut cursor,
|
|
1757
|
+
11,
|
|
1758
|
+
data.len() as u16,
|
|
1759
|
+
8,
|
|
1760
|
+
8,
|
|
1761
|
+
Some(2),
|
|
1762
|
+
&[5, 7],
|
|
1763
|
+
0,
|
|
1764
|
+
)
|
|
1765
|
+
.unwrap();
|
|
1766
|
+
match record {
|
|
1767
|
+
BTreeV2Record::ChunkedFiltered {
|
|
1768
|
+
address,
|
|
1769
|
+
chunk_size,
|
|
1770
|
+
filter_mask,
|
|
1771
|
+
offsets,
|
|
1772
|
+
} => {
|
|
1773
|
+
assert_eq!(address, 0x1234);
|
|
1774
|
+
assert_eq!(chunk_size, 99);
|
|
1775
|
+
assert_eq!(filter_mask, 0xA);
|
|
1776
|
+
assert_eq!(offsets, vec![15, 28]);
|
|
1777
|
+
}
|
|
1778
|
+
other => panic!("expected filtered chunk record, got {:?}", other),
|
|
1779
|
+
}
|
|
1780
|
+
}
|
|
1781
|
+
|
|
1782
|
+
#[test]
|
|
1783
|
+
fn parse_leaf_with_type5_records() {
|
|
1784
|
+
// Build a leaf node with 2 type-5 records (link name hash).
|
|
1785
|
+
// record_size = 12 (hash=4 + heap_id=8)
|
|
1786
|
+
let record_size: u16 = 12;
|
|
1787
|
+
let node_size: u32 = 4096;
|
|
1788
|
+
|
|
1789
|
+
let header = BTreeV2Header {
|
|
1790
|
+
btree_type: 5,
|
|
1791
|
+
node_size,
|
|
1792
|
+
record_size,
|
|
1793
|
+
depth: 0,
|
|
1794
|
+
split_percent: 75,
|
|
1795
|
+
merge_percent: 40,
|
|
1796
|
+
root_node_address: 0, // will point to our leaf
|
|
1797
|
+
num_records_in_root: 2,
|
|
1798
|
+
total_records: 2,
|
|
1799
|
+
};
|
|
1800
|
+
|
|
1801
|
+
// Build the leaf node.
|
|
1802
|
+
let mut leaf = Vec::new();
|
|
1803
|
+
leaf.extend_from_slice(b"BTLF"); // signature
|
|
1804
|
+
leaf.push(0); // version
|
|
1805
|
+
leaf.push(5); // type
|
|
1806
|
+
|
|
1807
|
+
// Record 1: hash=0xAABBCCDD, heap_id=[1,2,3,4,5,6,7,8]
|
|
1808
|
+
leaf.extend_from_slice(&0xAABBCCDDu32.to_le_bytes());
|
|
1809
|
+
leaf.extend_from_slice(&[1, 2, 3, 4, 5, 6, 7, 8]);
|
|
1810
|
+
|
|
1811
|
+
// Record 2: hash=0x11223344, heap_id=[9,10,11,12,13,14,15,16]
|
|
1812
|
+
leaf.extend_from_slice(&0x11223344u32.to_le_bytes());
|
|
1813
|
+
leaf.extend_from_slice(&[9, 10, 11, 12, 13, 14, 15, 16]);
|
|
1814
|
+
|
|
1815
|
+
// The checksum covers the populated node payload. The node allocation
|
|
1816
|
+
// may be larger and is padded after the checksum.
|
|
1817
|
+
let checksum = jenkins_lookup3(&leaf);
|
|
1818
|
+
leaf.extend_from_slice(&checksum.to_le_bytes());
|
|
1819
|
+
leaf.resize(node_size as usize, 0);
|
|
1820
|
+
|
|
1821
|
+
let mut records = Vec::new();
|
|
1822
|
+
let mut cursor = Cursor::new(&leaf);
|
|
1823
|
+
parse_leaf_node(
|
|
1824
|
+
&mut cursor,
|
|
1825
|
+
&header,
|
|
1826
|
+
8,
|
|
1827
|
+
8,
|
|
1828
|
+
None,
|
|
1829
|
+
&[],
|
|
1830
|
+
None,
|
|
1831
|
+
None,
|
|
1832
|
+
2,
|
|
1833
|
+
8, // heap_id_len
|
|
1834
|
+
&mut records,
|
|
1835
|
+
)
|
|
1836
|
+
.unwrap();
|
|
1837
|
+
|
|
1838
|
+
assert_eq!(records.len(), 2);
|
|
1839
|
+
match &records[0] {
|
|
1840
|
+
BTreeV2Record::LinkNameHash { hash, heap_id } => {
|
|
1841
|
+
assert_eq!(*hash, 0xAABBCCDD);
|
|
1842
|
+
assert_eq!(heap_id, &[1, 2, 3, 4, 5, 6, 7, 8]);
|
|
1843
|
+
}
|
|
1844
|
+
_ => panic!("expected LinkNameHash"),
|
|
1845
|
+
}
|
|
1846
|
+
match &records[1] {
|
|
1847
|
+
BTreeV2Record::LinkNameHash { hash, heap_id } => {
|
|
1848
|
+
assert_eq!(*hash, 0x11223344);
|
|
1849
|
+
assert_eq!(heap_id, &[9, 10, 11, 12, 13, 14, 15, 16]);
|
|
1850
|
+
}
|
|
1851
|
+
_ => panic!("expected LinkNameHash"),
|
|
1852
|
+
}
|
|
1853
|
+
}
|
|
1854
|
+
|
|
1855
|
+
fn build_type5_record(hash: u32, heap_id: [u8; 8]) -> Vec<u8> {
|
|
1856
|
+
let mut record = Vec::new();
|
|
1857
|
+
record.extend_from_slice(&hash.to_le_bytes());
|
|
1858
|
+
record.extend_from_slice(&heap_id);
|
|
1859
|
+
record
|
|
1860
|
+
}
|
|
1861
|
+
|
|
1862
|
+
fn build_type5_leaf(records: &[(u32, [u8; 8])], node_size: usize) -> Vec<u8> {
|
|
1863
|
+
let mut leaf = Vec::new();
|
|
1864
|
+
leaf.extend_from_slice(b"BTLF");
|
|
1865
|
+
leaf.push(0);
|
|
1866
|
+
leaf.push(5);
|
|
1867
|
+
for &(hash, heap_id) in records {
|
|
1868
|
+
leaf.extend_from_slice(&build_type5_record(hash, heap_id));
|
|
1869
|
+
}
|
|
1870
|
+
let checksum = jenkins_lookup3(&leaf);
|
|
1871
|
+
leaf.extend_from_slice(&checksum.to_le_bytes());
|
|
1872
|
+
leaf.resize(node_size, 0);
|
|
1873
|
+
leaf
|
|
1874
|
+
}
|
|
1875
|
+
|
|
1876
|
+
fn build_type5_internal(
|
|
1877
|
+
records: &[(u32, [u8; 8])],
|
|
1878
|
+
children: &[(u64, u8)],
|
|
1879
|
+
node_size: usize,
|
|
1880
|
+
) -> Vec<u8> {
|
|
1881
|
+
assert_eq!(children.len(), records.len() + 1);
|
|
1882
|
+
|
|
1883
|
+
let mut node = Vec::new();
|
|
1884
|
+
node.extend_from_slice(b"BTIN");
|
|
1885
|
+
node.push(0);
|
|
1886
|
+
node.push(5);
|
|
1887
|
+
for &(hash, heap_id) in records {
|
|
1888
|
+
node.extend_from_slice(&build_type5_record(hash, heap_id));
|
|
1889
|
+
}
|
|
1890
|
+
for &(address, child_records) in children {
|
|
1891
|
+
node.extend_from_slice(&address.to_le_bytes());
|
|
1892
|
+
node.push(child_records);
|
|
1893
|
+
}
|
|
1894
|
+
let checksum = jenkins_lookup3(&node);
|
|
1895
|
+
node.extend_from_slice(&checksum.to_le_bytes());
|
|
1896
|
+
node.resize(node_size, 0);
|
|
1897
|
+
node
|
|
1898
|
+
}
|
|
1899
|
+
|
|
1900
|
+
fn link_hashes(records: &[BTreeV2Record]) -> Vec<u32> {
|
|
1901
|
+
records
|
|
1902
|
+
.iter()
|
|
1903
|
+
.map(|record| match record {
|
|
1904
|
+
BTreeV2Record::LinkNameHash { hash, .. } => *hash,
|
|
1905
|
+
other => panic!("expected LinkNameHash, got {other:?}"),
|
|
1906
|
+
})
|
|
1907
|
+
.collect()
|
|
1908
|
+
}
|
|
1909
|
+
|
|
1910
|
+
#[test]
|
|
1911
|
+
fn collect_depth_one_tree_includes_internal_records_in_order() {
|
|
1912
|
+
// Mirrors HDF5 H5B2__iterate_node: recurse into child[i], visit
|
|
1913
|
+
// record[i], then recurse into the final child. Internal-node records
|
|
1914
|
+
// are callback records, not just separators.
|
|
1915
|
+
let node_size = 128usize;
|
|
1916
|
+
let left_addr = node_size as u64;
|
|
1917
|
+
let right_addr = (node_size * 2) as u64;
|
|
1918
|
+
let header = BTreeV2Header {
|
|
1919
|
+
btree_type: 5,
|
|
1920
|
+
node_size: node_size as u32,
|
|
1921
|
+
record_size: 12,
|
|
1922
|
+
depth: 1,
|
|
1923
|
+
split_percent: 75,
|
|
1924
|
+
merge_percent: 40,
|
|
1925
|
+
root_node_address: 0,
|
|
1926
|
+
num_records_in_root: 1,
|
|
1927
|
+
total_records: 3,
|
|
1928
|
+
};
|
|
1929
|
+
|
|
1930
|
+
let root = build_type5_internal(
|
|
1931
|
+
&[(20, [20; 8])],
|
|
1932
|
+
&[(left_addr, 1), (right_addr, 1)],
|
|
1933
|
+
node_size,
|
|
1934
|
+
);
|
|
1935
|
+
let left_leaf = build_type5_leaf(&[(10, [10; 8])], node_size);
|
|
1936
|
+
let right_leaf = build_type5_leaf(&[(30, [30; 8])], node_size);
|
|
1937
|
+
|
|
1938
|
+
let mut data = vec![0; node_size * 3];
|
|
1939
|
+
data[0..node_size].copy_from_slice(&root);
|
|
1940
|
+
data[node_size..node_size * 2].copy_from_slice(&left_leaf);
|
|
1941
|
+
data[node_size * 2..node_size * 3].copy_from_slice(&right_leaf);
|
|
1942
|
+
|
|
1943
|
+
let records = collect_btree_v2_records(&data, &header, 8, 8, None, &[], None).unwrap();
|
|
1944
|
+
assert_eq!(link_hashes(&records), vec![10, 20, 30]);
|
|
1945
|
+
|
|
1946
|
+
let storage = crate::storage::BytesStorage::new(data);
|
|
1947
|
+
let records =
|
|
1948
|
+
collect_btree_v2_records_storage(&storage, &header, 8, 8, None, &[], None).unwrap();
|
|
1949
|
+
assert_eq!(link_hashes(&records), vec![10, 20, 30]);
|
|
1950
|
+
}
|
|
1951
|
+
|
|
1952
|
+
#[test]
|
|
1953
|
+
fn collect_rejects_depth_above_traversal_limit() {
|
|
1954
|
+
let header = BTreeV2Header {
|
|
1955
|
+
btree_type: 5,
|
|
1956
|
+
node_size: 128,
|
|
1957
|
+
record_size: 12,
|
|
1958
|
+
depth: MAX_BTREE_V2_DEPTH + 1,
|
|
1959
|
+
split_percent: 75,
|
|
1960
|
+
merge_percent: 40,
|
|
1961
|
+
root_node_address: 0,
|
|
1962
|
+
num_records_in_root: 1,
|
|
1963
|
+
total_records: 1,
|
|
1964
|
+
};
|
|
1965
|
+
let data = vec![0; 128];
|
|
1966
|
+
|
|
1967
|
+
let err = collect_btree_v2_records(&data, &header, 8, 8, None, &[], None).unwrap_err();
|
|
1968
|
+
assert!(
|
|
1969
|
+
matches!(err, Error::InvalidData(message) if message.contains("exceeds traversal limit"))
|
|
1970
|
+
);
|
|
1971
|
+
}
|
|
1972
|
+
|
|
1973
|
+
#[test]
|
|
1974
|
+
fn collect_rejects_revisited_leaf_node() {
|
|
1975
|
+
let node_size = 128usize;
|
|
1976
|
+
let leaf_addr = node_size as u64;
|
|
1977
|
+
let header = BTreeV2Header {
|
|
1978
|
+
btree_type: 5,
|
|
1979
|
+
node_size: node_size as u32,
|
|
1980
|
+
record_size: 12,
|
|
1981
|
+
depth: 1,
|
|
1982
|
+
split_percent: 75,
|
|
1983
|
+
merge_percent: 40,
|
|
1984
|
+
root_node_address: 0,
|
|
1985
|
+
num_records_in_root: 1,
|
|
1986
|
+
total_records: 3,
|
|
1987
|
+
};
|
|
1988
|
+
|
|
1989
|
+
let root = build_type5_internal(
|
|
1990
|
+
&[(20, [20; 8])],
|
|
1991
|
+
&[(leaf_addr, 1), (leaf_addr, 1)],
|
|
1992
|
+
node_size,
|
|
1993
|
+
);
|
|
1994
|
+
let leaf = build_type5_leaf(&[(10, [10; 8])], node_size);
|
|
1995
|
+
|
|
1996
|
+
let mut data = vec![0; node_size * 2];
|
|
1997
|
+
data[0..node_size].copy_from_slice(&root);
|
|
1998
|
+
data[node_size..node_size * 2].copy_from_slice(&leaf);
|
|
1999
|
+
|
|
2000
|
+
let err = collect_btree_v2_records(&data, &header, 8, 8, None, &[], None).unwrap_err();
|
|
2001
|
+
assert!(matches!(err, Error::InvalidData(message) if message.contains("revisits node")));
|
|
2002
|
+
|
|
2003
|
+
let storage = crate::storage::BytesStorage::new(data);
|
|
2004
|
+
let err =
|
|
2005
|
+
collect_btree_v2_records_storage(&storage, &header, 8, 8, None, &[], None).unwrap_err();
|
|
2006
|
+
assert!(matches!(err, Error::InvalidData(message) if message.contains("revisits node")));
|
|
2007
|
+
}
|
|
2008
|
+
|
|
2009
|
+
#[test]
|
|
2010
|
+
fn collect_link_name_hash_records_filters_leaf_records() {
|
|
2011
|
+
let record_size: u16 = 12;
|
|
2012
|
+
let node_size: u32 = 4096;
|
|
2013
|
+
let header = BTreeV2Header {
|
|
2014
|
+
btree_type: 5,
|
|
2015
|
+
node_size,
|
|
2016
|
+
record_size,
|
|
2017
|
+
depth: 0,
|
|
2018
|
+
split_percent: 75,
|
|
2019
|
+
merge_percent: 40,
|
|
2020
|
+
root_node_address: 0,
|
|
2021
|
+
num_records_in_root: 2,
|
|
2022
|
+
total_records: 2,
|
|
2023
|
+
};
|
|
2024
|
+
|
|
2025
|
+
let mut leaf = Vec::new();
|
|
2026
|
+
leaf.extend_from_slice(b"BTLF");
|
|
2027
|
+
leaf.push(0);
|
|
2028
|
+
leaf.push(5);
|
|
2029
|
+
leaf.extend_from_slice(&0x1111_1111u32.to_le_bytes());
|
|
2030
|
+
leaf.extend_from_slice(&[1, 2, 3, 4, 5, 6, 7, 8]);
|
|
2031
|
+
leaf.extend_from_slice(&0x2222_2222u32.to_le_bytes());
|
|
2032
|
+
leaf.extend_from_slice(&[9, 10, 11, 12, 13, 14, 15, 16]);
|
|
2033
|
+
let checksum = jenkins_lookup3(&leaf);
|
|
2034
|
+
leaf.extend_from_slice(&checksum.to_le_bytes());
|
|
2035
|
+
leaf.resize(node_size as usize, 0);
|
|
2036
|
+
|
|
2037
|
+
let records =
|
|
2038
|
+
collect_btree_v2_link_name_hash_records(&leaf, &header, 8, 8, 0x2222_2222).unwrap();
|
|
2039
|
+
assert_eq!(records.len(), 1);
|
|
2040
|
+
match &records[0] {
|
|
2041
|
+
BTreeV2Record::LinkNameHash { hash, heap_id } => {
|
|
2042
|
+
assert_eq!(*hash, 0x2222_2222);
|
|
2043
|
+
assert_eq!(heap_id, &[9, 10, 11, 12, 13, 14, 15, 16]);
|
|
2044
|
+
}
|
|
2045
|
+
other => panic!("expected LinkNameHash, got {other:?}"),
|
|
2046
|
+
}
|
|
2047
|
+
}
|
|
2048
|
+
|
|
2049
|
+
#[test]
|
|
2050
|
+
fn child_link_hash_filter_prunes_disjoint_hash_ranges() {
|
|
2051
|
+
let records = vec![
|
|
2052
|
+
BTreeV2Record::LinkNameHash {
|
|
2053
|
+
hash: 10,
|
|
2054
|
+
heap_id: vec![],
|
|
2055
|
+
},
|
|
2056
|
+
BTreeV2Record::LinkNameHash {
|
|
2057
|
+
hash: 20,
|
|
2058
|
+
heap_id: vec![],
|
|
2059
|
+
},
|
|
2060
|
+
];
|
|
2061
|
+
|
|
2062
|
+
assert!(child_may_match_link_name_hash(&records, 0, Some(10)));
|
|
2063
|
+
assert!(!child_may_match_link_name_hash(&records, 0, Some(15)));
|
|
2064
|
+
assert!(child_may_match_link_name_hash(&records, 1, Some(15)));
|
|
2065
|
+
assert!(!child_may_match_link_name_hash(&records, 2, Some(15)));
|
|
2066
|
+
assert!(child_may_match_link_name_hash(&records, 2, Some(20)));
|
|
2067
|
+
}
|
|
2068
|
+
}
|