@nirs4all/formats-wasm 0.2.11 → 0.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD_PARTY_NOTICES.md +49 -0
- package/nirs4all_formats_wasm_bg.wasm +0 -0
- package/package.json +4 -2
- package/reader-backports/hdf5-reader/Cargo.toml +119 -0
- package/reader-backports/hdf5-reader/LICENSE-APACHE +164 -0
- package/reader-backports/hdf5-reader/LICENSE-MIT +21 -0
- package/reader-backports/hdf5-reader/PROVENANCE.json +80 -0
- package/reader-backports/hdf5-reader/README.md +308 -0
- package/reader-backports/hdf5-reader/UPSTREAM-Cargo.toml +44 -0
- package/reader-backports/hdf5-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/hdf5-reader/src/attribute_api.rs +847 -0
- package/reader-backports/hdf5-reader/src/btree_v1.rs +726 -0
- package/reader-backports/hdf5-reader/src/btree_v2.rs +2068 -0
- package/reader-backports/hdf5-reader/src/cache.rs +354 -0
- package/reader-backports/hdf5-reader/src/checksum.rs +56 -0
- package/reader-backports/hdf5-reader/src/chunk_index.rs +423 -0
- package/reader-backports/hdf5-reader/src/dataset.rs +4733 -0
- package/reader-backports/hdf5-reader/src/datatype_api.rs +497 -0
- package/reader-backports/hdf5-reader/src/error.rs +167 -0
- package/reader-backports/hdf5-reader/src/extensible_array.rs +1886 -0
- package/reader-backports/hdf5-reader/src/filters/deflate.rs +93 -0
- package/reader-backports/hdf5-reader/src/filters/fletcher32.rs +52 -0
- package/reader-backports/hdf5-reader/src/filters/lz4.rs +178 -0
- package/reader-backports/hdf5-reader/src/filters/mod.rs +326 -0
- package/reader-backports/hdf5-reader/src/filters/nbit.rs +586 -0
- package/reader-backports/hdf5-reader/src/filters/scaleoffset.rs +827 -0
- package/reader-backports/hdf5-reader/src/filters/shuffle.rs +99 -0
- package/reader-backports/hdf5-reader/src/fixed_array.rs +763 -0
- package/reader-backports/hdf5-reader/src/fractal_heap.rs +2029 -0
- package/reader-backports/hdf5-reader/src/global_heap.rs +284 -0
- package/reader-backports/hdf5-reader/src/group.rs +795 -0
- package/reader-backports/hdf5-reader/src/io.rs +413 -0
- package/reader-backports/hdf5-reader/src/lib.rs +916 -0
- package/reader-backports/hdf5-reader/src/local_heap.rs +261 -0
- package/reader-backports/hdf5-reader/src/messages/attribute.rs +304 -0
- package/reader-backports/hdf5-reader/src/messages/attribute_info.rs +96 -0
- package/reader-backports/hdf5-reader/src/messages/btree_k.rs +36 -0
- package/reader-backports/hdf5-reader/src/messages/continuation.rs +46 -0
- package/reader-backports/hdf5-reader/src/messages/dataspace.rs +259 -0
- package/reader-backports/hdf5-reader/src/messages/datatype.rs +938 -0
- package/reader-backports/hdf5-reader/src/messages/external_files.rs +50 -0
- package/reader-backports/hdf5-reader/src/messages/fill_value.rs +233 -0
- package/reader-backports/hdf5-reader/src/messages/filter_pipeline.rs +147 -0
- package/reader-backports/hdf5-reader/src/messages/group_info.rs +103 -0
- package/reader-backports/hdf5-reader/src/messages/layout.rs +476 -0
- package/reader-backports/hdf5-reader/src/messages/link.rs +170 -0
- package/reader-backports/hdf5-reader/src/messages/link_info.rs +127 -0
- package/reader-backports/hdf5-reader/src/messages/mod.rs +374 -0
- package/reader-backports/hdf5-reader/src/messages/modification_time.rs +83 -0
- package/reader-backports/hdf5-reader/src/messages/shared.rs +164 -0
- package/reader-backports/hdf5-reader/src/messages/shared_table.rs +67 -0
- package/reader-backports/hdf5-reader/src/messages/symbol_table_msg.rs +51 -0
- package/reader-backports/hdf5-reader/src/object_header.rs +1749 -0
- package/reader-backports/hdf5-reader/src/reference.rs +90 -0
- package/reader-backports/hdf5-reader/src/shared_message_table.rs +425 -0
- package/reader-backports/hdf5-reader/src/storage.rs +538 -0
- package/reader-backports/hdf5-reader/src/superblock.rs +285 -0
- package/reader-backports/hdf5-reader/src/symbol_table.rs +277 -0
- package/reader-backports/netcdf-reader/Cargo.toml +141 -0
- package/reader-backports/netcdf-reader/LICENSE-APACHE +164 -0
- package/reader-backports/netcdf-reader/LICENSE-MIT +21 -0
- package/reader-backports/netcdf-reader/PROVENANCE.json +55 -0
- package/reader-backports/netcdf-reader/README.md +308 -0
- package/reader-backports/netcdf-reader/UPSTREAM-Cargo.toml +48 -0
- package/reader-backports/netcdf-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/netcdf-reader/src/cf/axes.rs +419 -0
- package/reader-backports/netcdf-reader/src/cf/bounds.rs +108 -0
- package/reader-backports/netcdf-reader/src/cf/crs.rs +269 -0
- package/reader-backports/netcdf-reader/src/cf/mod.rs +30 -0
- package/reader-backports/netcdf-reader/src/cf/time.rs +1122 -0
- package/reader-backports/netcdf-reader/src/classic/data.rs +1465 -0
- package/reader-backports/netcdf-reader/src/classic/header.rs +1097 -0
- package/reader-backports/netcdf-reader/src/classic/mod.rs +240 -0
- package/reader-backports/netcdf-reader/src/classic/storage.rs +211 -0
- package/reader-backports/netcdf-reader/src/classic/types.rs +93 -0
- package/reader-backports/netcdf-reader/src/classic/variable.rs +1162 -0
- package/reader-backports/netcdf-reader/src/error.rs +59 -0
- package/reader-backports/netcdf-reader/src/lib.rs +1531 -0
- package/reader-backports/netcdf-reader/src/masked.rs +173 -0
- package/reader-backports/netcdf-reader/src/nc4/attributes.rs +203 -0
- package/reader-backports/netcdf-reader/src/nc4/dimensions.rs +139 -0
- package/reader-backports/netcdf-reader/src/nc4/groups.rs +308 -0
- package/reader-backports/netcdf-reader/src/nc4/mod.rs +572 -0
- package/reader-backports/netcdf-reader/src/nc4/types.rs +309 -0
- package/reader-backports/netcdf-reader/src/nc4/variables.rs +420 -0
- package/reader-backports/netcdf-reader/src/types.rs +18 -0
- package/reader-backports/netcdf-reader/src/unpack.rs +113 -0
- package/reader-backports/netcdf-reader/src/user_defined.rs +569 -0
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
use crate::checksum::jenkins_lookup3;
|
|
2
|
+
use crate::error::{Error, Result};
|
|
3
|
+
use crate::io::Cursor;
|
|
4
|
+
use crate::storage::Storage;
|
|
5
|
+
use crate::symbol_table::SymbolTableEntry;
|
|
6
|
+
|
|
7
|
+
/// HDF5 magic bytes: `\x89HDF\r\n\x1a\n`
|
|
8
|
+
pub const HDF5_MAGIC: [u8; 8] = [0x89, 0x48, 0x44, 0x46, 0x0d, 0x0a, 0x1a, 0x0a];
|
|
9
|
+
|
|
10
|
+
/// Parsed HDF5 superblock.
|
|
11
|
+
#[derive(Debug, Clone)]
|
|
12
|
+
pub struct Superblock {
|
|
13
|
+
/// Superblock version (0, 1, 2, or 3).
|
|
14
|
+
pub version: u8,
|
|
15
|
+
/// Size of offsets (addresses) in bytes: 2, 4, or 8.
|
|
16
|
+
pub offset_size: u8,
|
|
17
|
+
/// Size of lengths in bytes: 2, 4, or 8.
|
|
18
|
+
pub length_size: u8,
|
|
19
|
+
/// Group leaf node K (v0/v1 only).
|
|
20
|
+
pub group_leaf_node_k: u16,
|
|
21
|
+
/// Group internal node K (v0/v1 only).
|
|
22
|
+
pub group_internal_node_k: u16,
|
|
23
|
+
/// Indexed storage internal node K (v1 only).
|
|
24
|
+
pub indexed_storage_k: u16,
|
|
25
|
+
/// File consistency flags.
|
|
26
|
+
pub consistency_flags: u32,
|
|
27
|
+
/// Base address for offsets (usually 0).
|
|
28
|
+
pub base_address: u64,
|
|
29
|
+
/// Address of the file free-space info (undefined = not present).
|
|
30
|
+
pub free_space_address: u64,
|
|
31
|
+
/// End-of-file address.
|
|
32
|
+
pub eof_address: u64,
|
|
33
|
+
/// Driver information block address (v0/v1 only).
|
|
34
|
+
pub driver_info_address: u64,
|
|
35
|
+
/// Root group symbol table entry (v0/v1).
|
|
36
|
+
pub root_symbol_table_entry: Option<SymbolTableEntry>,
|
|
37
|
+
/// Root group object header address (v2/v3).
|
|
38
|
+
pub root_object_header_address: Option<u64>,
|
|
39
|
+
/// Superblock extension address (v2/v3).
|
|
40
|
+
pub extension_address: Option<u64>,
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
impl Superblock {
|
|
44
|
+
/// Parse the superblock from a cursor positioned at byte 0 (or where the magic starts).
|
|
45
|
+
///
|
|
46
|
+
/// The cursor should be positioned at the start of the file. The method will
|
|
47
|
+
/// search for the magic bytes at position 0, 512, 1024, 2048, etc.
|
|
48
|
+
pub fn parse(cursor: &mut Cursor<'_>) -> Result<Self> {
|
|
49
|
+
// Search for magic at positions 0, 512, 1024, 2048, ...
|
|
50
|
+
let magic_offset = find_magic(cursor)?;
|
|
51
|
+
cursor.set_position(magic_offset + 8);
|
|
52
|
+
|
|
53
|
+
let version = cursor.read_u8()?;
|
|
54
|
+
match version {
|
|
55
|
+
0 | 1 => Self::parse_v0_v1(cursor, version),
|
|
56
|
+
2 | 3 => Self::parse_v2_v3(cursor, version, magic_offset),
|
|
57
|
+
v => Err(Error::UnsupportedSuperblockVersion(v)),
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/// Parse the superblock from random-access storage.
|
|
62
|
+
pub fn parse_from_storage(storage: &dyn Storage) -> Result<Self> {
|
|
63
|
+
let magic_offset = find_magic_in_storage(storage)?;
|
|
64
|
+
let remaining = storage.len().saturating_sub(magic_offset);
|
|
65
|
+
let header_len = remaining.min(256) as usize;
|
|
66
|
+
let header = storage.read_range(magic_offset, header_len)?;
|
|
67
|
+
let mut cursor = Cursor::new(header.as_ref());
|
|
68
|
+
cursor.set_position(8);
|
|
69
|
+
|
|
70
|
+
let version = cursor.read_u8()?;
|
|
71
|
+
match version {
|
|
72
|
+
0 | 1 => Self::parse_v0_v1(&mut cursor, version),
|
|
73
|
+
2 | 3 => Self::parse_v2_v3(&mut cursor, version, 0),
|
|
74
|
+
v => Err(Error::UnsupportedSuperblockVersion(v)),
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
fn parse_v0_v1(cursor: &mut Cursor<'_>, version: u8) -> Result<Self> {
|
|
79
|
+
let _free_space_version = cursor.read_u8()?;
|
|
80
|
+
let _root_group_version = cursor.read_u8()?;
|
|
81
|
+
let _reserved1 = cursor.read_u8()?;
|
|
82
|
+
let _shared_header_version = cursor.read_u8()?;
|
|
83
|
+
|
|
84
|
+
let offset_size = cursor.read_u8()?;
|
|
85
|
+
let length_size = cursor.read_u8()?;
|
|
86
|
+
let _reserved2 = cursor.read_u8()?;
|
|
87
|
+
|
|
88
|
+
let group_leaf_node_k = cursor.read_u16_le()?;
|
|
89
|
+
let group_internal_node_k = cursor.read_u16_le()?;
|
|
90
|
+
let consistency_flags = cursor.read_u32_le()?;
|
|
91
|
+
|
|
92
|
+
let indexed_storage_k = if version == 1 {
|
|
93
|
+
let k = cursor.read_u16_le()?;
|
|
94
|
+
let _reserved = cursor.read_u16_le()?;
|
|
95
|
+
k
|
|
96
|
+
} else {
|
|
97
|
+
0
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
let base_address = cursor.read_offset(offset_size)?;
|
|
101
|
+
let free_space_address = cursor.read_offset(offset_size)?;
|
|
102
|
+
let eof_address = cursor.read_offset(offset_size)?;
|
|
103
|
+
let driver_info_address = cursor.read_offset(offset_size)?;
|
|
104
|
+
|
|
105
|
+
let root_entry = SymbolTableEntry::parse(cursor, offset_size, length_size)?;
|
|
106
|
+
|
|
107
|
+
Ok(Superblock {
|
|
108
|
+
version,
|
|
109
|
+
offset_size,
|
|
110
|
+
length_size,
|
|
111
|
+
group_leaf_node_k,
|
|
112
|
+
group_internal_node_k,
|
|
113
|
+
indexed_storage_k,
|
|
114
|
+
consistency_flags,
|
|
115
|
+
base_address,
|
|
116
|
+
free_space_address,
|
|
117
|
+
eof_address,
|
|
118
|
+
driver_info_address,
|
|
119
|
+
root_symbol_table_entry: Some(root_entry),
|
|
120
|
+
root_object_header_address: None,
|
|
121
|
+
extension_address: None,
|
|
122
|
+
})
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
fn parse_v2_v3(cursor: &mut Cursor<'_>, version: u8, magic_offset: u64) -> Result<Self> {
|
|
126
|
+
let offset_size = cursor.read_u8()?;
|
|
127
|
+
let length_size = cursor.read_u8()?;
|
|
128
|
+
let consistency_flags = cursor.read_u8()? as u32;
|
|
129
|
+
|
|
130
|
+
let base_address = cursor.read_offset(offset_size)?;
|
|
131
|
+
let extension_address = cursor.read_offset(offset_size)?;
|
|
132
|
+
let eof_address = cursor.read_offset(offset_size)?;
|
|
133
|
+
let root_object_header_address = cursor.read_offset(offset_size)?;
|
|
134
|
+
|
|
135
|
+
let stored_checksum = cursor.read_u32_le()?;
|
|
136
|
+
|
|
137
|
+
// Verify checksum: covers everything from magic to just before the checksum
|
|
138
|
+
let checksum_start = magic_offset as usize;
|
|
139
|
+
let checksum_end = cursor.position() as usize - 4;
|
|
140
|
+
let computed = jenkins_lookup3(&cursor.data()[checksum_start..checksum_end]);
|
|
141
|
+
if computed != stored_checksum {
|
|
142
|
+
return Err(Error::ChecksumMismatch {
|
|
143
|
+
expected: stored_checksum,
|
|
144
|
+
actual: computed,
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
let ext = if !Cursor::is_undefined_offset(extension_address, offset_size) {
|
|
149
|
+
Some(extension_address)
|
|
150
|
+
} else {
|
|
151
|
+
None
|
|
152
|
+
};
|
|
153
|
+
|
|
154
|
+
Ok(Superblock {
|
|
155
|
+
version,
|
|
156
|
+
offset_size,
|
|
157
|
+
length_size,
|
|
158
|
+
group_leaf_node_k: 0,
|
|
159
|
+
group_internal_node_k: 0,
|
|
160
|
+
indexed_storage_k: 0,
|
|
161
|
+
consistency_flags,
|
|
162
|
+
base_address,
|
|
163
|
+
free_space_address: u64::MAX,
|
|
164
|
+
eof_address,
|
|
165
|
+
driver_info_address: u64::MAX,
|
|
166
|
+
root_symbol_table_entry: None,
|
|
167
|
+
root_object_header_address: Some(root_object_header_address),
|
|
168
|
+
extension_address: ext,
|
|
169
|
+
})
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/// Get the root group's object header address.
|
|
173
|
+
pub fn root_object_header_address(&self) -> Result<u64> {
|
|
174
|
+
if let Some(addr) = self.root_object_header_address {
|
|
175
|
+
Ok(addr)
|
|
176
|
+
} else if let Some(ref entry) = self.root_symbol_table_entry {
|
|
177
|
+
Ok(entry.object_header_address)
|
|
178
|
+
} else {
|
|
179
|
+
Err(Error::InvalidData(
|
|
180
|
+
"superblock has no root group reference".into(),
|
|
181
|
+
))
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/// For v0/v1 superblocks, get the B-tree address from the root symbol table entry's
|
|
186
|
+
/// scratch-pad (used for root group navigation).
|
|
187
|
+
pub fn root_btree_address(&self) -> Option<u64> {
|
|
188
|
+
self.root_symbol_table_entry
|
|
189
|
+
.as_ref()
|
|
190
|
+
.and_then(|e| e.btree_address())
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/// For v0/v1 superblocks, get the local heap address from the root symbol table entry's
|
|
194
|
+
/// scratch-pad.
|
|
195
|
+
pub fn root_local_heap_address(&self) -> Option<u64> {
|
|
196
|
+
self.root_symbol_table_entry
|
|
197
|
+
.as_ref()
|
|
198
|
+
.and_then(|e| e.local_heap_address())
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/// Search for the HDF5 magic bytes. Per spec, the superblock can appear at
|
|
203
|
+
/// offsets 0, 512, 1024, 2048, etc. (powers of two times 512, plus 0).
|
|
204
|
+
fn find_magic(cursor: &Cursor<'_>) -> Result<u64> {
|
|
205
|
+
// Check offset 0
|
|
206
|
+
if cursor.len() >= 8 {
|
|
207
|
+
let bytes = cursor.peek_bytes(8)?;
|
|
208
|
+
if bytes == HDF5_MAGIC {
|
|
209
|
+
return Ok(0);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// Check 512, 1024, 2048, ...
|
|
214
|
+
let mut offset: u64 = 512;
|
|
215
|
+
while offset.checked_add(8).is_some_and(|end| end <= cursor.len()) {
|
|
216
|
+
let c = cursor.at_offset(offset)?;
|
|
217
|
+
let bytes = c.peek_bytes(8)?;
|
|
218
|
+
if bytes == HDF5_MAGIC {
|
|
219
|
+
return Ok(offset);
|
|
220
|
+
}
|
|
221
|
+
let Some(next_offset) = offset.checked_mul(2) else {
|
|
222
|
+
break;
|
|
223
|
+
};
|
|
224
|
+
offset = next_offset;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
Err(Error::InvalidMagic)
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
fn find_magic_in_storage(storage: &dyn Storage) -> Result<u64> {
|
|
231
|
+
if storage.len() >= 8 {
|
|
232
|
+
let bytes = storage.read_range(0, 8)?;
|
|
233
|
+
if bytes.as_ref() == HDF5_MAGIC {
|
|
234
|
+
return Ok(0);
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
let mut offset: u64 = 512;
|
|
239
|
+
while offset
|
|
240
|
+
.checked_add(8)
|
|
241
|
+
.is_some_and(|end| end <= storage.len())
|
|
242
|
+
{
|
|
243
|
+
let bytes = storage.read_range(offset, 8)?;
|
|
244
|
+
if bytes.as_ref() == HDF5_MAGIC {
|
|
245
|
+
return Ok(offset);
|
|
246
|
+
}
|
|
247
|
+
let Some(next_offset) = offset.checked_mul(2) else {
|
|
248
|
+
break;
|
|
249
|
+
};
|
|
250
|
+
offset = next_offset;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
Err(Error::InvalidMagic)
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
#[cfg(test)]
|
|
257
|
+
mod tests {
|
|
258
|
+
use super::*;
|
|
259
|
+
use crate::storage::RangeRequestStorage;
|
|
260
|
+
|
|
261
|
+
#[test]
|
|
262
|
+
fn magic_detection() {
|
|
263
|
+
// Valid magic at offset 0
|
|
264
|
+
let mut data = HDF5_MAGIC.to_vec();
|
|
265
|
+
data.extend_from_slice(&[0u8; 100]);
|
|
266
|
+
let cursor = Cursor::new(&data);
|
|
267
|
+
assert_eq!(find_magic(&cursor).unwrap(), 0);
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
#[test]
|
|
271
|
+
fn no_magic() {
|
|
272
|
+
let data = [0u8; 100];
|
|
273
|
+
let cursor = Cursor::new(&data);
|
|
274
|
+
assert!(find_magic(&cursor).is_err());
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
#[test]
|
|
278
|
+
fn storage_magic_search_handles_huge_length_without_overflow() {
|
|
279
|
+
let storage = RangeRequestStorage::new(u64::MAX, |_offset, len| Ok(vec![0; len]));
|
|
280
|
+
|
|
281
|
+
let err = find_magic_in_storage(&storage).unwrap_err();
|
|
282
|
+
|
|
283
|
+
assert!(matches!(err, Error::InvalidMagic));
|
|
284
|
+
}
|
|
285
|
+
}
|
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
//! HDF5 Symbol Table Entry and Symbol Table Node (SNOD).
|
|
2
|
+
//!
|
|
3
|
+
//! Symbol table entries appear in the superblock (root group) and inside
|
|
4
|
+
//! symbol table nodes. Each entry maps a link-name offset (within a local
|
|
5
|
+
//! heap) to an object header address, with optional scratch-pad data for
|
|
6
|
+
//! group B-tree/heap addresses or symbolic link values.
|
|
7
|
+
//!
|
|
8
|
+
//! Symbol table nodes (SNOD) are the leaf containers for these entries and
|
|
9
|
+
//! are stored at addresses referenced by v1 B-tree group nodes.
|
|
10
|
+
|
|
11
|
+
use crate::error::{Error, Result};
|
|
12
|
+
use crate::io::Cursor;
|
|
13
|
+
|
|
14
|
+
// ---------------------------------------------------------------------------
|
|
15
|
+
// Symbol Table Entry
|
|
16
|
+
// ---------------------------------------------------------------------------
|
|
17
|
+
|
|
18
|
+
/// A symbol table entry (used in v0/v1 superblock and in SNOD nodes).
|
|
19
|
+
///
|
|
20
|
+
/// When `cache_type` is 1 (group), the scratch-pad space contains the address
|
|
21
|
+
/// of the B-tree and local heap for the group. When `cache_type` is 2, the
|
|
22
|
+
/// scratch-pad contains a symbolic link offset. These are extracted during
|
|
23
|
+
/// parsing and cached for convenience.
|
|
24
|
+
#[derive(Debug, Clone)]
|
|
25
|
+
pub struct SymbolTableEntry {
|
|
26
|
+
/// Offset of the link name within the local heap data segment.
|
|
27
|
+
pub link_name_offset: u64,
|
|
28
|
+
/// Address of the object header for the target object.
|
|
29
|
+
pub object_header_address: u64,
|
|
30
|
+
/// Cache type: 0 = nothing cached, 1 = group info, 2 = symbolic link.
|
|
31
|
+
pub cache_type: u32,
|
|
32
|
+
/// Raw scratch-pad bytes (always 16 bytes in the file).
|
|
33
|
+
pub scratch: [u8; 16],
|
|
34
|
+
/// B-tree address extracted from scratch when `cache_type == 1`.
|
|
35
|
+
cached_btree_address: Option<u64>,
|
|
36
|
+
/// Local heap address extracted from scratch when `cache_type == 1`.
|
|
37
|
+
cached_heap_address: Option<u64>,
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
impl SymbolTableEntry {
|
|
41
|
+
/// Parse a single symbol table entry from the cursor.
|
|
42
|
+
///
|
|
43
|
+
/// The format is:
|
|
44
|
+
/// - Link name offset (`offset_size` bytes)
|
|
45
|
+
/// - Object header address (`offset_size` bytes)
|
|
46
|
+
/// - Cache type (u32 LE)
|
|
47
|
+
/// - Reserved (u32 LE)
|
|
48
|
+
/// - 16 bytes scratch-pad
|
|
49
|
+
pub fn parse(cursor: &mut Cursor, offset_size: u8, _length_size: u8) -> Result<Self> {
|
|
50
|
+
let link_name_offset = cursor.read_offset(offset_size)?;
|
|
51
|
+
let object_header_address = cursor.read_offset(offset_size)?;
|
|
52
|
+
let cache_type = cursor.read_u32_le()?;
|
|
53
|
+
let _reserved = cursor.read_u32_le()?;
|
|
54
|
+
|
|
55
|
+
let scratch_bytes = cursor.read_bytes(16)?;
|
|
56
|
+
let mut scratch = [0u8; 16];
|
|
57
|
+
scratch.copy_from_slice(scratch_bytes);
|
|
58
|
+
|
|
59
|
+
// Extract cached group addresses from scratch when cache_type == 1.
|
|
60
|
+
let (cached_btree_address, cached_heap_address) = if cache_type == 1 {
|
|
61
|
+
let mut sc = Cursor::new(&scratch);
|
|
62
|
+
let btree = sc.read_offset(offset_size)?;
|
|
63
|
+
let heap = sc.read_offset(offset_size)?;
|
|
64
|
+
(Some(btree), Some(heap))
|
|
65
|
+
} else {
|
|
66
|
+
(None, None)
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
Ok(SymbolTableEntry {
|
|
70
|
+
link_name_offset,
|
|
71
|
+
object_header_address,
|
|
72
|
+
cache_type,
|
|
73
|
+
scratch,
|
|
74
|
+
cached_btree_address,
|
|
75
|
+
cached_heap_address,
|
|
76
|
+
})
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/// Address of the B-tree for this group (only valid when `cache_type == 1`).
|
|
80
|
+
pub fn btree_address(&self) -> Option<u64> {
|
|
81
|
+
self.cached_btree_address
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/// Address of the local heap for this group (only valid when `cache_type == 1`).
|
|
85
|
+
pub fn local_heap_address(&self) -> Option<u64> {
|
|
86
|
+
self.cached_heap_address
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// ---------------------------------------------------------------------------
|
|
91
|
+
// Symbol Table Node (SNOD)
|
|
92
|
+
// ---------------------------------------------------------------------------
|
|
93
|
+
|
|
94
|
+
/// Signature bytes for a Symbol Table Node: ASCII `SNOD`.
|
|
95
|
+
const SNOD_SIGNATURE: [u8; 4] = *b"SNOD";
|
|
96
|
+
|
|
97
|
+
/// A symbol table node (SNOD) containing one or more symbol table entries.
|
|
98
|
+
///
|
|
99
|
+
/// These nodes are the leaves of v1 group B-trees and contain the actual
|
|
100
|
+
/// link-name-to-object mappings for groups stored in the old format.
|
|
101
|
+
#[derive(Debug, Clone)]
|
|
102
|
+
pub struct SymbolTableNode {
|
|
103
|
+
/// The entries contained in this node.
|
|
104
|
+
pub entries: Vec<SymbolTableEntry>,
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
impl SymbolTableNode {
|
|
108
|
+
/// Parse a symbol table node at the current cursor position.
|
|
109
|
+
///
|
|
110
|
+
/// The format is:
|
|
111
|
+
/// - Signature: `SNOD` (4 bytes)
|
|
112
|
+
/// - Version: 1 (1 byte)
|
|
113
|
+
/// - Reserved: 1 byte
|
|
114
|
+
/// - Number of entries (u16 LE)
|
|
115
|
+
/// - That many `SymbolTableEntry` values
|
|
116
|
+
pub fn parse(cursor: &mut Cursor, offset_size: u8, length_size: u8) -> Result<Self> {
|
|
117
|
+
let sig = cursor.read_bytes(4)?;
|
|
118
|
+
if sig != SNOD_SIGNATURE {
|
|
119
|
+
return Err(Error::InvalidSymbolTableNodeSignature);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
let version = cursor.read_u8()?;
|
|
123
|
+
if version != 1 {
|
|
124
|
+
return Err(Error::UnsupportedSymbolTableNodeVersion(version));
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
let _reserved = cursor.read_u8()?;
|
|
128
|
+
let num_entries = cursor.read_u16_le()?;
|
|
129
|
+
|
|
130
|
+
let mut entries = Vec::with_capacity(num_entries as usize);
|
|
131
|
+
for _ in 0..num_entries {
|
|
132
|
+
entries.push(SymbolTableEntry::parse(cursor, offset_size, length_size)?);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
Ok(SymbolTableNode { entries })
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
#[cfg(test)]
|
|
140
|
+
mod tests {
|
|
141
|
+
use super::*;
|
|
142
|
+
|
|
143
|
+
/// Build raw bytes for a symbol table entry with the given fields.
|
|
144
|
+
fn build_entry_bytes(
|
|
145
|
+
link_name_offset: u64,
|
|
146
|
+
obj_header_addr: u64,
|
|
147
|
+
cache_type: u32,
|
|
148
|
+
scratch: &[u8; 16],
|
|
149
|
+
offset_size: u8,
|
|
150
|
+
) -> Vec<u8> {
|
|
151
|
+
let mut buf = Vec::new();
|
|
152
|
+
// link name offset
|
|
153
|
+
match offset_size {
|
|
154
|
+
4 => buf.extend_from_slice(&(link_name_offset as u32).to_le_bytes()),
|
|
155
|
+
8 => buf.extend_from_slice(&link_name_offset.to_le_bytes()),
|
|
156
|
+
_ => panic!("test only supports 4/8 byte offsets"),
|
|
157
|
+
}
|
|
158
|
+
// object header address
|
|
159
|
+
match offset_size {
|
|
160
|
+
4 => buf.extend_from_slice(&(obj_header_addr as u32).to_le_bytes()),
|
|
161
|
+
8 => buf.extend_from_slice(&obj_header_addr.to_le_bytes()),
|
|
162
|
+
_ => panic!("test only supports 4/8 byte offsets"),
|
|
163
|
+
}
|
|
164
|
+
// cache type
|
|
165
|
+
buf.extend_from_slice(&cache_type.to_le_bytes());
|
|
166
|
+
// reserved
|
|
167
|
+
buf.extend_from_slice(&0u32.to_le_bytes());
|
|
168
|
+
// scratch pad
|
|
169
|
+
buf.extend_from_slice(scratch);
|
|
170
|
+
buf
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
#[test]
|
|
174
|
+
fn parse_entry_no_cache() {
|
|
175
|
+
let scratch = [0u8; 16];
|
|
176
|
+
let data = build_entry_bytes(42, 0x1000, 0, &scratch, 8);
|
|
177
|
+
|
|
178
|
+
let mut cursor = Cursor::new(&data);
|
|
179
|
+
let entry = SymbolTableEntry::parse(&mut cursor, 8, 8).unwrap();
|
|
180
|
+
|
|
181
|
+
assert_eq!(entry.link_name_offset, 42);
|
|
182
|
+
assert_eq!(entry.object_header_address, 0x1000);
|
|
183
|
+
assert_eq!(entry.cache_type, 0);
|
|
184
|
+
assert!(entry.btree_address().is_none());
|
|
185
|
+
assert!(entry.local_heap_address().is_none());
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
#[test]
|
|
189
|
+
fn parse_entry_group_cache_8byte() {
|
|
190
|
+
// Build scratch with btree=0x2000 and heap=0x3000 (8-byte offsets)
|
|
191
|
+
let mut scratch = [0u8; 16];
|
|
192
|
+
scratch[..8].copy_from_slice(&0x2000u64.to_le_bytes());
|
|
193
|
+
scratch[8..16].copy_from_slice(&0x3000u64.to_le_bytes());
|
|
194
|
+
|
|
195
|
+
let data = build_entry_bytes(0, 0x1000, 1, &scratch, 8);
|
|
196
|
+
|
|
197
|
+
let mut cursor = Cursor::new(&data);
|
|
198
|
+
let entry = SymbolTableEntry::parse(&mut cursor, 8, 8).unwrap();
|
|
199
|
+
|
|
200
|
+
assert_eq!(entry.cache_type, 1);
|
|
201
|
+
assert_eq!(entry.btree_address(), Some(0x2000));
|
|
202
|
+
assert_eq!(entry.local_heap_address(), Some(0x3000));
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
#[test]
|
|
206
|
+
fn parse_entry_group_cache_4byte() {
|
|
207
|
+
// Build scratch with btree=0x400 and heap=0x800 (4-byte offsets)
|
|
208
|
+
let mut scratch = [0u8; 16];
|
|
209
|
+
scratch[..4].copy_from_slice(&0x400u32.to_le_bytes());
|
|
210
|
+
scratch[4..8].copy_from_slice(&0x800u32.to_le_bytes());
|
|
211
|
+
|
|
212
|
+
let data = build_entry_bytes(0, 0x100, 1, &scratch, 4);
|
|
213
|
+
|
|
214
|
+
let mut cursor = Cursor::new(&data);
|
|
215
|
+
let entry = SymbolTableEntry::parse(&mut cursor, 4, 4).unwrap();
|
|
216
|
+
|
|
217
|
+
assert_eq!(entry.cache_type, 1);
|
|
218
|
+
assert_eq!(entry.btree_address(), Some(0x400));
|
|
219
|
+
assert_eq!(entry.local_heap_address(), Some(0x800));
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
#[test]
|
|
223
|
+
fn parse_snod_basic() {
|
|
224
|
+
let mut data = Vec::new();
|
|
225
|
+
// Signature
|
|
226
|
+
data.extend_from_slice(b"SNOD");
|
|
227
|
+
// Version
|
|
228
|
+
data.push(1);
|
|
229
|
+
// Reserved
|
|
230
|
+
data.push(0);
|
|
231
|
+
// Number of entries = 2
|
|
232
|
+
data.extend_from_slice(&2u16.to_le_bytes());
|
|
233
|
+
|
|
234
|
+
// Entry 1: cache_type=0
|
|
235
|
+
let scratch1 = [0u8; 16];
|
|
236
|
+
data.extend(build_entry_bytes(0, 0x1000, 0, &scratch1, 8));
|
|
237
|
+
|
|
238
|
+
// Entry 2: cache_type=1, btree=0xA000, heap=0xB000
|
|
239
|
+
let mut scratch2 = [0u8; 16];
|
|
240
|
+
scratch2[..8].copy_from_slice(&0xA000u64.to_le_bytes());
|
|
241
|
+
scratch2[8..16].copy_from_slice(&0xB000u64.to_le_bytes());
|
|
242
|
+
data.extend(build_entry_bytes(16, 0x2000, 1, &scratch2, 8));
|
|
243
|
+
|
|
244
|
+
let mut cursor = Cursor::new(&data);
|
|
245
|
+
let node = SymbolTableNode::parse(&mut cursor, 8, 8).unwrap();
|
|
246
|
+
|
|
247
|
+
assert_eq!(node.entries.len(), 2);
|
|
248
|
+
assert_eq!(node.entries[0].link_name_offset, 0);
|
|
249
|
+
assert_eq!(node.entries[0].object_header_address, 0x1000);
|
|
250
|
+
assert_eq!(node.entries[0].cache_type, 0);
|
|
251
|
+
|
|
252
|
+
assert_eq!(node.entries[1].link_name_offset, 16);
|
|
253
|
+
assert_eq!(node.entries[1].object_header_address, 0x2000);
|
|
254
|
+
assert_eq!(node.entries[1].btree_address(), Some(0xA000));
|
|
255
|
+
assert_eq!(node.entries[1].local_heap_address(), Some(0xB000));
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
#[test]
|
|
259
|
+
fn snod_bad_signature() {
|
|
260
|
+
let data = b"XNOD\x01\x00\x00\x00";
|
|
261
|
+
let mut cursor = Cursor::new(data);
|
|
262
|
+
assert!(matches!(
|
|
263
|
+
SymbolTableNode::parse(&mut cursor, 8, 8),
|
|
264
|
+
Err(Error::InvalidSymbolTableNodeSignature)
|
|
265
|
+
));
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
#[test]
|
|
269
|
+
fn snod_bad_version() {
|
|
270
|
+
let data = b"SNOD\x02\x00\x00\x00";
|
|
271
|
+
let mut cursor = Cursor::new(data);
|
|
272
|
+
assert!(matches!(
|
|
273
|
+
SymbolTableNode::parse(&mut cursor, 8, 8),
|
|
274
|
+
Err(Error::UnsupportedSymbolTableNodeVersion(2))
|
|
275
|
+
));
|
|
276
|
+
}
|
|
277
|
+
}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
|
|
2
|
+
#
|
|
3
|
+
# When uploading crates to the registry Cargo will automatically
|
|
4
|
+
# "normalize" Cargo.toml files for maximal compatibility
|
|
5
|
+
# with all versions of Cargo and also rewrite `path` dependencies
|
|
6
|
+
# to registry (e.g., crates.io) dependencies.
|
|
7
|
+
#
|
|
8
|
+
# If you are reading this file be aware that the original Cargo.toml
|
|
9
|
+
# will likely look very different (and much more reasonable).
|
|
10
|
+
# See UPSTREAM-Cargo.toml for the unmodified upstream manifest.
|
|
11
|
+
|
|
12
|
+
[package]
|
|
13
|
+
edition = "2021"
|
|
14
|
+
rust-version = "1.85"
|
|
15
|
+
name = "nirs4all-netcdf-reader"
|
|
16
|
+
version = "0.9.3-nirs4all.1"
|
|
17
|
+
build = false
|
|
18
|
+
autolib = false
|
|
19
|
+
autobins = false
|
|
20
|
+
autoexamples = false
|
|
21
|
+
autotests = false
|
|
22
|
+
autobenches = false
|
|
23
|
+
description = "nirs4all maintained security backport of netcdf-reader 0.9.3; see PROVENANCE.json"
|
|
24
|
+
homepage = "https://github.com/roteiro-gis/netcdf-rust"
|
|
25
|
+
documentation = "https://docs.rs/netcdf-reader"
|
|
26
|
+
readme = "README.md"
|
|
27
|
+
keywords = [
|
|
28
|
+
"netcdf",
|
|
29
|
+
"hdf5",
|
|
30
|
+
"decoder",
|
|
31
|
+
"scientific-data",
|
|
32
|
+
"read-only",
|
|
33
|
+
]
|
|
34
|
+
categories = [
|
|
35
|
+
"science",
|
|
36
|
+
"parser-implementations",
|
|
37
|
+
]
|
|
38
|
+
license = "MIT OR Apache-2.0"
|
|
39
|
+
repository = "https://github.com/GBeurier/nirs4all-formats"
|
|
40
|
+
|
|
41
|
+
[features]
|
|
42
|
+
bench-memory-profile = []
|
|
43
|
+
cf = ["dep:chrono"]
|
|
44
|
+
default = [
|
|
45
|
+
"netcdf4",
|
|
46
|
+
"rayon",
|
|
47
|
+
]
|
|
48
|
+
netcdf4 = ["dep:hdf5-reader"]
|
|
49
|
+
rayon = [
|
|
50
|
+
"dep:rayon",
|
|
51
|
+
"hdf5-reader?/rayon",
|
|
52
|
+
]
|
|
53
|
+
|
|
54
|
+
[lib]
|
|
55
|
+
name = "netcdf_reader"
|
|
56
|
+
path = "src/lib.rs"
|
|
57
|
+
|
|
58
|
+
[[test]]
|
|
59
|
+
name = "cf_integration"
|
|
60
|
+
path = "tests/cf_integration.rs"
|
|
61
|
+
|
|
62
|
+
[[test]]
|
|
63
|
+
name = "classic_slice_parity"
|
|
64
|
+
path = "tests/classic_slice_parity.rs"
|
|
65
|
+
|
|
66
|
+
[[test]]
|
|
67
|
+
name = "dos_resistance"
|
|
68
|
+
path = "tests/dos_resistance.rs"
|
|
69
|
+
|
|
70
|
+
[[test]]
|
|
71
|
+
name = "integration"
|
|
72
|
+
path = "tests/integration.rs"
|
|
73
|
+
|
|
74
|
+
[[test]]
|
|
75
|
+
name = "prop_mutation"
|
|
76
|
+
path = "tests/prop_mutation.rs"
|
|
77
|
+
|
|
78
|
+
[[test]]
|
|
79
|
+
name = "property_tests"
|
|
80
|
+
path = "tests/property_tests.rs"
|
|
81
|
+
|
|
82
|
+
[[bench]]
|
|
83
|
+
name = "compare_georust"
|
|
84
|
+
path = "benches/compare_georust.rs"
|
|
85
|
+
harness = false
|
|
86
|
+
|
|
87
|
+
[dependencies.chrono]
|
|
88
|
+
version = "0.4"
|
|
89
|
+
optional = true
|
|
90
|
+
|
|
91
|
+
[dependencies.hdf5-reader]
|
|
92
|
+
package = "nirs4all-hdf5-reader"
|
|
93
|
+
path = "../hdf5-reader"
|
|
94
|
+
version = "=0.9.3-nirs4all.1"
|
|
95
|
+
optional = true
|
|
96
|
+
|
|
97
|
+
[dependencies.memmap2]
|
|
98
|
+
version = "0.9"
|
|
99
|
+
|
|
100
|
+
[dependencies.ndarray]
|
|
101
|
+
version = "0.17"
|
|
102
|
+
|
|
103
|
+
[dependencies.netcdf-core]
|
|
104
|
+
version = "0.9.3"
|
|
105
|
+
|
|
106
|
+
[dependencies.rayon]
|
|
107
|
+
version = "1"
|
|
108
|
+
optional = true
|
|
109
|
+
|
|
110
|
+
[dependencies.thiserror]
|
|
111
|
+
version = "2"
|
|
112
|
+
|
|
113
|
+
[dev-dependencies.criterion]
|
|
114
|
+
version = "0.5"
|
|
115
|
+
|
|
116
|
+
[dev-dependencies.netcdf]
|
|
117
|
+
version = "0.12"
|
|
118
|
+
features = ["static"]
|
|
119
|
+
|
|
120
|
+
[dev-dependencies.peak_alloc]
|
|
121
|
+
version = "0.2"
|
|
122
|
+
|
|
123
|
+
[dev-dependencies.proptest]
|
|
124
|
+
version = "1"
|
|
125
|
+
|
|
126
|
+
[dev-dependencies.tempfile]
|
|
127
|
+
version = "3"
|
|
128
|
+
|
|
129
|
+
[lints.clippy]
|
|
130
|
+
undocumented_unsafe_blocks = "deny"
|
|
131
|
+
|
|
132
|
+
[lints.rust]
|
|
133
|
+
unsafe_op_in_unsafe_fn = "deny"
|
|
134
|
+
|
|
135
|
+
[lints.rust.unexpected_cfgs]
|
|
136
|
+
level = "warn"
|
|
137
|
+
priority = 0
|
|
138
|
+
check-cfg = ["cfg(test)"]
|
|
139
|
+
|
|
140
|
+
# Standalone, independently publishable downstream package.
|
|
141
|
+
[workspace]
|