@nirs4all/formats-wasm 0.2.12 → 0.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD_PARTY_NOTICES.md +49 -0
- package/nirs4all_formats_wasm_bg.wasm +0 -0
- package/package.json +4 -2
- package/reader-backports/hdf5-reader/Cargo.toml +119 -0
- package/reader-backports/hdf5-reader/LICENSE-APACHE +164 -0
- package/reader-backports/hdf5-reader/LICENSE-MIT +21 -0
- package/reader-backports/hdf5-reader/PROVENANCE.json +80 -0
- package/reader-backports/hdf5-reader/README.md +308 -0
- package/reader-backports/hdf5-reader/UPSTREAM-Cargo.toml +44 -0
- package/reader-backports/hdf5-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/hdf5-reader/src/attribute_api.rs +847 -0
- package/reader-backports/hdf5-reader/src/btree_v1.rs +726 -0
- package/reader-backports/hdf5-reader/src/btree_v2.rs +2068 -0
- package/reader-backports/hdf5-reader/src/cache.rs +354 -0
- package/reader-backports/hdf5-reader/src/checksum.rs +56 -0
- package/reader-backports/hdf5-reader/src/chunk_index.rs +423 -0
- package/reader-backports/hdf5-reader/src/dataset.rs +4733 -0
- package/reader-backports/hdf5-reader/src/datatype_api.rs +497 -0
- package/reader-backports/hdf5-reader/src/error.rs +167 -0
- package/reader-backports/hdf5-reader/src/extensible_array.rs +1886 -0
- package/reader-backports/hdf5-reader/src/filters/deflate.rs +93 -0
- package/reader-backports/hdf5-reader/src/filters/fletcher32.rs +52 -0
- package/reader-backports/hdf5-reader/src/filters/lz4.rs +178 -0
- package/reader-backports/hdf5-reader/src/filters/mod.rs +326 -0
- package/reader-backports/hdf5-reader/src/filters/nbit.rs +586 -0
- package/reader-backports/hdf5-reader/src/filters/scaleoffset.rs +827 -0
- package/reader-backports/hdf5-reader/src/filters/shuffle.rs +99 -0
- package/reader-backports/hdf5-reader/src/fixed_array.rs +763 -0
- package/reader-backports/hdf5-reader/src/fractal_heap.rs +2029 -0
- package/reader-backports/hdf5-reader/src/global_heap.rs +284 -0
- package/reader-backports/hdf5-reader/src/group.rs +795 -0
- package/reader-backports/hdf5-reader/src/io.rs +413 -0
- package/reader-backports/hdf5-reader/src/lib.rs +916 -0
- package/reader-backports/hdf5-reader/src/local_heap.rs +261 -0
- package/reader-backports/hdf5-reader/src/messages/attribute.rs +304 -0
- package/reader-backports/hdf5-reader/src/messages/attribute_info.rs +96 -0
- package/reader-backports/hdf5-reader/src/messages/btree_k.rs +36 -0
- package/reader-backports/hdf5-reader/src/messages/continuation.rs +46 -0
- package/reader-backports/hdf5-reader/src/messages/dataspace.rs +259 -0
- package/reader-backports/hdf5-reader/src/messages/datatype.rs +938 -0
- package/reader-backports/hdf5-reader/src/messages/external_files.rs +50 -0
- package/reader-backports/hdf5-reader/src/messages/fill_value.rs +233 -0
- package/reader-backports/hdf5-reader/src/messages/filter_pipeline.rs +147 -0
- package/reader-backports/hdf5-reader/src/messages/group_info.rs +103 -0
- package/reader-backports/hdf5-reader/src/messages/layout.rs +476 -0
- package/reader-backports/hdf5-reader/src/messages/link.rs +170 -0
- package/reader-backports/hdf5-reader/src/messages/link_info.rs +127 -0
- package/reader-backports/hdf5-reader/src/messages/mod.rs +374 -0
- package/reader-backports/hdf5-reader/src/messages/modification_time.rs +83 -0
- package/reader-backports/hdf5-reader/src/messages/shared.rs +164 -0
- package/reader-backports/hdf5-reader/src/messages/shared_table.rs +67 -0
- package/reader-backports/hdf5-reader/src/messages/symbol_table_msg.rs +51 -0
- package/reader-backports/hdf5-reader/src/object_header.rs +1749 -0
- package/reader-backports/hdf5-reader/src/reference.rs +90 -0
- package/reader-backports/hdf5-reader/src/shared_message_table.rs +425 -0
- package/reader-backports/hdf5-reader/src/storage.rs +538 -0
- package/reader-backports/hdf5-reader/src/superblock.rs +285 -0
- package/reader-backports/hdf5-reader/src/symbol_table.rs +277 -0
- package/reader-backports/netcdf-reader/Cargo.toml +141 -0
- package/reader-backports/netcdf-reader/LICENSE-APACHE +164 -0
- package/reader-backports/netcdf-reader/LICENSE-MIT +21 -0
- package/reader-backports/netcdf-reader/PROVENANCE.json +55 -0
- package/reader-backports/netcdf-reader/README.md +308 -0
- package/reader-backports/netcdf-reader/UPSTREAM-Cargo.toml +48 -0
- package/reader-backports/netcdf-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/netcdf-reader/src/cf/axes.rs +419 -0
- package/reader-backports/netcdf-reader/src/cf/bounds.rs +108 -0
- package/reader-backports/netcdf-reader/src/cf/crs.rs +269 -0
- package/reader-backports/netcdf-reader/src/cf/mod.rs +30 -0
- package/reader-backports/netcdf-reader/src/cf/time.rs +1122 -0
- package/reader-backports/netcdf-reader/src/classic/data.rs +1465 -0
- package/reader-backports/netcdf-reader/src/classic/header.rs +1097 -0
- package/reader-backports/netcdf-reader/src/classic/mod.rs +240 -0
- package/reader-backports/netcdf-reader/src/classic/storage.rs +211 -0
- package/reader-backports/netcdf-reader/src/classic/types.rs +93 -0
- package/reader-backports/netcdf-reader/src/classic/variable.rs +1162 -0
- package/reader-backports/netcdf-reader/src/error.rs +59 -0
- package/reader-backports/netcdf-reader/src/lib.rs +1531 -0
- package/reader-backports/netcdf-reader/src/masked.rs +173 -0
- package/reader-backports/netcdf-reader/src/nc4/attributes.rs +203 -0
- package/reader-backports/netcdf-reader/src/nc4/dimensions.rs +139 -0
- package/reader-backports/netcdf-reader/src/nc4/groups.rs +308 -0
- package/reader-backports/netcdf-reader/src/nc4/mod.rs +572 -0
- package/reader-backports/netcdf-reader/src/nc4/types.rs +309 -0
- package/reader-backports/netcdf-reader/src/nc4/variables.rs +420 -0
- package/reader-backports/netcdf-reader/src/types.rs +18 -0
- package/reader-backports/netcdf-reader/src/unpack.rs +113 -0
- package/reader-backports/netcdf-reader/src/user_defined.rs +569 -0
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
//! HDF5 Local Heap (HEAP).
|
|
2
|
+
//!
|
|
3
|
+
//! A local heap stores small, variable-length data — most commonly the link
|
|
4
|
+
//! names referenced by symbol table entries in old-style (v1) groups. Each
|
|
5
|
+
//! group has its own local heap pointed to by the Symbol Table message or the
|
|
6
|
+
//! root symbol table entry's scratch-pad.
|
|
7
|
+
//!
|
|
8
|
+
//! The heap header stores metadata while the actual string data lives in a
|
|
9
|
+
//! contiguous data segment elsewhere in the file.
|
|
10
|
+
|
|
11
|
+
use crate::error::{Error, Result};
|
|
12
|
+
use crate::io::Cursor;
|
|
13
|
+
use crate::storage::Storage;
|
|
14
|
+
|
|
15
|
+
/// Signature bytes for a Local Heap: ASCII `HEAP`.
|
|
16
|
+
const HEAP_SIGNATURE: [u8; 4] = *b"HEAP";
|
|
17
|
+
|
|
18
|
+
/// A parsed HDF5 local heap header.
|
|
19
|
+
///
|
|
20
|
+
/// The data segment (containing the actual strings) is stored separately in
|
|
21
|
+
/// the file at `data_segment_address`.
|
|
22
|
+
#[derive(Debug, Clone)]
|
|
23
|
+
pub struct LocalHeap {
|
|
24
|
+
/// Size in bytes of the data segment.
|
|
25
|
+
pub data_segment_size: u64,
|
|
26
|
+
/// Offset to the head of the free-list within the data segment
|
|
27
|
+
/// (relative to the start of the data segment).
|
|
28
|
+
pub free_list_offset: u64,
|
|
29
|
+
/// Absolute file address of the data segment.
|
|
30
|
+
pub data_segment_address: u64,
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
impl LocalHeap {
|
|
34
|
+
/// Parse a local heap header at the current cursor position.
|
|
35
|
+
///
|
|
36
|
+
/// Format:
|
|
37
|
+
/// - Signature: `HEAP` (4 bytes)
|
|
38
|
+
/// - Version: 0 (1 byte)
|
|
39
|
+
/// - Reserved: 3 bytes
|
|
40
|
+
/// - Data segment size (`length_size` bytes)
|
|
41
|
+
/// - Offset to head of free list (`length_size` bytes)
|
|
42
|
+
/// - Data segment address (`offset_size` bytes)
|
|
43
|
+
pub fn parse(cursor: &mut Cursor, offset_size: u8, length_size: u8) -> Result<Self> {
|
|
44
|
+
let sig = cursor.read_bytes(4)?;
|
|
45
|
+
if sig != HEAP_SIGNATURE {
|
|
46
|
+
return Err(Error::InvalidLocalHeapSignature);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
let version = cursor.read_u8()?;
|
|
50
|
+
if version != 0 {
|
|
51
|
+
return Err(Error::UnsupportedLocalHeapVersion(version));
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// Reserved 3 bytes
|
|
55
|
+
cursor.skip(3)?;
|
|
56
|
+
|
|
57
|
+
let data_segment_size = cursor.read_length(length_size)?;
|
|
58
|
+
let free_list_offset = cursor.read_length(length_size)?;
|
|
59
|
+
let data_segment_address = cursor.read_offset(offset_size)?;
|
|
60
|
+
|
|
61
|
+
Ok(LocalHeap {
|
|
62
|
+
data_segment_size,
|
|
63
|
+
free_list_offset,
|
|
64
|
+
data_segment_address,
|
|
65
|
+
})
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/// Parse a local heap header from random-access storage.
|
|
69
|
+
pub fn parse_at_storage(
|
|
70
|
+
storage: &dyn Storage,
|
|
71
|
+
address: u64,
|
|
72
|
+
offset_size: u8,
|
|
73
|
+
length_size: u8,
|
|
74
|
+
) -> Result<Self> {
|
|
75
|
+
let header_len = 4
|
|
76
|
+
+ 1
|
|
77
|
+
+ 3
|
|
78
|
+
+ usize::from(length_size)
|
|
79
|
+
+ usize::from(length_size)
|
|
80
|
+
+ usize::from(offset_size);
|
|
81
|
+
let bytes = storage.read_range(address, header_len)?;
|
|
82
|
+
let mut cursor = Cursor::new(bytes.as_ref());
|
|
83
|
+
Self::parse(&mut cursor, offset_size, length_size)
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/// Read a null-terminated string at the given offset within the heap's
|
|
87
|
+
/// data segment.
|
|
88
|
+
///
|
|
89
|
+
/// `offset` is relative to the start of the data segment (as stored in
|
|
90
|
+
/// `SymbolTableEntry::link_name_offset`). `file_data` must be the entire
|
|
91
|
+
/// file (or at least the portion containing the data segment).
|
|
92
|
+
pub fn get_string(&self, offset: u64, file_data: &[u8]) -> Result<String> {
|
|
93
|
+
let abs = self
|
|
94
|
+
.data_segment_address
|
|
95
|
+
.checked_add(offset)
|
|
96
|
+
.ok_or(Error::OffsetOutOfBounds(offset))?;
|
|
97
|
+
let start = abs as usize;
|
|
98
|
+
|
|
99
|
+
if start >= file_data.len() {
|
|
100
|
+
return Err(Error::OffsetOutOfBounds(abs));
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// Find the null terminator within the data segment bounds.
|
|
104
|
+
let segment_end = (self.data_segment_address as usize)
|
|
105
|
+
.saturating_add(self.data_segment_size as usize)
|
|
106
|
+
.min(file_data.len());
|
|
107
|
+
|
|
108
|
+
let search_region = &file_data[start..segment_end];
|
|
109
|
+
let null_pos = search_region.iter().position(|&b| b == 0).ok_or_else(|| {
|
|
110
|
+
Error::InvalidData("local heap string missing null terminator".into())
|
|
111
|
+
})?;
|
|
112
|
+
|
|
113
|
+
let s = std::str::from_utf8(&search_region[..null_pos])
|
|
114
|
+
.map_err(|e| Error::InvalidData(format!("invalid UTF-8 in local heap string: {e}")))?;
|
|
115
|
+
|
|
116
|
+
Ok(s.to_string())
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/// Read a null-terminated string from random-access storage.
|
|
120
|
+
pub fn get_string_storage(&self, offset: u64, storage: &dyn Storage) -> Result<String> {
|
|
121
|
+
if offset >= self.data_segment_size {
|
|
122
|
+
return Err(Error::OffsetOutOfBounds(offset));
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
let available = self
|
|
126
|
+
.data_segment_size
|
|
127
|
+
.checked_sub(offset)
|
|
128
|
+
.ok_or(Error::OffsetOutOfBounds(offset))?;
|
|
129
|
+
let len = usize::try_from(available).map_err(|_| {
|
|
130
|
+
Error::InvalidData("local heap string region exceeds platform usize capacity".into())
|
|
131
|
+
})?;
|
|
132
|
+
let abs = self
|
|
133
|
+
.data_segment_address
|
|
134
|
+
.checked_add(offset)
|
|
135
|
+
.ok_or(Error::OffsetOutOfBounds(offset))?;
|
|
136
|
+
let bytes = storage.read_range(abs, len)?;
|
|
137
|
+
let null_pos = bytes.iter().position(|&b| b == 0).ok_or_else(|| {
|
|
138
|
+
Error::InvalidData("local heap string missing null terminator".into())
|
|
139
|
+
})?;
|
|
140
|
+
let s = std::str::from_utf8(&bytes[..null_pos])
|
|
141
|
+
.map_err(|e| Error::InvalidData(format!("invalid UTF-8 in local heap string: {e}")))?;
|
|
142
|
+
Ok(s.to_string())
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
#[cfg(test)]
|
|
147
|
+
mod tests {
|
|
148
|
+
use super::*;
|
|
149
|
+
|
|
150
|
+
/// Build a local heap header with the given parameters (8-byte offset/length).
|
|
151
|
+
fn build_heap_header(
|
|
152
|
+
data_segment_size: u64,
|
|
153
|
+
free_list_offset: u64,
|
|
154
|
+
data_segment_address: u64,
|
|
155
|
+
) -> Vec<u8> {
|
|
156
|
+
let mut buf = Vec::new();
|
|
157
|
+
buf.extend_from_slice(b"HEAP");
|
|
158
|
+
buf.push(0); // version
|
|
159
|
+
buf.extend_from_slice(&[0, 0, 0]); // reserved
|
|
160
|
+
buf.extend_from_slice(&data_segment_size.to_le_bytes());
|
|
161
|
+
buf.extend_from_slice(&free_list_offset.to_le_bytes());
|
|
162
|
+
buf.extend_from_slice(&data_segment_address.to_le_bytes());
|
|
163
|
+
buf
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
#[test]
|
|
167
|
+
fn parse_local_heap() {
|
|
168
|
+
let data = build_heap_header(256, 128, 0x2000);
|
|
169
|
+
|
|
170
|
+
let mut cursor = Cursor::new(&data);
|
|
171
|
+
let heap = LocalHeap::parse(&mut cursor, 8, 8).unwrap();
|
|
172
|
+
|
|
173
|
+
assert_eq!(heap.data_segment_size, 256);
|
|
174
|
+
assert_eq!(heap.free_list_offset, 128);
|
|
175
|
+
assert_eq!(heap.data_segment_address, 0x2000);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
#[test]
|
|
179
|
+
fn parse_local_heap_4byte() {
|
|
180
|
+
let mut buf = Vec::new();
|
|
181
|
+
buf.extend_from_slice(b"HEAP");
|
|
182
|
+
buf.push(0); // version
|
|
183
|
+
buf.extend_from_slice(&[0, 0, 0]); // reserved
|
|
184
|
+
buf.extend_from_slice(&64u32.to_le_bytes()); // data segment size
|
|
185
|
+
buf.extend_from_slice(&32u32.to_le_bytes()); // free list offset
|
|
186
|
+
buf.extend_from_slice(&0x400u32.to_le_bytes()); // data segment address
|
|
187
|
+
|
|
188
|
+
let mut cursor = Cursor::new(&buf);
|
|
189
|
+
let heap = LocalHeap::parse(&mut cursor, 4, 4).unwrap();
|
|
190
|
+
|
|
191
|
+
assert_eq!(heap.data_segment_size, 64);
|
|
192
|
+
assert_eq!(heap.free_list_offset, 32);
|
|
193
|
+
assert_eq!(heap.data_segment_address, 0x400);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
#[test]
|
|
197
|
+
fn bad_signature() {
|
|
198
|
+
let mut data = build_heap_header(256, 128, 0x2000);
|
|
199
|
+
data[0] = b'X'; // corrupt signature
|
|
200
|
+
let mut cursor = Cursor::new(&data);
|
|
201
|
+
assert!(matches!(
|
|
202
|
+
LocalHeap::parse(&mut cursor, 8, 8),
|
|
203
|
+
Err(Error::InvalidLocalHeapSignature)
|
|
204
|
+
));
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
#[test]
|
|
208
|
+
fn bad_version() {
|
|
209
|
+
let mut data = build_heap_header(256, 128, 0x2000);
|
|
210
|
+
data[4] = 1; // version 1 (unsupported)
|
|
211
|
+
let mut cursor = Cursor::new(&data);
|
|
212
|
+
assert!(matches!(
|
|
213
|
+
LocalHeap::parse(&mut cursor, 8, 8),
|
|
214
|
+
Err(Error::UnsupportedLocalHeapVersion(1))
|
|
215
|
+
));
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
#[test]
|
|
219
|
+
fn local_heap_reads_null_terminated_strings() {
|
|
220
|
+
// Simulate a file where the data segment starts at offset 100.
|
|
221
|
+
let mut file_data = vec![0u8; 200];
|
|
222
|
+
// Place "hello\0world\0" at the data segment.
|
|
223
|
+
let seg_start = 100usize;
|
|
224
|
+
file_data[seg_start..seg_start + 6].copy_from_slice(b"hello\0");
|
|
225
|
+
file_data[seg_start + 6..seg_start + 12].copy_from_slice(b"world\0");
|
|
226
|
+
|
|
227
|
+
let heap = LocalHeap {
|
|
228
|
+
data_segment_size: 100,
|
|
229
|
+
free_list_offset: 50,
|
|
230
|
+
data_segment_address: seg_start as u64,
|
|
231
|
+
};
|
|
232
|
+
|
|
233
|
+
assert_eq!(heap.get_string(0, &file_data).unwrap(), "hello");
|
|
234
|
+
assert_eq!(heap.get_string(6, &file_data).unwrap(), "world");
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
#[test]
|
|
238
|
+
fn get_string_out_of_bounds() {
|
|
239
|
+
let file_data = vec![0u8; 50];
|
|
240
|
+
let heap = LocalHeap {
|
|
241
|
+
data_segment_size: 100,
|
|
242
|
+
free_list_offset: 0,
|
|
243
|
+
data_segment_address: 100, // beyond file_data
|
|
244
|
+
};
|
|
245
|
+
assert!(heap.get_string(0, &file_data).is_err());
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
#[test]
|
|
249
|
+
fn get_string_missing_null() {
|
|
250
|
+
// Data segment with no null terminator.
|
|
251
|
+
let mut file_data = vec![0xFFu8; 200];
|
|
252
|
+
file_data[100..105].copy_from_slice(b"abcde");
|
|
253
|
+
|
|
254
|
+
let heap = LocalHeap {
|
|
255
|
+
data_segment_size: 5,
|
|
256
|
+
free_list_offset: 0,
|
|
257
|
+
data_segment_address: 100,
|
|
258
|
+
};
|
|
259
|
+
assert!(heap.get_string(0, &file_data).is_err());
|
|
260
|
+
}
|
|
261
|
+
}
|
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
//! HDF5 Attribute message (type 0x000C).
|
|
2
|
+
//!
|
|
3
|
+
//! Attributes are small named data items attached to any HDF5 object.
|
|
4
|
+
//! Each attribute has a name, datatype, dataspace, and raw data.
|
|
5
|
+
|
|
6
|
+
use crate::error::{Error, Result};
|
|
7
|
+
use crate::io::Cursor;
|
|
8
|
+
|
|
9
|
+
use super::dataspace::{self, DataspaceMessage};
|
|
10
|
+
use super::datatype::{self, Datatype};
|
|
11
|
+
|
|
12
|
+
/// Parsed attribute message.
|
|
13
|
+
#[derive(Debug, Clone)]
|
|
14
|
+
pub struct AttributeMessage {
|
|
15
|
+
/// Attribute name.
|
|
16
|
+
pub name: String,
|
|
17
|
+
/// Datatype of the attribute value.
|
|
18
|
+
pub datatype: Datatype,
|
|
19
|
+
/// Dataspace describing the shape of the attribute data.
|
|
20
|
+
pub dataspace: DataspaceMessage,
|
|
21
|
+
/// Raw attribute data (un-decoded bytes).
|
|
22
|
+
pub raw_data: Vec<u8>,
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/// Parse an attribute message.
|
|
26
|
+
///
|
|
27
|
+
/// Attribute messages embed inline datatype and dataspace descriptions
|
|
28
|
+
/// followed by the raw data bytes.
|
|
29
|
+
pub fn parse(
|
|
30
|
+
cursor: &mut Cursor<'_>,
|
|
31
|
+
offset_size: u8,
|
|
32
|
+
length_size: u8,
|
|
33
|
+
msg_size: usize,
|
|
34
|
+
) -> Result<AttributeMessage> {
|
|
35
|
+
let start = cursor.position();
|
|
36
|
+
let version = cursor.read_u8()?;
|
|
37
|
+
|
|
38
|
+
let result = match version {
|
|
39
|
+
1 => parse_v1(cursor, offset_size, length_size),
|
|
40
|
+
2 => parse_v2(cursor, offset_size, length_size),
|
|
41
|
+
3 => parse_v3(cursor, offset_size, length_size),
|
|
42
|
+
v => Err(Error::UnsupportedAttributeVersion(v)),
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
result.and_then(|msg| {
|
|
46
|
+
let consumed = (cursor.position() - start) as usize;
|
|
47
|
+
if consumed < msg_size {
|
|
48
|
+
cursor.skip(msg_size - consumed)?;
|
|
49
|
+
}
|
|
50
|
+
Ok(msg)
|
|
51
|
+
})
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
fn parse_v1(cursor: &mut Cursor<'_>, offset_size: u8, length_size: u8) -> Result<AttributeMessage> {
|
|
55
|
+
let _reserved = cursor.read_u8()?;
|
|
56
|
+
let name_size = cursor.read_u16_le()? as usize;
|
|
57
|
+
let datatype_size = cursor.read_u16_le()? as usize;
|
|
58
|
+
let dataspace_size = cursor.read_u16_le()? as usize;
|
|
59
|
+
|
|
60
|
+
// Name — padded to 8-byte boundary
|
|
61
|
+
let name = cursor.read_fixed_string(name_size)?;
|
|
62
|
+
let name_padded = (name_size + 7) & !7;
|
|
63
|
+
if name_padded > name_size {
|
|
64
|
+
cursor.skip(name_padded - name_size)?;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// Datatype — padded to 8
|
|
68
|
+
let dt_msg = datatype::parse(cursor, datatype_size)?;
|
|
69
|
+
let dt_consumed = datatype_size; // parse() already handles its own size
|
|
70
|
+
let dt_padded = (dt_consumed + 7) & !7;
|
|
71
|
+
if dt_padded > dt_consumed {
|
|
72
|
+
cursor.skip(dt_padded - dt_consumed)?;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// Dataspace — padded to 8
|
|
76
|
+
let ds_msg = dataspace::parse(cursor, offset_size, length_size, dataspace_size)?;
|
|
77
|
+
let ds_consumed = dataspace_size;
|
|
78
|
+
let ds_padded = (ds_consumed + 7) & !7;
|
|
79
|
+
if ds_padded > ds_consumed {
|
|
80
|
+
cursor.skip(ds_padded - ds_consumed)?;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// Raw data — remaining bytes are the attribute data
|
|
84
|
+
let data_size = attribute_raw_data_size(&ds_msg, dt_msg.size)?;
|
|
85
|
+
let raw_data = if data_size > 0 {
|
|
86
|
+
cursor.read_bytes(data_size)?.to_vec()
|
|
87
|
+
} else {
|
|
88
|
+
vec![]
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
Ok(AttributeMessage {
|
|
92
|
+
name,
|
|
93
|
+
datatype: dt_msg.datatype,
|
|
94
|
+
dataspace: ds_msg,
|
|
95
|
+
raw_data,
|
|
96
|
+
})
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
fn parse_v2(cursor: &mut Cursor<'_>, offset_size: u8, length_size: u8) -> Result<AttributeMessage> {
|
|
100
|
+
let _flags = cursor.read_u8()?;
|
|
101
|
+
let name_size = cursor.read_u16_le()? as usize;
|
|
102
|
+
let datatype_size = cursor.read_u16_le()? as usize;
|
|
103
|
+
let dataspace_size = cursor.read_u16_le()? as usize;
|
|
104
|
+
|
|
105
|
+
// Name — NOT padded in v2
|
|
106
|
+
let name = cursor.read_fixed_string(name_size)?;
|
|
107
|
+
|
|
108
|
+
// Datatype
|
|
109
|
+
let dt_msg = datatype::parse(cursor, datatype_size)?;
|
|
110
|
+
|
|
111
|
+
// Dataspace
|
|
112
|
+
let ds_msg = dataspace::parse(cursor, offset_size, length_size, dataspace_size)?;
|
|
113
|
+
|
|
114
|
+
// Raw data
|
|
115
|
+
let data_size = attribute_raw_data_size(&ds_msg, dt_msg.size)?;
|
|
116
|
+
let raw_data = if data_size > 0 {
|
|
117
|
+
cursor.read_bytes(data_size)?.to_vec()
|
|
118
|
+
} else {
|
|
119
|
+
vec![]
|
|
120
|
+
};
|
|
121
|
+
|
|
122
|
+
Ok(AttributeMessage {
|
|
123
|
+
name,
|
|
124
|
+
datatype: dt_msg.datatype,
|
|
125
|
+
dataspace: ds_msg,
|
|
126
|
+
raw_data,
|
|
127
|
+
})
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
fn parse_v3(cursor: &mut Cursor<'_>, offset_size: u8, length_size: u8) -> Result<AttributeMessage> {
|
|
131
|
+
let flags = cursor.read_u8()?;
|
|
132
|
+
let name_size = cursor.read_u16_le()? as usize;
|
|
133
|
+
let datatype_size = cursor.read_u16_le()? as usize;
|
|
134
|
+
let dataspace_size = cursor.read_u16_le()? as usize;
|
|
135
|
+
let _name_encoding = cursor.read_u8()?;
|
|
136
|
+
|
|
137
|
+
if (flags & 0x03) != 0 {
|
|
138
|
+
return Err(Error::InvalidData(
|
|
139
|
+
"shared datatype/dataspace in attribute v3 is not supported".to_string(),
|
|
140
|
+
));
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// Name — NOT padded in v3
|
|
144
|
+
let name = cursor.read_fixed_string(name_size)?;
|
|
145
|
+
|
|
146
|
+
// Datatype
|
|
147
|
+
let dt_msg = datatype::parse(cursor, datatype_size)?;
|
|
148
|
+
|
|
149
|
+
// Dataspace
|
|
150
|
+
let ds_msg = dataspace::parse(cursor, offset_size, length_size, dataspace_size)?;
|
|
151
|
+
|
|
152
|
+
// Raw data
|
|
153
|
+
let data_size = attribute_raw_data_size(&ds_msg, dt_msg.size)?;
|
|
154
|
+
let raw_data = if data_size > 0 {
|
|
155
|
+
cursor.read_bytes(data_size)?.to_vec()
|
|
156
|
+
} else {
|
|
157
|
+
vec![]
|
|
158
|
+
};
|
|
159
|
+
|
|
160
|
+
Ok(AttributeMessage {
|
|
161
|
+
name,
|
|
162
|
+
datatype: dt_msg.datatype,
|
|
163
|
+
dataspace: ds_msg,
|
|
164
|
+
raw_data,
|
|
165
|
+
})
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
fn attribute_raw_data_size(ds_msg: &dataspace::DataspaceMessage, dtype_size: u32) -> Result<usize> {
|
|
169
|
+
let elements = usize::try_from(ds_msg.num_elements()?).map_err(|_| {
|
|
170
|
+
Error::InvalidData("attribute element count exceeds platform usize capacity".to_string())
|
|
171
|
+
})?;
|
|
172
|
+
elements.checked_mul(dtype_size as usize).ok_or_else(|| {
|
|
173
|
+
Error::InvalidData("attribute raw data size exceeds platform usize capacity".to_string())
|
|
174
|
+
})
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
#[cfg(test)]
|
|
178
|
+
mod tests {
|
|
179
|
+
use super::*;
|
|
180
|
+
use crate::error::ByteOrder;
|
|
181
|
+
use crate::messages::dataspace::DataspaceType;
|
|
182
|
+
|
|
183
|
+
/// Build a minimal u32-LE datatype description (8 bytes total: 4 class/ver/flags + 4 size).
|
|
184
|
+
fn u32_le_datatype() -> Vec<u8> {
|
|
185
|
+
let mut buf = Vec::new();
|
|
186
|
+
// class=0, version=1, flags=0 (LE unsigned)
|
|
187
|
+
let class_word: u32 = 0x01 << 4;
|
|
188
|
+
buf.extend_from_slice(&class_word.to_le_bytes());
|
|
189
|
+
buf.extend_from_slice(&4u32.to_le_bytes()); // size=4
|
|
190
|
+
// properties: offset=0, precision=32
|
|
191
|
+
buf.extend_from_slice(&0u16.to_le_bytes());
|
|
192
|
+
buf.extend_from_slice(&32u16.to_le_bytes());
|
|
193
|
+
buf
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/// Build a scalar dataspace (v2, rank=0, type=scalar).
|
|
197
|
+
fn scalar_dataspace() -> Vec<u8> {
|
|
198
|
+
vec![0x02, 0x00, 0x00, 0x00]
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
#[test]
|
|
202
|
+
fn parse_v1_scalar_u32_attr() {
|
|
203
|
+
let dt = u32_le_datatype();
|
|
204
|
+
let ds = scalar_dataspace();
|
|
205
|
+
|
|
206
|
+
let mut data = vec![
|
|
207
|
+
0x01, // version 1
|
|
208
|
+
0x00, // reserved
|
|
209
|
+
];
|
|
210
|
+
// name size = 5 ("temp\0")
|
|
211
|
+
data.extend_from_slice(&5u16.to_le_bytes());
|
|
212
|
+
// datatype size
|
|
213
|
+
data.extend_from_slice(&(dt.len() as u16).to_le_bytes());
|
|
214
|
+
// dataspace size
|
|
215
|
+
data.extend_from_slice(&(ds.len() as u16).to_le_bytes());
|
|
216
|
+
|
|
217
|
+
// Name "temp\0" padded to 8 bytes
|
|
218
|
+
data.extend_from_slice(b"temp\0\0\0\0");
|
|
219
|
+
|
|
220
|
+
// Datatype (12 bytes), padded to 16
|
|
221
|
+
data.extend_from_slice(&dt);
|
|
222
|
+
data.extend_from_slice(&[0u8; 4]); // padding to 16
|
|
223
|
+
|
|
224
|
+
// Dataspace (4 bytes), padded to 8
|
|
225
|
+
data.extend_from_slice(&ds);
|
|
226
|
+
data.extend_from_slice(&[0u8; 4]); // padding to 8
|
|
227
|
+
|
|
228
|
+
// Raw data: 1 scalar element * 4 bytes
|
|
229
|
+
data.extend_from_slice(&42u32.to_le_bytes());
|
|
230
|
+
|
|
231
|
+
let mut cursor = Cursor::new(&data);
|
|
232
|
+
let msg = parse(&mut cursor, 8, 8, data.len()).unwrap();
|
|
233
|
+
assert_eq!(msg.name, "temp");
|
|
234
|
+
assert_eq!(msg.dataspace.dataspace_type, DataspaceType::Scalar);
|
|
235
|
+
assert_eq!(msg.raw_data, 42u32.to_le_bytes());
|
|
236
|
+
match &msg.datatype {
|
|
237
|
+
Datatype::FixedPoint {
|
|
238
|
+
size: 4,
|
|
239
|
+
signed: false,
|
|
240
|
+
byte_order: ByteOrder::LittleEndian,
|
|
241
|
+
} => {}
|
|
242
|
+
other => panic!("unexpected datatype: {:?}", other),
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
#[test]
|
|
247
|
+
fn parse_v3_scalar_attr() {
|
|
248
|
+
let dt = u32_le_datatype();
|
|
249
|
+
let ds = scalar_dataspace();
|
|
250
|
+
|
|
251
|
+
let mut data = vec![
|
|
252
|
+
0x03, // version 3
|
|
253
|
+
0x00, // flags
|
|
254
|
+
];
|
|
255
|
+
// name size = 4 ("abc\0")
|
|
256
|
+
data.extend_from_slice(&4u16.to_le_bytes());
|
|
257
|
+
data.extend_from_slice(&(dt.len() as u16).to_le_bytes());
|
|
258
|
+
data.extend_from_slice(&(ds.len() as u16).to_le_bytes());
|
|
259
|
+
data.push(0x00); // ASCII name encoding
|
|
260
|
+
|
|
261
|
+
// Name (not padded in v3)
|
|
262
|
+
data.extend_from_slice(b"abc\0");
|
|
263
|
+
|
|
264
|
+
// Datatype
|
|
265
|
+
data.extend_from_slice(&dt);
|
|
266
|
+
|
|
267
|
+
// Dataspace
|
|
268
|
+
data.extend_from_slice(&ds);
|
|
269
|
+
|
|
270
|
+
// Raw data
|
|
271
|
+
data.extend_from_slice(&99u32.to_le_bytes());
|
|
272
|
+
|
|
273
|
+
let mut cursor = Cursor::new(&data);
|
|
274
|
+
let msg = parse(&mut cursor, 8, 8, data.len()).unwrap();
|
|
275
|
+
assert_eq!(msg.name, "abc");
|
|
276
|
+
assert_eq!(msg.dataspace.dataspace_type, DataspaceType::Scalar);
|
|
277
|
+
assert_eq!(msg.raw_data, 99u32.to_le_bytes());
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
#[test]
|
|
281
|
+
fn parse_v3_utf8_name_attr() {
|
|
282
|
+
let dt = u32_le_datatype();
|
|
283
|
+
let ds = scalar_dataspace();
|
|
284
|
+
|
|
285
|
+
let mut data = vec![
|
|
286
|
+
0x03, // version 3
|
|
287
|
+
0x00, // flags
|
|
288
|
+
];
|
|
289
|
+
data.extend_from_slice(&2u16.to_le_bytes()); // "x\0"
|
|
290
|
+
data.extend_from_slice(&(dt.len() as u16).to_le_bytes());
|
|
291
|
+
data.extend_from_slice(&(ds.len() as u16).to_le_bytes());
|
|
292
|
+
data.push(0x01); // UTF-8 name encoding
|
|
293
|
+
data.extend_from_slice(b"x\0");
|
|
294
|
+
data.extend_from_slice(&dt);
|
|
295
|
+
data.extend_from_slice(&ds);
|
|
296
|
+
data.extend_from_slice(&7u32.to_le_bytes());
|
|
297
|
+
|
|
298
|
+
let mut cursor = Cursor::new(&data);
|
|
299
|
+
let msg = parse(&mut cursor, 8, 8, data.len()).unwrap();
|
|
300
|
+
assert_eq!(msg.name, "x");
|
|
301
|
+
assert_eq!(msg.dataspace.dataspace_type, DataspaceType::Scalar);
|
|
302
|
+
assert_eq!(msg.raw_data, 7u32.to_le_bytes());
|
|
303
|
+
}
|
|
304
|
+
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
//! HDF5 Attribute Info message (type 0x0015).
|
|
2
|
+
//!
|
|
3
|
+
//! Provides information about how attributes are stored on an object:
|
|
4
|
+
//! compactly (inline) or densely (in a fractal heap + v2 B-tree).
|
|
5
|
+
|
|
6
|
+
use crate::error::{Error, Result};
|
|
7
|
+
use crate::io::Cursor;
|
|
8
|
+
|
|
9
|
+
/// Parsed attribute info message.
|
|
10
|
+
#[derive(Debug, Clone)]
|
|
11
|
+
pub struct AttributeInfoMessage {
|
|
12
|
+
/// Whether attribute creation order is tracked.
|
|
13
|
+
pub creation_order_tracked: bool,
|
|
14
|
+
/// Whether attribute creation order is indexed.
|
|
15
|
+
pub creation_order_indexed: bool,
|
|
16
|
+
/// Maximum creation order index, if tracked.
|
|
17
|
+
pub max_creation_index: Option<u64>,
|
|
18
|
+
/// Address of the fractal heap for attribute data.
|
|
19
|
+
pub fractal_heap_address: u64,
|
|
20
|
+
/// Address of the v2 B-tree for name-indexed lookups.
|
|
21
|
+
pub btree_name_index_address: u64,
|
|
22
|
+
/// Address of the v2 B-tree for creation-order lookups, if indexed.
|
|
23
|
+
pub btree_creation_order_address: Option<u64>,
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/// Parse an attribute info message.
|
|
27
|
+
pub fn parse(
|
|
28
|
+
cursor: &mut Cursor<'_>,
|
|
29
|
+
offset_size: u8,
|
|
30
|
+
_length_size: u8,
|
|
31
|
+
msg_size: usize,
|
|
32
|
+
) -> Result<AttributeInfoMessage> {
|
|
33
|
+
let start = cursor.position();
|
|
34
|
+
let version = cursor.read_u8()?;
|
|
35
|
+
|
|
36
|
+
if version != 0 {
|
|
37
|
+
return Err(Error::InvalidData(format!(
|
|
38
|
+
"unsupported attribute info version: {}",
|
|
39
|
+
version
|
|
40
|
+
)));
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
let flags = cursor.read_u8()?;
|
|
44
|
+
let creation_order_tracked = (flags & 0x01) != 0;
|
|
45
|
+
let creation_order_indexed = (flags & 0x02) != 0;
|
|
46
|
+
|
|
47
|
+
let max_creation_index = if creation_order_tracked {
|
|
48
|
+
Some(cursor.read_u16_le()? as u64)
|
|
49
|
+
} else {
|
|
50
|
+
None
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
let fractal_heap_address = cursor.read_offset(offset_size)?;
|
|
54
|
+
let btree_name_index_address = cursor.read_offset(offset_size)?;
|
|
55
|
+
|
|
56
|
+
let btree_creation_order_address = if creation_order_indexed {
|
|
57
|
+
Some(cursor.read_offset(offset_size)?)
|
|
58
|
+
} else {
|
|
59
|
+
None
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
let consumed = (cursor.position() - start) as usize;
|
|
63
|
+
if consumed < msg_size {
|
|
64
|
+
cursor.skip(msg_size - consumed)?;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
Ok(AttributeInfoMessage {
|
|
68
|
+
creation_order_tracked,
|
|
69
|
+
creation_order_indexed,
|
|
70
|
+
max_creation_index,
|
|
71
|
+
fractal_heap_address,
|
|
72
|
+
btree_name_index_address,
|
|
73
|
+
btree_creation_order_address,
|
|
74
|
+
})
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
#[cfg(test)]
|
|
78
|
+
mod tests {
|
|
79
|
+
use super::*;
|
|
80
|
+
|
|
81
|
+
#[test]
|
|
82
|
+
fn parse_attr_info_simple() {
|
|
83
|
+
let mut data = vec![
|
|
84
|
+
0x00, // version
|
|
85
|
+
0x00, // flags
|
|
86
|
+
];
|
|
87
|
+
data.extend_from_slice(&0xF000u64.to_le_bytes());
|
|
88
|
+
data.extend_from_slice(&0xF100u64.to_le_bytes());
|
|
89
|
+
|
|
90
|
+
let mut cursor = Cursor::new(&data);
|
|
91
|
+
let msg = parse(&mut cursor, 8, 8, data.len()).unwrap();
|
|
92
|
+
assert!(!msg.creation_order_tracked);
|
|
93
|
+
assert_eq!(msg.fractal_heap_address, 0xF000);
|
|
94
|
+
assert_eq!(msg.btree_name_index_address, 0xF100);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
//! HDF5 B-tree 'K' Values message (type 0x0013).
|
|
2
|
+
//!
|
|
3
|
+
//! Found in superblock extension objects, this message provides the 'K'
|
|
4
|
+
//! (branching factor) value for indexed storage internal nodes.
|
|
5
|
+
|
|
6
|
+
use crate::error::Result;
|
|
7
|
+
use crate::io::Cursor;
|
|
8
|
+
|
|
9
|
+
/// Parsed B-tree K values message.
|
|
10
|
+
#[derive(Debug, Clone)]
|
|
11
|
+
pub struct BTreeKMessage {
|
|
12
|
+
/// Internal node K value for indexed (chunked) storage B-trees.
|
|
13
|
+
pub indexed_storage_internal_k: u16,
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/// Parse a B-tree K values message.
|
|
17
|
+
pub fn parse(
|
|
18
|
+
cursor: &mut Cursor<'_>,
|
|
19
|
+
_offset_size: u8,
|
|
20
|
+
_length_size: u8,
|
|
21
|
+
msg_size: usize,
|
|
22
|
+
) -> Result<BTreeKMessage> {
|
|
23
|
+
let start = cursor.position();
|
|
24
|
+
|
|
25
|
+
let _version = cursor.read_u8()?;
|
|
26
|
+
let indexed_storage_internal_k = cursor.read_u16_le()?;
|
|
27
|
+
|
|
28
|
+
let consumed = (cursor.position() - start) as usize;
|
|
29
|
+
if consumed < msg_size {
|
|
30
|
+
cursor.skip(msg_size - consumed)?;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
Ok(BTreeKMessage {
|
|
34
|
+
indexed_storage_internal_k,
|
|
35
|
+
})
|
|
36
|
+
}
|