@nirs4all/formats-wasm 0.2.11 → 0.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD_PARTY_NOTICES.md +49 -0
- package/nirs4all_formats_wasm_bg.wasm +0 -0
- package/package.json +4 -2
- package/reader-backports/hdf5-reader/Cargo.toml +119 -0
- package/reader-backports/hdf5-reader/LICENSE-APACHE +164 -0
- package/reader-backports/hdf5-reader/LICENSE-MIT +21 -0
- package/reader-backports/hdf5-reader/PROVENANCE.json +80 -0
- package/reader-backports/hdf5-reader/README.md +308 -0
- package/reader-backports/hdf5-reader/UPSTREAM-Cargo.toml +44 -0
- package/reader-backports/hdf5-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/hdf5-reader/src/attribute_api.rs +847 -0
- package/reader-backports/hdf5-reader/src/btree_v1.rs +726 -0
- package/reader-backports/hdf5-reader/src/btree_v2.rs +2068 -0
- package/reader-backports/hdf5-reader/src/cache.rs +354 -0
- package/reader-backports/hdf5-reader/src/checksum.rs +56 -0
- package/reader-backports/hdf5-reader/src/chunk_index.rs +423 -0
- package/reader-backports/hdf5-reader/src/dataset.rs +4733 -0
- package/reader-backports/hdf5-reader/src/datatype_api.rs +497 -0
- package/reader-backports/hdf5-reader/src/error.rs +167 -0
- package/reader-backports/hdf5-reader/src/extensible_array.rs +1886 -0
- package/reader-backports/hdf5-reader/src/filters/deflate.rs +93 -0
- package/reader-backports/hdf5-reader/src/filters/fletcher32.rs +52 -0
- package/reader-backports/hdf5-reader/src/filters/lz4.rs +178 -0
- package/reader-backports/hdf5-reader/src/filters/mod.rs +326 -0
- package/reader-backports/hdf5-reader/src/filters/nbit.rs +586 -0
- package/reader-backports/hdf5-reader/src/filters/scaleoffset.rs +827 -0
- package/reader-backports/hdf5-reader/src/filters/shuffle.rs +99 -0
- package/reader-backports/hdf5-reader/src/fixed_array.rs +763 -0
- package/reader-backports/hdf5-reader/src/fractal_heap.rs +2029 -0
- package/reader-backports/hdf5-reader/src/global_heap.rs +284 -0
- package/reader-backports/hdf5-reader/src/group.rs +795 -0
- package/reader-backports/hdf5-reader/src/io.rs +413 -0
- package/reader-backports/hdf5-reader/src/lib.rs +916 -0
- package/reader-backports/hdf5-reader/src/local_heap.rs +261 -0
- package/reader-backports/hdf5-reader/src/messages/attribute.rs +304 -0
- package/reader-backports/hdf5-reader/src/messages/attribute_info.rs +96 -0
- package/reader-backports/hdf5-reader/src/messages/btree_k.rs +36 -0
- package/reader-backports/hdf5-reader/src/messages/continuation.rs +46 -0
- package/reader-backports/hdf5-reader/src/messages/dataspace.rs +259 -0
- package/reader-backports/hdf5-reader/src/messages/datatype.rs +938 -0
- package/reader-backports/hdf5-reader/src/messages/external_files.rs +50 -0
- package/reader-backports/hdf5-reader/src/messages/fill_value.rs +233 -0
- package/reader-backports/hdf5-reader/src/messages/filter_pipeline.rs +147 -0
- package/reader-backports/hdf5-reader/src/messages/group_info.rs +103 -0
- package/reader-backports/hdf5-reader/src/messages/layout.rs +476 -0
- package/reader-backports/hdf5-reader/src/messages/link.rs +170 -0
- package/reader-backports/hdf5-reader/src/messages/link_info.rs +127 -0
- package/reader-backports/hdf5-reader/src/messages/mod.rs +374 -0
- package/reader-backports/hdf5-reader/src/messages/modification_time.rs +83 -0
- package/reader-backports/hdf5-reader/src/messages/shared.rs +164 -0
- package/reader-backports/hdf5-reader/src/messages/shared_table.rs +67 -0
- package/reader-backports/hdf5-reader/src/messages/symbol_table_msg.rs +51 -0
- package/reader-backports/hdf5-reader/src/object_header.rs +1749 -0
- package/reader-backports/hdf5-reader/src/reference.rs +90 -0
- package/reader-backports/hdf5-reader/src/shared_message_table.rs +425 -0
- package/reader-backports/hdf5-reader/src/storage.rs +538 -0
- package/reader-backports/hdf5-reader/src/superblock.rs +285 -0
- package/reader-backports/hdf5-reader/src/symbol_table.rs +277 -0
- package/reader-backports/netcdf-reader/Cargo.toml +141 -0
- package/reader-backports/netcdf-reader/LICENSE-APACHE +164 -0
- package/reader-backports/netcdf-reader/LICENSE-MIT +21 -0
- package/reader-backports/netcdf-reader/PROVENANCE.json +55 -0
- package/reader-backports/netcdf-reader/README.md +308 -0
- package/reader-backports/netcdf-reader/UPSTREAM-Cargo.toml +48 -0
- package/reader-backports/netcdf-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/netcdf-reader/src/cf/axes.rs +419 -0
- package/reader-backports/netcdf-reader/src/cf/bounds.rs +108 -0
- package/reader-backports/netcdf-reader/src/cf/crs.rs +269 -0
- package/reader-backports/netcdf-reader/src/cf/mod.rs +30 -0
- package/reader-backports/netcdf-reader/src/cf/time.rs +1122 -0
- package/reader-backports/netcdf-reader/src/classic/data.rs +1465 -0
- package/reader-backports/netcdf-reader/src/classic/header.rs +1097 -0
- package/reader-backports/netcdf-reader/src/classic/mod.rs +240 -0
- package/reader-backports/netcdf-reader/src/classic/storage.rs +211 -0
- package/reader-backports/netcdf-reader/src/classic/types.rs +93 -0
- package/reader-backports/netcdf-reader/src/classic/variable.rs +1162 -0
- package/reader-backports/netcdf-reader/src/error.rs +59 -0
- package/reader-backports/netcdf-reader/src/lib.rs +1531 -0
- package/reader-backports/netcdf-reader/src/masked.rs +173 -0
- package/reader-backports/netcdf-reader/src/nc4/attributes.rs +203 -0
- package/reader-backports/netcdf-reader/src/nc4/dimensions.rs +139 -0
- package/reader-backports/netcdf-reader/src/nc4/groups.rs +308 -0
- package/reader-backports/netcdf-reader/src/nc4/mod.rs +572 -0
- package/reader-backports/netcdf-reader/src/nc4/types.rs +309 -0
- package/reader-backports/netcdf-reader/src/nc4/variables.rs +420 -0
- package/reader-backports/netcdf-reader/src/types.rs +18 -0
- package/reader-backports/netcdf-reader/src/unpack.rs +113 -0
- package/reader-backports/netcdf-reader/src/user_defined.rs +569 -0
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
//! HDF5 Header Continuation message (type 0x0010).
|
|
2
|
+
//!
|
|
3
|
+
//! Indicates that additional header messages are stored at another location
|
|
4
|
+
//! in the file. The parser follows these to read the complete object header.
|
|
5
|
+
|
|
6
|
+
use crate::error::Result;
|
|
7
|
+
use crate::io::Cursor;
|
|
8
|
+
|
|
9
|
+
/// Parsed continuation message.
|
|
10
|
+
#[derive(Debug, Clone)]
|
|
11
|
+
pub struct ContinuationMessage {
|
|
12
|
+
/// Absolute file offset of the continuation block.
|
|
13
|
+
pub offset: u64,
|
|
14
|
+
/// Length in bytes of the continuation block.
|
|
15
|
+
pub length: u64,
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/// Parse a header continuation message.
|
|
19
|
+
pub fn parse(
|
|
20
|
+
cursor: &mut Cursor<'_>,
|
|
21
|
+
offset_size: u8,
|
|
22
|
+
length_size: u8,
|
|
23
|
+
_msg_size: usize,
|
|
24
|
+
) -> Result<ContinuationMessage> {
|
|
25
|
+
let offset = cursor.read_offset(offset_size)?;
|
|
26
|
+
let length = cursor.read_length(length_size)?;
|
|
27
|
+
|
|
28
|
+
Ok(ContinuationMessage { offset, length })
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
#[cfg(test)]
|
|
32
|
+
mod tests {
|
|
33
|
+
use super::*;
|
|
34
|
+
|
|
35
|
+
#[test]
|
|
36
|
+
fn parse_continuation() {
|
|
37
|
+
let mut data = Vec::new();
|
|
38
|
+
data.extend_from_slice(&0x4000u64.to_le_bytes());
|
|
39
|
+
data.extend_from_slice(&256u64.to_le_bytes());
|
|
40
|
+
|
|
41
|
+
let mut cursor = Cursor::new(&data);
|
|
42
|
+
let msg = parse(&mut cursor, 8, 8, data.len()).unwrap();
|
|
43
|
+
assert_eq!(msg.offset, 0x4000);
|
|
44
|
+
assert_eq!(msg.length, 256);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
//! HDF5 Dataspace message (type 0x0001).
|
|
2
|
+
//!
|
|
3
|
+
//! A dataspace describes the shape of a dataset: scalar, null, or simple
|
|
4
|
+
//! (one or more dimensions with current and optional maximum sizes).
|
|
5
|
+
|
|
6
|
+
use crate::error::{Error, Result};
|
|
7
|
+
use crate::io::Cursor;
|
|
8
|
+
pub use hdf5_core::{DataspaceMessage, DataspaceType, UNLIMITED};
|
|
9
|
+
|
|
10
|
+
/// Parse a dataspace message.
|
|
11
|
+
///
|
|
12
|
+
/// `length_size` is needed for version 1 where dimensions are stored using
|
|
13
|
+
/// the file-global length size.
|
|
14
|
+
pub fn parse(
|
|
15
|
+
cursor: &mut Cursor<'_>,
|
|
16
|
+
_offset_size: u8,
|
|
17
|
+
length_size: u8,
|
|
18
|
+
msg_size: usize,
|
|
19
|
+
) -> Result<DataspaceMessage> {
|
|
20
|
+
let start = cursor.position();
|
|
21
|
+
let version = cursor.read_u8()?;
|
|
22
|
+
|
|
23
|
+
match version {
|
|
24
|
+
1 => parse_v1(cursor, length_size),
|
|
25
|
+
2 => parse_v2(cursor, length_size),
|
|
26
|
+
v => Err(Error::UnsupportedDataspaceVersion(v)),
|
|
27
|
+
}
|
|
28
|
+
.and_then(|msg| {
|
|
29
|
+
// Advance past any remaining bytes in the message
|
|
30
|
+
let consumed = (cursor.position() - start) as usize;
|
|
31
|
+
if consumed < msg_size {
|
|
32
|
+
cursor.skip(msg_size - consumed)?;
|
|
33
|
+
}
|
|
34
|
+
Ok(msg)
|
|
35
|
+
})
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/// Version 1 dataspace message.
|
|
39
|
+
fn parse_v1(cursor: &mut Cursor<'_>, length_size: u8) -> Result<DataspaceMessage> {
|
|
40
|
+
let rank = cursor.read_u8()?;
|
|
41
|
+
let flags = cursor.read_u8()?;
|
|
42
|
+
let _reserved = cursor.read_u8()?; // reserved
|
|
43
|
+
let _reserved2 = cursor.read_u32_le()?; // reserved
|
|
44
|
+
|
|
45
|
+
let has_max_dims = (flags & 0x01) != 0;
|
|
46
|
+
// Bit 1 was "permutation index present" in v1 but is never actually set
|
|
47
|
+
// in practice. We skip it if the flag is set.
|
|
48
|
+
let has_permutation = (flags & 0x02) != 0;
|
|
49
|
+
|
|
50
|
+
let dataspace_type = if rank == 0 {
|
|
51
|
+
DataspaceType::Scalar
|
|
52
|
+
} else {
|
|
53
|
+
DataspaceType::Simple
|
|
54
|
+
};
|
|
55
|
+
|
|
56
|
+
let mut dims = Vec::with_capacity(rank as usize);
|
|
57
|
+
for _ in 0..rank {
|
|
58
|
+
dims.push(cursor.read_length(length_size)?);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
let max_dims = if has_max_dims {
|
|
62
|
+
let mut md = Vec::with_capacity(rank as usize);
|
|
63
|
+
for _ in 0..rank {
|
|
64
|
+
md.push(cursor.read_length(length_size)?);
|
|
65
|
+
}
|
|
66
|
+
Some(md)
|
|
67
|
+
} else {
|
|
68
|
+
None
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
if has_permutation {
|
|
72
|
+
// Skip permutation indices — each is `length_size` bytes.
|
|
73
|
+
for _ in 0..rank {
|
|
74
|
+
cursor.read_length(length_size)?;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
Ok(DataspaceMessage {
|
|
79
|
+
rank,
|
|
80
|
+
dims,
|
|
81
|
+
max_dims,
|
|
82
|
+
dataspace_type,
|
|
83
|
+
})
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/// Version 2 dataspace message.
|
|
87
|
+
fn parse_v2(cursor: &mut Cursor<'_>, length_size: u8) -> Result<DataspaceMessage> {
|
|
88
|
+
let rank = cursor.read_u8()?;
|
|
89
|
+
let flags = cursor.read_u8()?;
|
|
90
|
+
let ds_type_byte = cursor.read_u8()?;
|
|
91
|
+
|
|
92
|
+
let has_max_dims = (flags & 0x01) != 0;
|
|
93
|
+
|
|
94
|
+
let dataspace_type = match ds_type_byte {
|
|
95
|
+
0 => DataspaceType::Scalar,
|
|
96
|
+
1 => DataspaceType::Simple,
|
|
97
|
+
2 => DataspaceType::Null,
|
|
98
|
+
_ => {
|
|
99
|
+
return Err(Error::InvalidData(format!(
|
|
100
|
+
"unknown dataspace type: {}",
|
|
101
|
+
ds_type_byte
|
|
102
|
+
)))
|
|
103
|
+
}
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
let mut dims = Vec::with_capacity(rank as usize);
|
|
107
|
+
for _ in 0..rank {
|
|
108
|
+
dims.push(cursor.read_length(length_size)?);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
let max_dims = if has_max_dims {
|
|
112
|
+
let mut md = Vec::with_capacity(rank as usize);
|
|
113
|
+
for _ in 0..rank {
|
|
114
|
+
md.push(cursor.read_length(length_size)?);
|
|
115
|
+
}
|
|
116
|
+
Some(md)
|
|
117
|
+
} else {
|
|
118
|
+
None
|
|
119
|
+
};
|
|
120
|
+
|
|
121
|
+
Ok(DataspaceMessage {
|
|
122
|
+
rank,
|
|
123
|
+
dims,
|
|
124
|
+
max_dims,
|
|
125
|
+
dataspace_type,
|
|
126
|
+
})
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
#[cfg(test)]
|
|
130
|
+
mod tests {
|
|
131
|
+
use super::*;
|
|
132
|
+
|
|
133
|
+
#[test]
|
|
134
|
+
fn parse_v1_scalar() {
|
|
135
|
+
// Version 1, rank=0 (scalar), flags=0, reserved bytes
|
|
136
|
+
let data = [
|
|
137
|
+
0x01, // version
|
|
138
|
+
0x00, // rank
|
|
139
|
+
0x00, // flags
|
|
140
|
+
0x00, // reserved
|
|
141
|
+
0x00, 0x00, 0x00, 0x00, // reserved u32
|
|
142
|
+
];
|
|
143
|
+
let mut cursor = Cursor::new(&data);
|
|
144
|
+
let msg = parse(&mut cursor, 8, 8, data.len()).unwrap();
|
|
145
|
+
assert_eq!(msg.rank, 0);
|
|
146
|
+
assert_eq!(msg.dataspace_type, DataspaceType::Scalar);
|
|
147
|
+
assert!(msg.dims.is_empty());
|
|
148
|
+
assert!(msg.max_dims.is_none());
|
|
149
|
+
assert_eq!(msg.num_elements().unwrap(), 1);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
#[test]
|
|
153
|
+
fn parse_v1_simple_2d() {
|
|
154
|
+
// Version 1, rank=2, flags=0x01 (has max dims), 8-byte lengths
|
|
155
|
+
let mut data = vec![
|
|
156
|
+
0x01, // version
|
|
157
|
+
0x02, // rank
|
|
158
|
+
0x01, // flags (has max dims)
|
|
159
|
+
0x00, // reserved
|
|
160
|
+
0x00, 0x00, 0x00, 0x00, // reserved u32
|
|
161
|
+
];
|
|
162
|
+
// dim[0] = 10
|
|
163
|
+
data.extend_from_slice(&10u64.to_le_bytes());
|
|
164
|
+
// dim[1] = 20
|
|
165
|
+
data.extend_from_slice(&20u64.to_le_bytes());
|
|
166
|
+
// max_dim[0] = 100
|
|
167
|
+
data.extend_from_slice(&100u64.to_le_bytes());
|
|
168
|
+
// max_dim[1] = UNLIMITED
|
|
169
|
+
data.extend_from_slice(&u64::MAX.to_le_bytes());
|
|
170
|
+
|
|
171
|
+
let mut cursor = Cursor::new(&data);
|
|
172
|
+
let msg = parse(&mut cursor, 8, 8, data.len()).unwrap();
|
|
173
|
+
assert_eq!(msg.rank, 2);
|
|
174
|
+
assert_eq!(msg.dims, vec![10, 20]);
|
|
175
|
+
assert_eq!(msg.max_dims.as_ref().unwrap(), &vec![100, UNLIMITED]);
|
|
176
|
+
assert_eq!(msg.dataspace_type, DataspaceType::Simple);
|
|
177
|
+
assert_eq!(msg.num_elements().unwrap(), 200);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
#[test]
|
|
181
|
+
fn parse_v2_simple_1d() {
|
|
182
|
+
// Version 2, rank=1, flags=0x00, type=1 (simple), 4-byte lengths
|
|
183
|
+
let mut data = vec![
|
|
184
|
+
0x02, // version
|
|
185
|
+
0x01, // rank
|
|
186
|
+
0x00, // flags
|
|
187
|
+
0x01, // type = simple
|
|
188
|
+
];
|
|
189
|
+
// dim[0] = 42
|
|
190
|
+
data.extend_from_slice(&42u32.to_le_bytes());
|
|
191
|
+
|
|
192
|
+
let mut cursor = Cursor::new(&data);
|
|
193
|
+
let msg = parse(&mut cursor, 4, 4, data.len()).unwrap();
|
|
194
|
+
assert_eq!(msg.rank, 1);
|
|
195
|
+
assert_eq!(msg.dims, vec![42]);
|
|
196
|
+
assert!(msg.max_dims.is_none());
|
|
197
|
+
assert_eq!(msg.dataspace_type, DataspaceType::Simple);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
#[test]
|
|
201
|
+
fn parse_v2_null() {
|
|
202
|
+
let data = [
|
|
203
|
+
0x02, // version
|
|
204
|
+
0x00, // rank
|
|
205
|
+
0x00, // flags
|
|
206
|
+
0x02, // type = null
|
|
207
|
+
];
|
|
208
|
+
let mut cursor = Cursor::new(&data);
|
|
209
|
+
let msg = parse(&mut cursor, 8, 8, data.len()).unwrap();
|
|
210
|
+
assert_eq!(msg.dataspace_type, DataspaceType::Null);
|
|
211
|
+
assert_eq!(msg.num_elements().unwrap(), 0);
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
#[test]
|
|
215
|
+
fn parse_v2_with_max_dims() {
|
|
216
|
+
let mut data = vec![
|
|
217
|
+
0x02, // version
|
|
218
|
+
0x03, // rank = 3
|
|
219
|
+
0x01, // flags = has max dims
|
|
220
|
+
0x01, // type = simple
|
|
221
|
+
];
|
|
222
|
+
// current dims: 5, 10, 15
|
|
223
|
+
for &d in &[5u64, 10, 15] {
|
|
224
|
+
data.extend_from_slice(&d.to_le_bytes());
|
|
225
|
+
}
|
|
226
|
+
// max dims: 50, 100, UNLIMITED
|
|
227
|
+
for &d in &[50u64, 100, u64::MAX] {
|
|
228
|
+
data.extend_from_slice(&d.to_le_bytes());
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
let mut cursor = Cursor::new(&data);
|
|
232
|
+
let msg = parse(&mut cursor, 8, 8, data.len()).unwrap();
|
|
233
|
+
assert_eq!(msg.rank, 3);
|
|
234
|
+
assert_eq!(msg.dims, vec![5, 10, 15]);
|
|
235
|
+
let md = msg.max_dims.clone().unwrap();
|
|
236
|
+
assert_eq!(md, vec![50, 100, UNLIMITED]);
|
|
237
|
+
assert_eq!(msg.num_elements().unwrap(), 750);
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
#[test]
|
|
241
|
+
fn unsupported_version() {
|
|
242
|
+
let data = [0x03, 0x00, 0x00, 0x00];
|
|
243
|
+
let mut cursor = Cursor::new(&data);
|
|
244
|
+
assert!(parse(&mut cursor, 8, 8, data.len()).is_err());
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
#[test]
|
|
248
|
+
fn num_elements_rejects_overflow() {
|
|
249
|
+
let msg = DataspaceMessage {
|
|
250
|
+
rank: 2,
|
|
251
|
+
dims: vec![u64::MAX, 2],
|
|
252
|
+
max_dims: None,
|
|
253
|
+
dataspace_type: DataspaceType::Simple,
|
|
254
|
+
};
|
|
255
|
+
|
|
256
|
+
let err = msg.num_elements().unwrap_err();
|
|
257
|
+
assert!(err.to_string().contains("element count"));
|
|
258
|
+
}
|
|
259
|
+
}
|