@nirs4all/formats-wasm 0.2.11 → 0.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD_PARTY_NOTICES.md +49 -0
- package/nirs4all_formats_wasm_bg.wasm +0 -0
- package/package.json +4 -2
- package/reader-backports/hdf5-reader/Cargo.toml +119 -0
- package/reader-backports/hdf5-reader/LICENSE-APACHE +164 -0
- package/reader-backports/hdf5-reader/LICENSE-MIT +21 -0
- package/reader-backports/hdf5-reader/PROVENANCE.json +80 -0
- package/reader-backports/hdf5-reader/README.md +308 -0
- package/reader-backports/hdf5-reader/UPSTREAM-Cargo.toml +44 -0
- package/reader-backports/hdf5-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/hdf5-reader/src/attribute_api.rs +847 -0
- package/reader-backports/hdf5-reader/src/btree_v1.rs +726 -0
- package/reader-backports/hdf5-reader/src/btree_v2.rs +2068 -0
- package/reader-backports/hdf5-reader/src/cache.rs +354 -0
- package/reader-backports/hdf5-reader/src/checksum.rs +56 -0
- package/reader-backports/hdf5-reader/src/chunk_index.rs +423 -0
- package/reader-backports/hdf5-reader/src/dataset.rs +4733 -0
- package/reader-backports/hdf5-reader/src/datatype_api.rs +497 -0
- package/reader-backports/hdf5-reader/src/error.rs +167 -0
- package/reader-backports/hdf5-reader/src/extensible_array.rs +1886 -0
- package/reader-backports/hdf5-reader/src/filters/deflate.rs +93 -0
- package/reader-backports/hdf5-reader/src/filters/fletcher32.rs +52 -0
- package/reader-backports/hdf5-reader/src/filters/lz4.rs +178 -0
- package/reader-backports/hdf5-reader/src/filters/mod.rs +326 -0
- package/reader-backports/hdf5-reader/src/filters/nbit.rs +586 -0
- package/reader-backports/hdf5-reader/src/filters/scaleoffset.rs +827 -0
- package/reader-backports/hdf5-reader/src/filters/shuffle.rs +99 -0
- package/reader-backports/hdf5-reader/src/fixed_array.rs +763 -0
- package/reader-backports/hdf5-reader/src/fractal_heap.rs +2029 -0
- package/reader-backports/hdf5-reader/src/global_heap.rs +284 -0
- package/reader-backports/hdf5-reader/src/group.rs +795 -0
- package/reader-backports/hdf5-reader/src/io.rs +413 -0
- package/reader-backports/hdf5-reader/src/lib.rs +916 -0
- package/reader-backports/hdf5-reader/src/local_heap.rs +261 -0
- package/reader-backports/hdf5-reader/src/messages/attribute.rs +304 -0
- package/reader-backports/hdf5-reader/src/messages/attribute_info.rs +96 -0
- package/reader-backports/hdf5-reader/src/messages/btree_k.rs +36 -0
- package/reader-backports/hdf5-reader/src/messages/continuation.rs +46 -0
- package/reader-backports/hdf5-reader/src/messages/dataspace.rs +259 -0
- package/reader-backports/hdf5-reader/src/messages/datatype.rs +938 -0
- package/reader-backports/hdf5-reader/src/messages/external_files.rs +50 -0
- package/reader-backports/hdf5-reader/src/messages/fill_value.rs +233 -0
- package/reader-backports/hdf5-reader/src/messages/filter_pipeline.rs +147 -0
- package/reader-backports/hdf5-reader/src/messages/group_info.rs +103 -0
- package/reader-backports/hdf5-reader/src/messages/layout.rs +476 -0
- package/reader-backports/hdf5-reader/src/messages/link.rs +170 -0
- package/reader-backports/hdf5-reader/src/messages/link_info.rs +127 -0
- package/reader-backports/hdf5-reader/src/messages/mod.rs +374 -0
- package/reader-backports/hdf5-reader/src/messages/modification_time.rs +83 -0
- package/reader-backports/hdf5-reader/src/messages/shared.rs +164 -0
- package/reader-backports/hdf5-reader/src/messages/shared_table.rs +67 -0
- package/reader-backports/hdf5-reader/src/messages/symbol_table_msg.rs +51 -0
- package/reader-backports/hdf5-reader/src/object_header.rs +1749 -0
- package/reader-backports/hdf5-reader/src/reference.rs +90 -0
- package/reader-backports/hdf5-reader/src/shared_message_table.rs +425 -0
- package/reader-backports/hdf5-reader/src/storage.rs +538 -0
- package/reader-backports/hdf5-reader/src/superblock.rs +285 -0
- package/reader-backports/hdf5-reader/src/symbol_table.rs +277 -0
- package/reader-backports/netcdf-reader/Cargo.toml +141 -0
- package/reader-backports/netcdf-reader/LICENSE-APACHE +164 -0
- package/reader-backports/netcdf-reader/LICENSE-MIT +21 -0
- package/reader-backports/netcdf-reader/PROVENANCE.json +55 -0
- package/reader-backports/netcdf-reader/README.md +308 -0
- package/reader-backports/netcdf-reader/UPSTREAM-Cargo.toml +48 -0
- package/reader-backports/netcdf-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/netcdf-reader/src/cf/axes.rs +419 -0
- package/reader-backports/netcdf-reader/src/cf/bounds.rs +108 -0
- package/reader-backports/netcdf-reader/src/cf/crs.rs +269 -0
- package/reader-backports/netcdf-reader/src/cf/mod.rs +30 -0
- package/reader-backports/netcdf-reader/src/cf/time.rs +1122 -0
- package/reader-backports/netcdf-reader/src/classic/data.rs +1465 -0
- package/reader-backports/netcdf-reader/src/classic/header.rs +1097 -0
- package/reader-backports/netcdf-reader/src/classic/mod.rs +240 -0
- package/reader-backports/netcdf-reader/src/classic/storage.rs +211 -0
- package/reader-backports/netcdf-reader/src/classic/types.rs +93 -0
- package/reader-backports/netcdf-reader/src/classic/variable.rs +1162 -0
- package/reader-backports/netcdf-reader/src/error.rs +59 -0
- package/reader-backports/netcdf-reader/src/lib.rs +1531 -0
- package/reader-backports/netcdf-reader/src/masked.rs +173 -0
- package/reader-backports/netcdf-reader/src/nc4/attributes.rs +203 -0
- package/reader-backports/netcdf-reader/src/nc4/dimensions.rs +139 -0
- package/reader-backports/netcdf-reader/src/nc4/groups.rs +308 -0
- package/reader-backports/netcdf-reader/src/nc4/mod.rs +572 -0
- package/reader-backports/netcdf-reader/src/nc4/types.rs +309 -0
- package/reader-backports/netcdf-reader/src/nc4/variables.rs +420 -0
- package/reader-backports/netcdf-reader/src/types.rs +18 -0
- package/reader-backports/netcdf-reader/src/unpack.rs +113 -0
- package/reader-backports/netcdf-reader/src/user_defined.rs +569 -0
|
@@ -0,0 +1,1097 @@
|
|
|
1
|
+
//! Parse the NetCDF classic (CDF-1/2/5) binary header.
|
|
2
|
+
//!
|
|
3
|
+
//! The classic header is a sequence of big-endian fields describing dimensions,
|
|
4
|
+
//! global attributes, and variables. All multi-byte integers are big-endian.
|
|
5
|
+
//! Strings are padded to 4-byte alignment. CDF-5 uses 8-byte counts and sizes
|
|
6
|
+
//! where CDF-1/2 use 4-byte values.
|
|
7
|
+
|
|
8
|
+
use crate::error::{Error, Result};
|
|
9
|
+
use crate::types::{
|
|
10
|
+
checked_mul_u64, checked_usize_from_u64, NcAttrValue, NcAttribute, NcDimension, NcType,
|
|
11
|
+
NcVariable,
|
|
12
|
+
};
|
|
13
|
+
use crate::NcFormat;
|
|
14
|
+
|
|
15
|
+
#[cfg(test)]
|
|
16
|
+
use super::types::pad_to_4;
|
|
17
|
+
use super::types::{nc_type_from_code, padding_to_4};
|
|
18
|
+
|
|
19
|
+
// Header tag constants.
|
|
20
|
+
const ABSENT: u32 = 0x0000_0000;
|
|
21
|
+
const NC_DIMENSION: u32 = 0x0000_000A;
|
|
22
|
+
const NC_VARIABLE: u32 = 0x0000_000B;
|
|
23
|
+
const NC_ATTRIBUTE: u32 = 0x0000_000C;
|
|
24
|
+
|
|
25
|
+
/// Streaming (indeterminate) record count sentinel.
|
|
26
|
+
const STREAMING: u32 = 0xFFFF_FFFF;
|
|
27
|
+
|
|
28
|
+
/// Result of parsing a classic NetCDF header.
|
|
29
|
+
pub struct ClassicHeader {
|
|
30
|
+
pub dimensions: Vec<NcDimension>,
|
|
31
|
+
pub global_attributes: Vec<NcAttribute>,
|
|
32
|
+
pub variables: Vec<NcVariable>,
|
|
33
|
+
pub numrecs: u64,
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/// A cursor for reading big-endian data from a byte slice.
|
|
37
|
+
struct Cursor<'a> {
|
|
38
|
+
data: &'a [u8],
|
|
39
|
+
pos: usize,
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
impl<'a> Cursor<'a> {
|
|
43
|
+
fn new(data: &'a [u8]) -> Self {
|
|
44
|
+
Cursor { data, pos: 0 }
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
fn remaining(&self) -> usize {
|
|
48
|
+
self.data.len().saturating_sub(self.pos)
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
fn ensure(&self, n: usize) -> Result<()> {
|
|
52
|
+
if self.remaining() < n {
|
|
53
|
+
Err(Error::UnexpectedEof {
|
|
54
|
+
offset: self.pos as u64,
|
|
55
|
+
needed: n as u64,
|
|
56
|
+
available: self.remaining() as u64,
|
|
57
|
+
})
|
|
58
|
+
} else {
|
|
59
|
+
Ok(())
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
#[allow(dead_code)]
|
|
64
|
+
fn read_u8(&mut self) -> Result<u8> {
|
|
65
|
+
self.ensure(1)?;
|
|
66
|
+
let v = self.data[self.pos];
|
|
67
|
+
self.pos += 1;
|
|
68
|
+
Ok(v)
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
fn read_u16_be(&mut self) -> Result<u16> {
|
|
72
|
+
self.ensure(2)?;
|
|
73
|
+
let v = u16::from_be_bytes([self.data[self.pos], self.data[self.pos + 1]]);
|
|
74
|
+
self.pos += 2;
|
|
75
|
+
Ok(v)
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
fn read_u32_be(&mut self) -> Result<u32> {
|
|
79
|
+
self.ensure(4)?;
|
|
80
|
+
let v = u32::from_be_bytes([
|
|
81
|
+
self.data[self.pos],
|
|
82
|
+
self.data[self.pos + 1],
|
|
83
|
+
self.data[self.pos + 2],
|
|
84
|
+
self.data[self.pos + 3],
|
|
85
|
+
]);
|
|
86
|
+
self.pos += 4;
|
|
87
|
+
Ok(v)
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
fn read_i32_be(&mut self) -> Result<i32> {
|
|
91
|
+
self.ensure(4)?;
|
|
92
|
+
let v = i32::from_be_bytes([
|
|
93
|
+
self.data[self.pos],
|
|
94
|
+
self.data[self.pos + 1],
|
|
95
|
+
self.data[self.pos + 2],
|
|
96
|
+
self.data[self.pos + 3],
|
|
97
|
+
]);
|
|
98
|
+
self.pos += 4;
|
|
99
|
+
Ok(v)
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
fn read_u64_be(&mut self) -> Result<u64> {
|
|
103
|
+
self.ensure(8)?;
|
|
104
|
+
let v = u64::from_be_bytes([
|
|
105
|
+
self.data[self.pos],
|
|
106
|
+
self.data[self.pos + 1],
|
|
107
|
+
self.data[self.pos + 2],
|
|
108
|
+
self.data[self.pos + 3],
|
|
109
|
+
self.data[self.pos + 4],
|
|
110
|
+
self.data[self.pos + 5],
|
|
111
|
+
self.data[self.pos + 6],
|
|
112
|
+
self.data[self.pos + 7],
|
|
113
|
+
]);
|
|
114
|
+
self.pos += 8;
|
|
115
|
+
Ok(v)
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
fn read_i64_be(&mut self) -> Result<i64> {
|
|
119
|
+
self.ensure(8)?;
|
|
120
|
+
let v = i64::from_be_bytes([
|
|
121
|
+
self.data[self.pos],
|
|
122
|
+
self.data[self.pos + 1],
|
|
123
|
+
self.data[self.pos + 2],
|
|
124
|
+
self.data[self.pos + 3],
|
|
125
|
+
self.data[self.pos + 4],
|
|
126
|
+
self.data[self.pos + 5],
|
|
127
|
+
self.data[self.pos + 6],
|
|
128
|
+
self.data[self.pos + 7],
|
|
129
|
+
]);
|
|
130
|
+
self.pos += 8;
|
|
131
|
+
Ok(v)
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
fn read_f32_be(&mut self) -> Result<f32> {
|
|
135
|
+
self.ensure(4)?;
|
|
136
|
+
let v = f32::from_be_bytes([
|
|
137
|
+
self.data[self.pos],
|
|
138
|
+
self.data[self.pos + 1],
|
|
139
|
+
self.data[self.pos + 2],
|
|
140
|
+
self.data[self.pos + 3],
|
|
141
|
+
]);
|
|
142
|
+
self.pos += 4;
|
|
143
|
+
Ok(v)
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
fn read_f64_be(&mut self) -> Result<f64> {
|
|
147
|
+
self.ensure(8)?;
|
|
148
|
+
let v = f64::from_be_bytes([
|
|
149
|
+
self.data[self.pos],
|
|
150
|
+
self.data[self.pos + 1],
|
|
151
|
+
self.data[self.pos + 2],
|
|
152
|
+
self.data[self.pos + 3],
|
|
153
|
+
self.data[self.pos + 4],
|
|
154
|
+
self.data[self.pos + 5],
|
|
155
|
+
self.data[self.pos + 6],
|
|
156
|
+
self.data[self.pos + 7],
|
|
157
|
+
]);
|
|
158
|
+
self.pos += 8;
|
|
159
|
+
Ok(v)
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
fn read_bytes(&mut self, n: usize) -> Result<&'a [u8]> {
|
|
163
|
+
self.ensure(n)?;
|
|
164
|
+
let slice = &self.data[self.pos..self.pos + n];
|
|
165
|
+
self.pos += n;
|
|
166
|
+
Ok(slice)
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
fn skip(&mut self, n: usize) -> Result<()> {
|
|
170
|
+
self.ensure(n)?;
|
|
171
|
+
self.pos += n;
|
|
172
|
+
Ok(())
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/// Read a count field: 4 bytes for CDF-1/2, 8 bytes for CDF-5.
|
|
176
|
+
fn read_count(&mut self, format: NcFormat) -> Result<u64> {
|
|
177
|
+
match format {
|
|
178
|
+
NcFormat::Cdf5 => self.read_u64_be(),
|
|
179
|
+
_ => self.read_u32_be().map(|v| v as u64),
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/// Read a padded name: 4-byte length, then chars, then padding to 4-byte boundary.
|
|
184
|
+
/// The name length prefix is always 4 bytes for CDF-1/2 and 8 bytes for CDF-5.
|
|
185
|
+
fn read_name(&mut self, format: NcFormat) -> Result<String> {
|
|
186
|
+
let len = checked_usize_from_u64(self.read_count(format)?, "classic name length")?;
|
|
187
|
+
let bytes = self.read_bytes(len)?;
|
|
188
|
+
let padded_len = checked_pad_to_4(len, "classic name length")?;
|
|
189
|
+
let pad = padded_len - len;
|
|
190
|
+
if pad > 0 {
|
|
191
|
+
self.skip(pad)?;
|
|
192
|
+
}
|
|
193
|
+
String::from_utf8(bytes.to_vec())
|
|
194
|
+
.map_err(|e| Error::InvalidData(format!("invalid UTF-8 name: {}", e)))
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
fn checked_pad_to_4(len: usize, context: &str) -> Result<usize> {
|
|
199
|
+
len.checked_add(padding_to_4(len)).ok_or_else(|| {
|
|
200
|
+
Error::InvalidData(format!("{context} padded length exceeds platform usize"))
|
|
201
|
+
})
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
fn read_list_count(
|
|
205
|
+
cur: &mut Cursor<'_>,
|
|
206
|
+
format: NcFormat,
|
|
207
|
+
min_bytes_per_entry: u64,
|
|
208
|
+
context: &str,
|
|
209
|
+
) -> Result<usize> {
|
|
210
|
+
let raw = cur.read_count(format)?;
|
|
211
|
+
let count = checked_usize_from_u64(raw, context)?;
|
|
212
|
+
let min_needed = checked_mul_u64(raw, min_bytes_per_entry, context)?;
|
|
213
|
+
if min_needed > cur.remaining() as u64 {
|
|
214
|
+
return Err(Error::UnexpectedEof {
|
|
215
|
+
offset: cur.pos as u64,
|
|
216
|
+
needed: min_needed,
|
|
217
|
+
available: cur.remaining() as u64,
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
Ok(count)
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/// Parse a complete classic NetCDF header from raw file bytes.
|
|
224
|
+
///
|
|
225
|
+
/// The `format` parameter must be one of `Classic`, `Offset64`, or `Cdf5`
|
|
226
|
+
/// (the caller has already read and validated the magic bytes).
|
|
227
|
+
pub fn parse_header(data: &[u8], format: NcFormat) -> Result<ClassicHeader> {
|
|
228
|
+
// Skip past the 4-byte magic (already validated by caller).
|
|
229
|
+
let mut cur = Cursor::new(data);
|
|
230
|
+
cur.skip(4)?;
|
|
231
|
+
|
|
232
|
+
// numrecs: 4 bytes for CDF-1/2, 8 bytes for CDF-5.
|
|
233
|
+
let numrecs_raw = cur.read_count(format)?;
|
|
234
|
+
let is_streaming = format != NcFormat::Cdf5 && (numrecs_raw as u32) == STREAMING;
|
|
235
|
+
let numrecs = if is_streaming { 0 } else { numrecs_raw };
|
|
236
|
+
|
|
237
|
+
// dim_list
|
|
238
|
+
let mut dimensions = parse_dim_list(&mut cur, format)?;
|
|
239
|
+
|
|
240
|
+
// att_list (global attributes)
|
|
241
|
+
let global_attributes = parse_att_list(&mut cur, format)?;
|
|
242
|
+
|
|
243
|
+
// var_list
|
|
244
|
+
let mut variables = parse_var_list(&mut cur, format, &dimensions)?;
|
|
245
|
+
|
|
246
|
+
if numrecs > 0 {
|
|
247
|
+
apply_unlimited_dimension_size(&mut dimensions, &mut variables, numrecs);
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
Ok(ClassicHeader {
|
|
251
|
+
dimensions,
|
|
252
|
+
global_attributes,
|
|
253
|
+
variables,
|
|
254
|
+
numrecs,
|
|
255
|
+
})
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
pub(crate) fn has_streaming_numrecs(data: &[u8], format: NcFormat) -> bool {
|
|
259
|
+
if format == NcFormat::Cdf5 {
|
|
260
|
+
return false;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
let Some(bytes) = data.get(4..8) else {
|
|
264
|
+
return false;
|
|
265
|
+
};
|
|
266
|
+
|
|
267
|
+
u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) == STREAMING
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/// Parse the dimension list.
|
|
271
|
+
fn parse_dim_list(cur: &mut Cursor<'_>, format: NcFormat) -> Result<Vec<NcDimension>> {
|
|
272
|
+
let tag = cur.read_u32_be()?;
|
|
273
|
+
|
|
274
|
+
if tag == ABSENT {
|
|
275
|
+
// ABSENT is a zero tag followed by a zero count.
|
|
276
|
+
let _zero = cur.read_count(format)?;
|
|
277
|
+
return Ok(Vec::new());
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
if tag != NC_DIMENSION {
|
|
281
|
+
return Err(Error::InvalidData(format!(
|
|
282
|
+
"expected NC_DIMENSION tag (0x{:08X}), got 0x{:08X}",
|
|
283
|
+
NC_DIMENSION, tag
|
|
284
|
+
)));
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
let nelems = read_list_count(cur, format, 16, "dimension count")?;
|
|
288
|
+
let mut dims = Vec::with_capacity(nelems);
|
|
289
|
+
|
|
290
|
+
for _ in 0..nelems {
|
|
291
|
+
let name = cur.read_name(format)?;
|
|
292
|
+
let size = cur.read_count(format)?;
|
|
293
|
+
// A dimension with size 0 is the unlimited (record) dimension.
|
|
294
|
+
let is_unlimited = size == 0;
|
|
295
|
+
dims.push(NcDimension {
|
|
296
|
+
name,
|
|
297
|
+
size,
|
|
298
|
+
is_unlimited,
|
|
299
|
+
});
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
Ok(dims)
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/// Parse an attribute list (used for both global and variable attributes).
|
|
306
|
+
fn parse_att_list(cur: &mut Cursor<'_>, format: NcFormat) -> Result<Vec<NcAttribute>> {
|
|
307
|
+
let tag = cur.read_u32_be()?;
|
|
308
|
+
|
|
309
|
+
if tag == ABSENT {
|
|
310
|
+
let _zero = cur.read_count(format)?;
|
|
311
|
+
return Ok(Vec::new());
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
if tag != NC_ATTRIBUTE {
|
|
315
|
+
return Err(Error::InvalidData(format!(
|
|
316
|
+
"expected NC_ATTRIBUTE tag (0x{:08X}), got 0x{:08X}",
|
|
317
|
+
NC_ATTRIBUTE, tag
|
|
318
|
+
)));
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
let nelems = read_list_count(cur, format, 12, "attribute count")?;
|
|
322
|
+
let mut attrs = Vec::with_capacity(nelems);
|
|
323
|
+
|
|
324
|
+
for _ in 0..nelems {
|
|
325
|
+
let name = cur.read_name(format)?;
|
|
326
|
+
let nc_type = cur.read_u32_be()?;
|
|
327
|
+
let nvalues = checked_usize_from_u64(cur.read_count(format)?, "attribute value count")?;
|
|
328
|
+
let value = read_attr_values(cur, nc_type, nvalues, format)?;
|
|
329
|
+
|
|
330
|
+
attrs.push(NcAttribute { name, value });
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
Ok(attrs)
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
/// Read attribute values of the given type and count.
|
|
337
|
+
/// Values are padded to a 4-byte boundary in the file.
|
|
338
|
+
fn read_attr_values(
|
|
339
|
+
cur: &mut Cursor<'_>,
|
|
340
|
+
nc_type: u32,
|
|
341
|
+
nvalues: usize,
|
|
342
|
+
_format: NcFormat,
|
|
343
|
+
) -> Result<NcAttrValue> {
|
|
344
|
+
let typ = nc_type_from_code(nc_type)?;
|
|
345
|
+
let elem_size = typ.size()?;
|
|
346
|
+
let raw_bytes = nvalues.checked_mul(elem_size).ok_or_else(|| {
|
|
347
|
+
Error::InvalidData("classic attribute byte count exceeds platform usize".to_string())
|
|
348
|
+
})?;
|
|
349
|
+
let padded = checked_pad_to_4(raw_bytes, "classic attribute byte count")?;
|
|
350
|
+
// Bound the declared value count by the bytes actually present before any
|
|
351
|
+
// arm allocates; a tiny crafted header must not force a huge allocation.
|
|
352
|
+
if padded > cur.remaining() {
|
|
353
|
+
return Err(Error::InvalidData(format!(
|
|
354
|
+
"classic attribute claims {padded} bytes but only {} remain",
|
|
355
|
+
cur.remaining()
|
|
356
|
+
)));
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
match typ {
|
|
360
|
+
NcType::Byte => {
|
|
361
|
+
let bytes = cur.read_bytes(raw_bytes)?;
|
|
362
|
+
let values: Vec<i8> = bytes.iter().map(|&b| b as i8).collect();
|
|
363
|
+
cur.skip(padded - raw_bytes)?;
|
|
364
|
+
Ok(NcAttrValue::Bytes(values))
|
|
365
|
+
}
|
|
366
|
+
NcType::Char => {
|
|
367
|
+
let bytes = cur.read_bytes(raw_bytes)?;
|
|
368
|
+
// Trim trailing null bytes (common in NetCDF char attributes).
|
|
369
|
+
let s = String::from_utf8_lossy(bytes);
|
|
370
|
+
let trimmed = s.trim_end_matches('\0').to_string();
|
|
371
|
+
cur.skip(padded - raw_bytes)?;
|
|
372
|
+
Ok(NcAttrValue::Chars(trimmed))
|
|
373
|
+
}
|
|
374
|
+
NcType::Short => {
|
|
375
|
+
let mut values = Vec::with_capacity(nvalues);
|
|
376
|
+
for _ in 0..nvalues {
|
|
377
|
+
values.push(cur.read_u16_be()? as i16);
|
|
378
|
+
}
|
|
379
|
+
let pad = padded - raw_bytes;
|
|
380
|
+
cur.skip(pad)?;
|
|
381
|
+
Ok(NcAttrValue::Shorts(values))
|
|
382
|
+
}
|
|
383
|
+
NcType::Int => {
|
|
384
|
+
let mut values = Vec::with_capacity(nvalues);
|
|
385
|
+
for _ in 0..nvalues {
|
|
386
|
+
values.push(cur.read_i32_be()?);
|
|
387
|
+
}
|
|
388
|
+
Ok(NcAttrValue::Ints(values))
|
|
389
|
+
}
|
|
390
|
+
NcType::Float => {
|
|
391
|
+
let mut values = Vec::with_capacity(nvalues);
|
|
392
|
+
for _ in 0..nvalues {
|
|
393
|
+
values.push(cur.read_f32_be()?);
|
|
394
|
+
}
|
|
395
|
+
Ok(NcAttrValue::Floats(values))
|
|
396
|
+
}
|
|
397
|
+
NcType::Double => {
|
|
398
|
+
let mut values = Vec::with_capacity(nvalues);
|
|
399
|
+
for _ in 0..nvalues {
|
|
400
|
+
values.push(cur.read_f64_be()?);
|
|
401
|
+
}
|
|
402
|
+
Ok(NcAttrValue::Doubles(values))
|
|
403
|
+
}
|
|
404
|
+
NcType::UByte => {
|
|
405
|
+
let bytes = cur.read_bytes(raw_bytes)?;
|
|
406
|
+
cur.skip(padded - raw_bytes)?;
|
|
407
|
+
Ok(NcAttrValue::UBytes(bytes.to_vec()))
|
|
408
|
+
}
|
|
409
|
+
NcType::UShort => {
|
|
410
|
+
let mut values = Vec::with_capacity(nvalues);
|
|
411
|
+
for _ in 0..nvalues {
|
|
412
|
+
values.push(cur.read_u16_be()?);
|
|
413
|
+
}
|
|
414
|
+
let pad = padded - raw_bytes;
|
|
415
|
+
cur.skip(pad)?;
|
|
416
|
+
Ok(NcAttrValue::UShorts(values))
|
|
417
|
+
}
|
|
418
|
+
NcType::UInt => {
|
|
419
|
+
let mut values = Vec::with_capacity(nvalues);
|
|
420
|
+
for _ in 0..nvalues {
|
|
421
|
+
values.push(cur.read_u32_be()?);
|
|
422
|
+
}
|
|
423
|
+
Ok(NcAttrValue::UInts(values))
|
|
424
|
+
}
|
|
425
|
+
NcType::Int64 => {
|
|
426
|
+
let mut values = Vec::with_capacity(nvalues);
|
|
427
|
+
for _ in 0..nvalues {
|
|
428
|
+
values.push(cur.read_i64_be()?);
|
|
429
|
+
}
|
|
430
|
+
Ok(NcAttrValue::Int64s(values))
|
|
431
|
+
}
|
|
432
|
+
NcType::UInt64 => {
|
|
433
|
+
let mut values = Vec::with_capacity(nvalues);
|
|
434
|
+
for _ in 0..nvalues {
|
|
435
|
+
values.push(cur.read_u64_be()?);
|
|
436
|
+
}
|
|
437
|
+
Ok(NcAttrValue::UInt64s(values))
|
|
438
|
+
}
|
|
439
|
+
NcType::String
|
|
440
|
+
| NcType::Enum { .. }
|
|
441
|
+
| NcType::Compound { .. }
|
|
442
|
+
| NcType::Opaque { .. }
|
|
443
|
+
| NcType::Array { .. }
|
|
444
|
+
| NcType::VLen { .. } => Err(Error::InvalidData(format!(
|
|
445
|
+
"{:?} is not valid in classic format attributes",
|
|
446
|
+
typ
|
|
447
|
+
))),
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
/// Parse the variable list.
|
|
452
|
+
fn parse_var_list(
|
|
453
|
+
cur: &mut Cursor<'_>,
|
|
454
|
+
format: NcFormat,
|
|
455
|
+
dims: &[NcDimension],
|
|
456
|
+
) -> Result<Vec<NcVariable>> {
|
|
457
|
+
let tag = cur.read_u32_be()?;
|
|
458
|
+
|
|
459
|
+
if tag == ABSENT {
|
|
460
|
+
let _zero = cur.read_count(format)?;
|
|
461
|
+
return Ok(Vec::new());
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
if tag != NC_VARIABLE {
|
|
465
|
+
return Err(Error::InvalidData(format!(
|
|
466
|
+
"expected NC_VARIABLE tag (0x{:08X}), got 0x{:08X}",
|
|
467
|
+
NC_VARIABLE, tag
|
|
468
|
+
)));
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
let nelems = read_list_count(cur, format, 28, "variable count")?;
|
|
472
|
+
let mut vars = Vec::with_capacity(nelems);
|
|
473
|
+
|
|
474
|
+
for _ in 0..nelems {
|
|
475
|
+
let name = cur.read_name(format)?;
|
|
476
|
+
|
|
477
|
+
// Number of dimensions for this variable. Each dimension id is a
|
|
478
|
+
// NON_NEG value (at least 4 bytes), so bound the count by the bytes
|
|
479
|
+
// remaining before allocating.
|
|
480
|
+
let ndims = read_list_count(cur, format, 4, "variable dimension count")?;
|
|
481
|
+
|
|
482
|
+
// Dimension IDs are NON_NEG values and widen to 64 bits in CDF-5.
|
|
483
|
+
let mut var_dims = Vec::with_capacity(ndims);
|
|
484
|
+
let mut is_record_var = false;
|
|
485
|
+
for _ in 0..ndims {
|
|
486
|
+
let dimid = checked_usize_from_u64(cur.read_count(format)?, "dimension id")?;
|
|
487
|
+
if dimid >= dims.len() {
|
|
488
|
+
return Err(Error::InvalidData(format!(
|
|
489
|
+
"variable '{}' references dimension index {} but only {} dimensions exist",
|
|
490
|
+
name,
|
|
491
|
+
dimid,
|
|
492
|
+
dims.len()
|
|
493
|
+
)));
|
|
494
|
+
}
|
|
495
|
+
if dims[dimid].is_unlimited {
|
|
496
|
+
is_record_var = true;
|
|
497
|
+
}
|
|
498
|
+
var_dims.push(dims[dimid].clone());
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
// Variable attributes.
|
|
502
|
+
let attributes = parse_att_list(cur, format)?;
|
|
503
|
+
|
|
504
|
+
// nc_type (always 4 bytes).
|
|
505
|
+
let nc_type_code = cur.read_u32_be()?;
|
|
506
|
+
let dtype = nc_type_from_code(nc_type_code)?;
|
|
507
|
+
|
|
508
|
+
// vsize: the padded size of one record's worth of data for this
|
|
509
|
+
// variable, or the padded total size for non-record variables.
|
|
510
|
+
// 4 bytes for CDF-1/2 (with 2^32 - 1 reserved as an overflow marker),
|
|
511
|
+
// 8 bytes for CDF-5. The field is redundant per the spec, so it is
|
|
512
|
+
// read for validation only; sizes are recomputed from the dimensions
|
|
513
|
+
// below, which also stays correct for writers that stored the
|
|
514
|
+
// unpadded value.
|
|
515
|
+
let _vsize = cur.read_count(format)?;
|
|
516
|
+
|
|
517
|
+
// begin (data offset): 4 bytes for CDF-1, 8 bytes for CDF-2/5.
|
|
518
|
+
let data_offset = match format {
|
|
519
|
+
NcFormat::Classic => cur.read_u32_be()? as u64,
|
|
520
|
+
NcFormat::Offset64 | NcFormat::Cdf5 => cur.read_u64_be()?,
|
|
521
|
+
_ => unreachable!("classic parser only handles CDF-1/2/5"),
|
|
522
|
+
};
|
|
523
|
+
|
|
524
|
+
// The unpadded per-record slab (record vars) or total byte size
|
|
525
|
+
// (fixed vars), derived from dimensions and element size.
|
|
526
|
+
let elem_size = dtype.size().map_err(|_| {
|
|
527
|
+
Error::InvalidData(format!(
|
|
528
|
+
"variable '{name}' has a type without a classic on-disk size"
|
|
529
|
+
))
|
|
530
|
+
})? as u64;
|
|
531
|
+
let slab_bytes = var_dims
|
|
532
|
+
.iter()
|
|
533
|
+
.skip(usize::from(is_record_var))
|
|
534
|
+
.try_fold(1u64, |acc, dim| acc.checked_mul(dim.size))
|
|
535
|
+
.and_then(|elements| elements.checked_mul(elem_size));
|
|
536
|
+
let (record_size, data_size) = if is_record_var {
|
|
537
|
+
// A record variable whose single record overflows u64 cannot
|
|
538
|
+
// exist in any file.
|
|
539
|
+
let slab_bytes = slab_bytes.ok_or_else(|| {
|
|
540
|
+
Error::InvalidData(format!(
|
|
541
|
+
"record variable '{name}' per-record byte size overflows u64"
|
|
542
|
+
))
|
|
543
|
+
})?;
|
|
544
|
+
(slab_bytes, 0)
|
|
545
|
+
} else {
|
|
546
|
+
// Huge fixed variables may overflow; full reads recompute the
|
|
547
|
+
// size and error, while metadata access and slicing still work.
|
|
548
|
+
(0, slab_bytes.unwrap_or(0))
|
|
549
|
+
};
|
|
550
|
+
|
|
551
|
+
vars.push(NcVariable {
|
|
552
|
+
name,
|
|
553
|
+
dimensions: var_dims,
|
|
554
|
+
dtype,
|
|
555
|
+
attributes,
|
|
556
|
+
data_offset,
|
|
557
|
+
_data_size: data_size,
|
|
558
|
+
is_record_var,
|
|
559
|
+
record_size,
|
|
560
|
+
});
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
Ok(vars)
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
pub(crate) fn apply_unlimited_dimension_size(
|
|
567
|
+
dimensions: &mut [NcDimension],
|
|
568
|
+
variables: &mut [NcVariable],
|
|
569
|
+
numrecs: u64,
|
|
570
|
+
) {
|
|
571
|
+
for dim in dimensions.iter_mut().filter(|dim| dim.is_unlimited) {
|
|
572
|
+
dim.size = numrecs;
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
for variable in variables {
|
|
576
|
+
for dim in variable
|
|
577
|
+
.dimensions
|
|
578
|
+
.iter_mut()
|
|
579
|
+
.filter(|dim| dim.is_unlimited)
|
|
580
|
+
{
|
|
581
|
+
dim.size = numrecs;
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
#[cfg(test)]
|
|
587
|
+
mod tests {
|
|
588
|
+
use super::*;
|
|
589
|
+
use crate::NcFormat;
|
|
590
|
+
|
|
591
|
+
/// Build a minimal CDF-1 file header in memory.
|
|
592
|
+
/// This helper constructs valid header bytes for testing.
|
|
593
|
+
fn build_cdf1_header(
|
|
594
|
+
dims: &[(&str, u32)],
|
|
595
|
+
attrs: &[(&str, u32, &[u8])], // (name, nc_type, raw_value_bytes)
|
|
596
|
+
vars: &[(&str, &[u32], u32, u32, u32)], // (name, dimids, nc_type, vsize, offset)
|
|
597
|
+
numrecs: u32,
|
|
598
|
+
) -> Vec<u8> {
|
|
599
|
+
let mut buf = Vec::new();
|
|
600
|
+
|
|
601
|
+
// Magic: CDF\x01
|
|
602
|
+
buf.extend_from_slice(b"CDF\x01");
|
|
603
|
+
|
|
604
|
+
// numrecs (4 bytes)
|
|
605
|
+
buf.extend_from_slice(&numrecs.to_be_bytes());
|
|
606
|
+
|
|
607
|
+
// dim_list
|
|
608
|
+
if dims.is_empty() {
|
|
609
|
+
// ABSENT
|
|
610
|
+
buf.extend_from_slice(&ABSENT.to_be_bytes());
|
|
611
|
+
buf.extend_from_slice(&0u32.to_be_bytes());
|
|
612
|
+
} else {
|
|
613
|
+
buf.extend_from_slice(&NC_DIMENSION.to_be_bytes());
|
|
614
|
+
buf.extend_from_slice(&(dims.len() as u32).to_be_bytes());
|
|
615
|
+
for (name, size) in dims {
|
|
616
|
+
write_name_cdf1(&mut buf, name);
|
|
617
|
+
buf.extend_from_slice(&size.to_be_bytes());
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
// att_list (global)
|
|
622
|
+
write_att_list_cdf1(&mut buf, attrs);
|
|
623
|
+
|
|
624
|
+
// var_list
|
|
625
|
+
if vars.is_empty() {
|
|
626
|
+
buf.extend_from_slice(&ABSENT.to_be_bytes());
|
|
627
|
+
buf.extend_from_slice(&0u32.to_be_bytes());
|
|
628
|
+
} else {
|
|
629
|
+
buf.extend_from_slice(&NC_VARIABLE.to_be_bytes());
|
|
630
|
+
buf.extend_from_slice(&(vars.len() as u32).to_be_bytes());
|
|
631
|
+
for (name, dimids, nc_type, vsize, offset) in vars {
|
|
632
|
+
write_name_cdf1(&mut buf, name);
|
|
633
|
+
// ndims
|
|
634
|
+
buf.extend_from_slice(&(dimids.len() as u32).to_be_bytes());
|
|
635
|
+
// dimids
|
|
636
|
+
for &did in *dimids {
|
|
637
|
+
buf.extend_from_slice(&did.to_be_bytes());
|
|
638
|
+
}
|
|
639
|
+
// att_list (empty for test vars)
|
|
640
|
+
buf.extend_from_slice(&ABSENT.to_be_bytes());
|
|
641
|
+
buf.extend_from_slice(&0u32.to_be_bytes());
|
|
642
|
+
// nc_type
|
|
643
|
+
buf.extend_from_slice(&nc_type.to_be_bytes());
|
|
644
|
+
// vsize
|
|
645
|
+
buf.extend_from_slice(&vsize.to_be_bytes());
|
|
646
|
+
// begin (offset) -- 4 bytes for CDF-1
|
|
647
|
+
buf.extend_from_slice(&offset.to_be_bytes());
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
buf
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
fn write_name_cdf1(buf: &mut Vec<u8>, name: &str) {
|
|
655
|
+
let name_bytes = name.as_bytes();
|
|
656
|
+
buf.extend_from_slice(&(name_bytes.len() as u32).to_be_bytes());
|
|
657
|
+
buf.extend_from_slice(name_bytes);
|
|
658
|
+
let pad = pad_to_4(name_bytes.len()) - name_bytes.len();
|
|
659
|
+
for _ in 0..pad {
|
|
660
|
+
buf.push(0);
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
fn write_att_list_cdf1(buf: &mut Vec<u8>, attrs: &[(&str, u32, &[u8])]) {
|
|
665
|
+
if attrs.is_empty() {
|
|
666
|
+
buf.extend_from_slice(&ABSENT.to_be_bytes());
|
|
667
|
+
buf.extend_from_slice(&0u32.to_be_bytes());
|
|
668
|
+
return;
|
|
669
|
+
}
|
|
670
|
+
buf.extend_from_slice(&NC_ATTRIBUTE.to_be_bytes());
|
|
671
|
+
buf.extend_from_slice(&(attrs.len() as u32).to_be_bytes());
|
|
672
|
+
for (name, nc_type, value_bytes) in attrs {
|
|
673
|
+
write_name_cdf1(buf, name);
|
|
674
|
+
buf.extend_from_slice(&nc_type.to_be_bytes());
|
|
675
|
+
// For simplicity, nvalues = 1 element (caller provides exactly one element's bytes)
|
|
676
|
+
let elem_size = match nc_type {
|
|
677
|
+
1 => 1, // byte
|
|
678
|
+
2 => 1, // char
|
|
679
|
+
3 => 2, // short
|
|
680
|
+
4 => 4, // int
|
|
681
|
+
5 => 4, // float
|
|
682
|
+
6 => 8, // double
|
|
683
|
+
_ => 1,
|
|
684
|
+
};
|
|
685
|
+
let nvalues = value_bytes.len() / elem_size;
|
|
686
|
+
buf.extend_from_slice(&(nvalues as u32).to_be_bytes());
|
|
687
|
+
buf.extend_from_slice(value_bytes);
|
|
688
|
+
let pad = pad_to_4(value_bytes.len()) - value_bytes.len();
|
|
689
|
+
for _ in 0..pad {
|
|
690
|
+
buf.push(0);
|
|
691
|
+
}
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
fn write_count_cdf5(buf: &mut Vec<u8>, value: u64) {
|
|
696
|
+
buf.extend_from_slice(&value.to_be_bytes());
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
fn write_name_cdf5(buf: &mut Vec<u8>, name: &str) {
|
|
700
|
+
let name_bytes = name.as_bytes();
|
|
701
|
+
write_count_cdf5(buf, name_bytes.len() as u64);
|
|
702
|
+
buf.extend_from_slice(name_bytes);
|
|
703
|
+
let pad = pad_to_4(name_bytes.len()) - name_bytes.len();
|
|
704
|
+
for _ in 0..pad {
|
|
705
|
+
buf.push(0);
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
fn build_cdf5_header(
|
|
710
|
+
dims: &[(&str, u64)],
|
|
711
|
+
vars: &[(&str, &[u64], u32, u64, u64)],
|
|
712
|
+
numrecs: u64,
|
|
713
|
+
) -> Vec<u8> {
|
|
714
|
+
let mut buf = Vec::new();
|
|
715
|
+
buf.extend_from_slice(b"CDF\x05");
|
|
716
|
+
write_count_cdf5(&mut buf, numrecs);
|
|
717
|
+
|
|
718
|
+
if dims.is_empty() {
|
|
719
|
+
buf.extend_from_slice(&ABSENT.to_be_bytes());
|
|
720
|
+
write_count_cdf5(&mut buf, 0);
|
|
721
|
+
} else {
|
|
722
|
+
buf.extend_from_slice(&NC_DIMENSION.to_be_bytes());
|
|
723
|
+
write_count_cdf5(&mut buf, dims.len() as u64);
|
|
724
|
+
for (name, size) in dims {
|
|
725
|
+
write_name_cdf5(&mut buf, name);
|
|
726
|
+
write_count_cdf5(&mut buf, *size);
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
buf.extend_from_slice(&ABSENT.to_be_bytes());
|
|
731
|
+
write_count_cdf5(&mut buf, 0);
|
|
732
|
+
|
|
733
|
+
if vars.is_empty() {
|
|
734
|
+
buf.extend_from_slice(&ABSENT.to_be_bytes());
|
|
735
|
+
write_count_cdf5(&mut buf, 0);
|
|
736
|
+
} else {
|
|
737
|
+
buf.extend_from_slice(&NC_VARIABLE.to_be_bytes());
|
|
738
|
+
write_count_cdf5(&mut buf, vars.len() as u64);
|
|
739
|
+
for (name, dimids, nc_type, vsize, offset) in vars {
|
|
740
|
+
write_name_cdf5(&mut buf, name);
|
|
741
|
+
write_count_cdf5(&mut buf, dimids.len() as u64);
|
|
742
|
+
for dimid in *dimids {
|
|
743
|
+
write_count_cdf5(&mut buf, *dimid);
|
|
744
|
+
}
|
|
745
|
+
buf.extend_from_slice(&ABSENT.to_be_bytes());
|
|
746
|
+
write_count_cdf5(&mut buf, 0);
|
|
747
|
+
buf.extend_from_slice(&nc_type.to_be_bytes());
|
|
748
|
+
write_count_cdf5(&mut buf, *vsize);
|
|
749
|
+
buf.extend_from_slice(&offset.to_be_bytes());
|
|
750
|
+
}
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
buf
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
#[test]
|
|
757
|
+
fn empty_header() {
|
|
758
|
+
let data = build_cdf1_header(&[], &[], &[], 0);
|
|
759
|
+
let header = parse_header(&data, NcFormat::Classic).unwrap();
|
|
760
|
+
assert!(header.dimensions.is_empty());
|
|
761
|
+
assert!(header.global_attributes.is_empty());
|
|
762
|
+
assert!(header.variables.is_empty());
|
|
763
|
+
assert_eq!(header.numrecs, 0);
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
#[test]
|
|
767
|
+
fn dimensions() {
|
|
768
|
+
let data = build_cdf1_header(
|
|
769
|
+
&[("x", 10), ("y", 20), ("time", 0)], // time is unlimited
|
|
770
|
+
&[],
|
|
771
|
+
&[],
|
|
772
|
+
5,
|
|
773
|
+
);
|
|
774
|
+
let header = parse_header(&data, NcFormat::Classic).unwrap();
|
|
775
|
+
assert_eq!(header.dimensions.len(), 3);
|
|
776
|
+
|
|
777
|
+
assert_eq!(header.dimensions[0].name, "x");
|
|
778
|
+
assert_eq!(header.dimensions[0].size, 10);
|
|
779
|
+
assert!(!header.dimensions[0].is_unlimited);
|
|
780
|
+
|
|
781
|
+
assert_eq!(header.dimensions[1].name, "y");
|
|
782
|
+
assert_eq!(header.dimensions[1].size, 20);
|
|
783
|
+
assert!(!header.dimensions[1].is_unlimited);
|
|
784
|
+
|
|
785
|
+
assert_eq!(header.dimensions[2].name, "time");
|
|
786
|
+
assert_eq!(header.dimensions[2].size, 5);
|
|
787
|
+
assert!(header.dimensions[2].is_unlimited);
|
|
788
|
+
|
|
789
|
+
assert_eq!(header.numrecs, 5);
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
#[test]
|
|
793
|
+
fn global_attributes() {
|
|
794
|
+
// One NC_INT attribute with value 42.
|
|
795
|
+
let value_bytes = 42i32.to_be_bytes();
|
|
796
|
+
let data = build_cdf1_header(
|
|
797
|
+
&[],
|
|
798
|
+
&[("answer", 4, &value_bytes)], // NC_INT = 4
|
|
799
|
+
&[],
|
|
800
|
+
0,
|
|
801
|
+
);
|
|
802
|
+
let header = parse_header(&data, NcFormat::Classic).unwrap();
|
|
803
|
+
assert_eq!(header.global_attributes.len(), 1);
|
|
804
|
+
assert_eq!(header.global_attributes[0].name, "answer");
|
|
805
|
+
if let NcAttrValue::Ints(ref v) = header.global_attributes[0].value {
|
|
806
|
+
assert_eq!(v, &[42]);
|
|
807
|
+
} else {
|
|
808
|
+
panic!("expected Ints attribute");
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
#[test]
|
|
813
|
+
fn char_attribute() {
|
|
814
|
+
let text = b"hello";
|
|
815
|
+
let data = build_cdf1_header(
|
|
816
|
+
&[],
|
|
817
|
+
&[("greeting", 2, text)], // NC_CHAR = 2
|
|
818
|
+
&[],
|
|
819
|
+
0,
|
|
820
|
+
);
|
|
821
|
+
let header = parse_header(&data, NcFormat::Classic).unwrap();
|
|
822
|
+
assert_eq!(header.global_attributes.len(), 1);
|
|
823
|
+
assert_eq!(header.global_attributes[0].name, "greeting");
|
|
824
|
+
if let NcAttrValue::Chars(ref s) = header.global_attributes[0].value {
|
|
825
|
+
assert_eq!(s, "hello");
|
|
826
|
+
} else {
|
|
827
|
+
panic!("expected Chars attribute");
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
#[test]
|
|
832
|
+
fn variables() {
|
|
833
|
+
let data = build_cdf1_header(
|
|
834
|
+
&[("x", 10), ("y", 20)],
|
|
835
|
+
&[],
|
|
836
|
+
&[
|
|
837
|
+
("temperature", &[0, 1], 5, 800, 200), // float, dimids=[x,y]
|
|
838
|
+
("pressure", &[0, 1], 6, 1600, 1000), // double, dimids=[x,y]
|
|
839
|
+
],
|
|
840
|
+
0,
|
|
841
|
+
);
|
|
842
|
+
let header = parse_header(&data, NcFormat::Classic).unwrap();
|
|
843
|
+
assert_eq!(header.variables.len(), 2);
|
|
844
|
+
|
|
845
|
+
let temp = &header.variables[0];
|
|
846
|
+
assert_eq!(temp.name, "temperature");
|
|
847
|
+
assert_eq!(temp.dtype, NcType::Float);
|
|
848
|
+
assert_eq!(temp.dimensions.len(), 2);
|
|
849
|
+
assert_eq!(temp.dimensions[0].name, "x");
|
|
850
|
+
assert_eq!(temp.dimensions[1].name, "y");
|
|
851
|
+
assert_eq!(temp.data_offset, 200);
|
|
852
|
+
assert_eq!(temp._data_size, 800);
|
|
853
|
+
assert!(!temp.is_record_var);
|
|
854
|
+
|
|
855
|
+
let pres = &header.variables[1];
|
|
856
|
+
assert_eq!(pres.name, "pressure");
|
|
857
|
+
assert_eq!(pres.dtype, NcType::Double);
|
|
858
|
+
assert_eq!(pres.data_offset, 1000);
|
|
859
|
+
assert_eq!(pres._data_size, 1600);
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
#[test]
|
|
863
|
+
fn record_variable() {
|
|
864
|
+
let data = build_cdf1_header(
|
|
865
|
+
&[("time", 0), ("x", 5)], // time is unlimited
|
|
866
|
+
&[],
|
|
867
|
+
&[
|
|
868
|
+
// record variable: first dim is unlimited
|
|
869
|
+
("values", &[0, 1], 5, 20, 100), // float, vsize=5*4=20 per record
|
|
870
|
+
],
|
|
871
|
+
10, // 10 records
|
|
872
|
+
);
|
|
873
|
+
let header = parse_header(&data, NcFormat::Classic).unwrap();
|
|
874
|
+
assert_eq!(header.numrecs, 10);
|
|
875
|
+
assert_eq!(header.variables.len(), 1);
|
|
876
|
+
|
|
877
|
+
let var = &header.variables[0];
|
|
878
|
+
assert_eq!(var.name, "values");
|
|
879
|
+
assert!(var.is_record_var);
|
|
880
|
+
assert_eq!(var.record_size, 20);
|
|
881
|
+
assert_eq!(var._data_size, 0); // data_size=0 for record vars (computed at read time)
|
|
882
|
+
assert_eq!(var.shape(), vec![10, 5]);
|
|
883
|
+
}
|
|
884
|
+
|
|
885
|
+
#[test]
|
|
886
|
+
fn cdf2_offset64() {
|
|
887
|
+
// Build a CDF-2 header manually.
|
|
888
|
+
// CDF-2 is mostly the same as CDF-1 but the data offset (begin) field is 8 bytes.
|
|
889
|
+
let mut buf = Vec::new();
|
|
890
|
+
buf.extend_from_slice(b"CDF\x02");
|
|
891
|
+
// numrecs (4 bytes)
|
|
892
|
+
buf.extend_from_slice(&0u32.to_be_bytes());
|
|
893
|
+
// dim_list: one dimension "x" with size 100
|
|
894
|
+
buf.extend_from_slice(&NC_DIMENSION.to_be_bytes());
|
|
895
|
+
buf.extend_from_slice(&1u32.to_be_bytes());
|
|
896
|
+
write_name_cdf1(&mut buf, "x");
|
|
897
|
+
buf.extend_from_slice(&100u32.to_be_bytes());
|
|
898
|
+
// att_list: absent
|
|
899
|
+
buf.extend_from_slice(&ABSENT.to_be_bytes());
|
|
900
|
+
buf.extend_from_slice(&0u32.to_be_bytes());
|
|
901
|
+
// var_list: one variable
|
|
902
|
+
buf.extend_from_slice(&NC_VARIABLE.to_be_bytes());
|
|
903
|
+
buf.extend_from_slice(&1u32.to_be_bytes());
|
|
904
|
+
write_name_cdf1(&mut buf, "data");
|
|
905
|
+
buf.extend_from_slice(&1u32.to_be_bytes()); // ndims=1
|
|
906
|
+
buf.extend_from_slice(&0u32.to_be_bytes()); // dimid=0
|
|
907
|
+
// att_list: absent
|
|
908
|
+
buf.extend_from_slice(&ABSENT.to_be_bytes());
|
|
909
|
+
buf.extend_from_slice(&0u32.to_be_bytes());
|
|
910
|
+
// nc_type = NC_FLOAT = 5
|
|
911
|
+
buf.extend_from_slice(&5u32.to_be_bytes());
|
|
912
|
+
// vsize (4 bytes for CDF-2)
|
|
913
|
+
buf.extend_from_slice(&400u32.to_be_bytes());
|
|
914
|
+
// begin (8 bytes for CDF-2!)
|
|
915
|
+
let offset: u64 = 0x1_0000_0000; // > 4 GB offset to test 64-bit
|
|
916
|
+
buf.extend_from_slice(&offset.to_be_bytes());
|
|
917
|
+
|
|
918
|
+
let header = parse_header(&buf, NcFormat::Offset64).unwrap();
|
|
919
|
+
assert_eq!(header.variables.len(), 1);
|
|
920
|
+
assert_eq!(header.variables[0].data_offset, 0x1_0000_0000);
|
|
921
|
+
assert_eq!(header.variables[0]._data_size, 400);
|
|
922
|
+
}
|
|
923
|
+
|
|
924
|
+
#[test]
|
|
925
|
+
fn cdf5_uses_64_bit_counts_for_var_metadata() {
|
|
926
|
+
let data = build_cdf5_header(
|
|
927
|
+
&[("n", 4)],
|
|
928
|
+
&[
|
|
929
|
+
("ubyte_var", &[0], 7, 4, 128),
|
|
930
|
+
("int64_var", &[0], 10, 32, 256),
|
|
931
|
+
],
|
|
932
|
+
0,
|
|
933
|
+
);
|
|
934
|
+
|
|
935
|
+
let header = parse_header(&data, NcFormat::Cdf5).unwrap();
|
|
936
|
+
assert_eq!(header.variables.len(), 2);
|
|
937
|
+
assert_eq!(header.variables[0].name, "ubyte_var");
|
|
938
|
+
assert_eq!(header.variables[0].dtype, NcType::UByte);
|
|
939
|
+
assert_eq!(header.variables[0].dimensions[0].name, "n");
|
|
940
|
+
assert_eq!(header.variables[1].name, "int64_var");
|
|
941
|
+
assert_eq!(header.variables[1].dtype, NcType::Int64);
|
|
942
|
+
assert_eq!(header.variables[1].data_offset, 256);
|
|
943
|
+
}
|
|
944
|
+
|
|
945
|
+
#[test]
|
|
946
|
+
fn unlimited_dimension_size_tracks_numrecs() {
|
|
947
|
+
let data = build_cdf1_header(
|
|
948
|
+
&[("time", 0), ("x", 5)],
|
|
949
|
+
&[],
|
|
950
|
+
&[("series", &[0, 1], 6, 40, 128)],
|
|
951
|
+
3,
|
|
952
|
+
);
|
|
953
|
+
|
|
954
|
+
let header = parse_header(&data, NcFormat::Classic).unwrap();
|
|
955
|
+
assert_eq!(header.dimensions[0].size, 3);
|
|
956
|
+
assert_eq!(header.variables[0].shape(), vec![3, 5]);
|
|
957
|
+
}
|
|
958
|
+
|
|
959
|
+
#[test]
|
|
960
|
+
fn double_attribute() {
|
|
961
|
+
let pi = std::f64::consts::PI;
|
|
962
|
+
let value_bytes = pi.to_be_bytes();
|
|
963
|
+
let data = build_cdf1_header(
|
|
964
|
+
&[],
|
|
965
|
+
&[("pi", 6, &value_bytes)], // NC_DOUBLE = 6
|
|
966
|
+
&[],
|
|
967
|
+
0,
|
|
968
|
+
);
|
|
969
|
+
let header = parse_header(&data, NcFormat::Classic).unwrap();
|
|
970
|
+
assert_eq!(header.global_attributes.len(), 1);
|
|
971
|
+
if let NcAttrValue::Doubles(ref v) = header.global_attributes[0].value {
|
|
972
|
+
assert_eq!(v.len(), 1);
|
|
973
|
+
assert!((v[0] - pi).abs() < 1e-15);
|
|
974
|
+
} else {
|
|
975
|
+
panic!("expected Doubles attribute");
|
|
976
|
+
}
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
#[test]
|
|
980
|
+
fn short_attribute_with_padding() {
|
|
981
|
+
// NC_SHORT (2 bytes) with 3 values = 6 bytes, padded to 8.
|
|
982
|
+
let mut value_bytes = Vec::new();
|
|
983
|
+
value_bytes.extend_from_slice(&1i16.to_be_bytes());
|
|
984
|
+
value_bytes.extend_from_slice(&2i16.to_be_bytes());
|
|
985
|
+
value_bytes.extend_from_slice(&3i16.to_be_bytes());
|
|
986
|
+
// The build helper will add padding.
|
|
987
|
+
|
|
988
|
+
let mut buf = Vec::new();
|
|
989
|
+
buf.extend_from_slice(b"CDF\x01");
|
|
990
|
+
buf.extend_from_slice(&0u32.to_be_bytes()); // numrecs
|
|
991
|
+
// dim_list: absent
|
|
992
|
+
buf.extend_from_slice(&ABSENT.to_be_bytes());
|
|
993
|
+
buf.extend_from_slice(&0u32.to_be_bytes());
|
|
994
|
+
// att_list: one short attribute with 3 values
|
|
995
|
+
buf.extend_from_slice(&NC_ATTRIBUTE.to_be_bytes());
|
|
996
|
+
buf.extend_from_slice(&1u32.to_be_bytes());
|
|
997
|
+
write_name_cdf1(&mut buf, "vals");
|
|
998
|
+
buf.extend_from_slice(&3u32.to_be_bytes()); // NC_SHORT
|
|
999
|
+
buf.extend_from_slice(&3u32.to_be_bytes()); // nvalues=3
|
|
1000
|
+
buf.extend_from_slice(&value_bytes);
|
|
1001
|
+
// Pad to 4-byte boundary: 6 bytes -> 2 bytes padding
|
|
1002
|
+
buf.extend_from_slice(&[0, 0]);
|
|
1003
|
+
// var_list: absent
|
|
1004
|
+
buf.extend_from_slice(&ABSENT.to_be_bytes());
|
|
1005
|
+
buf.extend_from_slice(&0u32.to_be_bytes());
|
|
1006
|
+
|
|
1007
|
+
let header = parse_header(&buf, NcFormat::Classic).unwrap();
|
|
1008
|
+
if let NcAttrValue::Shorts(ref v) = header.global_attributes[0].value {
|
|
1009
|
+
assert_eq!(v, &[1, 2, 3]);
|
|
1010
|
+
} else {
|
|
1011
|
+
panic!("expected Shorts attribute");
|
|
1012
|
+
}
|
|
1013
|
+
}
|
|
1014
|
+
|
|
1015
|
+
#[test]
|
|
1016
|
+
fn name_padding() {
|
|
1017
|
+
// Names with lengths 1, 2, 3, 4, 5 to test all padding cases.
|
|
1018
|
+
let data = build_cdf1_header(
|
|
1019
|
+
&[("a", 1), ("ab", 2), ("abc", 3), ("abcd", 4), ("abcde", 5)],
|
|
1020
|
+
&[],
|
|
1021
|
+
&[],
|
|
1022
|
+
0,
|
|
1023
|
+
);
|
|
1024
|
+
let header = parse_header(&data, NcFormat::Classic).unwrap();
|
|
1025
|
+
assert_eq!(header.dimensions.len(), 5);
|
|
1026
|
+
assert_eq!(header.dimensions[0].name, "a");
|
|
1027
|
+
assert_eq!(header.dimensions[1].name, "ab");
|
|
1028
|
+
assert_eq!(header.dimensions[2].name, "abc");
|
|
1029
|
+
assert_eq!(header.dimensions[3].name, "abcd");
|
|
1030
|
+
assert_eq!(header.dimensions[4].name, "abcde");
|
|
1031
|
+
}
|
|
1032
|
+
|
|
1033
|
+
#[test]
|
|
1034
|
+
fn invalid_dimension_reference() {
|
|
1035
|
+
// Variable referencing a non-existent dimension.
|
|
1036
|
+
let data = build_cdf1_header(
|
|
1037
|
+
&[("x", 10)], // only dim 0 exists
|
|
1038
|
+
&[],
|
|
1039
|
+
&[("bad_var", &[5], 4, 40, 100)], // dimid=5 is out of range
|
|
1040
|
+
0,
|
|
1041
|
+
);
|
|
1042
|
+
let result = parse_header(&data, NcFormat::Classic);
|
|
1043
|
+
assert!(result.is_err());
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
#[test]
|
|
1047
|
+
fn byte_attribute() {
|
|
1048
|
+
let value_bytes: &[u8] = &[0xFF]; // -1 as i8
|
|
1049
|
+
let data = build_cdf1_header(
|
|
1050
|
+
&[],
|
|
1051
|
+
&[("flag", 1, value_bytes)], // NC_BYTE = 1
|
|
1052
|
+
&[],
|
|
1053
|
+
0,
|
|
1054
|
+
);
|
|
1055
|
+
let header = parse_header(&data, NcFormat::Classic).unwrap();
|
|
1056
|
+
if let NcAttrValue::Bytes(ref v) = header.global_attributes[0].value {
|
|
1057
|
+
assert_eq!(v, &[-1i8]);
|
|
1058
|
+
} else {
|
|
1059
|
+
panic!("expected Bytes attribute");
|
|
1060
|
+
}
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
#[test]
|
|
1064
|
+
fn float_attribute() {
|
|
1065
|
+
let val = std::f32::consts::PI;
|
|
1066
|
+
let value_bytes = val.to_be_bytes();
|
|
1067
|
+
let data = build_cdf1_header(
|
|
1068
|
+
&[],
|
|
1069
|
+
&[("pi_approx", 5, &value_bytes)], // NC_FLOAT = 5
|
|
1070
|
+
&[],
|
|
1071
|
+
0,
|
|
1072
|
+
);
|
|
1073
|
+
let header = parse_header(&data, NcFormat::Classic).unwrap();
|
|
1074
|
+
if let NcAttrValue::Floats(ref v) = header.global_attributes[0].value {
|
|
1075
|
+
assert_eq!(v.len(), 1);
|
|
1076
|
+
assert!((v[0] - std::f32::consts::PI).abs() < 1e-6);
|
|
1077
|
+
} else {
|
|
1078
|
+
panic!("expected Floats attribute");
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
|
|
1082
|
+
#[test]
|
|
1083
|
+
fn multiple_global_attributes() {
|
|
1084
|
+
let int_val = 100i32.to_be_bytes();
|
|
1085
|
+
let float_val = 2.5f32.to_be_bytes();
|
|
1086
|
+
let data = build_cdf1_header(
|
|
1087
|
+
&[],
|
|
1088
|
+
&[("count", 4, &int_val), ("scale", 5, &float_val)],
|
|
1089
|
+
&[],
|
|
1090
|
+
0,
|
|
1091
|
+
);
|
|
1092
|
+
let header = parse_header(&data, NcFormat::Classic).unwrap();
|
|
1093
|
+
assert_eq!(header.global_attributes.len(), 2);
|
|
1094
|
+
assert_eq!(header.global_attributes[0].name, "count");
|
|
1095
|
+
assert_eq!(header.global_attributes[1].name, "scale");
|
|
1096
|
+
}
|
|
1097
|
+
}
|