@nirs4all/formats-wasm 0.2.12 → 0.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD_PARTY_NOTICES.md +49 -0
- package/nirs4all_formats_wasm.d.ts +4 -4
- package/nirs4all_formats_wasm.js +170 -149
- package/nirs4all_formats_wasm_bg.wasm +0 -0
- package/package.json +4 -2
- package/reader-backports/hdf5-reader/Cargo.toml +119 -0
- package/reader-backports/hdf5-reader/LICENSE-APACHE +164 -0
- package/reader-backports/hdf5-reader/LICENSE-MIT +21 -0
- package/reader-backports/hdf5-reader/PROVENANCE.json +80 -0
- package/reader-backports/hdf5-reader/README.md +308 -0
- package/reader-backports/hdf5-reader/UPSTREAM-Cargo.toml +44 -0
- package/reader-backports/hdf5-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/hdf5-reader/src/attribute_api.rs +847 -0
- package/reader-backports/hdf5-reader/src/btree_v1.rs +726 -0
- package/reader-backports/hdf5-reader/src/btree_v2.rs +2068 -0
- package/reader-backports/hdf5-reader/src/cache.rs +354 -0
- package/reader-backports/hdf5-reader/src/checksum.rs +56 -0
- package/reader-backports/hdf5-reader/src/chunk_index.rs +423 -0
- package/reader-backports/hdf5-reader/src/dataset.rs +4733 -0
- package/reader-backports/hdf5-reader/src/datatype_api.rs +497 -0
- package/reader-backports/hdf5-reader/src/error.rs +167 -0
- package/reader-backports/hdf5-reader/src/extensible_array.rs +1886 -0
- package/reader-backports/hdf5-reader/src/filters/deflate.rs +93 -0
- package/reader-backports/hdf5-reader/src/filters/fletcher32.rs +52 -0
- package/reader-backports/hdf5-reader/src/filters/lz4.rs +178 -0
- package/reader-backports/hdf5-reader/src/filters/mod.rs +326 -0
- package/reader-backports/hdf5-reader/src/filters/nbit.rs +586 -0
- package/reader-backports/hdf5-reader/src/filters/scaleoffset.rs +827 -0
- package/reader-backports/hdf5-reader/src/filters/shuffle.rs +99 -0
- package/reader-backports/hdf5-reader/src/fixed_array.rs +763 -0
- package/reader-backports/hdf5-reader/src/fractal_heap.rs +2029 -0
- package/reader-backports/hdf5-reader/src/global_heap.rs +284 -0
- package/reader-backports/hdf5-reader/src/group.rs +795 -0
- package/reader-backports/hdf5-reader/src/io.rs +413 -0
- package/reader-backports/hdf5-reader/src/lib.rs +916 -0
- package/reader-backports/hdf5-reader/src/local_heap.rs +261 -0
- package/reader-backports/hdf5-reader/src/messages/attribute.rs +304 -0
- package/reader-backports/hdf5-reader/src/messages/attribute_info.rs +96 -0
- package/reader-backports/hdf5-reader/src/messages/btree_k.rs +36 -0
- package/reader-backports/hdf5-reader/src/messages/continuation.rs +46 -0
- package/reader-backports/hdf5-reader/src/messages/dataspace.rs +259 -0
- package/reader-backports/hdf5-reader/src/messages/datatype.rs +938 -0
- package/reader-backports/hdf5-reader/src/messages/external_files.rs +50 -0
- package/reader-backports/hdf5-reader/src/messages/fill_value.rs +233 -0
- package/reader-backports/hdf5-reader/src/messages/filter_pipeline.rs +147 -0
- package/reader-backports/hdf5-reader/src/messages/group_info.rs +103 -0
- package/reader-backports/hdf5-reader/src/messages/layout.rs +476 -0
- package/reader-backports/hdf5-reader/src/messages/link.rs +170 -0
- package/reader-backports/hdf5-reader/src/messages/link_info.rs +127 -0
- package/reader-backports/hdf5-reader/src/messages/mod.rs +374 -0
- package/reader-backports/hdf5-reader/src/messages/modification_time.rs +83 -0
- package/reader-backports/hdf5-reader/src/messages/shared.rs +164 -0
- package/reader-backports/hdf5-reader/src/messages/shared_table.rs +67 -0
- package/reader-backports/hdf5-reader/src/messages/symbol_table_msg.rs +51 -0
- package/reader-backports/hdf5-reader/src/object_header.rs +1749 -0
- package/reader-backports/hdf5-reader/src/reference.rs +90 -0
- package/reader-backports/hdf5-reader/src/shared_message_table.rs +425 -0
- package/reader-backports/hdf5-reader/src/storage.rs +538 -0
- package/reader-backports/hdf5-reader/src/superblock.rs +285 -0
- package/reader-backports/hdf5-reader/src/symbol_table.rs +277 -0
- package/reader-backports/netcdf-reader/Cargo.toml +141 -0
- package/reader-backports/netcdf-reader/LICENSE-APACHE +164 -0
- package/reader-backports/netcdf-reader/LICENSE-MIT +21 -0
- package/reader-backports/netcdf-reader/PROVENANCE.json +55 -0
- package/reader-backports/netcdf-reader/README.md +308 -0
- package/reader-backports/netcdf-reader/UPSTREAM-Cargo.toml +48 -0
- package/reader-backports/netcdf-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/netcdf-reader/src/cf/axes.rs +419 -0
- package/reader-backports/netcdf-reader/src/cf/bounds.rs +108 -0
- package/reader-backports/netcdf-reader/src/cf/crs.rs +269 -0
- package/reader-backports/netcdf-reader/src/cf/mod.rs +30 -0
- package/reader-backports/netcdf-reader/src/cf/time.rs +1122 -0
- package/reader-backports/netcdf-reader/src/classic/data.rs +1465 -0
- package/reader-backports/netcdf-reader/src/classic/header.rs +1097 -0
- package/reader-backports/netcdf-reader/src/classic/mod.rs +240 -0
- package/reader-backports/netcdf-reader/src/classic/storage.rs +211 -0
- package/reader-backports/netcdf-reader/src/classic/types.rs +93 -0
- package/reader-backports/netcdf-reader/src/classic/variable.rs +1162 -0
- package/reader-backports/netcdf-reader/src/error.rs +59 -0
- package/reader-backports/netcdf-reader/src/lib.rs +1531 -0
- package/reader-backports/netcdf-reader/src/masked.rs +173 -0
- package/reader-backports/netcdf-reader/src/nc4/attributes.rs +203 -0
- package/reader-backports/netcdf-reader/src/nc4/dimensions.rs +139 -0
- package/reader-backports/netcdf-reader/src/nc4/groups.rs +308 -0
- package/reader-backports/netcdf-reader/src/nc4/mod.rs +572 -0
- package/reader-backports/netcdf-reader/src/nc4/types.rs +309 -0
- package/reader-backports/netcdf-reader/src/nc4/variables.rs +420 -0
- package/reader-backports/netcdf-reader/src/types.rs +18 -0
- package/reader-backports/netcdf-reader/src/unpack.rs +113 -0
- package/reader-backports/netcdf-reader/src/user_defined.rs +569 -0
|
@@ -0,0 +1,1465 @@
|
|
|
1
|
+
//! Data reading for classic (CDF-1/2/5) NetCDF files.
|
|
2
|
+
//!
|
|
3
|
+
//! Two layout types:
|
|
4
|
+
//! - **Non-record variables**: contiguous data at the offset stored in the variable header.
|
|
5
|
+
//! - **Record variables**: data is interleaved across records. Each record contains one
|
|
6
|
+
//! slice from every record variable, in the order they appear in the header. The total
|
|
7
|
+
//! record size is the sum of all record variables' vsize values (each padded to 4-byte
|
|
8
|
+
//! boundary in CDF-1/2).
|
|
9
|
+
|
|
10
|
+
use ndarray::{ArrayD, IxDyn};
|
|
11
|
+
#[cfg(feature = "rayon")]
|
|
12
|
+
use rayon::prelude::*;
|
|
13
|
+
|
|
14
|
+
use crate::error::{Error, Result};
|
|
15
|
+
use crate::types::{NcType, NcVariable};
|
|
16
|
+
|
|
17
|
+
use super::storage::ClassicStorage;
|
|
18
|
+
|
|
19
|
+
/// Trait for types that can be read from classic NetCDF data.
|
|
20
|
+
pub trait NcReadType: Clone + Default + Send + 'static {
|
|
21
|
+
/// The NetCDF type this Rust type corresponds to.
|
|
22
|
+
fn nc_type() -> NcType;
|
|
23
|
+
|
|
24
|
+
/// Read a single element from big-endian bytes.
|
|
25
|
+
fn from_be_bytes(bytes: &[u8]) -> Result<Self>;
|
|
26
|
+
|
|
27
|
+
/// Size in bytes of one element.
|
|
28
|
+
fn element_size() -> usize;
|
|
29
|
+
|
|
30
|
+
/// Bulk decode `count` elements from a contiguous big-endian byte slice.
|
|
31
|
+
///
|
|
32
|
+
/// Default implementation falls back to per-element decoding. Types with
|
|
33
|
+
/// multi-byte elements override this with an optimized bulk path using
|
|
34
|
+
/// `chunks_exact` + byte-swap (on LE hosts) or `copy_nonoverlapping`
|
|
35
|
+
/// (on BE hosts).
|
|
36
|
+
fn decode_bulk_be(raw: &[u8], count: usize) -> Result<Vec<Self>> {
|
|
37
|
+
let elem_size = Self::element_size();
|
|
38
|
+
let needed = count.checked_mul(elem_size).ok_or_else(|| {
|
|
39
|
+
Error::InvalidData("classic decode byte count exceeds platform usize".to_string())
|
|
40
|
+
})?;
|
|
41
|
+
if raw.len() < needed {
|
|
42
|
+
return Err(Error::InvalidData(format!(
|
|
43
|
+
"need {} bytes for {} elements, got {}",
|
|
44
|
+
needed,
|
|
45
|
+
count,
|
|
46
|
+
raw.len()
|
|
47
|
+
)));
|
|
48
|
+
}
|
|
49
|
+
let mut values = Vec::with_capacity(count);
|
|
50
|
+
for i in 0..count {
|
|
51
|
+
let start = i * elem_size;
|
|
52
|
+
values.push(Self::from_be_bytes(&raw[start..start + elem_size])?);
|
|
53
|
+
}
|
|
54
|
+
Ok(values)
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/// Bulk decode elements from a contiguous big-endian byte slice into a
|
|
58
|
+
/// caller-provided destination buffer.
|
|
59
|
+
fn decode_bulk_be_into(raw: &[u8], dst: &mut [Self]) -> Result<()> {
|
|
60
|
+
let elem_size = Self::element_size();
|
|
61
|
+
let needed = dst.len().checked_mul(elem_size).ok_or_else(|| {
|
|
62
|
+
Error::InvalidData("classic decode byte count exceeds platform usize".to_string())
|
|
63
|
+
})?;
|
|
64
|
+
if raw.len() < needed {
|
|
65
|
+
return Err(Error::InvalidData(format!(
|
|
66
|
+
"need {} bytes for {} elements, got {}",
|
|
67
|
+
needed,
|
|
68
|
+
dst.len(),
|
|
69
|
+
raw.len()
|
|
70
|
+
)));
|
|
71
|
+
}
|
|
72
|
+
for (out, chunk) in dst.iter_mut().zip(raw[..needed].chunks_exact(elem_size)) {
|
|
73
|
+
*out = Self::from_be_bytes(chunk)?;
|
|
74
|
+
}
|
|
75
|
+
Ok(())
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
macro_rules! impl_nc_read_type {
|
|
80
|
+
($ty:ty, $nc_type:path, $size:literal) => {
|
|
81
|
+
impl NcReadType for $ty {
|
|
82
|
+
fn nc_type() -> NcType {
|
|
83
|
+
$nc_type
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
fn from_be_bytes(bytes: &[u8]) -> Result<Self> {
|
|
87
|
+
if bytes.len() < $size {
|
|
88
|
+
return Err(Error::InvalidData(format!(
|
|
89
|
+
"need {} bytes for {}, got {}",
|
|
90
|
+
$size,
|
|
91
|
+
stringify!($ty),
|
|
92
|
+
bytes.len()
|
|
93
|
+
)));
|
|
94
|
+
}
|
|
95
|
+
let mut arr = [0u8; $size];
|
|
96
|
+
arr.copy_from_slice(&bytes[..$size]);
|
|
97
|
+
Ok(<$ty>::from_be_bytes(arr))
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
fn element_size() -> usize {
|
|
101
|
+
$size
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
fn decode_bulk_be(raw: &[u8], count: usize) -> Result<Vec<Self>> {
|
|
105
|
+
let total_bytes = count.checked_mul($size).ok_or_else(|| {
|
|
106
|
+
Error::InvalidData(
|
|
107
|
+
"classic decode byte count exceeds platform usize".to_string(),
|
|
108
|
+
)
|
|
109
|
+
})?;
|
|
110
|
+
if raw.len() < total_bytes {
|
|
111
|
+
return Err(Error::InvalidData(format!(
|
|
112
|
+
"need {} bytes for {} elements of {}, got {}",
|
|
113
|
+
total_bytes,
|
|
114
|
+
count,
|
|
115
|
+
stringify!($ty),
|
|
116
|
+
raw.len()
|
|
117
|
+
)));
|
|
118
|
+
}
|
|
119
|
+
let bytes = &raw[..total_bytes];
|
|
120
|
+
#[cfg(target_endian = "big")]
|
|
121
|
+
{
|
|
122
|
+
let mut values = Vec::<$ty>::with_capacity(count);
|
|
123
|
+
// SAFETY: `values` has capacity for `count` elements,
|
|
124
|
+
// `raw` contains exactly `count * size_of::<$ty>()` bytes,
|
|
125
|
+
// and numeric types admit every bit pattern.
|
|
126
|
+
unsafe {
|
|
127
|
+
std::ptr::copy_nonoverlapping(
|
|
128
|
+
bytes.as_ptr(),
|
|
129
|
+
values.as_mut_ptr() as *mut u8,
|
|
130
|
+
total_bytes,
|
|
131
|
+
);
|
|
132
|
+
values.set_len(count);
|
|
133
|
+
}
|
|
134
|
+
Ok(values)
|
|
135
|
+
}
|
|
136
|
+
#[cfg(target_endian = "little")]
|
|
137
|
+
{
|
|
138
|
+
// LE host reading BE data: chunks_exact + byte-swap.
|
|
139
|
+
Ok(bytes
|
|
140
|
+
.chunks_exact($size)
|
|
141
|
+
.map(|chunk| {
|
|
142
|
+
let mut arr = [0u8; $size];
|
|
143
|
+
arr.copy_from_slice(chunk);
|
|
144
|
+
<$ty>::from_be_bytes(arr)
|
|
145
|
+
})
|
|
146
|
+
.collect())
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
fn decode_bulk_be_into(raw: &[u8], dst: &mut [Self]) -> Result<()> {
|
|
151
|
+
let total_bytes = dst.len().checked_mul($size).ok_or_else(|| {
|
|
152
|
+
Error::InvalidData(
|
|
153
|
+
"classic decode byte count exceeds platform usize".to_string(),
|
|
154
|
+
)
|
|
155
|
+
})?;
|
|
156
|
+
if raw.len() < total_bytes {
|
|
157
|
+
return Err(Error::InvalidData(format!(
|
|
158
|
+
"need {} bytes for {} elements of {}, got {}",
|
|
159
|
+
total_bytes,
|
|
160
|
+
dst.len(),
|
|
161
|
+
stringify!($ty),
|
|
162
|
+
raw.len()
|
|
163
|
+
)));
|
|
164
|
+
}
|
|
165
|
+
let bytes = &raw[..total_bytes];
|
|
166
|
+
#[cfg(target_endian = "big")]
|
|
167
|
+
{
|
|
168
|
+
// SAFETY: `values` has capacity for `count` bytes and
|
|
169
|
+
// `bytes` contains exactly `count` initialized bytes.
|
|
170
|
+
unsafe {
|
|
171
|
+
std::ptr::copy_nonoverlapping(
|
|
172
|
+
bytes.as_ptr(),
|
|
173
|
+
dst.as_mut_ptr() as *mut u8,
|
|
174
|
+
total_bytes,
|
|
175
|
+
);
|
|
176
|
+
}
|
|
177
|
+
Ok(())
|
|
178
|
+
}
|
|
179
|
+
#[cfg(target_endian = "little")]
|
|
180
|
+
{
|
|
181
|
+
for (out, chunk) in dst.iter_mut().zip(bytes.chunks_exact($size)) {
|
|
182
|
+
let mut arr = [0u8; $size];
|
|
183
|
+
arr.copy_from_slice(chunk);
|
|
184
|
+
*out = <$ty>::from_be_bytes(arr);
|
|
185
|
+
}
|
|
186
|
+
Ok(())
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
impl_nc_read_type!(i8, NcType::Byte, 1);
|
|
194
|
+
impl_nc_read_type!(i16, NcType::Short, 2);
|
|
195
|
+
impl_nc_read_type!(i32, NcType::Int, 4);
|
|
196
|
+
impl_nc_read_type!(f32, NcType::Float, 4);
|
|
197
|
+
impl_nc_read_type!(f64, NcType::Double, 8);
|
|
198
|
+
impl_nc_read_type!(u8, NcType::UByte, 1);
|
|
199
|
+
impl_nc_read_type!(u16, NcType::UShort, 2);
|
|
200
|
+
impl_nc_read_type!(u32, NcType::UInt, 4);
|
|
201
|
+
impl_nc_read_type!(i64, NcType::Int64, 8);
|
|
202
|
+
impl_nc_read_type!(u64, NcType::UInt64, 8);
|
|
203
|
+
|
|
204
|
+
/// Read the entire data for a non-record variable into an ndarray.
|
|
205
|
+
///
|
|
206
|
+
/// The data is located at a contiguous region starting at `var.data_offset`
|
|
207
|
+
/// with total size `var.data_size`.
|
|
208
|
+
pub fn read_non_record_variable<T: NcReadType>(
|
|
209
|
+
file_data: &[u8],
|
|
210
|
+
var: &NcVariable,
|
|
211
|
+
) -> Result<ArrayD<T>> {
|
|
212
|
+
if var.is_record_var {
|
|
213
|
+
return Err(Error::InvalidData(
|
|
214
|
+
"use read_record_variable for record variables".to_string(),
|
|
215
|
+
));
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
let offset = crate::types::checked_usize_from_u64(var.data_offset, "variable data offset")?;
|
|
219
|
+
let total_elements = checked_non_record_element_count(var)?;
|
|
220
|
+
let elem_size = T::element_size();
|
|
221
|
+
let total_bytes = total_elements.checked_mul(elem_size).ok_or_else(|| {
|
|
222
|
+
Error::InvalidData(format!(
|
|
223
|
+
"variable '{}' size in bytes exceeds platform usize",
|
|
224
|
+
var.name
|
|
225
|
+
))
|
|
226
|
+
})?;
|
|
227
|
+
|
|
228
|
+
let end = offset.checked_add(total_bytes).ok_or_else(|| {
|
|
229
|
+
Error::InvalidData(format!(
|
|
230
|
+
"variable '{}' byte range exceeds platform usize",
|
|
231
|
+
var.name
|
|
232
|
+
))
|
|
233
|
+
})?;
|
|
234
|
+
if end > file_data.len() {
|
|
235
|
+
return Err(Error::InvalidData(format!(
|
|
236
|
+
"variable '{}' data extends beyond file: offset={}, size={}, file_len={}",
|
|
237
|
+
var.name,
|
|
238
|
+
offset,
|
|
239
|
+
total_bytes,
|
|
240
|
+
file_data.len()
|
|
241
|
+
)));
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
let data_slice = &file_data[offset..end];
|
|
245
|
+
let values = T::decode_bulk_be(data_slice, total_elements)?;
|
|
246
|
+
|
|
247
|
+
let shape: Vec<usize> = var
|
|
248
|
+
.shape()
|
|
249
|
+
.iter()
|
|
250
|
+
.map(|&s| crate::types::checked_usize_from_u64(s, "variable dimension"))
|
|
251
|
+
.collect::<Result<Vec<_>>>()?;
|
|
252
|
+
if shape.is_empty() {
|
|
253
|
+
// Scalar variable.
|
|
254
|
+
ArrayD::from_shape_vec(IxDyn(&[]), values)
|
|
255
|
+
} else {
|
|
256
|
+
ArrayD::from_shape_vec(IxDyn(&shape), values)
|
|
257
|
+
}
|
|
258
|
+
.map_err(|e| Error::InvalidData(format!("failed to create array: {}", e)))
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/// Read the entire data for a non-record variable from range-backed storage.
|
|
262
|
+
pub(crate) fn read_non_record_variable_from_storage<T: NcReadType>(
|
|
263
|
+
storage: &ClassicStorage,
|
|
264
|
+
var: &NcVariable,
|
|
265
|
+
) -> Result<ArrayD<T>> {
|
|
266
|
+
if var.is_record_var {
|
|
267
|
+
return Err(Error::InvalidData(
|
|
268
|
+
"use read_record_variable_from_storage for record variables".to_string(),
|
|
269
|
+
));
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
let total_elements = checked_non_record_element_count(var)?;
|
|
273
|
+
let total_bytes = variable_data_bytes::<T>(var.name.as_str(), total_elements)?;
|
|
274
|
+
let data = storage.read_range(var.data_offset, total_bytes)?;
|
|
275
|
+
let values = T::decode_bulk_be(data.as_ref(), total_elements)?;
|
|
276
|
+
let shape = checked_variable_shape(var)?;
|
|
277
|
+
|
|
278
|
+
ArrayD::from_shape_vec(IxDyn(&shape), values)
|
|
279
|
+
.map_err(|e| Error::InvalidData(format!("failed to create array: {}", e)))
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/// Read the entire data for a non-record variable using Rayon for large reads.
|
|
283
|
+
#[cfg(feature = "rayon")]
|
|
284
|
+
pub(crate) fn read_non_record_variable_parallel_from_storage<T: NcReadType>(
|
|
285
|
+
storage: &ClassicStorage,
|
|
286
|
+
var: &NcVariable,
|
|
287
|
+
) -> Result<ArrayD<T>> {
|
|
288
|
+
if var.is_record_var {
|
|
289
|
+
return Err(Error::InvalidData(
|
|
290
|
+
"use read_record_variable_parallel_from_storage for record variables".to_string(),
|
|
291
|
+
));
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
let total_elements = checked_non_record_element_count(var)?;
|
|
295
|
+
let total_bytes = variable_data_bytes::<T>(var.name.as_str(), total_elements)?;
|
|
296
|
+
let policy = storage.parallel_read_policy();
|
|
297
|
+
if total_bytes < policy.min_bytes || total_elements == 0 {
|
|
298
|
+
return read_non_record_variable_from_storage(storage, var);
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
// The parallel path allocates the whole result up front, so validate the
|
|
302
|
+
// declared byte range fits in storage before dispatching; the serial path
|
|
303
|
+
// already bounds-checks per read.
|
|
304
|
+
let end = var
|
|
305
|
+
.data_offset
|
|
306
|
+
.checked_add(total_bytes as u64)
|
|
307
|
+
.ok_or_else(|| {
|
|
308
|
+
Error::InvalidData(format!("variable '{}' byte range exceeds u64", var.name))
|
|
309
|
+
})?;
|
|
310
|
+
if end > storage.len() {
|
|
311
|
+
return Err(Error::InvalidData(format!(
|
|
312
|
+
"variable '{}' data extends beyond file: needs {end} bytes, have {}",
|
|
313
|
+
var.name,
|
|
314
|
+
storage.len()
|
|
315
|
+
)));
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
let values = read_contiguous_range_parallel::<T>(
|
|
319
|
+
storage,
|
|
320
|
+
var.data_offset,
|
|
321
|
+
total_elements,
|
|
322
|
+
policy.target_chunk_bytes,
|
|
323
|
+
)?;
|
|
324
|
+
let shape = checked_variable_shape(var)?;
|
|
325
|
+
|
|
326
|
+
ArrayD::from_shape_vec(IxDyn(&shape), values)
|
|
327
|
+
.map_err(|e| Error::InvalidData(format!("failed to create array: {}", e)))
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/// Read the entire data for a non-record variable into a caller-provided buffer.
|
|
331
|
+
pub fn read_non_record_variable_into<T: NcReadType>(
|
|
332
|
+
file_data: &[u8],
|
|
333
|
+
var: &NcVariable,
|
|
334
|
+
dst: &mut [T],
|
|
335
|
+
) -> Result<()> {
|
|
336
|
+
if var.is_record_var {
|
|
337
|
+
return Err(Error::InvalidData(
|
|
338
|
+
"use read_record_variable_into for record variables".to_string(),
|
|
339
|
+
));
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
let total_elements = checked_non_record_element_count(var)?;
|
|
343
|
+
if dst.len() != total_elements {
|
|
344
|
+
return Err(Error::InvalidData(format!(
|
|
345
|
+
"destination has {} elements, variable '{}' requires {}",
|
|
346
|
+
dst.len(),
|
|
347
|
+
var.name,
|
|
348
|
+
total_elements
|
|
349
|
+
)));
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
let offset = crate::types::checked_usize_from_u64(var.data_offset, "variable data offset")?;
|
|
353
|
+
let elem_size = T::element_size();
|
|
354
|
+
let total_bytes = total_elements.checked_mul(elem_size).ok_or_else(|| {
|
|
355
|
+
Error::InvalidData(format!(
|
|
356
|
+
"variable '{}' size in bytes exceeds platform usize",
|
|
357
|
+
var.name
|
|
358
|
+
))
|
|
359
|
+
})?;
|
|
360
|
+
|
|
361
|
+
let end = offset.checked_add(total_bytes).ok_or_else(|| {
|
|
362
|
+
Error::InvalidData(format!(
|
|
363
|
+
"variable '{}' byte range exceeds platform usize",
|
|
364
|
+
var.name
|
|
365
|
+
))
|
|
366
|
+
})?;
|
|
367
|
+
if end > file_data.len() {
|
|
368
|
+
return Err(Error::InvalidData(format!(
|
|
369
|
+
"variable '{}' data extends beyond file: offset={}, size={}, file_len={}",
|
|
370
|
+
var.name,
|
|
371
|
+
offset,
|
|
372
|
+
total_bytes,
|
|
373
|
+
file_data.len()
|
|
374
|
+
)));
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
T::decode_bulk_be_into(&file_data[offset..end], dst)
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
/// Read a non-record variable from range-backed storage into a caller-provided buffer.
|
|
381
|
+
pub(crate) fn read_non_record_variable_into_from_storage<T: NcReadType>(
|
|
382
|
+
storage: &ClassicStorage,
|
|
383
|
+
var: &NcVariable,
|
|
384
|
+
dst: &mut [T],
|
|
385
|
+
) -> Result<()> {
|
|
386
|
+
if var.is_record_var {
|
|
387
|
+
return Err(Error::InvalidData(
|
|
388
|
+
"use read_record_variable_into_from_storage for record variables".to_string(),
|
|
389
|
+
));
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
let total_elements = checked_non_record_element_count(var)?;
|
|
393
|
+
if dst.len() != total_elements {
|
|
394
|
+
return Err(Error::InvalidData(format!(
|
|
395
|
+
"destination has {} elements, variable '{}' requires {}",
|
|
396
|
+
dst.len(),
|
|
397
|
+
var.name,
|
|
398
|
+
total_elements
|
|
399
|
+
)));
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
let total_bytes = variable_data_bytes::<T>(var.name.as_str(), total_elements)?;
|
|
403
|
+
let data = storage.read_range(var.data_offset, total_bytes)?;
|
|
404
|
+
T::decode_bulk_be_into(data.as_ref(), dst)
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/// Read the entire data for a record variable into an ndarray.
|
|
408
|
+
///
|
|
409
|
+
/// Record variables are interleaved: for each of `numrecs` records, every record
|
|
410
|
+
/// variable contributes `record_size` bytes (padded to 4-byte alignment for CDF-1/2).
|
|
411
|
+
/// The `record_stride` is the total size of one record across all record variables.
|
|
412
|
+
///
|
|
413
|
+
/// Parameters:
|
|
414
|
+
/// - `file_data`: the raw file bytes
|
|
415
|
+
/// - `var`: the record variable to read
|
|
416
|
+
/// - `numrecs`: number of records (from the file header)
|
|
417
|
+
/// - `record_stride`: total bytes per record (sum of all record variables' padded vsizes)
|
|
418
|
+
pub fn read_record_variable<T: NcReadType>(
|
|
419
|
+
file_data: &[u8],
|
|
420
|
+
var: &NcVariable,
|
|
421
|
+
numrecs: u64,
|
|
422
|
+
record_stride: u64,
|
|
423
|
+
) -> Result<ArrayD<T>> {
|
|
424
|
+
if !var.is_record_var {
|
|
425
|
+
return Err(Error::InvalidData(
|
|
426
|
+
"use read_non_record_variable for non-record variables".to_string(),
|
|
427
|
+
));
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
let elem_size = T::element_size();
|
|
431
|
+
let base_offset =
|
|
432
|
+
crate::types::checked_usize_from_u64(var.data_offset, "record variable data offset")?;
|
|
433
|
+
let record_stride_usize = crate::types::checked_usize_from_u64(record_stride, "record stride")?;
|
|
434
|
+
|
|
435
|
+
// Shape: the first dimension is the unlimited dimension, replaced by numrecs.
|
|
436
|
+
let shape = checked_record_shape(var, numrecs)?;
|
|
437
|
+
let numrecs_usize = shape[0];
|
|
438
|
+
|
|
439
|
+
// Number of elements per record (product of all dims except the first).
|
|
440
|
+
let elements_per_record = checked_record_elements_per_record(var)?;
|
|
441
|
+
let bytes_per_record = elements_per_record.checked_mul(elem_size).ok_or_else(|| {
|
|
442
|
+
Error::InvalidData(format!(
|
|
443
|
+
"record variable '{}' bytes per record exceed platform usize",
|
|
444
|
+
var.name
|
|
445
|
+
))
|
|
446
|
+
})?;
|
|
447
|
+
let total_elements = numrecs_usize
|
|
448
|
+
.checked_mul(elements_per_record)
|
|
449
|
+
.ok_or_else(|| {
|
|
450
|
+
Error::InvalidData(format!(
|
|
451
|
+
"record variable '{}' element count exceeds platform usize",
|
|
452
|
+
var.name
|
|
453
|
+
))
|
|
454
|
+
})?;
|
|
455
|
+
|
|
456
|
+
// Validate that every record fits in the file before allocating, so a
|
|
457
|
+
// crafted `numrecs` cannot force a huge up-front allocation.
|
|
458
|
+
validate_record_span(
|
|
459
|
+
var,
|
|
460
|
+
numrecs,
|
|
461
|
+
record_stride,
|
|
462
|
+
bytes_per_record as u64,
|
|
463
|
+
file_data.len() as u64,
|
|
464
|
+
)?;
|
|
465
|
+
|
|
466
|
+
let mut values = Vec::with_capacity(total_elements);
|
|
467
|
+
|
|
468
|
+
for rec in 0..numrecs_usize {
|
|
469
|
+
let rec_offset = base_offset
|
|
470
|
+
.checked_add(rec.checked_mul(record_stride_usize).ok_or_else(|| {
|
|
471
|
+
Error::InvalidData(format!(
|
|
472
|
+
"record variable '{}' byte offset exceeds platform usize",
|
|
473
|
+
var.name
|
|
474
|
+
))
|
|
475
|
+
})?)
|
|
476
|
+
.ok_or_else(|| {
|
|
477
|
+
Error::InvalidData(format!(
|
|
478
|
+
"record variable '{}' byte offset exceeds platform usize",
|
|
479
|
+
var.name
|
|
480
|
+
))
|
|
481
|
+
})?;
|
|
482
|
+
let rec_end = rec_offset.checked_add(bytes_per_record).ok_or_else(|| {
|
|
483
|
+
Error::InvalidData(format!(
|
|
484
|
+
"record variable '{}' record range exceeds platform usize",
|
|
485
|
+
var.name
|
|
486
|
+
))
|
|
487
|
+
})?;
|
|
488
|
+
if rec_end > file_data.len() {
|
|
489
|
+
return Err(Error::InvalidData(format!(
|
|
490
|
+
"record {} for variable '{}' extends beyond file",
|
|
491
|
+
rec, var.name
|
|
492
|
+
)));
|
|
493
|
+
}
|
|
494
|
+
let rec_slice = &file_data[rec_offset..rec_end];
|
|
495
|
+
let rec_values = T::decode_bulk_be(rec_slice, elements_per_record)?;
|
|
496
|
+
values.extend(rec_values);
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
ArrayD::from_shape_vec(IxDyn(&shape), values)
|
|
500
|
+
.map_err(|e| Error::InvalidData(format!("failed to create array: {}", e)))
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
/// Read the entire data for a record variable from range-backed storage.
|
|
504
|
+
pub(crate) fn read_record_variable_from_storage<T: NcReadType>(
|
|
505
|
+
storage: &ClassicStorage,
|
|
506
|
+
var: &NcVariable,
|
|
507
|
+
numrecs: u64,
|
|
508
|
+
record_stride: u64,
|
|
509
|
+
) -> Result<ArrayD<T>> {
|
|
510
|
+
if !var.is_record_var {
|
|
511
|
+
return Err(Error::InvalidData(
|
|
512
|
+
"use read_non_record_variable_from_storage for non-record variables".to_string(),
|
|
513
|
+
));
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
let shape = checked_record_shape(var, numrecs)?;
|
|
517
|
+
let elements_per_record = checked_record_elements_per_record(var)?;
|
|
518
|
+
let bytes_per_record = variable_data_bytes::<T>(var.name.as_str(), elements_per_record)?;
|
|
519
|
+
let numrecs_usize = crate::types::checked_usize_from_u64(numrecs, "record count")?;
|
|
520
|
+
let total_elements = numrecs_usize
|
|
521
|
+
.checked_mul(elements_per_record)
|
|
522
|
+
.ok_or_else(|| {
|
|
523
|
+
Error::InvalidData(format!(
|
|
524
|
+
"record variable '{}' element count exceeds platform usize",
|
|
525
|
+
var.name
|
|
526
|
+
))
|
|
527
|
+
})?;
|
|
528
|
+
validate_record_span(
|
|
529
|
+
var,
|
|
530
|
+
numrecs,
|
|
531
|
+
record_stride,
|
|
532
|
+
bytes_per_record as u64,
|
|
533
|
+
storage.len(),
|
|
534
|
+
)?;
|
|
535
|
+
let mut values = Vec::with_capacity(total_elements);
|
|
536
|
+
|
|
537
|
+
for rec in 0..numrecs {
|
|
538
|
+
let rec_offset = record_byte_offset(var, rec, record_stride)?;
|
|
539
|
+
let rec_slice = storage.read_range(rec_offset, bytes_per_record)?;
|
|
540
|
+
let rec_values = T::decode_bulk_be(rec_slice.as_ref(), elements_per_record)?;
|
|
541
|
+
values.extend(rec_values);
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
ArrayD::from_shape_vec(IxDyn(&shape), values)
|
|
545
|
+
.map_err(|e| Error::InvalidData(format!("failed to create array: {}", e)))
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
/// Read the entire data for a record variable using Rayon for large reads.
|
|
549
|
+
#[cfg(feature = "rayon")]
|
|
550
|
+
pub(crate) fn read_record_variable_parallel_from_storage<T: NcReadType>(
|
|
551
|
+
storage: &ClassicStorage,
|
|
552
|
+
var: &NcVariable,
|
|
553
|
+
numrecs: u64,
|
|
554
|
+
record_stride: u64,
|
|
555
|
+
) -> Result<ArrayD<T>> {
|
|
556
|
+
if !var.is_record_var {
|
|
557
|
+
return Err(Error::InvalidData(
|
|
558
|
+
"use read_non_record_variable_parallel_from_storage for non-record variables"
|
|
559
|
+
.to_string(),
|
|
560
|
+
));
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
let shape = checked_record_shape(var, numrecs)?;
|
|
564
|
+
let elements_per_record = checked_record_elements_per_record(var)?;
|
|
565
|
+
let bytes_per_record = variable_data_bytes::<T>(var.name.as_str(), elements_per_record)?;
|
|
566
|
+
let numrecs_usize = crate::types::checked_usize_from_u64(numrecs, "record count")?;
|
|
567
|
+
let total_elements = numrecs_usize
|
|
568
|
+
.checked_mul(elements_per_record)
|
|
569
|
+
.ok_or_else(|| {
|
|
570
|
+
Error::InvalidData(format!(
|
|
571
|
+
"record variable '{}' element count exceeds platform usize",
|
|
572
|
+
var.name
|
|
573
|
+
))
|
|
574
|
+
})?;
|
|
575
|
+
let logical_bytes = numrecs_usize.checked_mul(bytes_per_record).ok_or_else(|| {
|
|
576
|
+
Error::InvalidData(format!(
|
|
577
|
+
"record variable '{}' logical byte count exceeds platform usize",
|
|
578
|
+
var.name
|
|
579
|
+
))
|
|
580
|
+
})?;
|
|
581
|
+
validate_record_span(
|
|
582
|
+
var,
|
|
583
|
+
numrecs,
|
|
584
|
+
record_stride,
|
|
585
|
+
bytes_per_record as u64,
|
|
586
|
+
storage.len(),
|
|
587
|
+
)?;
|
|
588
|
+
let policy = storage.parallel_read_policy();
|
|
589
|
+
if logical_bytes < policy.min_bytes || numrecs_usize <= 1 {
|
|
590
|
+
return read_record_variable_from_storage(storage, var, numrecs, record_stride);
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
let records_per_chunk = (policy.target_chunk_bytes / bytes_per_record.max(1)).max(1);
|
|
594
|
+
let elements_per_chunk = records_per_chunk
|
|
595
|
+
.checked_mul(elements_per_record)
|
|
596
|
+
.ok_or_else(|| {
|
|
597
|
+
Error::InvalidData(
|
|
598
|
+
"classic record chunk element count exceeds platform usize".to_string(),
|
|
599
|
+
)
|
|
600
|
+
})?;
|
|
601
|
+
let mut values = vec![T::default(); total_elements];
|
|
602
|
+
let chunk_plan = RecordChunkReadPlan {
|
|
603
|
+
var,
|
|
604
|
+
record_stride,
|
|
605
|
+
elements_per_record,
|
|
606
|
+
bytes_per_record,
|
|
607
|
+
};
|
|
608
|
+
values
|
|
609
|
+
.par_chunks_mut(elements_per_chunk)
|
|
610
|
+
.enumerate()
|
|
611
|
+
.try_for_each(|(chunk, dst)| {
|
|
612
|
+
let first_record = chunk.checked_mul(records_per_chunk).ok_or_else(|| {
|
|
613
|
+
Error::InvalidData("classic record chunk offset exceeds platform usize".to_string())
|
|
614
|
+
})?;
|
|
615
|
+
let records = dst.len().checked_div(elements_per_record).ok_or_else(|| {
|
|
616
|
+
Error::InvalidData("classic record elements per record is zero".to_string())
|
|
617
|
+
})?;
|
|
618
|
+
read_record_chunk_into::<T>(storage, &chunk_plan, first_record as u64, records, dst)
|
|
619
|
+
})?;
|
|
620
|
+
|
|
621
|
+
ArrayD::from_shape_vec(IxDyn(&shape), values)
|
|
622
|
+
.map_err(|e| Error::InvalidData(format!("failed to create array: {}", e)))
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
/// Read the entire data for a record variable into a caller-provided buffer.
|
|
626
|
+
pub fn read_record_variable_into<T: NcReadType>(
|
|
627
|
+
file_data: &[u8],
|
|
628
|
+
var: &NcVariable,
|
|
629
|
+
numrecs: u64,
|
|
630
|
+
record_stride: u64,
|
|
631
|
+
dst: &mut [T],
|
|
632
|
+
) -> Result<()> {
|
|
633
|
+
if !var.is_record_var {
|
|
634
|
+
return Err(Error::InvalidData(
|
|
635
|
+
"use read_non_record_variable_into for non-record variables".to_string(),
|
|
636
|
+
));
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
let elem_size = T::element_size();
|
|
640
|
+
let base_offset =
|
|
641
|
+
crate::types::checked_usize_from_u64(var.data_offset, "record variable data offset")?;
|
|
642
|
+
let numrecs_usize = crate::types::checked_usize_from_u64(numrecs, "record count")?;
|
|
643
|
+
let record_stride_usize = crate::types::checked_usize_from_u64(record_stride, "record stride")?;
|
|
644
|
+
|
|
645
|
+
if var.dimensions.is_empty() {
|
|
646
|
+
return Err(Error::InvalidData(
|
|
647
|
+
"record variable must have at least one dimension".to_string(),
|
|
648
|
+
));
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
let elements_per_record = checked_record_elements_per_record(var)?;
|
|
652
|
+
let bytes_per_record = elements_per_record.checked_mul(elem_size).ok_or_else(|| {
|
|
653
|
+
Error::InvalidData(format!(
|
|
654
|
+
"record variable '{}' bytes per record exceed platform usize",
|
|
655
|
+
var.name
|
|
656
|
+
))
|
|
657
|
+
})?;
|
|
658
|
+
let total_elements = numrecs_usize
|
|
659
|
+
.checked_mul(elements_per_record)
|
|
660
|
+
.ok_or_else(|| {
|
|
661
|
+
Error::InvalidData(format!(
|
|
662
|
+
"record variable '{}' element count exceeds platform usize",
|
|
663
|
+
var.name
|
|
664
|
+
))
|
|
665
|
+
})?;
|
|
666
|
+
if dst.len() != total_elements {
|
|
667
|
+
return Err(Error::InvalidData(format!(
|
|
668
|
+
"destination has {} elements, variable '{}' requires {}",
|
|
669
|
+
dst.len(),
|
|
670
|
+
var.name,
|
|
671
|
+
total_elements
|
|
672
|
+
)));
|
|
673
|
+
}
|
|
674
|
+
|
|
675
|
+
for rec in 0..numrecs_usize {
|
|
676
|
+
let rec_offset = base_offset
|
|
677
|
+
.checked_add(rec.checked_mul(record_stride_usize).ok_or_else(|| {
|
|
678
|
+
Error::InvalidData(format!(
|
|
679
|
+
"record variable '{}' byte offset exceeds platform usize",
|
|
680
|
+
var.name
|
|
681
|
+
))
|
|
682
|
+
})?)
|
|
683
|
+
.ok_or_else(|| {
|
|
684
|
+
Error::InvalidData(format!(
|
|
685
|
+
"record variable '{}' byte offset exceeds platform usize",
|
|
686
|
+
var.name
|
|
687
|
+
))
|
|
688
|
+
})?;
|
|
689
|
+
let rec_end = rec_offset.checked_add(bytes_per_record).ok_or_else(|| {
|
|
690
|
+
Error::InvalidData(format!(
|
|
691
|
+
"record variable '{}' record range exceeds platform usize",
|
|
692
|
+
var.name
|
|
693
|
+
))
|
|
694
|
+
})?;
|
|
695
|
+
if rec_end > file_data.len() {
|
|
696
|
+
return Err(Error::InvalidData(format!(
|
|
697
|
+
"record {} for variable '{}' extends beyond file",
|
|
698
|
+
rec, var.name
|
|
699
|
+
)));
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
let dst_start = rec.checked_mul(elements_per_record).ok_or_else(|| {
|
|
703
|
+
Error::InvalidData(format!(
|
|
704
|
+
"record variable '{}' destination offset exceeds platform usize",
|
|
705
|
+
var.name
|
|
706
|
+
))
|
|
707
|
+
})?;
|
|
708
|
+
let dst_end = dst_start.checked_add(elements_per_record).ok_or_else(|| {
|
|
709
|
+
Error::InvalidData(format!(
|
|
710
|
+
"record variable '{}' destination range exceeds platform usize",
|
|
711
|
+
var.name
|
|
712
|
+
))
|
|
713
|
+
})?;
|
|
714
|
+
T::decode_bulk_be_into(
|
|
715
|
+
&file_data[rec_offset..rec_end],
|
|
716
|
+
&mut dst[dst_start..dst_end],
|
|
717
|
+
)?;
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
Ok(())
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
/// Read a record variable from range-backed storage into a caller-provided buffer.
|
|
724
|
+
pub(crate) fn read_record_variable_into_from_storage<T: NcReadType>(
|
|
725
|
+
storage: &ClassicStorage,
|
|
726
|
+
var: &NcVariable,
|
|
727
|
+
numrecs: u64,
|
|
728
|
+
record_stride: u64,
|
|
729
|
+
dst: &mut [T],
|
|
730
|
+
) -> Result<()> {
|
|
731
|
+
if !var.is_record_var {
|
|
732
|
+
return Err(Error::InvalidData(
|
|
733
|
+
"use read_non_record_variable_into_from_storage for non-record variables".to_string(),
|
|
734
|
+
));
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
if var.dimensions.is_empty() {
|
|
738
|
+
return Err(Error::InvalidData(
|
|
739
|
+
"record variable must have at least one dimension".to_string(),
|
|
740
|
+
));
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
let elements_per_record = checked_record_elements_per_record(var)?;
|
|
744
|
+
let bytes_per_record = variable_data_bytes::<T>(var.name.as_str(), elements_per_record)?;
|
|
745
|
+
let numrecs_usize = crate::types::checked_usize_from_u64(numrecs, "record count")?;
|
|
746
|
+
let total_elements = numrecs_usize
|
|
747
|
+
.checked_mul(elements_per_record)
|
|
748
|
+
.ok_or_else(|| {
|
|
749
|
+
Error::InvalidData(format!(
|
|
750
|
+
"record variable '{}' element count exceeds platform usize",
|
|
751
|
+
var.name
|
|
752
|
+
))
|
|
753
|
+
})?;
|
|
754
|
+
if dst.len() != total_elements {
|
|
755
|
+
return Err(Error::InvalidData(format!(
|
|
756
|
+
"destination has {} elements, variable '{}' requires {}",
|
|
757
|
+
dst.len(),
|
|
758
|
+
var.name,
|
|
759
|
+
total_elements
|
|
760
|
+
)));
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
for rec in 0..numrecs {
|
|
764
|
+
let rec_offset = record_byte_offset(var, rec, record_stride)?;
|
|
765
|
+
let rec_slice = storage.read_range(rec_offset, bytes_per_record)?;
|
|
766
|
+
let dst_start = crate::types::checked_usize_from_u64(rec, "record index")?
|
|
767
|
+
.checked_mul(elements_per_record)
|
|
768
|
+
.ok_or_else(|| {
|
|
769
|
+
Error::InvalidData(format!(
|
|
770
|
+
"record variable '{}' destination offset exceeds platform usize",
|
|
771
|
+
var.name
|
|
772
|
+
))
|
|
773
|
+
})?;
|
|
774
|
+
let dst_end = dst_start.checked_add(elements_per_record).ok_or_else(|| {
|
|
775
|
+
Error::InvalidData(format!(
|
|
776
|
+
"record variable '{}' destination range exceeds platform usize",
|
|
777
|
+
var.name
|
|
778
|
+
))
|
|
779
|
+
})?;
|
|
780
|
+
T::decode_bulk_be_into(rec_slice.as_ref(), &mut dst[dst_start..dst_end])?;
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
Ok(())
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
/// Compute the record stride: total bytes per record across all record variables.
|
|
787
|
+
///
|
|
788
|
+
/// Each record variable's per-record contribution is its unpadded
|
|
789
|
+
/// `record_size` (derived from its dimensions), padded to a 4-byte boundary —
|
|
790
|
+
/// except in the spec's special case of exactly one record variable, whose
|
|
791
|
+
/// records are packed without padding.
|
|
792
|
+
pub fn compute_record_stride(variables: &[NcVariable]) -> Result<u64> {
|
|
793
|
+
let mut record_vars = variables.iter().filter(|v| v.is_record_var);
|
|
794
|
+
let (first, second) = (record_vars.next(), record_vars.next());
|
|
795
|
+
if let (Some(only), None) = (first, second) {
|
|
796
|
+
return Ok(only.record_size);
|
|
797
|
+
}
|
|
798
|
+
variables
|
|
799
|
+
.iter()
|
|
800
|
+
.filter(|v| v.is_record_var)
|
|
801
|
+
.try_fold(0u64, |stride, v| {
|
|
802
|
+
let size = v.record_size;
|
|
803
|
+
// Pad each variable's per-record size to 4-byte boundary.
|
|
804
|
+
let rem = size % 4;
|
|
805
|
+
let padded = if rem == 0 {
|
|
806
|
+
size
|
|
807
|
+
} else {
|
|
808
|
+
size.checked_add(4 - rem).ok_or_else(|| {
|
|
809
|
+
Error::InvalidData(format!(
|
|
810
|
+
"record variable '{}' padded record size exceeds u64",
|
|
811
|
+
v.name
|
|
812
|
+
))
|
|
813
|
+
})?
|
|
814
|
+
};
|
|
815
|
+
stride
|
|
816
|
+
.checked_add(padded)
|
|
817
|
+
.ok_or_else(|| Error::InvalidData("record stride exceeds u64".to_string()))
|
|
818
|
+
})
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
fn checked_non_record_element_count(var: &NcVariable) -> Result<usize> {
|
|
822
|
+
let mut total = 1u64;
|
|
823
|
+
for dim in &var.dimensions {
|
|
824
|
+
total = total.checked_mul(dim.size).ok_or_else(|| {
|
|
825
|
+
Error::InvalidData("variable element count overflows u64".to_string())
|
|
826
|
+
})?;
|
|
827
|
+
}
|
|
828
|
+
crate::types::checked_usize_from_u64(total, "variable element count")
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
pub(crate) fn checked_variable_shape(var: &NcVariable) -> Result<Vec<usize>> {
|
|
832
|
+
var.shape()
|
|
833
|
+
.iter()
|
|
834
|
+
.map(|&s| crate::types::checked_usize_from_u64(s, "variable dimension"))
|
|
835
|
+
.collect::<Result<Vec<_>>>()
|
|
836
|
+
}
|
|
837
|
+
|
|
838
|
+
fn checked_record_shape(var: &NcVariable, numrecs: u64) -> Result<Vec<usize>> {
|
|
839
|
+
let mut shape: Vec<usize> = var
|
|
840
|
+
.shape()
|
|
841
|
+
.iter()
|
|
842
|
+
.map(|&s| crate::types::checked_usize_from_u64(s, "record variable dimension"))
|
|
843
|
+
.collect::<Result<Vec<_>>>()?;
|
|
844
|
+
if shape.is_empty() {
|
|
845
|
+
return Err(Error::InvalidData(
|
|
846
|
+
"record variable must have at least one dimension".to_string(),
|
|
847
|
+
));
|
|
848
|
+
}
|
|
849
|
+
shape[0] = crate::types::checked_usize_from_u64(numrecs, "record count")?;
|
|
850
|
+
Ok(shape)
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
fn checked_record_elements_per_record(var: &NcVariable) -> Result<usize> {
|
|
854
|
+
let mut elements = 1usize;
|
|
855
|
+
for dim in var.dimensions.iter().skip(1) {
|
|
856
|
+
let size = crate::types::checked_usize_from_u64(dim.size, "record variable dimension")?;
|
|
857
|
+
elements = elements.checked_mul(size).ok_or_else(|| {
|
|
858
|
+
Error::InvalidData(format!(
|
|
859
|
+
"record variable '{}' elements per record exceed platform usize",
|
|
860
|
+
var.name
|
|
861
|
+
))
|
|
862
|
+
})?;
|
|
863
|
+
}
|
|
864
|
+
Ok(elements)
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
pub(crate) fn variable_data_bytes<T: NcReadType>(
|
|
868
|
+
var_name: &str,
|
|
869
|
+
element_count: usize,
|
|
870
|
+
) -> Result<usize> {
|
|
871
|
+
element_count.checked_mul(T::element_size()).ok_or_else(|| {
|
|
872
|
+
Error::InvalidData(format!(
|
|
873
|
+
"variable '{var_name}' size in bytes exceeds platform usize"
|
|
874
|
+
))
|
|
875
|
+
})
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
/// Validate that all `numrecs` records fit within `available` bytes before a
|
|
879
|
+
/// caller allocates the whole result buffer, so a crafted record count cannot
|
|
880
|
+
/// force a huge up-front allocation ahead of the per-record bounds checks.
|
|
881
|
+
pub(crate) fn validate_record_span(
|
|
882
|
+
var: &NcVariable,
|
|
883
|
+
numrecs: u64,
|
|
884
|
+
record_stride: u64,
|
|
885
|
+
bytes_per_record: u64,
|
|
886
|
+
available: u64,
|
|
887
|
+
) -> Result<()> {
|
|
888
|
+
if numrecs == 0 {
|
|
889
|
+
return Ok(());
|
|
890
|
+
}
|
|
891
|
+
let last_record_start = record_byte_offset(var, numrecs - 1, record_stride)?;
|
|
892
|
+
let end = last_record_start
|
|
893
|
+
.checked_add(bytes_per_record)
|
|
894
|
+
.ok_or_else(|| {
|
|
895
|
+
Error::InvalidData(format!(
|
|
896
|
+
"record variable '{}' data extent exceeds u64",
|
|
897
|
+
var.name
|
|
898
|
+
))
|
|
899
|
+
})?;
|
|
900
|
+
if end > available {
|
|
901
|
+
return Err(Error::InvalidData(format!(
|
|
902
|
+
"record variable '{}' needs {end} bytes but only {available} are available",
|
|
903
|
+
var.name
|
|
904
|
+
)));
|
|
905
|
+
}
|
|
906
|
+
Ok(())
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
pub(crate) fn record_byte_offset(var: &NcVariable, record: u64, record_stride: u64) -> Result<u64> {
|
|
910
|
+
var.data_offset
|
|
911
|
+
.checked_add(record.checked_mul(record_stride).ok_or_else(|| {
|
|
912
|
+
Error::InvalidData(format!(
|
|
913
|
+
"record variable '{}' byte offset exceeds u64",
|
|
914
|
+
var.name
|
|
915
|
+
))
|
|
916
|
+
})?)
|
|
917
|
+
.ok_or_else(|| {
|
|
918
|
+
Error::InvalidData(format!(
|
|
919
|
+
"record variable '{}' byte offset exceeds u64",
|
|
920
|
+
var.name
|
|
921
|
+
))
|
|
922
|
+
})
|
|
923
|
+
}
|
|
924
|
+
|
|
925
|
+
#[cfg(feature = "rayon")]
|
|
926
|
+
pub(crate) fn read_contiguous_range_parallel<T: NcReadType>(
|
|
927
|
+
storage: &ClassicStorage,
|
|
928
|
+
base_offset: u64,
|
|
929
|
+
total_elements: usize,
|
|
930
|
+
target_chunk_bytes: usize,
|
|
931
|
+
) -> Result<Vec<T>> {
|
|
932
|
+
if total_elements == 0 {
|
|
933
|
+
return Ok(Vec::new());
|
|
934
|
+
}
|
|
935
|
+
|
|
936
|
+
let elem_size = T::element_size();
|
|
937
|
+
let elements_per_chunk = (target_chunk_bytes / elem_size.max(1)).max(1);
|
|
938
|
+
let mut values = vec![T::default(); total_elements];
|
|
939
|
+
values
|
|
940
|
+
.par_chunks_mut(elements_per_chunk)
|
|
941
|
+
.enumerate()
|
|
942
|
+
.try_for_each(|(chunk, dst)| {
|
|
943
|
+
let start_element = chunk.checked_mul(elements_per_chunk).ok_or_else(|| {
|
|
944
|
+
Error::InvalidData(
|
|
945
|
+
"classic parallel chunk offset exceeds platform usize".to_string(),
|
|
946
|
+
)
|
|
947
|
+
})?;
|
|
948
|
+
let byte_offset = crate::types::checked_mul_u64(
|
|
949
|
+
start_element as u64,
|
|
950
|
+
elem_size as u64,
|
|
951
|
+
"classic parallel byte offset",
|
|
952
|
+
)?;
|
|
953
|
+
let offset = base_offset.checked_add(byte_offset).ok_or_else(|| {
|
|
954
|
+
Error::InvalidData("classic parallel byte offset exceeds u64".to_string())
|
|
955
|
+
})?;
|
|
956
|
+
let bytes = variable_data_bytes::<T>("parallel chunk", dst.len())?;
|
|
957
|
+
let data = storage.read_range(offset, bytes)?;
|
|
958
|
+
T::decode_bulk_be_into(data.as_ref(), dst)
|
|
959
|
+
})?;
|
|
960
|
+
|
|
961
|
+
Ok(values)
|
|
962
|
+
}
|
|
963
|
+
|
|
964
|
+
#[cfg(feature = "rayon")]
|
|
965
|
+
struct RecordChunkReadPlan<'a> {
|
|
966
|
+
var: &'a NcVariable,
|
|
967
|
+
record_stride: u64,
|
|
968
|
+
elements_per_record: usize,
|
|
969
|
+
bytes_per_record: usize,
|
|
970
|
+
}
|
|
971
|
+
|
|
972
|
+
#[cfg(feature = "rayon")]
|
|
973
|
+
fn read_record_chunk_into<T: NcReadType>(
|
|
974
|
+
storage: &ClassicStorage,
|
|
975
|
+
plan: &RecordChunkReadPlan<'_>,
|
|
976
|
+
first_record: u64,
|
|
977
|
+
records: usize,
|
|
978
|
+
dst: &mut [T],
|
|
979
|
+
) -> Result<()> {
|
|
980
|
+
if records == 0 {
|
|
981
|
+
return Ok(());
|
|
982
|
+
}
|
|
983
|
+
|
|
984
|
+
let expected_elements = records
|
|
985
|
+
.checked_mul(plan.elements_per_record)
|
|
986
|
+
.ok_or_else(|| {
|
|
987
|
+
Error::InvalidData(format!(
|
|
988
|
+
"record variable '{}' chunk element count exceeds platform usize",
|
|
989
|
+
plan.var.name
|
|
990
|
+
))
|
|
991
|
+
})?;
|
|
992
|
+
if dst.len() != expected_elements {
|
|
993
|
+
return Err(Error::InvalidData(format!(
|
|
994
|
+
"record variable '{}' chunk destination has {} elements, expected {}",
|
|
995
|
+
plan.var.name,
|
|
996
|
+
dst.len(),
|
|
997
|
+
expected_elements
|
|
998
|
+
)));
|
|
999
|
+
}
|
|
1000
|
+
|
|
1001
|
+
if plan.record_stride == plan.bytes_per_record as u64 {
|
|
1002
|
+
let offset = record_byte_offset(plan.var, first_record, plan.record_stride)?;
|
|
1003
|
+
let bytes = records.checked_mul(plan.bytes_per_record).ok_or_else(|| {
|
|
1004
|
+
Error::InvalidData(format!(
|
|
1005
|
+
"record variable '{}' chunk byte count exceeds platform usize",
|
|
1006
|
+
plan.var.name
|
|
1007
|
+
))
|
|
1008
|
+
})?;
|
|
1009
|
+
let data = storage.read_range(offset, bytes)?;
|
|
1010
|
+
return T::decode_bulk_be_into(data.as_ref(), dst);
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
for ordinal in 0..records {
|
|
1014
|
+
let record = first_record
|
|
1015
|
+
.checked_add(ordinal as u64)
|
|
1016
|
+
.ok_or_else(|| Error::InvalidData("classic record index exceeds u64".to_string()))?;
|
|
1017
|
+
let offset = record_byte_offset(plan.var, record, plan.record_stride)?;
|
|
1018
|
+
let data = storage.read_range(offset, plan.bytes_per_record)?;
|
|
1019
|
+
let dst_start = ordinal
|
|
1020
|
+
.checked_mul(plan.elements_per_record)
|
|
1021
|
+
.ok_or_else(|| {
|
|
1022
|
+
Error::InvalidData(format!(
|
|
1023
|
+
"record variable '{}' chunk destination offset exceeds platform usize",
|
|
1024
|
+
plan.var.name
|
|
1025
|
+
))
|
|
1026
|
+
})?;
|
|
1027
|
+
let dst_end = dst_start
|
|
1028
|
+
.checked_add(plan.elements_per_record)
|
|
1029
|
+
.ok_or_else(|| {
|
|
1030
|
+
Error::InvalidData(format!(
|
|
1031
|
+
"record variable '{}' chunk destination range exceeds platform usize",
|
|
1032
|
+
plan.var.name
|
|
1033
|
+
))
|
|
1034
|
+
})?;
|
|
1035
|
+
T::decode_bulk_be_into(data.as_ref(), &mut dst[dst_start..dst_end])?;
|
|
1036
|
+
}
|
|
1037
|
+
Ok(())
|
|
1038
|
+
}
|
|
1039
|
+
|
|
1040
|
+
#[cfg(test)]
|
|
1041
|
+
mod tests {
|
|
1042
|
+
use super::*;
|
|
1043
|
+
use crate::types::NcDimension;
|
|
1044
|
+
|
|
1045
|
+
#[test]
|
|
1046
|
+
fn read_non_record_1d_float() {
|
|
1047
|
+
// Create a fake file with 3 floats starting at offset 100.
|
|
1048
|
+
let mut file_data = vec![0u8; 200];
|
|
1049
|
+
let values = [1.0f32, 2.0f32, 3.0f32];
|
|
1050
|
+
for (i, &v) in values.iter().enumerate() {
|
|
1051
|
+
let bytes = v.to_be_bytes();
|
|
1052
|
+
file_data[100 + i * 4..100 + i * 4 + 4].copy_from_slice(&bytes);
|
|
1053
|
+
}
|
|
1054
|
+
|
|
1055
|
+
let var = NcVariable {
|
|
1056
|
+
name: "temp".to_string(),
|
|
1057
|
+
dimensions: vec![NcDimension {
|
|
1058
|
+
name: "x".to_string(),
|
|
1059
|
+
size: 3,
|
|
1060
|
+
is_unlimited: false,
|
|
1061
|
+
}],
|
|
1062
|
+
dtype: NcType::Float,
|
|
1063
|
+
attributes: vec![],
|
|
1064
|
+
data_offset: 100,
|
|
1065
|
+
_data_size: 12,
|
|
1066
|
+
is_record_var: false,
|
|
1067
|
+
record_size: 0,
|
|
1068
|
+
};
|
|
1069
|
+
|
|
1070
|
+
let arr: ArrayD<f32> = read_non_record_variable(&file_data, &var).unwrap();
|
|
1071
|
+
assert_eq!(arr.shape(), &[3]);
|
|
1072
|
+
assert_eq!(arr[[0]], 1.0f32);
|
|
1073
|
+
assert_eq!(arr[[1]], 2.0f32);
|
|
1074
|
+
assert_eq!(arr[[2]], 3.0f32);
|
|
1075
|
+
}
|
|
1076
|
+
|
|
1077
|
+
#[test]
|
|
1078
|
+
fn non_record_variable_into_copies_values() {
|
|
1079
|
+
let mut file_data = vec![0u8; 200];
|
|
1080
|
+
let values = [1.0f32, 2.0f32, 3.0f32];
|
|
1081
|
+
for (i, &v) in values.iter().enumerate() {
|
|
1082
|
+
file_data[100 + i * 4..100 + i * 4 + 4].copy_from_slice(&v.to_be_bytes());
|
|
1083
|
+
}
|
|
1084
|
+
|
|
1085
|
+
let var = NcVariable {
|
|
1086
|
+
name: "temp".to_string(),
|
|
1087
|
+
dimensions: vec![NcDimension {
|
|
1088
|
+
name: "x".to_string(),
|
|
1089
|
+
size: 3,
|
|
1090
|
+
is_unlimited: false,
|
|
1091
|
+
}],
|
|
1092
|
+
dtype: NcType::Float,
|
|
1093
|
+
attributes: vec![],
|
|
1094
|
+
data_offset: 100,
|
|
1095
|
+
_data_size: 12,
|
|
1096
|
+
is_record_var: false,
|
|
1097
|
+
record_size: 0,
|
|
1098
|
+
};
|
|
1099
|
+
|
|
1100
|
+
let mut dst = [0.0f32; 3];
|
|
1101
|
+
read_non_record_variable_into(&file_data, &var, &mut dst).unwrap();
|
|
1102
|
+
assert_eq!(dst, values);
|
|
1103
|
+
}
|
|
1104
|
+
|
|
1105
|
+
#[test]
|
|
1106
|
+
fn read_non_record_2d_int() {
|
|
1107
|
+
// 2x3 array of i32 at offset 0
|
|
1108
|
+
let values: Vec<i32> = vec![10, 20, 30, 40, 50, 60];
|
|
1109
|
+
let mut file_data = Vec::new();
|
|
1110
|
+
for &v in &values {
|
|
1111
|
+
file_data.extend_from_slice(&v.to_be_bytes());
|
|
1112
|
+
}
|
|
1113
|
+
|
|
1114
|
+
let var = NcVariable {
|
|
1115
|
+
name: "grid".to_string(),
|
|
1116
|
+
dimensions: vec![
|
|
1117
|
+
NcDimension {
|
|
1118
|
+
name: "y".to_string(),
|
|
1119
|
+
size: 2,
|
|
1120
|
+
is_unlimited: false,
|
|
1121
|
+
},
|
|
1122
|
+
NcDimension {
|
|
1123
|
+
name: "x".to_string(),
|
|
1124
|
+
size: 3,
|
|
1125
|
+
is_unlimited: false,
|
|
1126
|
+
},
|
|
1127
|
+
],
|
|
1128
|
+
dtype: NcType::Int,
|
|
1129
|
+
attributes: vec![],
|
|
1130
|
+
data_offset: 0,
|
|
1131
|
+
_data_size: 24,
|
|
1132
|
+
is_record_var: false,
|
|
1133
|
+
record_size: 0,
|
|
1134
|
+
};
|
|
1135
|
+
|
|
1136
|
+
let arr: ArrayD<i32> = read_non_record_variable(&file_data, &var).unwrap();
|
|
1137
|
+
assert_eq!(arr.shape(), &[2, 3]);
|
|
1138
|
+
assert_eq!(arr[[0, 0]], 10);
|
|
1139
|
+
assert_eq!(arr[[0, 2]], 30);
|
|
1140
|
+
assert_eq!(arr[[1, 0]], 40);
|
|
1141
|
+
assert_eq!(arr[[1, 2]], 60);
|
|
1142
|
+
}
|
|
1143
|
+
|
|
1144
|
+
#[test]
|
|
1145
|
+
fn read_non_record_variable_into_rejects_wrong_destination_len() {
|
|
1146
|
+
let var = NcVariable {
|
|
1147
|
+
name: "grid".to_string(),
|
|
1148
|
+
dimensions: vec![NcDimension {
|
|
1149
|
+
name: "x".to_string(),
|
|
1150
|
+
size: 3,
|
|
1151
|
+
is_unlimited: false,
|
|
1152
|
+
}],
|
|
1153
|
+
dtype: NcType::Float,
|
|
1154
|
+
attributes: vec![],
|
|
1155
|
+
data_offset: 0,
|
|
1156
|
+
_data_size: 12,
|
|
1157
|
+
is_record_var: false,
|
|
1158
|
+
record_size: 0,
|
|
1159
|
+
};
|
|
1160
|
+
|
|
1161
|
+
let mut dst = [0.0f32; 2];
|
|
1162
|
+
let err = read_non_record_variable_into(&[0; 12], &var, &mut dst).unwrap_err();
|
|
1163
|
+
assert!(matches!(err, Error::InvalidData(_)));
|
|
1164
|
+
}
|
|
1165
|
+
|
|
1166
|
+
#[test]
|
|
1167
|
+
fn record_stride_sums_padded_record_variables() {
|
|
1168
|
+
let vars = vec![
|
|
1169
|
+
NcVariable {
|
|
1170
|
+
name: "a".to_string(),
|
|
1171
|
+
dimensions: vec![],
|
|
1172
|
+
dtype: NcType::Float,
|
|
1173
|
+
attributes: vec![],
|
|
1174
|
+
data_offset: 0,
|
|
1175
|
+
_data_size: 0,
|
|
1176
|
+
is_record_var: true,
|
|
1177
|
+
record_size: 20, // 5 floats
|
|
1178
|
+
},
|
|
1179
|
+
NcVariable {
|
|
1180
|
+
name: "b".to_string(),
|
|
1181
|
+
dimensions: vec![],
|
|
1182
|
+
dtype: NcType::Short,
|
|
1183
|
+
attributes: vec![],
|
|
1184
|
+
data_offset: 0,
|
|
1185
|
+
_data_size: 0,
|
|
1186
|
+
is_record_var: true,
|
|
1187
|
+
record_size: 6, // 3 shorts -> padded to 8
|
|
1188
|
+
},
|
|
1189
|
+
NcVariable {
|
|
1190
|
+
name: "c".to_string(),
|
|
1191
|
+
dimensions: vec![],
|
|
1192
|
+
dtype: NcType::Double,
|
|
1193
|
+
attributes: vec![],
|
|
1194
|
+
data_offset: 0,
|
|
1195
|
+
_data_size: 100,
|
|
1196
|
+
is_record_var: false, // not a record var, should be excluded
|
|
1197
|
+
record_size: 0,
|
|
1198
|
+
},
|
|
1199
|
+
];
|
|
1200
|
+
// a: 20 (already 4-aligned), b: 6 -> 8 = total 28
|
|
1201
|
+
assert_eq!(compute_record_stride(&vars).unwrap(), 28);
|
|
1202
|
+
}
|
|
1203
|
+
|
|
1204
|
+
#[test]
|
|
1205
|
+
fn record_stride_rejects_padded_size_overflow() {
|
|
1206
|
+
// Two record variables so per-variable padding applies (a lone
|
|
1207
|
+
// record variable is packed without padding).
|
|
1208
|
+
let vars = vec![
|
|
1209
|
+
NcVariable {
|
|
1210
|
+
name: "huge".to_string(),
|
|
1211
|
+
dimensions: vec![],
|
|
1212
|
+
dtype: NcType::Byte,
|
|
1213
|
+
attributes: vec![],
|
|
1214
|
+
data_offset: 0,
|
|
1215
|
+
_data_size: 0,
|
|
1216
|
+
is_record_var: true,
|
|
1217
|
+
record_size: u64::MAX,
|
|
1218
|
+
},
|
|
1219
|
+
NcVariable {
|
|
1220
|
+
name: "small".to_string(),
|
|
1221
|
+
dimensions: vec![],
|
|
1222
|
+
dtype: NcType::Byte,
|
|
1223
|
+
attributes: vec![],
|
|
1224
|
+
data_offset: 0,
|
|
1225
|
+
_data_size: 0,
|
|
1226
|
+
is_record_var: true,
|
|
1227
|
+
record_size: 4,
|
|
1228
|
+
},
|
|
1229
|
+
];
|
|
1230
|
+
|
|
1231
|
+
let err = compute_record_stride(&vars).unwrap_err();
|
|
1232
|
+
assert!(matches!(err, Error::InvalidData(_)));
|
|
1233
|
+
}
|
|
1234
|
+
|
|
1235
|
+
#[test]
|
|
1236
|
+
fn record_stride_rejects_sum_overflow() {
|
|
1237
|
+
let vars = vec![
|
|
1238
|
+
NcVariable {
|
|
1239
|
+
name: "a".to_string(),
|
|
1240
|
+
dimensions: vec![],
|
|
1241
|
+
dtype: NcType::Byte,
|
|
1242
|
+
attributes: vec![],
|
|
1243
|
+
data_offset: 0,
|
|
1244
|
+
_data_size: 0,
|
|
1245
|
+
is_record_var: true,
|
|
1246
|
+
record_size: u64::MAX - 7,
|
|
1247
|
+
},
|
|
1248
|
+
NcVariable {
|
|
1249
|
+
name: "b".to_string(),
|
|
1250
|
+
dimensions: vec![],
|
|
1251
|
+
dtype: NcType::Byte,
|
|
1252
|
+
attributes: vec![],
|
|
1253
|
+
data_offset: 0,
|
|
1254
|
+
_data_size: 0,
|
|
1255
|
+
is_record_var: true,
|
|
1256
|
+
record_size: 8,
|
|
1257
|
+
},
|
|
1258
|
+
];
|
|
1259
|
+
|
|
1260
|
+
let err = compute_record_stride(&vars).unwrap_err();
|
|
1261
|
+
assert!(matches!(err, Error::InvalidData(_)));
|
|
1262
|
+
}
|
|
1263
|
+
|
|
1264
|
+
#[test]
|
|
1265
|
+
fn record_variable_reads_all_records() {
|
|
1266
|
+
// Single record variable "temp" with shape [time, x] where x=2.
|
|
1267
|
+
// 3 records, each with 2 floats = 8 bytes per record.
|
|
1268
|
+
// Record stride = 8 (only one record var, already 4-aligned).
|
|
1269
|
+
let mut file_data = vec![0u8; 200];
|
|
1270
|
+
let base = 100usize;
|
|
1271
|
+
let record_values: Vec<Vec<f32>> = vec![vec![1.0, 2.0], vec![3.0, 4.0], vec![5.0, 6.0]];
|
|
1272
|
+
for (rec, vals) in record_values.iter().enumerate() {
|
|
1273
|
+
for (i, &v) in vals.iter().enumerate() {
|
|
1274
|
+
let offset = base + rec * 8 + i * 4;
|
|
1275
|
+
file_data[offset..offset + 4].copy_from_slice(&v.to_be_bytes());
|
|
1276
|
+
}
|
|
1277
|
+
}
|
|
1278
|
+
|
|
1279
|
+
let var = NcVariable {
|
|
1280
|
+
name: "temp".to_string(),
|
|
1281
|
+
dimensions: vec![
|
|
1282
|
+
NcDimension {
|
|
1283
|
+
name: "time".to_string(),
|
|
1284
|
+
size: 0, // unlimited
|
|
1285
|
+
is_unlimited: true,
|
|
1286
|
+
},
|
|
1287
|
+
NcDimension {
|
|
1288
|
+
name: "x".to_string(),
|
|
1289
|
+
size: 2,
|
|
1290
|
+
is_unlimited: false,
|
|
1291
|
+
},
|
|
1292
|
+
],
|
|
1293
|
+
dtype: NcType::Float,
|
|
1294
|
+
attributes: vec![],
|
|
1295
|
+
data_offset: 100,
|
|
1296
|
+
_data_size: 0,
|
|
1297
|
+
is_record_var: true,
|
|
1298
|
+
record_size: 8,
|
|
1299
|
+
};
|
|
1300
|
+
|
|
1301
|
+
let arr: ArrayD<f32> = read_record_variable(&file_data, &var, 3, 8).unwrap();
|
|
1302
|
+
assert_eq!(arr.shape(), &[3, 2]);
|
|
1303
|
+
assert_eq!(arr[[0, 0]], 1.0);
|
|
1304
|
+
assert_eq!(arr[[0, 1]], 2.0);
|
|
1305
|
+
assert_eq!(arr[[1, 0]], 3.0);
|
|
1306
|
+
assert_eq!(arr[[2, 1]], 6.0);
|
|
1307
|
+
}
|
|
1308
|
+
|
|
1309
|
+
#[test]
|
|
1310
|
+
fn record_variable_into_copies_values() {
|
|
1311
|
+
let mut file_data = vec![0u8; 200];
|
|
1312
|
+
let base = 100usize;
|
|
1313
|
+
let record_values: Vec<Vec<f32>> = vec![vec![1.0, 2.0], vec![3.0, 4.0], vec![5.0, 6.0]];
|
|
1314
|
+
for (rec, vals) in record_values.iter().enumerate() {
|
|
1315
|
+
for (i, &v) in vals.iter().enumerate() {
|
|
1316
|
+
let offset = base + rec * 8 + i * 4;
|
|
1317
|
+
file_data[offset..offset + 4].copy_from_slice(&v.to_be_bytes());
|
|
1318
|
+
}
|
|
1319
|
+
}
|
|
1320
|
+
|
|
1321
|
+
let var = NcVariable {
|
|
1322
|
+
name: "temp".to_string(),
|
|
1323
|
+
dimensions: vec![
|
|
1324
|
+
NcDimension {
|
|
1325
|
+
name: "time".to_string(),
|
|
1326
|
+
size: 0,
|
|
1327
|
+
is_unlimited: true,
|
|
1328
|
+
},
|
|
1329
|
+
NcDimension {
|
|
1330
|
+
name: "x".to_string(),
|
|
1331
|
+
size: 2,
|
|
1332
|
+
is_unlimited: false,
|
|
1333
|
+
},
|
|
1334
|
+
],
|
|
1335
|
+
dtype: NcType::Float,
|
|
1336
|
+
attributes: vec![],
|
|
1337
|
+
data_offset: 100,
|
|
1338
|
+
_data_size: 0,
|
|
1339
|
+
is_record_var: true,
|
|
1340
|
+
record_size: 8,
|
|
1341
|
+
};
|
|
1342
|
+
|
|
1343
|
+
let mut dst = [0.0f32; 6];
|
|
1344
|
+
read_record_variable_into(&file_data, &var, 3, 8, &mut dst).unwrap();
|
|
1345
|
+
assert_eq!(dst, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]);
|
|
1346
|
+
}
|
|
1347
|
+
|
|
1348
|
+
#[test]
|
|
1349
|
+
fn read_record_variable_into_rejects_wrong_destination_len() {
|
|
1350
|
+
let var = NcVariable {
|
|
1351
|
+
name: "temp".to_string(),
|
|
1352
|
+
dimensions: vec![
|
|
1353
|
+
NcDimension {
|
|
1354
|
+
name: "time".to_string(),
|
|
1355
|
+
size: 0,
|
|
1356
|
+
is_unlimited: true,
|
|
1357
|
+
},
|
|
1358
|
+
NcDimension {
|
|
1359
|
+
name: "x".to_string(),
|
|
1360
|
+
size: 2,
|
|
1361
|
+
is_unlimited: false,
|
|
1362
|
+
},
|
|
1363
|
+
],
|
|
1364
|
+
dtype: NcType::Float,
|
|
1365
|
+
attributes: vec![],
|
|
1366
|
+
data_offset: 0,
|
|
1367
|
+
_data_size: 0,
|
|
1368
|
+
is_record_var: true,
|
|
1369
|
+
record_size: 8,
|
|
1370
|
+
};
|
|
1371
|
+
|
|
1372
|
+
let mut dst = [0.0f32; 5];
|
|
1373
|
+
let err = read_record_variable_into(&[0; 24], &var, 3, 8, &mut dst).unwrap_err();
|
|
1374
|
+
assert!(matches!(err, Error::InvalidData(_)));
|
|
1375
|
+
}
|
|
1376
|
+
|
|
1377
|
+
#[test]
|
|
1378
|
+
fn read_non_record_variable_rejects_element_count_overflow() {
|
|
1379
|
+
let var = NcVariable {
|
|
1380
|
+
name: "huge".to_string(),
|
|
1381
|
+
dimensions: vec![
|
|
1382
|
+
NcDimension {
|
|
1383
|
+
name: "y".to_string(),
|
|
1384
|
+
size: u64::MAX,
|
|
1385
|
+
is_unlimited: false,
|
|
1386
|
+
},
|
|
1387
|
+
NcDimension {
|
|
1388
|
+
name: "x".to_string(),
|
|
1389
|
+
size: 2,
|
|
1390
|
+
is_unlimited: false,
|
|
1391
|
+
},
|
|
1392
|
+
],
|
|
1393
|
+
dtype: NcType::Float,
|
|
1394
|
+
attributes: vec![],
|
|
1395
|
+
data_offset: 0,
|
|
1396
|
+
_data_size: 0,
|
|
1397
|
+
is_record_var: false,
|
|
1398
|
+
record_size: 0,
|
|
1399
|
+
};
|
|
1400
|
+
|
|
1401
|
+
let err = read_non_record_variable::<f32>(&[], &var).unwrap_err();
|
|
1402
|
+
assert!(matches!(err, Error::InvalidData(_)));
|
|
1403
|
+
}
|
|
1404
|
+
|
|
1405
|
+
#[test]
|
|
1406
|
+
fn read_record_variable_rejects_elements_per_record_overflow() {
|
|
1407
|
+
let var = NcVariable {
|
|
1408
|
+
name: "huge_record".to_string(),
|
|
1409
|
+
dimensions: vec![
|
|
1410
|
+
NcDimension {
|
|
1411
|
+
name: "time".to_string(),
|
|
1412
|
+
size: 0,
|
|
1413
|
+
is_unlimited: true,
|
|
1414
|
+
},
|
|
1415
|
+
NcDimension {
|
|
1416
|
+
name: "y".to_string(),
|
|
1417
|
+
size: usize::MAX as u64,
|
|
1418
|
+
is_unlimited: false,
|
|
1419
|
+
},
|
|
1420
|
+
NcDimension {
|
|
1421
|
+
name: "x".to_string(),
|
|
1422
|
+
size: 2,
|
|
1423
|
+
is_unlimited: false,
|
|
1424
|
+
},
|
|
1425
|
+
],
|
|
1426
|
+
dtype: NcType::Float,
|
|
1427
|
+
attributes: vec![],
|
|
1428
|
+
data_offset: 0,
|
|
1429
|
+
_data_size: 0,
|
|
1430
|
+
is_record_var: true,
|
|
1431
|
+
record_size: 4,
|
|
1432
|
+
};
|
|
1433
|
+
|
|
1434
|
+
let err = read_record_variable::<f32>(&[], &var, 1, 4).unwrap_err();
|
|
1435
|
+
assert!(matches!(err, Error::InvalidData(_)));
|
|
1436
|
+
}
|
|
1437
|
+
|
|
1438
|
+
#[test]
|
|
1439
|
+
fn read_record_variable_rejects_record_offset_overflow() {
|
|
1440
|
+
let var = NcVariable {
|
|
1441
|
+
name: "huge_record".to_string(),
|
|
1442
|
+
dimensions: vec![
|
|
1443
|
+
NcDimension {
|
|
1444
|
+
name: "time".to_string(),
|
|
1445
|
+
size: 0,
|
|
1446
|
+
is_unlimited: true,
|
|
1447
|
+
},
|
|
1448
|
+
NcDimension {
|
|
1449
|
+
name: "x".to_string(),
|
|
1450
|
+
size: 1,
|
|
1451
|
+
is_unlimited: false,
|
|
1452
|
+
},
|
|
1453
|
+
],
|
|
1454
|
+
dtype: NcType::Float,
|
|
1455
|
+
attributes: vec![],
|
|
1456
|
+
data_offset: u64::MAX,
|
|
1457
|
+
_data_size: 0,
|
|
1458
|
+
is_record_var: true,
|
|
1459
|
+
record_size: 4,
|
|
1460
|
+
};
|
|
1461
|
+
|
|
1462
|
+
let err = read_record_variable::<f32>(&[], &var, 1, 4).unwrap_err();
|
|
1463
|
+
assert!(matches!(err, Error::InvalidData(_)));
|
|
1464
|
+
}
|
|
1465
|
+
}
|