@nirs4all/formats-wasm 0.2.11 → 0.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD_PARTY_NOTICES.md +49 -0
- package/nirs4all_formats_wasm_bg.wasm +0 -0
- package/package.json +4 -2
- package/reader-backports/hdf5-reader/Cargo.toml +119 -0
- package/reader-backports/hdf5-reader/LICENSE-APACHE +164 -0
- package/reader-backports/hdf5-reader/LICENSE-MIT +21 -0
- package/reader-backports/hdf5-reader/PROVENANCE.json +80 -0
- package/reader-backports/hdf5-reader/README.md +308 -0
- package/reader-backports/hdf5-reader/UPSTREAM-Cargo.toml +44 -0
- package/reader-backports/hdf5-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/hdf5-reader/src/attribute_api.rs +847 -0
- package/reader-backports/hdf5-reader/src/btree_v1.rs +726 -0
- package/reader-backports/hdf5-reader/src/btree_v2.rs +2068 -0
- package/reader-backports/hdf5-reader/src/cache.rs +354 -0
- package/reader-backports/hdf5-reader/src/checksum.rs +56 -0
- package/reader-backports/hdf5-reader/src/chunk_index.rs +423 -0
- package/reader-backports/hdf5-reader/src/dataset.rs +4733 -0
- package/reader-backports/hdf5-reader/src/datatype_api.rs +497 -0
- package/reader-backports/hdf5-reader/src/error.rs +167 -0
- package/reader-backports/hdf5-reader/src/extensible_array.rs +1886 -0
- package/reader-backports/hdf5-reader/src/filters/deflate.rs +93 -0
- package/reader-backports/hdf5-reader/src/filters/fletcher32.rs +52 -0
- package/reader-backports/hdf5-reader/src/filters/lz4.rs +178 -0
- package/reader-backports/hdf5-reader/src/filters/mod.rs +326 -0
- package/reader-backports/hdf5-reader/src/filters/nbit.rs +586 -0
- package/reader-backports/hdf5-reader/src/filters/scaleoffset.rs +827 -0
- package/reader-backports/hdf5-reader/src/filters/shuffle.rs +99 -0
- package/reader-backports/hdf5-reader/src/fixed_array.rs +763 -0
- package/reader-backports/hdf5-reader/src/fractal_heap.rs +2029 -0
- package/reader-backports/hdf5-reader/src/global_heap.rs +284 -0
- package/reader-backports/hdf5-reader/src/group.rs +795 -0
- package/reader-backports/hdf5-reader/src/io.rs +413 -0
- package/reader-backports/hdf5-reader/src/lib.rs +916 -0
- package/reader-backports/hdf5-reader/src/local_heap.rs +261 -0
- package/reader-backports/hdf5-reader/src/messages/attribute.rs +304 -0
- package/reader-backports/hdf5-reader/src/messages/attribute_info.rs +96 -0
- package/reader-backports/hdf5-reader/src/messages/btree_k.rs +36 -0
- package/reader-backports/hdf5-reader/src/messages/continuation.rs +46 -0
- package/reader-backports/hdf5-reader/src/messages/dataspace.rs +259 -0
- package/reader-backports/hdf5-reader/src/messages/datatype.rs +938 -0
- package/reader-backports/hdf5-reader/src/messages/external_files.rs +50 -0
- package/reader-backports/hdf5-reader/src/messages/fill_value.rs +233 -0
- package/reader-backports/hdf5-reader/src/messages/filter_pipeline.rs +147 -0
- package/reader-backports/hdf5-reader/src/messages/group_info.rs +103 -0
- package/reader-backports/hdf5-reader/src/messages/layout.rs +476 -0
- package/reader-backports/hdf5-reader/src/messages/link.rs +170 -0
- package/reader-backports/hdf5-reader/src/messages/link_info.rs +127 -0
- package/reader-backports/hdf5-reader/src/messages/mod.rs +374 -0
- package/reader-backports/hdf5-reader/src/messages/modification_time.rs +83 -0
- package/reader-backports/hdf5-reader/src/messages/shared.rs +164 -0
- package/reader-backports/hdf5-reader/src/messages/shared_table.rs +67 -0
- package/reader-backports/hdf5-reader/src/messages/symbol_table_msg.rs +51 -0
- package/reader-backports/hdf5-reader/src/object_header.rs +1749 -0
- package/reader-backports/hdf5-reader/src/reference.rs +90 -0
- package/reader-backports/hdf5-reader/src/shared_message_table.rs +425 -0
- package/reader-backports/hdf5-reader/src/storage.rs +538 -0
- package/reader-backports/hdf5-reader/src/superblock.rs +285 -0
- package/reader-backports/hdf5-reader/src/symbol_table.rs +277 -0
- package/reader-backports/netcdf-reader/Cargo.toml +141 -0
- package/reader-backports/netcdf-reader/LICENSE-APACHE +164 -0
- package/reader-backports/netcdf-reader/LICENSE-MIT +21 -0
- package/reader-backports/netcdf-reader/PROVENANCE.json +55 -0
- package/reader-backports/netcdf-reader/README.md +308 -0
- package/reader-backports/netcdf-reader/UPSTREAM-Cargo.toml +48 -0
- package/reader-backports/netcdf-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/netcdf-reader/src/cf/axes.rs +419 -0
- package/reader-backports/netcdf-reader/src/cf/bounds.rs +108 -0
- package/reader-backports/netcdf-reader/src/cf/crs.rs +269 -0
- package/reader-backports/netcdf-reader/src/cf/mod.rs +30 -0
- package/reader-backports/netcdf-reader/src/cf/time.rs +1122 -0
- package/reader-backports/netcdf-reader/src/classic/data.rs +1465 -0
- package/reader-backports/netcdf-reader/src/classic/header.rs +1097 -0
- package/reader-backports/netcdf-reader/src/classic/mod.rs +240 -0
- package/reader-backports/netcdf-reader/src/classic/storage.rs +211 -0
- package/reader-backports/netcdf-reader/src/classic/types.rs +93 -0
- package/reader-backports/netcdf-reader/src/classic/variable.rs +1162 -0
- package/reader-backports/netcdf-reader/src/error.rs +59 -0
- package/reader-backports/netcdf-reader/src/lib.rs +1531 -0
- package/reader-backports/netcdf-reader/src/masked.rs +173 -0
- package/reader-backports/netcdf-reader/src/nc4/attributes.rs +203 -0
- package/reader-backports/netcdf-reader/src/nc4/dimensions.rs +139 -0
- package/reader-backports/netcdf-reader/src/nc4/groups.rs +308 -0
- package/reader-backports/netcdf-reader/src/nc4/mod.rs +572 -0
- package/reader-backports/netcdf-reader/src/nc4/types.rs +309 -0
- package/reader-backports/netcdf-reader/src/nc4/variables.rs +420 -0
- package/reader-backports/netcdf-reader/src/types.rs +18 -0
- package/reader-backports/netcdf-reader/src/unpack.rs +113 -0
- package/reader-backports/netcdf-reader/src/user_defined.rs +569 -0
|
@@ -0,0 +1,1162 @@
|
|
|
1
|
+
//! Variable-level read methods for classic NetCDF files.
|
|
2
|
+
//!
|
|
3
|
+
//! Provides convenience methods for reading variable data from a `ClassicFile`,
|
|
4
|
+
//! with type-checked access and support for both record and non-record variables.
|
|
5
|
+
|
|
6
|
+
use ndarray::ArrayD;
|
|
7
|
+
#[cfg(feature = "rayon")]
|
|
8
|
+
use rayon::prelude::*;
|
|
9
|
+
|
|
10
|
+
use crate::error::{Error, Result};
|
|
11
|
+
use crate::types::{
|
|
12
|
+
checked_mul_u64, checked_shape_elements, checked_usize_from_u64, NcType, NcVariable,
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
use super::data::{self, compute_record_stride, NcReadType};
|
|
16
|
+
use super::storage::ClassicStorage;
|
|
17
|
+
use super::ClassicFile;
|
|
18
|
+
|
|
19
|
+
#[derive(Clone, Debug)]
|
|
20
|
+
enum ResolvedClassicSelectionDim {
|
|
21
|
+
Index(u64),
|
|
22
|
+
Slice {
|
|
23
|
+
start: u64,
|
|
24
|
+
step: u64,
|
|
25
|
+
count: usize,
|
|
26
|
+
is_full_unit_stride: bool,
|
|
27
|
+
},
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
impl ResolvedClassicSelectionDim {
|
|
31
|
+
fn is_full_unit_stride(&self) -> bool {
|
|
32
|
+
matches!(
|
|
33
|
+
self,
|
|
34
|
+
Self::Slice {
|
|
35
|
+
is_full_unit_stride: true,
|
|
36
|
+
..
|
|
37
|
+
}
|
|
38
|
+
)
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
#[derive(Clone, Debug)]
|
|
43
|
+
struct ResolvedClassicSelection {
|
|
44
|
+
dims: Vec<ResolvedClassicSelectionDim>,
|
|
45
|
+
result_shape: Vec<usize>,
|
|
46
|
+
result_elements: usize,
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
struct BlockReadContext<'a> {
|
|
50
|
+
base_offset: u64,
|
|
51
|
+
plan: &'a ContiguousSelectionPlan,
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
struct ContiguousSelectionPlan {
|
|
55
|
+
dims: Vec<ResolvedClassicSelectionDim>,
|
|
56
|
+
strides: Vec<u64>,
|
|
57
|
+
tail_start: usize,
|
|
58
|
+
block_elements: usize,
|
|
59
|
+
block_bytes: usize,
|
|
60
|
+
elem_size: u64,
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
#[derive(Clone, Debug)]
|
|
64
|
+
struct PlannedReadSpan {
|
|
65
|
+
offset: u64,
|
|
66
|
+
elements: usize,
|
|
67
|
+
bytes: usize,
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
#[cfg(feature = "rayon")]
|
|
71
|
+
#[derive(Debug)]
|
|
72
|
+
struct PlannedReadChunk {
|
|
73
|
+
spans: Vec<PlannedReadSpan>,
|
|
74
|
+
elements: usize,
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
struct RecordSliceContext<'a> {
|
|
78
|
+
storage: &'a ClassicStorage,
|
|
79
|
+
base_offset: u64,
|
|
80
|
+
record_stride: u64,
|
|
81
|
+
inner_resolved: &'a ResolvedClassicSelection,
|
|
82
|
+
inner_plan: &'a ContiguousSelectionPlan,
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
impl ClassicFile {
|
|
86
|
+
/// Read a variable's data as an ndarray of the specified type.
|
|
87
|
+
///
|
|
88
|
+
/// The type parameter `T` must match the variable's NetCDF type. For example,
|
|
89
|
+
/// use `f32` for NC_FLOAT variables and `f64` for NC_DOUBLE variables.
|
|
90
|
+
pub fn read_variable<T: NcReadType>(&self, name: &str) -> Result<ArrayD<T>> {
|
|
91
|
+
let var = self.find_variable(name)?;
|
|
92
|
+
|
|
93
|
+
// Check type compatibility.
|
|
94
|
+
let expected = T::nc_type();
|
|
95
|
+
if var.dtype != expected {
|
|
96
|
+
return Err(Error::TypeMismatch {
|
|
97
|
+
expected: format!("{:?}", expected),
|
|
98
|
+
actual: format!("{:?}", var.dtype),
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
if var.is_record_var {
|
|
103
|
+
let record_stride = compute_record_stride(&self.root_group.variables)?;
|
|
104
|
+
data::read_record_variable_from_storage(&self.storage, var, self.numrecs, record_stride)
|
|
105
|
+
} else {
|
|
106
|
+
data::read_non_record_variable_from_storage(&self.storage, var)
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/// Read a variable's data using Rayon for large classic byte ranges.
|
|
111
|
+
#[cfg(feature = "rayon")]
|
|
112
|
+
pub fn read_variable_parallel<T: NcReadType>(&self, name: &str) -> Result<ArrayD<T>> {
|
|
113
|
+
let var = self.find_variable(name)?;
|
|
114
|
+
|
|
115
|
+
let expected = T::nc_type();
|
|
116
|
+
if var.dtype != expected {
|
|
117
|
+
return Err(Error::TypeMismatch {
|
|
118
|
+
expected: format!("{:?}", expected),
|
|
119
|
+
actual: format!("{:?}", var.dtype),
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
if var.is_record_var {
|
|
124
|
+
let record_stride = compute_record_stride(&self.root_group.variables)?;
|
|
125
|
+
data::read_record_variable_parallel_from_storage(
|
|
126
|
+
&self.storage,
|
|
127
|
+
var,
|
|
128
|
+
self.numrecs,
|
|
129
|
+
record_stride,
|
|
130
|
+
)
|
|
131
|
+
} else {
|
|
132
|
+
data::read_non_record_variable_parallel_from_storage(&self.storage, var)
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/// Read a variable's data into a caller-provided typed buffer.
|
|
137
|
+
pub fn read_variable_into<T: NcReadType>(&self, name: &str, dst: &mut [T]) -> Result<()> {
|
|
138
|
+
let var = self.find_variable(name)?;
|
|
139
|
+
|
|
140
|
+
let expected = T::nc_type();
|
|
141
|
+
if var.dtype != expected {
|
|
142
|
+
return Err(Error::TypeMismatch {
|
|
143
|
+
expected: format!("{:?}", expected),
|
|
144
|
+
actual: format!("{:?}", var.dtype),
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
if var.is_record_var {
|
|
149
|
+
let record_stride = compute_record_stride(&self.root_group.variables)?;
|
|
150
|
+
data::read_record_variable_into_from_storage(
|
|
151
|
+
&self.storage,
|
|
152
|
+
var,
|
|
153
|
+
self.numrecs,
|
|
154
|
+
record_stride,
|
|
155
|
+
dst,
|
|
156
|
+
)
|
|
157
|
+
} else {
|
|
158
|
+
data::read_non_record_variable_into_from_storage(&self.storage, var, dst)
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/// Read a variable's data with automatic type promotion to f64.
|
|
163
|
+
///
|
|
164
|
+
/// This reads any numeric variable and converts all values to f64,
|
|
165
|
+
/// which is convenient for analysis but may lose precision for i64/u64.
|
|
166
|
+
pub fn read_variable_as_f64(&self, name: &str) -> Result<ArrayD<f64>> {
|
|
167
|
+
let var = self.find_variable(name)?;
|
|
168
|
+
|
|
169
|
+
match var.dtype {
|
|
170
|
+
NcType::Byte => {
|
|
171
|
+
let arr = self.read_typed_variable::<i8>(var)?;
|
|
172
|
+
Ok(arr.mapv(|v| v as f64))
|
|
173
|
+
}
|
|
174
|
+
NcType::Short => {
|
|
175
|
+
let arr = self.read_typed_variable::<i16>(var)?;
|
|
176
|
+
Ok(arr.mapv(|v| v as f64))
|
|
177
|
+
}
|
|
178
|
+
NcType::Int => {
|
|
179
|
+
let arr = self.read_typed_variable::<i32>(var)?;
|
|
180
|
+
Ok(arr.mapv(|v| v as f64))
|
|
181
|
+
}
|
|
182
|
+
NcType::Float => {
|
|
183
|
+
let arr = self.read_typed_variable::<f32>(var)?;
|
|
184
|
+
Ok(arr.mapv(|v| v as f64))
|
|
185
|
+
}
|
|
186
|
+
NcType::Double => self.read_typed_variable::<f64>(var),
|
|
187
|
+
NcType::UByte => {
|
|
188
|
+
let arr = self.read_typed_variable::<u8>(var)?;
|
|
189
|
+
Ok(arr.mapv(|v| v as f64))
|
|
190
|
+
}
|
|
191
|
+
NcType::UShort => {
|
|
192
|
+
let arr = self.read_typed_variable::<u16>(var)?;
|
|
193
|
+
Ok(arr.mapv(|v| v as f64))
|
|
194
|
+
}
|
|
195
|
+
NcType::UInt => {
|
|
196
|
+
let arr = self.read_typed_variable::<u32>(var)?;
|
|
197
|
+
Ok(arr.mapv(|v| v as f64))
|
|
198
|
+
}
|
|
199
|
+
NcType::Int64 => {
|
|
200
|
+
let arr = self.read_typed_variable::<i64>(var)?;
|
|
201
|
+
Ok(arr.mapv(|v| v as f64))
|
|
202
|
+
}
|
|
203
|
+
NcType::UInt64 => {
|
|
204
|
+
let arr = self.read_typed_variable::<u64>(var)?;
|
|
205
|
+
Ok(arr.mapv(|v| v as f64))
|
|
206
|
+
}
|
|
207
|
+
NcType::Char => Err(Error::TypeMismatch {
|
|
208
|
+
expected: "numeric type".to_string(),
|
|
209
|
+
actual: "Char".to_string(),
|
|
210
|
+
}),
|
|
211
|
+
NcType::String => Err(Error::TypeMismatch {
|
|
212
|
+
expected: "numeric type".to_string(),
|
|
213
|
+
actual: "String".to_string(),
|
|
214
|
+
}),
|
|
215
|
+
_ => Err(Error::TypeMismatch {
|
|
216
|
+
expected: "numeric type".to_string(),
|
|
217
|
+
actual: format!("{:?}", var.dtype),
|
|
218
|
+
}),
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/// Read a char variable as a `String` (or `Vec<String>` for multi-dimensional).
|
|
223
|
+
pub fn read_variable_as_string(&self, name: &str) -> Result<String> {
|
|
224
|
+
let mut strings = self.read_variable_as_strings(name)?;
|
|
225
|
+
match strings.len() {
|
|
226
|
+
1 => Ok(strings.swap_remove(0)),
|
|
227
|
+
0 => Err(Error::InvalidData(format!(
|
|
228
|
+
"variable '{}' contains no string elements",
|
|
229
|
+
name
|
|
230
|
+
))),
|
|
231
|
+
count => Err(Error::InvalidData(format!(
|
|
232
|
+
"variable '{}' contains {count} strings; use read_variable_as_strings()",
|
|
233
|
+
name
|
|
234
|
+
))),
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/// Read a char variable as a flat vector of strings.
|
|
239
|
+
///
|
|
240
|
+
/// For 2-D and higher char arrays, the last dimension is interpreted as the
|
|
241
|
+
/// string length and the leading dimensions are flattened.
|
|
242
|
+
pub fn read_variable_as_strings(&self, name: &str) -> Result<Vec<String>> {
|
|
243
|
+
let var = self.find_variable(name)?;
|
|
244
|
+
if var.dtype != NcType::Char {
|
|
245
|
+
return Err(Error::TypeMismatch {
|
|
246
|
+
expected: "Char".to_string(),
|
|
247
|
+
actual: format!("{:?}", var.dtype),
|
|
248
|
+
});
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
let arr = self.read_typed_variable::<u8>(var)?;
|
|
252
|
+
let bytes: Vec<u8> = arr.iter().copied().collect();
|
|
253
|
+
decode_char_variable_strings(var, &bytes)
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/// Read a slice (hyperslab) of a variable.
|
|
257
|
+
///
|
|
258
|
+
/// Classic variables are read directly from the on-disk byte ranges for
|
|
259
|
+
/// arbitrary selections.
|
|
260
|
+
pub fn read_variable_slice<T: NcReadType>(
|
|
261
|
+
&self,
|
|
262
|
+
name: &str,
|
|
263
|
+
selection: &crate::types::NcSliceInfo,
|
|
264
|
+
) -> Result<ArrayD<T>> {
|
|
265
|
+
let var = self.find_variable(name)?;
|
|
266
|
+
let expected = T::nc_type();
|
|
267
|
+
if var.dtype != expected {
|
|
268
|
+
return Err(Error::TypeMismatch {
|
|
269
|
+
expected: format!("{:?}", expected),
|
|
270
|
+
actual: format!("{:?}", var.dtype),
|
|
271
|
+
});
|
|
272
|
+
}
|
|
273
|
+
let resolved = resolve_classic_selection(
|
|
274
|
+
var,
|
|
275
|
+
selection,
|
|
276
|
+
if var.is_record_var { self.numrecs } else { 0 },
|
|
277
|
+
)?;
|
|
278
|
+
check_selection_fits_file::<T>(var, &resolved, self.storage.len())?;
|
|
279
|
+
|
|
280
|
+
if !var.is_record_var {
|
|
281
|
+
return read_non_record_variable_slice_direct(&self.storage, var, &resolved);
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
let record_stride = compute_record_stride(&self.root_group.variables)?;
|
|
285
|
+
read_record_variable_slice_direct(
|
|
286
|
+
&self.storage,
|
|
287
|
+
var,
|
|
288
|
+
self.numrecs,
|
|
289
|
+
record_stride,
|
|
290
|
+
&resolved,
|
|
291
|
+
)
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/// Read a variable slice using Rayon for large planned byte ranges.
|
|
295
|
+
#[cfg(feature = "rayon")]
|
|
296
|
+
pub fn read_variable_slice_parallel<T: NcReadType>(
|
|
297
|
+
&self,
|
|
298
|
+
name: &str,
|
|
299
|
+
selection: &crate::types::NcSliceInfo,
|
|
300
|
+
) -> Result<ArrayD<T>> {
|
|
301
|
+
let var = self.find_variable(name)?;
|
|
302
|
+
let expected = T::nc_type();
|
|
303
|
+
if var.dtype != expected {
|
|
304
|
+
return Err(Error::TypeMismatch {
|
|
305
|
+
expected: format!("{:?}", expected),
|
|
306
|
+
actual: format!("{:?}", var.dtype),
|
|
307
|
+
});
|
|
308
|
+
}
|
|
309
|
+
let resolved = resolve_classic_selection(
|
|
310
|
+
var,
|
|
311
|
+
selection,
|
|
312
|
+
if var.is_record_var { self.numrecs } else { 0 },
|
|
313
|
+
)?;
|
|
314
|
+
check_selection_fits_file::<T>(var, &resolved, self.storage.len())?;
|
|
315
|
+
|
|
316
|
+
if !var.is_record_var {
|
|
317
|
+
return read_non_record_variable_slice_parallel(&self.storage, var, &resolved);
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
let record_stride = compute_record_stride(&self.root_group.variables)?;
|
|
321
|
+
read_record_variable_slice_parallel(
|
|
322
|
+
&self.storage,
|
|
323
|
+
var,
|
|
324
|
+
self.numrecs,
|
|
325
|
+
record_stride,
|
|
326
|
+
&resolved,
|
|
327
|
+
)
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/// Read a slice with automatic type promotion to f64.
|
|
331
|
+
pub fn read_variable_slice_as_f64(
|
|
332
|
+
&self,
|
|
333
|
+
name: &str,
|
|
334
|
+
selection: &crate::types::NcSliceInfo,
|
|
335
|
+
) -> Result<ArrayD<f64>> {
|
|
336
|
+
let var = self.find_variable(name)?;
|
|
337
|
+
|
|
338
|
+
macro_rules! slice_promoted {
|
|
339
|
+
($ty:ty) => {{
|
|
340
|
+
let sliced = self.read_variable_slice::<$ty>(name, selection)?;
|
|
341
|
+
Ok(sliced.mapv(|v| v as f64))
|
|
342
|
+
}};
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
match var.dtype {
|
|
346
|
+
NcType::Byte => slice_promoted!(i8),
|
|
347
|
+
NcType::Short => slice_promoted!(i16),
|
|
348
|
+
NcType::Int => slice_promoted!(i32),
|
|
349
|
+
NcType::Float => slice_promoted!(f32),
|
|
350
|
+
NcType::Double => slice_promoted!(f64),
|
|
351
|
+
NcType::UByte => slice_promoted!(u8),
|
|
352
|
+
NcType::UShort => slice_promoted!(u16),
|
|
353
|
+
NcType::UInt => slice_promoted!(u32),
|
|
354
|
+
NcType::Int64 => slice_promoted!(i64),
|
|
355
|
+
NcType::UInt64 => slice_promoted!(u64),
|
|
356
|
+
NcType::Char => Err(Error::TypeMismatch {
|
|
357
|
+
expected: "numeric type".to_string(),
|
|
358
|
+
actual: "Char".to_string(),
|
|
359
|
+
}),
|
|
360
|
+
_ => Err(Error::TypeMismatch {
|
|
361
|
+
expected: "numeric type".to_string(),
|
|
362
|
+
actual: format!("{:?}", var.dtype),
|
|
363
|
+
}),
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
/// Internal: find a variable by name.
|
|
368
|
+
fn find_variable(&self, name: &str) -> Result<&NcVariable> {
|
|
369
|
+
self.root_group
|
|
370
|
+
.variables
|
|
371
|
+
.iter()
|
|
372
|
+
.find(|v| v.name == name)
|
|
373
|
+
.ok_or_else(|| Error::VariableNotFound(name.to_string()))
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
/// Internal: read a variable with the correct record handling.
|
|
377
|
+
fn read_typed_variable<T: NcReadType>(&self, var: &NcVariable) -> Result<ArrayD<T>> {
|
|
378
|
+
if var.is_record_var {
|
|
379
|
+
let record_stride = compute_record_stride(&self.root_group.variables)?;
|
|
380
|
+
data::read_record_variable_from_storage(&self.storage, var, self.numrecs, record_stride)
|
|
381
|
+
} else {
|
|
382
|
+
data::read_non_record_variable_from_storage(&self.storage, var)
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
fn decode_char_variable_strings(var: &NcVariable, bytes: &[u8]) -> Result<Vec<String>> {
|
|
388
|
+
let shape = var.shape();
|
|
389
|
+
if shape.len() <= 1 {
|
|
390
|
+
return Ok(vec![decode_char_string(bytes)]);
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
let string_len = checked_usize_from_u64(
|
|
394
|
+
*shape
|
|
395
|
+
.last()
|
|
396
|
+
.ok_or_else(|| Error::InvalidData("char variable missing string axis".into()))?,
|
|
397
|
+
"char string length",
|
|
398
|
+
)?;
|
|
399
|
+
let string_count_u64 = checked_shape_elements(&shape[..shape.len() - 1], "char string count")?;
|
|
400
|
+
let string_count = checked_usize_from_u64(string_count_u64, "char string count")?;
|
|
401
|
+
let expected_bytes = string_count.checked_mul(string_len).ok_or_else(|| {
|
|
402
|
+
Error::InvalidData("char string byte count exceeds platform usize".to_string())
|
|
403
|
+
})?;
|
|
404
|
+
|
|
405
|
+
if bytes.len() < expected_bytes {
|
|
406
|
+
return Err(Error::InvalidData(format!(
|
|
407
|
+
"char variable '{}' data too short: need {} bytes, have {}",
|
|
408
|
+
var.name,
|
|
409
|
+
expected_bytes,
|
|
410
|
+
bytes.len()
|
|
411
|
+
)));
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
if string_len == 0 {
|
|
415
|
+
return Ok(vec![String::new(); string_count]);
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
Ok(bytes[..expected_bytes]
|
|
419
|
+
.chunks_exact(string_len)
|
|
420
|
+
.map(decode_char_string)
|
|
421
|
+
.collect())
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
fn decode_char_string(bytes: &[u8]) -> String {
|
|
425
|
+
String::from_utf8_lossy(bytes)
|
|
426
|
+
.trim_end_matches('\0')
|
|
427
|
+
.to_string()
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
fn variable_shape_for_selection(var: &NcVariable, numrecs: u64) -> Vec<u64> {
|
|
431
|
+
let mut shape = var.shape();
|
|
432
|
+
if var.is_record_var && !shape.is_empty() {
|
|
433
|
+
shape[0] = numrecs;
|
|
434
|
+
}
|
|
435
|
+
shape
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
fn row_major_strides(shape: &[u64], context: &str) -> Result<Vec<u64>> {
|
|
439
|
+
let ndim = shape.len();
|
|
440
|
+
if ndim == 0 {
|
|
441
|
+
return Ok(Vec::new());
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
let mut strides = vec![1u64; ndim];
|
|
445
|
+
for i in (0..ndim - 1).rev() {
|
|
446
|
+
strides[i] = checked_mul_u64(strides[i + 1], shape[i + 1], context)?;
|
|
447
|
+
}
|
|
448
|
+
Ok(strides)
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
fn resolve_classic_selection(
|
|
452
|
+
var: &NcVariable,
|
|
453
|
+
selection: &crate::types::NcSliceInfo,
|
|
454
|
+
numrecs: u64,
|
|
455
|
+
) -> Result<ResolvedClassicSelection> {
|
|
456
|
+
use crate::types::NcSliceInfoElem;
|
|
457
|
+
|
|
458
|
+
let shape = variable_shape_for_selection(var, numrecs);
|
|
459
|
+
if selection.selections.len() != shape.len() {
|
|
460
|
+
return Err(Error::InvalidData(format!(
|
|
461
|
+
"selection has {} dimensions but variable '{}' has {}",
|
|
462
|
+
selection.selections.len(),
|
|
463
|
+
var.name,
|
|
464
|
+
shape.len()
|
|
465
|
+
)));
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
let mut dims = Vec::with_capacity(shape.len());
|
|
469
|
+
let mut result_shape = Vec::new();
|
|
470
|
+
let mut result_elements = 1usize;
|
|
471
|
+
|
|
472
|
+
for (dim, (sel, &dim_size)) in selection.selections.iter().zip(shape.iter()).enumerate() {
|
|
473
|
+
match sel {
|
|
474
|
+
NcSliceInfoElem::Index(idx) => {
|
|
475
|
+
if *idx >= dim_size {
|
|
476
|
+
return Err(Error::InvalidData(format!(
|
|
477
|
+
"index {} out of bounds for dimension {} (size {})",
|
|
478
|
+
idx, dim, dim_size
|
|
479
|
+
)));
|
|
480
|
+
}
|
|
481
|
+
dims.push(ResolvedClassicSelectionDim::Index(*idx));
|
|
482
|
+
}
|
|
483
|
+
NcSliceInfoElem::Slice { start, end, step } => {
|
|
484
|
+
if *step == 0 {
|
|
485
|
+
return Err(Error::InvalidData("slice step cannot be 0".to_string()));
|
|
486
|
+
}
|
|
487
|
+
if *start > dim_size {
|
|
488
|
+
return Err(Error::InvalidData(format!(
|
|
489
|
+
"slice start {} out of bounds for dimension {} (size {})",
|
|
490
|
+
start, dim, dim_size
|
|
491
|
+
)));
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
let actual_end = if *end == u64::MAX {
|
|
495
|
+
dim_size
|
|
496
|
+
} else {
|
|
497
|
+
(*end).min(dim_size)
|
|
498
|
+
};
|
|
499
|
+
let count_u64 = if *start >= actual_end {
|
|
500
|
+
0
|
|
501
|
+
} else {
|
|
502
|
+
(actual_end - *start).div_ceil(*step)
|
|
503
|
+
};
|
|
504
|
+
let count = checked_usize_from_u64(count_u64, "classic slice result dimension")?;
|
|
505
|
+
|
|
506
|
+
result_shape.push(count);
|
|
507
|
+
result_elements = result_elements.checked_mul(count).ok_or_else(|| {
|
|
508
|
+
Error::InvalidData(
|
|
509
|
+
"classic slice result element count exceeds platform usize".to_string(),
|
|
510
|
+
)
|
|
511
|
+
})?;
|
|
512
|
+
dims.push(ResolvedClassicSelectionDim::Slice {
|
|
513
|
+
start: *start,
|
|
514
|
+
step: *step,
|
|
515
|
+
count,
|
|
516
|
+
is_full_unit_stride: *start == 0 && actual_end == dim_size && *step == 1,
|
|
517
|
+
});
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
Ok(ResolvedClassicSelection {
|
|
523
|
+
dims,
|
|
524
|
+
result_shape,
|
|
525
|
+
result_elements,
|
|
526
|
+
})
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
/// Classic data are uncompressed: every selected element is its own byte range in the file, so
|
|
530
|
+
/// a selection needing more bytes than the file holds comes from a corrupt header (a dimension
|
|
531
|
+
/// of 4.4e12 asked for a 26 TB buffer and aborted the process; ModernViewer MV-110 fuzzing).
|
|
532
|
+
fn check_selection_fits_file<T: NcReadType>(
|
|
533
|
+
var: &NcVariable,
|
|
534
|
+
resolved: &ResolvedClassicSelection,
|
|
535
|
+
file_len: u64,
|
|
536
|
+
) -> Result<()> {
|
|
537
|
+
// `T` matches the variable's type (checked by the callers), so this is its size on disk.
|
|
538
|
+
let bytes = (resolved.result_elements as u64).checked_mul(T::element_size() as u64);
|
|
539
|
+
match bytes {
|
|
540
|
+
Some(bytes) if bytes <= file_len => Ok(()),
|
|
541
|
+
_ => Err(Error::InvalidData(format!(
|
|
542
|
+
"selection of variable '{}' ({} elements) exceeds the {file_len}-byte file; the header is corrupt",
|
|
543
|
+
var.name, resolved.result_elements
|
|
544
|
+
))),
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
fn read_non_record_variable_slice_direct<T: NcReadType>(
|
|
549
|
+
storage: &ClassicStorage,
|
|
550
|
+
var: &NcVariable,
|
|
551
|
+
resolved: &ResolvedClassicSelection,
|
|
552
|
+
) -> Result<ArrayD<T>> {
|
|
553
|
+
let shape = variable_shape_for_selection(var, 0);
|
|
554
|
+
build_array_from_contiguous_selection::<T>(storage, var.data_offset, &shape, resolved)
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
#[cfg(feature = "rayon")]
|
|
558
|
+
fn read_non_record_variable_slice_parallel<T: NcReadType>(
|
|
559
|
+
storage: &ClassicStorage,
|
|
560
|
+
var: &NcVariable,
|
|
561
|
+
resolved: &ResolvedClassicSelection,
|
|
562
|
+
) -> Result<ArrayD<T>> {
|
|
563
|
+
use ndarray::IxDyn;
|
|
564
|
+
|
|
565
|
+
let shape = variable_shape_for_selection(var, 0);
|
|
566
|
+
let plan = build_contiguous_selection_plan::<T>(&shape, &resolved.dims)?;
|
|
567
|
+
let spans = plan_contiguous_selection_spans(var.data_offset, &plan, resolved.result_elements)?;
|
|
568
|
+
let values = read_planned_spans_maybe_parallel::<T>(storage, &spans, resolved.result_elements)?;
|
|
569
|
+
|
|
570
|
+
ArrayD::from_shape_vec(IxDyn(&resolved.result_shape), values)
|
|
571
|
+
.map_err(|e| Error::InvalidData(format!("failed to create array: {e}")))
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
fn read_record_variable_slice_direct<T: NcReadType>(
|
|
575
|
+
storage: &ClassicStorage,
|
|
576
|
+
var: &NcVariable,
|
|
577
|
+
numrecs: u64,
|
|
578
|
+
record_stride: u64,
|
|
579
|
+
resolved: &ResolvedClassicSelection,
|
|
580
|
+
) -> Result<ArrayD<T>> {
|
|
581
|
+
use ndarray::IxDyn;
|
|
582
|
+
|
|
583
|
+
if resolved.result_elements == 0 {
|
|
584
|
+
return ArrayD::from_shape_vec(IxDyn(&resolved.result_shape), Vec::new())
|
|
585
|
+
.map_err(|e| Error::InvalidData(format!("failed to create array: {e}")));
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
let shape = variable_shape_for_selection(var, numrecs);
|
|
589
|
+
let inner_shape = &shape[1..];
|
|
590
|
+
let inner_dims = resolved.dims[1..].to_vec();
|
|
591
|
+
let inner_resolved = ResolvedClassicSelection {
|
|
592
|
+
result_shape: selection_result_shape(&inner_dims),
|
|
593
|
+
result_elements: selection_result_elements(&inner_dims)?,
|
|
594
|
+
dims: inner_dims,
|
|
595
|
+
};
|
|
596
|
+
let inner_plan = build_contiguous_selection_plan::<T>(inner_shape, &inner_resolved.dims)?;
|
|
597
|
+
let mut values = Vec::with_capacity(resolved.result_elements);
|
|
598
|
+
let context = RecordSliceContext {
|
|
599
|
+
storage,
|
|
600
|
+
base_offset: var.data_offset,
|
|
601
|
+
record_stride,
|
|
602
|
+
inner_resolved: &inner_resolved,
|
|
603
|
+
inner_plan: &inner_plan,
|
|
604
|
+
};
|
|
605
|
+
|
|
606
|
+
match &resolved.dims[0] {
|
|
607
|
+
ResolvedClassicSelectionDim::Index(record) => {
|
|
608
|
+
append_one_record_slice::<T>(&context, *record, &mut values)?
|
|
609
|
+
}
|
|
610
|
+
ResolvedClassicSelectionDim::Slice {
|
|
611
|
+
start, step, count, ..
|
|
612
|
+
} => {
|
|
613
|
+
for ordinal in 0..*count {
|
|
614
|
+
let record = start
|
|
615
|
+
.checked_add(checked_mul_u64(
|
|
616
|
+
ordinal as u64,
|
|
617
|
+
*step,
|
|
618
|
+
"classic record slice coordinate",
|
|
619
|
+
)?)
|
|
620
|
+
.ok_or_else(|| {
|
|
621
|
+
Error::InvalidData(
|
|
622
|
+
"classic record slice coordinate exceeds u64".to_string(),
|
|
623
|
+
)
|
|
624
|
+
})?;
|
|
625
|
+
append_one_record_slice::<T>(&context, record, &mut values)?;
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
debug_assert_eq!(values.len(), resolved.result_elements);
|
|
631
|
+
ArrayD::from_shape_vec(IxDyn(&resolved.result_shape), values)
|
|
632
|
+
.map_err(|e| Error::InvalidData(format!("failed to create array: {e}")))
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
#[cfg(feature = "rayon")]
|
|
636
|
+
fn read_record_variable_slice_parallel<T: NcReadType>(
|
|
637
|
+
storage: &ClassicStorage,
|
|
638
|
+
var: &NcVariable,
|
|
639
|
+
numrecs: u64,
|
|
640
|
+
record_stride: u64,
|
|
641
|
+
resolved: &ResolvedClassicSelection,
|
|
642
|
+
) -> Result<ArrayD<T>> {
|
|
643
|
+
use ndarray::IxDyn;
|
|
644
|
+
|
|
645
|
+
if resolved.result_elements == 0 {
|
|
646
|
+
return ArrayD::from_shape_vec(IxDyn(&resolved.result_shape), Vec::new())
|
|
647
|
+
.map_err(|e| Error::InvalidData(format!("failed to create array: {e}")));
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
let shape = variable_shape_for_selection(var, numrecs);
|
|
651
|
+
let inner_shape = &shape[1..];
|
|
652
|
+
let inner_dims = resolved.dims[1..].to_vec();
|
|
653
|
+
let inner_resolved = ResolvedClassicSelection {
|
|
654
|
+
result_shape: selection_result_shape(&inner_dims),
|
|
655
|
+
result_elements: selection_result_elements(&inner_dims)?,
|
|
656
|
+
dims: inner_dims,
|
|
657
|
+
};
|
|
658
|
+
let inner_plan = build_contiguous_selection_plan::<T>(inner_shape, &inner_resolved.dims)?;
|
|
659
|
+
let mut spans = Vec::new();
|
|
660
|
+
|
|
661
|
+
match &resolved.dims[0] {
|
|
662
|
+
ResolvedClassicSelectionDim::Index(record) => append_one_record_slice_spans(
|
|
663
|
+
var.data_offset,
|
|
664
|
+
record_stride,
|
|
665
|
+
&inner_plan,
|
|
666
|
+
*record,
|
|
667
|
+
&mut spans,
|
|
668
|
+
)?,
|
|
669
|
+
ResolvedClassicSelectionDim::Slice {
|
|
670
|
+
start, step, count, ..
|
|
671
|
+
} => {
|
|
672
|
+
for ordinal in 0..*count {
|
|
673
|
+
let record = start
|
|
674
|
+
.checked_add(checked_mul_u64(
|
|
675
|
+
ordinal as u64,
|
|
676
|
+
*step,
|
|
677
|
+
"classic record slice coordinate",
|
|
678
|
+
)?)
|
|
679
|
+
.ok_or_else(|| {
|
|
680
|
+
Error::InvalidData(
|
|
681
|
+
"classic record slice coordinate exceeds u64".to_string(),
|
|
682
|
+
)
|
|
683
|
+
})?;
|
|
684
|
+
append_one_record_slice_spans(
|
|
685
|
+
var.data_offset,
|
|
686
|
+
record_stride,
|
|
687
|
+
&inner_plan,
|
|
688
|
+
record,
|
|
689
|
+
&mut spans,
|
|
690
|
+
)?;
|
|
691
|
+
}
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
let values = read_planned_spans_maybe_parallel::<T>(storage, &spans, resolved.result_elements)?;
|
|
696
|
+
ArrayD::from_shape_vec(IxDyn(&resolved.result_shape), values)
|
|
697
|
+
.map_err(|e| Error::InvalidData(format!("failed to create array: {e}")))
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
fn selection_result_shape(dims: &[ResolvedClassicSelectionDim]) -> Vec<usize> {
|
|
701
|
+
dims.iter()
|
|
702
|
+
.filter_map(|dim| match dim {
|
|
703
|
+
ResolvedClassicSelectionDim::Index(_) => None,
|
|
704
|
+
ResolvedClassicSelectionDim::Slice { count, .. } => Some(*count),
|
|
705
|
+
})
|
|
706
|
+
.collect()
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
fn selection_result_elements(dims: &[ResolvedClassicSelectionDim]) -> Result<usize> {
|
|
710
|
+
let mut elements = 1usize;
|
|
711
|
+
for dim in dims {
|
|
712
|
+
if let ResolvedClassicSelectionDim::Slice { count, .. } = dim {
|
|
713
|
+
elements = elements.checked_mul(*count).ok_or_else(|| {
|
|
714
|
+
Error::InvalidData(
|
|
715
|
+
"classic slice result element count exceeds platform usize".to_string(),
|
|
716
|
+
)
|
|
717
|
+
})?;
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
Ok(elements)
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
fn build_array_from_contiguous_selection<T: NcReadType>(
|
|
724
|
+
storage: &ClassicStorage,
|
|
725
|
+
base_offset: u64,
|
|
726
|
+
shape: &[u64],
|
|
727
|
+
resolved: &ResolvedClassicSelection,
|
|
728
|
+
) -> Result<ArrayD<T>> {
|
|
729
|
+
use ndarray::IxDyn;
|
|
730
|
+
|
|
731
|
+
let plan = build_contiguous_selection_plan::<T>(shape, &resolved.dims)?;
|
|
732
|
+
let values = read_contiguous_selection_values_with_plan::<T>(
|
|
733
|
+
storage,
|
|
734
|
+
base_offset,
|
|
735
|
+
&plan,
|
|
736
|
+
resolved.result_elements,
|
|
737
|
+
)?;
|
|
738
|
+
ArrayD::from_shape_vec(IxDyn(&resolved.result_shape), values)
|
|
739
|
+
.map_err(|e| Error::InvalidData(format!("failed to create array: {e}")))
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
fn build_contiguous_selection_plan<T: NcReadType>(
|
|
743
|
+
shape: &[u64],
|
|
744
|
+
dims: &[ResolvedClassicSelectionDim],
|
|
745
|
+
) -> Result<ContiguousSelectionPlan> {
|
|
746
|
+
let strides = row_major_strides(shape, "classic slice stride")?;
|
|
747
|
+
let tail_start = dims
|
|
748
|
+
.iter()
|
|
749
|
+
.rposition(|dim| !dim.is_full_unit_stride())
|
|
750
|
+
.map_or(0, |idx| idx + 1);
|
|
751
|
+
let block_elements_u64 = checked_shape_elements(
|
|
752
|
+
&shape[tail_start..],
|
|
753
|
+
"classic slice contiguous block element count",
|
|
754
|
+
)?;
|
|
755
|
+
let block_elements = checked_usize_from_u64(
|
|
756
|
+
block_elements_u64,
|
|
757
|
+
"classic slice contiguous block element count",
|
|
758
|
+
)?;
|
|
759
|
+
let block_bytes = checked_usize_from_u64(
|
|
760
|
+
checked_mul_u64(
|
|
761
|
+
block_elements_u64,
|
|
762
|
+
T::element_size() as u64,
|
|
763
|
+
"classic slice contiguous block size in bytes",
|
|
764
|
+
)?,
|
|
765
|
+
"classic slice contiguous block size in bytes",
|
|
766
|
+
)?;
|
|
767
|
+
|
|
768
|
+
Ok(ContiguousSelectionPlan {
|
|
769
|
+
dims: dims.to_vec(),
|
|
770
|
+
strides,
|
|
771
|
+
tail_start,
|
|
772
|
+
block_elements,
|
|
773
|
+
block_bytes,
|
|
774
|
+
elem_size: T::element_size() as u64,
|
|
775
|
+
})
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
fn read_contiguous_selection_values_with_plan<T: NcReadType>(
|
|
779
|
+
storage: &ClassicStorage,
|
|
780
|
+
base_offset: u64,
|
|
781
|
+
plan: &ContiguousSelectionPlan,
|
|
782
|
+
result_elements: usize,
|
|
783
|
+
) -> Result<Vec<T>> {
|
|
784
|
+
if result_elements == 0 {
|
|
785
|
+
return Ok(Vec::new());
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
let spans = plan_contiguous_selection_spans(base_offset, plan, result_elements)?;
|
|
789
|
+
read_planned_spans_serial::<T>(storage, &spans, result_elements)
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
fn plan_contiguous_selection_spans(
|
|
793
|
+
base_offset: u64,
|
|
794
|
+
plan: &ContiguousSelectionPlan,
|
|
795
|
+
result_elements: usize,
|
|
796
|
+
) -> Result<Vec<PlannedReadSpan>> {
|
|
797
|
+
if result_elements == 0 {
|
|
798
|
+
return Ok(Vec::new());
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
let mut spans = Vec::new();
|
|
802
|
+
let context = BlockReadContext { base_offset, plan };
|
|
803
|
+
plan_selected_blocks_recursive(0, 0, &context, &mut spans)?;
|
|
804
|
+
Ok(spans)
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
fn read_planned_spans_serial<T: NcReadType>(
|
|
808
|
+
storage: &ClassicStorage,
|
|
809
|
+
spans: &[PlannedReadSpan],
|
|
810
|
+
result_elements: usize,
|
|
811
|
+
) -> Result<Vec<T>> {
|
|
812
|
+
let total_elements = planned_spans_total_elements(spans)?;
|
|
813
|
+
if total_elements != result_elements {
|
|
814
|
+
return Err(Error::InvalidData(format!(
|
|
815
|
+
"classic planned span element count {total_elements} does not match result size {result_elements}"
|
|
816
|
+
)));
|
|
817
|
+
}
|
|
818
|
+
|
|
819
|
+
let mut values = vec![T::default(); result_elements];
|
|
820
|
+
let mut dst_start = 0usize;
|
|
821
|
+
for span in spans {
|
|
822
|
+
let data = storage.read_range(span.offset, span.bytes)?;
|
|
823
|
+
let dst_end = dst_start.checked_add(span.elements).ok_or_else(|| {
|
|
824
|
+
Error::InvalidData(
|
|
825
|
+
"classic planned destination range exceeds platform usize".to_string(),
|
|
826
|
+
)
|
|
827
|
+
})?;
|
|
828
|
+
T::decode_bulk_be_into(data.as_ref(), &mut values[dst_start..dst_end])?;
|
|
829
|
+
dst_start = dst_end;
|
|
830
|
+
}
|
|
831
|
+
debug_assert_eq!(values.len(), result_elements);
|
|
832
|
+
Ok(values)
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
fn planned_spans_total_elements(spans: &[PlannedReadSpan]) -> Result<usize> {
|
|
836
|
+
let mut total = 0usize;
|
|
837
|
+
for span in spans {
|
|
838
|
+
total = total.checked_add(span.elements).ok_or_else(|| {
|
|
839
|
+
Error::InvalidData("classic planned element count exceeds platform usize".to_string())
|
|
840
|
+
})?;
|
|
841
|
+
}
|
|
842
|
+
Ok(total)
|
|
843
|
+
}
|
|
844
|
+
|
|
845
|
+
fn append_one_record_slice<T: NcReadType>(
|
|
846
|
+
context: &RecordSliceContext<'_>,
|
|
847
|
+
record: u64,
|
|
848
|
+
values: &mut Vec<T>,
|
|
849
|
+
) -> Result<()> {
|
|
850
|
+
let record_offset = context
|
|
851
|
+
.base_offset
|
|
852
|
+
.checked_add(record.checked_mul(context.record_stride).ok_or_else(|| {
|
|
853
|
+
Error::InvalidData("classic record byte offset exceeds u64".to_string())
|
|
854
|
+
})?)
|
|
855
|
+
.ok_or_else(|| Error::InvalidData("classic record byte offset exceeds u64".to_string()))?;
|
|
856
|
+
let mut decoded = read_contiguous_selection_values_with_plan::<T>(
|
|
857
|
+
context.storage,
|
|
858
|
+
record_offset,
|
|
859
|
+
context.inner_plan,
|
|
860
|
+
context.inner_resolved.result_elements,
|
|
861
|
+
)?;
|
|
862
|
+
values.append(&mut decoded);
|
|
863
|
+
Ok(())
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
#[cfg(feature = "rayon")]
|
|
867
|
+
fn append_one_record_slice_spans(
|
|
868
|
+
base_offset: u64,
|
|
869
|
+
record_stride: u64,
|
|
870
|
+
inner_plan: &ContiguousSelectionPlan,
|
|
871
|
+
record: u64,
|
|
872
|
+
spans: &mut Vec<PlannedReadSpan>,
|
|
873
|
+
) -> Result<()> {
|
|
874
|
+
let record_offset = base_offset
|
|
875
|
+
.checked_add(record.checked_mul(record_stride).ok_or_else(|| {
|
|
876
|
+
Error::InvalidData("classic record byte offset exceeds u64".to_string())
|
|
877
|
+
})?)
|
|
878
|
+
.ok_or_else(|| Error::InvalidData("classic record byte offset exceeds u64".to_string()))?;
|
|
879
|
+
let record_spans = plan_contiguous_selection_spans(record_offset, inner_plan, 1)?;
|
|
880
|
+
for span in record_spans {
|
|
881
|
+
push_planned_span(spans, span)?;
|
|
882
|
+
}
|
|
883
|
+
Ok(())
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
fn plan_selected_blocks_recursive(
|
|
887
|
+
level: usize,
|
|
888
|
+
current_offset: u64,
|
|
889
|
+
context: &BlockReadContext<'_>,
|
|
890
|
+
spans: &mut Vec<PlannedReadSpan>,
|
|
891
|
+
) -> Result<()> {
|
|
892
|
+
if level == context.plan.tail_start {
|
|
893
|
+
let byte_offset = checked_mul_u64(
|
|
894
|
+
current_offset,
|
|
895
|
+
context.plan.elem_size,
|
|
896
|
+
"classic slice element byte offset",
|
|
897
|
+
)?;
|
|
898
|
+
let start = context
|
|
899
|
+
.base_offset
|
|
900
|
+
.checked_add(byte_offset)
|
|
901
|
+
.ok_or_else(|| {
|
|
902
|
+
Error::InvalidData("classic slice byte offset exceeds u64".to_string())
|
|
903
|
+
})?;
|
|
904
|
+
|
|
905
|
+
push_planned_span(
|
|
906
|
+
spans,
|
|
907
|
+
PlannedReadSpan {
|
|
908
|
+
offset: start,
|
|
909
|
+
elements: context.plan.block_elements,
|
|
910
|
+
bytes: context.plan.block_bytes,
|
|
911
|
+
},
|
|
912
|
+
)?;
|
|
913
|
+
return Ok(());
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
match &context.plan.dims[level] {
|
|
917
|
+
ResolvedClassicSelectionDim::Index(idx) => plan_selected_blocks_recursive(
|
|
918
|
+
level + 1,
|
|
919
|
+
current_offset
|
|
920
|
+
.checked_add(checked_mul_u64(
|
|
921
|
+
*idx,
|
|
922
|
+
context.plan.strides[level],
|
|
923
|
+
"classic slice logical element offset",
|
|
924
|
+
)?)
|
|
925
|
+
.ok_or_else(|| {
|
|
926
|
+
Error::InvalidData(
|
|
927
|
+
"classic slice logical element offset exceeds u64".to_string(),
|
|
928
|
+
)
|
|
929
|
+
})?,
|
|
930
|
+
context,
|
|
931
|
+
spans,
|
|
932
|
+
),
|
|
933
|
+
ResolvedClassicSelectionDim::Slice {
|
|
934
|
+
start, step, count, ..
|
|
935
|
+
} => {
|
|
936
|
+
let start = *start;
|
|
937
|
+
let step = *step;
|
|
938
|
+
let count = *count;
|
|
939
|
+
for ordinal in 0..count {
|
|
940
|
+
let coord = start
|
|
941
|
+
.checked_add(checked_mul_u64(
|
|
942
|
+
ordinal as u64,
|
|
943
|
+
step,
|
|
944
|
+
"classic slice coordinate",
|
|
945
|
+
)?)
|
|
946
|
+
.ok_or_else(|| {
|
|
947
|
+
Error::InvalidData("classic slice coordinate exceeds u64".to_string())
|
|
948
|
+
})?;
|
|
949
|
+
plan_selected_blocks_recursive(
|
|
950
|
+
level + 1,
|
|
951
|
+
current_offset
|
|
952
|
+
.checked_add(checked_mul_u64(
|
|
953
|
+
coord,
|
|
954
|
+
context.plan.strides[level],
|
|
955
|
+
"classic slice logical element offset",
|
|
956
|
+
)?)
|
|
957
|
+
.ok_or_else(|| {
|
|
958
|
+
Error::InvalidData(
|
|
959
|
+
"classic slice logical element offset exceeds u64".to_string(),
|
|
960
|
+
)
|
|
961
|
+
})?,
|
|
962
|
+
context,
|
|
963
|
+
spans,
|
|
964
|
+
)?;
|
|
965
|
+
}
|
|
966
|
+
Ok(())
|
|
967
|
+
}
|
|
968
|
+
}
|
|
969
|
+
}
|
|
970
|
+
|
|
971
|
+
fn push_planned_span(spans: &mut Vec<PlannedReadSpan>, span: PlannedReadSpan) -> Result<()> {
|
|
972
|
+
if span.bytes == 0 {
|
|
973
|
+
return Ok(());
|
|
974
|
+
}
|
|
975
|
+
|
|
976
|
+
if let Some(last) = spans.last_mut() {
|
|
977
|
+
let last_end = last.offset.checked_add(last.bytes as u64).ok_or_else(|| {
|
|
978
|
+
Error::InvalidData("classic planned byte span exceeds u64".to_string())
|
|
979
|
+
})?;
|
|
980
|
+
if last_end == span.offset {
|
|
981
|
+
last.bytes = last.bytes.checked_add(span.bytes).ok_or_else(|| {
|
|
982
|
+
Error::InvalidData("classic planned byte span exceeds platform usize".to_string())
|
|
983
|
+
})?;
|
|
984
|
+
last.elements = last.elements.checked_add(span.elements).ok_or_else(|| {
|
|
985
|
+
Error::InvalidData(
|
|
986
|
+
"classic planned element span exceeds platform usize".to_string(),
|
|
987
|
+
)
|
|
988
|
+
})?;
|
|
989
|
+
return Ok(());
|
|
990
|
+
}
|
|
991
|
+
}
|
|
992
|
+
|
|
993
|
+
spans.push(span);
|
|
994
|
+
Ok(())
|
|
995
|
+
}
|
|
996
|
+
|
|
997
|
+
#[cfg(feature = "rayon")]
|
|
998
|
+
fn read_planned_spans_maybe_parallel<T: NcReadType>(
|
|
999
|
+
storage: &ClassicStorage,
|
|
1000
|
+
spans: &[PlannedReadSpan],
|
|
1001
|
+
result_elements: usize,
|
|
1002
|
+
) -> Result<Vec<T>> {
|
|
1003
|
+
let total_bytes = planned_spans_total_bytes(spans)?;
|
|
1004
|
+
let policy = storage.parallel_read_policy();
|
|
1005
|
+
if total_bytes < policy.min_bytes || spans.is_empty() {
|
|
1006
|
+
return read_planned_spans_serial::<T>(storage, spans, result_elements);
|
|
1007
|
+
}
|
|
1008
|
+
|
|
1009
|
+
let total_elements = planned_spans_total_elements(spans)?;
|
|
1010
|
+
if total_elements != result_elements {
|
|
1011
|
+
return Err(Error::InvalidData(format!(
|
|
1012
|
+
"classic planned span element count {total_elements} does not match result size {result_elements}"
|
|
1013
|
+
)));
|
|
1014
|
+
}
|
|
1015
|
+
|
|
1016
|
+
let (chunks, elements_per_chunk) =
|
|
1017
|
+
split_planned_spans_for_parallel::<T>(spans, policy.target_chunk_bytes)?;
|
|
1018
|
+
let mut values = vec![T::default(); result_elements];
|
|
1019
|
+
debug_assert_eq!(values.len().div_ceil(elements_per_chunk), chunks.len());
|
|
1020
|
+
values
|
|
1021
|
+
.par_chunks_mut(elements_per_chunk)
|
|
1022
|
+
.zip(chunks.par_iter())
|
|
1023
|
+
.try_for_each(|(dst, chunk)| read_planned_chunk_into::<T>(storage, chunk, dst))?;
|
|
1024
|
+
Ok(values)
|
|
1025
|
+
}
|
|
1026
|
+
|
|
1027
|
+
#[cfg(feature = "rayon")]
|
|
1028
|
+
fn planned_spans_total_bytes(spans: &[PlannedReadSpan]) -> Result<usize> {
|
|
1029
|
+
let mut total = 0usize;
|
|
1030
|
+
for span in spans {
|
|
1031
|
+
total = total.checked_add(span.bytes).ok_or_else(|| {
|
|
1032
|
+
Error::InvalidData("classic planned byte count exceeds platform usize".to_string())
|
|
1033
|
+
})?;
|
|
1034
|
+
}
|
|
1035
|
+
Ok(total)
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
#[cfg(feature = "rayon")]
|
|
1039
|
+
fn split_planned_spans_for_parallel<T: NcReadType>(
|
|
1040
|
+
spans: &[PlannedReadSpan],
|
|
1041
|
+
target_chunk_bytes: usize,
|
|
1042
|
+
) -> Result<(Vec<PlannedReadChunk>, usize)> {
|
|
1043
|
+
let elem_size = T::element_size();
|
|
1044
|
+
let elements_per_chunk = (target_chunk_bytes / elem_size.max(1)).max(1);
|
|
1045
|
+
let mut chunks = Vec::new();
|
|
1046
|
+
let mut current = PlannedReadChunk {
|
|
1047
|
+
spans: Vec::new(),
|
|
1048
|
+
elements: 0,
|
|
1049
|
+
};
|
|
1050
|
+
|
|
1051
|
+
for span in spans {
|
|
1052
|
+
let mut remaining = span.elements;
|
|
1053
|
+
let mut offset = span.offset;
|
|
1054
|
+
while remaining > 0 {
|
|
1055
|
+
if current.elements == elements_per_chunk {
|
|
1056
|
+
chunks.push(current);
|
|
1057
|
+
current = PlannedReadChunk {
|
|
1058
|
+
spans: Vec::new(),
|
|
1059
|
+
elements: 0,
|
|
1060
|
+
};
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
let available = elements_per_chunk - current.elements;
|
|
1064
|
+
let elements = remaining.min(available);
|
|
1065
|
+
let bytes = elements.checked_mul(elem_size).ok_or_else(|| {
|
|
1066
|
+
Error::InvalidData(
|
|
1067
|
+
"classic planned chunk byte count exceeds platform usize".to_string(),
|
|
1068
|
+
)
|
|
1069
|
+
})?;
|
|
1070
|
+
|
|
1071
|
+
current.spans.push(PlannedReadSpan {
|
|
1072
|
+
offset,
|
|
1073
|
+
elements,
|
|
1074
|
+
bytes,
|
|
1075
|
+
});
|
|
1076
|
+
current.elements = current.elements.checked_add(elements).ok_or_else(|| {
|
|
1077
|
+
Error::InvalidData(
|
|
1078
|
+
"classic planned chunk element count exceeds platform usize".to_string(),
|
|
1079
|
+
)
|
|
1080
|
+
})?;
|
|
1081
|
+
remaining -= elements;
|
|
1082
|
+
offset = offset.checked_add(bytes as u64).ok_or_else(|| {
|
|
1083
|
+
Error::InvalidData("classic planned chunk byte offset exceeds u64".to_string())
|
|
1084
|
+
})?;
|
|
1085
|
+
}
|
|
1086
|
+
}
|
|
1087
|
+
|
|
1088
|
+
if current.elements > 0 {
|
|
1089
|
+
chunks.push(current);
|
|
1090
|
+
}
|
|
1091
|
+
|
|
1092
|
+
Ok((chunks, elements_per_chunk))
|
|
1093
|
+
}
|
|
1094
|
+
|
|
1095
|
+
#[cfg(feature = "rayon")]
|
|
1096
|
+
fn read_planned_chunk_into<T: NcReadType>(
|
|
1097
|
+
storage: &ClassicStorage,
|
|
1098
|
+
chunk: &PlannedReadChunk,
|
|
1099
|
+
dst: &mut [T],
|
|
1100
|
+
) -> Result<()> {
|
|
1101
|
+
if dst.len() != chunk.elements {
|
|
1102
|
+
return Err(Error::InvalidData(format!(
|
|
1103
|
+
"classic planned chunk destination has {} elements, expected {}",
|
|
1104
|
+
dst.len(),
|
|
1105
|
+
chunk.elements
|
|
1106
|
+
)));
|
|
1107
|
+
}
|
|
1108
|
+
|
|
1109
|
+
let mut dst_start = 0usize;
|
|
1110
|
+
for span in &chunk.spans {
|
|
1111
|
+
let data = storage.read_range(span.offset, span.bytes)?;
|
|
1112
|
+
let dst_end = dst_start.checked_add(span.elements).ok_or_else(|| {
|
|
1113
|
+
Error::InvalidData(
|
|
1114
|
+
"classic planned chunk destination range exceeds platform usize".to_string(),
|
|
1115
|
+
)
|
|
1116
|
+
})?;
|
|
1117
|
+
T::decode_bulk_be_into(data.as_ref(), &mut dst[dst_start..dst_end])?;
|
|
1118
|
+
dst_start = dst_end;
|
|
1119
|
+
}
|
|
1120
|
+
Ok(())
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
#[cfg(test)]
|
|
1124
|
+
mod tests {
|
|
1125
|
+
use super::*;
|
|
1126
|
+
use crate::types::NcDimension;
|
|
1127
|
+
|
|
1128
|
+
fn char_variable(shape: &[u64]) -> NcVariable {
|
|
1129
|
+
NcVariable {
|
|
1130
|
+
name: "chars".to_string(),
|
|
1131
|
+
dimensions: shape
|
|
1132
|
+
.iter()
|
|
1133
|
+
.enumerate()
|
|
1134
|
+
.map(|(i, &size)| NcDimension {
|
|
1135
|
+
name: format!("d{i}"),
|
|
1136
|
+
size,
|
|
1137
|
+
is_unlimited: false,
|
|
1138
|
+
})
|
|
1139
|
+
.collect(),
|
|
1140
|
+
dtype: NcType::Char,
|
|
1141
|
+
attributes: vec![],
|
|
1142
|
+
data_offset: 0,
|
|
1143
|
+
_data_size: 0,
|
|
1144
|
+
is_record_var: false,
|
|
1145
|
+
record_size: 0,
|
|
1146
|
+
}
|
|
1147
|
+
}
|
|
1148
|
+
|
|
1149
|
+
#[test]
|
|
1150
|
+
fn decode_char_variable_strings_1d() {
|
|
1151
|
+
let var = char_variable(&[5]);
|
|
1152
|
+
let strings = decode_char_variable_strings(&var, b"alpha").unwrap();
|
|
1153
|
+
assert_eq!(strings, vec!["alpha"]);
|
|
1154
|
+
}
|
|
1155
|
+
|
|
1156
|
+
#[test]
|
|
1157
|
+
fn decode_char_variable_strings_2d() {
|
|
1158
|
+
let var = char_variable(&[2, 5]);
|
|
1159
|
+
let strings = decode_char_variable_strings(&var, b"alphabeta\0").unwrap();
|
|
1160
|
+
assert_eq!(strings, vec!["alpha", "beta"]);
|
|
1161
|
+
}
|
|
1162
|
+
}
|