@nirs4all/formats-wasm 0.2.12 → 0.2.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/THIRD_PARTY_NOTICES.md +49 -0
  2. package/nirs4all_formats_wasm_bg.wasm +0 -0
  3. package/package.json +4 -2
  4. package/reader-backports/hdf5-reader/Cargo.toml +119 -0
  5. package/reader-backports/hdf5-reader/LICENSE-APACHE +164 -0
  6. package/reader-backports/hdf5-reader/LICENSE-MIT +21 -0
  7. package/reader-backports/hdf5-reader/PROVENANCE.json +80 -0
  8. package/reader-backports/hdf5-reader/README.md +308 -0
  9. package/reader-backports/hdf5-reader/UPSTREAM-Cargo.toml +44 -0
  10. package/reader-backports/hdf5-reader/UPSTREAM-VCS.json +6 -0
  11. package/reader-backports/hdf5-reader/src/attribute_api.rs +847 -0
  12. package/reader-backports/hdf5-reader/src/btree_v1.rs +726 -0
  13. package/reader-backports/hdf5-reader/src/btree_v2.rs +2068 -0
  14. package/reader-backports/hdf5-reader/src/cache.rs +354 -0
  15. package/reader-backports/hdf5-reader/src/checksum.rs +56 -0
  16. package/reader-backports/hdf5-reader/src/chunk_index.rs +423 -0
  17. package/reader-backports/hdf5-reader/src/dataset.rs +4733 -0
  18. package/reader-backports/hdf5-reader/src/datatype_api.rs +497 -0
  19. package/reader-backports/hdf5-reader/src/error.rs +167 -0
  20. package/reader-backports/hdf5-reader/src/extensible_array.rs +1886 -0
  21. package/reader-backports/hdf5-reader/src/filters/deflate.rs +93 -0
  22. package/reader-backports/hdf5-reader/src/filters/fletcher32.rs +52 -0
  23. package/reader-backports/hdf5-reader/src/filters/lz4.rs +178 -0
  24. package/reader-backports/hdf5-reader/src/filters/mod.rs +326 -0
  25. package/reader-backports/hdf5-reader/src/filters/nbit.rs +586 -0
  26. package/reader-backports/hdf5-reader/src/filters/scaleoffset.rs +827 -0
  27. package/reader-backports/hdf5-reader/src/filters/shuffle.rs +99 -0
  28. package/reader-backports/hdf5-reader/src/fixed_array.rs +763 -0
  29. package/reader-backports/hdf5-reader/src/fractal_heap.rs +2029 -0
  30. package/reader-backports/hdf5-reader/src/global_heap.rs +284 -0
  31. package/reader-backports/hdf5-reader/src/group.rs +795 -0
  32. package/reader-backports/hdf5-reader/src/io.rs +413 -0
  33. package/reader-backports/hdf5-reader/src/lib.rs +916 -0
  34. package/reader-backports/hdf5-reader/src/local_heap.rs +261 -0
  35. package/reader-backports/hdf5-reader/src/messages/attribute.rs +304 -0
  36. package/reader-backports/hdf5-reader/src/messages/attribute_info.rs +96 -0
  37. package/reader-backports/hdf5-reader/src/messages/btree_k.rs +36 -0
  38. package/reader-backports/hdf5-reader/src/messages/continuation.rs +46 -0
  39. package/reader-backports/hdf5-reader/src/messages/dataspace.rs +259 -0
  40. package/reader-backports/hdf5-reader/src/messages/datatype.rs +938 -0
  41. package/reader-backports/hdf5-reader/src/messages/external_files.rs +50 -0
  42. package/reader-backports/hdf5-reader/src/messages/fill_value.rs +233 -0
  43. package/reader-backports/hdf5-reader/src/messages/filter_pipeline.rs +147 -0
  44. package/reader-backports/hdf5-reader/src/messages/group_info.rs +103 -0
  45. package/reader-backports/hdf5-reader/src/messages/layout.rs +476 -0
  46. package/reader-backports/hdf5-reader/src/messages/link.rs +170 -0
  47. package/reader-backports/hdf5-reader/src/messages/link_info.rs +127 -0
  48. package/reader-backports/hdf5-reader/src/messages/mod.rs +374 -0
  49. package/reader-backports/hdf5-reader/src/messages/modification_time.rs +83 -0
  50. package/reader-backports/hdf5-reader/src/messages/shared.rs +164 -0
  51. package/reader-backports/hdf5-reader/src/messages/shared_table.rs +67 -0
  52. package/reader-backports/hdf5-reader/src/messages/symbol_table_msg.rs +51 -0
  53. package/reader-backports/hdf5-reader/src/object_header.rs +1749 -0
  54. package/reader-backports/hdf5-reader/src/reference.rs +90 -0
  55. package/reader-backports/hdf5-reader/src/shared_message_table.rs +425 -0
  56. package/reader-backports/hdf5-reader/src/storage.rs +538 -0
  57. package/reader-backports/hdf5-reader/src/superblock.rs +285 -0
  58. package/reader-backports/hdf5-reader/src/symbol_table.rs +277 -0
  59. package/reader-backports/netcdf-reader/Cargo.toml +141 -0
  60. package/reader-backports/netcdf-reader/LICENSE-APACHE +164 -0
  61. package/reader-backports/netcdf-reader/LICENSE-MIT +21 -0
  62. package/reader-backports/netcdf-reader/PROVENANCE.json +55 -0
  63. package/reader-backports/netcdf-reader/README.md +308 -0
  64. package/reader-backports/netcdf-reader/UPSTREAM-Cargo.toml +48 -0
  65. package/reader-backports/netcdf-reader/UPSTREAM-VCS.json +6 -0
  66. package/reader-backports/netcdf-reader/src/cf/axes.rs +419 -0
  67. package/reader-backports/netcdf-reader/src/cf/bounds.rs +108 -0
  68. package/reader-backports/netcdf-reader/src/cf/crs.rs +269 -0
  69. package/reader-backports/netcdf-reader/src/cf/mod.rs +30 -0
  70. package/reader-backports/netcdf-reader/src/cf/time.rs +1122 -0
  71. package/reader-backports/netcdf-reader/src/classic/data.rs +1465 -0
  72. package/reader-backports/netcdf-reader/src/classic/header.rs +1097 -0
  73. package/reader-backports/netcdf-reader/src/classic/mod.rs +240 -0
  74. package/reader-backports/netcdf-reader/src/classic/storage.rs +211 -0
  75. package/reader-backports/netcdf-reader/src/classic/types.rs +93 -0
  76. package/reader-backports/netcdf-reader/src/classic/variable.rs +1162 -0
  77. package/reader-backports/netcdf-reader/src/error.rs +59 -0
  78. package/reader-backports/netcdf-reader/src/lib.rs +1531 -0
  79. package/reader-backports/netcdf-reader/src/masked.rs +173 -0
  80. package/reader-backports/netcdf-reader/src/nc4/attributes.rs +203 -0
  81. package/reader-backports/netcdf-reader/src/nc4/dimensions.rs +139 -0
  82. package/reader-backports/netcdf-reader/src/nc4/groups.rs +308 -0
  83. package/reader-backports/netcdf-reader/src/nc4/mod.rs +572 -0
  84. package/reader-backports/netcdf-reader/src/nc4/types.rs +309 -0
  85. package/reader-backports/netcdf-reader/src/nc4/variables.rs +420 -0
  86. package/reader-backports/netcdf-reader/src/types.rs +18 -0
  87. package/reader-backports/netcdf-reader/src/unpack.rs +113 -0
  88. package/reader-backports/netcdf-reader/src/user_defined.rs +569 -0
@@ -0,0 +1,1465 @@
1
+ //! Data reading for classic (CDF-1/2/5) NetCDF files.
2
+ //!
3
+ //! Two layout types:
4
+ //! - **Non-record variables**: contiguous data at the offset stored in the variable header.
5
+ //! - **Record variables**: data is interleaved across records. Each record contains one
6
+ //! slice from every record variable, in the order they appear in the header. The total
7
+ //! record size is the sum of all record variables' vsize values (each padded to 4-byte
8
+ //! boundary in CDF-1/2).
9
+
10
+ use ndarray::{ArrayD, IxDyn};
11
+ #[cfg(feature = "rayon")]
12
+ use rayon::prelude::*;
13
+
14
+ use crate::error::{Error, Result};
15
+ use crate::types::{NcType, NcVariable};
16
+
17
+ use super::storage::ClassicStorage;
18
+
19
+ /// Trait for types that can be read from classic NetCDF data.
20
+ pub trait NcReadType: Clone + Default + Send + 'static {
21
+ /// The NetCDF type this Rust type corresponds to.
22
+ fn nc_type() -> NcType;
23
+
24
+ /// Read a single element from big-endian bytes.
25
+ fn from_be_bytes(bytes: &[u8]) -> Result<Self>;
26
+
27
+ /// Size in bytes of one element.
28
+ fn element_size() -> usize;
29
+
30
+ /// Bulk decode `count` elements from a contiguous big-endian byte slice.
31
+ ///
32
+ /// Default implementation falls back to per-element decoding. Types with
33
+ /// multi-byte elements override this with an optimized bulk path using
34
+ /// `chunks_exact` + byte-swap (on LE hosts) or `copy_nonoverlapping`
35
+ /// (on BE hosts).
36
+ fn decode_bulk_be(raw: &[u8], count: usize) -> Result<Vec<Self>> {
37
+ let elem_size = Self::element_size();
38
+ let needed = count.checked_mul(elem_size).ok_or_else(|| {
39
+ Error::InvalidData("classic decode byte count exceeds platform usize".to_string())
40
+ })?;
41
+ if raw.len() < needed {
42
+ return Err(Error::InvalidData(format!(
43
+ "need {} bytes for {} elements, got {}",
44
+ needed,
45
+ count,
46
+ raw.len()
47
+ )));
48
+ }
49
+ let mut values = Vec::with_capacity(count);
50
+ for i in 0..count {
51
+ let start = i * elem_size;
52
+ values.push(Self::from_be_bytes(&raw[start..start + elem_size])?);
53
+ }
54
+ Ok(values)
55
+ }
56
+
57
+ /// Bulk decode elements from a contiguous big-endian byte slice into a
58
+ /// caller-provided destination buffer.
59
+ fn decode_bulk_be_into(raw: &[u8], dst: &mut [Self]) -> Result<()> {
60
+ let elem_size = Self::element_size();
61
+ let needed = dst.len().checked_mul(elem_size).ok_or_else(|| {
62
+ Error::InvalidData("classic decode byte count exceeds platform usize".to_string())
63
+ })?;
64
+ if raw.len() < needed {
65
+ return Err(Error::InvalidData(format!(
66
+ "need {} bytes for {} elements, got {}",
67
+ needed,
68
+ dst.len(),
69
+ raw.len()
70
+ )));
71
+ }
72
+ for (out, chunk) in dst.iter_mut().zip(raw[..needed].chunks_exact(elem_size)) {
73
+ *out = Self::from_be_bytes(chunk)?;
74
+ }
75
+ Ok(())
76
+ }
77
+ }
78
+
79
+ macro_rules! impl_nc_read_type {
80
+ ($ty:ty, $nc_type:path, $size:literal) => {
81
+ impl NcReadType for $ty {
82
+ fn nc_type() -> NcType {
83
+ $nc_type
84
+ }
85
+
86
+ fn from_be_bytes(bytes: &[u8]) -> Result<Self> {
87
+ if bytes.len() < $size {
88
+ return Err(Error::InvalidData(format!(
89
+ "need {} bytes for {}, got {}",
90
+ $size,
91
+ stringify!($ty),
92
+ bytes.len()
93
+ )));
94
+ }
95
+ let mut arr = [0u8; $size];
96
+ arr.copy_from_slice(&bytes[..$size]);
97
+ Ok(<$ty>::from_be_bytes(arr))
98
+ }
99
+
100
+ fn element_size() -> usize {
101
+ $size
102
+ }
103
+
104
+ fn decode_bulk_be(raw: &[u8], count: usize) -> Result<Vec<Self>> {
105
+ let total_bytes = count.checked_mul($size).ok_or_else(|| {
106
+ Error::InvalidData(
107
+ "classic decode byte count exceeds platform usize".to_string(),
108
+ )
109
+ })?;
110
+ if raw.len() < total_bytes {
111
+ return Err(Error::InvalidData(format!(
112
+ "need {} bytes for {} elements of {}, got {}",
113
+ total_bytes,
114
+ count,
115
+ stringify!($ty),
116
+ raw.len()
117
+ )));
118
+ }
119
+ let bytes = &raw[..total_bytes];
120
+ #[cfg(target_endian = "big")]
121
+ {
122
+ let mut values = Vec::<$ty>::with_capacity(count);
123
+ // SAFETY: `values` has capacity for `count` elements,
124
+ // `raw` contains exactly `count * size_of::<$ty>()` bytes,
125
+ // and numeric types admit every bit pattern.
126
+ unsafe {
127
+ std::ptr::copy_nonoverlapping(
128
+ bytes.as_ptr(),
129
+ values.as_mut_ptr() as *mut u8,
130
+ total_bytes,
131
+ );
132
+ values.set_len(count);
133
+ }
134
+ Ok(values)
135
+ }
136
+ #[cfg(target_endian = "little")]
137
+ {
138
+ // LE host reading BE data: chunks_exact + byte-swap.
139
+ Ok(bytes
140
+ .chunks_exact($size)
141
+ .map(|chunk| {
142
+ let mut arr = [0u8; $size];
143
+ arr.copy_from_slice(chunk);
144
+ <$ty>::from_be_bytes(arr)
145
+ })
146
+ .collect())
147
+ }
148
+ }
149
+
150
+ fn decode_bulk_be_into(raw: &[u8], dst: &mut [Self]) -> Result<()> {
151
+ let total_bytes = dst.len().checked_mul($size).ok_or_else(|| {
152
+ Error::InvalidData(
153
+ "classic decode byte count exceeds platform usize".to_string(),
154
+ )
155
+ })?;
156
+ if raw.len() < total_bytes {
157
+ return Err(Error::InvalidData(format!(
158
+ "need {} bytes for {} elements of {}, got {}",
159
+ total_bytes,
160
+ dst.len(),
161
+ stringify!($ty),
162
+ raw.len()
163
+ )));
164
+ }
165
+ let bytes = &raw[..total_bytes];
166
+ #[cfg(target_endian = "big")]
167
+ {
168
+ // SAFETY: `values` has capacity for `count` bytes and
169
+ // `bytes` contains exactly `count` initialized bytes.
170
+ unsafe {
171
+ std::ptr::copy_nonoverlapping(
172
+ bytes.as_ptr(),
173
+ dst.as_mut_ptr() as *mut u8,
174
+ total_bytes,
175
+ );
176
+ }
177
+ Ok(())
178
+ }
179
+ #[cfg(target_endian = "little")]
180
+ {
181
+ for (out, chunk) in dst.iter_mut().zip(bytes.chunks_exact($size)) {
182
+ let mut arr = [0u8; $size];
183
+ arr.copy_from_slice(chunk);
184
+ *out = <$ty>::from_be_bytes(arr);
185
+ }
186
+ Ok(())
187
+ }
188
+ }
189
+ }
190
+ };
191
+ }
192
+
193
+ impl_nc_read_type!(i8, NcType::Byte, 1);
194
+ impl_nc_read_type!(i16, NcType::Short, 2);
195
+ impl_nc_read_type!(i32, NcType::Int, 4);
196
+ impl_nc_read_type!(f32, NcType::Float, 4);
197
+ impl_nc_read_type!(f64, NcType::Double, 8);
198
+ impl_nc_read_type!(u8, NcType::UByte, 1);
199
+ impl_nc_read_type!(u16, NcType::UShort, 2);
200
+ impl_nc_read_type!(u32, NcType::UInt, 4);
201
+ impl_nc_read_type!(i64, NcType::Int64, 8);
202
+ impl_nc_read_type!(u64, NcType::UInt64, 8);
203
+
204
+ /// Read the entire data for a non-record variable into an ndarray.
205
+ ///
206
+ /// The data is located at a contiguous region starting at `var.data_offset`
207
+ /// with total size `var.data_size`.
208
+ pub fn read_non_record_variable<T: NcReadType>(
209
+ file_data: &[u8],
210
+ var: &NcVariable,
211
+ ) -> Result<ArrayD<T>> {
212
+ if var.is_record_var {
213
+ return Err(Error::InvalidData(
214
+ "use read_record_variable for record variables".to_string(),
215
+ ));
216
+ }
217
+
218
+ let offset = crate::types::checked_usize_from_u64(var.data_offset, "variable data offset")?;
219
+ let total_elements = checked_non_record_element_count(var)?;
220
+ let elem_size = T::element_size();
221
+ let total_bytes = total_elements.checked_mul(elem_size).ok_or_else(|| {
222
+ Error::InvalidData(format!(
223
+ "variable '{}' size in bytes exceeds platform usize",
224
+ var.name
225
+ ))
226
+ })?;
227
+
228
+ let end = offset.checked_add(total_bytes).ok_or_else(|| {
229
+ Error::InvalidData(format!(
230
+ "variable '{}' byte range exceeds platform usize",
231
+ var.name
232
+ ))
233
+ })?;
234
+ if end > file_data.len() {
235
+ return Err(Error::InvalidData(format!(
236
+ "variable '{}' data extends beyond file: offset={}, size={}, file_len={}",
237
+ var.name,
238
+ offset,
239
+ total_bytes,
240
+ file_data.len()
241
+ )));
242
+ }
243
+
244
+ let data_slice = &file_data[offset..end];
245
+ let values = T::decode_bulk_be(data_slice, total_elements)?;
246
+
247
+ let shape: Vec<usize> = var
248
+ .shape()
249
+ .iter()
250
+ .map(|&s| crate::types::checked_usize_from_u64(s, "variable dimension"))
251
+ .collect::<Result<Vec<_>>>()?;
252
+ if shape.is_empty() {
253
+ // Scalar variable.
254
+ ArrayD::from_shape_vec(IxDyn(&[]), values)
255
+ } else {
256
+ ArrayD::from_shape_vec(IxDyn(&shape), values)
257
+ }
258
+ .map_err(|e| Error::InvalidData(format!("failed to create array: {}", e)))
259
+ }
260
+
261
+ /// Read the entire data for a non-record variable from range-backed storage.
262
+ pub(crate) fn read_non_record_variable_from_storage<T: NcReadType>(
263
+ storage: &ClassicStorage,
264
+ var: &NcVariable,
265
+ ) -> Result<ArrayD<T>> {
266
+ if var.is_record_var {
267
+ return Err(Error::InvalidData(
268
+ "use read_record_variable_from_storage for record variables".to_string(),
269
+ ));
270
+ }
271
+
272
+ let total_elements = checked_non_record_element_count(var)?;
273
+ let total_bytes = variable_data_bytes::<T>(var.name.as_str(), total_elements)?;
274
+ let data = storage.read_range(var.data_offset, total_bytes)?;
275
+ let values = T::decode_bulk_be(data.as_ref(), total_elements)?;
276
+ let shape = checked_variable_shape(var)?;
277
+
278
+ ArrayD::from_shape_vec(IxDyn(&shape), values)
279
+ .map_err(|e| Error::InvalidData(format!("failed to create array: {}", e)))
280
+ }
281
+
282
+ /// Read the entire data for a non-record variable using Rayon for large reads.
283
+ #[cfg(feature = "rayon")]
284
+ pub(crate) fn read_non_record_variable_parallel_from_storage<T: NcReadType>(
285
+ storage: &ClassicStorage,
286
+ var: &NcVariable,
287
+ ) -> Result<ArrayD<T>> {
288
+ if var.is_record_var {
289
+ return Err(Error::InvalidData(
290
+ "use read_record_variable_parallel_from_storage for record variables".to_string(),
291
+ ));
292
+ }
293
+
294
+ let total_elements = checked_non_record_element_count(var)?;
295
+ let total_bytes = variable_data_bytes::<T>(var.name.as_str(), total_elements)?;
296
+ let policy = storage.parallel_read_policy();
297
+ if total_bytes < policy.min_bytes || total_elements == 0 {
298
+ return read_non_record_variable_from_storage(storage, var);
299
+ }
300
+
301
+ // The parallel path allocates the whole result up front, so validate the
302
+ // declared byte range fits in storage before dispatching; the serial path
303
+ // already bounds-checks per read.
304
+ let end = var
305
+ .data_offset
306
+ .checked_add(total_bytes as u64)
307
+ .ok_or_else(|| {
308
+ Error::InvalidData(format!("variable '{}' byte range exceeds u64", var.name))
309
+ })?;
310
+ if end > storage.len() {
311
+ return Err(Error::InvalidData(format!(
312
+ "variable '{}' data extends beyond file: needs {end} bytes, have {}",
313
+ var.name,
314
+ storage.len()
315
+ )));
316
+ }
317
+
318
+ let values = read_contiguous_range_parallel::<T>(
319
+ storage,
320
+ var.data_offset,
321
+ total_elements,
322
+ policy.target_chunk_bytes,
323
+ )?;
324
+ let shape = checked_variable_shape(var)?;
325
+
326
+ ArrayD::from_shape_vec(IxDyn(&shape), values)
327
+ .map_err(|e| Error::InvalidData(format!("failed to create array: {}", e)))
328
+ }
329
+
330
+ /// Read the entire data for a non-record variable into a caller-provided buffer.
331
+ pub fn read_non_record_variable_into<T: NcReadType>(
332
+ file_data: &[u8],
333
+ var: &NcVariable,
334
+ dst: &mut [T],
335
+ ) -> Result<()> {
336
+ if var.is_record_var {
337
+ return Err(Error::InvalidData(
338
+ "use read_record_variable_into for record variables".to_string(),
339
+ ));
340
+ }
341
+
342
+ let total_elements = checked_non_record_element_count(var)?;
343
+ if dst.len() != total_elements {
344
+ return Err(Error::InvalidData(format!(
345
+ "destination has {} elements, variable '{}' requires {}",
346
+ dst.len(),
347
+ var.name,
348
+ total_elements
349
+ )));
350
+ }
351
+
352
+ let offset = crate::types::checked_usize_from_u64(var.data_offset, "variable data offset")?;
353
+ let elem_size = T::element_size();
354
+ let total_bytes = total_elements.checked_mul(elem_size).ok_or_else(|| {
355
+ Error::InvalidData(format!(
356
+ "variable '{}' size in bytes exceeds platform usize",
357
+ var.name
358
+ ))
359
+ })?;
360
+
361
+ let end = offset.checked_add(total_bytes).ok_or_else(|| {
362
+ Error::InvalidData(format!(
363
+ "variable '{}' byte range exceeds platform usize",
364
+ var.name
365
+ ))
366
+ })?;
367
+ if end > file_data.len() {
368
+ return Err(Error::InvalidData(format!(
369
+ "variable '{}' data extends beyond file: offset={}, size={}, file_len={}",
370
+ var.name,
371
+ offset,
372
+ total_bytes,
373
+ file_data.len()
374
+ )));
375
+ }
376
+
377
+ T::decode_bulk_be_into(&file_data[offset..end], dst)
378
+ }
379
+
380
+ /// Read a non-record variable from range-backed storage into a caller-provided buffer.
381
+ pub(crate) fn read_non_record_variable_into_from_storage<T: NcReadType>(
382
+ storage: &ClassicStorage,
383
+ var: &NcVariable,
384
+ dst: &mut [T],
385
+ ) -> Result<()> {
386
+ if var.is_record_var {
387
+ return Err(Error::InvalidData(
388
+ "use read_record_variable_into_from_storage for record variables".to_string(),
389
+ ));
390
+ }
391
+
392
+ let total_elements = checked_non_record_element_count(var)?;
393
+ if dst.len() != total_elements {
394
+ return Err(Error::InvalidData(format!(
395
+ "destination has {} elements, variable '{}' requires {}",
396
+ dst.len(),
397
+ var.name,
398
+ total_elements
399
+ )));
400
+ }
401
+
402
+ let total_bytes = variable_data_bytes::<T>(var.name.as_str(), total_elements)?;
403
+ let data = storage.read_range(var.data_offset, total_bytes)?;
404
+ T::decode_bulk_be_into(data.as_ref(), dst)
405
+ }
406
+
407
+ /// Read the entire data for a record variable into an ndarray.
408
+ ///
409
+ /// Record variables are interleaved: for each of `numrecs` records, every record
410
+ /// variable contributes `record_size` bytes (padded to 4-byte alignment for CDF-1/2).
411
+ /// The `record_stride` is the total size of one record across all record variables.
412
+ ///
413
+ /// Parameters:
414
+ /// - `file_data`: the raw file bytes
415
+ /// - `var`: the record variable to read
416
+ /// - `numrecs`: number of records (from the file header)
417
+ /// - `record_stride`: total bytes per record (sum of all record variables' padded vsizes)
418
+ pub fn read_record_variable<T: NcReadType>(
419
+ file_data: &[u8],
420
+ var: &NcVariable,
421
+ numrecs: u64,
422
+ record_stride: u64,
423
+ ) -> Result<ArrayD<T>> {
424
+ if !var.is_record_var {
425
+ return Err(Error::InvalidData(
426
+ "use read_non_record_variable for non-record variables".to_string(),
427
+ ));
428
+ }
429
+
430
+ let elem_size = T::element_size();
431
+ let base_offset =
432
+ crate::types::checked_usize_from_u64(var.data_offset, "record variable data offset")?;
433
+ let record_stride_usize = crate::types::checked_usize_from_u64(record_stride, "record stride")?;
434
+
435
+ // Shape: the first dimension is the unlimited dimension, replaced by numrecs.
436
+ let shape = checked_record_shape(var, numrecs)?;
437
+ let numrecs_usize = shape[0];
438
+
439
+ // Number of elements per record (product of all dims except the first).
440
+ let elements_per_record = checked_record_elements_per_record(var)?;
441
+ let bytes_per_record = elements_per_record.checked_mul(elem_size).ok_or_else(|| {
442
+ Error::InvalidData(format!(
443
+ "record variable '{}' bytes per record exceed platform usize",
444
+ var.name
445
+ ))
446
+ })?;
447
+ let total_elements = numrecs_usize
448
+ .checked_mul(elements_per_record)
449
+ .ok_or_else(|| {
450
+ Error::InvalidData(format!(
451
+ "record variable '{}' element count exceeds platform usize",
452
+ var.name
453
+ ))
454
+ })?;
455
+
456
+ // Validate that every record fits in the file before allocating, so a
457
+ // crafted `numrecs` cannot force a huge up-front allocation.
458
+ validate_record_span(
459
+ var,
460
+ numrecs,
461
+ record_stride,
462
+ bytes_per_record as u64,
463
+ file_data.len() as u64,
464
+ )?;
465
+
466
+ let mut values = Vec::with_capacity(total_elements);
467
+
468
+ for rec in 0..numrecs_usize {
469
+ let rec_offset = base_offset
470
+ .checked_add(rec.checked_mul(record_stride_usize).ok_or_else(|| {
471
+ Error::InvalidData(format!(
472
+ "record variable '{}' byte offset exceeds platform usize",
473
+ var.name
474
+ ))
475
+ })?)
476
+ .ok_or_else(|| {
477
+ Error::InvalidData(format!(
478
+ "record variable '{}' byte offset exceeds platform usize",
479
+ var.name
480
+ ))
481
+ })?;
482
+ let rec_end = rec_offset.checked_add(bytes_per_record).ok_or_else(|| {
483
+ Error::InvalidData(format!(
484
+ "record variable '{}' record range exceeds platform usize",
485
+ var.name
486
+ ))
487
+ })?;
488
+ if rec_end > file_data.len() {
489
+ return Err(Error::InvalidData(format!(
490
+ "record {} for variable '{}' extends beyond file",
491
+ rec, var.name
492
+ )));
493
+ }
494
+ let rec_slice = &file_data[rec_offset..rec_end];
495
+ let rec_values = T::decode_bulk_be(rec_slice, elements_per_record)?;
496
+ values.extend(rec_values);
497
+ }
498
+
499
+ ArrayD::from_shape_vec(IxDyn(&shape), values)
500
+ .map_err(|e| Error::InvalidData(format!("failed to create array: {}", e)))
501
+ }
502
+
503
+ /// Read the entire data for a record variable from range-backed storage.
504
+ pub(crate) fn read_record_variable_from_storage<T: NcReadType>(
505
+ storage: &ClassicStorage,
506
+ var: &NcVariable,
507
+ numrecs: u64,
508
+ record_stride: u64,
509
+ ) -> Result<ArrayD<T>> {
510
+ if !var.is_record_var {
511
+ return Err(Error::InvalidData(
512
+ "use read_non_record_variable_from_storage for non-record variables".to_string(),
513
+ ));
514
+ }
515
+
516
+ let shape = checked_record_shape(var, numrecs)?;
517
+ let elements_per_record = checked_record_elements_per_record(var)?;
518
+ let bytes_per_record = variable_data_bytes::<T>(var.name.as_str(), elements_per_record)?;
519
+ let numrecs_usize = crate::types::checked_usize_from_u64(numrecs, "record count")?;
520
+ let total_elements = numrecs_usize
521
+ .checked_mul(elements_per_record)
522
+ .ok_or_else(|| {
523
+ Error::InvalidData(format!(
524
+ "record variable '{}' element count exceeds platform usize",
525
+ var.name
526
+ ))
527
+ })?;
528
+ validate_record_span(
529
+ var,
530
+ numrecs,
531
+ record_stride,
532
+ bytes_per_record as u64,
533
+ storage.len(),
534
+ )?;
535
+ let mut values = Vec::with_capacity(total_elements);
536
+
537
+ for rec in 0..numrecs {
538
+ let rec_offset = record_byte_offset(var, rec, record_stride)?;
539
+ let rec_slice = storage.read_range(rec_offset, bytes_per_record)?;
540
+ let rec_values = T::decode_bulk_be(rec_slice.as_ref(), elements_per_record)?;
541
+ values.extend(rec_values);
542
+ }
543
+
544
+ ArrayD::from_shape_vec(IxDyn(&shape), values)
545
+ .map_err(|e| Error::InvalidData(format!("failed to create array: {}", e)))
546
+ }
547
+
548
+ /// Read the entire data for a record variable using Rayon for large reads.
549
+ #[cfg(feature = "rayon")]
550
+ pub(crate) fn read_record_variable_parallel_from_storage<T: NcReadType>(
551
+ storage: &ClassicStorage,
552
+ var: &NcVariable,
553
+ numrecs: u64,
554
+ record_stride: u64,
555
+ ) -> Result<ArrayD<T>> {
556
+ if !var.is_record_var {
557
+ return Err(Error::InvalidData(
558
+ "use read_non_record_variable_parallel_from_storage for non-record variables"
559
+ .to_string(),
560
+ ));
561
+ }
562
+
563
+ let shape = checked_record_shape(var, numrecs)?;
564
+ let elements_per_record = checked_record_elements_per_record(var)?;
565
+ let bytes_per_record = variable_data_bytes::<T>(var.name.as_str(), elements_per_record)?;
566
+ let numrecs_usize = crate::types::checked_usize_from_u64(numrecs, "record count")?;
567
+ let total_elements = numrecs_usize
568
+ .checked_mul(elements_per_record)
569
+ .ok_or_else(|| {
570
+ Error::InvalidData(format!(
571
+ "record variable '{}' element count exceeds platform usize",
572
+ var.name
573
+ ))
574
+ })?;
575
+ let logical_bytes = numrecs_usize.checked_mul(bytes_per_record).ok_or_else(|| {
576
+ Error::InvalidData(format!(
577
+ "record variable '{}' logical byte count exceeds platform usize",
578
+ var.name
579
+ ))
580
+ })?;
581
+ validate_record_span(
582
+ var,
583
+ numrecs,
584
+ record_stride,
585
+ bytes_per_record as u64,
586
+ storage.len(),
587
+ )?;
588
+ let policy = storage.parallel_read_policy();
589
+ if logical_bytes < policy.min_bytes || numrecs_usize <= 1 {
590
+ return read_record_variable_from_storage(storage, var, numrecs, record_stride);
591
+ }
592
+
593
+ let records_per_chunk = (policy.target_chunk_bytes / bytes_per_record.max(1)).max(1);
594
+ let elements_per_chunk = records_per_chunk
595
+ .checked_mul(elements_per_record)
596
+ .ok_or_else(|| {
597
+ Error::InvalidData(
598
+ "classic record chunk element count exceeds platform usize".to_string(),
599
+ )
600
+ })?;
601
+ let mut values = vec![T::default(); total_elements];
602
+ let chunk_plan = RecordChunkReadPlan {
603
+ var,
604
+ record_stride,
605
+ elements_per_record,
606
+ bytes_per_record,
607
+ };
608
+ values
609
+ .par_chunks_mut(elements_per_chunk)
610
+ .enumerate()
611
+ .try_for_each(|(chunk, dst)| {
612
+ let first_record = chunk.checked_mul(records_per_chunk).ok_or_else(|| {
613
+ Error::InvalidData("classic record chunk offset exceeds platform usize".to_string())
614
+ })?;
615
+ let records = dst.len().checked_div(elements_per_record).ok_or_else(|| {
616
+ Error::InvalidData("classic record elements per record is zero".to_string())
617
+ })?;
618
+ read_record_chunk_into::<T>(storage, &chunk_plan, first_record as u64, records, dst)
619
+ })?;
620
+
621
+ ArrayD::from_shape_vec(IxDyn(&shape), values)
622
+ .map_err(|e| Error::InvalidData(format!("failed to create array: {}", e)))
623
+ }
624
+
625
+ /// Read the entire data for a record variable into a caller-provided buffer.
626
+ pub fn read_record_variable_into<T: NcReadType>(
627
+ file_data: &[u8],
628
+ var: &NcVariable,
629
+ numrecs: u64,
630
+ record_stride: u64,
631
+ dst: &mut [T],
632
+ ) -> Result<()> {
633
+ if !var.is_record_var {
634
+ return Err(Error::InvalidData(
635
+ "use read_non_record_variable_into for non-record variables".to_string(),
636
+ ));
637
+ }
638
+
639
+ let elem_size = T::element_size();
640
+ let base_offset =
641
+ crate::types::checked_usize_from_u64(var.data_offset, "record variable data offset")?;
642
+ let numrecs_usize = crate::types::checked_usize_from_u64(numrecs, "record count")?;
643
+ let record_stride_usize = crate::types::checked_usize_from_u64(record_stride, "record stride")?;
644
+
645
+ if var.dimensions.is_empty() {
646
+ return Err(Error::InvalidData(
647
+ "record variable must have at least one dimension".to_string(),
648
+ ));
649
+ }
650
+
651
+ let elements_per_record = checked_record_elements_per_record(var)?;
652
+ let bytes_per_record = elements_per_record.checked_mul(elem_size).ok_or_else(|| {
653
+ Error::InvalidData(format!(
654
+ "record variable '{}' bytes per record exceed platform usize",
655
+ var.name
656
+ ))
657
+ })?;
658
+ let total_elements = numrecs_usize
659
+ .checked_mul(elements_per_record)
660
+ .ok_or_else(|| {
661
+ Error::InvalidData(format!(
662
+ "record variable '{}' element count exceeds platform usize",
663
+ var.name
664
+ ))
665
+ })?;
666
+ if dst.len() != total_elements {
667
+ return Err(Error::InvalidData(format!(
668
+ "destination has {} elements, variable '{}' requires {}",
669
+ dst.len(),
670
+ var.name,
671
+ total_elements
672
+ )));
673
+ }
674
+
675
+ for rec in 0..numrecs_usize {
676
+ let rec_offset = base_offset
677
+ .checked_add(rec.checked_mul(record_stride_usize).ok_or_else(|| {
678
+ Error::InvalidData(format!(
679
+ "record variable '{}' byte offset exceeds platform usize",
680
+ var.name
681
+ ))
682
+ })?)
683
+ .ok_or_else(|| {
684
+ Error::InvalidData(format!(
685
+ "record variable '{}' byte offset exceeds platform usize",
686
+ var.name
687
+ ))
688
+ })?;
689
+ let rec_end = rec_offset.checked_add(bytes_per_record).ok_or_else(|| {
690
+ Error::InvalidData(format!(
691
+ "record variable '{}' record range exceeds platform usize",
692
+ var.name
693
+ ))
694
+ })?;
695
+ if rec_end > file_data.len() {
696
+ return Err(Error::InvalidData(format!(
697
+ "record {} for variable '{}' extends beyond file",
698
+ rec, var.name
699
+ )));
700
+ }
701
+
702
+ let dst_start = rec.checked_mul(elements_per_record).ok_or_else(|| {
703
+ Error::InvalidData(format!(
704
+ "record variable '{}' destination offset exceeds platform usize",
705
+ var.name
706
+ ))
707
+ })?;
708
+ let dst_end = dst_start.checked_add(elements_per_record).ok_or_else(|| {
709
+ Error::InvalidData(format!(
710
+ "record variable '{}' destination range exceeds platform usize",
711
+ var.name
712
+ ))
713
+ })?;
714
+ T::decode_bulk_be_into(
715
+ &file_data[rec_offset..rec_end],
716
+ &mut dst[dst_start..dst_end],
717
+ )?;
718
+ }
719
+
720
+ Ok(())
721
+ }
722
+
723
+ /// Read a record variable from range-backed storage into a caller-provided buffer.
724
+ pub(crate) fn read_record_variable_into_from_storage<T: NcReadType>(
725
+ storage: &ClassicStorage,
726
+ var: &NcVariable,
727
+ numrecs: u64,
728
+ record_stride: u64,
729
+ dst: &mut [T],
730
+ ) -> Result<()> {
731
+ if !var.is_record_var {
732
+ return Err(Error::InvalidData(
733
+ "use read_non_record_variable_into_from_storage for non-record variables".to_string(),
734
+ ));
735
+ }
736
+
737
+ if var.dimensions.is_empty() {
738
+ return Err(Error::InvalidData(
739
+ "record variable must have at least one dimension".to_string(),
740
+ ));
741
+ }
742
+
743
+ let elements_per_record = checked_record_elements_per_record(var)?;
744
+ let bytes_per_record = variable_data_bytes::<T>(var.name.as_str(), elements_per_record)?;
745
+ let numrecs_usize = crate::types::checked_usize_from_u64(numrecs, "record count")?;
746
+ let total_elements = numrecs_usize
747
+ .checked_mul(elements_per_record)
748
+ .ok_or_else(|| {
749
+ Error::InvalidData(format!(
750
+ "record variable '{}' element count exceeds platform usize",
751
+ var.name
752
+ ))
753
+ })?;
754
+ if dst.len() != total_elements {
755
+ return Err(Error::InvalidData(format!(
756
+ "destination has {} elements, variable '{}' requires {}",
757
+ dst.len(),
758
+ var.name,
759
+ total_elements
760
+ )));
761
+ }
762
+
763
+ for rec in 0..numrecs {
764
+ let rec_offset = record_byte_offset(var, rec, record_stride)?;
765
+ let rec_slice = storage.read_range(rec_offset, bytes_per_record)?;
766
+ let dst_start = crate::types::checked_usize_from_u64(rec, "record index")?
767
+ .checked_mul(elements_per_record)
768
+ .ok_or_else(|| {
769
+ Error::InvalidData(format!(
770
+ "record variable '{}' destination offset exceeds platform usize",
771
+ var.name
772
+ ))
773
+ })?;
774
+ let dst_end = dst_start.checked_add(elements_per_record).ok_or_else(|| {
775
+ Error::InvalidData(format!(
776
+ "record variable '{}' destination range exceeds platform usize",
777
+ var.name
778
+ ))
779
+ })?;
780
+ T::decode_bulk_be_into(rec_slice.as_ref(), &mut dst[dst_start..dst_end])?;
781
+ }
782
+
783
+ Ok(())
784
+ }
785
+
786
+ /// Compute the record stride: total bytes per record across all record variables.
787
+ ///
788
+ /// Each record variable's per-record contribution is its unpadded
789
+ /// `record_size` (derived from its dimensions), padded to a 4-byte boundary —
790
+ /// except in the spec's special case of exactly one record variable, whose
791
+ /// records are packed without padding.
792
+ pub fn compute_record_stride(variables: &[NcVariable]) -> Result<u64> {
793
+ let mut record_vars = variables.iter().filter(|v| v.is_record_var);
794
+ let (first, second) = (record_vars.next(), record_vars.next());
795
+ if let (Some(only), None) = (first, second) {
796
+ return Ok(only.record_size);
797
+ }
798
+ variables
799
+ .iter()
800
+ .filter(|v| v.is_record_var)
801
+ .try_fold(0u64, |stride, v| {
802
+ let size = v.record_size;
803
+ // Pad each variable's per-record size to 4-byte boundary.
804
+ let rem = size % 4;
805
+ let padded = if rem == 0 {
806
+ size
807
+ } else {
808
+ size.checked_add(4 - rem).ok_or_else(|| {
809
+ Error::InvalidData(format!(
810
+ "record variable '{}' padded record size exceeds u64",
811
+ v.name
812
+ ))
813
+ })?
814
+ };
815
+ stride
816
+ .checked_add(padded)
817
+ .ok_or_else(|| Error::InvalidData("record stride exceeds u64".to_string()))
818
+ })
819
+ }
820
+
821
+ fn checked_non_record_element_count(var: &NcVariable) -> Result<usize> {
822
+ let mut total = 1u64;
823
+ for dim in &var.dimensions {
824
+ total = total.checked_mul(dim.size).ok_or_else(|| {
825
+ Error::InvalidData("variable element count overflows u64".to_string())
826
+ })?;
827
+ }
828
+ crate::types::checked_usize_from_u64(total, "variable element count")
829
+ }
830
+
831
+ pub(crate) fn checked_variable_shape(var: &NcVariable) -> Result<Vec<usize>> {
832
+ var.shape()
833
+ .iter()
834
+ .map(|&s| crate::types::checked_usize_from_u64(s, "variable dimension"))
835
+ .collect::<Result<Vec<_>>>()
836
+ }
837
+
838
+ fn checked_record_shape(var: &NcVariable, numrecs: u64) -> Result<Vec<usize>> {
839
+ let mut shape: Vec<usize> = var
840
+ .shape()
841
+ .iter()
842
+ .map(|&s| crate::types::checked_usize_from_u64(s, "record variable dimension"))
843
+ .collect::<Result<Vec<_>>>()?;
844
+ if shape.is_empty() {
845
+ return Err(Error::InvalidData(
846
+ "record variable must have at least one dimension".to_string(),
847
+ ));
848
+ }
849
+ shape[0] = crate::types::checked_usize_from_u64(numrecs, "record count")?;
850
+ Ok(shape)
851
+ }
852
+
853
+ fn checked_record_elements_per_record(var: &NcVariable) -> Result<usize> {
854
+ let mut elements = 1usize;
855
+ for dim in var.dimensions.iter().skip(1) {
856
+ let size = crate::types::checked_usize_from_u64(dim.size, "record variable dimension")?;
857
+ elements = elements.checked_mul(size).ok_or_else(|| {
858
+ Error::InvalidData(format!(
859
+ "record variable '{}' elements per record exceed platform usize",
860
+ var.name
861
+ ))
862
+ })?;
863
+ }
864
+ Ok(elements)
865
+ }
866
+
867
+ pub(crate) fn variable_data_bytes<T: NcReadType>(
868
+ var_name: &str,
869
+ element_count: usize,
870
+ ) -> Result<usize> {
871
+ element_count.checked_mul(T::element_size()).ok_or_else(|| {
872
+ Error::InvalidData(format!(
873
+ "variable '{var_name}' size in bytes exceeds platform usize"
874
+ ))
875
+ })
876
+ }
877
+
878
+ /// Validate that all `numrecs` records fit within `available` bytes before a
879
+ /// caller allocates the whole result buffer, so a crafted record count cannot
880
+ /// force a huge up-front allocation ahead of the per-record bounds checks.
881
+ pub(crate) fn validate_record_span(
882
+ var: &NcVariable,
883
+ numrecs: u64,
884
+ record_stride: u64,
885
+ bytes_per_record: u64,
886
+ available: u64,
887
+ ) -> Result<()> {
888
+ if numrecs == 0 {
889
+ return Ok(());
890
+ }
891
+ let last_record_start = record_byte_offset(var, numrecs - 1, record_stride)?;
892
+ let end = last_record_start
893
+ .checked_add(bytes_per_record)
894
+ .ok_or_else(|| {
895
+ Error::InvalidData(format!(
896
+ "record variable '{}' data extent exceeds u64",
897
+ var.name
898
+ ))
899
+ })?;
900
+ if end > available {
901
+ return Err(Error::InvalidData(format!(
902
+ "record variable '{}' needs {end} bytes but only {available} are available",
903
+ var.name
904
+ )));
905
+ }
906
+ Ok(())
907
+ }
908
+
909
+ pub(crate) fn record_byte_offset(var: &NcVariable, record: u64, record_stride: u64) -> Result<u64> {
910
+ var.data_offset
911
+ .checked_add(record.checked_mul(record_stride).ok_or_else(|| {
912
+ Error::InvalidData(format!(
913
+ "record variable '{}' byte offset exceeds u64",
914
+ var.name
915
+ ))
916
+ })?)
917
+ .ok_or_else(|| {
918
+ Error::InvalidData(format!(
919
+ "record variable '{}' byte offset exceeds u64",
920
+ var.name
921
+ ))
922
+ })
923
+ }
924
+
925
+ #[cfg(feature = "rayon")]
926
+ pub(crate) fn read_contiguous_range_parallel<T: NcReadType>(
927
+ storage: &ClassicStorage,
928
+ base_offset: u64,
929
+ total_elements: usize,
930
+ target_chunk_bytes: usize,
931
+ ) -> Result<Vec<T>> {
932
+ if total_elements == 0 {
933
+ return Ok(Vec::new());
934
+ }
935
+
936
+ let elem_size = T::element_size();
937
+ let elements_per_chunk = (target_chunk_bytes / elem_size.max(1)).max(1);
938
+ let mut values = vec![T::default(); total_elements];
939
+ values
940
+ .par_chunks_mut(elements_per_chunk)
941
+ .enumerate()
942
+ .try_for_each(|(chunk, dst)| {
943
+ let start_element = chunk.checked_mul(elements_per_chunk).ok_or_else(|| {
944
+ Error::InvalidData(
945
+ "classic parallel chunk offset exceeds platform usize".to_string(),
946
+ )
947
+ })?;
948
+ let byte_offset = crate::types::checked_mul_u64(
949
+ start_element as u64,
950
+ elem_size as u64,
951
+ "classic parallel byte offset",
952
+ )?;
953
+ let offset = base_offset.checked_add(byte_offset).ok_or_else(|| {
954
+ Error::InvalidData("classic parallel byte offset exceeds u64".to_string())
955
+ })?;
956
+ let bytes = variable_data_bytes::<T>("parallel chunk", dst.len())?;
957
+ let data = storage.read_range(offset, bytes)?;
958
+ T::decode_bulk_be_into(data.as_ref(), dst)
959
+ })?;
960
+
961
+ Ok(values)
962
+ }
963
+
964
+ #[cfg(feature = "rayon")]
965
+ struct RecordChunkReadPlan<'a> {
966
+ var: &'a NcVariable,
967
+ record_stride: u64,
968
+ elements_per_record: usize,
969
+ bytes_per_record: usize,
970
+ }
971
+
972
+ #[cfg(feature = "rayon")]
973
+ fn read_record_chunk_into<T: NcReadType>(
974
+ storage: &ClassicStorage,
975
+ plan: &RecordChunkReadPlan<'_>,
976
+ first_record: u64,
977
+ records: usize,
978
+ dst: &mut [T],
979
+ ) -> Result<()> {
980
+ if records == 0 {
981
+ return Ok(());
982
+ }
983
+
984
+ let expected_elements = records
985
+ .checked_mul(plan.elements_per_record)
986
+ .ok_or_else(|| {
987
+ Error::InvalidData(format!(
988
+ "record variable '{}' chunk element count exceeds platform usize",
989
+ plan.var.name
990
+ ))
991
+ })?;
992
+ if dst.len() != expected_elements {
993
+ return Err(Error::InvalidData(format!(
994
+ "record variable '{}' chunk destination has {} elements, expected {}",
995
+ plan.var.name,
996
+ dst.len(),
997
+ expected_elements
998
+ )));
999
+ }
1000
+
1001
+ if plan.record_stride == plan.bytes_per_record as u64 {
1002
+ let offset = record_byte_offset(plan.var, first_record, plan.record_stride)?;
1003
+ let bytes = records.checked_mul(plan.bytes_per_record).ok_or_else(|| {
1004
+ Error::InvalidData(format!(
1005
+ "record variable '{}' chunk byte count exceeds platform usize",
1006
+ plan.var.name
1007
+ ))
1008
+ })?;
1009
+ let data = storage.read_range(offset, bytes)?;
1010
+ return T::decode_bulk_be_into(data.as_ref(), dst);
1011
+ }
1012
+
1013
+ for ordinal in 0..records {
1014
+ let record = first_record
1015
+ .checked_add(ordinal as u64)
1016
+ .ok_or_else(|| Error::InvalidData("classic record index exceeds u64".to_string()))?;
1017
+ let offset = record_byte_offset(plan.var, record, plan.record_stride)?;
1018
+ let data = storage.read_range(offset, plan.bytes_per_record)?;
1019
+ let dst_start = ordinal
1020
+ .checked_mul(plan.elements_per_record)
1021
+ .ok_or_else(|| {
1022
+ Error::InvalidData(format!(
1023
+ "record variable '{}' chunk destination offset exceeds platform usize",
1024
+ plan.var.name
1025
+ ))
1026
+ })?;
1027
+ let dst_end = dst_start
1028
+ .checked_add(plan.elements_per_record)
1029
+ .ok_or_else(|| {
1030
+ Error::InvalidData(format!(
1031
+ "record variable '{}' chunk destination range exceeds platform usize",
1032
+ plan.var.name
1033
+ ))
1034
+ })?;
1035
+ T::decode_bulk_be_into(data.as_ref(), &mut dst[dst_start..dst_end])?;
1036
+ }
1037
+ Ok(())
1038
+ }
1039
+
1040
+ #[cfg(test)]
1041
+ mod tests {
1042
+ use super::*;
1043
+ use crate::types::NcDimension;
1044
+
1045
+ #[test]
1046
+ fn read_non_record_1d_float() {
1047
+ // Create a fake file with 3 floats starting at offset 100.
1048
+ let mut file_data = vec![0u8; 200];
1049
+ let values = [1.0f32, 2.0f32, 3.0f32];
1050
+ for (i, &v) in values.iter().enumerate() {
1051
+ let bytes = v.to_be_bytes();
1052
+ file_data[100 + i * 4..100 + i * 4 + 4].copy_from_slice(&bytes);
1053
+ }
1054
+
1055
+ let var = NcVariable {
1056
+ name: "temp".to_string(),
1057
+ dimensions: vec![NcDimension {
1058
+ name: "x".to_string(),
1059
+ size: 3,
1060
+ is_unlimited: false,
1061
+ }],
1062
+ dtype: NcType::Float,
1063
+ attributes: vec![],
1064
+ data_offset: 100,
1065
+ _data_size: 12,
1066
+ is_record_var: false,
1067
+ record_size: 0,
1068
+ };
1069
+
1070
+ let arr: ArrayD<f32> = read_non_record_variable(&file_data, &var).unwrap();
1071
+ assert_eq!(arr.shape(), &[3]);
1072
+ assert_eq!(arr[[0]], 1.0f32);
1073
+ assert_eq!(arr[[1]], 2.0f32);
1074
+ assert_eq!(arr[[2]], 3.0f32);
1075
+ }
1076
+
1077
+ #[test]
1078
+ fn non_record_variable_into_copies_values() {
1079
+ let mut file_data = vec![0u8; 200];
1080
+ let values = [1.0f32, 2.0f32, 3.0f32];
1081
+ for (i, &v) in values.iter().enumerate() {
1082
+ file_data[100 + i * 4..100 + i * 4 + 4].copy_from_slice(&v.to_be_bytes());
1083
+ }
1084
+
1085
+ let var = NcVariable {
1086
+ name: "temp".to_string(),
1087
+ dimensions: vec![NcDimension {
1088
+ name: "x".to_string(),
1089
+ size: 3,
1090
+ is_unlimited: false,
1091
+ }],
1092
+ dtype: NcType::Float,
1093
+ attributes: vec![],
1094
+ data_offset: 100,
1095
+ _data_size: 12,
1096
+ is_record_var: false,
1097
+ record_size: 0,
1098
+ };
1099
+
1100
+ let mut dst = [0.0f32; 3];
1101
+ read_non_record_variable_into(&file_data, &var, &mut dst).unwrap();
1102
+ assert_eq!(dst, values);
1103
+ }
1104
+
1105
+ #[test]
1106
+ fn read_non_record_2d_int() {
1107
+ // 2x3 array of i32 at offset 0
1108
+ let values: Vec<i32> = vec![10, 20, 30, 40, 50, 60];
1109
+ let mut file_data = Vec::new();
1110
+ for &v in &values {
1111
+ file_data.extend_from_slice(&v.to_be_bytes());
1112
+ }
1113
+
1114
+ let var = NcVariable {
1115
+ name: "grid".to_string(),
1116
+ dimensions: vec![
1117
+ NcDimension {
1118
+ name: "y".to_string(),
1119
+ size: 2,
1120
+ is_unlimited: false,
1121
+ },
1122
+ NcDimension {
1123
+ name: "x".to_string(),
1124
+ size: 3,
1125
+ is_unlimited: false,
1126
+ },
1127
+ ],
1128
+ dtype: NcType::Int,
1129
+ attributes: vec![],
1130
+ data_offset: 0,
1131
+ _data_size: 24,
1132
+ is_record_var: false,
1133
+ record_size: 0,
1134
+ };
1135
+
1136
+ let arr: ArrayD<i32> = read_non_record_variable(&file_data, &var).unwrap();
1137
+ assert_eq!(arr.shape(), &[2, 3]);
1138
+ assert_eq!(arr[[0, 0]], 10);
1139
+ assert_eq!(arr[[0, 2]], 30);
1140
+ assert_eq!(arr[[1, 0]], 40);
1141
+ assert_eq!(arr[[1, 2]], 60);
1142
+ }
1143
+
1144
+ #[test]
1145
+ fn read_non_record_variable_into_rejects_wrong_destination_len() {
1146
+ let var = NcVariable {
1147
+ name: "grid".to_string(),
1148
+ dimensions: vec![NcDimension {
1149
+ name: "x".to_string(),
1150
+ size: 3,
1151
+ is_unlimited: false,
1152
+ }],
1153
+ dtype: NcType::Float,
1154
+ attributes: vec![],
1155
+ data_offset: 0,
1156
+ _data_size: 12,
1157
+ is_record_var: false,
1158
+ record_size: 0,
1159
+ };
1160
+
1161
+ let mut dst = [0.0f32; 2];
1162
+ let err = read_non_record_variable_into(&[0; 12], &var, &mut dst).unwrap_err();
1163
+ assert!(matches!(err, Error::InvalidData(_)));
1164
+ }
1165
+
1166
+ #[test]
1167
+ fn record_stride_sums_padded_record_variables() {
1168
+ let vars = vec![
1169
+ NcVariable {
1170
+ name: "a".to_string(),
1171
+ dimensions: vec![],
1172
+ dtype: NcType::Float,
1173
+ attributes: vec![],
1174
+ data_offset: 0,
1175
+ _data_size: 0,
1176
+ is_record_var: true,
1177
+ record_size: 20, // 5 floats
1178
+ },
1179
+ NcVariable {
1180
+ name: "b".to_string(),
1181
+ dimensions: vec![],
1182
+ dtype: NcType::Short,
1183
+ attributes: vec![],
1184
+ data_offset: 0,
1185
+ _data_size: 0,
1186
+ is_record_var: true,
1187
+ record_size: 6, // 3 shorts -> padded to 8
1188
+ },
1189
+ NcVariable {
1190
+ name: "c".to_string(),
1191
+ dimensions: vec![],
1192
+ dtype: NcType::Double,
1193
+ attributes: vec![],
1194
+ data_offset: 0,
1195
+ _data_size: 100,
1196
+ is_record_var: false, // not a record var, should be excluded
1197
+ record_size: 0,
1198
+ },
1199
+ ];
1200
+ // a: 20 (already 4-aligned), b: 6 -> 8 = total 28
1201
+ assert_eq!(compute_record_stride(&vars).unwrap(), 28);
1202
+ }
1203
+
1204
+ #[test]
1205
+ fn record_stride_rejects_padded_size_overflow() {
1206
+ // Two record variables so per-variable padding applies (a lone
1207
+ // record variable is packed without padding).
1208
+ let vars = vec![
1209
+ NcVariable {
1210
+ name: "huge".to_string(),
1211
+ dimensions: vec![],
1212
+ dtype: NcType::Byte,
1213
+ attributes: vec![],
1214
+ data_offset: 0,
1215
+ _data_size: 0,
1216
+ is_record_var: true,
1217
+ record_size: u64::MAX,
1218
+ },
1219
+ NcVariable {
1220
+ name: "small".to_string(),
1221
+ dimensions: vec![],
1222
+ dtype: NcType::Byte,
1223
+ attributes: vec![],
1224
+ data_offset: 0,
1225
+ _data_size: 0,
1226
+ is_record_var: true,
1227
+ record_size: 4,
1228
+ },
1229
+ ];
1230
+
1231
+ let err = compute_record_stride(&vars).unwrap_err();
1232
+ assert!(matches!(err, Error::InvalidData(_)));
1233
+ }
1234
+
1235
+ #[test]
1236
+ fn record_stride_rejects_sum_overflow() {
1237
+ let vars = vec![
1238
+ NcVariable {
1239
+ name: "a".to_string(),
1240
+ dimensions: vec![],
1241
+ dtype: NcType::Byte,
1242
+ attributes: vec![],
1243
+ data_offset: 0,
1244
+ _data_size: 0,
1245
+ is_record_var: true,
1246
+ record_size: u64::MAX - 7,
1247
+ },
1248
+ NcVariable {
1249
+ name: "b".to_string(),
1250
+ dimensions: vec![],
1251
+ dtype: NcType::Byte,
1252
+ attributes: vec![],
1253
+ data_offset: 0,
1254
+ _data_size: 0,
1255
+ is_record_var: true,
1256
+ record_size: 8,
1257
+ },
1258
+ ];
1259
+
1260
+ let err = compute_record_stride(&vars).unwrap_err();
1261
+ assert!(matches!(err, Error::InvalidData(_)));
1262
+ }
1263
+
1264
+ #[test]
1265
+ fn record_variable_reads_all_records() {
1266
+ // Single record variable "temp" with shape [time, x] where x=2.
1267
+ // 3 records, each with 2 floats = 8 bytes per record.
1268
+ // Record stride = 8 (only one record var, already 4-aligned).
1269
+ let mut file_data = vec![0u8; 200];
1270
+ let base = 100usize;
1271
+ let record_values: Vec<Vec<f32>> = vec![vec![1.0, 2.0], vec![3.0, 4.0], vec![5.0, 6.0]];
1272
+ for (rec, vals) in record_values.iter().enumerate() {
1273
+ for (i, &v) in vals.iter().enumerate() {
1274
+ let offset = base + rec * 8 + i * 4;
1275
+ file_data[offset..offset + 4].copy_from_slice(&v.to_be_bytes());
1276
+ }
1277
+ }
1278
+
1279
+ let var = NcVariable {
1280
+ name: "temp".to_string(),
1281
+ dimensions: vec![
1282
+ NcDimension {
1283
+ name: "time".to_string(),
1284
+ size: 0, // unlimited
1285
+ is_unlimited: true,
1286
+ },
1287
+ NcDimension {
1288
+ name: "x".to_string(),
1289
+ size: 2,
1290
+ is_unlimited: false,
1291
+ },
1292
+ ],
1293
+ dtype: NcType::Float,
1294
+ attributes: vec![],
1295
+ data_offset: 100,
1296
+ _data_size: 0,
1297
+ is_record_var: true,
1298
+ record_size: 8,
1299
+ };
1300
+
1301
+ let arr: ArrayD<f32> = read_record_variable(&file_data, &var, 3, 8).unwrap();
1302
+ assert_eq!(arr.shape(), &[3, 2]);
1303
+ assert_eq!(arr[[0, 0]], 1.0);
1304
+ assert_eq!(arr[[0, 1]], 2.0);
1305
+ assert_eq!(arr[[1, 0]], 3.0);
1306
+ assert_eq!(arr[[2, 1]], 6.0);
1307
+ }
1308
+
1309
+ #[test]
1310
+ fn record_variable_into_copies_values() {
1311
+ let mut file_data = vec![0u8; 200];
1312
+ let base = 100usize;
1313
+ let record_values: Vec<Vec<f32>> = vec![vec![1.0, 2.0], vec![3.0, 4.0], vec![5.0, 6.0]];
1314
+ for (rec, vals) in record_values.iter().enumerate() {
1315
+ for (i, &v) in vals.iter().enumerate() {
1316
+ let offset = base + rec * 8 + i * 4;
1317
+ file_data[offset..offset + 4].copy_from_slice(&v.to_be_bytes());
1318
+ }
1319
+ }
1320
+
1321
+ let var = NcVariable {
1322
+ name: "temp".to_string(),
1323
+ dimensions: vec![
1324
+ NcDimension {
1325
+ name: "time".to_string(),
1326
+ size: 0,
1327
+ is_unlimited: true,
1328
+ },
1329
+ NcDimension {
1330
+ name: "x".to_string(),
1331
+ size: 2,
1332
+ is_unlimited: false,
1333
+ },
1334
+ ],
1335
+ dtype: NcType::Float,
1336
+ attributes: vec![],
1337
+ data_offset: 100,
1338
+ _data_size: 0,
1339
+ is_record_var: true,
1340
+ record_size: 8,
1341
+ };
1342
+
1343
+ let mut dst = [0.0f32; 6];
1344
+ read_record_variable_into(&file_data, &var, 3, 8, &mut dst).unwrap();
1345
+ assert_eq!(dst, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]);
1346
+ }
1347
+
1348
+ #[test]
1349
+ fn read_record_variable_into_rejects_wrong_destination_len() {
1350
+ let var = NcVariable {
1351
+ name: "temp".to_string(),
1352
+ dimensions: vec![
1353
+ NcDimension {
1354
+ name: "time".to_string(),
1355
+ size: 0,
1356
+ is_unlimited: true,
1357
+ },
1358
+ NcDimension {
1359
+ name: "x".to_string(),
1360
+ size: 2,
1361
+ is_unlimited: false,
1362
+ },
1363
+ ],
1364
+ dtype: NcType::Float,
1365
+ attributes: vec![],
1366
+ data_offset: 0,
1367
+ _data_size: 0,
1368
+ is_record_var: true,
1369
+ record_size: 8,
1370
+ };
1371
+
1372
+ let mut dst = [0.0f32; 5];
1373
+ let err = read_record_variable_into(&[0; 24], &var, 3, 8, &mut dst).unwrap_err();
1374
+ assert!(matches!(err, Error::InvalidData(_)));
1375
+ }
1376
+
1377
+ #[test]
1378
+ fn read_non_record_variable_rejects_element_count_overflow() {
1379
+ let var = NcVariable {
1380
+ name: "huge".to_string(),
1381
+ dimensions: vec![
1382
+ NcDimension {
1383
+ name: "y".to_string(),
1384
+ size: u64::MAX,
1385
+ is_unlimited: false,
1386
+ },
1387
+ NcDimension {
1388
+ name: "x".to_string(),
1389
+ size: 2,
1390
+ is_unlimited: false,
1391
+ },
1392
+ ],
1393
+ dtype: NcType::Float,
1394
+ attributes: vec![],
1395
+ data_offset: 0,
1396
+ _data_size: 0,
1397
+ is_record_var: false,
1398
+ record_size: 0,
1399
+ };
1400
+
1401
+ let err = read_non_record_variable::<f32>(&[], &var).unwrap_err();
1402
+ assert!(matches!(err, Error::InvalidData(_)));
1403
+ }
1404
+
1405
+ #[test]
1406
+ fn read_record_variable_rejects_elements_per_record_overflow() {
1407
+ let var = NcVariable {
1408
+ name: "huge_record".to_string(),
1409
+ dimensions: vec![
1410
+ NcDimension {
1411
+ name: "time".to_string(),
1412
+ size: 0,
1413
+ is_unlimited: true,
1414
+ },
1415
+ NcDimension {
1416
+ name: "y".to_string(),
1417
+ size: usize::MAX as u64,
1418
+ is_unlimited: false,
1419
+ },
1420
+ NcDimension {
1421
+ name: "x".to_string(),
1422
+ size: 2,
1423
+ is_unlimited: false,
1424
+ },
1425
+ ],
1426
+ dtype: NcType::Float,
1427
+ attributes: vec![],
1428
+ data_offset: 0,
1429
+ _data_size: 0,
1430
+ is_record_var: true,
1431
+ record_size: 4,
1432
+ };
1433
+
1434
+ let err = read_record_variable::<f32>(&[], &var, 1, 4).unwrap_err();
1435
+ assert!(matches!(err, Error::InvalidData(_)));
1436
+ }
1437
+
1438
+ #[test]
1439
+ fn read_record_variable_rejects_record_offset_overflow() {
1440
+ let var = NcVariable {
1441
+ name: "huge_record".to_string(),
1442
+ dimensions: vec![
1443
+ NcDimension {
1444
+ name: "time".to_string(),
1445
+ size: 0,
1446
+ is_unlimited: true,
1447
+ },
1448
+ NcDimension {
1449
+ name: "x".to_string(),
1450
+ size: 1,
1451
+ is_unlimited: false,
1452
+ },
1453
+ ],
1454
+ dtype: NcType::Float,
1455
+ attributes: vec![],
1456
+ data_offset: u64::MAX,
1457
+ _data_size: 0,
1458
+ is_record_var: true,
1459
+ record_size: 4,
1460
+ };
1461
+
1462
+ let err = read_record_variable::<f32>(&[], &var, 1, 4).unwrap_err();
1463
+ assert!(matches!(err, Error::InvalidData(_)));
1464
+ }
1465
+ }