@nirs4all/formats-wasm 0.2.11 → 0.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD_PARTY_NOTICES.md +49 -0
- package/nirs4all_formats_wasm_bg.wasm +0 -0
- package/package.json +4 -2
- package/reader-backports/hdf5-reader/Cargo.toml +119 -0
- package/reader-backports/hdf5-reader/LICENSE-APACHE +164 -0
- package/reader-backports/hdf5-reader/LICENSE-MIT +21 -0
- package/reader-backports/hdf5-reader/PROVENANCE.json +80 -0
- package/reader-backports/hdf5-reader/README.md +308 -0
- package/reader-backports/hdf5-reader/UPSTREAM-Cargo.toml +44 -0
- package/reader-backports/hdf5-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/hdf5-reader/src/attribute_api.rs +847 -0
- package/reader-backports/hdf5-reader/src/btree_v1.rs +726 -0
- package/reader-backports/hdf5-reader/src/btree_v2.rs +2068 -0
- package/reader-backports/hdf5-reader/src/cache.rs +354 -0
- package/reader-backports/hdf5-reader/src/checksum.rs +56 -0
- package/reader-backports/hdf5-reader/src/chunk_index.rs +423 -0
- package/reader-backports/hdf5-reader/src/dataset.rs +4733 -0
- package/reader-backports/hdf5-reader/src/datatype_api.rs +497 -0
- package/reader-backports/hdf5-reader/src/error.rs +167 -0
- package/reader-backports/hdf5-reader/src/extensible_array.rs +1886 -0
- package/reader-backports/hdf5-reader/src/filters/deflate.rs +93 -0
- package/reader-backports/hdf5-reader/src/filters/fletcher32.rs +52 -0
- package/reader-backports/hdf5-reader/src/filters/lz4.rs +178 -0
- package/reader-backports/hdf5-reader/src/filters/mod.rs +326 -0
- package/reader-backports/hdf5-reader/src/filters/nbit.rs +586 -0
- package/reader-backports/hdf5-reader/src/filters/scaleoffset.rs +827 -0
- package/reader-backports/hdf5-reader/src/filters/shuffle.rs +99 -0
- package/reader-backports/hdf5-reader/src/fixed_array.rs +763 -0
- package/reader-backports/hdf5-reader/src/fractal_heap.rs +2029 -0
- package/reader-backports/hdf5-reader/src/global_heap.rs +284 -0
- package/reader-backports/hdf5-reader/src/group.rs +795 -0
- package/reader-backports/hdf5-reader/src/io.rs +413 -0
- package/reader-backports/hdf5-reader/src/lib.rs +916 -0
- package/reader-backports/hdf5-reader/src/local_heap.rs +261 -0
- package/reader-backports/hdf5-reader/src/messages/attribute.rs +304 -0
- package/reader-backports/hdf5-reader/src/messages/attribute_info.rs +96 -0
- package/reader-backports/hdf5-reader/src/messages/btree_k.rs +36 -0
- package/reader-backports/hdf5-reader/src/messages/continuation.rs +46 -0
- package/reader-backports/hdf5-reader/src/messages/dataspace.rs +259 -0
- package/reader-backports/hdf5-reader/src/messages/datatype.rs +938 -0
- package/reader-backports/hdf5-reader/src/messages/external_files.rs +50 -0
- package/reader-backports/hdf5-reader/src/messages/fill_value.rs +233 -0
- package/reader-backports/hdf5-reader/src/messages/filter_pipeline.rs +147 -0
- package/reader-backports/hdf5-reader/src/messages/group_info.rs +103 -0
- package/reader-backports/hdf5-reader/src/messages/layout.rs +476 -0
- package/reader-backports/hdf5-reader/src/messages/link.rs +170 -0
- package/reader-backports/hdf5-reader/src/messages/link_info.rs +127 -0
- package/reader-backports/hdf5-reader/src/messages/mod.rs +374 -0
- package/reader-backports/hdf5-reader/src/messages/modification_time.rs +83 -0
- package/reader-backports/hdf5-reader/src/messages/shared.rs +164 -0
- package/reader-backports/hdf5-reader/src/messages/shared_table.rs +67 -0
- package/reader-backports/hdf5-reader/src/messages/symbol_table_msg.rs +51 -0
- package/reader-backports/hdf5-reader/src/object_header.rs +1749 -0
- package/reader-backports/hdf5-reader/src/reference.rs +90 -0
- package/reader-backports/hdf5-reader/src/shared_message_table.rs +425 -0
- package/reader-backports/hdf5-reader/src/storage.rs +538 -0
- package/reader-backports/hdf5-reader/src/superblock.rs +285 -0
- package/reader-backports/hdf5-reader/src/symbol_table.rs +277 -0
- package/reader-backports/netcdf-reader/Cargo.toml +141 -0
- package/reader-backports/netcdf-reader/LICENSE-APACHE +164 -0
- package/reader-backports/netcdf-reader/LICENSE-MIT +21 -0
- package/reader-backports/netcdf-reader/PROVENANCE.json +55 -0
- package/reader-backports/netcdf-reader/README.md +308 -0
- package/reader-backports/netcdf-reader/UPSTREAM-Cargo.toml +48 -0
- package/reader-backports/netcdf-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/netcdf-reader/src/cf/axes.rs +419 -0
- package/reader-backports/netcdf-reader/src/cf/bounds.rs +108 -0
- package/reader-backports/netcdf-reader/src/cf/crs.rs +269 -0
- package/reader-backports/netcdf-reader/src/cf/mod.rs +30 -0
- package/reader-backports/netcdf-reader/src/cf/time.rs +1122 -0
- package/reader-backports/netcdf-reader/src/classic/data.rs +1465 -0
- package/reader-backports/netcdf-reader/src/classic/header.rs +1097 -0
- package/reader-backports/netcdf-reader/src/classic/mod.rs +240 -0
- package/reader-backports/netcdf-reader/src/classic/storage.rs +211 -0
- package/reader-backports/netcdf-reader/src/classic/types.rs +93 -0
- package/reader-backports/netcdf-reader/src/classic/variable.rs +1162 -0
- package/reader-backports/netcdf-reader/src/error.rs +59 -0
- package/reader-backports/netcdf-reader/src/lib.rs +1531 -0
- package/reader-backports/netcdf-reader/src/masked.rs +173 -0
- package/reader-backports/netcdf-reader/src/nc4/attributes.rs +203 -0
- package/reader-backports/netcdf-reader/src/nc4/dimensions.rs +139 -0
- package/reader-backports/netcdf-reader/src/nc4/groups.rs +308 -0
- package/reader-backports/netcdf-reader/src/nc4/mod.rs +572 -0
- package/reader-backports/netcdf-reader/src/nc4/types.rs +309 -0
- package/reader-backports/netcdf-reader/src/nc4/variables.rs +420 -0
- package/reader-backports/netcdf-reader/src/types.rs +18 -0
- package/reader-backports/netcdf-reader/src/unpack.rs +113 -0
- package/reader-backports/netcdf-reader/src/user_defined.rs +569 -0
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
//! _FillValue / missing_value masking for NetCDF variables.
|
|
2
|
+
//!
|
|
3
|
+
//! Replaces fill/missing values with NaN according to CF conventions.
|
|
4
|
+
//! Checks `_FillValue`, `missing_value`, `valid_min`, `valid_max`, and `valid_range`.
|
|
5
|
+
|
|
6
|
+
use ndarray::ArrayD;
|
|
7
|
+
|
|
8
|
+
use crate::types::NcVariable;
|
|
9
|
+
|
|
10
|
+
/// Parameters for masking invalid data.
|
|
11
|
+
#[derive(Debug, Clone)]
|
|
12
|
+
pub struct MaskParams {
|
|
13
|
+
pub fill_value: Option<f64>,
|
|
14
|
+
pub missing_value: Option<f64>,
|
|
15
|
+
pub valid_min: Option<f64>,
|
|
16
|
+
pub valid_max: Option<f64>,
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
impl MaskParams {
|
|
20
|
+
/// Extract masking parameters from a variable's attributes.
|
|
21
|
+
///
|
|
22
|
+
/// Returns `None` if no masking attributes are present.
|
|
23
|
+
pub fn from_variable(var: &NcVariable) -> Option<Self> {
|
|
24
|
+
let fill = var.attribute("_FillValue").and_then(|a| a.value.as_f64());
|
|
25
|
+
let missing = var
|
|
26
|
+
.attribute("missing_value")
|
|
27
|
+
.and_then(|a| a.value.as_f64());
|
|
28
|
+
|
|
29
|
+
// valid_range takes precedence over valid_min/valid_max individually
|
|
30
|
+
let (vmin, vmax) = if let Some(range) = var
|
|
31
|
+
.attribute("valid_range")
|
|
32
|
+
.and_then(|a| a.value.as_f64_vec())
|
|
33
|
+
{
|
|
34
|
+
if range.len() >= 2 {
|
|
35
|
+
(Some(range[0]), Some(range[1]))
|
|
36
|
+
} else {
|
|
37
|
+
(None, None)
|
|
38
|
+
}
|
|
39
|
+
} else {
|
|
40
|
+
let vmin = var.attribute("valid_min").and_then(|a| a.value.as_f64());
|
|
41
|
+
let vmax = var.attribute("valid_max").and_then(|a| a.value.as_f64());
|
|
42
|
+
(vmin, vmax)
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
if fill.is_none() && missing.is_none() && vmin.is_none() && vmax.is_none() {
|
|
46
|
+
return None;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
Some(MaskParams {
|
|
50
|
+
fill_value: fill,
|
|
51
|
+
missing_value: missing,
|
|
52
|
+
valid_min: vmin,
|
|
53
|
+
valid_max: vmax,
|
|
54
|
+
})
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/// Replace fill/missing values with NaN and mask values outside valid range.
|
|
58
|
+
///
|
|
59
|
+
/// Uses bit-exact comparison for fill/missing values to correctly handle
|
|
60
|
+
/// NaN fill values (since `NaN != NaN` with normal `==`).
|
|
61
|
+
pub fn apply(&self, data: &mut ArrayD<f64>) {
|
|
62
|
+
data.mapv_inplace(|v| {
|
|
63
|
+
if let Some(fill) = self.fill_value {
|
|
64
|
+
if v.to_bits() == fill.to_bits() {
|
|
65
|
+
return f64::NAN;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
if let Some(miss) = self.missing_value {
|
|
69
|
+
if v.to_bits() == miss.to_bits() {
|
|
70
|
+
return f64::NAN;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
if let Some(vmin) = self.valid_min {
|
|
74
|
+
if v < vmin {
|
|
75
|
+
return f64::NAN;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
if let Some(vmax) = self.valid_max {
|
|
79
|
+
if v > vmax {
|
|
80
|
+
return f64::NAN;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
v
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
#[cfg(test)]
|
|
89
|
+
mod tests {
|
|
90
|
+
use super::*;
|
|
91
|
+
use crate::types::{NcAttrValue, NcAttribute, NcType, NcVariable};
|
|
92
|
+
use ndarray::arr1;
|
|
93
|
+
|
|
94
|
+
fn make_var(attrs: Vec<NcAttribute>) -> NcVariable {
|
|
95
|
+
NcVariable {
|
|
96
|
+
name: "test".into(),
|
|
97
|
+
dimensions: vec![],
|
|
98
|
+
dtype: NcType::Float,
|
|
99
|
+
attributes: attrs,
|
|
100
|
+
data_offset: 0,
|
|
101
|
+
_data_size: 0,
|
|
102
|
+
is_record_var: false,
|
|
103
|
+
record_size: 0,
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
#[test]
|
|
108
|
+
fn no_mask_attrs() {
|
|
109
|
+
let var = make_var(vec![]);
|
|
110
|
+
assert!(MaskParams::from_variable(&var).is_none());
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
#[test]
|
|
114
|
+
fn fill_value() {
|
|
115
|
+
let var = make_var(vec![NcAttribute {
|
|
116
|
+
name: "_FillValue".into(),
|
|
117
|
+
value: NcAttrValue::Floats(vec![-9999.0]),
|
|
118
|
+
}]);
|
|
119
|
+
let params = MaskParams::from_variable(&var).unwrap();
|
|
120
|
+
let mut data = arr1(&[1.0, -9999.0, 3.0]).into_dyn();
|
|
121
|
+
params.apply(&mut data);
|
|
122
|
+
assert_eq!(data[[0]], 1.0);
|
|
123
|
+
assert!(data[[1]].is_nan());
|
|
124
|
+
assert_eq!(data[[2]], 3.0);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
#[test]
|
|
128
|
+
fn missing_value() {
|
|
129
|
+
let var = make_var(vec![NcAttribute {
|
|
130
|
+
name: "missing_value".into(),
|
|
131
|
+
value: NcAttrValue::Doubles(vec![-999.0]),
|
|
132
|
+
}]);
|
|
133
|
+
let params = MaskParams::from_variable(&var).unwrap();
|
|
134
|
+
let mut data = arr1(&[-999.0, 5.0]).into_dyn();
|
|
135
|
+
params.apply(&mut data);
|
|
136
|
+
assert!(data[[0]].is_nan());
|
|
137
|
+
assert_eq!(data[[1]], 5.0);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
#[test]
|
|
141
|
+
fn valid_range() {
|
|
142
|
+
let var = make_var(vec![NcAttribute {
|
|
143
|
+
name: "valid_range".into(),
|
|
144
|
+
value: NcAttrValue::Doubles(vec![0.0, 100.0]),
|
|
145
|
+
}]);
|
|
146
|
+
let params = MaskParams::from_variable(&var).unwrap();
|
|
147
|
+
let mut data = arr1(&[-5.0, 50.0, 150.0]).into_dyn();
|
|
148
|
+
params.apply(&mut data);
|
|
149
|
+
assert!(data[[0]].is_nan());
|
|
150
|
+
assert_eq!(data[[1]], 50.0);
|
|
151
|
+
assert!(data[[2]].is_nan());
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
#[test]
|
|
155
|
+
fn valid_min_max() {
|
|
156
|
+
let var = make_var(vec![
|
|
157
|
+
NcAttribute {
|
|
158
|
+
name: "valid_min".into(),
|
|
159
|
+
value: NcAttrValue::Doubles(vec![0.0]),
|
|
160
|
+
},
|
|
161
|
+
NcAttribute {
|
|
162
|
+
name: "valid_max".into(),
|
|
163
|
+
value: NcAttrValue::Doubles(vec![50.0]),
|
|
164
|
+
},
|
|
165
|
+
]);
|
|
166
|
+
let params = MaskParams::from_variable(&var).unwrap();
|
|
167
|
+
let mut data = arr1(&[-1.0, 25.0, 51.0]).into_dyn();
|
|
168
|
+
params.apply(&mut data);
|
|
169
|
+
assert!(data[[0]].is_nan());
|
|
170
|
+
assert_eq!(data[[1]], 25.0);
|
|
171
|
+
assert!(data[[2]].is_nan());
|
|
172
|
+
}
|
|
173
|
+
}
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
//! Filter and convert HDF5 attributes to NetCDF-4 attributes.
|
|
2
|
+
//!
|
|
3
|
+
//! NetCDF-4 uses several internal HDF5 attributes that should not be exposed
|
|
4
|
+
//! to users:
|
|
5
|
+
//! - `_NCProperties`: file provenance metadata
|
|
6
|
+
//! - `_Netcdf4Dimid`: dimension ID assignment
|
|
7
|
+
//! - `_Netcdf4Coordinates`: coordinate variable tracking
|
|
8
|
+
//! - `DIMENSION_LIST`: dimension-to-variable references
|
|
9
|
+
//! - `REFERENCE_LIST`: variable-to-dimension back-references
|
|
10
|
+
//! - `CLASS`: dimension scale marker
|
|
11
|
+
//! - `NAME`: dimension scale name
|
|
12
|
+
|
|
13
|
+
use hdf5_reader::group::Group;
|
|
14
|
+
use hdf5_reader::messages::datatype::{Datatype, VarLenKind};
|
|
15
|
+
|
|
16
|
+
use crate::error::{Error, Result};
|
|
17
|
+
use crate::types::{NcAttrValue, NcAttribute};
|
|
18
|
+
|
|
19
|
+
/// Internal attribute names that should be hidden from the NetCDF user API.
|
|
20
|
+
const INTERNAL_ATTRIBUTES: &[&str] = &[
|
|
21
|
+
"_NCProperties",
|
|
22
|
+
"_Netcdf4Dimid",
|
|
23
|
+
"_Netcdf4Coordinates",
|
|
24
|
+
"_nc3_strict",
|
|
25
|
+
"DIMENSION_LIST",
|
|
26
|
+
"REFERENCE_LIST",
|
|
27
|
+
"CLASS",
|
|
28
|
+
"NAME",
|
|
29
|
+
];
|
|
30
|
+
|
|
31
|
+
/// Returns true if the given attribute name is an internal NetCDF-4/HDF5 attribute
|
|
32
|
+
/// that should not be exposed to users.
|
|
33
|
+
pub fn is_internal_attribute(name: &str) -> bool {
|
|
34
|
+
INTERNAL_ATTRIBUTES.contains(&name)
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/// Extract user-visible attributes from an HDF5 group, filtering out
|
|
38
|
+
/// internal NetCDF-4 attributes.
|
|
39
|
+
pub fn extract_group_attributes(
|
|
40
|
+
group: &Group,
|
|
41
|
+
metadata_mode: crate::NcMetadataMode,
|
|
42
|
+
) -> Result<Vec<NcAttribute>> {
|
|
43
|
+
collect_visible_attributes(&group.attributes()?, metadata_mode)
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/// Extract user-visible attributes from an HDF5 dataset, filtering out
|
|
47
|
+
/// internal attributes.
|
|
48
|
+
pub fn extract_variable_attributes(
|
|
49
|
+
dataset: &hdf5_reader::Dataset,
|
|
50
|
+
metadata_mode: crate::NcMetadataMode,
|
|
51
|
+
) -> Result<Vec<NcAttribute>> {
|
|
52
|
+
collect_visible_attributes(&dataset.attributes(), metadata_mode)
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
fn collect_visible_attributes(
|
|
56
|
+
attrs: &[hdf5_reader::Attribute],
|
|
57
|
+
metadata_mode: crate::NcMetadataMode,
|
|
58
|
+
) -> Result<Vec<NcAttribute>> {
|
|
59
|
+
let mut nc_attrs = Vec::new();
|
|
60
|
+
for attr in attrs {
|
|
61
|
+
if is_internal_attribute(&attr.name) {
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
if let Some(nc_attr) = convert_attribute(attr, metadata_mode)? {
|
|
65
|
+
nc_attrs.push(nc_attr);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
Ok(nc_attrs)
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
pub fn convert_visible_attribute(
|
|
72
|
+
attr: &hdf5_reader::Attribute,
|
|
73
|
+
metadata_mode: crate::NcMetadataMode,
|
|
74
|
+
) -> Result<Option<NcAttribute>> {
|
|
75
|
+
if is_internal_attribute(&attr.name) {
|
|
76
|
+
return Ok(None);
|
|
77
|
+
}
|
|
78
|
+
convert_attribute(attr, metadata_mode)
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/// Convert an HDF5 attribute to a NetCDF attribute.
|
|
82
|
+
fn convert_attribute(
|
|
83
|
+
attr: &hdf5_reader::Attribute,
|
|
84
|
+
metadata_mode: crate::NcMetadataMode,
|
|
85
|
+
) -> Result<Option<NcAttribute>> {
|
|
86
|
+
let Some(value) = convert_attribute_value(attr, metadata_mode)? else {
|
|
87
|
+
return Ok(None);
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
Ok(Some(NcAttribute {
|
|
91
|
+
name: attr.name.clone(),
|
|
92
|
+
value,
|
|
93
|
+
}))
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/// Convert an HDF5 attribute's value to an NcAttrValue.
|
|
97
|
+
fn convert_attribute_value(
|
|
98
|
+
attr: &hdf5_reader::Attribute,
|
|
99
|
+
metadata_mode: crate::NcMetadataMode,
|
|
100
|
+
) -> Result<Option<NcAttrValue>> {
|
|
101
|
+
let strict = metadata_mode == crate::NcMetadataMode::Strict;
|
|
102
|
+
|
|
103
|
+
let unsupported = || {
|
|
104
|
+
Error::InvalidData(format!(
|
|
105
|
+
"attribute '{}' uses unsupported NetCDF-4 datatype {:?}",
|
|
106
|
+
attr.name, attr.datatype
|
|
107
|
+
))
|
|
108
|
+
};
|
|
109
|
+
|
|
110
|
+
let read_attr = |result: hdf5_reader::error::Result<NcAttrValue>| match result {
|
|
111
|
+
Ok(value) => Ok(Some(value)),
|
|
112
|
+
Err(err) if strict => Err(Error::InvalidData(format!(
|
|
113
|
+
"attribute '{}' could not be decoded: {err}",
|
|
114
|
+
attr.name
|
|
115
|
+
))),
|
|
116
|
+
Err(_) => Ok(None),
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
match &attr.datatype {
|
|
120
|
+
Datatype::FixedPoint { size, signed, .. } => match (size, signed) {
|
|
121
|
+
(1, true) => read_attr(attr.read_1d::<i8>().map(NcAttrValue::Bytes)),
|
|
122
|
+
(1, false) => read_attr(attr.read_1d::<u8>().map(NcAttrValue::UBytes)),
|
|
123
|
+
(2, true) => read_attr(attr.read_1d::<i16>().map(NcAttrValue::Shorts)),
|
|
124
|
+
(2, false) => read_attr(attr.read_1d::<u16>().map(NcAttrValue::UShorts)),
|
|
125
|
+
(4, true) => read_attr(attr.read_1d::<i32>().map(NcAttrValue::Ints)),
|
|
126
|
+
(4, false) => read_attr(attr.read_1d::<u32>().map(NcAttrValue::UInts)),
|
|
127
|
+
(8, true) => read_attr(attr.read_1d::<i64>().map(NcAttrValue::Int64s)),
|
|
128
|
+
(8, false) => read_attr(attr.read_1d::<u64>().map(NcAttrValue::UInt64s)),
|
|
129
|
+
_ if strict => Err(unsupported()),
|
|
130
|
+
_ => Ok(None),
|
|
131
|
+
},
|
|
132
|
+
Datatype::FloatingPoint { size, .. } => match size {
|
|
133
|
+
4 => read_attr(attr.read_1d::<f32>().map(NcAttrValue::Floats)),
|
|
134
|
+
8 => read_attr(attr.read_1d::<f64>().map(NcAttrValue::Doubles)),
|
|
135
|
+
_ if strict => Err(unsupported()),
|
|
136
|
+
_ => Ok(None),
|
|
137
|
+
},
|
|
138
|
+
Datatype::String { .. } => {
|
|
139
|
+
if attr.shape.is_empty() {
|
|
140
|
+
read_attr(attr.read_string().map(NcAttrValue::Chars))
|
|
141
|
+
} else {
|
|
142
|
+
read_attr(attr.read_strings().map(NcAttrValue::Strings))
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
Datatype::VarLen {
|
|
146
|
+
base,
|
|
147
|
+
kind: VarLenKind::String,
|
|
148
|
+
..
|
|
149
|
+
} if base.is_vlen_string_base() => {
|
|
150
|
+
if attr.shape.is_empty() {
|
|
151
|
+
read_attr(attr.read_string().map(NcAttrValue::Chars))
|
|
152
|
+
} else {
|
|
153
|
+
read_attr(attr.read_strings().map(NcAttrValue::Strings))
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
_ if strict => Err(unsupported()),
|
|
157
|
+
_ => Ok(None),
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
#[cfg(test)]
|
|
162
|
+
mod tests {
|
|
163
|
+
use hdf5_reader::Attribute;
|
|
164
|
+
use hdf5_reader::ByteOrder;
|
|
165
|
+
|
|
166
|
+
use super::*;
|
|
167
|
+
|
|
168
|
+
#[test]
|
|
169
|
+
fn strict_mode_rejects_unsupported_attribute_types() {
|
|
170
|
+
let attr = Attribute {
|
|
171
|
+
name: "opaque_attr".to_string(),
|
|
172
|
+
datatype: Datatype::Opaque {
|
|
173
|
+
size: 4,
|
|
174
|
+
tag: "test".to_string(),
|
|
175
|
+
},
|
|
176
|
+
shape: vec![1],
|
|
177
|
+
raw_data: vec![0, 0, 0, 0],
|
|
178
|
+
decoded_strings: None,
|
|
179
|
+
};
|
|
180
|
+
|
|
181
|
+
let err = convert_attribute_value(&attr, crate::NcMetadataMode::Strict).unwrap_err();
|
|
182
|
+
assert!(matches!(err, Error::InvalidData(_)));
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
#[test]
|
|
186
|
+
fn lossy_mode_skips_unsupported_attribute_types() {
|
|
187
|
+
let attr = Attribute {
|
|
188
|
+
name: "bad_int".to_string(),
|
|
189
|
+
datatype: Datatype::FixedPoint {
|
|
190
|
+
size: 16,
|
|
191
|
+
signed: true,
|
|
192
|
+
byte_order: ByteOrder::LittleEndian,
|
|
193
|
+
},
|
|
194
|
+
shape: vec![1],
|
|
195
|
+
raw_data: vec![0; 16],
|
|
196
|
+
decoded_strings: None,
|
|
197
|
+
};
|
|
198
|
+
|
|
199
|
+
assert!(convert_attribute_value(&attr, crate::NcMetadataMode::Lossy)
|
|
200
|
+
.unwrap()
|
|
201
|
+
.is_none());
|
|
202
|
+
}
|
|
203
|
+
}
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
//! Reconstruct NetCDF-4 dimensions from HDF5 metadata.
|
|
2
|
+
//!
|
|
3
|
+
//! NetCDF-4 stores dimension information as:
|
|
4
|
+
//! - Scale datasets (one per dimension) with `CLASS=DIMENSION_SCALE` attribute
|
|
5
|
+
//! - `DIMENSION_LIST` attribute on each variable dataset referencing the scales
|
|
6
|
+
//! - `REFERENCE_LIST` attribute on scale datasets (back-references)
|
|
7
|
+
//! - `_Netcdf4Dimid` attribute to assign stable dimension IDs
|
|
8
|
+
//! - `NAME` attribute on scale datasets gives the dimension name
|
|
9
|
+
//!
|
|
10
|
+
//! Unlimited dimensions are represented by chunked datasets whose maximum
|
|
11
|
+
//! dimension in the dataspace is `H5S_UNLIMITED`.
|
|
12
|
+
|
|
13
|
+
use std::collections::HashMap;
|
|
14
|
+
|
|
15
|
+
use hdf5_reader::group::Group;
|
|
16
|
+
|
|
17
|
+
use crate::error::{Error, Result};
|
|
18
|
+
use crate::types::NcDimension;
|
|
19
|
+
|
|
20
|
+
fn leaf_name(name: &str) -> &str {
|
|
21
|
+
name.rsplit('/').next().unwrap_or(name)
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
pub(crate) fn is_dimension_without_variable_name(name: &str) -> bool {
|
|
25
|
+
name.starts_with("This is a netCDF dimension but not a netCDF variable")
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/// Extract dimensions from an HDF5 group.
|
|
29
|
+
///
|
|
30
|
+
/// Returns a tuple of:
|
|
31
|
+
/// - The list of dimensions (sorted by `_Netcdf4Dimid` if available)
|
|
32
|
+
/// - A map from dataset object-header address to the corresponding dimension
|
|
33
|
+
///
|
|
34
|
+
/// The address map is used by `extract_variables` to resolve `DIMENSION_LIST`
|
|
35
|
+
/// references back to the correct dimension by address rather than by size.
|
|
36
|
+
pub fn extract_dimensions(
|
|
37
|
+
group: &Group,
|
|
38
|
+
metadata_mode: crate::NcMetadataMode,
|
|
39
|
+
) -> Result<(Vec<NcDimension>, HashMap<u64, NcDimension>)> {
|
|
40
|
+
let datasets = group.datasets()?;
|
|
41
|
+
extract_dimensions_from_datasets(&datasets, metadata_mode)
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
pub fn extract_dimensions_from_datasets(
|
|
45
|
+
datasets: &[hdf5_reader::Dataset],
|
|
46
|
+
metadata_mode: crate::NcMetadataMode,
|
|
47
|
+
) -> Result<(Vec<NcDimension>, HashMap<u64, NcDimension>)> {
|
|
48
|
+
let mut dims: Vec<(Option<i64>, NcDimension, u64)> = Vec::new();
|
|
49
|
+
|
|
50
|
+
for ds in datasets {
|
|
51
|
+
if let Some((dimid, dim, address)) = extract_dimension_entry(ds, metadata_mode)? {
|
|
52
|
+
dims.push((dimid, dim, address));
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// Sort by _Netcdf4Dimid if available, otherwise preserve order
|
|
57
|
+
dims.sort_by_key(|(id, _, _)| id.unwrap_or(i64::MAX));
|
|
58
|
+
|
|
59
|
+
let addr_map: HashMap<u64, NcDimension> =
|
|
60
|
+
dims.iter().map(|(_, d, addr)| (*addr, d.clone())).collect();
|
|
61
|
+
|
|
62
|
+
let dim_list: Vec<NcDimension> = dims.into_iter().map(|(_, d, _)| d).collect();
|
|
63
|
+
|
|
64
|
+
Ok((dim_list, addr_map))
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
fn extract_dimension_entry(
|
|
68
|
+
ds: &hdf5_reader::Dataset,
|
|
69
|
+
metadata_mode: crate::NcMetadataMode,
|
|
70
|
+
) -> Result<Option<(Option<i64>, NcDimension, u64)>> {
|
|
71
|
+
let strict = metadata_mode == crate::NcMetadataMode::Strict;
|
|
72
|
+
|
|
73
|
+
let is_dim_scale = match ds.attribute("CLASS") {
|
|
74
|
+
Ok(attr) => match attr.read_string() {
|
|
75
|
+
Ok(value) => value == "DIMENSION_SCALE",
|
|
76
|
+
Err(err) if strict => {
|
|
77
|
+
return Err(Error::InvalidData(format!(
|
|
78
|
+
"dataset '{}' has unreadable CLASS attribute: {err}",
|
|
79
|
+
ds.name()
|
|
80
|
+
)))
|
|
81
|
+
}
|
|
82
|
+
Err(_) => false,
|
|
83
|
+
},
|
|
84
|
+
Err(_) => false,
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
if !is_dim_scale {
|
|
88
|
+
return Ok(None);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
let name = match ds.attribute("NAME") {
|
|
92
|
+
Ok(attr) => match attr.read_string() {
|
|
93
|
+
Ok(value) => {
|
|
94
|
+
if is_dimension_without_variable_name(&value) {
|
|
95
|
+
leaf_name(ds.name()).to_string()
|
|
96
|
+
} else {
|
|
97
|
+
value
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
Err(err) if strict => {
|
|
101
|
+
return Err(Error::InvalidData(format!(
|
|
102
|
+
"dimension scale '{}' has unreadable NAME attribute: {err}",
|
|
103
|
+
ds.name()
|
|
104
|
+
)))
|
|
105
|
+
}
|
|
106
|
+
Err(_) => leaf_name(ds.name()).to_string(),
|
|
107
|
+
},
|
|
108
|
+
Err(_) => leaf_name(ds.name()).to_string(),
|
|
109
|
+
};
|
|
110
|
+
|
|
111
|
+
let shape = ds.shape();
|
|
112
|
+
let size = if shape.is_empty() { 0 } else { shape[0] };
|
|
113
|
+
let is_unlimited = ds
|
|
114
|
+
.max_dims()
|
|
115
|
+
.is_some_and(|md| !md.is_empty() && md[0] == u64::MAX);
|
|
116
|
+
let dimid = match ds.attribute("_Netcdf4Dimid") {
|
|
117
|
+
Ok(attr) => match attr.read_scalar::<i32>() {
|
|
118
|
+
Ok(id) => Some(id as i64),
|
|
119
|
+
Err(err) if strict => {
|
|
120
|
+
return Err(Error::InvalidData(format!(
|
|
121
|
+
"dimension scale '{}' has unreadable _Netcdf4Dimid attribute: {err}",
|
|
122
|
+
ds.name()
|
|
123
|
+
)))
|
|
124
|
+
}
|
|
125
|
+
Err(_) => None,
|
|
126
|
+
},
|
|
127
|
+
Err(_) => None,
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
Ok(Some((
|
|
131
|
+
dimid,
|
|
132
|
+
NcDimension {
|
|
133
|
+
name,
|
|
134
|
+
size,
|
|
135
|
+
is_unlimited,
|
|
136
|
+
},
|
|
137
|
+
ds.address(),
|
|
138
|
+
)))
|
|
139
|
+
}
|