@nirs4all/formats-wasm 0.2.11 → 0.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD_PARTY_NOTICES.md +49 -0
- package/nirs4all_formats_wasm_bg.wasm +0 -0
- package/package.json +4 -2
- package/reader-backports/hdf5-reader/Cargo.toml +119 -0
- package/reader-backports/hdf5-reader/LICENSE-APACHE +164 -0
- package/reader-backports/hdf5-reader/LICENSE-MIT +21 -0
- package/reader-backports/hdf5-reader/PROVENANCE.json +80 -0
- package/reader-backports/hdf5-reader/README.md +308 -0
- package/reader-backports/hdf5-reader/UPSTREAM-Cargo.toml +44 -0
- package/reader-backports/hdf5-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/hdf5-reader/src/attribute_api.rs +847 -0
- package/reader-backports/hdf5-reader/src/btree_v1.rs +726 -0
- package/reader-backports/hdf5-reader/src/btree_v2.rs +2068 -0
- package/reader-backports/hdf5-reader/src/cache.rs +354 -0
- package/reader-backports/hdf5-reader/src/checksum.rs +56 -0
- package/reader-backports/hdf5-reader/src/chunk_index.rs +423 -0
- package/reader-backports/hdf5-reader/src/dataset.rs +4733 -0
- package/reader-backports/hdf5-reader/src/datatype_api.rs +497 -0
- package/reader-backports/hdf5-reader/src/error.rs +167 -0
- package/reader-backports/hdf5-reader/src/extensible_array.rs +1886 -0
- package/reader-backports/hdf5-reader/src/filters/deflate.rs +93 -0
- package/reader-backports/hdf5-reader/src/filters/fletcher32.rs +52 -0
- package/reader-backports/hdf5-reader/src/filters/lz4.rs +178 -0
- package/reader-backports/hdf5-reader/src/filters/mod.rs +326 -0
- package/reader-backports/hdf5-reader/src/filters/nbit.rs +586 -0
- package/reader-backports/hdf5-reader/src/filters/scaleoffset.rs +827 -0
- package/reader-backports/hdf5-reader/src/filters/shuffle.rs +99 -0
- package/reader-backports/hdf5-reader/src/fixed_array.rs +763 -0
- package/reader-backports/hdf5-reader/src/fractal_heap.rs +2029 -0
- package/reader-backports/hdf5-reader/src/global_heap.rs +284 -0
- package/reader-backports/hdf5-reader/src/group.rs +795 -0
- package/reader-backports/hdf5-reader/src/io.rs +413 -0
- package/reader-backports/hdf5-reader/src/lib.rs +916 -0
- package/reader-backports/hdf5-reader/src/local_heap.rs +261 -0
- package/reader-backports/hdf5-reader/src/messages/attribute.rs +304 -0
- package/reader-backports/hdf5-reader/src/messages/attribute_info.rs +96 -0
- package/reader-backports/hdf5-reader/src/messages/btree_k.rs +36 -0
- package/reader-backports/hdf5-reader/src/messages/continuation.rs +46 -0
- package/reader-backports/hdf5-reader/src/messages/dataspace.rs +259 -0
- package/reader-backports/hdf5-reader/src/messages/datatype.rs +938 -0
- package/reader-backports/hdf5-reader/src/messages/external_files.rs +50 -0
- package/reader-backports/hdf5-reader/src/messages/fill_value.rs +233 -0
- package/reader-backports/hdf5-reader/src/messages/filter_pipeline.rs +147 -0
- package/reader-backports/hdf5-reader/src/messages/group_info.rs +103 -0
- package/reader-backports/hdf5-reader/src/messages/layout.rs +476 -0
- package/reader-backports/hdf5-reader/src/messages/link.rs +170 -0
- package/reader-backports/hdf5-reader/src/messages/link_info.rs +127 -0
- package/reader-backports/hdf5-reader/src/messages/mod.rs +374 -0
- package/reader-backports/hdf5-reader/src/messages/modification_time.rs +83 -0
- package/reader-backports/hdf5-reader/src/messages/shared.rs +164 -0
- package/reader-backports/hdf5-reader/src/messages/shared_table.rs +67 -0
- package/reader-backports/hdf5-reader/src/messages/symbol_table_msg.rs +51 -0
- package/reader-backports/hdf5-reader/src/object_header.rs +1749 -0
- package/reader-backports/hdf5-reader/src/reference.rs +90 -0
- package/reader-backports/hdf5-reader/src/shared_message_table.rs +425 -0
- package/reader-backports/hdf5-reader/src/storage.rs +538 -0
- package/reader-backports/hdf5-reader/src/superblock.rs +285 -0
- package/reader-backports/hdf5-reader/src/symbol_table.rs +277 -0
- package/reader-backports/netcdf-reader/Cargo.toml +141 -0
- package/reader-backports/netcdf-reader/LICENSE-APACHE +164 -0
- package/reader-backports/netcdf-reader/LICENSE-MIT +21 -0
- package/reader-backports/netcdf-reader/PROVENANCE.json +55 -0
- package/reader-backports/netcdf-reader/README.md +308 -0
- package/reader-backports/netcdf-reader/UPSTREAM-Cargo.toml +48 -0
- package/reader-backports/netcdf-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/netcdf-reader/src/cf/axes.rs +419 -0
- package/reader-backports/netcdf-reader/src/cf/bounds.rs +108 -0
- package/reader-backports/netcdf-reader/src/cf/crs.rs +269 -0
- package/reader-backports/netcdf-reader/src/cf/mod.rs +30 -0
- package/reader-backports/netcdf-reader/src/cf/time.rs +1122 -0
- package/reader-backports/netcdf-reader/src/classic/data.rs +1465 -0
- package/reader-backports/netcdf-reader/src/classic/header.rs +1097 -0
- package/reader-backports/netcdf-reader/src/classic/mod.rs +240 -0
- package/reader-backports/netcdf-reader/src/classic/storage.rs +211 -0
- package/reader-backports/netcdf-reader/src/classic/types.rs +93 -0
- package/reader-backports/netcdf-reader/src/classic/variable.rs +1162 -0
- package/reader-backports/netcdf-reader/src/error.rs +59 -0
- package/reader-backports/netcdf-reader/src/lib.rs +1531 -0
- package/reader-backports/netcdf-reader/src/masked.rs +173 -0
- package/reader-backports/netcdf-reader/src/nc4/attributes.rs +203 -0
- package/reader-backports/netcdf-reader/src/nc4/dimensions.rs +139 -0
- package/reader-backports/netcdf-reader/src/nc4/groups.rs +308 -0
- package/reader-backports/netcdf-reader/src/nc4/mod.rs +572 -0
- package/reader-backports/netcdf-reader/src/nc4/types.rs +309 -0
- package/reader-backports/netcdf-reader/src/nc4/variables.rs +420 -0
- package/reader-backports/netcdf-reader/src/types.rs +18 -0
- package/reader-backports/netcdf-reader/src/unpack.rs +113 -0
- package/reader-backports/netcdf-reader/src/user_defined.rs +569 -0
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
//! Map HDF5 groups to NetCDF-4 groups.
|
|
2
|
+
//!
|
|
3
|
+
//! Each HDF5 group becomes an `NcGroup`. The root group is special: it may
|
|
4
|
+
//! contain `_NCProperties` and other internal attributes that should be filtered.
|
|
5
|
+
//! Sub-groups are traversed recursively.
|
|
6
|
+
|
|
7
|
+
use std::collections::HashMap;
|
|
8
|
+
|
|
9
|
+
use hdf5_reader::Hdf5File;
|
|
10
|
+
|
|
11
|
+
use crate::error::{Error, Result};
|
|
12
|
+
use crate::types::{NcDimension, NcGroup};
|
|
13
|
+
|
|
14
|
+
use super::attributes;
|
|
15
|
+
use super::dimensions;
|
|
16
|
+
use super::variables;
|
|
17
|
+
|
|
18
|
+
#[allow(dead_code)]
|
|
19
|
+
pub(crate) struct GroupContext {
|
|
20
|
+
pub(crate) group: hdf5_reader::group::Group,
|
|
21
|
+
pub(crate) visible_dimensions: Vec<NcDimension>,
|
|
22
|
+
pub(crate) visible_dim_addr_map: HashMap<u64, NcDimension>,
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
fn leaf_name(name: &str) -> &str {
|
|
26
|
+
name.rsplit('/').next().unwrap_or(name)
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
fn visible_dimensions(
|
|
30
|
+
local_dimensions: &[NcDimension],
|
|
31
|
+
inherited_dimensions: &[NcDimension],
|
|
32
|
+
) -> Vec<NcDimension> {
|
|
33
|
+
let mut visible_dimensions = local_dimensions.to_vec();
|
|
34
|
+
visible_dimensions.extend(
|
|
35
|
+
inherited_dimensions
|
|
36
|
+
.iter()
|
|
37
|
+
.filter(|dim| {
|
|
38
|
+
!local_dimensions
|
|
39
|
+
.iter()
|
|
40
|
+
.any(|local_dim| local_dim.name == dim.name)
|
|
41
|
+
})
|
|
42
|
+
.cloned(),
|
|
43
|
+
);
|
|
44
|
+
visible_dimensions
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
fn visible_dim_addr_map(
|
|
48
|
+
local_dim_addr_map: HashMap<u64, NcDimension>,
|
|
49
|
+
inherited_dim_addr_map: &HashMap<u64, NcDimension>,
|
|
50
|
+
) -> HashMap<u64, NcDimension> {
|
|
51
|
+
let mut visible_dim_addr_map = inherited_dim_addr_map.clone();
|
|
52
|
+
visible_dim_addr_map.extend(local_dim_addr_map);
|
|
53
|
+
visible_dim_addr_map
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/// Build the root NcGroup from an HDF5 file.
|
|
57
|
+
pub fn build_root_group(hdf5: &Hdf5File, metadata_mode: crate::NcMetadataMode) -> Result<NcGroup> {
|
|
58
|
+
build_group_at_path(hdf5, "/", metadata_mode, true)
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/// Build only the root group's local metadata.
|
|
62
|
+
pub fn build_root_group_metadata(
|
|
63
|
+
hdf5: &Hdf5File,
|
|
64
|
+
metadata_mode: crate::NcMetadataMode,
|
|
65
|
+
) -> Result<NcGroup> {
|
|
66
|
+
build_group_at_path(hdf5, "/", metadata_mode, false)
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/// Build metadata for a group path.
|
|
70
|
+
pub fn build_group_at_path(
|
|
71
|
+
hdf5: &Hdf5File,
|
|
72
|
+
path: &str,
|
|
73
|
+
metadata_mode: crate::NcMetadataMode,
|
|
74
|
+
recursive: bool,
|
|
75
|
+
) -> Result<NcGroup> {
|
|
76
|
+
let normalized = normalize_group_path(path);
|
|
77
|
+
let root = hdf5.root_group()?;
|
|
78
|
+
let mut group = root;
|
|
79
|
+
let mut inherited_dimensions = Vec::new();
|
|
80
|
+
let mut inherited_dim_addr_map = HashMap::new();
|
|
81
|
+
|
|
82
|
+
for component in normalized.split('/').filter(|part| !part.is_empty()) {
|
|
83
|
+
let datasets = group.datasets()?;
|
|
84
|
+
let (local_dimensions, local_dim_addr_map) =
|
|
85
|
+
dimensions::extract_dimensions_from_datasets(&datasets, metadata_mode)?;
|
|
86
|
+
inherited_dimensions = visible_dimensions(&local_dimensions, &inherited_dimensions);
|
|
87
|
+
inherited_dim_addr_map = visible_dim_addr_map(local_dim_addr_map, &inherited_dim_addr_map);
|
|
88
|
+
group = group
|
|
89
|
+
.group(component)
|
|
90
|
+
.map_err(|_| Error::GroupNotFound(path.to_string()))?;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
let group_name = if normalized.is_empty() {
|
|
94
|
+
"/".to_string()
|
|
95
|
+
} else {
|
|
96
|
+
leaf_name(group.name()).to_string()
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
if recursive {
|
|
100
|
+
build_group_recursive(
|
|
101
|
+
&group,
|
|
102
|
+
&group_name,
|
|
103
|
+
&inherited_dimensions,
|
|
104
|
+
&inherited_dim_addr_map,
|
|
105
|
+
metadata_mode,
|
|
106
|
+
)
|
|
107
|
+
} else {
|
|
108
|
+
build_group_metadata(
|
|
109
|
+
&group,
|
|
110
|
+
&group_name,
|
|
111
|
+
&inherited_dimensions,
|
|
112
|
+
&inherited_dim_addr_map,
|
|
113
|
+
metadata_mode,
|
|
114
|
+
)
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
#[allow(dead_code)]
|
|
119
|
+
pub(crate) fn group_context_at_path(
|
|
120
|
+
hdf5: &Hdf5File,
|
|
121
|
+
path: &str,
|
|
122
|
+
metadata_mode: crate::NcMetadataMode,
|
|
123
|
+
) -> Result<GroupContext> {
|
|
124
|
+
let normalized = normalize_group_path(path);
|
|
125
|
+
let root = hdf5.root_group()?;
|
|
126
|
+
let mut group = root;
|
|
127
|
+
let mut inherited_dimensions = Vec::new();
|
|
128
|
+
let mut inherited_dim_addr_map = HashMap::new();
|
|
129
|
+
|
|
130
|
+
for component in normalized.split('/').filter(|part| !part.is_empty()) {
|
|
131
|
+
let datasets = group.datasets()?;
|
|
132
|
+
let (local_dimensions, local_dim_addr_map) =
|
|
133
|
+
dimensions::extract_dimensions_from_datasets(&datasets, metadata_mode)?;
|
|
134
|
+
inherited_dimensions = visible_dimensions(&local_dimensions, &inherited_dimensions);
|
|
135
|
+
inherited_dim_addr_map = visible_dim_addr_map(local_dim_addr_map, &inherited_dim_addr_map);
|
|
136
|
+
group = group
|
|
137
|
+
.group(component)
|
|
138
|
+
.map_err(|_| Error::GroupNotFound(path.to_string()))?;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
let datasets = group.datasets()?;
|
|
142
|
+
let (local_dimensions, local_dim_addr_map) =
|
|
143
|
+
dimensions::extract_dimensions_from_datasets(&datasets, metadata_mode)?;
|
|
144
|
+
let visible_dimensions = visible_dimensions(&local_dimensions, &inherited_dimensions);
|
|
145
|
+
let visible_dim_addr_map = visible_dim_addr_map(local_dim_addr_map, &inherited_dim_addr_map);
|
|
146
|
+
|
|
147
|
+
Ok(GroupContext {
|
|
148
|
+
group,
|
|
149
|
+
visible_dimensions,
|
|
150
|
+
visible_dim_addr_map,
|
|
151
|
+
})
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/// Recursively build an NcGroup from an HDF5 Group.
|
|
155
|
+
fn build_group_recursive(
|
|
156
|
+
hdf5_group: &hdf5_reader::group::Group,
|
|
157
|
+
name: &str,
|
|
158
|
+
inherited_dimensions: &[NcDimension],
|
|
159
|
+
inherited_dim_addr_map: &HashMap<u64, NcDimension>,
|
|
160
|
+
metadata_mode: crate::NcMetadataMode,
|
|
161
|
+
) -> Result<NcGroup> {
|
|
162
|
+
let (hdf5_children, datasets) = hdf5_group.members()?;
|
|
163
|
+
|
|
164
|
+
// Extract dimensions declared locally in this group, then combine them
|
|
165
|
+
// with dimensions inherited from ancestor groups for lookups and variable
|
|
166
|
+
// reconstruction.
|
|
167
|
+
let (local_dimensions, local_dim_addr_map) =
|
|
168
|
+
dimensions::extract_dimensions_from_datasets(&datasets, metadata_mode)?;
|
|
169
|
+
let visible_dimensions = visible_dimensions(&local_dimensions, inherited_dimensions);
|
|
170
|
+
let visible_dim_addr_map = visible_dim_addr_map(local_dim_addr_map, inherited_dim_addr_map);
|
|
171
|
+
|
|
172
|
+
// Extract variables (non-dimension-scale datasets).
|
|
173
|
+
let variables = variables::extract_variables_from_datasets(
|
|
174
|
+
&datasets,
|
|
175
|
+
hdf5_group,
|
|
176
|
+
&visible_dimensions,
|
|
177
|
+
&visible_dim_addr_map,
|
|
178
|
+
metadata_mode,
|
|
179
|
+
)?;
|
|
180
|
+
|
|
181
|
+
// Extract group-level attributes, filtering internal NetCDF-4 attributes.
|
|
182
|
+
let nc_attributes = attributes::extract_group_attributes(hdf5_group, metadata_mode)?;
|
|
183
|
+
|
|
184
|
+
// Recurse into child groups.
|
|
185
|
+
let mut child_groups = Vec::new();
|
|
186
|
+
for child in &hdf5_children {
|
|
187
|
+
let child_name = leaf_name(child.name()).to_string();
|
|
188
|
+
let nc_child = build_group_recursive(
|
|
189
|
+
child,
|
|
190
|
+
&child_name,
|
|
191
|
+
&visible_dimensions,
|
|
192
|
+
&visible_dim_addr_map,
|
|
193
|
+
metadata_mode,
|
|
194
|
+
)?;
|
|
195
|
+
child_groups.push(nc_child);
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
Ok(NcGroup {
|
|
199
|
+
name: name.to_string(),
|
|
200
|
+
dimensions: visible_dimensions,
|
|
201
|
+
variables,
|
|
202
|
+
attributes: nc_attributes,
|
|
203
|
+
groups: child_groups,
|
|
204
|
+
})
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
fn build_group_metadata(
|
|
208
|
+
hdf5_group: &hdf5_reader::group::Group,
|
|
209
|
+
name: &str,
|
|
210
|
+
inherited_dimensions: &[NcDimension],
|
|
211
|
+
inherited_dim_addr_map: &HashMap<u64, NcDimension>,
|
|
212
|
+
metadata_mode: crate::NcMetadataMode,
|
|
213
|
+
) -> Result<NcGroup> {
|
|
214
|
+
let datasets = hdf5_group.datasets()?;
|
|
215
|
+
let (local_dimensions, local_dim_addr_map) =
|
|
216
|
+
dimensions::extract_dimensions_from_datasets(&datasets, metadata_mode)?;
|
|
217
|
+
let visible_dimensions = visible_dimensions(&local_dimensions, inherited_dimensions);
|
|
218
|
+
let visible_dim_addr_map = visible_dim_addr_map(local_dim_addr_map, inherited_dim_addr_map);
|
|
219
|
+
let variables = variables::extract_variables_from_datasets(
|
|
220
|
+
&datasets,
|
|
221
|
+
hdf5_group,
|
|
222
|
+
&visible_dimensions,
|
|
223
|
+
&visible_dim_addr_map,
|
|
224
|
+
metadata_mode,
|
|
225
|
+
)?;
|
|
226
|
+
let attributes = attributes::extract_group_attributes(hdf5_group, metadata_mode)?;
|
|
227
|
+
|
|
228
|
+
Ok(NcGroup {
|
|
229
|
+
name: name.to_string(),
|
|
230
|
+
dimensions: visible_dimensions,
|
|
231
|
+
variables,
|
|
232
|
+
attributes,
|
|
233
|
+
groups: Vec::new(),
|
|
234
|
+
})
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
fn normalize_group_path(path: &str) -> &str {
|
|
238
|
+
path.trim_matches('/')
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
#[cfg(test)]
|
|
242
|
+
mod tests {
|
|
243
|
+
use super::*;
|
|
244
|
+
|
|
245
|
+
#[test]
|
|
246
|
+
fn visible_dimensions_include_inherited_without_duplicates() {
|
|
247
|
+
let local = vec![NcDimension {
|
|
248
|
+
name: "y".to_string(),
|
|
249
|
+
size: 4,
|
|
250
|
+
is_unlimited: false,
|
|
251
|
+
}];
|
|
252
|
+
let inherited = vec![
|
|
253
|
+
NcDimension {
|
|
254
|
+
name: "x".to_string(),
|
|
255
|
+
size: 3,
|
|
256
|
+
is_unlimited: false,
|
|
257
|
+
},
|
|
258
|
+
NcDimension {
|
|
259
|
+
name: "y".to_string(),
|
|
260
|
+
size: 99,
|
|
261
|
+
is_unlimited: true,
|
|
262
|
+
},
|
|
263
|
+
];
|
|
264
|
+
|
|
265
|
+
let merged = visible_dimensions(&local, &inherited);
|
|
266
|
+
let names: Vec<&str> = merged.iter().map(|dim| dim.name.as_str()).collect();
|
|
267
|
+
assert_eq!(names, vec!["y", "x"]);
|
|
268
|
+
assert_eq!(merged[0].size, 4);
|
|
269
|
+
assert!(!merged[0].is_unlimited);
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
#[test]
|
|
273
|
+
fn visible_dim_addr_map_prefers_local_dimensions() {
|
|
274
|
+
let mut inherited = HashMap::new();
|
|
275
|
+
inherited.insert(
|
|
276
|
+
10,
|
|
277
|
+
NcDimension {
|
|
278
|
+
name: "x".to_string(),
|
|
279
|
+
size: 3,
|
|
280
|
+
is_unlimited: false,
|
|
281
|
+
},
|
|
282
|
+
);
|
|
283
|
+
inherited.insert(
|
|
284
|
+
20,
|
|
285
|
+
NcDimension {
|
|
286
|
+
name: "shared".to_string(),
|
|
287
|
+
size: 1,
|
|
288
|
+
is_unlimited: false,
|
|
289
|
+
},
|
|
290
|
+
);
|
|
291
|
+
|
|
292
|
+
let mut local = HashMap::new();
|
|
293
|
+
local.insert(
|
|
294
|
+
20,
|
|
295
|
+
NcDimension {
|
|
296
|
+
name: "shared".to_string(),
|
|
297
|
+
size: 2,
|
|
298
|
+
is_unlimited: true,
|
|
299
|
+
},
|
|
300
|
+
);
|
|
301
|
+
|
|
302
|
+
let merged = visible_dim_addr_map(local, &inherited);
|
|
303
|
+
assert_eq!(merged.len(), 2);
|
|
304
|
+
assert_eq!(merged.get(&10).unwrap().name, "x");
|
|
305
|
+
assert_eq!(merged.get(&20).unwrap().size, 2);
|
|
306
|
+
assert!(merged.get(&20).unwrap().is_unlimited);
|
|
307
|
+
}
|
|
308
|
+
}
|