@nirs4all/formats-wasm 0.2.12 → 0.2.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/THIRD_PARTY_NOTICES.md +49 -0
  2. package/nirs4all_formats_wasm.d.ts +4 -4
  3. package/nirs4all_formats_wasm.js +170 -149
  4. package/nirs4all_formats_wasm_bg.wasm +0 -0
  5. package/package.json +4 -2
  6. package/reader-backports/hdf5-reader/Cargo.toml +119 -0
  7. package/reader-backports/hdf5-reader/LICENSE-APACHE +164 -0
  8. package/reader-backports/hdf5-reader/LICENSE-MIT +21 -0
  9. package/reader-backports/hdf5-reader/PROVENANCE.json +80 -0
  10. package/reader-backports/hdf5-reader/README.md +308 -0
  11. package/reader-backports/hdf5-reader/UPSTREAM-Cargo.toml +44 -0
  12. package/reader-backports/hdf5-reader/UPSTREAM-VCS.json +6 -0
  13. package/reader-backports/hdf5-reader/src/attribute_api.rs +847 -0
  14. package/reader-backports/hdf5-reader/src/btree_v1.rs +726 -0
  15. package/reader-backports/hdf5-reader/src/btree_v2.rs +2068 -0
  16. package/reader-backports/hdf5-reader/src/cache.rs +354 -0
  17. package/reader-backports/hdf5-reader/src/checksum.rs +56 -0
  18. package/reader-backports/hdf5-reader/src/chunk_index.rs +423 -0
  19. package/reader-backports/hdf5-reader/src/dataset.rs +4733 -0
  20. package/reader-backports/hdf5-reader/src/datatype_api.rs +497 -0
  21. package/reader-backports/hdf5-reader/src/error.rs +167 -0
  22. package/reader-backports/hdf5-reader/src/extensible_array.rs +1886 -0
  23. package/reader-backports/hdf5-reader/src/filters/deflate.rs +93 -0
  24. package/reader-backports/hdf5-reader/src/filters/fletcher32.rs +52 -0
  25. package/reader-backports/hdf5-reader/src/filters/lz4.rs +178 -0
  26. package/reader-backports/hdf5-reader/src/filters/mod.rs +326 -0
  27. package/reader-backports/hdf5-reader/src/filters/nbit.rs +586 -0
  28. package/reader-backports/hdf5-reader/src/filters/scaleoffset.rs +827 -0
  29. package/reader-backports/hdf5-reader/src/filters/shuffle.rs +99 -0
  30. package/reader-backports/hdf5-reader/src/fixed_array.rs +763 -0
  31. package/reader-backports/hdf5-reader/src/fractal_heap.rs +2029 -0
  32. package/reader-backports/hdf5-reader/src/global_heap.rs +284 -0
  33. package/reader-backports/hdf5-reader/src/group.rs +795 -0
  34. package/reader-backports/hdf5-reader/src/io.rs +413 -0
  35. package/reader-backports/hdf5-reader/src/lib.rs +916 -0
  36. package/reader-backports/hdf5-reader/src/local_heap.rs +261 -0
  37. package/reader-backports/hdf5-reader/src/messages/attribute.rs +304 -0
  38. package/reader-backports/hdf5-reader/src/messages/attribute_info.rs +96 -0
  39. package/reader-backports/hdf5-reader/src/messages/btree_k.rs +36 -0
  40. package/reader-backports/hdf5-reader/src/messages/continuation.rs +46 -0
  41. package/reader-backports/hdf5-reader/src/messages/dataspace.rs +259 -0
  42. package/reader-backports/hdf5-reader/src/messages/datatype.rs +938 -0
  43. package/reader-backports/hdf5-reader/src/messages/external_files.rs +50 -0
  44. package/reader-backports/hdf5-reader/src/messages/fill_value.rs +233 -0
  45. package/reader-backports/hdf5-reader/src/messages/filter_pipeline.rs +147 -0
  46. package/reader-backports/hdf5-reader/src/messages/group_info.rs +103 -0
  47. package/reader-backports/hdf5-reader/src/messages/layout.rs +476 -0
  48. package/reader-backports/hdf5-reader/src/messages/link.rs +170 -0
  49. package/reader-backports/hdf5-reader/src/messages/link_info.rs +127 -0
  50. package/reader-backports/hdf5-reader/src/messages/mod.rs +374 -0
  51. package/reader-backports/hdf5-reader/src/messages/modification_time.rs +83 -0
  52. package/reader-backports/hdf5-reader/src/messages/shared.rs +164 -0
  53. package/reader-backports/hdf5-reader/src/messages/shared_table.rs +67 -0
  54. package/reader-backports/hdf5-reader/src/messages/symbol_table_msg.rs +51 -0
  55. package/reader-backports/hdf5-reader/src/object_header.rs +1749 -0
  56. package/reader-backports/hdf5-reader/src/reference.rs +90 -0
  57. package/reader-backports/hdf5-reader/src/shared_message_table.rs +425 -0
  58. package/reader-backports/hdf5-reader/src/storage.rs +538 -0
  59. package/reader-backports/hdf5-reader/src/superblock.rs +285 -0
  60. package/reader-backports/hdf5-reader/src/symbol_table.rs +277 -0
  61. package/reader-backports/netcdf-reader/Cargo.toml +141 -0
  62. package/reader-backports/netcdf-reader/LICENSE-APACHE +164 -0
  63. package/reader-backports/netcdf-reader/LICENSE-MIT +21 -0
  64. package/reader-backports/netcdf-reader/PROVENANCE.json +55 -0
  65. package/reader-backports/netcdf-reader/README.md +308 -0
  66. package/reader-backports/netcdf-reader/UPSTREAM-Cargo.toml +48 -0
  67. package/reader-backports/netcdf-reader/UPSTREAM-VCS.json +6 -0
  68. package/reader-backports/netcdf-reader/src/cf/axes.rs +419 -0
  69. package/reader-backports/netcdf-reader/src/cf/bounds.rs +108 -0
  70. package/reader-backports/netcdf-reader/src/cf/crs.rs +269 -0
  71. package/reader-backports/netcdf-reader/src/cf/mod.rs +30 -0
  72. package/reader-backports/netcdf-reader/src/cf/time.rs +1122 -0
  73. package/reader-backports/netcdf-reader/src/classic/data.rs +1465 -0
  74. package/reader-backports/netcdf-reader/src/classic/header.rs +1097 -0
  75. package/reader-backports/netcdf-reader/src/classic/mod.rs +240 -0
  76. package/reader-backports/netcdf-reader/src/classic/storage.rs +211 -0
  77. package/reader-backports/netcdf-reader/src/classic/types.rs +93 -0
  78. package/reader-backports/netcdf-reader/src/classic/variable.rs +1162 -0
  79. package/reader-backports/netcdf-reader/src/error.rs +59 -0
  80. package/reader-backports/netcdf-reader/src/lib.rs +1531 -0
  81. package/reader-backports/netcdf-reader/src/masked.rs +173 -0
  82. package/reader-backports/netcdf-reader/src/nc4/attributes.rs +203 -0
  83. package/reader-backports/netcdf-reader/src/nc4/dimensions.rs +139 -0
  84. package/reader-backports/netcdf-reader/src/nc4/groups.rs +308 -0
  85. package/reader-backports/netcdf-reader/src/nc4/mod.rs +572 -0
  86. package/reader-backports/netcdf-reader/src/nc4/types.rs +309 -0
  87. package/reader-backports/netcdf-reader/src/nc4/variables.rs +420 -0
  88. package/reader-backports/netcdf-reader/src/types.rs +18 -0
  89. package/reader-backports/netcdf-reader/src/unpack.rs +113 -0
  90. package/reader-backports/netcdf-reader/src/user_defined.rs +569 -0
@@ -0,0 +1,309 @@
1
+ //! Map HDF5 datatypes to NetCDF-4 types.
2
+ //!
3
+ //! HDF5 has a richer type system than NetCDF-4. This module maps the subset
4
+ //! of HDF5 types that are valid in NetCDF-4 to `NcType`:
5
+ //!
6
+ //! | HDF5 Datatype | NcType |
7
+ //! |-------------------------------|----------|
8
+ //! | FixedPoint { size=1, signed } | Byte |
9
+ //! | FixedPoint { size=1, !signed} | UByte |
10
+ //! | FixedPoint { size=2, signed } | Short |
11
+ //! | FixedPoint { size=2, !signed} | UShort |
12
+ //! | FixedPoint { size=4, signed } | Int |
13
+ //! | FixedPoint { size=4, !signed} | UInt |
14
+ //! | FixedPoint { size=8, signed } | Int64 |
15
+ //! | FixedPoint { size=8, !signed} | UInt64 |
16
+ //! | FloatingPoint { size=4 } | Float |
17
+ //! | FloatingPoint { size=8 } | Double |
18
+ //! | String (any) | String |
19
+ //! | Enum { base=byte } | Enum |
20
+ //! | Compound { .. } | Compound |
21
+ //! | Opaque { .. } | Opaque |
22
+ //! | Array { base, dims } | Array |
23
+ //! | VarLen { kind=String, base=u8 } | String* |
24
+ //! | VarLen { base } | VLen |
25
+ //!
26
+ //! * Some NetCDF-4 string variables are stored as HDF5 vlen bytes.
27
+
28
+ use hdf5_reader::messages::datatype::{Datatype, StringSize, VarLenKind};
29
+ use hdf5_reader::ByteOrder;
30
+
31
+ use crate::error::{Error, Result};
32
+ use crate::types::{NcCompoundField, NcEnumMember, NcIntegerValue, NcType};
33
+
34
+ /// Map an HDF5 datatype to a NetCDF type.
35
+ pub fn hdf5_to_nc_type(dtype: &Datatype) -> Result<NcType> {
36
+ match dtype {
37
+ Datatype::FixedPoint { size, signed, .. } => match (size, signed) {
38
+ (1, true) => Ok(NcType::Byte),
39
+ (1, false) => Ok(NcType::UByte),
40
+ (2, true) => Ok(NcType::Short),
41
+ (2, false) => Ok(NcType::UShort),
42
+ (4, true) => Ok(NcType::Int),
43
+ (4, false) => Ok(NcType::UInt),
44
+ (8, true) => Ok(NcType::Int64),
45
+ (8, false) => Ok(NcType::UInt64),
46
+ _ => Err(Error::InvalidData(format!(
47
+ "unsupported HDF5 integer size {} for NetCDF-4",
48
+ size
49
+ ))),
50
+ },
51
+ Datatype::FloatingPoint { size, .. } => match size {
52
+ 4 => Ok(NcType::Float),
53
+ 8 => Ok(NcType::Double),
54
+ _ => Err(Error::InvalidData(format!(
55
+ "unsupported HDF5 float size {} for NetCDF-4",
56
+ size
57
+ ))),
58
+ },
59
+ Datatype::String {
60
+ size: StringSize::Fixed(1),
61
+ ..
62
+ } => Ok(NcType::Char),
63
+ Datatype::String { .. } => Ok(NcType::String),
64
+ Datatype::Enum { base, members } => Ok(NcType::Enum {
65
+ base: Box::new(hdf5_to_nc_type(base)?),
66
+ members: members
67
+ .iter()
68
+ .map(|member| {
69
+ Ok(NcEnumMember {
70
+ name: member.name.clone(),
71
+ value: decode_enum_integer(base, &member.value)?,
72
+ })
73
+ })
74
+ .collect::<Result<Vec<_>>>()?,
75
+ }),
76
+ Datatype::Compound { size, fields } => {
77
+ let mut nc_fields = Vec::with_capacity(fields.len());
78
+ for f in fields {
79
+ nc_fields.push(NcCompoundField {
80
+ name: f.name.clone(),
81
+ offset: f.byte_offset as u64,
82
+ dtype: hdf5_to_nc_type(&f.datatype)?,
83
+ });
84
+ }
85
+ Ok(NcType::Compound {
86
+ size: *size,
87
+ fields: nc_fields,
88
+ })
89
+ }
90
+ Datatype::Opaque { size, tag } => Ok(NcType::Opaque {
91
+ size: *size,
92
+ tag: tag.clone(),
93
+ }),
94
+ Datatype::Array { base, dims } => {
95
+ let base_nc = hdf5_to_nc_type(base)?;
96
+ Ok(NcType::Array {
97
+ base: Box::new(base_nc),
98
+ dims: dims.clone(),
99
+ })
100
+ }
101
+ Datatype::VarLen {
102
+ base,
103
+ kind: VarLenKind::String,
104
+ ..
105
+ } if base.is_vlen_string_base() => Ok(NcType::String),
106
+ Datatype::VarLen { base, .. } => {
107
+ let base_nc = hdf5_to_nc_type(base)?;
108
+ Ok(NcType::VLen {
109
+ base: Box::new(base_nc),
110
+ })
111
+ }
112
+ _ => Err(Error::InvalidData(format!(
113
+ "HDF5 datatype {:?} has no NetCDF-4 equivalent",
114
+ dtype
115
+ ))),
116
+ }
117
+ }
118
+
119
+ pub(crate) fn decode_enum_integer(base: &Datatype, bytes: &[u8]) -> Result<NcIntegerValue> {
120
+ match base {
121
+ Datatype::FixedPoint {
122
+ size,
123
+ signed,
124
+ byte_order,
125
+ } => decode_fixed_point_integer(bytes, *size, *signed, *byte_order),
126
+ other => Err(Error::InvalidData(format!(
127
+ "NetCDF-4 enum base type must be integer, got {other:?}"
128
+ ))),
129
+ }
130
+ }
131
+
132
+ pub(crate) fn decode_fixed_point_integer(
133
+ bytes: &[u8],
134
+ size: u8,
135
+ signed: bool,
136
+ byte_order: ByteOrder,
137
+ ) -> Result<NcIntegerValue> {
138
+ fn read<const N: usize>(bytes: &[u8], byte_order: ByteOrder) -> Result<[u8; N]> {
139
+ if bytes.len() < N {
140
+ return Err(Error::InvalidData(format!(
141
+ "integer value too short: need {} bytes, have {}",
142
+ N,
143
+ bytes.len()
144
+ )));
145
+ }
146
+ let mut out = [0u8; N];
147
+ out.copy_from_slice(&bytes[..N]);
148
+ #[cfg(target_endian = "little")]
149
+ if byte_order == ByteOrder::BigEndian {
150
+ out.reverse();
151
+ }
152
+ #[cfg(target_endian = "big")]
153
+ if byte_order == ByteOrder::LittleEndian {
154
+ out.reverse();
155
+ }
156
+ Ok(out)
157
+ }
158
+
159
+ match (size, signed) {
160
+ (1, true) => Ok(NcIntegerValue::I8(i8::from_ne_bytes(read::<1>(
161
+ bytes, byte_order,
162
+ )?))),
163
+ (1, false) => Ok(NcIntegerValue::U8(u8::from_ne_bytes(read::<1>(
164
+ bytes, byte_order,
165
+ )?))),
166
+ (2, true) => Ok(NcIntegerValue::I16(i16::from_ne_bytes(read::<2>(
167
+ bytes, byte_order,
168
+ )?))),
169
+ (2, false) => Ok(NcIntegerValue::U16(u16::from_ne_bytes(read::<2>(
170
+ bytes, byte_order,
171
+ )?))),
172
+ (4, true) => Ok(NcIntegerValue::I32(i32::from_ne_bytes(read::<4>(
173
+ bytes, byte_order,
174
+ )?))),
175
+ (4, false) => Ok(NcIntegerValue::U32(u32::from_ne_bytes(read::<4>(
176
+ bytes, byte_order,
177
+ )?))),
178
+ (8, true) => Ok(NcIntegerValue::I64(i64::from_ne_bytes(read::<8>(
179
+ bytes, byte_order,
180
+ )?))),
181
+ (8, false) => Ok(NcIntegerValue::U64(u64::from_ne_bytes(read::<8>(
182
+ bytes, byte_order,
183
+ )?))),
184
+ _ => Err(Error::InvalidData(format!(
185
+ "unsupported NetCDF-4 enum integer size {size}"
186
+ ))),
187
+ }
188
+ }
189
+
190
+ #[cfg(test)]
191
+ mod tests {
192
+ use super::*;
193
+ use hdf5_reader::error::ByteOrder;
194
+
195
+ #[test]
196
+ fn integer_types() {
197
+ let bo = ByteOrder::LittleEndian;
198
+ assert_eq!(
199
+ hdf5_to_nc_type(&Datatype::FixedPoint {
200
+ size: 1,
201
+ signed: true,
202
+ byte_order: bo
203
+ })
204
+ .unwrap(),
205
+ NcType::Byte
206
+ );
207
+ assert_eq!(
208
+ hdf5_to_nc_type(&Datatype::FixedPoint {
209
+ size: 1,
210
+ signed: false,
211
+ byte_order: bo
212
+ })
213
+ .unwrap(),
214
+ NcType::UByte
215
+ );
216
+ assert_eq!(
217
+ hdf5_to_nc_type(&Datatype::FixedPoint {
218
+ size: 4,
219
+ signed: true,
220
+ byte_order: bo
221
+ })
222
+ .unwrap(),
223
+ NcType::Int
224
+ );
225
+ assert_eq!(
226
+ hdf5_to_nc_type(&Datatype::FixedPoint {
227
+ size: 8,
228
+ signed: false,
229
+ byte_order: bo
230
+ })
231
+ .unwrap(),
232
+ NcType::UInt64
233
+ );
234
+ }
235
+
236
+ #[test]
237
+ fn float_types() {
238
+ let bo = ByteOrder::LittleEndian;
239
+ assert_eq!(
240
+ hdf5_to_nc_type(&Datatype::FloatingPoint {
241
+ size: 4,
242
+ byte_order: bo
243
+ })
244
+ .unwrap(),
245
+ NcType::Float
246
+ );
247
+ assert_eq!(
248
+ hdf5_to_nc_type(&Datatype::FloatingPoint {
249
+ size: 8,
250
+ byte_order: bo
251
+ })
252
+ .unwrap(),
253
+ NcType::Double
254
+ );
255
+ }
256
+
257
+ #[test]
258
+ fn varlen_u8_maps_to_string() {
259
+ let bo = ByteOrder::LittleEndian;
260
+ assert_eq!(
261
+ hdf5_to_nc_type(&Datatype::VarLen {
262
+ base: Box::new(Datatype::FixedPoint {
263
+ size: 1,
264
+ signed: false,
265
+ byte_order: bo,
266
+ }),
267
+ kind: VarLenKind::String,
268
+ encoding: hdf5_reader::StringEncoding::Utf8,
269
+ padding: hdf5_reader::StringPadding::NullTerminate,
270
+ })
271
+ .unwrap(),
272
+ NcType::String
273
+ );
274
+ }
275
+
276
+ #[test]
277
+ fn fixed_one_byte_string_maps_to_char() {
278
+ assert_eq!(
279
+ hdf5_to_nc_type(&Datatype::String {
280
+ size: StringSize::Fixed(1),
281
+ encoding: hdf5_reader::StringEncoding::Ascii,
282
+ padding: hdf5_reader::StringPadding::NullPad,
283
+ })
284
+ .unwrap(),
285
+ NcType::Char
286
+ );
287
+ }
288
+
289
+ #[test]
290
+ fn sequence_varlen_u8_maps_to_vlen() {
291
+ let bo = ByteOrder::LittleEndian;
292
+ assert_eq!(
293
+ hdf5_to_nc_type(&Datatype::VarLen {
294
+ base: Box::new(Datatype::FixedPoint {
295
+ size: 1,
296
+ signed: false,
297
+ byte_order: bo,
298
+ }),
299
+ kind: VarLenKind::Sequence,
300
+ encoding: hdf5_reader::StringEncoding::Ascii,
301
+ padding: hdf5_reader::StringPadding::NullTerminate,
302
+ })
303
+ .unwrap(),
304
+ NcType::VLen {
305
+ base: Box::new(NcType::UByte)
306
+ }
307
+ );
308
+ }
309
+ }