@nirs4all/formats-wasm 0.2.12 → 0.2.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/THIRD_PARTY_NOTICES.md +49 -0
  2. package/nirs4all_formats_wasm.d.ts +4 -4
  3. package/nirs4all_formats_wasm.js +170 -149
  4. package/nirs4all_formats_wasm_bg.wasm +0 -0
  5. package/package.json +4 -2
  6. package/reader-backports/hdf5-reader/Cargo.toml +119 -0
  7. package/reader-backports/hdf5-reader/LICENSE-APACHE +164 -0
  8. package/reader-backports/hdf5-reader/LICENSE-MIT +21 -0
  9. package/reader-backports/hdf5-reader/PROVENANCE.json +80 -0
  10. package/reader-backports/hdf5-reader/README.md +308 -0
  11. package/reader-backports/hdf5-reader/UPSTREAM-Cargo.toml +44 -0
  12. package/reader-backports/hdf5-reader/UPSTREAM-VCS.json +6 -0
  13. package/reader-backports/hdf5-reader/src/attribute_api.rs +847 -0
  14. package/reader-backports/hdf5-reader/src/btree_v1.rs +726 -0
  15. package/reader-backports/hdf5-reader/src/btree_v2.rs +2068 -0
  16. package/reader-backports/hdf5-reader/src/cache.rs +354 -0
  17. package/reader-backports/hdf5-reader/src/checksum.rs +56 -0
  18. package/reader-backports/hdf5-reader/src/chunk_index.rs +423 -0
  19. package/reader-backports/hdf5-reader/src/dataset.rs +4733 -0
  20. package/reader-backports/hdf5-reader/src/datatype_api.rs +497 -0
  21. package/reader-backports/hdf5-reader/src/error.rs +167 -0
  22. package/reader-backports/hdf5-reader/src/extensible_array.rs +1886 -0
  23. package/reader-backports/hdf5-reader/src/filters/deflate.rs +93 -0
  24. package/reader-backports/hdf5-reader/src/filters/fletcher32.rs +52 -0
  25. package/reader-backports/hdf5-reader/src/filters/lz4.rs +178 -0
  26. package/reader-backports/hdf5-reader/src/filters/mod.rs +326 -0
  27. package/reader-backports/hdf5-reader/src/filters/nbit.rs +586 -0
  28. package/reader-backports/hdf5-reader/src/filters/scaleoffset.rs +827 -0
  29. package/reader-backports/hdf5-reader/src/filters/shuffle.rs +99 -0
  30. package/reader-backports/hdf5-reader/src/fixed_array.rs +763 -0
  31. package/reader-backports/hdf5-reader/src/fractal_heap.rs +2029 -0
  32. package/reader-backports/hdf5-reader/src/global_heap.rs +284 -0
  33. package/reader-backports/hdf5-reader/src/group.rs +795 -0
  34. package/reader-backports/hdf5-reader/src/io.rs +413 -0
  35. package/reader-backports/hdf5-reader/src/lib.rs +916 -0
  36. package/reader-backports/hdf5-reader/src/local_heap.rs +261 -0
  37. package/reader-backports/hdf5-reader/src/messages/attribute.rs +304 -0
  38. package/reader-backports/hdf5-reader/src/messages/attribute_info.rs +96 -0
  39. package/reader-backports/hdf5-reader/src/messages/btree_k.rs +36 -0
  40. package/reader-backports/hdf5-reader/src/messages/continuation.rs +46 -0
  41. package/reader-backports/hdf5-reader/src/messages/dataspace.rs +259 -0
  42. package/reader-backports/hdf5-reader/src/messages/datatype.rs +938 -0
  43. package/reader-backports/hdf5-reader/src/messages/external_files.rs +50 -0
  44. package/reader-backports/hdf5-reader/src/messages/fill_value.rs +233 -0
  45. package/reader-backports/hdf5-reader/src/messages/filter_pipeline.rs +147 -0
  46. package/reader-backports/hdf5-reader/src/messages/group_info.rs +103 -0
  47. package/reader-backports/hdf5-reader/src/messages/layout.rs +476 -0
  48. package/reader-backports/hdf5-reader/src/messages/link.rs +170 -0
  49. package/reader-backports/hdf5-reader/src/messages/link_info.rs +127 -0
  50. package/reader-backports/hdf5-reader/src/messages/mod.rs +374 -0
  51. package/reader-backports/hdf5-reader/src/messages/modification_time.rs +83 -0
  52. package/reader-backports/hdf5-reader/src/messages/shared.rs +164 -0
  53. package/reader-backports/hdf5-reader/src/messages/shared_table.rs +67 -0
  54. package/reader-backports/hdf5-reader/src/messages/symbol_table_msg.rs +51 -0
  55. package/reader-backports/hdf5-reader/src/object_header.rs +1749 -0
  56. package/reader-backports/hdf5-reader/src/reference.rs +90 -0
  57. package/reader-backports/hdf5-reader/src/shared_message_table.rs +425 -0
  58. package/reader-backports/hdf5-reader/src/storage.rs +538 -0
  59. package/reader-backports/hdf5-reader/src/superblock.rs +285 -0
  60. package/reader-backports/hdf5-reader/src/symbol_table.rs +277 -0
  61. package/reader-backports/netcdf-reader/Cargo.toml +141 -0
  62. package/reader-backports/netcdf-reader/LICENSE-APACHE +164 -0
  63. package/reader-backports/netcdf-reader/LICENSE-MIT +21 -0
  64. package/reader-backports/netcdf-reader/PROVENANCE.json +55 -0
  65. package/reader-backports/netcdf-reader/README.md +308 -0
  66. package/reader-backports/netcdf-reader/UPSTREAM-Cargo.toml +48 -0
  67. package/reader-backports/netcdf-reader/UPSTREAM-VCS.json +6 -0
  68. package/reader-backports/netcdf-reader/src/cf/axes.rs +419 -0
  69. package/reader-backports/netcdf-reader/src/cf/bounds.rs +108 -0
  70. package/reader-backports/netcdf-reader/src/cf/crs.rs +269 -0
  71. package/reader-backports/netcdf-reader/src/cf/mod.rs +30 -0
  72. package/reader-backports/netcdf-reader/src/cf/time.rs +1122 -0
  73. package/reader-backports/netcdf-reader/src/classic/data.rs +1465 -0
  74. package/reader-backports/netcdf-reader/src/classic/header.rs +1097 -0
  75. package/reader-backports/netcdf-reader/src/classic/mod.rs +240 -0
  76. package/reader-backports/netcdf-reader/src/classic/storage.rs +211 -0
  77. package/reader-backports/netcdf-reader/src/classic/types.rs +93 -0
  78. package/reader-backports/netcdf-reader/src/classic/variable.rs +1162 -0
  79. package/reader-backports/netcdf-reader/src/error.rs +59 -0
  80. package/reader-backports/netcdf-reader/src/lib.rs +1531 -0
  81. package/reader-backports/netcdf-reader/src/masked.rs +173 -0
  82. package/reader-backports/netcdf-reader/src/nc4/attributes.rs +203 -0
  83. package/reader-backports/netcdf-reader/src/nc4/dimensions.rs +139 -0
  84. package/reader-backports/netcdf-reader/src/nc4/groups.rs +308 -0
  85. package/reader-backports/netcdf-reader/src/nc4/mod.rs +572 -0
  86. package/reader-backports/netcdf-reader/src/nc4/types.rs +309 -0
  87. package/reader-backports/netcdf-reader/src/nc4/variables.rs +420 -0
  88. package/reader-backports/netcdf-reader/src/types.rs +18 -0
  89. package/reader-backports/netcdf-reader/src/unpack.rs +113 -0
  90. package/reader-backports/netcdf-reader/src/user_defined.rs +569 -0
@@ -0,0 +1,938 @@
1
+ //! HDF5 Datatype message (type 0x0003).
2
+ //!
3
+ //! The datatype message describes the type of each element in a dataset.
4
+ //! The first 4 bytes encode class (bits 0-3), version (bits 4-7), and
5
+ //! class-specific bit flags (bits 8-31). The remaining bytes carry
6
+ //! class-specific properties.
7
+ //!
8
+ //! Supported classes:
9
+ //! - 0: Fixed-point (integer)
10
+ //! - 1: Floating-point
11
+ //! - 2: Time (treated as opaque)
12
+ //! - 3: String
13
+ //! - 4: Bitfield
14
+ //! - 5: Opaque
15
+ //! - 6: Compound
16
+ //! - 7: Reference
17
+ //! - 8: Enum
18
+ //! - 9: Variable-length
19
+ //! - 10: Array
20
+
21
+ use crate::error::{ByteOrder, Error, Result};
22
+ use crate::io::Cursor;
23
+ pub use hdf5_core::{
24
+ CompoundField, Datatype, DatatypeMessage, EnumMember, ReferenceType, StringEncoding,
25
+ StringPadding, StringSize, VarLenKind,
26
+ };
27
+
28
+ // ---------------------------------------------------------------------------
29
+ // Parsing
30
+ // ---------------------------------------------------------------------------
31
+
32
+ /// Parse a datatype message starting at the current cursor position.
33
+ ///
34
+ /// The `msg_size` is the total number of bytes allocated for this message
35
+ /// (used to skip any trailing padding).
36
+ pub fn parse(cursor: &mut Cursor<'_>, msg_size: usize) -> Result<DatatypeMessage> {
37
+ let start = cursor.position();
38
+ let (dt, size) = parse_datatype_description(cursor)?;
39
+
40
+ let consumed = (cursor.position() - start) as usize;
41
+ if consumed < msg_size {
42
+ cursor.skip(msg_size - consumed)?;
43
+ }
44
+
45
+ Ok(DatatypeMessage { datatype: dt, size })
46
+ }
47
+
48
+ /// Maximum nesting depth for composite datatypes (compound members, enum,
49
+ /// variable-length, and array bases). A malformed file cannot recurse the
50
+ /// parser into a stack overflow.
51
+ const MAX_DATATYPE_NESTING_DEPTH: usize = 32;
52
+
53
+ /// Parse a single datatype description (the 4-byte header + properties).
54
+ ///
55
+ /// This is also called recursively for compound members, arrays, enums, etc.
56
+ pub fn parse_datatype_description(cursor: &mut Cursor<'_>) -> Result<(Datatype, u32)> {
57
+ parse_datatype_description_at_depth(cursor, 0)
58
+ }
59
+
60
+ fn parse_datatype_description_at_depth(
61
+ cursor: &mut Cursor<'_>,
62
+ depth: usize,
63
+ ) -> Result<(Datatype, u32)> {
64
+ if depth > MAX_DATATYPE_NESTING_DEPTH {
65
+ return Err(Error::InvalidData(format!(
66
+ "datatype nesting exceeds {MAX_DATATYPE_NESTING_DEPTH} levels"
67
+ )));
68
+ }
69
+ let class_and_flags = cursor.read_u32_le()?;
70
+ let class = (class_and_flags & 0x0F) as u8;
71
+ let version = ((class_and_flags >> 4) & 0x0F) as u8;
72
+ let class_flags = class_and_flags >> 8; // upper 24 bits
73
+ let size = cursor.read_u32_le()?;
74
+
75
+ let dt = match class {
76
+ 0 => parse_fixed_point(cursor, class_flags, size)?,
77
+ 1 => parse_floating_point(cursor, class_flags, size)?,
78
+ 2 => parse_time(cursor, size)?,
79
+ 3 => parse_string(class_flags, size)?,
80
+ 4 => parse_bitfield(cursor, class_flags, size)?,
81
+ 5 => parse_opaque(cursor, class_flags, size)?,
82
+ 6 => parse_compound(cursor, class_flags, size, version, depth)?,
83
+ 7 => parse_reference(class_flags, size)?,
84
+ 8 => parse_enum(cursor, class_flags, size, depth)?,
85
+ 9 => parse_varlen(cursor, class_flags, size, depth)?,
86
+ 10 => parse_array(cursor, size, version, depth)?,
87
+ c => return Err(Error::UnsupportedDatatypeClass(c)),
88
+ };
89
+
90
+ Ok((dt, size))
91
+ }
92
+
93
+ // ---------------------------------------------------------------------------
94
+ // Class 0: Fixed-point (integer)
95
+ // ---------------------------------------------------------------------------
96
+
97
+ fn parse_fixed_point(cursor: &mut Cursor<'_>, flags: u32, size: u32) -> Result<Datatype> {
98
+ // Bit 0 of class flags: byte order (0 = LE, 1 = BE)
99
+ let byte_order = if (flags & 0x01) != 0 {
100
+ ByteOrder::BigEndian
101
+ } else {
102
+ ByteOrder::LittleEndian
103
+ };
104
+ // Bit 3: signed (0 = unsigned, 1 = signed)
105
+ let signed = (flags & 0x08) != 0;
106
+
107
+ // Properties: bit offset (u16) + bit precision (u16)
108
+ let _bit_offset = cursor.read_u16_le()?;
109
+ let _bit_precision = cursor.read_u16_le()?;
110
+
111
+ Ok(Datatype::FixedPoint {
112
+ size: size as u8,
113
+ signed,
114
+ byte_order,
115
+ })
116
+ }
117
+
118
+ // ---------------------------------------------------------------------------
119
+ // Class 1: Floating-point
120
+ // ---------------------------------------------------------------------------
121
+
122
+ fn parse_floating_point(cursor: &mut Cursor<'_>, flags: u32, size: u32) -> Result<Datatype> {
123
+ // Byte order: bit 0 low-order, bit 6 high-order
124
+ // 00 = LE, 01 = BE, 10 = VAX (treated as LE for our purposes)
125
+ let bo_lo = flags & 0x01;
126
+ let bo_hi = (flags >> 6) & 0x01;
127
+ let byte_order = match (bo_hi, bo_lo) {
128
+ (0, 0) => ByteOrder::LittleEndian,
129
+ (0, 1) => ByteOrder::BigEndian,
130
+ // VAX order — map to little endian (close enough for decoding)
131
+ _ => ByteOrder::LittleEndian,
132
+ };
133
+
134
+ // Properties: 12 bytes
135
+ // bit offset (u16), bit precision (u16), exponent location (u8),
136
+ // exponent size (u8), mantissa location (u8), mantissa size (u8),
137
+ // exponent bias (u32)
138
+ let _bit_offset = cursor.read_u16_le()?;
139
+ let _bit_precision = cursor.read_u16_le()?;
140
+ let _exp_location = cursor.read_u8()?;
141
+ let _exp_size = cursor.read_u8()?;
142
+ let _mant_location = cursor.read_u8()?;
143
+ let _mant_size = cursor.read_u8()?;
144
+ let _exp_bias = cursor.read_u32_le()?;
145
+
146
+ Ok(Datatype::FloatingPoint {
147
+ size: size as u8,
148
+ byte_order,
149
+ })
150
+ }
151
+
152
+ // ---------------------------------------------------------------------------
153
+ // Class 2: Time (rarely used, treat as opaque)
154
+ // ---------------------------------------------------------------------------
155
+
156
+ fn parse_time(cursor: &mut Cursor<'_>, size: u32) -> Result<Datatype> {
157
+ // Properties: bit precision (u16)
158
+ let _bit_precision = cursor.read_u16_le()?;
159
+ Ok(Datatype::Opaque {
160
+ size,
161
+ tag: "HDF5_TIME".to_string(),
162
+ })
163
+ }
164
+
165
+ // ---------------------------------------------------------------------------
166
+ // Class 3: String
167
+ // ---------------------------------------------------------------------------
168
+
169
+ fn parse_string(flags: u32, size: u32) -> Result<Datatype> {
170
+ // Bits 0-3: padding type
171
+ let padding = string_padding_from_bits(flags & 0x0F);
172
+
173
+ // Bits 4-7: character set
174
+ let encoding = string_encoding_from_bits((flags >> 4) & 0x0F);
175
+
176
+ // No additional property bytes for string class.
177
+
178
+ let string_size = if size == 0 {
179
+ // Size 0 can indicate variable-length when used with vlen wrapper,
180
+ // but for the string class itself we treat it as Variable.
181
+ StringSize::Variable
182
+ } else {
183
+ StringSize::Fixed(size)
184
+ };
185
+
186
+ Ok(Datatype::String {
187
+ size: string_size,
188
+ encoding,
189
+ padding,
190
+ })
191
+ }
192
+
193
+ // ---------------------------------------------------------------------------
194
+ // Class 4: Bitfield
195
+ // ---------------------------------------------------------------------------
196
+
197
+ fn parse_bitfield(cursor: &mut Cursor<'_>, flags: u32, size: u32) -> Result<Datatype> {
198
+ let byte_order = if (flags & 0x01) != 0 {
199
+ ByteOrder::BigEndian
200
+ } else {
201
+ ByteOrder::LittleEndian
202
+ };
203
+
204
+ // Properties: bit offset (u16) + bit precision (u16)
205
+ let _bit_offset = cursor.read_u16_le()?;
206
+ let _bit_precision = cursor.read_u16_le()?;
207
+
208
+ Ok(Datatype::Bitfield {
209
+ size: size as u8,
210
+ byte_order,
211
+ })
212
+ }
213
+
214
+ // ---------------------------------------------------------------------------
215
+ // Class 5: Opaque
216
+ // ---------------------------------------------------------------------------
217
+
218
+ fn parse_opaque(cursor: &mut Cursor<'_>, flags: u32, size: u32) -> Result<Datatype> {
219
+ // The class flags encode the length of the tag (in the lower bits).
220
+ let tag_len = (flags & 0xFF) as usize;
221
+
222
+ let tag = if tag_len > 0 {
223
+ let tag_bytes = cursor.read_bytes(tag_len)?;
224
+ // Trim trailing nulls
225
+ let end = tag_bytes.iter().rposition(|&b| b != 0).map_or(0, |i| i + 1);
226
+ String::from_utf8_lossy(&tag_bytes[..end]).into_owned()
227
+ } else {
228
+ String::new()
229
+ };
230
+
231
+ // Pad to 8-byte alignment
232
+ let padded = (tag_len + 7) & !7;
233
+ if padded > tag_len {
234
+ cursor.skip(padded - tag_len)?;
235
+ }
236
+
237
+ Ok(Datatype::Opaque { size, tag })
238
+ }
239
+
240
+ // ---------------------------------------------------------------------------
241
+ // Class 6: Compound
242
+ // ---------------------------------------------------------------------------
243
+
244
+ fn parse_compound(
245
+ cursor: &mut Cursor<'_>,
246
+ flags: u32,
247
+ size: u32,
248
+ version: u8,
249
+ depth: usize,
250
+ ) -> Result<Datatype> {
251
+ // Lower 16 bits of class flags = number of members
252
+ let n_members = (flags & 0xFFFF) as usize;
253
+ let byte_offset_size = compound_member_offset_size(size);
254
+
255
+ let mut fields = Vec::with_capacity(n_members);
256
+
257
+ for _ in 0..n_members {
258
+ let name_start = cursor.position();
259
+ let name = cursor.read_null_terminated_string()?;
260
+
261
+ if version < 3 {
262
+ // V1/V2: the name, null terminator included, is padded to a multiple of
263
+ // eight bytes counted from the start of the name, not from the start of the
264
+ // buffer. A member whose datatype description is not a multiple of eight
265
+ // bytes long (a fixed-length string's is 8, a float's 20) leaves the next
266
+ // name off the buffer's 8-byte grid.
267
+ let used = (cursor.position() - name_start) as usize;
268
+ cursor.skip(used.next_multiple_of(8) - used)?;
269
+ }
270
+
271
+ let byte_offset = if version == 1 {
272
+ // V1: byte offset is `size of offsets` (4 bytes)
273
+ cursor.read_u32_le()?
274
+ } else if version >= 3 {
275
+ cursor.read_uvar(byte_offset_size)? as u32
276
+ } else {
277
+ // V2/V3: byte offset is 4 bytes
278
+ cursor.read_u32_le()?
279
+ };
280
+
281
+ if version == 1 {
282
+ // V1: dimensionality (1 byte), reserved (3 bytes), dim perm (4 bytes),
283
+ // reserved (4 bytes), dim sizes (4 * 4 = 16 bytes)
284
+ let _dimensionality = cursor.read_u8()?;
285
+ cursor.skip(3)?; // reserved
286
+ cursor.skip(4)?; // dimension permutation
287
+ cursor.skip(4)?; // reserved
288
+ cursor.skip(16)?; // 4 dimension sizes (each u32)
289
+ }
290
+
291
+ let (member_dt, _member_size) = parse_datatype_description_at_depth(cursor, depth + 1)?;
292
+
293
+ fields.push(CompoundField {
294
+ name,
295
+ byte_offset,
296
+ datatype: member_dt,
297
+ });
298
+ }
299
+
300
+ Ok(Datatype::Compound { size, fields })
301
+ }
302
+
303
+ fn compound_member_offset_size(size: u32) -> usize {
304
+ match size {
305
+ 0..=0xFF => 1,
306
+ 0x100..=0xFFFF => 2,
307
+ 0x1_0000..=0xFF_FFFF => 3,
308
+ _ => 4,
309
+ }
310
+ }
311
+
312
+ // ---------------------------------------------------------------------------
313
+ // Class 7: Reference
314
+ // ---------------------------------------------------------------------------
315
+
316
+ fn parse_reference(flags: u32, size: u32) -> Result<Datatype> {
317
+ // Bit 0-3: reference type (0 = object, 1 = dataset region)
318
+ let ref_type = match flags & 0x0F {
319
+ 0 => ReferenceType::Object,
320
+ 1 => ReferenceType::DatasetRegion,
321
+ _ => ReferenceType::Object,
322
+ };
323
+
324
+ // No property bytes for reference class.
325
+
326
+ Ok(Datatype::Reference {
327
+ ref_type,
328
+ size: size as u8,
329
+ })
330
+ }
331
+
332
+ // ---------------------------------------------------------------------------
333
+ // Class 8: Enum
334
+ // ---------------------------------------------------------------------------
335
+
336
+ fn parse_enum(cursor: &mut Cursor<'_>, flags: u32, size: u32, depth: usize) -> Result<Datatype> {
337
+ let n_members = (flags & 0xFFFF) as usize;
338
+
339
+ // Base type
340
+ let (base_dt, _base_size) = parse_datatype_description_at_depth(cursor, depth + 1)?;
341
+
342
+ // Member names (null-terminated)
343
+ let mut names = Vec::with_capacity(n_members);
344
+ for _ in 0..n_members {
345
+ names.push(cursor.read_null_terminated_string()?);
346
+ }
347
+
348
+ // Member values (each is `size` bytes, matching the base type size)
349
+ let member_value_size = size as usize;
350
+ let mut members = Vec::with_capacity(n_members);
351
+ for name in names {
352
+ let value = cursor.read_bytes(member_value_size)?.to_vec();
353
+ members.push(EnumMember { name, value });
354
+ }
355
+
356
+ Ok(Datatype::Enum {
357
+ base: Box::new(base_dt),
358
+ members,
359
+ })
360
+ }
361
+
362
+ // ---------------------------------------------------------------------------
363
+ // Class 9: Variable-length
364
+ // ---------------------------------------------------------------------------
365
+
366
+ fn parse_varlen(cursor: &mut Cursor<'_>, flags: u32, _size: u32, depth: usize) -> Result<Datatype> {
367
+ // Bits 0-3: type (0 = sequence, 1 = string)
368
+ let kind = match flags & 0x0F {
369
+ 0 => VarLenKind::Sequence,
370
+ 1 => VarLenKind::String,
371
+ other => VarLenKind::Unknown(other as u8),
372
+ };
373
+ // Bits 4-7: padding type (for strings)
374
+ let padding = string_padding_from_bits((flags >> 4) & 0x0F);
375
+ // Bits 8-11: character set (for strings)
376
+ let encoding = string_encoding_from_bits((flags >> 8) & 0x0F);
377
+
378
+ // Base type follows
379
+ let (base_dt, _base_size) = parse_datatype_description_at_depth(cursor, depth + 1)?;
380
+
381
+ Ok(Datatype::VarLen {
382
+ base: Box::new(base_dt),
383
+ kind,
384
+ encoding,
385
+ padding,
386
+ })
387
+ }
388
+
389
+ fn string_padding_from_bits(bits: u32) -> StringPadding {
390
+ match bits {
391
+ 0 => StringPadding::NullTerminate,
392
+ 1 => StringPadding::NullPad,
393
+ 2 => StringPadding::SpacePad,
394
+ _ => StringPadding::NullTerminate,
395
+ }
396
+ }
397
+
398
+ fn string_encoding_from_bits(bits: u32) -> StringEncoding {
399
+ match bits {
400
+ 0 => StringEncoding::Ascii,
401
+ 1 => StringEncoding::Utf8,
402
+ _ => StringEncoding::Ascii,
403
+ }
404
+ }
405
+
406
+ // ---------------------------------------------------------------------------
407
+ // Class 10: Array
408
+ // ---------------------------------------------------------------------------
409
+
410
+ fn parse_array(cursor: &mut Cursor<'_>, _size: u32, version: u8, depth: usize) -> Result<Datatype> {
411
+ let rank = cursor.read_u8()? as usize;
412
+
413
+ if version < 3 {
414
+ // Version 1/2: 3 reserved bytes after rank
415
+ cursor.skip(3)?;
416
+ }
417
+
418
+ let mut dims = Vec::with_capacity(rank);
419
+ for _ in 0..rank {
420
+ dims.push(cursor.read_u32_le()? as u64);
421
+ }
422
+
423
+ if version < 3 {
424
+ // Version 1: permutation indices (rank * u32) — skip them
425
+ cursor.skip(rank * 4)?;
426
+ }
427
+
428
+ // Base type
429
+ let (base_dt, _base_size) = parse_datatype_description_at_depth(cursor, depth + 1)?;
430
+
431
+ Ok(Datatype::Array {
432
+ base: Box::new(base_dt),
433
+ dims,
434
+ })
435
+ }
436
+
437
+ // ---------------------------------------------------------------------------
438
+ // Tests
439
+ // ---------------------------------------------------------------------------
440
+
441
+ #[cfg(test)]
442
+ mod tests {
443
+ use super::*;
444
+
445
+ /// Build the 4-byte class+version+flags word.
446
+ fn class_word(class: u8, version: u8, flags: u32) -> u32 {
447
+ (class as u32) | ((version as u32) << 4) | (flags << 8)
448
+ }
449
+
450
+ /// Encode `levels` nested v3 array datatypes around a 4-byte fixed-point
451
+ /// base.
452
+ fn nested_array_datatype(levels: usize) -> Vec<u8> {
453
+ let mut data = Vec::new();
454
+ for _ in 0..levels {
455
+ data.extend_from_slice(&class_word(10, 3, 0x00).to_le_bytes());
456
+ data.extend_from_slice(&4u32.to_le_bytes());
457
+ data.push(1); // rank
458
+ data.extend_from_slice(&2u32.to_le_bytes()); // dim
459
+ }
460
+ // Fixed-point base.
461
+ data.extend_from_slice(&class_word(0, 1, 0x00).to_le_bytes());
462
+ data.extend_from_slice(&4u32.to_le_bytes());
463
+ data.extend_from_slice(&0u16.to_le_bytes());
464
+ data.extend_from_slice(&32u16.to_le_bytes());
465
+ data
466
+ }
467
+
468
+ #[test]
469
+ fn deeply_nested_datatype_is_rejected() {
470
+ let data = nested_array_datatype(64);
471
+ let mut cursor = Cursor::new(&data);
472
+ let err = parse(&mut cursor, data.len()).unwrap_err();
473
+ assert!(
474
+ err.to_string().contains("nesting exceeds"),
475
+ "unexpected error: {err}"
476
+ );
477
+ }
478
+
479
+ #[test]
480
+ fn moderately_nested_datatype_parses() {
481
+ let data = nested_array_datatype(4);
482
+ let mut cursor = Cursor::new(&data);
483
+ let msg = parse(&mut cursor, data.len()).unwrap();
484
+ let mut dt = &msg.datatype;
485
+ for _ in 0..4 {
486
+ match dt {
487
+ Datatype::Array { base, dims } => {
488
+ assert_eq!(dims.as_slice(), &[2]);
489
+ dt = base;
490
+ }
491
+ other => panic!("expected Array, got {other:?}"),
492
+ }
493
+ }
494
+ assert!(matches!(dt, Datatype::FixedPoint { size: 4, .. }));
495
+ }
496
+
497
+ #[test]
498
+ fn parse_u32_le() {
499
+ let mut data = Vec::new();
500
+ // class=0 (fixed-point), version=1, flags: bit0=0 (LE), bit3=0 (unsigned)
501
+ data.extend_from_slice(&class_word(0, 1, 0x00).to_le_bytes());
502
+ // size = 4
503
+ data.extend_from_slice(&4u32.to_le_bytes());
504
+ // properties: bit offset=0, bit precision=32
505
+ data.extend_from_slice(&0u16.to_le_bytes());
506
+ data.extend_from_slice(&32u16.to_le_bytes());
507
+
508
+ let mut cursor = Cursor::new(&data);
509
+ let msg = parse(&mut cursor, data.len()).unwrap();
510
+ assert_eq!(msg.size, 4);
511
+ match &msg.datatype {
512
+ Datatype::FixedPoint {
513
+ size,
514
+ signed,
515
+ byte_order,
516
+ } => {
517
+ assert_eq!(*size, 4);
518
+ assert!(!*signed);
519
+ assert_eq!(*byte_order, ByteOrder::LittleEndian);
520
+ }
521
+ other => panic!("expected FixedPoint, got {:?}", other),
522
+ }
523
+ }
524
+
525
+ #[test]
526
+ fn parse_i64_be() {
527
+ let mut data = Vec::new();
528
+ // class=0 (fixed-point), version=1, flags: bit0=1 (BE), bit3=1 (signed)
529
+ data.extend_from_slice(&class_word(0, 1, 0x09).to_le_bytes());
530
+ // size = 8
531
+ data.extend_from_slice(&8u32.to_le_bytes());
532
+ // properties: bit offset=0, bit precision=64
533
+ data.extend_from_slice(&0u16.to_le_bytes());
534
+ data.extend_from_slice(&64u16.to_le_bytes());
535
+
536
+ let mut cursor = Cursor::new(&data);
537
+ let msg = parse(&mut cursor, data.len()).unwrap();
538
+ assert_eq!(msg.size, 8);
539
+ match &msg.datatype {
540
+ Datatype::FixedPoint {
541
+ size,
542
+ signed,
543
+ byte_order,
544
+ } => {
545
+ assert_eq!(*size, 8);
546
+ assert!(*signed);
547
+ assert_eq!(*byte_order, ByteOrder::BigEndian);
548
+ }
549
+ other => panic!("expected FixedPoint, got {:?}", other),
550
+ }
551
+ }
552
+
553
+ #[test]
554
+ fn parse_f32_le() {
555
+ let mut data = Vec::new();
556
+ // class=1 (float), version=1, flags: bit0=0 (LE), bit6=0
557
+ data.extend_from_slice(&class_word(1, 1, 0x20).to_le_bytes());
558
+ // size = 4
559
+ data.extend_from_slice(&4u32.to_le_bytes());
560
+ // properties: bit offset=0, bit precision=32
561
+ data.extend_from_slice(&0u16.to_le_bytes());
562
+ data.extend_from_slice(&32u16.to_le_bytes());
563
+ // exp location=23, exp size=8
564
+ data.push(23);
565
+ data.push(8);
566
+ // mant location=0, mant size=23
567
+ data.push(0);
568
+ data.push(23);
569
+ // exp bias=127
570
+ data.extend_from_slice(&127u32.to_le_bytes());
571
+
572
+ let mut cursor = Cursor::new(&data);
573
+ let msg = parse(&mut cursor, data.len()).unwrap();
574
+ assert_eq!(msg.size, 4);
575
+ match &msg.datatype {
576
+ Datatype::FloatingPoint { size, byte_order } => {
577
+ assert_eq!(*size, 4);
578
+ assert_eq!(*byte_order, ByteOrder::LittleEndian);
579
+ }
580
+ other => panic!("expected FloatingPoint, got {:?}", other),
581
+ }
582
+ }
583
+
584
+ #[test]
585
+ fn parse_f64_be() {
586
+ let mut data = Vec::new();
587
+ // class=1 (float), version=1, flags: bit0=1 (BE), bit6=0
588
+ data.extend_from_slice(&class_word(1, 1, 0x01).to_le_bytes());
589
+ // size = 8
590
+ data.extend_from_slice(&8u32.to_le_bytes());
591
+ // properties
592
+ data.extend_from_slice(&0u16.to_le_bytes());
593
+ data.extend_from_slice(&64u16.to_le_bytes());
594
+ data.push(52);
595
+ data.push(11);
596
+ data.push(0);
597
+ data.push(52);
598
+ data.extend_from_slice(&1023u32.to_le_bytes());
599
+
600
+ let mut cursor = Cursor::new(&data);
601
+ let msg = parse(&mut cursor, data.len()).unwrap();
602
+ assert_eq!(msg.size, 8);
603
+ match &msg.datatype {
604
+ Datatype::FloatingPoint { size, byte_order } => {
605
+ assert_eq!(*size, 8);
606
+ assert_eq!(*byte_order, ByteOrder::BigEndian);
607
+ }
608
+ other => panic!("expected FloatingPoint, got {:?}", other),
609
+ }
610
+ }
611
+
612
+ #[test]
613
+ fn parse_string_fixed_ascii() {
614
+ let mut data = Vec::new();
615
+ // class=3 (string), version=1, flags: padding=0 (null-term), charset=0 (ascii)
616
+ data.extend_from_slice(&class_word(3, 1, 0x00).to_le_bytes());
617
+ // size = 32
618
+ data.extend_from_slice(&32u32.to_le_bytes());
619
+ // No property bytes for strings.
620
+
621
+ let mut cursor = Cursor::new(&data);
622
+ let msg = parse(&mut cursor, data.len()).unwrap();
623
+ assert_eq!(msg.size, 32);
624
+ match &msg.datatype {
625
+ Datatype::String {
626
+ size,
627
+ encoding,
628
+ padding,
629
+ } => {
630
+ assert_eq!(*size, StringSize::Fixed(32));
631
+ assert_eq!(*encoding, StringEncoding::Ascii);
632
+ assert_eq!(*padding, StringPadding::NullTerminate);
633
+ }
634
+ other => panic!("expected String, got {:?}", other),
635
+ }
636
+ }
637
+
638
+ #[test]
639
+ fn parse_string_utf8_space_pad() {
640
+ let mut data = Vec::new();
641
+ // class=3, version=1, flags: padding=2 (space-pad), charset=1 (utf8)
642
+ // padding bits 0-3 = 2, charset bits 4-7 = 1
643
+ let flags: u32 = 0x02 | (0x01 << 4);
644
+ data.extend_from_slice(&class_word(3, 1, flags).to_le_bytes());
645
+ data.extend_from_slice(&16u32.to_le_bytes());
646
+
647
+ let mut cursor = Cursor::new(&data);
648
+ let msg = parse(&mut cursor, data.len()).unwrap();
649
+ match &msg.datatype {
650
+ Datatype::String {
651
+ size,
652
+ encoding,
653
+ padding,
654
+ } => {
655
+ assert_eq!(*size, StringSize::Fixed(16));
656
+ assert_eq!(*encoding, StringEncoding::Utf8);
657
+ assert_eq!(*padding, StringPadding::SpacePad);
658
+ }
659
+ other => panic!("expected String, got {:?}", other),
660
+ }
661
+ }
662
+
663
+ #[test]
664
+ fn parse_varlen_string_preserves_flags() {
665
+ let mut data = Vec::new();
666
+ // class=9, version=1, flags: kind=string, padding=space-pad, charset=utf8
667
+ let flags: u32 = 0x01 | (0x02 << 4) | (0x01 << 8);
668
+ data.extend_from_slice(&class_word(9, 1, flags).to_le_bytes());
669
+ data.extend_from_slice(&16u32.to_le_bytes());
670
+ data.extend_from_slice(&class_word(0, 1, 0x00).to_le_bytes());
671
+ data.extend_from_slice(&1u32.to_le_bytes());
672
+ data.extend_from_slice(&0u16.to_le_bytes());
673
+ data.extend_from_slice(&8u16.to_le_bytes());
674
+
675
+ let mut cursor = Cursor::new(&data);
676
+ let msg = parse(&mut cursor, data.len()).unwrap();
677
+ match &msg.datatype {
678
+ Datatype::VarLen {
679
+ base,
680
+ kind,
681
+ encoding,
682
+ padding,
683
+ } => {
684
+ assert_eq!(*kind, VarLenKind::String);
685
+ assert_eq!(*encoding, StringEncoding::Utf8);
686
+ assert_eq!(*padding, StringPadding::SpacePad);
687
+ assert!(matches!(
688
+ base.as_ref(),
689
+ Datatype::FixedPoint {
690
+ size: 1,
691
+ signed: false,
692
+ byte_order: ByteOrder::LittleEndian
693
+ }
694
+ ));
695
+ }
696
+ other => panic!("expected VarLen, got {:?}", other),
697
+ }
698
+ }
699
+
700
+ #[test]
701
+ fn parse_varlen_sequence_preserves_kind() {
702
+ let mut data = Vec::new();
703
+ data.extend_from_slice(&class_word(9, 1, 0x00).to_le_bytes());
704
+ data.extend_from_slice(&16u32.to_le_bytes());
705
+ data.extend_from_slice(&class_word(0, 1, 0x00).to_le_bytes());
706
+ data.extend_from_slice(&4u32.to_le_bytes());
707
+ data.extend_from_slice(&0u16.to_le_bytes());
708
+ data.extend_from_slice(&32u16.to_le_bytes());
709
+
710
+ let mut cursor = Cursor::new(&data);
711
+ let msg = parse(&mut cursor, data.len()).unwrap();
712
+ match &msg.datatype {
713
+ Datatype::VarLen { kind, base, .. } => {
714
+ assert_eq!(*kind, VarLenKind::Sequence);
715
+ assert!(matches!(
716
+ base.as_ref(),
717
+ Datatype::FixedPoint {
718
+ size: 4,
719
+ signed: false,
720
+ byte_order: ByteOrder::LittleEndian
721
+ }
722
+ ));
723
+ }
724
+ other => panic!("expected VarLen, got {:?}", other),
725
+ }
726
+ }
727
+
728
+ #[test]
729
+ fn parse_reference_object() {
730
+ let mut data = Vec::new();
731
+ // class=7, version=1, flags: ref_type=0 (object)
732
+ data.extend_from_slice(&class_word(7, 1, 0x00).to_le_bytes());
733
+ data.extend_from_slice(&8u32.to_le_bytes());
734
+
735
+ let mut cursor = Cursor::new(&data);
736
+ let msg = parse(&mut cursor, data.len()).unwrap();
737
+ match &msg.datatype {
738
+ Datatype::Reference { ref_type, size } => {
739
+ assert_eq!(*ref_type, ReferenceType::Object);
740
+ assert_eq!(*size, 8);
741
+ }
742
+ other => panic!("expected Reference, got {:?}", other),
743
+ }
744
+ }
745
+
746
+ #[test]
747
+ fn parse_reference_region() {
748
+ let mut data = Vec::new();
749
+ // class=7, version=1, flags: ref_type=1 (dataset region)
750
+ data.extend_from_slice(&class_word(7, 1, 0x01).to_le_bytes());
751
+ data.extend_from_slice(&12u32.to_le_bytes());
752
+
753
+ let mut cursor = Cursor::new(&data);
754
+ let msg = parse(&mut cursor, data.len()).unwrap();
755
+ match &msg.datatype {
756
+ Datatype::Reference { ref_type, size } => {
757
+ assert_eq!(*ref_type, ReferenceType::DatasetRegion);
758
+ assert_eq!(*size, 12);
759
+ }
760
+ other => panic!("expected Reference, got {:?}", other),
761
+ }
762
+ }
763
+
764
+ /// A version 1 compound member: its name padded to a multiple of eight bytes, the
765
+ /// byte offset, then dimensionality, reserved, permutation, reserved and four
766
+ /// dimension sizes, then the member's own datatype description.
767
+ fn compound_v1_member(data: &mut Vec<u8>, name: &str, offset: u32, member: &[u8]) {
768
+ let mut padded = name.as_bytes().to_vec();
769
+ padded.push(0);
770
+ padded.resize(padded.len().next_multiple_of(8), 0);
771
+ data.extend_from_slice(&padded);
772
+ data.extend_from_slice(&offset.to_le_bytes());
773
+ data.extend_from_slice(&[0u8; 28]);
774
+ data.extend_from_slice(member);
775
+ }
776
+
777
+ #[test]
778
+ fn parse_compound_v1_pads_names_relative_to_their_start() {
779
+ // A float description is 20 bytes and a fixed string's is 8, so after the first
780
+ // two members the third name starts four bytes off the buffer's 8-byte grid
781
+ // (as in HEC-RAS attribute tables). Its padding must still be counted from the
782
+ // start of the name.
783
+ let mut float = Vec::new();
784
+ float.extend_from_slice(&class_word(1, 1, 0x20).to_le_bytes());
785
+ float.extend_from_slice(&4u32.to_le_bytes());
786
+ float.extend_from_slice(&0u16.to_le_bytes());
787
+ float.extend_from_slice(&32u16.to_le_bytes());
788
+ float.extend_from_slice(&[23, 8, 0, 23]);
789
+ float.extend_from_slice(&127u32.to_le_bytes());
790
+ let mut string = Vec::new();
791
+ string.extend_from_slice(&class_word(3, 1, 0x00).to_le_bytes());
792
+ string.extend_from_slice(&16u32.to_le_bytes());
793
+ let mut int = Vec::new();
794
+ int.extend_from_slice(&class_word(0, 1, 0x08).to_le_bytes());
795
+ int.extend_from_slice(&4u32.to_le_bytes());
796
+ int.extend_from_slice(&0u16.to_le_bytes());
797
+ int.extend_from_slice(&32u16.to_le_bytes());
798
+
799
+ let mut data = Vec::new();
800
+ data.extend_from_slice(&class_word(6, 1, 3).to_le_bytes());
801
+ data.extend_from_slice(&24u32.to_le_bytes());
802
+ compound_v1_member(&mut data, "x", 0, &float);
803
+ compound_v1_member(&mut data, "name", 4, &string);
804
+ compound_v1_member(&mut data, "count", 20, &int);
805
+
806
+ let mut cursor = Cursor::new(&data);
807
+ let msg = parse(&mut cursor, data.len()).unwrap();
808
+ match &msg.datatype {
809
+ Datatype::Compound { size, fields } => {
810
+ assert_eq!(*size, 24);
811
+ let names: Vec<_> = fields
812
+ .iter()
813
+ .map(|f| (f.name.as_str(), f.byte_offset))
814
+ .collect();
815
+ assert_eq!(names, [("x", 0), ("name", 4), ("count", 20)]);
816
+ assert!(matches!(
817
+ fields[2].datatype,
818
+ Datatype::FixedPoint {
819
+ size: 4,
820
+ signed: true,
821
+ ..
822
+ }
823
+ ));
824
+ }
825
+ other => panic!("expected Compound, got {:?}", other),
826
+ }
827
+ assert_eq!(cursor.position() as usize, data.len());
828
+ }
829
+
830
+ #[test]
831
+ fn parse_compound_v3_variable_member_offsets() {
832
+ let mut data = Vec::new();
833
+ data.extend_from_slice(&class_word(6, 3, 2).to_le_bytes());
834
+ data.extend_from_slice(&16u32.to_le_bytes());
835
+
836
+ data.extend_from_slice(b"dataset\0");
837
+ data.push(0x00);
838
+ data.extend_from_slice(&class_word(7, 1, 0x00).to_le_bytes());
839
+ data.extend_from_slice(&8u32.to_le_bytes());
840
+
841
+ data.extend_from_slice(b"dimension\0");
842
+ data.push(0x08);
843
+ data.extend_from_slice(&class_word(0, 1, 0x00).to_le_bytes());
844
+ data.extend_from_slice(&4u32.to_le_bytes());
845
+ data.extend_from_slice(&0u16.to_le_bytes());
846
+ data.extend_from_slice(&32u16.to_le_bytes());
847
+
848
+ let mut cursor = Cursor::new(&data);
849
+ let msg = parse(&mut cursor, data.len()).unwrap();
850
+ match &msg.datatype {
851
+ Datatype::Compound { size, fields } => {
852
+ assert_eq!(*size, 16);
853
+ assert_eq!(fields.len(), 2);
854
+ assert_eq!(fields[0].name, "dataset");
855
+ assert_eq!(fields[0].byte_offset, 0);
856
+ assert_eq!(fields[1].name, "dimension");
857
+ assert_eq!(fields[1].byte_offset, 8);
858
+ }
859
+ other => panic!("expected Compound, got {:?}", other),
860
+ }
861
+ }
862
+
863
+ #[test]
864
+ fn parse_enum_u8() {
865
+ let mut data = Vec::new();
866
+ // class=8 (enum), version=3, flags: n_members=2
867
+ data.extend_from_slice(&class_word(8, 3, 2).to_le_bytes());
868
+ // size = 1
869
+ data.extend_from_slice(&1u32.to_le_bytes());
870
+
871
+ // Base type: u8 (class=0, version=1, flags=0, size=1, props: offset=0 precision=8)
872
+ data.extend_from_slice(&class_word(0, 1, 0).to_le_bytes());
873
+ data.extend_from_slice(&1u32.to_le_bytes());
874
+ data.extend_from_slice(&0u16.to_le_bytes());
875
+ data.extend_from_slice(&8u16.to_le_bytes());
876
+
877
+ // Member names
878
+ data.extend_from_slice(b"OFF\0");
879
+ data.extend_from_slice(b"ON\0");
880
+
881
+ // Member values (1 byte each, matching size=1)
882
+ data.push(0x00);
883
+ data.push(0x01);
884
+
885
+ let mut cursor = Cursor::new(&data);
886
+ let msg = parse(&mut cursor, data.len()).unwrap();
887
+ match &msg.datatype {
888
+ Datatype::Enum { base, members } => {
889
+ assert!(matches!(
890
+ base.as_ref(),
891
+ Datatype::FixedPoint {
892
+ size: 1,
893
+ signed: false,
894
+ ..
895
+ }
896
+ ));
897
+ assert_eq!(members.len(), 2);
898
+ assert_eq!(members[0].name, "OFF");
899
+ assert_eq!(members[0].value, vec![0x00]);
900
+ assert_eq!(members[1].name, "ON");
901
+ assert_eq!(members[1].value, vec![0x01]);
902
+ }
903
+ other => panic!("expected Enum, got {:?}", other),
904
+ }
905
+ }
906
+
907
+ #[test]
908
+ fn bitfield_datatype_parses_size_and_byte_order() {
909
+ let mut data = Vec::new();
910
+ // class=4 (bitfield), version=1, flags: bit0=0 (LE)
911
+ data.extend_from_slice(&class_word(4, 1, 0x00).to_le_bytes());
912
+ data.extend_from_slice(&2u32.to_le_bytes());
913
+ // properties: bit offset=0, bit precision=16
914
+ data.extend_from_slice(&0u16.to_le_bytes());
915
+ data.extend_from_slice(&16u16.to_le_bytes());
916
+
917
+ let mut cursor = Cursor::new(&data);
918
+ let msg = parse(&mut cursor, data.len()).unwrap();
919
+ match &msg.datatype {
920
+ Datatype::Bitfield { size, byte_order } => {
921
+ assert_eq!(*size, 2);
922
+ assert_eq!(*byte_order, ByteOrder::LittleEndian);
923
+ }
924
+ other => panic!("expected Bitfield, got {:?}", other),
925
+ }
926
+ }
927
+
928
+ #[test]
929
+ fn unsupported_class() {
930
+ let mut data = Vec::new();
931
+ // class=15 (invalid), version=1, flags=0
932
+ data.extend_from_slice(&class_word(15, 1, 0).to_le_bytes());
933
+ data.extend_from_slice(&0u32.to_le_bytes());
934
+
935
+ let mut cursor = Cursor::new(&data);
936
+ assert!(parse(&mut cursor, data.len()).is_err());
937
+ }
938
+ }