@nirs4all/formats-wasm 0.2.12 → 0.2.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/THIRD_PARTY_NOTICES.md +49 -0
  2. package/nirs4all_formats_wasm.d.ts +4 -4
  3. package/nirs4all_formats_wasm.js +170 -149
  4. package/nirs4all_formats_wasm_bg.wasm +0 -0
  5. package/package.json +4 -2
  6. package/reader-backports/hdf5-reader/Cargo.toml +119 -0
  7. package/reader-backports/hdf5-reader/LICENSE-APACHE +164 -0
  8. package/reader-backports/hdf5-reader/LICENSE-MIT +21 -0
  9. package/reader-backports/hdf5-reader/PROVENANCE.json +80 -0
  10. package/reader-backports/hdf5-reader/README.md +308 -0
  11. package/reader-backports/hdf5-reader/UPSTREAM-Cargo.toml +44 -0
  12. package/reader-backports/hdf5-reader/UPSTREAM-VCS.json +6 -0
  13. package/reader-backports/hdf5-reader/src/attribute_api.rs +847 -0
  14. package/reader-backports/hdf5-reader/src/btree_v1.rs +726 -0
  15. package/reader-backports/hdf5-reader/src/btree_v2.rs +2068 -0
  16. package/reader-backports/hdf5-reader/src/cache.rs +354 -0
  17. package/reader-backports/hdf5-reader/src/checksum.rs +56 -0
  18. package/reader-backports/hdf5-reader/src/chunk_index.rs +423 -0
  19. package/reader-backports/hdf5-reader/src/dataset.rs +4733 -0
  20. package/reader-backports/hdf5-reader/src/datatype_api.rs +497 -0
  21. package/reader-backports/hdf5-reader/src/error.rs +167 -0
  22. package/reader-backports/hdf5-reader/src/extensible_array.rs +1886 -0
  23. package/reader-backports/hdf5-reader/src/filters/deflate.rs +93 -0
  24. package/reader-backports/hdf5-reader/src/filters/fletcher32.rs +52 -0
  25. package/reader-backports/hdf5-reader/src/filters/lz4.rs +178 -0
  26. package/reader-backports/hdf5-reader/src/filters/mod.rs +326 -0
  27. package/reader-backports/hdf5-reader/src/filters/nbit.rs +586 -0
  28. package/reader-backports/hdf5-reader/src/filters/scaleoffset.rs +827 -0
  29. package/reader-backports/hdf5-reader/src/filters/shuffle.rs +99 -0
  30. package/reader-backports/hdf5-reader/src/fixed_array.rs +763 -0
  31. package/reader-backports/hdf5-reader/src/fractal_heap.rs +2029 -0
  32. package/reader-backports/hdf5-reader/src/global_heap.rs +284 -0
  33. package/reader-backports/hdf5-reader/src/group.rs +795 -0
  34. package/reader-backports/hdf5-reader/src/io.rs +413 -0
  35. package/reader-backports/hdf5-reader/src/lib.rs +916 -0
  36. package/reader-backports/hdf5-reader/src/local_heap.rs +261 -0
  37. package/reader-backports/hdf5-reader/src/messages/attribute.rs +304 -0
  38. package/reader-backports/hdf5-reader/src/messages/attribute_info.rs +96 -0
  39. package/reader-backports/hdf5-reader/src/messages/btree_k.rs +36 -0
  40. package/reader-backports/hdf5-reader/src/messages/continuation.rs +46 -0
  41. package/reader-backports/hdf5-reader/src/messages/dataspace.rs +259 -0
  42. package/reader-backports/hdf5-reader/src/messages/datatype.rs +938 -0
  43. package/reader-backports/hdf5-reader/src/messages/external_files.rs +50 -0
  44. package/reader-backports/hdf5-reader/src/messages/fill_value.rs +233 -0
  45. package/reader-backports/hdf5-reader/src/messages/filter_pipeline.rs +147 -0
  46. package/reader-backports/hdf5-reader/src/messages/group_info.rs +103 -0
  47. package/reader-backports/hdf5-reader/src/messages/layout.rs +476 -0
  48. package/reader-backports/hdf5-reader/src/messages/link.rs +170 -0
  49. package/reader-backports/hdf5-reader/src/messages/link_info.rs +127 -0
  50. package/reader-backports/hdf5-reader/src/messages/mod.rs +374 -0
  51. package/reader-backports/hdf5-reader/src/messages/modification_time.rs +83 -0
  52. package/reader-backports/hdf5-reader/src/messages/shared.rs +164 -0
  53. package/reader-backports/hdf5-reader/src/messages/shared_table.rs +67 -0
  54. package/reader-backports/hdf5-reader/src/messages/symbol_table_msg.rs +51 -0
  55. package/reader-backports/hdf5-reader/src/object_header.rs +1749 -0
  56. package/reader-backports/hdf5-reader/src/reference.rs +90 -0
  57. package/reader-backports/hdf5-reader/src/shared_message_table.rs +425 -0
  58. package/reader-backports/hdf5-reader/src/storage.rs +538 -0
  59. package/reader-backports/hdf5-reader/src/superblock.rs +285 -0
  60. package/reader-backports/hdf5-reader/src/symbol_table.rs +277 -0
  61. package/reader-backports/netcdf-reader/Cargo.toml +141 -0
  62. package/reader-backports/netcdf-reader/LICENSE-APACHE +164 -0
  63. package/reader-backports/netcdf-reader/LICENSE-MIT +21 -0
  64. package/reader-backports/netcdf-reader/PROVENANCE.json +55 -0
  65. package/reader-backports/netcdf-reader/README.md +308 -0
  66. package/reader-backports/netcdf-reader/UPSTREAM-Cargo.toml +48 -0
  67. package/reader-backports/netcdf-reader/UPSTREAM-VCS.json +6 -0
  68. package/reader-backports/netcdf-reader/src/cf/axes.rs +419 -0
  69. package/reader-backports/netcdf-reader/src/cf/bounds.rs +108 -0
  70. package/reader-backports/netcdf-reader/src/cf/crs.rs +269 -0
  71. package/reader-backports/netcdf-reader/src/cf/mod.rs +30 -0
  72. package/reader-backports/netcdf-reader/src/cf/time.rs +1122 -0
  73. package/reader-backports/netcdf-reader/src/classic/data.rs +1465 -0
  74. package/reader-backports/netcdf-reader/src/classic/header.rs +1097 -0
  75. package/reader-backports/netcdf-reader/src/classic/mod.rs +240 -0
  76. package/reader-backports/netcdf-reader/src/classic/storage.rs +211 -0
  77. package/reader-backports/netcdf-reader/src/classic/types.rs +93 -0
  78. package/reader-backports/netcdf-reader/src/classic/variable.rs +1162 -0
  79. package/reader-backports/netcdf-reader/src/error.rs +59 -0
  80. package/reader-backports/netcdf-reader/src/lib.rs +1531 -0
  81. package/reader-backports/netcdf-reader/src/masked.rs +173 -0
  82. package/reader-backports/netcdf-reader/src/nc4/attributes.rs +203 -0
  83. package/reader-backports/netcdf-reader/src/nc4/dimensions.rs +139 -0
  84. package/reader-backports/netcdf-reader/src/nc4/groups.rs +308 -0
  85. package/reader-backports/netcdf-reader/src/nc4/mod.rs +572 -0
  86. package/reader-backports/netcdf-reader/src/nc4/types.rs +309 -0
  87. package/reader-backports/netcdf-reader/src/nc4/variables.rs +420 -0
  88. package/reader-backports/netcdf-reader/src/types.rs +18 -0
  89. package/reader-backports/netcdf-reader/src/unpack.rs +113 -0
  90. package/reader-backports/netcdf-reader/src/user_defined.rs +569 -0
@@ -0,0 +1,847 @@
1
+ use crate::error::{Error, Result};
2
+ use crate::filters::FilterRegistry;
3
+ use crate::fractal_heap::FractalHeap;
4
+ use crate::global_heap::GlobalHeapCollection;
5
+ use crate::io::Cursor;
6
+ use crate::messages::attribute::AttributeMessage;
7
+ use crate::messages::attribute_info::AttributeInfoMessage;
8
+ use crate::messages::dataspace::DataspaceType;
9
+ use crate::messages::datatype::{Datatype, StringEncoding, StringPadding, StringSize, VarLenKind};
10
+ use crate::messages::HdfMessage;
11
+ use crate::object_header::ObjectHeader;
12
+ use crate::storage::Storage;
13
+ use crate::{btree_v2, messages};
14
+
15
+ fn checked_usize(value: u64, context: &str) -> Result<usize> {
16
+ usize::try_from(value).map_err(|_| {
17
+ Error::InvalidData(format!(
18
+ "{context} value {value} exceeds platform usize capacity"
19
+ ))
20
+ })
21
+ }
22
+
23
+ /// A parsed, high-level HDF5 attribute.
24
+ #[derive(Debug, Clone)]
25
+ pub struct Attribute {
26
+ pub name: String,
27
+ pub datatype: Datatype,
28
+ pub shape: Vec<u64>,
29
+ pub raw_data: Vec<u8>,
30
+ pub decoded_strings: Option<Vec<String>>,
31
+ }
32
+
33
+ impl Attribute {
34
+ /// Create from a parsed attribute message.
35
+ pub fn from_message(msg: AttributeMessage) -> Self {
36
+ Self::from_message_with_context(msg, None, 0)
37
+ }
38
+
39
+ /// Create from a parsed attribute message with optional file context for
40
+ /// resolving variable-length byte attributes stored in the global heap.
41
+ pub fn from_message_with_context(
42
+ msg: AttributeMessage,
43
+ file_data: Option<&[u8]>,
44
+ offset_size: u8,
45
+ ) -> Self {
46
+ let shape = match msg.dataspace.dataspace_type {
47
+ DataspaceType::Scalar => vec![],
48
+ DataspaceType::Null => vec![0],
49
+ DataspaceType::Simple => msg.dataspace.dims.clone(),
50
+ };
51
+ let decoded_strings = file_data.and_then(|file_data| {
52
+ decoded_vlen_strings(
53
+ &msg.datatype,
54
+ &msg.raw_data,
55
+ file_data,
56
+ offset_size,
57
+ msg.dataspace.num_elements().ok()?,
58
+ )
59
+ });
60
+ let raw_data = if let (Some(file_data), Datatype::VarLen { base, kind, .. }) =
61
+ (file_data, &msg.datatype)
62
+ {
63
+ if *kind == VarLenKind::String && is_byte_vlen(base) && shape.is_empty() {
64
+ resolve_vlen_bytes(&msg.raw_data, file_data, offset_size)
65
+ .unwrap_or_else(|| msg.raw_data.clone())
66
+ } else {
67
+ msg.raw_data.clone()
68
+ }
69
+ } else {
70
+ msg.raw_data.clone()
71
+ };
72
+ Attribute {
73
+ name: msg.name,
74
+ datatype: msg.datatype,
75
+ shape,
76
+ raw_data,
77
+ decoded_strings,
78
+ }
79
+ }
80
+
81
+ /// Total number of elements.
82
+ pub fn num_elements(&self) -> Result<u64> {
83
+ if self.shape.is_empty() {
84
+ Ok(1) // scalar
85
+ } else {
86
+ self.shape.iter().try_fold(1u64, |acc, &dim| {
87
+ acc.checked_mul(dim).ok_or_else(|| {
88
+ Error::InvalidData("attribute element count overflows u64".to_string())
89
+ })
90
+ })
91
+ }
92
+ }
93
+
94
+ /// Read the attribute value as a scalar of the given type.
95
+ pub fn read_scalar<T: crate::datatype_api::H5Type>(&self) -> Result<T> {
96
+ T::from_bytes(&self.raw_data, &self.datatype)
97
+ }
98
+
99
+ /// Read the attribute as a 1-D vector of the given type.
100
+ pub fn read_1d<T: crate::datatype_api::H5Type>(&self) -> Result<Vec<T>> {
101
+ let elem_size = T::element_size(&self.datatype);
102
+ let n = checked_usize(self.num_elements()?, "attribute element count")?;
103
+ let mut result = Vec::with_capacity(n);
104
+ for i in 0..n {
105
+ let start = i * elem_size;
106
+ let end = start + elem_size;
107
+ if end > self.raw_data.len() {
108
+ return Err(Error::InvalidData(format!(
109
+ "attribute data too short: need {} bytes, have {}",
110
+ end,
111
+ self.raw_data.len()
112
+ )));
113
+ }
114
+ result.push(T::from_bytes(&self.raw_data[start..end], &self.datatype)?);
115
+ }
116
+ Ok(result)
117
+ }
118
+
119
+ /// Read the attribute as a string (for string-typed attributes).
120
+ ///
121
+ /// For variable-length strings, use `read_vlen_string()` with the file data
122
+ /// and offset_size — this method will return an error directing you there.
123
+ pub fn read_string(&self) -> Result<String> {
124
+ if let Some(strings) = &self.decoded_strings {
125
+ return match strings.len() {
126
+ 1 => Ok(strings[0].clone()),
127
+ 0 => Err(Error::InvalidData(format!(
128
+ "attribute '{}' contains no string elements",
129
+ self.name
130
+ ))),
131
+ count => Err(Error::InvalidData(format!(
132
+ "attribute '{}' contains {count} string elements; use read_strings()",
133
+ self.name
134
+ ))),
135
+ };
136
+ }
137
+ match &self.datatype {
138
+ Datatype::VarLen {
139
+ base,
140
+ kind: VarLenKind::String,
141
+ ..
142
+ } if is_byte_vlen(base) => decode_varlen_byte_string(&self.raw_data),
143
+ Datatype::String {
144
+ size,
145
+ encoding,
146
+ padding,
147
+ } => match size {
148
+ StringSize::Fixed(len) => {
149
+ let len = *len as usize;
150
+ let bytes = if self.raw_data.len() < len {
151
+ &self.raw_data
152
+ } else {
153
+ &self.raw_data[..len]
154
+ };
155
+ decode_string(bytes, *padding, *encoding)
156
+ }
157
+ StringSize::Variable => {
158
+ // For inline vlen strings in attributes, try direct decode.
159
+ // If it looks like a global heap reference (>= 12 bytes for
160
+ // seq_len + addr + index), suggest read_vlen_string instead.
161
+ if self.raw_data.len() >= 12 {
162
+ // Try to decode directly first — some files inline the string
163
+ let trimmed = match padding {
164
+ StringPadding::NullTerminate => {
165
+ let end = self
166
+ .raw_data
167
+ .iter()
168
+ .position(|&b| b == 0)
169
+ .unwrap_or(self.raw_data.len());
170
+ &self.raw_data[..end]
171
+ }
172
+ _ => &self.raw_data,
173
+ };
174
+ if let Ok(s) = String::from_utf8(trimmed.to_vec()) {
175
+ if s.chars()
176
+ .all(|c| !c.is_control() || c == '\n' || c == '\r' || c == '\t')
177
+ {
178
+ return Ok(s);
179
+ }
180
+ }
181
+ }
182
+ decode_string(&self.raw_data, *padding, *encoding)
183
+ }
184
+ },
185
+ _ => Err(Error::TypeMismatch {
186
+ expected: "String".into(),
187
+ actual: format!("{:?}", self.datatype),
188
+ }),
189
+ }
190
+ }
191
+
192
+ /// Read a variable-length string attribute from the global heap.
193
+ ///
194
+ /// Variable-length strings in HDF5 are stored as references into a global
195
+ /// heap collection. Each reference is: `seq_len(u32) + heap_addr(offset_size) + index(u32)`.
196
+ pub fn read_vlen_string(&self, file_data: &[u8], offset_size: u8) -> Result<String> {
197
+ if let Some(strings) = &self.decoded_strings {
198
+ return match strings.len() {
199
+ 1 => Ok(strings[0].clone()),
200
+ 0 => Err(Error::InvalidData(format!(
201
+ "attribute '{}' contains no string elements",
202
+ self.name
203
+ ))),
204
+ count => Err(Error::InvalidData(format!(
205
+ "attribute '{}' contains {count} string elements; use read_vlen_strings()",
206
+ self.name
207
+ ))),
208
+ };
209
+ }
210
+ match &self.datatype {
211
+ Datatype::String {
212
+ size: StringSize::Variable,
213
+ encoding,
214
+ padding,
215
+ } => {
216
+ let ref_size = 4 + offset_size as usize + 4; // seq_len + addr + index
217
+ if self.raw_data.len() < ref_size {
218
+ // Fallback: try direct decode
219
+ return decode_string(&self.raw_data, *padding, *encoding);
220
+ }
221
+ let bytes = read_one_vlen_string(
222
+ &self.raw_data,
223
+ 0,
224
+ file_data,
225
+ offset_size,
226
+ *padding,
227
+ *encoding,
228
+ )?;
229
+ Ok(bytes)
230
+ }
231
+ Datatype::String {
232
+ size: StringSize::Fixed(_),
233
+ ..
234
+ } => self.read_string(),
235
+ _ => Err(Error::TypeMismatch {
236
+ expected: "String".into(),
237
+ actual: format!("{:?}", self.datatype),
238
+ }),
239
+ }
240
+ }
241
+
242
+ /// Read an array of variable-length strings from the global heap.
243
+ pub fn read_vlen_strings(&self, file_data: &[u8], offset_size: u8) -> Result<Vec<String>> {
244
+ if let Some(strings) = &self.decoded_strings {
245
+ return Ok(strings.clone());
246
+ }
247
+ match &self.datatype {
248
+ Datatype::String {
249
+ size: StringSize::Variable,
250
+ encoding,
251
+ padding,
252
+ } => {
253
+ let ref_size = 4 + offset_size as usize + 4;
254
+ let n = checked_usize(self.num_elements()?, "attribute string element count")?;
255
+ let mut result = Vec::with_capacity(n);
256
+ for i in 0..n {
257
+ let offset = i * ref_size;
258
+ if offset + ref_size > self.raw_data.len() {
259
+ break;
260
+ }
261
+ result.push(read_one_vlen_string(
262
+ &self.raw_data,
263
+ offset,
264
+ file_data,
265
+ offset_size,
266
+ *padding,
267
+ *encoding,
268
+ )?);
269
+ }
270
+ Ok(result)
271
+ }
272
+ Datatype::String {
273
+ size: StringSize::Fixed(_),
274
+ ..
275
+ } => self.read_strings(),
276
+ _ => Err(Error::TypeMismatch {
277
+ expected: "String array".into(),
278
+ actual: format!("{:?}", self.datatype),
279
+ }),
280
+ }
281
+ }
282
+
283
+ /// Read the attribute as a vector of strings.
284
+ pub fn read_strings(&self) -> Result<Vec<String>> {
285
+ if let Some(strings) = &self.decoded_strings {
286
+ return Ok(strings.clone());
287
+ }
288
+ match &self.datatype {
289
+ Datatype::String {
290
+ size: StringSize::Fixed(len),
291
+ encoding,
292
+ padding,
293
+ } => {
294
+ let len = *len as usize;
295
+ let n = checked_usize(self.num_elements()?, "attribute string element count")?;
296
+ let mut result = Vec::with_capacity(n);
297
+ for i in 0..n {
298
+ let start = i * len;
299
+ let end = (start + len).min(self.raw_data.len());
300
+ if start >= self.raw_data.len() {
301
+ break;
302
+ }
303
+ result.push(decode_string(
304
+ &self.raw_data[start..end],
305
+ *padding,
306
+ *encoding,
307
+ )?);
308
+ }
309
+ Ok(result)
310
+ }
311
+ _ => Err(Error::TypeMismatch {
312
+ expected: "String array".into(),
313
+ actual: format!("{:?}", self.datatype),
314
+ }),
315
+ }
316
+ }
317
+
318
+ /// Read an attribute as f64 (with automatic promotion from int types).
319
+ pub fn read_as_f64(&self) -> Result<f64> {
320
+ match &self.datatype {
321
+ Datatype::FloatingPoint { size, .. } => {
322
+ let val: f64 = match size {
323
+ 4 => {
324
+ let v = self.read_scalar::<f32>()?;
325
+ v as f64
326
+ }
327
+ 8 => self.read_scalar::<f64>()?,
328
+ _ => {
329
+ return Err(Error::TypeMismatch {
330
+ expected: "f32 or f64".into(),
331
+ actual: format!("FloatingPoint(size={})", size),
332
+ })
333
+ }
334
+ };
335
+ Ok(val)
336
+ }
337
+ Datatype::FixedPoint { size, signed, .. } => {
338
+ let val = match (size, signed) {
339
+ (1, true) => self.read_scalar::<i8>()? as f64,
340
+ (1, false) => self.read_scalar::<u8>()? as f64,
341
+ (2, true) => self.read_scalar::<i16>()? as f64,
342
+ (2, false) => self.read_scalar::<u16>()? as f64,
343
+ (4, true) => self.read_scalar::<i32>()? as f64,
344
+ (4, false) => self.read_scalar::<u32>()? as f64,
345
+ (8, true) => self.read_scalar::<i64>()? as f64,
346
+ (8, false) => self.read_scalar::<u64>()? as f64,
347
+ _ => {
348
+ return Err(Error::TypeMismatch {
349
+ expected: "numeric".into(),
350
+ actual: format!("FixedPoint(size={})", size),
351
+ })
352
+ }
353
+ };
354
+ Ok(val)
355
+ }
356
+ _ => Err(Error::TypeMismatch {
357
+ expected: "numeric".into(),
358
+ actual: format!("{:?}", self.datatype),
359
+ }),
360
+ }
361
+ }
362
+ }
363
+
364
+ pub(crate) fn collect_attribute_messages_storage(
365
+ header: &ObjectHeader,
366
+ storage: &dyn Storage,
367
+ offset_size: u8,
368
+ length_size: u8,
369
+ filter_registry: Option<&FilterRegistry>,
370
+ ) -> Result<Vec<AttributeMessage>> {
371
+ let mut attributes = Vec::new();
372
+ let mut attribute_info = None;
373
+
374
+ for msg in &header.messages {
375
+ match msg {
376
+ HdfMessage::Attribute(attr) => attributes.push(attr.clone()),
377
+ HdfMessage::AttributeInfo(info) => attribute_info = Some(info.clone()),
378
+ _ => {}
379
+ }
380
+ }
381
+
382
+ if let Some(info) = attribute_info {
383
+ attributes.extend(load_dense_attribute_messages_storage(
384
+ &info,
385
+ storage,
386
+ offset_size,
387
+ length_size,
388
+ filter_registry,
389
+ )?);
390
+ }
391
+
392
+ Ok(attributes)
393
+ }
394
+
395
+ fn load_dense_attribute_messages_storage(
396
+ info: &AttributeInfoMessage,
397
+ storage: &dyn Storage,
398
+ offset_size: u8,
399
+ length_size: u8,
400
+ filter_registry: Option<&FilterRegistry>,
401
+ ) -> Result<Vec<AttributeMessage>> {
402
+ if Cursor::is_undefined_offset(info.fractal_heap_address, offset_size) {
403
+ return Ok(Vec::new());
404
+ }
405
+
406
+ let heap = FractalHeap::parse_at_storage(
407
+ storage,
408
+ info.fractal_heap_address,
409
+ offset_size,
410
+ length_size,
411
+ )?;
412
+
413
+ let records = load_dense_attribute_records_storage(info, storage, offset_size, length_size)?;
414
+
415
+ let mut attributes = Vec::new();
416
+ for record in records {
417
+ let heap_id = match record {
418
+ btree_v2::BTreeV2Record::AttributeNameHash { heap_id, .. }
419
+ | btree_v2::BTreeV2Record::AttributeCreationOrder { heap_id, .. } => heap_id,
420
+ _ => continue,
421
+ };
422
+
423
+ let managed_bytes = heap.get_object_storage_with_registry(
424
+ &heap_id,
425
+ storage,
426
+ offset_size,
427
+ length_size,
428
+ filter_registry,
429
+ )?;
430
+
431
+ let mut attr_cursor = Cursor::new(&managed_bytes);
432
+ let attr = messages::attribute::parse(
433
+ &mut attr_cursor,
434
+ offset_size,
435
+ length_size,
436
+ managed_bytes.len(),
437
+ )?;
438
+ attributes.push(attr);
439
+ }
440
+
441
+ Ok(attributes)
442
+ }
443
+
444
+ fn load_dense_attribute_records_storage(
445
+ info: &AttributeInfoMessage,
446
+ storage: &dyn Storage,
447
+ offset_size: u8,
448
+ length_size: u8,
449
+ ) -> Result<Vec<btree_v2::BTreeV2Record>> {
450
+ let mut addrs = Vec::new();
451
+ // Match compact-attribute and NetCDF enumeration order when the object
452
+ // provides a creation-order index; the name index remains the fallback.
453
+ if let Some(creation_order_addr) = info.btree_creation_order_address {
454
+ addrs.push(("creation-order", creation_order_addr));
455
+ }
456
+ addrs.push(("name", info.btree_name_index_address));
457
+
458
+ let mut last_error = None;
459
+ for (index_name, addr) in addrs {
460
+ if Cursor::is_undefined_offset(addr, offset_size) {
461
+ continue;
462
+ }
463
+
464
+ let header = match btree_v2::BTreeV2Header::parse_at_storage(
465
+ storage,
466
+ addr,
467
+ offset_size,
468
+ length_size,
469
+ ) {
470
+ Ok(header) => header,
471
+ Err(err) => {
472
+ last_error = Some(format!(
473
+ "failed to parse dense attribute {index_name} B-tree at {addr:#x}: {err}"
474
+ ));
475
+ continue;
476
+ }
477
+ };
478
+
479
+ match btree_v2::collect_btree_v2_records_storage(
480
+ storage,
481
+ &header,
482
+ offset_size,
483
+ length_size,
484
+ None,
485
+ &[],
486
+ None,
487
+ ) {
488
+ Ok(records) => return Ok(records),
489
+ Err(err) => {
490
+ last_error = Some(format!(
491
+ "failed to read dense attribute {index_name} B-tree at {addr:#x}: {err}"
492
+ ));
493
+ }
494
+ }
495
+ }
496
+
497
+ if let Some(err) = last_error {
498
+ Err(Error::InvalidData(format!(
499
+ "failed to load dense attribute records: {err}"
500
+ )))
501
+ } else {
502
+ Ok(Vec::new())
503
+ }
504
+ }
505
+
506
+ /// Read one variable-length string from a vlen reference in raw_data.
507
+ pub(crate) fn read_one_vlen_string(
508
+ raw_data: &[u8],
509
+ offset: usize,
510
+ file_data: &[u8],
511
+ offset_size: u8,
512
+ padding: StringPadding,
513
+ encoding: StringEncoding,
514
+ ) -> Result<String> {
515
+ let mut cursor = Cursor::new(&raw_data[offset..]);
516
+ let _seq_len = cursor.read_u32_le()?;
517
+ let heap_addr = cursor.read_offset(offset_size)?;
518
+ let obj_index = cursor.read_u32_le()?;
519
+
520
+ if Cursor::is_undefined_offset(heap_addr, offset_size) || obj_index == 0 {
521
+ return Ok(String::new());
522
+ }
523
+
524
+ let mut heap_cursor = Cursor::new(file_data);
525
+ heap_cursor.set_position(heap_addr);
526
+ let collection = GlobalHeapCollection::parse(&mut heap_cursor, offset_size, offset_size)?;
527
+
528
+ match collection.get_object(obj_index as u16) {
529
+ Some(obj) => decode_string(&obj.data, padding, encoding),
530
+ None => Ok(String::new()),
531
+ }
532
+ }
533
+
534
+ pub(crate) fn read_one_vlen_string_storage(
535
+ raw_data: &[u8],
536
+ offset: usize,
537
+ storage: &dyn Storage,
538
+ offset_size: u8,
539
+ length_size: u8,
540
+ padding: StringPadding,
541
+ encoding: StringEncoding,
542
+ ) -> Result<String> {
543
+ let mut cursor = Cursor::new(&raw_data[offset..]);
544
+ let _seq_len = cursor.read_u32_le()?;
545
+ let heap_addr = cursor.read_offset(offset_size)?;
546
+ let obj_index = cursor.read_u32_le()?;
547
+
548
+ if Cursor::is_undefined_offset(heap_addr, offset_size) || obj_index == 0 {
549
+ return Ok(String::new());
550
+ }
551
+
552
+ let collection =
553
+ GlobalHeapCollection::parse_at_storage(storage, heap_addr, offset_size, length_size)?;
554
+ match collection.get_object(obj_index as u16) {
555
+ Some(obj) => decode_string(&obj.data, padding, encoding),
556
+ None => Ok(String::new()),
557
+ }
558
+ }
559
+
560
+ pub(crate) fn decoded_vlen_strings_storage(
561
+ datatype: &Datatype,
562
+ raw_data: &[u8],
563
+ storage: &dyn Storage,
564
+ offset_size: u8,
565
+ length_size: u8,
566
+ element_count: u64,
567
+ ) -> Option<Vec<String>> {
568
+ let (padding, encoding) = vlen_string_format(datatype)?;
569
+ let ref_size = 4 + offset_size as usize + 4;
570
+ let n = checked_usize(element_count, "attribute string element count").ok()?;
571
+ if raw_data.len() < n.checked_mul(ref_size)? {
572
+ return None;
573
+ }
574
+ let mut result = Vec::with_capacity(n);
575
+ for i in 0..n {
576
+ result.push(
577
+ read_one_vlen_string_storage(
578
+ raw_data,
579
+ i * ref_size,
580
+ storage,
581
+ offset_size,
582
+ length_size,
583
+ padding,
584
+ encoding,
585
+ )
586
+ .ok()?,
587
+ );
588
+ }
589
+ Some(result)
590
+ }
591
+
592
+ fn decoded_vlen_strings(
593
+ datatype: &Datatype,
594
+ raw_data: &[u8],
595
+ file_data: &[u8],
596
+ offset_size: u8,
597
+ element_count: u64,
598
+ ) -> Option<Vec<String>> {
599
+ let (padding, encoding) = vlen_string_format(datatype)?;
600
+ let ref_size = 4 + offset_size as usize + 4;
601
+ let n = checked_usize(element_count, "attribute string element count").ok()?;
602
+ if raw_data.len() < n.checked_mul(ref_size)? {
603
+ return None;
604
+ }
605
+ let mut result = Vec::with_capacity(n);
606
+ for i in 0..n {
607
+ result.push(
608
+ read_one_vlen_string(
609
+ raw_data,
610
+ i * ref_size,
611
+ file_data,
612
+ offset_size,
613
+ padding,
614
+ encoding,
615
+ )
616
+ .ok()?,
617
+ );
618
+ }
619
+ Some(result)
620
+ }
621
+
622
+ fn vlen_string_format(datatype: &Datatype) -> Option<(StringPadding, StringEncoding)> {
623
+ match datatype {
624
+ Datatype::String {
625
+ size: StringSize::Variable,
626
+ padding,
627
+ encoding,
628
+ } => Some((*padding, *encoding)),
629
+ Datatype::VarLen {
630
+ base,
631
+ kind: VarLenKind::String,
632
+ padding,
633
+ encoding,
634
+ } if is_byte_vlen(base) => Some((*padding, *encoding)),
635
+ _ => None,
636
+ }
637
+ }
638
+
639
+ /// Decode a byte slice into a String, handling padding and encoding.
640
+ ///
641
+ /// HDF5 supports ASCII and UTF-8 string encodings. Both are valid UTF-8
642
+ /// (ASCII is a strict subset), so we decode uniformly via `from_utf8`.
643
+ pub(crate) fn decode_string(
644
+ bytes: &[u8],
645
+ padding: StringPadding,
646
+ _encoding: StringEncoding,
647
+ ) -> Result<String> {
648
+ let trimmed = match padding {
649
+ StringPadding::NullTerminate => {
650
+ let end = bytes.iter().position(|&b| b == 0).unwrap_or(bytes.len());
651
+ &bytes[..end]
652
+ }
653
+ StringPadding::NullPad => {
654
+ let end = bytes.iter().rposition(|&b| b != 0).map_or(0, |i| i + 1);
655
+ &bytes[..end]
656
+ }
657
+ StringPadding::SpacePad => {
658
+ let end = bytes.iter().rposition(|&b| b != b' ').map_or(0, |i| i + 1);
659
+ &bytes[..end]
660
+ }
661
+ };
662
+
663
+ String::from_utf8(trimmed.to_vec())
664
+ .map_err(|e| Error::InvalidData(format!("invalid string data: {e}")))
665
+ }
666
+
667
+ fn is_byte_vlen(base: &Datatype) -> bool {
668
+ base.is_vlen_string_base()
669
+ }
670
+
671
+ pub(crate) fn decode_varlen_byte_string(bytes: &[u8]) -> Result<String> {
672
+ let end = bytes.iter().position(|&b| b == 0).unwrap_or(bytes.len());
673
+ String::from_utf8(bytes[..end].to_vec())
674
+ .map_err(|e| Error::InvalidData(format!("invalid string data: {e}")))
675
+ }
676
+
677
+ pub(crate) fn resolve_vlen_bytes(
678
+ raw_data: &[u8],
679
+ file_data: &[u8],
680
+ offset_size: u8,
681
+ ) -> Option<Vec<u8>> {
682
+ if raw_data.len() < 4 + offset_size as usize + 4 {
683
+ return None;
684
+ }
685
+
686
+ let mut cursor = Cursor::new(raw_data);
687
+ let seq_len = cursor.read_u32_le().ok()? as usize;
688
+ let heap_addr = cursor.read_offset(offset_size).ok()?;
689
+ let obj_index = cursor.read_u32_le().ok()? as u16;
690
+
691
+ if Cursor::is_undefined_offset(heap_addr, offset_size) || obj_index == 0 {
692
+ return Some(Vec::new());
693
+ }
694
+
695
+ let mut heap_cursor = Cursor::new(file_data);
696
+ heap_cursor.set_position(heap_addr);
697
+ let collection =
698
+ GlobalHeapCollection::parse(&mut heap_cursor, offset_size, offset_size).ok()?;
699
+ let object = collection.get_object(obj_index)?;
700
+ Some(object.data[..object.data.len().min(seq_len)].to_vec())
701
+ }
702
+
703
+ pub(crate) fn resolve_vlen_bytes_storage(
704
+ raw_data: &[u8],
705
+ storage: &dyn Storage,
706
+ offset_size: u8,
707
+ length_size: u8,
708
+ ) -> Option<Vec<u8>> {
709
+ if raw_data.len() < 4 + offset_size as usize + 4 {
710
+ return None;
711
+ }
712
+
713
+ let mut cursor = Cursor::new(raw_data);
714
+ let seq_len = cursor.read_u32_le().ok()? as usize;
715
+ let heap_addr = cursor.read_offset(offset_size).ok()?;
716
+ let obj_index = cursor.read_u32_le().ok()? as u16;
717
+
718
+ if Cursor::is_undefined_offset(heap_addr, offset_size) || obj_index == 0 {
719
+ return Some(Vec::new());
720
+ }
721
+
722
+ let collection =
723
+ GlobalHeapCollection::parse_at_storage(storage, heap_addr, offset_size, length_size)
724
+ .ok()?;
725
+ let object = collection.get_object(obj_index)?;
726
+ Some(object.data[..object.data.len().min(seq_len)].to_vec())
727
+ }
728
+
729
+ #[cfg(test)]
730
+ mod tests {
731
+ use super::*;
732
+ use crate::error::ByteOrder;
733
+ use crate::storage::BytesStorage;
734
+ use std::f64::consts::PI;
735
+
736
+ #[test]
737
+ fn scalar_f64_attribute() {
738
+ let value: f64 = PI;
739
+ let raw_data = value.to_le_bytes().to_vec();
740
+ let attr = Attribute {
741
+ name: "pi".to_string(),
742
+ datatype: Datatype::FloatingPoint {
743
+ size: 8,
744
+ byte_order: ByteOrder::LittleEndian,
745
+ },
746
+ shape: vec![],
747
+ raw_data,
748
+ decoded_strings: None,
749
+ };
750
+ let val = attr.read_scalar::<f64>().unwrap();
751
+ assert!((val - PI).abs() < 1e-10);
752
+ }
753
+
754
+ #[test]
755
+ fn one_dimensional_i32_attribute() {
756
+ let values = [1i32, 2, 3, 4];
757
+ let mut raw_data = Vec::new();
758
+ for v in &values {
759
+ raw_data.extend_from_slice(&v.to_le_bytes());
760
+ }
761
+ let attr = Attribute {
762
+ name: "data".to_string(),
763
+ datatype: Datatype::FixedPoint {
764
+ size: 4,
765
+ signed: true,
766
+ byte_order: ByteOrder::LittleEndian,
767
+ },
768
+ shape: vec![4],
769
+ raw_data,
770
+ decoded_strings: None,
771
+ };
772
+ let result = attr.read_1d::<i32>().unwrap();
773
+ assert_eq!(result, vec![1, 2, 3, 4]);
774
+ }
775
+
776
+ #[test]
777
+ fn string_attribute() {
778
+ let attr = Attribute {
779
+ name: "units".to_string(),
780
+ datatype: Datatype::String {
781
+ size: StringSize::Fixed(10),
782
+ encoding: StringEncoding::Ascii,
783
+ padding: StringPadding::NullPad,
784
+ },
785
+ shape: vec![],
786
+ raw_data: b"meters\0\0\0\0".to_vec(),
787
+ decoded_strings: None,
788
+ };
789
+ assert_eq!(attr.read_string().unwrap(), "meters");
790
+ }
791
+
792
+ #[test]
793
+ fn varlen_byte_string_attribute() {
794
+ let attr = Attribute {
795
+ name: "name".to_string(),
796
+ datatype: Datatype::VarLen {
797
+ base: Box::new(Datatype::FixedPoint {
798
+ size: 1,
799
+ signed: false,
800
+ byte_order: ByteOrder::LittleEndian,
801
+ }),
802
+ kind: VarLenKind::String,
803
+ encoding: StringEncoding::Utf8,
804
+ padding: StringPadding::NullTerminate,
805
+ },
806
+ shape: vec![],
807
+ raw_data: b"test_dataset".to_vec(),
808
+ decoded_strings: None,
809
+ };
810
+ assert_eq!(attr.read_string().unwrap(), "test_dataset");
811
+ }
812
+
813
+ #[test]
814
+ fn read_as_f64_from_int() {
815
+ let raw_data = 42i32.to_le_bytes().to_vec();
816
+ let attr = Attribute {
817
+ name: "count".to_string(),
818
+ datatype: Datatype::FixedPoint {
819
+ size: 4,
820
+ signed: true,
821
+ byte_order: ByteOrder::LittleEndian,
822
+ },
823
+ shape: vec![],
824
+ raw_data,
825
+ decoded_strings: None,
826
+ };
827
+ let val = attr.read_as_f64().unwrap();
828
+ assert!((val - 42.0).abs() < 1e-10);
829
+ }
830
+
831
+ #[test]
832
+ fn dense_attribute_btree_errors_surface() {
833
+ let info = AttributeInfoMessage {
834
+ creation_order_tracked: false,
835
+ creation_order_indexed: false,
836
+ max_creation_index: None,
837
+ fractal_heap_address: 0,
838
+ btree_name_index_address: 0,
839
+ btree_creation_order_address: None,
840
+ };
841
+ let storage = BytesStorage::new(Vec::new());
842
+
843
+ let err = load_dense_attribute_records_storage(&info, &storage, 8, 8).unwrap_err();
844
+ assert!(matches!(err, Error::InvalidData(_)));
845
+ assert!(err.to_string().contains("dense attribute"));
846
+ }
847
+ }