@nirs4all/formats-wasm 0.2.12 → 0.2.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/THIRD_PARTY_NOTICES.md +49 -0
  2. package/nirs4all_formats_wasm.d.ts +4 -4
  3. package/nirs4all_formats_wasm.js +170 -149
  4. package/nirs4all_formats_wasm_bg.wasm +0 -0
  5. package/package.json +4 -2
  6. package/reader-backports/hdf5-reader/Cargo.toml +119 -0
  7. package/reader-backports/hdf5-reader/LICENSE-APACHE +164 -0
  8. package/reader-backports/hdf5-reader/LICENSE-MIT +21 -0
  9. package/reader-backports/hdf5-reader/PROVENANCE.json +80 -0
  10. package/reader-backports/hdf5-reader/README.md +308 -0
  11. package/reader-backports/hdf5-reader/UPSTREAM-Cargo.toml +44 -0
  12. package/reader-backports/hdf5-reader/UPSTREAM-VCS.json +6 -0
  13. package/reader-backports/hdf5-reader/src/attribute_api.rs +847 -0
  14. package/reader-backports/hdf5-reader/src/btree_v1.rs +726 -0
  15. package/reader-backports/hdf5-reader/src/btree_v2.rs +2068 -0
  16. package/reader-backports/hdf5-reader/src/cache.rs +354 -0
  17. package/reader-backports/hdf5-reader/src/checksum.rs +56 -0
  18. package/reader-backports/hdf5-reader/src/chunk_index.rs +423 -0
  19. package/reader-backports/hdf5-reader/src/dataset.rs +4733 -0
  20. package/reader-backports/hdf5-reader/src/datatype_api.rs +497 -0
  21. package/reader-backports/hdf5-reader/src/error.rs +167 -0
  22. package/reader-backports/hdf5-reader/src/extensible_array.rs +1886 -0
  23. package/reader-backports/hdf5-reader/src/filters/deflate.rs +93 -0
  24. package/reader-backports/hdf5-reader/src/filters/fletcher32.rs +52 -0
  25. package/reader-backports/hdf5-reader/src/filters/lz4.rs +178 -0
  26. package/reader-backports/hdf5-reader/src/filters/mod.rs +326 -0
  27. package/reader-backports/hdf5-reader/src/filters/nbit.rs +586 -0
  28. package/reader-backports/hdf5-reader/src/filters/scaleoffset.rs +827 -0
  29. package/reader-backports/hdf5-reader/src/filters/shuffle.rs +99 -0
  30. package/reader-backports/hdf5-reader/src/fixed_array.rs +763 -0
  31. package/reader-backports/hdf5-reader/src/fractal_heap.rs +2029 -0
  32. package/reader-backports/hdf5-reader/src/global_heap.rs +284 -0
  33. package/reader-backports/hdf5-reader/src/group.rs +795 -0
  34. package/reader-backports/hdf5-reader/src/io.rs +413 -0
  35. package/reader-backports/hdf5-reader/src/lib.rs +916 -0
  36. package/reader-backports/hdf5-reader/src/local_heap.rs +261 -0
  37. package/reader-backports/hdf5-reader/src/messages/attribute.rs +304 -0
  38. package/reader-backports/hdf5-reader/src/messages/attribute_info.rs +96 -0
  39. package/reader-backports/hdf5-reader/src/messages/btree_k.rs +36 -0
  40. package/reader-backports/hdf5-reader/src/messages/continuation.rs +46 -0
  41. package/reader-backports/hdf5-reader/src/messages/dataspace.rs +259 -0
  42. package/reader-backports/hdf5-reader/src/messages/datatype.rs +938 -0
  43. package/reader-backports/hdf5-reader/src/messages/external_files.rs +50 -0
  44. package/reader-backports/hdf5-reader/src/messages/fill_value.rs +233 -0
  45. package/reader-backports/hdf5-reader/src/messages/filter_pipeline.rs +147 -0
  46. package/reader-backports/hdf5-reader/src/messages/group_info.rs +103 -0
  47. package/reader-backports/hdf5-reader/src/messages/layout.rs +476 -0
  48. package/reader-backports/hdf5-reader/src/messages/link.rs +170 -0
  49. package/reader-backports/hdf5-reader/src/messages/link_info.rs +127 -0
  50. package/reader-backports/hdf5-reader/src/messages/mod.rs +374 -0
  51. package/reader-backports/hdf5-reader/src/messages/modification_time.rs +83 -0
  52. package/reader-backports/hdf5-reader/src/messages/shared.rs +164 -0
  53. package/reader-backports/hdf5-reader/src/messages/shared_table.rs +67 -0
  54. package/reader-backports/hdf5-reader/src/messages/symbol_table_msg.rs +51 -0
  55. package/reader-backports/hdf5-reader/src/object_header.rs +1749 -0
  56. package/reader-backports/hdf5-reader/src/reference.rs +90 -0
  57. package/reader-backports/hdf5-reader/src/shared_message_table.rs +425 -0
  58. package/reader-backports/hdf5-reader/src/storage.rs +538 -0
  59. package/reader-backports/hdf5-reader/src/superblock.rs +285 -0
  60. package/reader-backports/hdf5-reader/src/symbol_table.rs +277 -0
  61. package/reader-backports/netcdf-reader/Cargo.toml +141 -0
  62. package/reader-backports/netcdf-reader/LICENSE-APACHE +164 -0
  63. package/reader-backports/netcdf-reader/LICENSE-MIT +21 -0
  64. package/reader-backports/netcdf-reader/PROVENANCE.json +55 -0
  65. package/reader-backports/netcdf-reader/README.md +308 -0
  66. package/reader-backports/netcdf-reader/UPSTREAM-Cargo.toml +48 -0
  67. package/reader-backports/netcdf-reader/UPSTREAM-VCS.json +6 -0
  68. package/reader-backports/netcdf-reader/src/cf/axes.rs +419 -0
  69. package/reader-backports/netcdf-reader/src/cf/bounds.rs +108 -0
  70. package/reader-backports/netcdf-reader/src/cf/crs.rs +269 -0
  71. package/reader-backports/netcdf-reader/src/cf/mod.rs +30 -0
  72. package/reader-backports/netcdf-reader/src/cf/time.rs +1122 -0
  73. package/reader-backports/netcdf-reader/src/classic/data.rs +1465 -0
  74. package/reader-backports/netcdf-reader/src/classic/header.rs +1097 -0
  75. package/reader-backports/netcdf-reader/src/classic/mod.rs +240 -0
  76. package/reader-backports/netcdf-reader/src/classic/storage.rs +211 -0
  77. package/reader-backports/netcdf-reader/src/classic/types.rs +93 -0
  78. package/reader-backports/netcdf-reader/src/classic/variable.rs +1162 -0
  79. package/reader-backports/netcdf-reader/src/error.rs +59 -0
  80. package/reader-backports/netcdf-reader/src/lib.rs +1531 -0
  81. package/reader-backports/netcdf-reader/src/masked.rs +173 -0
  82. package/reader-backports/netcdf-reader/src/nc4/attributes.rs +203 -0
  83. package/reader-backports/netcdf-reader/src/nc4/dimensions.rs +139 -0
  84. package/reader-backports/netcdf-reader/src/nc4/groups.rs +308 -0
  85. package/reader-backports/netcdf-reader/src/nc4/mod.rs +572 -0
  86. package/reader-backports/netcdf-reader/src/nc4/types.rs +309 -0
  87. package/reader-backports/netcdf-reader/src/nc4/variables.rs +420 -0
  88. package/reader-backports/netcdf-reader/src/types.rs +18 -0
  89. package/reader-backports/netcdf-reader/src/unpack.rs +113 -0
  90. package/reader-backports/netcdf-reader/src/user_defined.rs +569 -0
@@ -0,0 +1,423 @@
1
+ //! Chunk indexing — resolves chunk locations from various storage strategies.
2
+ //!
3
+ //! Supports all six HDF5 chunk indexing types:
4
+ //! - V1 B-tree chunk indexing (btree_v1 type 1) — dispatched externally
5
+ //! - V2 B-tree chunk indexing (btree_v2 types 10 and 11)
6
+ //! - Single chunk indexing
7
+ //! - Implicit chunk indexing
8
+ //! - Fixed array indexing (fixed_array module)
9
+ //! - Extensible array indexing (extensible_array module)
10
+
11
+ use crate::error::{Error, Result};
12
+ use crate::storage::Storage;
13
+
14
+ /// A resolved chunk location within the file.
15
+ #[derive(Debug, Clone)]
16
+ pub struct ChunkEntry {
17
+ /// Absolute file address of the chunk data.
18
+ pub address: u64,
19
+ /// Size of the chunk data in bytes (after filtering, i.e., on-disk size).
20
+ pub size: u64,
21
+ /// Filter mask — each bit indicates whether the corresponding filter
22
+ /// in the pipeline was skipped (1 = skipped).
23
+ pub filter_mask: u32,
24
+ /// Chunk offsets within the dataset (one per dimension).
25
+ pub offsets: Vec<u64>,
26
+ }
27
+
28
+ fn chunk_overlaps_bounds(
29
+ offsets: &[u64],
30
+ chunk_dims: &[u32],
31
+ chunk_bounds: Option<(&[u64], &[u64])>,
32
+ ) -> bool {
33
+ let Some((first_chunk, last_chunk)) = chunk_bounds else {
34
+ return true;
35
+ };
36
+
37
+ offsets.iter().enumerate().all(|(dim, offset)| {
38
+ let chunk_index = *offset / u64::from(chunk_dims[dim]);
39
+ chunk_index >= first_chunk[dim] && chunk_index <= last_chunk[dim]
40
+ })
41
+ }
42
+
43
+ fn checked_mul_u64(lhs: u64, rhs: u64, context: &str) -> Result<u64> {
44
+ lhs.checked_mul(rhs)
45
+ .ok_or_else(|| Error::InvalidData(format!("{context} overflows u64")))
46
+ }
47
+
48
+ fn checked_add_u64(lhs: u64, rhs: u64, context: &str) -> Result<u64> {
49
+ lhs.checked_add(rhs)
50
+ .ok_or_else(|| Error::InvalidData(format!("{context} overflows u64")))
51
+ }
52
+
53
+ fn checked_usize(value: u64, context: &str) -> Result<usize> {
54
+ usize::try_from(value).map_err(|_| {
55
+ Error::InvalidData(format!(
56
+ "{context} value {value} exceeds platform usize capacity"
57
+ ))
58
+ })
59
+ }
60
+
61
+ fn chunk_linear_index(chunk_indices: &[u64], chunks_per_dim: &[u64]) -> Result<u64> {
62
+ let mut linear = 0u64;
63
+ for (dim, chunk_index) in chunk_indices.iter().enumerate() {
64
+ linear = checked_mul_u64(linear, chunks_per_dim[dim], "implicit chunk linear index")?;
65
+ linear = checked_add_u64(linear, *chunk_index, "implicit chunk linear index")?;
66
+ }
67
+ Ok(linear)
68
+ }
69
+
70
+ /// Collect chunk entries from a B-tree v2 chunk index.
71
+ pub fn collect_v2_chunk_entries(
72
+ data: &[u8],
73
+ btree_address: u64,
74
+ offset_size: u8,
75
+ length_size: u8,
76
+ ndim: u32,
77
+ chunk_dims: &[u32],
78
+ chunk_bounds: Option<(&[u64], &[u64])>,
79
+ ) -> Result<Vec<ChunkEntry>> {
80
+ let mut cursor = crate::io::Cursor::new(data);
81
+ cursor.set_position(btree_address);
82
+ let header = crate::btree_v2::BTreeV2Header::parse(&mut cursor, offset_size, length_size)?;
83
+
84
+ let records = crate::btree_v2::collect_btree_v2_records(
85
+ data,
86
+ &header,
87
+ offset_size,
88
+ length_size,
89
+ Some(ndim),
90
+ chunk_dims,
91
+ chunk_bounds,
92
+ )?;
93
+
94
+ let mut entries = Vec::with_capacity(records.len());
95
+ for record in records {
96
+ match record {
97
+ crate::btree_v2::BTreeV2Record::ChunkedNonFiltered { address, offsets }
98
+ if chunk_overlaps_bounds(&offsets, chunk_dims, chunk_bounds) =>
99
+ {
100
+ entries.push(ChunkEntry {
101
+ address,
102
+ size: 0, // caller must compute from chunk dims * elem_size
103
+ filter_mask: 0,
104
+ offsets,
105
+ });
106
+ }
107
+ crate::btree_v2::BTreeV2Record::ChunkedFiltered {
108
+ address,
109
+ chunk_size,
110
+ filter_mask,
111
+ offsets,
112
+ } if chunk_overlaps_bounds(&offsets, chunk_dims, chunk_bounds) => {
113
+ entries.push(ChunkEntry {
114
+ address,
115
+ size: chunk_size,
116
+ filter_mask,
117
+ offsets,
118
+ });
119
+ }
120
+ _ => {
121
+ // Skip non-chunk records
122
+ }
123
+ }
124
+ }
125
+
126
+ Ok(entries)
127
+ }
128
+
129
+ /// Collect chunk entries from a B-tree v2 chunk index using random-access storage.
130
+ pub fn collect_v2_chunk_entries_storage(
131
+ storage: &dyn Storage,
132
+ btree_address: u64,
133
+ offset_size: u8,
134
+ length_size: u8,
135
+ ndim: u32,
136
+ chunk_dims: &[u32],
137
+ chunk_bounds: Option<(&[u64], &[u64])>,
138
+ ) -> Result<Vec<ChunkEntry>> {
139
+ let header = crate::btree_v2::BTreeV2Header::parse_at_storage(
140
+ storage,
141
+ btree_address,
142
+ offset_size,
143
+ length_size,
144
+ )?;
145
+ let records = crate::btree_v2::collect_btree_v2_records_storage(
146
+ storage,
147
+ &header,
148
+ offset_size,
149
+ length_size,
150
+ Some(ndim),
151
+ chunk_dims,
152
+ chunk_bounds,
153
+ )?;
154
+
155
+ let mut entries = Vec::with_capacity(records.len());
156
+ for record in records {
157
+ match record {
158
+ crate::btree_v2::BTreeV2Record::ChunkedNonFiltered { address, offsets }
159
+ if chunk_overlaps_bounds(&offsets, chunk_dims, chunk_bounds) =>
160
+ {
161
+ entries.push(ChunkEntry {
162
+ address,
163
+ size: 0,
164
+ filter_mask: 0,
165
+ offsets,
166
+ });
167
+ }
168
+ crate::btree_v2::BTreeV2Record::ChunkedFiltered {
169
+ address,
170
+ chunk_size,
171
+ filter_mask,
172
+ offsets,
173
+ } if chunk_overlaps_bounds(&offsets, chunk_dims, chunk_bounds) => {
174
+ entries.push(ChunkEntry {
175
+ address,
176
+ size: chunk_size,
177
+ filter_mask,
178
+ offsets,
179
+ });
180
+ }
181
+ _ => {}
182
+ }
183
+ }
184
+
185
+ Ok(entries)
186
+ }
187
+
188
+ /// Collect chunk entries for implicit indexing.
189
+ ///
190
+ /// Implicit chunks are laid out sequentially starting at the given address.
191
+ /// Each chunk has the same size = product(chunk_dims) * elem_size.
192
+ pub fn collect_implicit_chunk_entries(
193
+ start_address: u64,
194
+ dataset_shape: &[u64],
195
+ chunk_dims: &[u32],
196
+ elem_size: usize,
197
+ chunk_bounds: Option<(&[u64], &[u64])>,
198
+ storage_len: u64,
199
+ ) -> Result<Vec<ChunkEntry>> {
200
+ let ndim = dataset_shape.len();
201
+ if chunk_dims.len() != ndim {
202
+ return Err(Error::InvalidData(format!(
203
+ "implicit chunk rank {} does not match dataset rank {ndim}",
204
+ chunk_dims.len()
205
+ )));
206
+ }
207
+
208
+ // Compute how many chunks along each dimension
209
+ let mut chunks_per_dim = Vec::with_capacity(ndim);
210
+ for i in 0..ndim {
211
+ let chunk_dim = u64::from(chunk_dims[i]);
212
+ if chunk_dim == 0 {
213
+ return Err(Error::InvalidData(format!(
214
+ "implicit chunk dimension {i} has zero extent"
215
+ )));
216
+ }
217
+ chunks_per_dim.push(dataset_shape[i].div_ceil(chunk_dim));
218
+ }
219
+ if dataset_shape.contains(&0) {
220
+ return Ok(Vec::new());
221
+ }
222
+
223
+ let chunk_elements = chunk_dims.iter().try_fold(1u64, |acc, &dim| {
224
+ checked_mul_u64(acc, u64::from(dim), "implicit chunk element count")
225
+ })?;
226
+ let elem_size = u64::try_from(elem_size).map_err(|_| {
227
+ Error::InvalidData("implicit chunk element size exceeds u64 capacity".to_string())
228
+ })?;
229
+ let chunk_bytes = checked_mul_u64(chunk_elements, elem_size, "implicit chunk byte size")?;
230
+
231
+ if ndim == 0 {
232
+ return Ok(vec![ChunkEntry {
233
+ address: start_address,
234
+ size: chunk_bytes,
235
+ filter_mask: 0,
236
+ offsets: Vec::new(),
237
+ }]);
238
+ }
239
+
240
+ let (first_chunk, last_chunk): (Vec<u64>, Vec<u64>) = match chunk_bounds {
241
+ Some((first, last)) => (first.to_vec(), last.to_vec()),
242
+ None => (
243
+ vec![0u64; ndim],
244
+ chunks_per_dim
245
+ .iter()
246
+ .map(|count| count.saturating_sub(1))
247
+ .collect(),
248
+ ),
249
+ };
250
+
251
+ let mut chunk_counts = Vec::with_capacity(ndim);
252
+ for dim in 0..ndim {
253
+ let selected = last_chunk[dim]
254
+ .checked_sub(first_chunk[dim])
255
+ .and_then(|value| value.checked_add(1))
256
+ .ok_or_else(|| {
257
+ Error::InvalidData("implicit chunk selection bounds are invalid".to_string())
258
+ })?;
259
+ chunk_counts.push(selected);
260
+ }
261
+ let total_selected_chunks = chunk_counts.iter().try_fold(1u64, |acc, &count| {
262
+ checked_mul_u64(acc, count, "implicit selected chunk count")
263
+ })?;
264
+ // An implicit index stores its chunks contiguously and unfiltered, so the
265
+ // total chunk data cannot exceed the file. Reject a declared chunk count
266
+ // whose data would not fit before pre-allocating the entry vector, so a
267
+ // tiny file cannot declare billions of chunks.
268
+ if chunk_bytes > 0 {
269
+ let max_chunks = storage_len / chunk_bytes;
270
+ if total_selected_chunks > max_chunks.saturating_add(1) {
271
+ return Err(Error::InvalidData(format!(
272
+ "implicit index declares {total_selected_chunks} chunks ({} bytes each) but the \
273
+ file has only {storage_len} bytes",
274
+ chunk_bytes
275
+ )));
276
+ }
277
+ }
278
+ let mut entries = Vec::with_capacity(checked_usize(
279
+ total_selected_chunks,
280
+ "implicit selected chunk count",
281
+ )?);
282
+ let mut chunk_indices = first_chunk.clone();
283
+
284
+ loop {
285
+ let chunk_idx = chunk_linear_index(&chunk_indices, &chunks_per_dim)?;
286
+ let offsets = chunk_indices
287
+ .iter()
288
+ .enumerate()
289
+ .map(|(dim, chunk_index)| {
290
+ checked_mul_u64(
291
+ *chunk_index,
292
+ u64::from(chunk_dims[dim]),
293
+ "implicit chunk offset",
294
+ )
295
+ })
296
+ .collect::<Result<Vec<_>>>()?;
297
+ let chunk_data_offset =
298
+ checked_mul_u64(chunk_idx, chunk_bytes, "implicit chunk byte offset")?;
299
+
300
+ entries.push(ChunkEntry {
301
+ address: checked_add_u64(start_address, chunk_data_offset, "implicit chunk address")?,
302
+ size: chunk_bytes,
303
+ filter_mask: 0,
304
+ offsets,
305
+ });
306
+
307
+ let mut advanced = false;
308
+ for dim in (0..ndim).rev() {
309
+ if chunk_indices[dim] < last_chunk[dim] {
310
+ chunk_indices[dim] += 1;
311
+ if dim + 1 < ndim {
312
+ chunk_indices[(dim + 1)..ndim].copy_from_slice(&first_chunk[(dim + 1)..ndim]);
313
+ }
314
+ advanced = true;
315
+ break;
316
+ }
317
+ }
318
+
319
+ if !advanced {
320
+ break;
321
+ }
322
+ }
323
+
324
+ Ok(entries)
325
+ }
326
+
327
+ /// Resolve a single-chunk layout.
328
+ ///
329
+ /// The entire dataset is stored as one chunk at the given address.
330
+ pub fn single_chunk_entry(
331
+ address: u64,
332
+ filtered_size: u64,
333
+ filter_mask: u32,
334
+ ndim: usize,
335
+ ) -> ChunkEntry {
336
+ ChunkEntry {
337
+ address,
338
+ size: filtered_size,
339
+ filter_mask,
340
+ offsets: vec![0u64; ndim],
341
+ }
342
+ }
343
+
344
+ #[cfg(test)]
345
+ mod tests {
346
+ use super::*;
347
+
348
+ #[test]
349
+ fn chunk_entry_debug_clone() {
350
+ let entry = ChunkEntry {
351
+ address: 0x1000,
352
+ size: 4096,
353
+ filter_mask: 0,
354
+ offsets: vec![0, 0],
355
+ };
356
+ let entry2 = entry.clone();
357
+ assert_eq!(entry2.address, 0x1000);
358
+ let _ = format!("{:?}", entry);
359
+ }
360
+
361
+ #[test]
362
+ fn implicit_chunk_entries() {
363
+ let entries =
364
+ collect_implicit_chunk_entries(1000, &[10, 20], &[5, 10], 4, None, u64::MAX).unwrap();
365
+ // 2 chunks along dim 0, 2 chunks along dim 1 = 4 total
366
+ assert_eq!(entries.len(), 4);
367
+ assert_eq!(entries[0].address, 1000);
368
+ assert_eq!(entries[0].offsets, vec![0, 0]);
369
+ assert_eq!(entries[1].address, 1000 + 200); // 5*10*4 = 200
370
+ assert_eq!(entries[1].offsets, vec![0, 10]);
371
+ assert_eq!(entries[2].offsets, vec![5, 0]);
372
+ assert_eq!(entries[3].offsets, vec![5, 10]);
373
+ }
374
+
375
+ #[test]
376
+ fn implicit_chunk_entries_empty_extent() {
377
+ let entries =
378
+ collect_implicit_chunk_entries(1000, &[0, 20], &[5, 10], 4, None, u64::MAX).unwrap();
379
+ assert!(entries.is_empty());
380
+ }
381
+
382
+ #[test]
383
+ fn implicit_chunk_entries_reject_chunk_byte_overflow() {
384
+ let err = collect_implicit_chunk_entries(
385
+ 1000,
386
+ &[10, 10],
387
+ &[u32::MAX, u32::MAX],
388
+ 2,
389
+ None,
390
+ u64::MAX,
391
+ )
392
+ .unwrap_err();
393
+ assert!(err.to_string().contains("implicit chunk byte size"));
394
+ }
395
+
396
+ #[test]
397
+ fn implicit_chunk_entries_reject_address_overflow() {
398
+ let err =
399
+ collect_implicit_chunk_entries(u64::MAX, &[2], &[1], 1, Some((&[1], &[1])), u64::MAX)
400
+ .unwrap_err();
401
+ assert!(err.to_string().contains("implicit chunk address"));
402
+ }
403
+
404
+ #[test]
405
+ fn implicit_chunk_entries_reject_declared_count_beyond_file() {
406
+ // 2^32 chunks of 4 bytes each declared in a 1 KiB file must be
407
+ // rejected before the entry vector is allocated.
408
+ let err = collect_implicit_chunk_entries(1000, &[1u64 << 34, 1], &[4, 1], 1, None, 1024)
409
+ .unwrap_err();
410
+ assert!(
411
+ err.to_string().contains("file has only"),
412
+ "unexpected error: {err}"
413
+ );
414
+ }
415
+
416
+ #[test]
417
+ fn single_chunk_entry_uses_origin_offsets() {
418
+ let entry = single_chunk_entry(0x2000, 8192, 0, 3);
419
+ assert_eq!(entry.address, 0x2000);
420
+ assert_eq!(entry.size, 8192);
421
+ assert_eq!(entry.offsets, vec![0, 0, 0]);
422
+ }
423
+ }