@nirs4all/formats-wasm 0.2.12 → 0.2.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/THIRD_PARTY_NOTICES.md +49 -0
  2. package/nirs4all_formats_wasm.d.ts +4 -4
  3. package/nirs4all_formats_wasm.js +170 -149
  4. package/nirs4all_formats_wasm_bg.wasm +0 -0
  5. package/package.json +4 -2
  6. package/reader-backports/hdf5-reader/Cargo.toml +119 -0
  7. package/reader-backports/hdf5-reader/LICENSE-APACHE +164 -0
  8. package/reader-backports/hdf5-reader/LICENSE-MIT +21 -0
  9. package/reader-backports/hdf5-reader/PROVENANCE.json +80 -0
  10. package/reader-backports/hdf5-reader/README.md +308 -0
  11. package/reader-backports/hdf5-reader/UPSTREAM-Cargo.toml +44 -0
  12. package/reader-backports/hdf5-reader/UPSTREAM-VCS.json +6 -0
  13. package/reader-backports/hdf5-reader/src/attribute_api.rs +847 -0
  14. package/reader-backports/hdf5-reader/src/btree_v1.rs +726 -0
  15. package/reader-backports/hdf5-reader/src/btree_v2.rs +2068 -0
  16. package/reader-backports/hdf5-reader/src/cache.rs +354 -0
  17. package/reader-backports/hdf5-reader/src/checksum.rs +56 -0
  18. package/reader-backports/hdf5-reader/src/chunk_index.rs +423 -0
  19. package/reader-backports/hdf5-reader/src/dataset.rs +4733 -0
  20. package/reader-backports/hdf5-reader/src/datatype_api.rs +497 -0
  21. package/reader-backports/hdf5-reader/src/error.rs +167 -0
  22. package/reader-backports/hdf5-reader/src/extensible_array.rs +1886 -0
  23. package/reader-backports/hdf5-reader/src/filters/deflate.rs +93 -0
  24. package/reader-backports/hdf5-reader/src/filters/fletcher32.rs +52 -0
  25. package/reader-backports/hdf5-reader/src/filters/lz4.rs +178 -0
  26. package/reader-backports/hdf5-reader/src/filters/mod.rs +326 -0
  27. package/reader-backports/hdf5-reader/src/filters/nbit.rs +586 -0
  28. package/reader-backports/hdf5-reader/src/filters/scaleoffset.rs +827 -0
  29. package/reader-backports/hdf5-reader/src/filters/shuffle.rs +99 -0
  30. package/reader-backports/hdf5-reader/src/fixed_array.rs +763 -0
  31. package/reader-backports/hdf5-reader/src/fractal_heap.rs +2029 -0
  32. package/reader-backports/hdf5-reader/src/global_heap.rs +284 -0
  33. package/reader-backports/hdf5-reader/src/group.rs +795 -0
  34. package/reader-backports/hdf5-reader/src/io.rs +413 -0
  35. package/reader-backports/hdf5-reader/src/lib.rs +916 -0
  36. package/reader-backports/hdf5-reader/src/local_heap.rs +261 -0
  37. package/reader-backports/hdf5-reader/src/messages/attribute.rs +304 -0
  38. package/reader-backports/hdf5-reader/src/messages/attribute_info.rs +96 -0
  39. package/reader-backports/hdf5-reader/src/messages/btree_k.rs +36 -0
  40. package/reader-backports/hdf5-reader/src/messages/continuation.rs +46 -0
  41. package/reader-backports/hdf5-reader/src/messages/dataspace.rs +259 -0
  42. package/reader-backports/hdf5-reader/src/messages/datatype.rs +938 -0
  43. package/reader-backports/hdf5-reader/src/messages/external_files.rs +50 -0
  44. package/reader-backports/hdf5-reader/src/messages/fill_value.rs +233 -0
  45. package/reader-backports/hdf5-reader/src/messages/filter_pipeline.rs +147 -0
  46. package/reader-backports/hdf5-reader/src/messages/group_info.rs +103 -0
  47. package/reader-backports/hdf5-reader/src/messages/layout.rs +476 -0
  48. package/reader-backports/hdf5-reader/src/messages/link.rs +170 -0
  49. package/reader-backports/hdf5-reader/src/messages/link_info.rs +127 -0
  50. package/reader-backports/hdf5-reader/src/messages/mod.rs +374 -0
  51. package/reader-backports/hdf5-reader/src/messages/modification_time.rs +83 -0
  52. package/reader-backports/hdf5-reader/src/messages/shared.rs +164 -0
  53. package/reader-backports/hdf5-reader/src/messages/shared_table.rs +67 -0
  54. package/reader-backports/hdf5-reader/src/messages/symbol_table_msg.rs +51 -0
  55. package/reader-backports/hdf5-reader/src/object_header.rs +1749 -0
  56. package/reader-backports/hdf5-reader/src/reference.rs +90 -0
  57. package/reader-backports/hdf5-reader/src/shared_message_table.rs +425 -0
  58. package/reader-backports/hdf5-reader/src/storage.rs +538 -0
  59. package/reader-backports/hdf5-reader/src/superblock.rs +285 -0
  60. package/reader-backports/hdf5-reader/src/symbol_table.rs +277 -0
  61. package/reader-backports/netcdf-reader/Cargo.toml +141 -0
  62. package/reader-backports/netcdf-reader/LICENSE-APACHE +164 -0
  63. package/reader-backports/netcdf-reader/LICENSE-MIT +21 -0
  64. package/reader-backports/netcdf-reader/PROVENANCE.json +55 -0
  65. package/reader-backports/netcdf-reader/README.md +308 -0
  66. package/reader-backports/netcdf-reader/UPSTREAM-Cargo.toml +48 -0
  67. package/reader-backports/netcdf-reader/UPSTREAM-VCS.json +6 -0
  68. package/reader-backports/netcdf-reader/src/cf/axes.rs +419 -0
  69. package/reader-backports/netcdf-reader/src/cf/bounds.rs +108 -0
  70. package/reader-backports/netcdf-reader/src/cf/crs.rs +269 -0
  71. package/reader-backports/netcdf-reader/src/cf/mod.rs +30 -0
  72. package/reader-backports/netcdf-reader/src/cf/time.rs +1122 -0
  73. package/reader-backports/netcdf-reader/src/classic/data.rs +1465 -0
  74. package/reader-backports/netcdf-reader/src/classic/header.rs +1097 -0
  75. package/reader-backports/netcdf-reader/src/classic/mod.rs +240 -0
  76. package/reader-backports/netcdf-reader/src/classic/storage.rs +211 -0
  77. package/reader-backports/netcdf-reader/src/classic/types.rs +93 -0
  78. package/reader-backports/netcdf-reader/src/classic/variable.rs +1162 -0
  79. package/reader-backports/netcdf-reader/src/error.rs +59 -0
  80. package/reader-backports/netcdf-reader/src/lib.rs +1531 -0
  81. package/reader-backports/netcdf-reader/src/masked.rs +173 -0
  82. package/reader-backports/netcdf-reader/src/nc4/attributes.rs +203 -0
  83. package/reader-backports/netcdf-reader/src/nc4/dimensions.rs +139 -0
  84. package/reader-backports/netcdf-reader/src/nc4/groups.rs +308 -0
  85. package/reader-backports/netcdf-reader/src/nc4/mod.rs +572 -0
  86. package/reader-backports/netcdf-reader/src/nc4/types.rs +309 -0
  87. package/reader-backports/netcdf-reader/src/nc4/variables.rs +420 -0
  88. package/reader-backports/netcdf-reader/src/types.rs +18 -0
  89. package/reader-backports/netcdf-reader/src/unpack.rs +113 -0
  90. package/reader-backports/netcdf-reader/src/user_defined.rs +569 -0
@@ -0,0 +1,93 @@
1
+ use crate::error::{Error, Result};
2
+ use flate2::read::ZlibDecoder;
3
+ use flate2::{Decompress, FlushDecompress, Status};
4
+ use std::io::Read;
5
+
6
+ /// Decompress DEFLATE/zlib-compressed data.
7
+ pub fn decompress(data: &[u8]) -> Result<Vec<u8>> {
8
+ decompress_inner(data, None)
9
+ }
10
+
11
+ /// Decompress DEFLATE/zlib-compressed data, reading at most `max_output_len`
12
+ /// bytes from the decoded stream.
13
+ pub fn decompress_with_limit(data: &[u8], max_output_len: usize) -> Result<Vec<u8>> {
14
+ decompress_inner(data, Some(max_output_len))
15
+ }
16
+
17
+ fn decompress_inner(data: &[u8], max_output_len: Option<usize>) -> Result<Vec<u8>> {
18
+ if let Some(max_output_len) = max_output_len {
19
+ return decompress_bounded(data, max_output_len);
20
+ }
21
+
22
+ let mut decoder = ZlibDecoder::new(data);
23
+ let mut output = Vec::new();
24
+ decoder
25
+ .read_to_end(&mut output)
26
+ .map_err(|e| Error::DecompressionError(format!("DEFLATE decompression failed: {e}")))?;
27
+ Ok(output)
28
+ }
29
+
30
+ fn decompress_bounded(data: &[u8], max_output_len: usize) -> Result<Vec<u8>> {
31
+ let mut decoder = Decompress::new(true);
32
+ // DEFLATE expands at most ~1032:1, so no valid stream needs more room than this; a corrupted
33
+ // chunk shape can no longer ask for a multi-gigabyte buffer before a byte is decoded.
34
+ let capacity = max_output_len.min(data.len().saturating_mul(1032).saturating_add(64));
35
+ let mut output = Vec::new();
36
+ output.try_reserve_exact(capacity).map_err(|_| {
37
+ Error::DecompressionError(format!("DEFLATE: cannot allocate {capacity} output bytes"))
38
+ })?;
39
+ let status = decoder
40
+ .decompress_vec(data, &mut output, FlushDecompress::Finish)
41
+ .map_err(|e| Error::DecompressionError(format!("DEFLATE decompression failed: {e}")))?;
42
+
43
+ if matches!(status, Status::StreamEnd) || output.len() == max_output_len {
44
+ return Ok(output);
45
+ }
46
+
47
+ Err(Error::DecompressionError(
48
+ "DEFLATE decompression failed: stream did not finish".into(),
49
+ ))
50
+ }
51
+
52
+ #[cfg(test)]
53
+ mod tests {
54
+ use super::*;
55
+
56
+ /// A corrupted chunk shape gives an absurd output limit: the stream still decodes,
57
+ /// without reserving the limit up front (it used to abort on allocation failure).
58
+ #[test]
59
+ fn absurd_output_limit_does_not_preallocate() {
60
+ let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
61
+ std::io::Write::write_all(&mut encoder, &[7u8; 300]).unwrap();
62
+ let compressed = encoder.finish().unwrap();
63
+ let out = decompress_with_limit(&compressed, usize::MAX / 2).unwrap();
64
+ assert_eq!(out, vec![7u8; 300]);
65
+ assert!(out.capacity() <= compressed.len() * 1032 + 64);
66
+ }
67
+ use flate2::write::ZlibEncoder;
68
+ use flate2::Compression;
69
+ use std::io::Write;
70
+
71
+ #[test]
72
+ fn roundtrip() {
73
+ let original = b"hello world, this is a test of zlib compression!";
74
+ let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
75
+ encoder.write_all(original).unwrap();
76
+ let compressed = encoder.finish().unwrap();
77
+
78
+ let decompressed = decompress(&compressed).unwrap();
79
+ assert_eq!(decompressed, original);
80
+ }
81
+
82
+ #[test]
83
+ fn decompress_with_limit_stops_at_limit() {
84
+ let original = vec![0x5a; 4096];
85
+ let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
86
+ encoder.write_all(&original).unwrap();
87
+ let compressed = encoder.finish().unwrap();
88
+
89
+ let decompressed = decompress_with_limit(&compressed, 17).unwrap();
90
+ assert_eq!(decompressed.len(), 17);
91
+ assert_eq!(decompressed, original[..17]);
92
+ }
93
+ }
@@ -0,0 +1,52 @@
1
+ use crate::error::{Error, Result};
2
+
3
+ /// Verify Fletcher-32 checksum and strip it from the data.
4
+ ///
5
+ /// The checksum is stored as the last 4 bytes of the chunk data.
6
+ pub fn verify_and_strip(data: &[u8]) -> Result<Vec<u8>> {
7
+ if data.len() < 4 {
8
+ return Err(Error::FilterError(
9
+ "data too short for Fletcher-32 checksum".into(),
10
+ ));
11
+ }
12
+
13
+ let payload = &data[..data.len() - 4];
14
+ let stored = u32::from_le_bytes([
15
+ data[data.len() - 4],
16
+ data[data.len() - 3],
17
+ data[data.len() - 2],
18
+ data[data.len() - 1],
19
+ ]);
20
+
21
+ let computed = crate::checksum::fletcher32(payload);
22
+ if computed != stored {
23
+ return Err(Error::ChecksumMismatch {
24
+ expected: stored,
25
+ actual: computed,
26
+ });
27
+ }
28
+
29
+ Ok(payload.to_vec())
30
+ }
31
+
32
+ #[cfg(test)]
33
+ mod tests {
34
+ use super::*;
35
+
36
+ #[test]
37
+ fn verify_valid() {
38
+ let payload = vec![0x01, 0x02, 0x03, 0x04, 0x05, 0x06];
39
+ let checksum = crate::checksum::fletcher32(&payload);
40
+ let mut data = payload.clone();
41
+ data.extend_from_slice(&checksum.to_le_bytes());
42
+ let result = verify_and_strip(&data).unwrap();
43
+ assert_eq!(result, payload);
44
+ }
45
+
46
+ #[test]
47
+ fn verify_invalid() {
48
+ let mut data = vec![0x01, 0x02, 0x03, 0x04, 0x05, 0x06];
49
+ data.extend_from_slice(&[0xFF, 0xFF, 0xFF, 0xFF]);
50
+ assert!(verify_and_strip(&data).is_err());
51
+ }
52
+ }
@@ -0,0 +1,178 @@
1
+ //! HDF5 LZ4 filter (filter ID 32004).
2
+ //!
3
+ //! On-disk format per the [HDF5 LZ4 plugin](https://github.com/HDFGroup/hdf5_plugins):
4
+ //! - `orig_size: u64 BE` — total uncompressed size of the chunk
5
+ //! - Repeated blocks:
6
+ //! - `orig_block_size: u32 BE` — uncompressed size of this block
7
+ //! - `compressed_size: u32 BE` — compressed size of this block
8
+ //! - `compressed_data: [u8; compressed_size]`
9
+ //!
10
+ //! When `compressed_size == orig_block_size` the block is stored uncompressed.
11
+
12
+ use crate::error::{Error, Result};
13
+
14
+ /// Decompress HDF5 LZ4-filtered data.
15
+ /// LZ4 cannot expand data by more than ~255x; bound the unlimited path's
16
+ /// declared output size by this multiple of the input so a crafted header
17
+ /// cannot force a huge up-front allocation.
18
+ const LZ4_MAX_EXPANSION: usize = 256;
19
+
20
+ pub fn decompress(data: &[u8]) -> Result<Vec<u8>> {
21
+ // Even without a caller-supplied limit, cap the declared output size by
22
+ // what the input could plausibly expand to.
23
+ let implicit_limit = data.len().saturating_mul(LZ4_MAX_EXPANSION);
24
+ decompress_inner(data, Some(implicit_limit))
25
+ }
26
+
27
+ /// Decompress HDF5 LZ4-filtered data, rejecting declared output sizes above
28
+ /// `max_output_len` before allocating the output buffer.
29
+ pub fn decompress_with_limit(data: &[u8], max_output_len: usize) -> Result<Vec<u8>> {
30
+ decompress_inner(data, Some(max_output_len))
31
+ }
32
+
33
+ fn decompress_inner(data: &[u8], max_output_len: Option<usize>) -> Result<Vec<u8>> {
34
+ if data.len() < 8 {
35
+ return Err(Error::DecompressionError(
36
+ "LZ4: input too short for header".into(),
37
+ ));
38
+ }
39
+
40
+ let declared_orig_size = u64::from_be_bytes([
41
+ data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7],
42
+ ]);
43
+ let orig_size = usize::try_from(declared_orig_size).map_err(|_| {
44
+ Error::DecompressionError(format!(
45
+ "LZ4: declared output size {declared_orig_size} exceeds platform usize capacity"
46
+ ))
47
+ })?;
48
+ if let Some(max_output_len) = max_output_len {
49
+ if orig_size > max_output_len {
50
+ return Err(Error::DecompressionError(format!(
51
+ "LZ4: declared output size {orig_size} exceeds limit {max_output_len}"
52
+ )));
53
+ }
54
+ }
55
+
56
+ // The output grows as blocks are decoded; reserve no more than LZ4 can expand to
57
+ // (~255:1), so a corrupted size field cannot abort the process.
58
+ let mut output = Vec::new();
59
+ let capacity = orig_size.min(data.len().saturating_mul(255).saturating_add(64));
60
+ output.try_reserve_exact(capacity).map_err(|_| {
61
+ Error::DecompressionError(format!("LZ4: cannot allocate {capacity} output bytes"))
62
+ })?;
63
+ let mut pos = 8;
64
+
65
+ while pos < data.len() && output.len() < orig_size {
66
+ if pos + 8 > data.len() {
67
+ return Err(Error::DecompressionError(
68
+ "LZ4: truncated block header".into(),
69
+ ));
70
+ }
71
+
72
+ let orig_block_size =
73
+ u32::from_be_bytes([data[pos], data[pos + 1], data[pos + 2], data[pos + 3]]) as usize;
74
+ pos += 4;
75
+
76
+ let compressed_size =
77
+ u32::from_be_bytes([data[pos], data[pos + 1], data[pos + 2], data[pos + 3]]) as usize;
78
+ pos += 4;
79
+
80
+ let block_end = output
81
+ .len()
82
+ .checked_add(orig_block_size)
83
+ .ok_or_else(|| Error::DecompressionError("LZ4: block output size overflowed".into()))?;
84
+ if block_end > orig_size {
85
+ return Err(Error::DecompressionError(
86
+ "LZ4: block output exceeds declared output size".into(),
87
+ ));
88
+ }
89
+
90
+ if pos + compressed_size > data.len() {
91
+ return Err(Error::DecompressionError(
92
+ "LZ4: truncated block data".into(),
93
+ ));
94
+ }
95
+
96
+ if compressed_size == orig_block_size {
97
+ // Stored uncompressed
98
+ output.extend_from_slice(&data[pos..pos + compressed_size]);
99
+ } else {
100
+ let decompressed =
101
+ lz4_flex::decompress(&data[pos..pos + compressed_size], orig_block_size)
102
+ .map_err(|e| Error::DecompressionError(format!("LZ4: {}", e)))?;
103
+ output.extend_from_slice(&decompressed);
104
+ }
105
+ pos += compressed_size;
106
+ }
107
+
108
+ Ok(output)
109
+ }
110
+
111
+ #[cfg(test)]
112
+ mod tests {
113
+ use super::*;
114
+
115
+ #[test]
116
+ fn lz4_round_trip() {
117
+ let original = b"hello world hello world hello world!";
118
+ let raw_compressed = lz4_flex::compress(original);
119
+
120
+ let mut hdf5_data = Vec::new();
121
+ hdf5_data.extend_from_slice(&(original.len() as u64).to_be_bytes());
122
+ // Block header: orig_block_size + compressed_size
123
+ hdf5_data.extend_from_slice(&(original.len() as u32).to_be_bytes());
124
+ hdf5_data.extend_from_slice(&(raw_compressed.len() as u32).to_be_bytes());
125
+ hdf5_data.extend_from_slice(&raw_compressed);
126
+
127
+ let result = decompress(&hdf5_data).unwrap();
128
+ assert_eq!(result, original);
129
+ }
130
+
131
+ #[test]
132
+ fn lz4_uncompressed_block() {
133
+ let original = b"short";
134
+ let mut hdf5_data = Vec::new();
135
+ hdf5_data.extend_from_slice(&(original.len() as u64).to_be_bytes());
136
+ // compressed_size == orig_block_size => uncompressed
137
+ hdf5_data.extend_from_slice(&(original.len() as u32).to_be_bytes());
138
+ hdf5_data.extend_from_slice(&(original.len() as u32).to_be_bytes());
139
+ hdf5_data.extend_from_slice(original);
140
+
141
+ let result = decompress(&hdf5_data).unwrap();
142
+ assert_eq!(result, original);
143
+ }
144
+
145
+ #[test]
146
+ fn lz4_too_short() {
147
+ let data = &[0; 7];
148
+ assert!(decompress(data).is_err());
149
+ }
150
+
151
+ #[test]
152
+ fn lz4_truncated_block() {
153
+ let mut data = vec![0, 0, 0, 0, 0, 0, 0, 10]; // orig_size = 10
154
+ data.extend_from_slice(&10u32.to_be_bytes()); // orig_block_size = 10
155
+ data.extend_from_slice(&100u32.to_be_bytes()); // compressed_size = 100
156
+ data.extend_from_slice(&[0; 5]); // only 5 bytes of "100"
157
+ assert!(decompress(&data).is_err());
158
+ }
159
+
160
+ #[test]
161
+ fn lz4_declared_size_above_limit_errors_before_allocation() {
162
+ let data = (4096u64).to_be_bytes();
163
+ let err = decompress_with_limit(&data, 32).unwrap_err();
164
+ assert!(err.to_string().contains("exceeds limit"));
165
+ }
166
+
167
+ #[test]
168
+ fn lz4_block_cannot_exceed_declared_size() {
169
+ let mut data = Vec::new();
170
+ data.extend_from_slice(&4u64.to_be_bytes());
171
+ data.extend_from_slice(&8u32.to_be_bytes());
172
+ data.extend_from_slice(&8u32.to_be_bytes());
173
+ data.extend_from_slice(&[0; 8]);
174
+
175
+ let err = decompress(&data).unwrap_err();
176
+ assert!(err.to_string().contains("exceeds declared output size"));
177
+ }
178
+ }
@@ -0,0 +1,326 @@
1
+ pub mod deflate;
2
+ pub mod fletcher32;
3
+ #[cfg(feature = "lz4")]
4
+ pub mod lz4;
5
+ pub mod nbit;
6
+ pub mod scaleoffset;
7
+ pub mod shuffle;
8
+
9
+ use std::collections::HashMap;
10
+
11
+ use crate::error::{Error, Result};
12
+ use crate::messages::filter_pipeline::FilterDescription;
13
+
14
+ /// Standard HDF5 filter IDs.
15
+ pub const FILTER_DEFLATE: u16 = 1;
16
+ pub const FILTER_SHUFFLE: u16 = 2;
17
+ pub const FILTER_FLETCHER32: u16 = 3;
18
+ pub const FILTER_SZIP: u16 = 4;
19
+ pub const FILTER_NBIT: u16 = 5;
20
+ pub const FILTER_SCALEOFFSET: u16 = 6;
21
+ /// HDF5 registered LZ4 filter.
22
+ pub const FILTER_LZ4: u16 = 32004;
23
+
24
+ /// A user-supplied filter function.
25
+ ///
26
+ /// Takes the filter description, input data, and element size, then returns
27
+ /// the decoded output.
28
+ pub type FilterFn = Box<dyn Fn(&FilterDescription, &[u8], usize) -> Result<Vec<u8>> + Send + Sync>;
29
+
30
+ enum FilterImplementation {
31
+ Builtin,
32
+ Custom(FilterFn),
33
+ }
34
+
35
+ /// A registry of filter implementations.
36
+ ///
37
+ /// Comes pre-loaded with deflate, shuffle, and fletcher32. Users can register
38
+ /// additional filters (e.g., Blosc, LZ4, ZFP) before reading datasets.
39
+ pub struct FilterRegistry {
40
+ filters: HashMap<u16, FilterImplementation>,
41
+ }
42
+
43
+ impl FilterRegistry {
44
+ /// Create a new registry with the built-in filters pre-registered.
45
+ pub fn new() -> Self {
46
+ let mut registry = FilterRegistry {
47
+ filters: HashMap::new(),
48
+ };
49
+ registry.register_builtin(FILTER_DEFLATE);
50
+ registry.register_builtin(FILTER_SHUFFLE);
51
+ registry.register_builtin(FILTER_FLETCHER32);
52
+ registry.register_builtin(FILTER_NBIT);
53
+ registry.register_builtin(FILTER_SCALEOFFSET);
54
+ #[cfg(feature = "lz4")]
55
+ registry.register_builtin(FILTER_LZ4);
56
+ registry
57
+ }
58
+
59
+ fn register_builtin(&mut self, id: u16) {
60
+ self.filters.insert(id, FilterImplementation::Builtin);
61
+ }
62
+
63
+ /// Register a custom filter implementation for the given filter ID.
64
+ ///
65
+ /// Overwrites any previously registered filter with the same ID.
66
+ pub fn register(&mut self, id: u16, f: FilterFn) {
67
+ self.filters.insert(id, FilterImplementation::Custom(f));
68
+ }
69
+
70
+ /// Apply a single filter by ID.
71
+ pub fn apply(
72
+ &self,
73
+ filter: &FilterDescription,
74
+ data: &[u8],
75
+ element_size: usize,
76
+ ) -> Result<Vec<u8>> {
77
+ self.apply_with_limit(filter, data, element_size, None)
78
+ }
79
+
80
+ /// Apply a single filter by ID, passing a maximum decoded output length to
81
+ /// built-in filters that can enforce it while decoding.
82
+ pub fn apply_with_limit(
83
+ &self,
84
+ filter: &FilterDescription,
85
+ data: &[u8],
86
+ element_size: usize,
87
+ max_output_len: Option<usize>,
88
+ ) -> Result<Vec<u8>> {
89
+ match self.filters.get(&filter.id) {
90
+ Some(FilterImplementation::Builtin) => {
91
+ apply_builtin_filter_with_limit(filter, data, element_size, max_output_len)
92
+ }
93
+ Some(FilterImplementation::Custom(f)) => f(filter, data, element_size),
94
+ None => Err(Error::UnsupportedFilter(format!("filter id {}", filter.id))),
95
+ }
96
+ }
97
+ }
98
+
99
+ impl Default for FilterRegistry {
100
+ fn default() -> Self {
101
+ Self::new()
102
+ }
103
+ }
104
+
105
+ /// Apply the filter pipeline in reverse (decompression direction) to a chunk.
106
+ ///
107
+ /// HDF5 stores filters in the order they were applied during writing.
108
+ /// On read, we apply them in reverse order.
109
+ ///
110
+ /// If `registry` is `None`, the built-in filter set is used.
111
+ ///
112
+ /// `filter_mask` is a bitmask where bit N being set means filter N should be skipped.
113
+ pub fn apply_pipeline(
114
+ data: &[u8],
115
+ filters: &[FilterDescription],
116
+ filter_mask: u32,
117
+ element_size: usize,
118
+ registry: Option<&FilterRegistry>,
119
+ ) -> Result<Vec<u8>> {
120
+ apply_pipeline_with_limit(data, filters, filter_mask, element_size, registry, None)
121
+ }
122
+
123
+ /// Apply the filter pipeline in reverse (decompression direction) to a chunk,
124
+ /// passing a maximum decoded output length to built-in filters that support
125
+ /// bounded decompression.
126
+ pub fn apply_pipeline_with_limit(
127
+ data: &[u8],
128
+ filters: &[FilterDescription],
129
+ filter_mask: u32,
130
+ element_size: usize,
131
+ registry: Option<&FilterRegistry>,
132
+ max_output_len: Option<usize>,
133
+ ) -> Result<Vec<u8>> {
134
+ // Count active filters so an all-skipped pipeline can avoid the loop.
135
+ let active_count = filters
136
+ .iter()
137
+ .enumerate()
138
+ .rev()
139
+ .filter(|(i, _)| filter_mask & (1 << i) == 0)
140
+ .count();
141
+
142
+ if active_count == 0 {
143
+ validate_output_limit("filter pipeline", data.len(), max_output_len)?;
144
+ return Ok(data.to_vec());
145
+ }
146
+
147
+ // For a single active filter, avoid the double-buffer loop overhead.
148
+ if active_count == 1 {
149
+ for (i, filter) in filters.iter().enumerate().rev() {
150
+ if filter_mask & (1 << i) != 0 {
151
+ continue;
152
+ }
153
+ let output = if let Some(reg) = registry {
154
+ reg.apply_with_limit(filter, data, element_size, max_output_len)?
155
+ } else {
156
+ apply_builtin_filter_with_limit(filter, data, element_size, max_output_len)?
157
+ };
158
+ validate_output_limit("filter pipeline", output.len(), max_output_len)?;
159
+ return Ok(output);
160
+ }
161
+ }
162
+
163
+ // Multi-filter pipeline: the first stage reads from the borrowed input
164
+ // slice (avoiding a copy), subsequent stages consume the previous output.
165
+ // Each filter stage necessarily allocates (output sizes are unpredictable),
166
+ // but we avoid the initial data.to_vec() copy.
167
+ let mut owned: Option<Vec<u8>> = None;
168
+
169
+ for (i, filter) in filters.iter().enumerate().rev() {
170
+ if filter_mask & (1 << i) != 0 {
171
+ continue;
172
+ }
173
+
174
+ let input: &[u8] = match &owned {
175
+ Some(buf) => buf,
176
+ None => data,
177
+ };
178
+
179
+ owned = Some(if let Some(reg) = registry {
180
+ reg.apply_with_limit(filter, input, element_size, max_output_len)?
181
+ } else {
182
+ apply_builtin_filter_with_limit(filter, input, element_size, max_output_len)?
183
+ });
184
+ }
185
+
186
+ let output = owned.unwrap_or_else(|| data.to_vec());
187
+ validate_output_limit("filter pipeline", output.len(), max_output_len)?;
188
+ Ok(output)
189
+ }
190
+
191
+ fn apply_builtin_filter_with_limit(
192
+ filter: &FilterDescription,
193
+ data: &[u8],
194
+ element_size: usize,
195
+ max_output_len: Option<usize>,
196
+ ) -> Result<Vec<u8>> {
197
+ match filter.id {
198
+ FILTER_DEFLATE => match max_output_len {
199
+ Some(max_output_len) => deflate::decompress_with_limit(data, max_output_len),
200
+ None => deflate::decompress(data),
201
+ },
202
+ FILTER_SHUFFLE => Ok(shuffle::unshuffle(data, element_size)),
203
+ FILTER_FLETCHER32 => fletcher32::verify_and_strip(data),
204
+ FILTER_SZIP => Err(Error::UnsupportedFilter("szip".into())),
205
+ FILTER_NBIT => match max_output_len {
206
+ Some(max_output_len) => {
207
+ nbit::decompress_with_limit(data, &filter.client_data, max_output_len)
208
+ }
209
+ None => nbit::decompress(data, &filter.client_data),
210
+ },
211
+ FILTER_SCALEOFFSET => match max_output_len {
212
+ Some(max_output_len) => {
213
+ scaleoffset::decompress_with_limit(data, &filter.client_data, max_output_len)
214
+ }
215
+ None => scaleoffset::decompress(data, &filter.client_data),
216
+ },
217
+ #[cfg(feature = "lz4")]
218
+ FILTER_LZ4 => match max_output_len {
219
+ Some(max_output_len) => lz4::decompress_with_limit(data, max_output_len),
220
+ None => lz4::decompress(data),
221
+ },
222
+ id => Err(Error::UnsupportedFilter(format!("filter id {}", id))),
223
+ }
224
+ }
225
+
226
+ /// A zero-filled filter output buffer, or an error when it cannot be allocated: a corrupted
227
+ /// chunk shape or size field must fail the read, not abort the process.
228
+ pub(crate) fn zeroed_output(len: usize, context: &str) -> Result<Vec<u8>> {
229
+ let mut out = Vec::new();
230
+ out.try_reserve_exact(len).map_err(|_| {
231
+ Error::DecompressionError(format!("{context}: cannot allocate {len} output bytes"))
232
+ })?;
233
+ out.resize(len, 0);
234
+ Ok(out)
235
+ }
236
+
237
+ fn validate_output_limit(context: &str, len: usize, max_output_len: Option<usize>) -> Result<()> {
238
+ if let Some(max_output_len) = max_output_len {
239
+ if len > max_output_len {
240
+ return Err(Error::DecompressionError(format!(
241
+ "{context} decoded to {len} bytes, limit {max_output_len}"
242
+ )));
243
+ }
244
+ }
245
+ Ok(())
246
+ }
247
+
248
+ #[cfg(test)]
249
+ mod tests {
250
+ use super::*;
251
+ use flate2::write::ZlibEncoder;
252
+ use flate2::Compression;
253
+ use std::io::Write;
254
+
255
+ #[test]
256
+ fn filter_registry_default() {
257
+ let registry = FilterRegistry::new();
258
+ // Built-in filters should be registered
259
+ assert!(registry.filters.contains_key(&FILTER_DEFLATE));
260
+ assert!(registry.filters.contains_key(&FILTER_SHUFFLE));
261
+ assert!(registry.filters.contains_key(&FILTER_FLETCHER32));
262
+ assert!(registry.filters.contains_key(&FILTER_NBIT));
263
+ assert!(registry.filters.contains_key(&FILTER_SCALEOFFSET));
264
+ }
265
+
266
+ #[test]
267
+ fn filter_registry_custom() {
268
+ let mut registry = FilterRegistry::new();
269
+ // Register a no-op custom filter
270
+ registry.register(32000, Box::new(|_, data, _| Ok(data.to_vec())));
271
+ let filter = FilterDescription {
272
+ id: 32000,
273
+ name: None,
274
+ client_data: Vec::new(),
275
+ };
276
+ let result = registry.apply(&filter, &[1, 2, 3], 1).unwrap();
277
+ assert_eq!(result, vec![1, 2, 3]);
278
+ }
279
+
280
+ #[test]
281
+ fn filter_registry_unknown() {
282
+ let registry = FilterRegistry::new();
283
+ let filter = FilterDescription {
284
+ id: 9999,
285
+ name: None,
286
+ client_data: Vec::new(),
287
+ };
288
+ let err = registry.apply(&filter, &[1, 2, 3], 1).unwrap_err();
289
+ assert!(matches!(err, Error::UnsupportedFilter(_)));
290
+ }
291
+
292
+ #[test]
293
+ fn apply_pipeline_with_limit_caps_registry_deflate_output() {
294
+ let original = vec![0u8; 4096];
295
+ let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
296
+ encoder.write_all(&original).unwrap();
297
+ let compressed = encoder.finish().unwrap();
298
+ let filter = FilterDescription {
299
+ id: FILTER_DEFLATE,
300
+ name: None,
301
+ client_data: vec![6],
302
+ };
303
+ let registry = FilterRegistry::new();
304
+
305
+ let decoded =
306
+ apply_pipeline_with_limit(&compressed, &[filter], 0, 1, Some(&registry), Some(65))
307
+ .unwrap();
308
+
309
+ assert_eq!(decoded.len(), 65);
310
+ assert_eq!(decoded, original[..65]);
311
+ }
312
+
313
+ #[test]
314
+ fn apply_pipeline_with_limit_rejects_oversized_final_output() {
315
+ let filter = FilterDescription {
316
+ id: FILTER_SHUFFLE,
317
+ name: None,
318
+ client_data: Vec::new(),
319
+ };
320
+
321
+ let err =
322
+ apply_pipeline_with_limit(&[1, 2, 3, 4], &[filter], 0, 1, None, Some(3)).unwrap_err();
323
+
324
+ assert!(err.to_string().contains("limit 3"));
325
+ }
326
+ }