@nirs4all/formats-wasm 0.2.12 → 0.2.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/THIRD_PARTY_NOTICES.md +49 -0
  2. package/nirs4all_formats_wasm_bg.wasm +0 -0
  3. package/package.json +4 -2
  4. package/reader-backports/hdf5-reader/Cargo.toml +119 -0
  5. package/reader-backports/hdf5-reader/LICENSE-APACHE +164 -0
  6. package/reader-backports/hdf5-reader/LICENSE-MIT +21 -0
  7. package/reader-backports/hdf5-reader/PROVENANCE.json +80 -0
  8. package/reader-backports/hdf5-reader/README.md +308 -0
  9. package/reader-backports/hdf5-reader/UPSTREAM-Cargo.toml +44 -0
  10. package/reader-backports/hdf5-reader/UPSTREAM-VCS.json +6 -0
  11. package/reader-backports/hdf5-reader/src/attribute_api.rs +847 -0
  12. package/reader-backports/hdf5-reader/src/btree_v1.rs +726 -0
  13. package/reader-backports/hdf5-reader/src/btree_v2.rs +2068 -0
  14. package/reader-backports/hdf5-reader/src/cache.rs +354 -0
  15. package/reader-backports/hdf5-reader/src/checksum.rs +56 -0
  16. package/reader-backports/hdf5-reader/src/chunk_index.rs +423 -0
  17. package/reader-backports/hdf5-reader/src/dataset.rs +4733 -0
  18. package/reader-backports/hdf5-reader/src/datatype_api.rs +497 -0
  19. package/reader-backports/hdf5-reader/src/error.rs +167 -0
  20. package/reader-backports/hdf5-reader/src/extensible_array.rs +1886 -0
  21. package/reader-backports/hdf5-reader/src/filters/deflate.rs +93 -0
  22. package/reader-backports/hdf5-reader/src/filters/fletcher32.rs +52 -0
  23. package/reader-backports/hdf5-reader/src/filters/lz4.rs +178 -0
  24. package/reader-backports/hdf5-reader/src/filters/mod.rs +326 -0
  25. package/reader-backports/hdf5-reader/src/filters/nbit.rs +586 -0
  26. package/reader-backports/hdf5-reader/src/filters/scaleoffset.rs +827 -0
  27. package/reader-backports/hdf5-reader/src/filters/shuffle.rs +99 -0
  28. package/reader-backports/hdf5-reader/src/fixed_array.rs +763 -0
  29. package/reader-backports/hdf5-reader/src/fractal_heap.rs +2029 -0
  30. package/reader-backports/hdf5-reader/src/global_heap.rs +284 -0
  31. package/reader-backports/hdf5-reader/src/group.rs +795 -0
  32. package/reader-backports/hdf5-reader/src/io.rs +413 -0
  33. package/reader-backports/hdf5-reader/src/lib.rs +916 -0
  34. package/reader-backports/hdf5-reader/src/local_heap.rs +261 -0
  35. package/reader-backports/hdf5-reader/src/messages/attribute.rs +304 -0
  36. package/reader-backports/hdf5-reader/src/messages/attribute_info.rs +96 -0
  37. package/reader-backports/hdf5-reader/src/messages/btree_k.rs +36 -0
  38. package/reader-backports/hdf5-reader/src/messages/continuation.rs +46 -0
  39. package/reader-backports/hdf5-reader/src/messages/dataspace.rs +259 -0
  40. package/reader-backports/hdf5-reader/src/messages/datatype.rs +938 -0
  41. package/reader-backports/hdf5-reader/src/messages/external_files.rs +50 -0
  42. package/reader-backports/hdf5-reader/src/messages/fill_value.rs +233 -0
  43. package/reader-backports/hdf5-reader/src/messages/filter_pipeline.rs +147 -0
  44. package/reader-backports/hdf5-reader/src/messages/group_info.rs +103 -0
  45. package/reader-backports/hdf5-reader/src/messages/layout.rs +476 -0
  46. package/reader-backports/hdf5-reader/src/messages/link.rs +170 -0
  47. package/reader-backports/hdf5-reader/src/messages/link_info.rs +127 -0
  48. package/reader-backports/hdf5-reader/src/messages/mod.rs +374 -0
  49. package/reader-backports/hdf5-reader/src/messages/modification_time.rs +83 -0
  50. package/reader-backports/hdf5-reader/src/messages/shared.rs +164 -0
  51. package/reader-backports/hdf5-reader/src/messages/shared_table.rs +67 -0
  52. package/reader-backports/hdf5-reader/src/messages/symbol_table_msg.rs +51 -0
  53. package/reader-backports/hdf5-reader/src/object_header.rs +1749 -0
  54. package/reader-backports/hdf5-reader/src/reference.rs +90 -0
  55. package/reader-backports/hdf5-reader/src/shared_message_table.rs +425 -0
  56. package/reader-backports/hdf5-reader/src/storage.rs +538 -0
  57. package/reader-backports/hdf5-reader/src/superblock.rs +285 -0
  58. package/reader-backports/hdf5-reader/src/symbol_table.rs +277 -0
  59. package/reader-backports/netcdf-reader/Cargo.toml +141 -0
  60. package/reader-backports/netcdf-reader/LICENSE-APACHE +164 -0
  61. package/reader-backports/netcdf-reader/LICENSE-MIT +21 -0
  62. package/reader-backports/netcdf-reader/PROVENANCE.json +55 -0
  63. package/reader-backports/netcdf-reader/README.md +308 -0
  64. package/reader-backports/netcdf-reader/UPSTREAM-Cargo.toml +48 -0
  65. package/reader-backports/netcdf-reader/UPSTREAM-VCS.json +6 -0
  66. package/reader-backports/netcdf-reader/src/cf/axes.rs +419 -0
  67. package/reader-backports/netcdf-reader/src/cf/bounds.rs +108 -0
  68. package/reader-backports/netcdf-reader/src/cf/crs.rs +269 -0
  69. package/reader-backports/netcdf-reader/src/cf/mod.rs +30 -0
  70. package/reader-backports/netcdf-reader/src/cf/time.rs +1122 -0
  71. package/reader-backports/netcdf-reader/src/classic/data.rs +1465 -0
  72. package/reader-backports/netcdf-reader/src/classic/header.rs +1097 -0
  73. package/reader-backports/netcdf-reader/src/classic/mod.rs +240 -0
  74. package/reader-backports/netcdf-reader/src/classic/storage.rs +211 -0
  75. package/reader-backports/netcdf-reader/src/classic/types.rs +93 -0
  76. package/reader-backports/netcdf-reader/src/classic/variable.rs +1162 -0
  77. package/reader-backports/netcdf-reader/src/error.rs +59 -0
  78. package/reader-backports/netcdf-reader/src/lib.rs +1531 -0
  79. package/reader-backports/netcdf-reader/src/masked.rs +173 -0
  80. package/reader-backports/netcdf-reader/src/nc4/attributes.rs +203 -0
  81. package/reader-backports/netcdf-reader/src/nc4/dimensions.rs +139 -0
  82. package/reader-backports/netcdf-reader/src/nc4/groups.rs +308 -0
  83. package/reader-backports/netcdf-reader/src/nc4/mod.rs +572 -0
  84. package/reader-backports/netcdf-reader/src/nc4/types.rs +309 -0
  85. package/reader-backports/netcdf-reader/src/nc4/variables.rs +420 -0
  86. package/reader-backports/netcdf-reader/src/types.rs +18 -0
  87. package/reader-backports/netcdf-reader/src/unpack.rs +113 -0
  88. package/reader-backports/netcdf-reader/src/user_defined.rs +569 -0
@@ -0,0 +1,827 @@
1
+ use crate::error::{Error, Result};
2
+
3
+ const SCALE_FLOAT_DSCALE: u32 = 0;
4
+ const SCALE_FLOAT_ESCALE: u32 = 1;
5
+ const SCALE_INT: u32 = 2;
6
+
7
+ const CLASS_INTEGER: u32 = 0;
8
+ const CLASS_FLOAT: u32 = 1;
9
+
10
+ const SIGN_UNSIGNED: u32 = 0;
11
+ const SIGN_SIGNED: u32 = 1;
12
+
13
+ const ORDER_LE: u32 = 0;
14
+ const ORDER_BE: u32 = 1;
15
+
16
+ const FILL_UNDEFINED: u32 = 0;
17
+ const HEADER_SIZE: usize = 21;
18
+ const TOTAL_PARAMS: usize = 20;
19
+
20
+ #[derive(Clone, Copy)]
21
+ struct ScaleOffsetParams<'a> {
22
+ raw: &'a [u32],
23
+ element_count: usize,
24
+ class: u32,
25
+ size: usize,
26
+ sign: u32,
27
+ order: u32,
28
+ fill_available: u32,
29
+ scale_type: u32,
30
+ scale_factor: i32,
31
+ }
32
+
33
+ #[derive(Clone, Copy)]
34
+ struct PackedParams {
35
+ size: usize,
36
+ minbits: usize,
37
+ native_order: u32,
38
+ }
39
+
40
+ struct BitReader<'a> {
41
+ data: &'a [u8],
42
+ byte_idx: usize,
43
+ bits_left: u8,
44
+ }
45
+
46
+ impl<'a> BitReader<'a> {
47
+ fn new(data: &'a [u8]) -> Self {
48
+ Self {
49
+ data,
50
+ byte_idx: 0,
51
+ bits_left: 8,
52
+ }
53
+ }
54
+
55
+ fn read_bits(&mut self, count: usize) -> Result<u8> {
56
+ if count > 8 {
57
+ return Err(filter_error(
58
+ "scaleoffset attempted to read more than one byte of packed bits",
59
+ ));
60
+ }
61
+ if count == 0 {
62
+ return Ok(0);
63
+ }
64
+
65
+ let mut remaining = count;
66
+ let mut value = 0u16;
67
+
68
+ while remaining > 0 {
69
+ let current = self
70
+ .data
71
+ .get(self.byte_idx)
72
+ .copied()
73
+ .ok_or_else(|| filter_error("scaleoffset packed stream ended early"))?;
74
+ let take = remaining.min(self.bits_left as usize);
75
+ let shift = self.bits_left as usize - take;
76
+ let chunk = (current >> shift) & low_mask(take);
77
+ value = (value << take) | u16::from(chunk);
78
+ self.bits_left -= take as u8;
79
+ remaining -= take;
80
+
81
+ if self.bits_left == 0 {
82
+ self.byte_idx += 1;
83
+ self.bits_left = 8;
84
+ }
85
+ }
86
+
87
+ Ok(value as u8)
88
+ }
89
+ }
90
+
91
+ pub fn decompress(data: &[u8], client_data: &[u32]) -> Result<Vec<u8>> {
92
+ decompress_inner(data, client_data, None)
93
+ }
94
+
95
+ pub fn decompress_with_limit(
96
+ data: &[u8],
97
+ client_data: &[u32],
98
+ max_output_len: usize,
99
+ ) -> Result<Vec<u8>> {
100
+ decompress_inner(data, client_data, Some(max_output_len))
101
+ }
102
+
103
+ fn decompress_inner(
104
+ data: &[u8],
105
+ client_data: &[u32],
106
+ max_output_len: Option<usize>,
107
+ ) -> Result<Vec<u8>> {
108
+ let params = parse_params(client_data)?;
109
+ let full_width_bits = params
110
+ .size
111
+ .checked_mul(8)
112
+ .ok_or_else(|| filter_error("scaleoffset datatype width overflowed"))?;
113
+
114
+ if params.class == CLASS_FLOAT && params.scale_type == SCALE_FLOAT_ESCALE {
115
+ return Err(Error::UnsupportedFilter("scaleoffset float E-scale".into()));
116
+ }
117
+
118
+ if params.scale_type != SCALE_FLOAT_DSCALE && params.scale_factor as usize == full_width_bits {
119
+ validate_output_limit(data.len(), max_output_len)?;
120
+ return Ok(data.to_vec());
121
+ }
122
+
123
+ if params.class == CLASS_FLOAT {
124
+ if params.scale_type != SCALE_FLOAT_DSCALE && params.scale_type != SCALE_FLOAT_ESCALE {
125
+ return Err(filter_error(
126
+ "scaleoffset float filter has an invalid scale type",
127
+ ));
128
+ }
129
+ } else if params.class == CLASS_INTEGER {
130
+ if params.scale_type != SCALE_INT {
131
+ return Err(filter_error(
132
+ "scaleoffset integer filter has an invalid scale type",
133
+ ));
134
+ }
135
+ } else {
136
+ return Err(filter_error("scaleoffset datatype class is not supported"));
137
+ }
138
+
139
+ if data.len() < HEADER_SIZE {
140
+ return Err(filter_error(
141
+ "scaleoffset payload is shorter than the filter header",
142
+ ));
143
+ }
144
+
145
+ let minbits = u32::from_le_bytes([data[0], data[1], data[2], data[3]]) as usize;
146
+ if minbits > full_width_bits {
147
+ return Err(filter_error(
148
+ "scaleoffset header encodes more bits than the datatype can hold",
149
+ ));
150
+ }
151
+
152
+ let minval_size = usize::min(std::mem::size_of::<u64>(), data[4] as usize);
153
+ if HEADER_SIZE > data.len() || 5 + minval_size > data.len() {
154
+ return Err(filter_error("scaleoffset header is truncated"));
155
+ }
156
+
157
+ let mut minval_bytes = [0u8; 8];
158
+ minval_bytes[..minval_size].copy_from_slice(&data[5..5 + minval_size]);
159
+ let minval = u64::from_le_bytes(minval_bytes);
160
+
161
+ let total_size = params
162
+ .element_count
163
+ .checked_mul(params.size)
164
+ .ok_or_else(|| filter_error("scaleoffset output size overflowed"))?;
165
+ validate_output_limit(total_size, max_output_len)?;
166
+ let mut out = super::zeroed_output(total_size, "scaleoffset")?;
167
+
168
+ if minbits == full_width_bits {
169
+ let payload_end = HEADER_SIZE
170
+ .checked_add(total_size)
171
+ .ok_or_else(|| filter_error("scaleoffset payload size overflowed"))?;
172
+ let payload = data.get(HEADER_SIZE..payload_end).ok_or_else(|| {
173
+ filter_error("scaleoffset payload is shorter than the expected output")
174
+ })?;
175
+ out.copy_from_slice(payload);
176
+
177
+ if params.order != native_order() {
178
+ swap_endian_in_place(&mut out, params.size);
179
+ }
180
+
181
+ return Ok(out);
182
+ } else if minbits != 0 {
183
+ let packed = PackedParams {
184
+ size: params.size,
185
+ minbits,
186
+ native_order: native_order(),
187
+ };
188
+ unpack_packed(&mut out, params.element_count, &data[HEADER_SIZE..], packed)?;
189
+ }
190
+
191
+ match params.class {
192
+ CLASS_INTEGER => postprocess_integers(&mut out, params, minbits, minval)?,
193
+ CLASS_FLOAT => postprocess_floats(&mut out, params, minbits, minval)?,
194
+ _ => unreachable!(),
195
+ }
196
+
197
+ if params.order != native_order() {
198
+ swap_endian_in_place(&mut out, params.size);
199
+ }
200
+
201
+ Ok(out)
202
+ }
203
+
204
+ fn validate_output_limit(len: usize, max_output_len: Option<usize>) -> Result<()> {
205
+ if let Some(max_output_len) = max_output_len {
206
+ if len > max_output_len {
207
+ return Err(filter_error(&format!(
208
+ "scaleoffset output size {len} exceeds limit {max_output_len}"
209
+ )));
210
+ }
211
+ }
212
+ Ok(())
213
+ }
214
+
215
+ fn parse_params(client_data: &[u32]) -> Result<ScaleOffsetParams<'_>> {
216
+ if client_data.len() != TOTAL_PARAMS {
217
+ return Err(filter_error(
218
+ "scaleoffset client data length does not match the required parameter count",
219
+ ));
220
+ }
221
+
222
+ Ok(ScaleOffsetParams {
223
+ raw: client_data,
224
+ element_count: usize_from_u32(client_data[2], "scaleoffset element count")?,
225
+ class: client_data[3],
226
+ size: usize_from_u32(client_data[4], "scaleoffset datatype size")?,
227
+ sign: client_data[5],
228
+ order: client_data[6],
229
+ fill_available: client_data[7],
230
+ scale_type: client_data[0],
231
+ scale_factor: client_data[1] as i32,
232
+ })
233
+ }
234
+
235
+ fn unpack_packed(
236
+ out: &mut [u8],
237
+ element_count: usize,
238
+ data: &[u8],
239
+ packed: PackedParams,
240
+ ) -> Result<()> {
241
+ let mut bits = BitReader::new(data);
242
+ let dtype_len = packed
243
+ .size
244
+ .checked_mul(8)
245
+ .ok_or_else(|| filter_error("scaleoffset datatype width overflowed"))?;
246
+
247
+ for i in 0..element_count {
248
+ let offset = i
249
+ .checked_mul(packed.size)
250
+ .ok_or_else(|| filter_error("scaleoffset output offset overflowed"))?;
251
+ let slice = out
252
+ .get_mut(offset..offset + packed.size)
253
+ .ok_or_else(|| filter_error("scaleoffset output range is out of bounds"))?;
254
+
255
+ match packed.native_order {
256
+ ORDER_LE => {
257
+ let begin = packed.size - 1 - (dtype_len - packed.minbits) / 8;
258
+ for k in (0..=begin).rev() {
259
+ let bit_count = if k == begin {
260
+ let remainder = (dtype_len - packed.minbits) % 8;
261
+ if remainder == 0 {
262
+ 8
263
+ } else {
264
+ 8 - remainder
265
+ }
266
+ } else {
267
+ 8
268
+ };
269
+ slice[k] = bits.read_bits(bit_count)?;
270
+ }
271
+ }
272
+ ORDER_BE => {
273
+ let begin = (dtype_len - packed.minbits) / 8;
274
+ for (k, byte) in slice.iter_mut().enumerate().take(packed.size).skip(begin) {
275
+ let bit_count = if k == begin {
276
+ let remainder = (dtype_len - packed.minbits) % 8;
277
+ if remainder == 0 {
278
+ 8
279
+ } else {
280
+ 8 - remainder
281
+ }
282
+ } else {
283
+ 8
284
+ };
285
+ *byte = bits.read_bits(bit_count)?;
286
+ }
287
+ }
288
+ _ => {
289
+ return Err(filter_error(
290
+ "scaleoffset encountered an unknown byte order",
291
+ ))
292
+ }
293
+ }
294
+ }
295
+
296
+ Ok(())
297
+ }
298
+
299
+ fn postprocess_integers(
300
+ out: &mut [u8],
301
+ params: ScaleOffsetParams<'_>,
302
+ minbits: usize,
303
+ minval: u64,
304
+ ) -> Result<()> {
305
+ let fill = if params.fill_available != FILL_UNDEFINED {
306
+ Some(extract_fill_bytes(params.raw, params.size)?)
307
+ } else {
308
+ None
309
+ };
310
+ let fill_marker = bit_marker(minbits);
311
+
312
+ match params.sign {
313
+ SIGN_UNSIGNED => {
314
+ let min = truncate_unsigned(minval, params.size)?;
315
+ for chunk in out.chunks_exact_mut(params.size) {
316
+ let raw = read_unsigned_native(chunk)?;
317
+ let value = if let Some(fill_bytes) = &fill {
318
+ if raw == fill_marker {
319
+ read_unsigned_native(fill_bytes)?
320
+ } else {
321
+ raw.wrapping_add(min)
322
+ }
323
+ } else {
324
+ raw.wrapping_add(min)
325
+ };
326
+ write_unsigned_native(chunk, value)?;
327
+ }
328
+ }
329
+ SIGN_SIGNED => {
330
+ let min = truncate_signed(minval, params.size)?;
331
+ for chunk in out.chunks_exact_mut(params.size) {
332
+ let raw_unsigned = read_unsigned_native(chunk)?;
333
+ let value = if let Some(fill_bytes) = &fill {
334
+ if raw_unsigned == fill_marker {
335
+ read_signed_native(fill_bytes)?
336
+ } else {
337
+ read_signed_native(chunk)?.wrapping_add(min)
338
+ }
339
+ } else {
340
+ read_signed_native(chunk)?.wrapping_add(min)
341
+ };
342
+ write_signed_native(chunk, value)?;
343
+ }
344
+ }
345
+ _ => {
346
+ return Err(filter_error(
347
+ "scaleoffset integer sign code is not supported",
348
+ ))
349
+ }
350
+ }
351
+
352
+ Ok(())
353
+ }
354
+
355
+ fn postprocess_floats(
356
+ out: &mut [u8],
357
+ params: ScaleOffsetParams<'_>,
358
+ minbits: usize,
359
+ minval: u64,
360
+ ) -> Result<()> {
361
+ if params.scale_type != SCALE_FLOAT_DSCALE {
362
+ return Err(Error::UnsupportedFilter("scaleoffset float E-scale".into()));
363
+ }
364
+
365
+ let scale = 10f64.powi(params.scale_factor);
366
+ let fill_marker = bit_marker(minbits);
367
+ let fill = if params.fill_available != FILL_UNDEFINED {
368
+ Some(extract_fill_bytes(params.raw, params.size)?)
369
+ } else {
370
+ None
371
+ };
372
+
373
+ match params.size {
374
+ 4 => {
375
+ let min_bytes = truncate_minval_bytes(minval, 4)?;
376
+ let min = f32::from_ne_bytes(min_bytes[..4].try_into().unwrap());
377
+ for chunk in out.chunks_exact_mut(4) {
378
+ let raw = i32::from_ne_bytes(chunk.try_into().unwrap());
379
+ let value = if let Some(fill_bytes) = &fill {
380
+ if raw as u32 as u64 == fill_marker {
381
+ f32::from_ne_bytes(fill_bytes.clone().try_into().unwrap())
382
+ } else {
383
+ (raw as f64 / scale) as f32 + min
384
+ }
385
+ } else {
386
+ (raw as f64 / scale) as f32 + min
387
+ };
388
+ chunk.copy_from_slice(&value.to_ne_bytes());
389
+ }
390
+ }
391
+ 8 => {
392
+ let min_bytes = truncate_minval_bytes(minval, 8)?;
393
+ let min = f64::from_ne_bytes(min_bytes[..8].try_into().unwrap());
394
+ for chunk in out.chunks_exact_mut(8) {
395
+ let raw = i64::from_ne_bytes(chunk.try_into().unwrap());
396
+ let value = if let Some(fill_bytes) = &fill {
397
+ if raw as u64 == fill_marker {
398
+ f64::from_ne_bytes(fill_bytes.clone().try_into().unwrap())
399
+ } else {
400
+ raw as f64 / scale + min
401
+ }
402
+ } else {
403
+ raw as f64 / scale + min
404
+ };
405
+ chunk.copy_from_slice(&value.to_ne_bytes());
406
+ }
407
+ }
408
+ _ => {
409
+ return Err(filter_error(
410
+ "scaleoffset floating-point decode only supports 4-byte and 8-byte values",
411
+ ))
412
+ }
413
+ }
414
+
415
+ Ok(())
416
+ }
417
+
418
+ fn extract_fill_bytes(params: &[u32], size: usize) -> Result<Vec<u8>> {
419
+ let mut out = vec![0u8; size];
420
+ let mut idx = 8usize;
421
+ if native_order() == ORDER_LE {
422
+ let mut pos = 0usize;
423
+ while pos < size {
424
+ let word = *params
425
+ .get(idx)
426
+ .ok_or_else(|| filter_error("scaleoffset fill value is truncated"))?;
427
+ let bytes = word.to_ne_bytes();
428
+ let take = (size - pos).min(4);
429
+ out[pos..pos + take].copy_from_slice(&bytes[..take]);
430
+ pos += take;
431
+ idx += 1;
432
+ }
433
+ } else {
434
+ let mut remaining = size;
435
+ let mut pos = size.saturating_sub(remaining.min(4));
436
+ while remaining >= 4 {
437
+ let word = *params
438
+ .get(idx)
439
+ .ok_or_else(|| filter_error("scaleoffset fill value is truncated"))?;
440
+ out[pos..pos + 4].copy_from_slice(&word.to_ne_bytes());
441
+ idx += 1;
442
+ remaining -= 4;
443
+ if remaining >= 4 {
444
+ pos -= 4;
445
+ } else if remaining > 0 {
446
+ pos -= remaining;
447
+ }
448
+ }
449
+ if remaining > 0 {
450
+ let word = *params
451
+ .get(idx)
452
+ .ok_or_else(|| filter_error("scaleoffset fill value is truncated"))?;
453
+ let bytes = word.to_ne_bytes();
454
+ out[..remaining].copy_from_slice(&bytes[4 - remaining..]);
455
+ }
456
+ }
457
+ Ok(out)
458
+ }
459
+
460
+ fn truncate_minval_bytes(minval: u64, size: usize) -> Result<Vec<u8>> {
461
+ match size {
462
+ 4 | 8 => {
463
+ let bytes = minval.to_ne_bytes();
464
+ if native_order() == ORDER_LE {
465
+ Ok(bytes[..size].to_vec())
466
+ } else {
467
+ Ok(bytes[8 - size..].to_vec())
468
+ }
469
+ }
470
+ _ => Err(filter_error(
471
+ "scaleoffset floating-point size is not supported",
472
+ )),
473
+ }
474
+ }
475
+
476
+ fn truncate_unsigned(minval: u64, size: usize) -> Result<u64> {
477
+ Ok(match size {
478
+ 1 => minval & 0xFF,
479
+ 2 => minval & 0xFFFF,
480
+ 4 => minval & 0xFFFF_FFFF,
481
+ 8 => minval,
482
+ _ => {
483
+ return Err(filter_error(
484
+ "scaleoffset integer decode only supports 1-, 2-, 4-, and 8-byte values",
485
+ ))
486
+ }
487
+ })
488
+ }
489
+
490
+ fn truncate_signed(minval: u64, size: usize) -> Result<i64> {
491
+ Ok(match size {
492
+ 1 => i8::from_ne_bytes([minval.to_ne_bytes()[0]]) as i64,
493
+ 2 => i16::from_ne_bytes(minval.to_ne_bytes()[..2].try_into().unwrap()) as i64,
494
+ 4 => i32::from_ne_bytes(minval.to_ne_bytes()[..4].try_into().unwrap()) as i64,
495
+ 8 => i64::from_ne_bytes(minval.to_ne_bytes()),
496
+ _ => {
497
+ return Err(filter_error(
498
+ "scaleoffset integer decode only supports 1-, 2-, 4-, and 8-byte values",
499
+ ))
500
+ }
501
+ })
502
+ }
503
+
504
+ fn read_unsigned_native(bytes: &[u8]) -> Result<u64> {
505
+ Ok(match bytes.len() {
506
+ 1 => bytes[0] as u64,
507
+ 2 => u16::from_ne_bytes(bytes.try_into().unwrap()) as u64,
508
+ 4 => u32::from_ne_bytes(bytes.try_into().unwrap()) as u64,
509
+ 8 => u64::from_ne_bytes(bytes.try_into().unwrap()),
510
+ _ => {
511
+ return Err(filter_error(
512
+ "scaleoffset integer decode only supports 1-, 2-, 4-, and 8-byte values",
513
+ ))
514
+ }
515
+ })
516
+ }
517
+
518
+ fn write_unsigned_native(bytes: &mut [u8], value: u64) -> Result<()> {
519
+ match bytes.len() {
520
+ 1 => bytes[0] = value as u8,
521
+ 2 => bytes.copy_from_slice(&(value as u16).to_ne_bytes()),
522
+ 4 => bytes.copy_from_slice(&(value as u32).to_ne_bytes()),
523
+ 8 => bytes.copy_from_slice(&value.to_ne_bytes()),
524
+ _ => {
525
+ return Err(filter_error(
526
+ "scaleoffset integer decode only supports 1-, 2-, 4-, and 8-byte values",
527
+ ))
528
+ }
529
+ }
530
+ Ok(())
531
+ }
532
+
533
+ fn read_signed_native(bytes: &[u8]) -> Result<i64> {
534
+ Ok(match bytes.len() {
535
+ 1 => i8::from_ne_bytes([bytes[0]]) as i64,
536
+ 2 => i16::from_ne_bytes(bytes.try_into().unwrap()) as i64,
537
+ 4 => i32::from_ne_bytes(bytes.try_into().unwrap()) as i64,
538
+ 8 => i64::from_ne_bytes(bytes.try_into().unwrap()),
539
+ _ => {
540
+ return Err(filter_error(
541
+ "scaleoffset integer decode only supports 1-, 2-, 4-, and 8-byte values",
542
+ ))
543
+ }
544
+ })
545
+ }
546
+
547
+ fn write_signed_native(bytes: &mut [u8], value: i64) -> Result<()> {
548
+ match bytes.len() {
549
+ 1 => bytes[0] = value as i8 as u8,
550
+ 2 => bytes.copy_from_slice(&(value as i16).to_ne_bytes()),
551
+ 4 => bytes.copy_from_slice(&(value as i32).to_ne_bytes()),
552
+ 8 => bytes.copy_from_slice(&value.to_ne_bytes()),
553
+ _ => {
554
+ return Err(filter_error(
555
+ "scaleoffset integer decode only supports 1-, 2-, 4-, and 8-byte values",
556
+ ))
557
+ }
558
+ }
559
+ Ok(())
560
+ }
561
+
562
+ fn swap_endian_in_place(bytes: &mut [u8], element_size: usize) {
563
+ if element_size <= 1 {
564
+ return;
565
+ }
566
+ for chunk in bytes.chunks_exact_mut(element_size) {
567
+ chunk.reverse();
568
+ }
569
+ }
570
+
571
+ fn bit_marker(minbits: usize) -> u64 {
572
+ if minbits == 0 {
573
+ 0
574
+ } else if minbits >= 64 {
575
+ u64::MAX
576
+ } else {
577
+ (1u64 << minbits) - 1
578
+ }
579
+ }
580
+
581
+ fn native_order() -> u32 {
582
+ if cfg!(target_endian = "little") {
583
+ ORDER_LE
584
+ } else {
585
+ ORDER_BE
586
+ }
587
+ }
588
+
589
+ fn usize_from_u32(value: u32, what: &str) -> Result<usize> {
590
+ usize::try_from(value).map_err(|_| filter_error(&format!("{what} does not fit in usize")))
591
+ }
592
+
593
+ fn low_mask(bits: usize) -> u8 {
594
+ if bits >= 8 {
595
+ u8::MAX
596
+ } else {
597
+ ((1u16 << bits) - 1) as u8
598
+ }
599
+ }
600
+
601
+ fn filter_error(message: &str) -> Error {
602
+ Error::FilterError(message.into())
603
+ }
604
+
605
+ #[cfg(test)]
606
+ mod tests {
607
+ use super::*;
608
+
609
+ struct BitWriter {
610
+ data: Vec<u8>,
611
+ bits_left: u8,
612
+ }
613
+
614
+ impl BitWriter {
615
+ fn new() -> Self {
616
+ Self {
617
+ data: vec![0],
618
+ bits_left: 8,
619
+ }
620
+ }
621
+
622
+ fn write_bits(&mut self, value: u8, count: usize) {
623
+ if count == 0 {
624
+ return;
625
+ }
626
+
627
+ let mut remaining = count;
628
+ while remaining > 0 {
629
+ let take = remaining.min(self.bits_left as usize);
630
+ let shift = remaining - take;
631
+ let chunk = (value >> shift) & low_mask(take);
632
+ let idx = self.data.len() - 1;
633
+ self.data[idx] |= chunk << (self.bits_left as usize - take);
634
+ self.bits_left -= take as u8;
635
+ remaining -= take;
636
+ if self.bits_left == 0 {
637
+ self.data.push(0);
638
+ self.bits_left = 8;
639
+ }
640
+ }
641
+ }
642
+
643
+ fn finish(mut self) -> Vec<u8> {
644
+ if self.bits_left == 8 {
645
+ self.data.pop();
646
+ }
647
+ self.data
648
+ }
649
+ }
650
+
651
+ fn pack_values(elements: &[Vec<u8>], size: usize, minbits: usize) -> Vec<u8> {
652
+ let mut writer = BitWriter::new();
653
+ let dtype_len = size * 8;
654
+ match native_order() {
655
+ ORDER_LE => {
656
+ let begin = size - 1 - (dtype_len - minbits) / 8;
657
+ for value in elements {
658
+ for k in (0..=begin).rev() {
659
+ let bit_count = if k == begin {
660
+ let remainder = (dtype_len - minbits) % 8;
661
+ if remainder == 0 {
662
+ 8
663
+ } else {
664
+ 8 - remainder
665
+ }
666
+ } else {
667
+ 8
668
+ };
669
+ writer.write_bits(value[k] & low_mask(bit_count), bit_count);
670
+ }
671
+ }
672
+ }
673
+ ORDER_BE => {
674
+ let begin = (dtype_len - minbits) / 8;
675
+ for value in elements {
676
+ for (k, byte) in value.iter().enumerate().take(size).skip(begin) {
677
+ let bit_count = if k == begin {
678
+ let remainder = (dtype_len - minbits) % 8;
679
+ if remainder == 0 {
680
+ 8
681
+ } else {
682
+ 8 - remainder
683
+ }
684
+ } else {
685
+ 8
686
+ };
687
+ writer.write_bits(*byte & low_mask(bit_count), bit_count);
688
+ }
689
+ }
690
+ }
691
+ _ => unreachable!(),
692
+ }
693
+ writer.finish()
694
+ }
695
+
696
+ fn header_with_minval(minbits: u32, minval: u64, payload: &[u8]) -> Vec<u8> {
697
+ let mut data = vec![0u8; HEADER_SIZE];
698
+ data[..4].copy_from_slice(&minbits.to_le_bytes());
699
+ data[4] = 8;
700
+ data[5..13].copy_from_slice(&minval.to_le_bytes());
701
+ data.extend_from_slice(payload);
702
+ data
703
+ }
704
+
705
+ #[test]
706
+ fn decompresses_unsigned_integer_scaleoffset() {
707
+ let encoded_values = vec![
708
+ 0u16.to_ne_bytes().to_vec(),
709
+ 1u16.to_ne_bytes().to_vec(),
710
+ 7u16.to_ne_bytes().to_vec(),
711
+ 10u16.to_ne_bytes().to_vec(),
712
+ ];
713
+ let packed = pack_values(&encoded_values, 2, 4);
714
+ let input = header_with_minval(4, 100, &packed);
715
+ let client_data = vec![
716
+ SCALE_INT,
717
+ 0,
718
+ 4,
719
+ CLASS_INTEGER,
720
+ 2,
721
+ SIGN_UNSIGNED,
722
+ native_order(),
723
+ FILL_UNDEFINED,
724
+ 0,
725
+ 0,
726
+ 0,
727
+ 0,
728
+ 0,
729
+ 0,
730
+ 0,
731
+ 0,
732
+ 0,
733
+ 0,
734
+ 0,
735
+ 0,
736
+ ];
737
+
738
+ let decoded = decompress(&input, &client_data).unwrap();
739
+ let values: Vec<u16> = decoded
740
+ .chunks_exact(2)
741
+ .map(|chunk| u16::from_ne_bytes(chunk.try_into().unwrap()))
742
+ .collect();
743
+ assert_eq!(values, vec![100, 101, 107, 110]);
744
+ }
745
+
746
+ #[test]
747
+ fn decompresses_float_dscale_values() {
748
+ let encoded_values = vec![
749
+ 0i32.to_ne_bytes().to_vec(),
750
+ 25i32.to_ne_bytes().to_vec(),
751
+ 75i32.to_ne_bytes().to_vec(),
752
+ ];
753
+ let packed = pack_values(&encoded_values, 4, 7);
754
+ let min = 1.25f32;
755
+ let minval = u64::from_ne_bytes({
756
+ let mut bytes = [0u8; 8];
757
+ bytes[..4].copy_from_slice(&min.to_ne_bytes());
758
+ bytes
759
+ });
760
+ let input = header_with_minval(7, minval, &packed);
761
+ let client_data = vec![
762
+ SCALE_FLOAT_DSCALE,
763
+ 2,
764
+ 3,
765
+ CLASS_FLOAT,
766
+ 4,
767
+ SIGN_SIGNED,
768
+ native_order(),
769
+ FILL_UNDEFINED,
770
+ 0,
771
+ 0,
772
+ 0,
773
+ 0,
774
+ 0,
775
+ 0,
776
+ 0,
777
+ 0,
778
+ 0,
779
+ 0,
780
+ 0,
781
+ 0,
782
+ ];
783
+
784
+ let decoded = decompress(&input, &client_data).unwrap();
785
+ let values: Vec<f32> = decoded
786
+ .chunks_exact(4)
787
+ .map(|chunk| f32::from_ne_bytes(chunk.try_into().unwrap()))
788
+ .collect();
789
+ assert_eq!(values, vec![1.25, 1.5, 2.0]);
790
+ }
791
+
792
+ #[test]
793
+ fn full_precision_integer_payload_skips_postprocess() {
794
+ let raw_values = [300u16.to_ne_bytes(), 511u16.to_ne_bytes()];
795
+ let payload: Vec<u8> = raw_values.iter().flat_map(|v| v.iter().copied()).collect();
796
+ let input = header_with_minval(16, 700, &payload);
797
+ let client_data = vec![
798
+ SCALE_INT,
799
+ 0,
800
+ 2,
801
+ CLASS_INTEGER,
802
+ 2,
803
+ SIGN_UNSIGNED,
804
+ native_order(),
805
+ FILL_UNDEFINED,
806
+ 0,
807
+ 0,
808
+ 0,
809
+ 0,
810
+ 0,
811
+ 0,
812
+ 0,
813
+ 0,
814
+ 0,
815
+ 0,
816
+ 0,
817
+ 0,
818
+ ];
819
+
820
+ let decoded = decompress(&input, &client_data).unwrap();
821
+ let values: Vec<u16> = decoded
822
+ .chunks_exact(2)
823
+ .map(|chunk| u16::from_ne_bytes(chunk.try_into().unwrap()))
824
+ .collect();
825
+ assert_eq!(values, vec![300, 511]);
826
+ }
827
+ }