@nirs4all/formats-wasm 0.2.12 → 0.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD_PARTY_NOTICES.md +49 -0
- package/nirs4all_formats_wasm.d.ts +4 -4
- package/nirs4all_formats_wasm.js +170 -149
- package/nirs4all_formats_wasm_bg.wasm +0 -0
- package/package.json +4 -2
- package/reader-backports/hdf5-reader/Cargo.toml +119 -0
- package/reader-backports/hdf5-reader/LICENSE-APACHE +164 -0
- package/reader-backports/hdf5-reader/LICENSE-MIT +21 -0
- package/reader-backports/hdf5-reader/PROVENANCE.json +80 -0
- package/reader-backports/hdf5-reader/README.md +308 -0
- package/reader-backports/hdf5-reader/UPSTREAM-Cargo.toml +44 -0
- package/reader-backports/hdf5-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/hdf5-reader/src/attribute_api.rs +847 -0
- package/reader-backports/hdf5-reader/src/btree_v1.rs +726 -0
- package/reader-backports/hdf5-reader/src/btree_v2.rs +2068 -0
- package/reader-backports/hdf5-reader/src/cache.rs +354 -0
- package/reader-backports/hdf5-reader/src/checksum.rs +56 -0
- package/reader-backports/hdf5-reader/src/chunk_index.rs +423 -0
- package/reader-backports/hdf5-reader/src/dataset.rs +4733 -0
- package/reader-backports/hdf5-reader/src/datatype_api.rs +497 -0
- package/reader-backports/hdf5-reader/src/error.rs +167 -0
- package/reader-backports/hdf5-reader/src/extensible_array.rs +1886 -0
- package/reader-backports/hdf5-reader/src/filters/deflate.rs +93 -0
- package/reader-backports/hdf5-reader/src/filters/fletcher32.rs +52 -0
- package/reader-backports/hdf5-reader/src/filters/lz4.rs +178 -0
- package/reader-backports/hdf5-reader/src/filters/mod.rs +326 -0
- package/reader-backports/hdf5-reader/src/filters/nbit.rs +586 -0
- package/reader-backports/hdf5-reader/src/filters/scaleoffset.rs +827 -0
- package/reader-backports/hdf5-reader/src/filters/shuffle.rs +99 -0
- package/reader-backports/hdf5-reader/src/fixed_array.rs +763 -0
- package/reader-backports/hdf5-reader/src/fractal_heap.rs +2029 -0
- package/reader-backports/hdf5-reader/src/global_heap.rs +284 -0
- package/reader-backports/hdf5-reader/src/group.rs +795 -0
- package/reader-backports/hdf5-reader/src/io.rs +413 -0
- package/reader-backports/hdf5-reader/src/lib.rs +916 -0
- package/reader-backports/hdf5-reader/src/local_heap.rs +261 -0
- package/reader-backports/hdf5-reader/src/messages/attribute.rs +304 -0
- package/reader-backports/hdf5-reader/src/messages/attribute_info.rs +96 -0
- package/reader-backports/hdf5-reader/src/messages/btree_k.rs +36 -0
- package/reader-backports/hdf5-reader/src/messages/continuation.rs +46 -0
- package/reader-backports/hdf5-reader/src/messages/dataspace.rs +259 -0
- package/reader-backports/hdf5-reader/src/messages/datatype.rs +938 -0
- package/reader-backports/hdf5-reader/src/messages/external_files.rs +50 -0
- package/reader-backports/hdf5-reader/src/messages/fill_value.rs +233 -0
- package/reader-backports/hdf5-reader/src/messages/filter_pipeline.rs +147 -0
- package/reader-backports/hdf5-reader/src/messages/group_info.rs +103 -0
- package/reader-backports/hdf5-reader/src/messages/layout.rs +476 -0
- package/reader-backports/hdf5-reader/src/messages/link.rs +170 -0
- package/reader-backports/hdf5-reader/src/messages/link_info.rs +127 -0
- package/reader-backports/hdf5-reader/src/messages/mod.rs +374 -0
- package/reader-backports/hdf5-reader/src/messages/modification_time.rs +83 -0
- package/reader-backports/hdf5-reader/src/messages/shared.rs +164 -0
- package/reader-backports/hdf5-reader/src/messages/shared_table.rs +67 -0
- package/reader-backports/hdf5-reader/src/messages/symbol_table_msg.rs +51 -0
- package/reader-backports/hdf5-reader/src/object_header.rs +1749 -0
- package/reader-backports/hdf5-reader/src/reference.rs +90 -0
- package/reader-backports/hdf5-reader/src/shared_message_table.rs +425 -0
- package/reader-backports/hdf5-reader/src/storage.rs +538 -0
- package/reader-backports/hdf5-reader/src/superblock.rs +285 -0
- package/reader-backports/hdf5-reader/src/symbol_table.rs +277 -0
- package/reader-backports/netcdf-reader/Cargo.toml +141 -0
- package/reader-backports/netcdf-reader/LICENSE-APACHE +164 -0
- package/reader-backports/netcdf-reader/LICENSE-MIT +21 -0
- package/reader-backports/netcdf-reader/PROVENANCE.json +55 -0
- package/reader-backports/netcdf-reader/README.md +308 -0
- package/reader-backports/netcdf-reader/UPSTREAM-Cargo.toml +48 -0
- package/reader-backports/netcdf-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/netcdf-reader/src/cf/axes.rs +419 -0
- package/reader-backports/netcdf-reader/src/cf/bounds.rs +108 -0
- package/reader-backports/netcdf-reader/src/cf/crs.rs +269 -0
- package/reader-backports/netcdf-reader/src/cf/mod.rs +30 -0
- package/reader-backports/netcdf-reader/src/cf/time.rs +1122 -0
- package/reader-backports/netcdf-reader/src/classic/data.rs +1465 -0
- package/reader-backports/netcdf-reader/src/classic/header.rs +1097 -0
- package/reader-backports/netcdf-reader/src/classic/mod.rs +240 -0
- package/reader-backports/netcdf-reader/src/classic/storage.rs +211 -0
- package/reader-backports/netcdf-reader/src/classic/types.rs +93 -0
- package/reader-backports/netcdf-reader/src/classic/variable.rs +1162 -0
- package/reader-backports/netcdf-reader/src/error.rs +59 -0
- package/reader-backports/netcdf-reader/src/lib.rs +1531 -0
- package/reader-backports/netcdf-reader/src/masked.rs +173 -0
- package/reader-backports/netcdf-reader/src/nc4/attributes.rs +203 -0
- package/reader-backports/netcdf-reader/src/nc4/dimensions.rs +139 -0
- package/reader-backports/netcdf-reader/src/nc4/groups.rs +308 -0
- package/reader-backports/netcdf-reader/src/nc4/mod.rs +572 -0
- package/reader-backports/netcdf-reader/src/nc4/types.rs +309 -0
- package/reader-backports/netcdf-reader/src/nc4/variables.rs +420 -0
- package/reader-backports/netcdf-reader/src/types.rs +18 -0
- package/reader-backports/netcdf-reader/src/unpack.rs +113 -0
- package/reader-backports/netcdf-reader/src/user_defined.rs +569 -0
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
/// Unshuffle bytes — reverses the HDF5 shuffle filter.
|
|
2
|
+
///
|
|
3
|
+
/// The shuffle filter rearranges bytes so that byte 0 of all elements comes first,
|
|
4
|
+
/// then byte 1 of all elements, etc. This improves compression of numeric arrays
|
|
5
|
+
/// where adjacent elements often have similar high bytes.
|
|
6
|
+
///
|
|
7
|
+
/// On read, we need to unshuffle: given the shuffled bytes, reconstruct the
|
|
8
|
+
/// original element order.
|
|
9
|
+
pub fn unshuffle(data: &[u8], element_size: usize) -> Vec<u8> {
|
|
10
|
+
if element_size <= 1 || data.is_empty() {
|
|
11
|
+
return data.to_vec();
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
let n_elements = data.len() / element_size;
|
|
15
|
+
if n_elements == 0 {
|
|
16
|
+
return data.to_vec();
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
let mut output = vec![0u8; data.len()];
|
|
20
|
+
|
|
21
|
+
for byte_idx in 0..element_size {
|
|
22
|
+
let src_start = byte_idx * n_elements;
|
|
23
|
+
for elem in 0..n_elements {
|
|
24
|
+
if src_start + elem < data.len() && elem * element_size + byte_idx < output.len() {
|
|
25
|
+
output[elem * element_size + byte_idx] = data[src_start + elem];
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
// Copy any trailing bytes that don't form a complete element
|
|
31
|
+
let complete = n_elements * element_size;
|
|
32
|
+
if complete < data.len() {
|
|
33
|
+
output[complete..].copy_from_slice(&data[complete..]);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
output
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/// Shuffle bytes — for testing purposes.
|
|
40
|
+
#[cfg(test)]
|
|
41
|
+
fn shuffle(data: &[u8], element_size: usize) -> Vec<u8> {
|
|
42
|
+
if element_size <= 1 || data.is_empty() {
|
|
43
|
+
return data.to_vec();
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
let n_elements = data.len() / element_size;
|
|
47
|
+
let mut output = vec![0u8; data.len()];
|
|
48
|
+
|
|
49
|
+
for byte_idx in 0..element_size {
|
|
50
|
+
let dst_start = byte_idx * n_elements;
|
|
51
|
+
for elem in 0..n_elements {
|
|
52
|
+
output[dst_start + elem] = data[elem * element_size + byte_idx];
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
let complete = n_elements * element_size;
|
|
57
|
+
if complete < data.len() {
|
|
58
|
+
output[complete..].copy_from_slice(&data[complete..]);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
output
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
#[cfg(test)]
|
|
65
|
+
mod tests {
|
|
66
|
+
use super::*;
|
|
67
|
+
|
|
68
|
+
#[test]
|
|
69
|
+
fn roundtrip_4byte() {
|
|
70
|
+
let original: Vec<u8> = vec![
|
|
71
|
+
0x01, 0x02, 0x03, 0x04, // element 0
|
|
72
|
+
0x05, 0x06, 0x07, 0x08, // element 1
|
|
73
|
+
0x09, 0x0A, 0x0B, 0x0C, // element 2
|
|
74
|
+
];
|
|
75
|
+
let shuffled = shuffle(&original, 4);
|
|
76
|
+
// After shuffle: [0x01,0x05,0x09, 0x02,0x06,0x0A, 0x03,0x07,0x0B, 0x04,0x08,0x0C]
|
|
77
|
+
assert_eq!(
|
|
78
|
+
shuffled,
|
|
79
|
+
vec![0x01, 0x05, 0x09, 0x02, 0x06, 0x0A, 0x03, 0x07, 0x0B, 0x04, 0x08, 0x0C]
|
|
80
|
+
);
|
|
81
|
+
|
|
82
|
+
let unshuffled = unshuffle(&shuffled, 4);
|
|
83
|
+
assert_eq!(unshuffled, original);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
#[test]
|
|
87
|
+
fn single_element() {
|
|
88
|
+
let data = vec![0x01, 0x02, 0x03, 0x04];
|
|
89
|
+
let result = unshuffle(&data, 4);
|
|
90
|
+
assert_eq!(result, data);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
#[test]
|
|
94
|
+
fn element_size_1() {
|
|
95
|
+
let data = vec![0x01, 0x02, 0x03];
|
|
96
|
+
let result = unshuffle(&data, 1);
|
|
97
|
+
assert_eq!(result, data);
|
|
98
|
+
}
|
|
99
|
+
}
|