@nirs4all/formats-wasm 0.2.12 → 0.2.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/THIRD_PARTY_NOTICES.md +49 -0
  2. package/nirs4all_formats_wasm_bg.wasm +0 -0
  3. package/package.json +4 -2
  4. package/reader-backports/hdf5-reader/Cargo.toml +119 -0
  5. package/reader-backports/hdf5-reader/LICENSE-APACHE +164 -0
  6. package/reader-backports/hdf5-reader/LICENSE-MIT +21 -0
  7. package/reader-backports/hdf5-reader/PROVENANCE.json +80 -0
  8. package/reader-backports/hdf5-reader/README.md +308 -0
  9. package/reader-backports/hdf5-reader/UPSTREAM-Cargo.toml +44 -0
  10. package/reader-backports/hdf5-reader/UPSTREAM-VCS.json +6 -0
  11. package/reader-backports/hdf5-reader/src/attribute_api.rs +847 -0
  12. package/reader-backports/hdf5-reader/src/btree_v1.rs +726 -0
  13. package/reader-backports/hdf5-reader/src/btree_v2.rs +2068 -0
  14. package/reader-backports/hdf5-reader/src/cache.rs +354 -0
  15. package/reader-backports/hdf5-reader/src/checksum.rs +56 -0
  16. package/reader-backports/hdf5-reader/src/chunk_index.rs +423 -0
  17. package/reader-backports/hdf5-reader/src/dataset.rs +4733 -0
  18. package/reader-backports/hdf5-reader/src/datatype_api.rs +497 -0
  19. package/reader-backports/hdf5-reader/src/error.rs +167 -0
  20. package/reader-backports/hdf5-reader/src/extensible_array.rs +1886 -0
  21. package/reader-backports/hdf5-reader/src/filters/deflate.rs +93 -0
  22. package/reader-backports/hdf5-reader/src/filters/fletcher32.rs +52 -0
  23. package/reader-backports/hdf5-reader/src/filters/lz4.rs +178 -0
  24. package/reader-backports/hdf5-reader/src/filters/mod.rs +326 -0
  25. package/reader-backports/hdf5-reader/src/filters/nbit.rs +586 -0
  26. package/reader-backports/hdf5-reader/src/filters/scaleoffset.rs +827 -0
  27. package/reader-backports/hdf5-reader/src/filters/shuffle.rs +99 -0
  28. package/reader-backports/hdf5-reader/src/fixed_array.rs +763 -0
  29. package/reader-backports/hdf5-reader/src/fractal_heap.rs +2029 -0
  30. package/reader-backports/hdf5-reader/src/global_heap.rs +284 -0
  31. package/reader-backports/hdf5-reader/src/group.rs +795 -0
  32. package/reader-backports/hdf5-reader/src/io.rs +413 -0
  33. package/reader-backports/hdf5-reader/src/lib.rs +916 -0
  34. package/reader-backports/hdf5-reader/src/local_heap.rs +261 -0
  35. package/reader-backports/hdf5-reader/src/messages/attribute.rs +304 -0
  36. package/reader-backports/hdf5-reader/src/messages/attribute_info.rs +96 -0
  37. package/reader-backports/hdf5-reader/src/messages/btree_k.rs +36 -0
  38. package/reader-backports/hdf5-reader/src/messages/continuation.rs +46 -0
  39. package/reader-backports/hdf5-reader/src/messages/dataspace.rs +259 -0
  40. package/reader-backports/hdf5-reader/src/messages/datatype.rs +938 -0
  41. package/reader-backports/hdf5-reader/src/messages/external_files.rs +50 -0
  42. package/reader-backports/hdf5-reader/src/messages/fill_value.rs +233 -0
  43. package/reader-backports/hdf5-reader/src/messages/filter_pipeline.rs +147 -0
  44. package/reader-backports/hdf5-reader/src/messages/group_info.rs +103 -0
  45. package/reader-backports/hdf5-reader/src/messages/layout.rs +476 -0
  46. package/reader-backports/hdf5-reader/src/messages/link.rs +170 -0
  47. package/reader-backports/hdf5-reader/src/messages/link_info.rs +127 -0
  48. package/reader-backports/hdf5-reader/src/messages/mod.rs +374 -0
  49. package/reader-backports/hdf5-reader/src/messages/modification_time.rs +83 -0
  50. package/reader-backports/hdf5-reader/src/messages/shared.rs +164 -0
  51. package/reader-backports/hdf5-reader/src/messages/shared_table.rs +67 -0
  52. package/reader-backports/hdf5-reader/src/messages/symbol_table_msg.rs +51 -0
  53. package/reader-backports/hdf5-reader/src/object_header.rs +1749 -0
  54. package/reader-backports/hdf5-reader/src/reference.rs +90 -0
  55. package/reader-backports/hdf5-reader/src/shared_message_table.rs +425 -0
  56. package/reader-backports/hdf5-reader/src/storage.rs +538 -0
  57. package/reader-backports/hdf5-reader/src/superblock.rs +285 -0
  58. package/reader-backports/hdf5-reader/src/symbol_table.rs +277 -0
  59. package/reader-backports/netcdf-reader/Cargo.toml +141 -0
  60. package/reader-backports/netcdf-reader/LICENSE-APACHE +164 -0
  61. package/reader-backports/netcdf-reader/LICENSE-MIT +21 -0
  62. package/reader-backports/netcdf-reader/PROVENANCE.json +55 -0
  63. package/reader-backports/netcdf-reader/README.md +308 -0
  64. package/reader-backports/netcdf-reader/UPSTREAM-Cargo.toml +48 -0
  65. package/reader-backports/netcdf-reader/UPSTREAM-VCS.json +6 -0
  66. package/reader-backports/netcdf-reader/src/cf/axes.rs +419 -0
  67. package/reader-backports/netcdf-reader/src/cf/bounds.rs +108 -0
  68. package/reader-backports/netcdf-reader/src/cf/crs.rs +269 -0
  69. package/reader-backports/netcdf-reader/src/cf/mod.rs +30 -0
  70. package/reader-backports/netcdf-reader/src/cf/time.rs +1122 -0
  71. package/reader-backports/netcdf-reader/src/classic/data.rs +1465 -0
  72. package/reader-backports/netcdf-reader/src/classic/header.rs +1097 -0
  73. package/reader-backports/netcdf-reader/src/classic/mod.rs +240 -0
  74. package/reader-backports/netcdf-reader/src/classic/storage.rs +211 -0
  75. package/reader-backports/netcdf-reader/src/classic/types.rs +93 -0
  76. package/reader-backports/netcdf-reader/src/classic/variable.rs +1162 -0
  77. package/reader-backports/netcdf-reader/src/error.rs +59 -0
  78. package/reader-backports/netcdf-reader/src/lib.rs +1531 -0
  79. package/reader-backports/netcdf-reader/src/masked.rs +173 -0
  80. package/reader-backports/netcdf-reader/src/nc4/attributes.rs +203 -0
  81. package/reader-backports/netcdf-reader/src/nc4/dimensions.rs +139 -0
  82. package/reader-backports/netcdf-reader/src/nc4/groups.rs +308 -0
  83. package/reader-backports/netcdf-reader/src/nc4/mod.rs +572 -0
  84. package/reader-backports/netcdf-reader/src/nc4/types.rs +309 -0
  85. package/reader-backports/netcdf-reader/src/nc4/variables.rs +420 -0
  86. package/reader-backports/netcdf-reader/src/types.rs +18 -0
  87. package/reader-backports/netcdf-reader/src/unpack.rs +113 -0
  88. package/reader-backports/netcdf-reader/src/user_defined.rs +569 -0
@@ -0,0 +1,354 @@
1
+ use std::collections::HashMap;
2
+ use std::num::NonZeroUsize;
3
+ use std::sync::Arc;
4
+
5
+ use lru::LruCache;
6
+ use parking_lot::{Condvar, Mutex};
7
+ use smallvec::SmallVec;
8
+
9
+ use crate::error::Result;
10
+
11
+ /// Key for the chunk cache: (dataset object header address, chunk offset tuple).
12
+ ///
13
+ /// Uses `SmallVec<[u64; 4]>` to avoid heap allocation for datasets with up to
14
+ /// 4 dimensions (the common case for climate/science data).
15
+ #[derive(Debug, Clone, PartialEq, Eq, Hash)]
16
+ pub struct ChunkKey {
17
+ pub dataset_addr: u64,
18
+ pub chunk_offsets: SmallVec<[u64; 4]>,
19
+ }
20
+
21
+ /// LRU cache for decompressed chunks.
22
+ ///
23
+ /// Thread-safe via `parking_lot::Mutex` (non-poisoning). Values are
24
+ /// `Arc<Vec<u8>>` so multiple readers can share the same decompressed chunk data.
25
+ pub struct ChunkCache {
26
+ inner: Mutex<ChunkCacheState>,
27
+ max_bytes: usize,
28
+ max_slots: usize,
29
+ }
30
+
31
+ struct ChunkCacheState {
32
+ cache: LruCache<ChunkKey, Arc<Vec<u8>>>,
33
+ current_bytes: usize,
34
+ in_flight: HashMap<ChunkKey, Arc<InFlightLoad>>,
35
+ hits: u64,
36
+ misses: u64,
37
+ inserts: u64,
38
+ evictions: u64,
39
+ }
40
+
41
+ struct InFlightLoad {
42
+ completed: Mutex<bool>,
43
+ ready: Condvar,
44
+ }
45
+
46
+ /// Point-in-time statistics for a [`ChunkCache`].
47
+ #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
48
+ pub struct ChunkCacheStats {
49
+ pub hits: u64,
50
+ pub misses: u64,
51
+ pub inserts: u64,
52
+ pub evictions: u64,
53
+ pub current_bytes: usize,
54
+ pub entries: usize,
55
+ pub in_flight: usize,
56
+ pub max_bytes: usize,
57
+ pub max_slots: usize,
58
+ }
59
+
60
+ impl ChunkCache {
61
+ /// Create a new chunk cache.
62
+ ///
63
+ /// - `max_bytes`: maximum total bytes of decompressed data to cache (default 64 MiB)
64
+ /// - `max_slots`: maximum number of entries (default 521)
65
+ pub fn new(max_bytes: usize, max_slots: usize) -> Self {
66
+ let slots = NonZeroUsize::new(max_slots).unwrap_or(NonZeroUsize::new(521).unwrap());
67
+ ChunkCache {
68
+ inner: Mutex::new(ChunkCacheState {
69
+ cache: LruCache::new(slots),
70
+ current_bytes: 0,
71
+ in_flight: HashMap::new(),
72
+ hits: 0,
73
+ misses: 0,
74
+ inserts: 0,
75
+ evictions: 0,
76
+ }),
77
+ max_bytes,
78
+ max_slots: slots.get(),
79
+ }
80
+ }
81
+
82
+ /// Get a cached chunk, if present. Promotes the entry in LRU order.
83
+ pub fn get(&self, key: &ChunkKey) -> Option<Arc<Vec<u8>>> {
84
+ let mut cache = self.inner.lock();
85
+ let result = cache.cache.get(key).cloned();
86
+ if result.is_some() {
87
+ cache.hits += 1;
88
+ } else {
89
+ cache.misses += 1;
90
+ }
91
+ result
92
+ }
93
+
94
+ /// Insert a chunk into the cache. Evicts LRU entries if over capacity.
95
+ pub fn insert(&self, key: ChunkKey, data: Vec<u8>) -> Arc<Vec<u8>> {
96
+ let data_len = data.len();
97
+ let arc = Arc::new(data);
98
+
99
+ if self.max_bytes == 0 || data_len > self.max_bytes {
100
+ return arc;
101
+ }
102
+
103
+ let mut state = self.inner.lock();
104
+ // Evict until we have room
105
+ while state.current_bytes + data_len > self.max_bytes && !state.cache.is_empty() {
106
+ if let Some((_, evicted)) = state.cache.pop_lru() {
107
+ state.current_bytes = state.current_bytes.saturating_sub(evicted.len());
108
+ state.evictions += 1;
109
+ }
110
+ }
111
+
112
+ if let Some(replaced) = state.cache.peek(&key) {
113
+ state.current_bytes = state.current_bytes.saturating_sub(replaced.len());
114
+ }
115
+ state.current_bytes += data_len;
116
+ state.inserts += 1;
117
+ state.cache.put(key, arc.clone());
118
+
119
+ arc
120
+ }
121
+
122
+ /// Return a cached chunk or compute it once across concurrent callers.
123
+ pub fn get_or_insert_with<F>(&self, key: ChunkKey, load: F) -> Result<Arc<Vec<u8>>>
124
+ where
125
+ F: FnOnce() -> Result<Vec<u8>>,
126
+ {
127
+ loop {
128
+ let in_flight = {
129
+ let mut state = self.inner.lock();
130
+ if let Some(cached) = state.cache.get(&key).cloned() {
131
+ state.hits += 1;
132
+ return Ok(cached);
133
+ }
134
+ state.misses += 1;
135
+
136
+ if let Some(in_flight) = state.in_flight.get(&key) {
137
+ Arc::clone(in_flight)
138
+ } else {
139
+ let in_flight = Arc::new(InFlightLoad {
140
+ completed: Mutex::new(false),
141
+ ready: Condvar::new(),
142
+ });
143
+ state.in_flight.insert(key.clone(), Arc::clone(&in_flight));
144
+ drop(state);
145
+
146
+ let result = load().map(|data| self.insert(key.clone(), data));
147
+
148
+ let mut state = self.inner.lock();
149
+ state.in_flight.remove(&key);
150
+ let mut completed = in_flight.completed.lock();
151
+ *completed = true;
152
+ in_flight.ready.notify_all();
153
+
154
+ return result;
155
+ }
156
+ };
157
+
158
+ let mut completed = in_flight.completed.lock();
159
+ while !*completed {
160
+ in_flight.ready.wait(&mut completed);
161
+ }
162
+ }
163
+ }
164
+
165
+ /// Return current cache statistics.
166
+ pub fn stats(&self) -> ChunkCacheStats {
167
+ let state = self.inner.lock();
168
+ ChunkCacheStats {
169
+ hits: state.hits,
170
+ misses: state.misses,
171
+ inserts: state.inserts,
172
+ evictions: state.evictions,
173
+ current_bytes: state.current_bytes,
174
+ entries: state.cache.len(),
175
+ in_flight: state.in_flight.len(),
176
+ max_bytes: self.max_bytes,
177
+ max_slots: self.max_slots,
178
+ }
179
+ }
180
+ }
181
+
182
+ impl Default for ChunkCache {
183
+ fn default() -> Self {
184
+ Self::new(64 * 1024 * 1024, 521)
185
+ }
186
+ }
187
+
188
+ #[cfg(test)]
189
+ mod tests {
190
+ use super::*;
191
+
192
+ #[test]
193
+ fn cache_insert_and_get() {
194
+ let cache = ChunkCache::new(1024, 10);
195
+ let key = ChunkKey {
196
+ dataset_addr: 100,
197
+ chunk_offsets: SmallVec::from_vec(vec![0, 0]),
198
+ };
199
+ cache.insert(key.clone(), vec![1, 2, 3]);
200
+ let val = cache.get(&key).unwrap();
201
+ assert_eq!(&**val, &[1, 2, 3]);
202
+ }
203
+
204
+ #[test]
205
+ fn cache_eviction() {
206
+ let cache = ChunkCache::new(10, 10); // 10 bytes max
207
+ for i in 0..5 {
208
+ let key = ChunkKey {
209
+ dataset_addr: 100,
210
+ chunk_offsets: SmallVec::from_vec(vec![i]),
211
+ };
212
+ cache.insert(key, vec![0; 4]); // 4 bytes each
213
+ }
214
+ // Should have evicted older entries to stay under 10 bytes
215
+ // At most 2 entries of 4 bytes each = 8 bytes
216
+ let first_key = ChunkKey {
217
+ dataset_addr: 100,
218
+ chunk_offsets: SmallVec::from_vec(vec![0]),
219
+ };
220
+ assert!(cache.get(&first_key).is_none()); // should be evicted
221
+ }
222
+
223
+ #[test]
224
+ fn cache_disabled_bypasses_storage() {
225
+ let cache = ChunkCache::new(0, 10);
226
+ let key = ChunkKey {
227
+ dataset_addr: 100,
228
+ chunk_offsets: SmallVec::from_vec(vec![0]),
229
+ };
230
+ cache.insert(key.clone(), vec![1, 2, 3]);
231
+ assert!(cache.get(&key).is_none());
232
+ }
233
+
234
+ #[test]
235
+ fn cache_promotes_on_get() {
236
+ // Verify that get() promotes entries in LRU order (the bug fix).
237
+ let cache = ChunkCache::new(12, 10); // room for 3 entries of 4 bytes
238
+ let key_a = ChunkKey {
239
+ dataset_addr: 1,
240
+ chunk_offsets: SmallVec::from_vec(vec![0]),
241
+ };
242
+ let key_b = ChunkKey {
243
+ dataset_addr: 2,
244
+ chunk_offsets: SmallVec::from_vec(vec![0]),
245
+ };
246
+ let key_c = ChunkKey {
247
+ dataset_addr: 3,
248
+ chunk_offsets: SmallVec::from_vec(vec![0]),
249
+ };
250
+
251
+ cache.insert(key_a.clone(), vec![0; 4]); // LRU order: a
252
+ cache.insert(key_b.clone(), vec![0; 4]); // LRU order: a, b
253
+ cache.insert(key_c.clone(), vec![0; 4]); // LRU order: a, b, c
254
+
255
+ // Access key_a to promote it
256
+ assert!(cache.get(&key_a).is_some()); // LRU order: b, c, a
257
+
258
+ // Insert a new entry that forces eviction
259
+ let key_d = ChunkKey {
260
+ dataset_addr: 4,
261
+ chunk_offsets: SmallVec::from_vec(vec![0]),
262
+ };
263
+ cache.insert(key_d, vec![0; 4]); // Should evict b (LRU)
264
+
265
+ assert!(cache.get(&key_a).is_some()); // a was promoted, should survive
266
+ assert!(cache.get(&key_b).is_none()); // b was LRU, should be evicted
267
+ }
268
+
269
+ #[test]
270
+ fn cache_replacement_updates_accounting() {
271
+ let cache = ChunkCache::new(8, 10);
272
+ let key = ChunkKey {
273
+ dataset_addr: 100,
274
+ chunk_offsets: SmallVec::from_vec(vec![0]),
275
+ };
276
+
277
+ cache.insert(key.clone(), vec![1, 2, 3, 4]);
278
+ cache.insert(key.clone(), vec![5, 6]);
279
+
280
+ let other = ChunkKey {
281
+ dataset_addr: 100,
282
+ chunk_offsets: SmallVec::from_vec(vec![1]),
283
+ };
284
+ cache.insert(other.clone(), vec![7, 8, 9, 10]);
285
+
286
+ assert_eq!(&**cache.get(&key).unwrap(), &[5, 6]);
287
+ assert!(cache.get(&other).is_some());
288
+ }
289
+
290
+ #[test]
291
+ fn cache_get_or_insert_with_deduplicates_concurrent_loads() {
292
+ use std::sync::atomic::{AtomicUsize, Ordering};
293
+
294
+ let cache = Arc::new(ChunkCache::new(1024, 10));
295
+ let key = ChunkKey {
296
+ dataset_addr: 100,
297
+ chunk_offsets: SmallVec::from_vec(vec![0, 0]),
298
+ };
299
+ let load_count = Arc::new(AtomicUsize::new(0));
300
+
301
+ std::thread::scope(|scope| {
302
+ for _ in 0..8 {
303
+ let cache = Arc::clone(&cache);
304
+ let key = key.clone();
305
+ let load_count = Arc::clone(&load_count);
306
+ scope.spawn(move || {
307
+ let value = cache
308
+ .get_or_insert_with(key, || {
309
+ load_count.fetch_add(1, Ordering::SeqCst);
310
+ std::thread::sleep(std::time::Duration::from_millis(10));
311
+ Ok(vec![1, 2, 3, 4])
312
+ })
313
+ .unwrap();
314
+ assert_eq!(&*value, &[1, 2, 3, 4]);
315
+ });
316
+ }
317
+ });
318
+
319
+ assert_eq!(load_count.load(Ordering::SeqCst), 1);
320
+ }
321
+
322
+ #[test]
323
+ fn cache_stats_track_hits_misses_and_evictions() {
324
+ let cache = ChunkCache::new(8, 2);
325
+ let key_a = ChunkKey {
326
+ dataset_addr: 1,
327
+ chunk_offsets: SmallVec::from_vec(vec![0]),
328
+ };
329
+ let key_b = ChunkKey {
330
+ dataset_addr: 1,
331
+ chunk_offsets: SmallVec::from_vec(vec![1]),
332
+ };
333
+ let key_c = ChunkKey {
334
+ dataset_addr: 1,
335
+ chunk_offsets: SmallVec::from_vec(vec![2]),
336
+ };
337
+
338
+ assert!(cache.get(&key_a).is_none());
339
+ cache.insert(key_a.clone(), vec![1, 2, 3, 4]);
340
+ assert!(cache.get(&key_a).is_some());
341
+ cache.insert(key_b, vec![5, 6, 7, 8]);
342
+ cache.insert(key_c, vec![9, 10, 11, 12]);
343
+
344
+ let stats = cache.stats();
345
+ assert_eq!(stats.hits, 1);
346
+ assert_eq!(stats.misses, 1);
347
+ assert_eq!(stats.inserts, 3);
348
+ assert_eq!(stats.evictions, 1);
349
+ assert_eq!(stats.entries, 2);
350
+ assert_eq!(stats.current_bytes, 8);
351
+ assert_eq!(stats.max_bytes, 8);
352
+ assert_eq!(stats.max_slots, 2);
353
+ }
354
+ }
@@ -0,0 +1,56 @@
1
+ pub use hdf5_core::{fletcher32, jenkins_lookup3};
2
+
3
+ #[cfg(test)]
4
+ mod tests {
5
+ use super::*;
6
+
7
+ #[test]
8
+ fn jenkins_empty() {
9
+ let hash = jenkins_lookup3(b"");
10
+ // Empty input produces a deterministic hash from the initial state
11
+ assert_ne!(hash, 0);
12
+ }
13
+
14
+ #[test]
15
+ fn jenkins_known_value() {
16
+ // Test with known input — the hash is deterministic
17
+ let h1 = jenkins_lookup3(b"hello");
18
+ let h2 = jenkins_lookup3(b"hello");
19
+ assert_eq!(h1, h2);
20
+
21
+ // Different input -> different hash (with overwhelming probability)
22
+ let h3 = jenkins_lookup3(b"world");
23
+ assert_ne!(h1, h3);
24
+ }
25
+
26
+ #[test]
27
+ fn fletcher32_simple() {
28
+ let data = [0x01, 0x02, 0x03, 0x04];
29
+ let checksum = fletcher32(&data);
30
+ // Verify it's deterministic
31
+ assert_eq!(checksum, fletcher32(&data));
32
+ }
33
+
34
+ #[test]
35
+ fn fletcher32_known_reference() {
36
+ // Known reference: 4 bytes [0x00, 0x01, 0x00, 0x02] interpreted as
37
+ // big-endian u16 words: 0x0001 and 0x0002.
38
+ // sum1 = (0 + 1) % 65535 = 1; sum2 = (0 + 1) % 65535 = 1
39
+ // sum1 = (1 + 2) % 65535 = 3; sum2 = (1 + 3) % 65535 = 4
40
+ // result = (4 << 16) | 3 = 0x0004_0003
41
+ let data = [0x00, 0x01, 0x00, 0x02];
42
+ assert_eq!(fletcher32(&data), 0x0004_0003);
43
+ }
44
+
45
+ #[test]
46
+ fn fletcher32_roundtrip_with_filter() {
47
+ // Build a payload + checksum and verify the filter can strip it
48
+ let payload = vec![0x00u8, 0x80, 0x3F, 0x80, 0x00, 0x00, 0x40, 0x00];
49
+ let ck = fletcher32(&payload);
50
+ let mut data = payload.clone();
51
+ data.extend_from_slice(&ck.to_le_bytes());
52
+ // The filter stores checksum in little-endian.
53
+ let stripped = crate::filters::fletcher32::verify_and_strip(&data).unwrap();
54
+ assert_eq!(stripped, payload);
55
+ }
56
+ }