@nirs4all/formats-wasm 0.2.12 → 0.2.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/THIRD_PARTY_NOTICES.md +49 -0
  2. package/nirs4all_formats_wasm.d.ts +4 -4
  3. package/nirs4all_formats_wasm.js +170 -149
  4. package/nirs4all_formats_wasm_bg.wasm +0 -0
  5. package/package.json +4 -2
  6. package/reader-backports/hdf5-reader/Cargo.toml +119 -0
  7. package/reader-backports/hdf5-reader/LICENSE-APACHE +164 -0
  8. package/reader-backports/hdf5-reader/LICENSE-MIT +21 -0
  9. package/reader-backports/hdf5-reader/PROVENANCE.json +80 -0
  10. package/reader-backports/hdf5-reader/README.md +308 -0
  11. package/reader-backports/hdf5-reader/UPSTREAM-Cargo.toml +44 -0
  12. package/reader-backports/hdf5-reader/UPSTREAM-VCS.json +6 -0
  13. package/reader-backports/hdf5-reader/src/attribute_api.rs +847 -0
  14. package/reader-backports/hdf5-reader/src/btree_v1.rs +726 -0
  15. package/reader-backports/hdf5-reader/src/btree_v2.rs +2068 -0
  16. package/reader-backports/hdf5-reader/src/cache.rs +354 -0
  17. package/reader-backports/hdf5-reader/src/checksum.rs +56 -0
  18. package/reader-backports/hdf5-reader/src/chunk_index.rs +423 -0
  19. package/reader-backports/hdf5-reader/src/dataset.rs +4733 -0
  20. package/reader-backports/hdf5-reader/src/datatype_api.rs +497 -0
  21. package/reader-backports/hdf5-reader/src/error.rs +167 -0
  22. package/reader-backports/hdf5-reader/src/extensible_array.rs +1886 -0
  23. package/reader-backports/hdf5-reader/src/filters/deflate.rs +93 -0
  24. package/reader-backports/hdf5-reader/src/filters/fletcher32.rs +52 -0
  25. package/reader-backports/hdf5-reader/src/filters/lz4.rs +178 -0
  26. package/reader-backports/hdf5-reader/src/filters/mod.rs +326 -0
  27. package/reader-backports/hdf5-reader/src/filters/nbit.rs +586 -0
  28. package/reader-backports/hdf5-reader/src/filters/scaleoffset.rs +827 -0
  29. package/reader-backports/hdf5-reader/src/filters/shuffle.rs +99 -0
  30. package/reader-backports/hdf5-reader/src/fixed_array.rs +763 -0
  31. package/reader-backports/hdf5-reader/src/fractal_heap.rs +2029 -0
  32. package/reader-backports/hdf5-reader/src/global_heap.rs +284 -0
  33. package/reader-backports/hdf5-reader/src/group.rs +795 -0
  34. package/reader-backports/hdf5-reader/src/io.rs +413 -0
  35. package/reader-backports/hdf5-reader/src/lib.rs +916 -0
  36. package/reader-backports/hdf5-reader/src/local_heap.rs +261 -0
  37. package/reader-backports/hdf5-reader/src/messages/attribute.rs +304 -0
  38. package/reader-backports/hdf5-reader/src/messages/attribute_info.rs +96 -0
  39. package/reader-backports/hdf5-reader/src/messages/btree_k.rs +36 -0
  40. package/reader-backports/hdf5-reader/src/messages/continuation.rs +46 -0
  41. package/reader-backports/hdf5-reader/src/messages/dataspace.rs +259 -0
  42. package/reader-backports/hdf5-reader/src/messages/datatype.rs +938 -0
  43. package/reader-backports/hdf5-reader/src/messages/external_files.rs +50 -0
  44. package/reader-backports/hdf5-reader/src/messages/fill_value.rs +233 -0
  45. package/reader-backports/hdf5-reader/src/messages/filter_pipeline.rs +147 -0
  46. package/reader-backports/hdf5-reader/src/messages/group_info.rs +103 -0
  47. package/reader-backports/hdf5-reader/src/messages/layout.rs +476 -0
  48. package/reader-backports/hdf5-reader/src/messages/link.rs +170 -0
  49. package/reader-backports/hdf5-reader/src/messages/link_info.rs +127 -0
  50. package/reader-backports/hdf5-reader/src/messages/mod.rs +374 -0
  51. package/reader-backports/hdf5-reader/src/messages/modification_time.rs +83 -0
  52. package/reader-backports/hdf5-reader/src/messages/shared.rs +164 -0
  53. package/reader-backports/hdf5-reader/src/messages/shared_table.rs +67 -0
  54. package/reader-backports/hdf5-reader/src/messages/symbol_table_msg.rs +51 -0
  55. package/reader-backports/hdf5-reader/src/object_header.rs +1749 -0
  56. package/reader-backports/hdf5-reader/src/reference.rs +90 -0
  57. package/reader-backports/hdf5-reader/src/shared_message_table.rs +425 -0
  58. package/reader-backports/hdf5-reader/src/storage.rs +538 -0
  59. package/reader-backports/hdf5-reader/src/superblock.rs +285 -0
  60. package/reader-backports/hdf5-reader/src/symbol_table.rs +277 -0
  61. package/reader-backports/netcdf-reader/Cargo.toml +141 -0
  62. package/reader-backports/netcdf-reader/LICENSE-APACHE +164 -0
  63. package/reader-backports/netcdf-reader/LICENSE-MIT +21 -0
  64. package/reader-backports/netcdf-reader/PROVENANCE.json +55 -0
  65. package/reader-backports/netcdf-reader/README.md +308 -0
  66. package/reader-backports/netcdf-reader/UPSTREAM-Cargo.toml +48 -0
  67. package/reader-backports/netcdf-reader/UPSTREAM-VCS.json +6 -0
  68. package/reader-backports/netcdf-reader/src/cf/axes.rs +419 -0
  69. package/reader-backports/netcdf-reader/src/cf/bounds.rs +108 -0
  70. package/reader-backports/netcdf-reader/src/cf/crs.rs +269 -0
  71. package/reader-backports/netcdf-reader/src/cf/mod.rs +30 -0
  72. package/reader-backports/netcdf-reader/src/cf/time.rs +1122 -0
  73. package/reader-backports/netcdf-reader/src/classic/data.rs +1465 -0
  74. package/reader-backports/netcdf-reader/src/classic/header.rs +1097 -0
  75. package/reader-backports/netcdf-reader/src/classic/mod.rs +240 -0
  76. package/reader-backports/netcdf-reader/src/classic/storage.rs +211 -0
  77. package/reader-backports/netcdf-reader/src/classic/types.rs +93 -0
  78. package/reader-backports/netcdf-reader/src/classic/variable.rs +1162 -0
  79. package/reader-backports/netcdf-reader/src/error.rs +59 -0
  80. package/reader-backports/netcdf-reader/src/lib.rs +1531 -0
  81. package/reader-backports/netcdf-reader/src/masked.rs +173 -0
  82. package/reader-backports/netcdf-reader/src/nc4/attributes.rs +203 -0
  83. package/reader-backports/netcdf-reader/src/nc4/dimensions.rs +139 -0
  84. package/reader-backports/netcdf-reader/src/nc4/groups.rs +308 -0
  85. package/reader-backports/netcdf-reader/src/nc4/mod.rs +572 -0
  86. package/reader-backports/netcdf-reader/src/nc4/types.rs +309 -0
  87. package/reader-backports/netcdf-reader/src/nc4/variables.rs +420 -0
  88. package/reader-backports/netcdf-reader/src/types.rs +18 -0
  89. package/reader-backports/netcdf-reader/src/unpack.rs +113 -0
  90. package/reader-backports/netcdf-reader/src/user_defined.rs +569 -0
@@ -0,0 +1,916 @@
1
+ pub mod checksum;
2
+ pub mod error;
3
+ pub mod io;
4
+
5
+ // Level 0 — File Metadata
6
+ pub mod superblock;
7
+
8
+ // Level 1 — File Infrastructure
9
+ pub mod btree_v1;
10
+ pub mod btree_v2;
11
+ pub mod chunk_index;
12
+ pub mod extensible_array;
13
+ pub mod fixed_array;
14
+ pub mod fractal_heap;
15
+ pub mod global_heap;
16
+ pub mod local_heap;
17
+ pub mod shared_message_table;
18
+ pub mod symbol_table;
19
+
20
+ // Level 2 — Data Objects
21
+ pub mod messages;
22
+ pub mod object_header;
23
+
24
+ // High-level API
25
+ pub mod attribute_api;
26
+ pub mod dataset;
27
+ pub mod datatype_api;
28
+ pub mod group;
29
+ pub mod reference;
30
+ pub mod storage;
31
+
32
+ // Filters
33
+ pub mod filters;
34
+
35
+ // Utilities
36
+ pub mod cache;
37
+
38
+ use std::collections::HashMap;
39
+ #[cfg(unix)]
40
+ use std::ffi::{CString, OsStr};
41
+ use std::fs::File;
42
+ use std::io::ErrorKind;
43
+ #[cfg(unix)]
44
+ use std::os::fd::{AsRawFd, FromRawFd};
45
+ #[cfg(unix)]
46
+ use std::os::unix::ffi::OsStrExt;
47
+ use std::path::{Component, Path, PathBuf};
48
+ use std::sync::{Arc, OnceLock};
49
+
50
+ use memmap2::Mmap;
51
+ // parking_lot::Mutex used via fully-qualified paths in HeaderCache and constructors.
52
+
53
+ use cache::ChunkCache;
54
+ use error::{Error, Result};
55
+ use group::Group;
56
+ use messages::HdfMessage;
57
+ use object_header::ObjectHeader;
58
+ use shared_message_table::SharedMessageTableRef;
59
+ use storage::DynStorage;
60
+ use superblock::Superblock;
61
+
62
+ // Re-exports
63
+ pub use attribute_api::Attribute;
64
+ pub use cache::ChunkCacheStats;
65
+ use dataset::DatasetTemplate;
66
+ pub use dataset::{Dataset, DatasetChunk, DatasetChunkIterator, SliceInfo, SliceInfoElem};
67
+ pub use datatype_api::{
68
+ dtype_element_size, CompoundField, EnumMember, H5Type, ReferenceType, StringEncoding,
69
+ StringPadding, StringSize, VarLenKind,
70
+ };
71
+ pub use error::ByteOrder;
72
+ pub use filters::FilterRegistry;
73
+ pub use messages::datatype::Datatype;
74
+ pub use storage::{
75
+ BlockCacheStats, BlockCacheStorage, BytesStorage, FileStorage, MmapStorage,
76
+ RangeRequestStorage, Storage, StorageBuffer,
77
+ };
78
+
79
+ /// Configuration options for opening an HDF5 file.
80
+ pub struct OpenOptions {
81
+ /// Maximum bytes for the chunk cache. Default: 64 MiB.
82
+ pub chunk_cache_bytes: usize,
83
+ /// Maximum number of chunk cache slots. Default: 521.
84
+ pub chunk_cache_slots: usize,
85
+ /// Custom filter registry. If `None`, the default built-in filters are used.
86
+ pub filter_registry: Option<FilterRegistry>,
87
+ /// Resolver for HDF5 external raw data files. If `None`, external raw data
88
+ /// files are not resolved.
89
+ pub external_file_resolver: Option<Arc<dyn ExternalFileResolver>>,
90
+ /// Optional resolver for HDF5 external links.
91
+ pub external_link_resolver: Option<Arc<dyn ExternalLinkResolver>>,
92
+ }
93
+
94
+ impl Default for OpenOptions {
95
+ fn default() -> Self {
96
+ OpenOptions {
97
+ chunk_cache_bytes: 64 * 1024 * 1024,
98
+ chunk_cache_slots: 521,
99
+ filter_registry: None,
100
+ external_file_resolver: None,
101
+ external_link_resolver: None,
102
+ }
103
+ }
104
+ }
105
+
106
+ /// Resolves file names from HDF5 External Data Files messages to storage.
107
+ ///
108
+ /// Implementations are responsible for their own path security policy. The
109
+ /// built-in [`FilesystemExternalFileResolver`] confines normal paths to a base
110
+ /// directory and, on Unix, opens paths through an anchored directory handle
111
+ /// without following symlinks. On non-Unix platforms it falls back to
112
+ /// canonicalize-then-open, so attacker-writable resolver roots are out of
113
+ /// scope there.
114
+ pub trait ExternalFileResolver: Send + Sync {
115
+ fn resolve_external_file(&self, filename: &str) -> Result<Option<DynStorage>>;
116
+ }
117
+
118
+ /// Resolves HDF5 external links to another opened file.
119
+ ///
120
+ /// Implementations are responsible for their own path security policy. The
121
+ /// built-in [`FilesystemExternalLinkResolver`] confines normal paths to a base
122
+ /// directory and, on Unix, opens paths through an anchored directory handle
123
+ /// without following symlinks. On non-Unix platforms it falls back to
124
+ /// canonicalize-then-open, so attacker-writable resolver roots are out of
125
+ /// scope there.
126
+ pub trait ExternalLinkResolver: Send + Sync {
127
+ fn resolve_external_link(&self, filename: &str) -> Result<Option<Hdf5File>>;
128
+ }
129
+
130
+ fn normalize_resolver_path(filename: &str, description: &str) -> Result<PathBuf> {
131
+ let path = Path::new(filename);
132
+ if path.as_os_str().is_empty() {
133
+ return Err(Error::InvalidData(format!("{description} path is empty")));
134
+ }
135
+
136
+ if path.is_absolute() {
137
+ return Err(Error::InvalidData(format!(
138
+ "{description} path must be relative: {filename}"
139
+ )));
140
+ }
141
+
142
+ let mut normalized = PathBuf::new();
143
+ for component in path.components() {
144
+ match component {
145
+ Component::Normal(name) => normalized.push(name),
146
+ Component::CurDir => {}
147
+ Component::Prefix(_) | Component::RootDir | Component::ParentDir => {
148
+ return Err(Error::InvalidData(format!(
149
+ "{description} path must stay within the resolver base directory: {filename}"
150
+ )));
151
+ }
152
+ }
153
+ }
154
+
155
+ if normalized.as_os_str().is_empty() {
156
+ return Err(Error::InvalidData(format!("{description} path is empty")));
157
+ }
158
+
159
+ Ok(normalized)
160
+ }
161
+
162
+ #[cfg(not(unix))]
163
+ fn open_external_file_within_base(
164
+ base_dir: &Path,
165
+ relative_path: &Path,
166
+ description: &str,
167
+ filename: &str,
168
+ ) -> Result<Option<File>> {
169
+ let base = match base_dir.canonicalize() {
170
+ Ok(path) => path,
171
+ Err(err) if err.kind() == ErrorKind::NotFound => return Ok(None),
172
+ Err(err) => return Err(err.into()),
173
+ };
174
+ let candidate = base.join(relative_path);
175
+ let resolved = match candidate.canonicalize() {
176
+ Ok(path) => path,
177
+ Err(err) if err.kind() == ErrorKind::NotFound => return Ok(None),
178
+ Err(err) => return Err(err.into()),
179
+ };
180
+
181
+ if !resolved.starts_with(&base) {
182
+ return Err(Error::InvalidData(format!(
183
+ "{description} path escapes the resolver base directory: {filename}"
184
+ )));
185
+ }
186
+
187
+ Ok(Some(File::open(resolved)?))
188
+ }
189
+
190
+ #[cfg(unix)]
191
+ fn open_external_file_within_base(
192
+ base_dir: &Path,
193
+ relative_path: &Path,
194
+ description: &str,
195
+ filename: &str,
196
+ ) -> Result<Option<File>> {
197
+ let mut dir = match open_unix_path(
198
+ base_dir,
199
+ libc::O_RDONLY | libc::O_CLOEXEC | libc::O_NOFOLLOW,
200
+ ) {
201
+ Ok(file) => file,
202
+ Err(err) if path_lookup_is_missing(&err) => return Ok(None),
203
+ Err(err) if path_lookup_is_symlink(&err) => {
204
+ return Err(Error::InvalidData(format!(
205
+ "{description} resolver base directory must not be a symlink"
206
+ )));
207
+ }
208
+ Err(err) => return Err(err.into()),
209
+ };
210
+ if !dir.metadata()?.is_dir() {
211
+ return Ok(None);
212
+ }
213
+
214
+ let parts: Vec<&OsStr> = relative_path
215
+ .components()
216
+ .filter_map(|component| match component {
217
+ Component::Normal(name) => Some(name),
218
+ _ => None,
219
+ })
220
+ .collect();
221
+
222
+ let Some((leaf, parents)) = parts.split_last() else {
223
+ return Err(Error::InvalidData(format!("{description} path is empty")));
224
+ };
225
+
226
+ for parent in parents {
227
+ dir = match open_unix_child(
228
+ &dir,
229
+ parent,
230
+ libc::O_RDONLY | libc::O_CLOEXEC | libc::O_NOFOLLOW,
231
+ ) {
232
+ Ok(file) => file,
233
+ Err(err) if path_lookup_is_missing(&err) => return Ok(None),
234
+ Err(err) if path_lookup_is_symlink(&err) => {
235
+ return Err(symlink_resolver_error(description, filename));
236
+ }
237
+ Err(err) => return Err(err.into()),
238
+ };
239
+ if !dir.metadata()?.is_dir() {
240
+ return Ok(None);
241
+ }
242
+ }
243
+
244
+ let file = match open_unix_child(
245
+ &dir,
246
+ leaf,
247
+ libc::O_RDONLY | libc::O_CLOEXEC | libc::O_NOFOLLOW,
248
+ ) {
249
+ Ok(file) => file,
250
+ Err(err) if path_lookup_is_missing(&err) => return Ok(None),
251
+ Err(err) if path_lookup_is_symlink(&err) => {
252
+ return Err(symlink_resolver_error(description, filename));
253
+ }
254
+ Err(err) => return Err(err.into()),
255
+ };
256
+
257
+ if file.metadata()?.is_dir() {
258
+ return Err(Error::InvalidData(format!(
259
+ "{description} path resolves to a directory: {filename}"
260
+ )));
261
+ }
262
+
263
+ Ok(Some(file))
264
+ }
265
+
266
+ #[cfg(unix)]
267
+ fn open_unix_path(path: &Path, flags: libc::c_int) -> std::io::Result<File> {
268
+ let path = CString::new(path.as_os_str().as_bytes()).map_err(|_| {
269
+ std::io::Error::new(
270
+ std::io::ErrorKind::InvalidInput,
271
+ "filesystem path contains an interior NUL byte",
272
+ )
273
+ })?;
274
+ // SAFETY: `path` is NUL-terminated, `flags` is passed through unchanged,
275
+ // and the returned descriptor is checked before ownership is assumed.
276
+ let fd = unsafe { libc::open(path.as_ptr(), flags) };
277
+ file_from_unix_fd(fd)
278
+ }
279
+
280
+ #[cfg(unix)]
281
+ fn open_unix_child(dir: &File, name: &OsStr, flags: libc::c_int) -> std::io::Result<File> {
282
+ let name = CString::new(name.as_bytes()).map_err(|_| {
283
+ std::io::Error::new(
284
+ std::io::ErrorKind::InvalidInput,
285
+ "filesystem path contains an interior NUL byte",
286
+ )
287
+ })?;
288
+ // SAFETY: `dir` owns a live descriptor, `name` is NUL-terminated, and the
289
+ // returned descriptor is checked before ownership is assumed.
290
+ let fd = unsafe { libc::openat(dir.as_raw_fd(), name.as_ptr(), flags) };
291
+ file_from_unix_fd(fd)
292
+ }
293
+
294
+ #[cfg(unix)]
295
+ fn file_from_unix_fd(fd: libc::c_int) -> std::io::Result<File> {
296
+ if fd < 0 {
297
+ Err(std::io::Error::last_os_error())
298
+ } else {
299
+ // SAFETY: a successful `open`/`openat` returned an owned descriptor,
300
+ // and this transfers that ownership exactly once to `File`.
301
+ Ok(unsafe { File::from_raw_fd(fd) })
302
+ }
303
+ }
304
+
305
+ #[cfg(unix)]
306
+ fn path_lookup_is_missing(err: &std::io::Error) -> bool {
307
+ err.kind() == ErrorKind::NotFound
308
+ || matches!(err.raw_os_error(), Some(code) if code == libc::ENOTDIR)
309
+ }
310
+
311
+ #[cfg(unix)]
312
+ fn path_lookup_is_symlink(err: &std::io::Error) -> bool {
313
+ matches!(err.raw_os_error(), Some(code) if code == libc::ELOOP)
314
+ }
315
+
316
+ #[cfg(unix)]
317
+ fn symlink_resolver_error(description: &str, filename: &str) -> Error {
318
+ Error::InvalidData(format!(
319
+ "{description} path escapes the resolver base directory or uses a symlink: {filename}"
320
+ ))
321
+ }
322
+
323
+ /// Filesystem resolver for external raw data files.
324
+ ///
325
+ /// The resolver rejects absolute paths and `..` components. On Unix, it opens
326
+ /// paths relative to `base_dir` using `openat` and `O_NOFOLLOW`, so symlinks
327
+ /// are rejected instead of being followed. On non-Unix platforms,
328
+ /// attacker-writable resolver roots are out of scope.
329
+ #[derive(Debug, Clone)]
330
+ pub struct FilesystemExternalFileResolver {
331
+ base_dir: PathBuf,
332
+ }
333
+
334
+ impl FilesystemExternalFileResolver {
335
+ pub fn new(base_dir: impl Into<PathBuf>) -> Self {
336
+ Self {
337
+ base_dir: base_dir.into(),
338
+ }
339
+ }
340
+
341
+ fn relative_path_for(&self, filename: &str) -> Result<PathBuf> {
342
+ normalize_resolver_path(filename, "external raw data file")
343
+ }
344
+ }
345
+
346
+ impl ExternalFileResolver for FilesystemExternalFileResolver {
347
+ fn resolve_external_file(&self, filename: &str) -> Result<Option<DynStorage>> {
348
+ let relative_path = self.relative_path_for(filename)?;
349
+ let Some(file) = open_external_file_within_base(
350
+ &self.base_dir,
351
+ &relative_path,
352
+ "external raw data file",
353
+ filename,
354
+ )?
355
+ else {
356
+ return Ok(None);
357
+ };
358
+ Ok(Some(Arc::new(FileStorage::from_file(file)?)))
359
+ }
360
+ }
361
+
362
+ /// Filesystem resolver for external links. Linked files are cached after the
363
+ /// first successful open.
364
+ ///
365
+ /// The resolver rejects absolute paths and `..` components. On Unix, it opens
366
+ /// paths relative to `base_dir` using `openat` and `O_NOFOLLOW`, so symlinks
367
+ /// are rejected instead of being followed. On non-Unix platforms,
368
+ /// attacker-writable resolver roots are out of scope.
369
+ pub struct FilesystemExternalLinkResolver {
370
+ base_dir: PathBuf,
371
+ cache: parking_lot::Mutex<HashMap<PathBuf, Hdf5File>>,
372
+ }
373
+
374
+ impl FilesystemExternalLinkResolver {
375
+ pub fn new(base_dir: impl Into<PathBuf>) -> Self {
376
+ Self {
377
+ base_dir: base_dir.into(),
378
+ cache: parking_lot::Mutex::new(HashMap::new()),
379
+ }
380
+ }
381
+
382
+ fn relative_path_for(&self, filename: &str) -> Result<PathBuf> {
383
+ normalize_resolver_path(filename, "external link")
384
+ }
385
+ }
386
+
387
+ impl ExternalLinkResolver for FilesystemExternalLinkResolver {
388
+ fn resolve_external_link(&self, filename: &str) -> Result<Option<Hdf5File>> {
389
+ let relative_path = self.relative_path_for(filename)?;
390
+
391
+ if let Some(file) = self.cache.lock().get(&relative_path).cloned() {
392
+ return Ok(Some(file));
393
+ }
394
+
395
+ let Some(opened) = open_external_file_within_base(
396
+ &self.base_dir,
397
+ &relative_path,
398
+ "external link",
399
+ filename,
400
+ )?
401
+ else {
402
+ return Ok(None);
403
+ };
404
+
405
+ let file = Hdf5File::from_storage(Arc::new(FileStorage::from_file(opened)?))?;
406
+ self.cache.lock().insert(relative_path, file.clone());
407
+ Ok(Some(file))
408
+ }
409
+ }
410
+
411
+ /// Cache for parsed object headers, keyed by file address.
412
+ pub type HeaderCache = Arc<parking_lot::Mutex<HashMap<u64, Arc<ObjectHeader>>>>;
413
+
414
+ /// An opened HDF5 file.
415
+ ///
416
+ /// This is the main entry point for reading HDF5 files. Storage is random-
417
+ /// access and range-based, so metadata and data reads do not require an eager
418
+ /// whole-file mapping.
419
+ #[derive(Clone)]
420
+ pub struct Hdf5File {
421
+ context: Arc<FileContext>,
422
+ }
423
+
424
+ pub(crate) struct FileContext {
425
+ pub(crate) storage: DynStorage,
426
+ pub(crate) superblock: Superblock,
427
+ pub(crate) chunk_cache: Arc<ChunkCache>,
428
+ pub(crate) header_cache: HeaderCache,
429
+ pub(crate) dataset_path_cache: Arc<parking_lot::Mutex<HashMap<String, Arc<DatasetTemplate>>>>,
430
+ pub(crate) filter_registry: Arc<FilterRegistry>,
431
+ pub(crate) external_file_resolver: Option<Arc<dyn ExternalFileResolver>>,
432
+ pub(crate) external_link_resolver: Option<Arc<dyn ExternalLinkResolver>>,
433
+ pub(crate) external_file_cache: parking_lot::Mutex<HashMap<String, DynStorage>>,
434
+ sohm_table: OnceLock<std::result::Result<Option<SharedMessageTableRef>, String>>,
435
+ full_file_cache: OnceLock<StorageBuffer>,
436
+ }
437
+
438
+ impl FileContext {
439
+ pub(crate) fn read_range(&self, offset: u64, len: usize) -> Result<StorageBuffer> {
440
+ self.storage.read_range(offset, len)
441
+ }
442
+
443
+ pub(crate) fn full_file_data(&self) -> Result<StorageBuffer> {
444
+ if let Some(buffer) = self.full_file_cache.get() {
445
+ return Ok(buffer.clone());
446
+ }
447
+
448
+ let len = usize::try_from(self.storage.len()).map_err(|_| {
449
+ Error::InvalidData("file size exceeds platform usize capacity".to_string())
450
+ })?;
451
+ let buffer = self.storage.read_range(0, len)?;
452
+ let _ = self.full_file_cache.set(buffer);
453
+ Ok(self
454
+ .full_file_cache
455
+ .get()
456
+ .expect("full-file buffer must exist after successful initialization")
457
+ .clone())
458
+ }
459
+
460
+ pub(crate) fn get_or_parse_header(&self, addr: u64) -> Result<Arc<ObjectHeader>> {
461
+ {
462
+ let cache = self.header_cache.lock();
463
+ if let Some(hdr) = cache.get(&addr) {
464
+ return Ok(Arc::clone(hdr));
465
+ }
466
+ }
467
+
468
+ let mut hdr = ObjectHeader::parse_at_storage(
469
+ self.storage.as_ref(),
470
+ addr,
471
+ self.superblock.offset_size,
472
+ self.superblock.length_size,
473
+ )?;
474
+ hdr.resolve_shared_messages_storage_with_sohm(
475
+ self.storage.as_ref(),
476
+ self.superblock.offset_size,
477
+ self.superblock.length_size,
478
+ |heap_id, message_type| self.resolve_sohm_message(heap_id, message_type),
479
+ )?;
480
+ let arc = Arc::new(hdr);
481
+ let mut cache = self.header_cache.lock();
482
+ cache.insert(addr, Arc::clone(&arc));
483
+ Ok(arc)
484
+ }
485
+
486
+ fn resolve_sohm_message(
487
+ &self,
488
+ heap_id: &[u8],
489
+ message_type: u16,
490
+ ) -> Result<Option<HdfMessage>> {
491
+ let Some(table) = self.sohm_table()? else {
492
+ return Ok(None);
493
+ };
494
+ table.resolve_heap_message(
495
+ heap_id,
496
+ message_type,
497
+ self.storage.as_ref(),
498
+ self.superblock.offset_size,
499
+ self.superblock.length_size,
500
+ Some(self.filter_registry.as_ref()),
501
+ )
502
+ }
503
+
504
+ fn sohm_table(&self) -> Result<Option<SharedMessageTableRef>> {
505
+ let cached = self
506
+ .sohm_table
507
+ .get_or_init(|| self.load_sohm_table().map_err(|err| err.to_string()));
508
+ match cached {
509
+ Ok(table) => Ok(table.clone()),
510
+ Err(message) => Err(Error::InvalidData(format!(
511
+ "failed to load SOHM table: {message}"
512
+ ))),
513
+ }
514
+ }
515
+
516
+ fn load_sohm_table(&self) -> Result<Option<SharedMessageTableRef>> {
517
+ let Some(extension_address) = self.superblock.extension_address else {
518
+ return Ok(None);
519
+ };
520
+ let extension = ObjectHeader::parse_at_storage(
521
+ self.storage.as_ref(),
522
+ extension_address,
523
+ self.superblock.offset_size,
524
+ self.superblock.length_size,
525
+ )?;
526
+
527
+ let shared_table = extension.messages.iter().find_map(|message| match message {
528
+ HdfMessage::SharedTable(table) => Some(table),
529
+ _ => None,
530
+ });
531
+ let Some(shared_table) = shared_table else {
532
+ return Ok(None);
533
+ };
534
+
535
+ let table = crate::shared_message_table::SharedMessageTable::parse_at_storage(
536
+ self.storage.as_ref(),
537
+ shared_table.table_address,
538
+ shared_table.num_indices,
539
+ self.superblock.offset_size,
540
+ )?;
541
+ Ok(Some(Arc::new(table)))
542
+ }
543
+
544
+ pub(crate) fn resolve_external_file(&self, filename: &str) -> Result<Option<DynStorage>> {
545
+ if let Some(storage) = self.external_file_cache.lock().get(filename).cloned() {
546
+ return Ok(Some(storage));
547
+ }
548
+
549
+ let Some(resolver) = self.external_file_resolver.as_ref() else {
550
+ return Ok(None);
551
+ };
552
+ let Some(storage) = resolver.resolve_external_file(filename)? else {
553
+ return Ok(None);
554
+ };
555
+ self.external_file_cache
556
+ .lock()
557
+ .insert(filename.to_string(), storage.clone());
558
+ Ok(Some(storage))
559
+ }
560
+ }
561
+
562
+ impl Hdf5File {
563
+ fn from_storage_impl(storage: DynStorage, options: OpenOptions) -> Result<Self> {
564
+ let superblock = Superblock::parse_from_storage(storage.as_ref())?;
565
+ let cache = Arc::new(ChunkCache::new(
566
+ options.chunk_cache_bytes,
567
+ options.chunk_cache_slots,
568
+ ));
569
+ let registry = options.filter_registry.unwrap_or_default();
570
+ let external_file_resolver = options.external_file_resolver;
571
+ let external_link_resolver = options.external_link_resolver;
572
+
573
+ Ok(Hdf5File {
574
+ context: Arc::new(FileContext {
575
+ storage,
576
+ superblock,
577
+ chunk_cache: cache,
578
+ header_cache: Arc::new(parking_lot::Mutex::new(HashMap::new())),
579
+ dataset_path_cache: Arc::new(parking_lot::Mutex::new(HashMap::new())),
580
+ filter_registry: Arc::new(registry),
581
+ external_file_resolver,
582
+ external_link_resolver,
583
+ external_file_cache: parking_lot::Mutex::new(HashMap::new()),
584
+ sohm_table: OnceLock::new(),
585
+ full_file_cache: OnceLock::new(),
586
+ }),
587
+ })
588
+ }
589
+
590
+ /// Open an HDF5 file with default options.
591
+ pub fn open(path: impl AsRef<Path>) -> Result<Self> {
592
+ Self::open_with_options(path, OpenOptions::default())
593
+ }
594
+
595
+ /// Open an HDF5 file with custom options.
596
+ pub fn open_with_options(path: impl AsRef<Path>, options: OpenOptions) -> Result<Self> {
597
+ let path = path.as_ref();
598
+ Self::from_storage_with_options(Arc::new(FileStorage::open(path)?), options)
599
+ }
600
+
601
+ /// Open an HDF5 file from an in-memory byte slice.
602
+ ///
603
+ /// The data is copied into an owned buffer.
604
+ pub fn from_bytes(data: &[u8]) -> Result<Self> {
605
+ Self::from_bytes_with_options(data, OpenOptions::default())
606
+ }
607
+
608
+ /// Open an HDF5 file from an in-memory byte slice with custom options.
609
+ ///
610
+ /// The data is copied into an owned buffer.
611
+ pub fn from_bytes_with_options(data: &[u8], options: OpenOptions) -> Result<Self> {
612
+ Self::from_vec_with_options(data.to_vec(), options)
613
+ }
614
+
615
+ /// Open an HDF5 file from an owned byte vector without copying.
616
+ pub fn from_vec(data: Vec<u8>) -> Result<Self> {
617
+ Self::from_vec_with_options(data, OpenOptions::default())
618
+ }
619
+
620
+ /// Open an HDF5 file from an owned byte vector with custom options.
621
+ pub fn from_vec_with_options(data: Vec<u8>, options: OpenOptions) -> Result<Self> {
622
+ Self::from_storage_with_options(Arc::new(BytesStorage::new(data)), options)
623
+ }
624
+
625
+ /// Open an HDF5 file from an existing memory map with custom options.
626
+ ///
627
+ /// This avoids remapping when the caller already owns a read-only mapping.
628
+ pub fn from_mmap_with_options(mmap: Mmap, options: OpenOptions) -> Result<Self> {
629
+ Self::from_storage_with_options(Arc::new(MmapStorage::new(mmap)), options)
630
+ }
631
+
632
+ /// Open an HDF5 file from a custom random-access storage backend.
633
+ pub fn from_storage(storage: DynStorage) -> Result<Self> {
634
+ Self::from_storage_with_options(storage, OpenOptions::default())
635
+ }
636
+
637
+ /// Open an HDF5 file from a custom random-access storage backend.
638
+ pub fn from_storage_with_options(storage: DynStorage, options: OpenOptions) -> Result<Self> {
639
+ Self::from_storage_impl(storage, options)
640
+ }
641
+
642
+ /// Get the parsed superblock.
643
+ pub fn superblock(&self) -> &Superblock {
644
+ &self.context.superblock
645
+ }
646
+
647
+ /// Access the underlying random-access storage backend.
648
+ pub fn storage(&self) -> &dyn Storage {
649
+ self.context.storage.as_ref()
650
+ }
651
+
652
+ /// Return current chunk-cache statistics for this file.
653
+ pub fn chunk_cache_stats(&self) -> ChunkCacheStats {
654
+ self.context.chunk_cache.stats()
655
+ }
656
+
657
+ /// Look up or parse an object header at the given address.
658
+ ///
659
+ /// Uses the internal cache to avoid re-parsing the same header.
660
+ pub fn get_or_parse_header(&self, addr: u64) -> Result<Arc<ObjectHeader>> {
661
+ self.context.get_or_parse_header(addr)
662
+ }
663
+
664
+ /// Get the root group of the file.
665
+ pub fn root_group(&self) -> Result<Group> {
666
+ let addr = self.context.superblock.root_object_header_address()?;
667
+
668
+ Ok(Group::new(
669
+ self.context.clone(),
670
+ addr,
671
+ "/".to_string(),
672
+ addr, // root_address = self
673
+ ))
674
+ }
675
+
676
+ /// Convenience: get a dataset at a path like "/group1/dataset".
677
+ pub fn dataset(&self, path: &str) -> Result<Dataset> {
678
+ let parts: Vec<&str> = path
679
+ .trim_start_matches('/')
680
+ .split('/')
681
+ .filter(|s| !s.is_empty())
682
+ .collect();
683
+ let normalized_path = format!("/{}", parts.join("/"));
684
+
685
+ if parts.is_empty() {
686
+ return Err(Error::DatasetNotFound(path.to_string()).with_context(path));
687
+ }
688
+
689
+ if let Some(template) = self
690
+ .context
691
+ .dataset_path_cache
692
+ .lock()
693
+ .get(&normalized_path)
694
+ .cloned()
695
+ {
696
+ return Ok(Dataset::from_template(self.context.clone(), template));
697
+ }
698
+
699
+ let mut group = self.root_group()?;
700
+ for &part in &parts[..parts.len() - 1] {
701
+ group = group.group(part).map_err(|e| e.with_context(path))?;
702
+ }
703
+
704
+ let dataset = group
705
+ .dataset(parts[parts.len() - 1])
706
+ .map_err(|e| e.with_context(path))?;
707
+ if Arc::ptr_eq(&dataset.context, &self.context) {
708
+ self.context
709
+ .dataset_path_cache
710
+ .lock()
711
+ .insert(normalized_path, dataset.template());
712
+ }
713
+ Ok(dataset)
714
+ }
715
+
716
+ /// Convenience: get a group at a path like "/group1/subgroup".
717
+ pub fn group(&self, path: &str) -> Result<Group> {
718
+ let parts: Vec<&str> = path
719
+ .trim_start_matches('/')
720
+ .split('/')
721
+ .filter(|s| !s.is_empty())
722
+ .collect();
723
+
724
+ let mut group = self.root_group()?;
725
+ for &part in &parts {
726
+ group = group.group(part)?;
727
+ }
728
+
729
+ Ok(group)
730
+ }
731
+ }
732
+
733
+ #[cfg(test)]
734
+ mod tests {
735
+ use super::*;
736
+
737
+ #[test]
738
+ fn open_options_default() {
739
+ let opts = OpenOptions::default();
740
+ assert_eq!(opts.chunk_cache_bytes, 64 * 1024 * 1024);
741
+ assert_eq!(opts.chunk_cache_slots, 521);
742
+ assert!(opts.external_file_resolver.is_none());
743
+ }
744
+
745
+ #[test]
746
+ fn invalid_file() {
747
+ let data = b"this is not an HDF5 file";
748
+ let result = Hdf5File::from_bytes(data);
749
+ assert!(result.is_err());
750
+ }
751
+
752
+ #[test]
753
+ fn btree_v2_chunked_fixture_uses_btree_v2_index() {
754
+ use crate::messages::layout::{ChunkIndexing, DataLayout};
755
+
756
+ let path = Path::new(env!("CARGO_MANIFEST_DIR"))
757
+ .parent()
758
+ .unwrap()
759
+ .join("testdata/hdf5/btree_v2_chunked.h5");
760
+ if !path.exists() {
761
+ eprintln!("SKIPPED: fixture btree_v2_chunked.h5 not found");
762
+ return;
763
+ }
764
+
765
+ let file = Hdf5File::open(path).unwrap();
766
+ let dataset = file.dataset("/data").unwrap();
767
+ assert!(matches!(
768
+ dataset.layout,
769
+ DataLayout::Chunked {
770
+ chunk_indexing: Some(ChunkIndexing::BTreeV2),
771
+ ..
772
+ }
773
+ ));
774
+ }
775
+
776
+ #[test]
777
+ fn filesystem_external_file_resolver_reads_relative_file() {
778
+ let dir = tempfile::tempdir().unwrap();
779
+ let path = dir.path().join("raw.bin");
780
+ std::fs::write(&path, b"abcdef").unwrap();
781
+
782
+ let resolver = FilesystemExternalFileResolver::new(dir.path());
783
+ let storage = resolver
784
+ .resolve_external_file("raw.bin")
785
+ .unwrap()
786
+ .expect("raw file should resolve");
787
+ let bytes = storage.read_range(2, 3).unwrap();
788
+ assert_eq!(bytes.as_ref(), b"cde");
789
+ }
790
+
791
+ #[test]
792
+ fn filesystem_external_file_resolver_rejects_absolute_path() {
793
+ let dir = tempfile::tempdir().unwrap();
794
+ let path = dir.path().join("raw.bin");
795
+ std::fs::write(&path, b"abcdef").unwrap();
796
+
797
+ let resolver = FilesystemExternalFileResolver::new(dir.path());
798
+ let err = match resolver.resolve_external_file(path.to_str().unwrap()) {
799
+ Ok(_) => panic!("absolute external file path should be rejected"),
800
+ Err(err) => err,
801
+ };
802
+ assert!(err.to_string().contains("must be relative"));
803
+ }
804
+
805
+ #[test]
806
+ fn filesystem_external_file_resolver_rejects_parent_component() {
807
+ let dir = tempfile::tempdir().unwrap();
808
+ let resolver = FilesystemExternalFileResolver::new(dir.path());
809
+
810
+ let err = match resolver.resolve_external_file("../raw.bin") {
811
+ Ok(_) => panic!("parent external file path should be rejected"),
812
+ Err(err) => err,
813
+ };
814
+ assert!(err.to_string().contains("resolver base directory"));
815
+ }
816
+
817
+ #[cfg(unix)]
818
+ #[test]
819
+ fn filesystem_external_file_resolver_rejects_symlink_escape() {
820
+ use std::os::unix::fs::symlink;
821
+
822
+ let dir = tempfile::tempdir().unwrap();
823
+ let outside = tempfile::tempdir().unwrap();
824
+ let outside_path = outside.path().join("raw.bin");
825
+ std::fs::write(&outside_path, b"abcdef").unwrap();
826
+ symlink(&outside_path, dir.path().join("raw.bin")).unwrap();
827
+
828
+ let resolver = FilesystemExternalFileResolver::new(dir.path());
829
+ let err = match resolver.resolve_external_file("raw.bin") {
830
+ Ok(_) => panic!("symlink escape should be rejected"),
831
+ Err(err) => err,
832
+ };
833
+ assert!(err.to_string().contains("escapes"));
834
+ }
835
+
836
+ #[cfg(unix)]
837
+ #[test]
838
+ fn filesystem_external_file_resolver_rejects_symlink_inside_base() {
839
+ use std::os::unix::fs::symlink;
840
+
841
+ let dir = tempfile::tempdir().unwrap();
842
+ std::fs::write(dir.path().join("raw.bin"), b"abcdef").unwrap();
843
+ symlink("raw.bin", dir.path().join("link.bin")).unwrap();
844
+
845
+ let resolver = FilesystemExternalFileResolver::new(dir.path());
846
+ let err = match resolver.resolve_external_file("link.bin") {
847
+ Ok(_) => panic!("symlinks should be rejected even when they point inside the base"),
848
+ Err(err) => err,
849
+ };
850
+ assert!(err.to_string().contains("symlink"));
851
+ }
852
+
853
+ #[cfg(unix)]
854
+ #[test]
855
+ fn filesystem_external_file_resolver_rejects_symlink_directory_component() {
856
+ use std::os::unix::fs::symlink;
857
+
858
+ let dir = tempfile::tempdir().unwrap();
859
+ let real_dir = dir.path().join("real");
860
+ std::fs::create_dir(&real_dir).unwrap();
861
+ std::fs::write(real_dir.join("raw.bin"), b"abcdef").unwrap();
862
+ symlink("real", dir.path().join("linkdir")).unwrap();
863
+
864
+ let resolver = FilesystemExternalFileResolver::new(dir.path());
865
+ let err = match resolver.resolve_external_file("linkdir/raw.bin") {
866
+ Ok(_) => panic!("symlinked directory components should be rejected"),
867
+ Err(err) => err,
868
+ };
869
+ assert!(err.to_string().contains("symlink"));
870
+ }
871
+
872
+ #[test]
873
+ fn filesystem_external_link_resolver_rejects_absolute_path() {
874
+ let dir = tempfile::tempdir().unwrap();
875
+ let path = dir.path().join("linked.h5");
876
+ std::fs::write(&path, b"not really hdf5").unwrap();
877
+
878
+ let resolver = FilesystemExternalLinkResolver::new(dir.path());
879
+ let err = match resolver.resolve_external_link(path.to_str().unwrap()) {
880
+ Ok(_) => panic!("absolute external link path should be rejected"),
881
+ Err(err) => err,
882
+ };
883
+ assert!(err.to_string().contains("must be relative"));
884
+ }
885
+
886
+ #[test]
887
+ fn filesystem_external_link_resolver_rejects_parent_component() {
888
+ let dir = tempfile::tempdir().unwrap();
889
+ let resolver = FilesystemExternalLinkResolver::new(dir.path());
890
+
891
+ let err = match resolver.resolve_external_link("../linked.h5") {
892
+ Ok(_) => panic!("parent external link path should be rejected"),
893
+ Err(err) => err,
894
+ };
895
+ assert!(err.to_string().contains("resolver base directory"));
896
+ }
897
+
898
+ #[cfg(unix)]
899
+ #[test]
900
+ fn filesystem_external_link_resolver_rejects_symlink_escape() {
901
+ use std::os::unix::fs::symlink;
902
+
903
+ let dir = tempfile::tempdir().unwrap();
904
+ let outside = tempfile::tempdir().unwrap();
905
+ let outside_path = outside.path().join("linked.h5");
906
+ std::fs::write(&outside_path, b"not really hdf5").unwrap();
907
+ symlink(&outside_path, dir.path().join("linked.h5")).unwrap();
908
+
909
+ let resolver = FilesystemExternalLinkResolver::new(dir.path());
910
+ let err = match resolver.resolve_external_link("linked.h5") {
911
+ Ok(_) => panic!("symlink escape should be rejected"),
912
+ Err(err) => err,
913
+ };
914
+ assert!(err.to_string().contains("escapes"));
915
+ }
916
+ }