@nirs4all/formats-wasm 0.2.12 → 0.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD_PARTY_NOTICES.md +49 -0
- package/nirs4all_formats_wasm_bg.wasm +0 -0
- package/package.json +4 -2
- package/reader-backports/hdf5-reader/Cargo.toml +119 -0
- package/reader-backports/hdf5-reader/LICENSE-APACHE +164 -0
- package/reader-backports/hdf5-reader/LICENSE-MIT +21 -0
- package/reader-backports/hdf5-reader/PROVENANCE.json +80 -0
- package/reader-backports/hdf5-reader/README.md +308 -0
- package/reader-backports/hdf5-reader/UPSTREAM-Cargo.toml +44 -0
- package/reader-backports/hdf5-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/hdf5-reader/src/attribute_api.rs +847 -0
- package/reader-backports/hdf5-reader/src/btree_v1.rs +726 -0
- package/reader-backports/hdf5-reader/src/btree_v2.rs +2068 -0
- package/reader-backports/hdf5-reader/src/cache.rs +354 -0
- package/reader-backports/hdf5-reader/src/checksum.rs +56 -0
- package/reader-backports/hdf5-reader/src/chunk_index.rs +423 -0
- package/reader-backports/hdf5-reader/src/dataset.rs +4733 -0
- package/reader-backports/hdf5-reader/src/datatype_api.rs +497 -0
- package/reader-backports/hdf5-reader/src/error.rs +167 -0
- package/reader-backports/hdf5-reader/src/extensible_array.rs +1886 -0
- package/reader-backports/hdf5-reader/src/filters/deflate.rs +93 -0
- package/reader-backports/hdf5-reader/src/filters/fletcher32.rs +52 -0
- package/reader-backports/hdf5-reader/src/filters/lz4.rs +178 -0
- package/reader-backports/hdf5-reader/src/filters/mod.rs +326 -0
- package/reader-backports/hdf5-reader/src/filters/nbit.rs +586 -0
- package/reader-backports/hdf5-reader/src/filters/scaleoffset.rs +827 -0
- package/reader-backports/hdf5-reader/src/filters/shuffle.rs +99 -0
- package/reader-backports/hdf5-reader/src/fixed_array.rs +763 -0
- package/reader-backports/hdf5-reader/src/fractal_heap.rs +2029 -0
- package/reader-backports/hdf5-reader/src/global_heap.rs +284 -0
- package/reader-backports/hdf5-reader/src/group.rs +795 -0
- package/reader-backports/hdf5-reader/src/io.rs +413 -0
- package/reader-backports/hdf5-reader/src/lib.rs +916 -0
- package/reader-backports/hdf5-reader/src/local_heap.rs +261 -0
- package/reader-backports/hdf5-reader/src/messages/attribute.rs +304 -0
- package/reader-backports/hdf5-reader/src/messages/attribute_info.rs +96 -0
- package/reader-backports/hdf5-reader/src/messages/btree_k.rs +36 -0
- package/reader-backports/hdf5-reader/src/messages/continuation.rs +46 -0
- package/reader-backports/hdf5-reader/src/messages/dataspace.rs +259 -0
- package/reader-backports/hdf5-reader/src/messages/datatype.rs +938 -0
- package/reader-backports/hdf5-reader/src/messages/external_files.rs +50 -0
- package/reader-backports/hdf5-reader/src/messages/fill_value.rs +233 -0
- package/reader-backports/hdf5-reader/src/messages/filter_pipeline.rs +147 -0
- package/reader-backports/hdf5-reader/src/messages/group_info.rs +103 -0
- package/reader-backports/hdf5-reader/src/messages/layout.rs +476 -0
- package/reader-backports/hdf5-reader/src/messages/link.rs +170 -0
- package/reader-backports/hdf5-reader/src/messages/link_info.rs +127 -0
- package/reader-backports/hdf5-reader/src/messages/mod.rs +374 -0
- package/reader-backports/hdf5-reader/src/messages/modification_time.rs +83 -0
- package/reader-backports/hdf5-reader/src/messages/shared.rs +164 -0
- package/reader-backports/hdf5-reader/src/messages/shared_table.rs +67 -0
- package/reader-backports/hdf5-reader/src/messages/symbol_table_msg.rs +51 -0
- package/reader-backports/hdf5-reader/src/object_header.rs +1749 -0
- package/reader-backports/hdf5-reader/src/reference.rs +90 -0
- package/reader-backports/hdf5-reader/src/shared_message_table.rs +425 -0
- package/reader-backports/hdf5-reader/src/storage.rs +538 -0
- package/reader-backports/hdf5-reader/src/superblock.rs +285 -0
- package/reader-backports/hdf5-reader/src/symbol_table.rs +277 -0
- package/reader-backports/netcdf-reader/Cargo.toml +141 -0
- package/reader-backports/netcdf-reader/LICENSE-APACHE +164 -0
- package/reader-backports/netcdf-reader/LICENSE-MIT +21 -0
- package/reader-backports/netcdf-reader/PROVENANCE.json +55 -0
- package/reader-backports/netcdf-reader/README.md +308 -0
- package/reader-backports/netcdf-reader/UPSTREAM-Cargo.toml +48 -0
- package/reader-backports/netcdf-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/netcdf-reader/src/cf/axes.rs +419 -0
- package/reader-backports/netcdf-reader/src/cf/bounds.rs +108 -0
- package/reader-backports/netcdf-reader/src/cf/crs.rs +269 -0
- package/reader-backports/netcdf-reader/src/cf/mod.rs +30 -0
- package/reader-backports/netcdf-reader/src/cf/time.rs +1122 -0
- package/reader-backports/netcdf-reader/src/classic/data.rs +1465 -0
- package/reader-backports/netcdf-reader/src/classic/header.rs +1097 -0
- package/reader-backports/netcdf-reader/src/classic/mod.rs +240 -0
- package/reader-backports/netcdf-reader/src/classic/storage.rs +211 -0
- package/reader-backports/netcdf-reader/src/classic/types.rs +93 -0
- package/reader-backports/netcdf-reader/src/classic/variable.rs +1162 -0
- package/reader-backports/netcdf-reader/src/error.rs +59 -0
- package/reader-backports/netcdf-reader/src/lib.rs +1531 -0
- package/reader-backports/netcdf-reader/src/masked.rs +173 -0
- package/reader-backports/netcdf-reader/src/nc4/attributes.rs +203 -0
- package/reader-backports/netcdf-reader/src/nc4/dimensions.rs +139 -0
- package/reader-backports/netcdf-reader/src/nc4/groups.rs +308 -0
- package/reader-backports/netcdf-reader/src/nc4/mod.rs +572 -0
- package/reader-backports/netcdf-reader/src/nc4/types.rs +309 -0
- package/reader-backports/netcdf-reader/src/nc4/variables.rs +420 -0
- package/reader-backports/netcdf-reader/src/types.rs +18 -0
- package/reader-backports/netcdf-reader/src/unpack.rs +113 -0
- package/reader-backports/netcdf-reader/src/user_defined.rs +569 -0
|
@@ -0,0 +1,795 @@
|
|
|
1
|
+
use std::sync::Arc;
|
|
2
|
+
|
|
3
|
+
use crate::attribute_api::{
|
|
4
|
+
collect_attribute_messages_storage, decoded_vlen_strings_storage, resolve_vlen_bytes_storage,
|
|
5
|
+
Attribute,
|
|
6
|
+
};
|
|
7
|
+
use crate::btree_v1;
|
|
8
|
+
use crate::btree_v2;
|
|
9
|
+
use crate::checksum::jenkins_lookup3;
|
|
10
|
+
use crate::dataset::Dataset;
|
|
11
|
+
use crate::error::{Error, Result};
|
|
12
|
+
use crate::fractal_heap::{FractalHeap, FractalHeapDirectBlockCache};
|
|
13
|
+
use crate::io::Cursor;
|
|
14
|
+
use crate::local_heap::LocalHeap;
|
|
15
|
+
use crate::messages::datatype::VarLenKind;
|
|
16
|
+
use crate::messages::link::{self, LinkMessage, LinkTarget};
|
|
17
|
+
use crate::messages::link_info::LinkInfoMessage;
|
|
18
|
+
use crate::messages::symbol_table_msg::SymbolTableMessage;
|
|
19
|
+
use crate::messages::HdfMessage;
|
|
20
|
+
use crate::storage::Storage;
|
|
21
|
+
use crate::FileContext;
|
|
22
|
+
|
|
23
|
+
/// A group within an HDF5 file.
|
|
24
|
+
#[derive(Clone)]
|
|
25
|
+
pub struct Group {
|
|
26
|
+
context: Arc<FileContext>,
|
|
27
|
+
pub(crate) name: String,
|
|
28
|
+
pub(crate) address: u64,
|
|
29
|
+
/// Address of the root group's object header, used for resolving soft links.
|
|
30
|
+
pub(crate) root_address: u64,
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
#[derive(Clone)]
|
|
34
|
+
struct ChildEntry {
|
|
35
|
+
name: String,
|
|
36
|
+
location: ObjectLocation,
|
|
37
|
+
/// The link's stored creation order, when its group tracks it.
|
|
38
|
+
creation_order: Option<u64>,
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
#[derive(Clone)]
|
|
42
|
+
struct ObjectLocation {
|
|
43
|
+
context: Arc<FileContext>,
|
|
44
|
+
address: u64,
|
|
45
|
+
root_address: u64,
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
49
|
+
enum ChildObjectKind {
|
|
50
|
+
Group,
|
|
51
|
+
Dataset,
|
|
52
|
+
Other,
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
impl Group {
|
|
56
|
+
/// Create a group from a known object header address.
|
|
57
|
+
pub(crate) fn new(
|
|
58
|
+
context: Arc<FileContext>,
|
|
59
|
+
address: u64,
|
|
60
|
+
name: String,
|
|
61
|
+
root_address: u64,
|
|
62
|
+
) -> Self {
|
|
63
|
+
Group {
|
|
64
|
+
context,
|
|
65
|
+
name,
|
|
66
|
+
address,
|
|
67
|
+
root_address,
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/// Group name.
|
|
72
|
+
pub fn name(&self) -> &str {
|
|
73
|
+
&self.name
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/// Object header address of this group within the file.
|
|
77
|
+
pub fn address(&self) -> u64 {
|
|
78
|
+
self.address
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/// Materialize the full file backing this group.
|
|
82
|
+
pub fn file_data(&self) -> Result<crate::storage::StorageBuffer> {
|
|
83
|
+
self.context.full_file_data()
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/// Access the underlying random-access storage backend.
|
|
87
|
+
pub fn storage(&self) -> &dyn Storage {
|
|
88
|
+
self.context.storage.as_ref()
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/// Size of file offsets in bytes.
|
|
92
|
+
pub fn offset_size(&self) -> u8 {
|
|
93
|
+
self.context.superblock.offset_size
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/// Size of file lengths in bytes.
|
|
97
|
+
pub fn length_size(&self) -> u8 {
|
|
98
|
+
self.context.superblock.length_size
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/// Parse (or retrieve from cache) the object header at the given address.
|
|
102
|
+
fn cached_header(&self, addr: u64) -> Result<Arc<crate::object_header::ObjectHeader>> {
|
|
103
|
+
self.context.get_or_parse_header(addr)
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
fn local_location(&self, address: u64) -> ObjectLocation {
|
|
107
|
+
ObjectLocation {
|
|
108
|
+
context: self.context.clone(),
|
|
109
|
+
address,
|
|
110
|
+
root_address: self.root_address,
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/// List all child groups.
|
|
115
|
+
pub fn groups(&self) -> Result<Vec<Group>> {
|
|
116
|
+
let (groups, _) = self.resolve_member_objects()?;
|
|
117
|
+
Ok(groups)
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/// List all child members, partitioned into groups and datasets.
|
|
121
|
+
pub fn members(&self) -> Result<(Vec<Group>, Vec<Dataset>)> {
|
|
122
|
+
self.resolve_member_objects()
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
fn resolve_member_objects(&self) -> Result<(Vec<Group>, Vec<Dataset>)> {
|
|
126
|
+
let children = self.resolve_children()?;
|
|
127
|
+
let mut groups = Vec::new();
|
|
128
|
+
let mut datasets = Vec::new();
|
|
129
|
+
for child in &children {
|
|
130
|
+
match self.child_object_kind(child)? {
|
|
131
|
+
ChildObjectKind::Group => {
|
|
132
|
+
groups.push(Group::new(
|
|
133
|
+
child.location.context.clone(),
|
|
134
|
+
child.location.address,
|
|
135
|
+
child.name.clone(),
|
|
136
|
+
child.location.root_address,
|
|
137
|
+
));
|
|
138
|
+
}
|
|
139
|
+
ChildObjectKind::Dataset => {
|
|
140
|
+
if let Some(dataset) = self.try_open_child_dataset(child)? {
|
|
141
|
+
datasets.push(dataset);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
ChildObjectKind::Other => {}
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
Ok((groups, datasets))
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/// Get a child group by name.
|
|
151
|
+
pub fn group(&self, name: &str) -> Result<Group> {
|
|
152
|
+
if let Some(child) = self.resolve_child(name)? {
|
|
153
|
+
return match self.child_object_kind(&child)? {
|
|
154
|
+
ChildObjectKind::Group => Ok(Group::new(
|
|
155
|
+
child.location.context.clone(),
|
|
156
|
+
child.location.address,
|
|
157
|
+
child.name.clone(),
|
|
158
|
+
child.location.root_address,
|
|
159
|
+
)),
|
|
160
|
+
ChildObjectKind::Dataset => Err(Error::GroupNotFound(format!(
|
|
161
|
+
"'{}' is a dataset, not a group",
|
|
162
|
+
name
|
|
163
|
+
))),
|
|
164
|
+
ChildObjectKind::Other => {
|
|
165
|
+
Err(Error::GroupNotFound(format!("'{}' is not a group", name)))
|
|
166
|
+
}
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
Err(Error::GroupNotFound(name.to_string()))
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/// List all child datasets.
|
|
173
|
+
pub fn datasets(&self) -> Result<Vec<Dataset>> {
|
|
174
|
+
let (_, datasets) = self.resolve_member_objects()?;
|
|
175
|
+
Ok(datasets)
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/// Get a child dataset by name.
|
|
179
|
+
pub fn dataset(&self, name: &str) -> Result<Dataset> {
|
|
180
|
+
if let Some(child) = self.resolve_child(name)? {
|
|
181
|
+
if let Some(dataset) = self.try_open_child_dataset(&child)? {
|
|
182
|
+
return Ok(dataset);
|
|
183
|
+
}
|
|
184
|
+
return Err(Error::DatasetNotFound(name.to_string()));
|
|
185
|
+
}
|
|
186
|
+
Err(Error::DatasetNotFound(name.to_string()))
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/// List attributes on this group.
|
|
190
|
+
pub fn attributes(&self) -> Result<Vec<Attribute>> {
|
|
191
|
+
// `cached_header` already resolves shared messages, including those
|
|
192
|
+
// stored in the shared object header message (SOHM) table, so the
|
|
193
|
+
// group attribute path supports the same sharing the dataset path does.
|
|
194
|
+
let header = self.cached_header(self.address)?;
|
|
195
|
+
Ok(collect_attribute_messages_storage(
|
|
196
|
+
&header,
|
|
197
|
+
self.context.storage.as_ref(),
|
|
198
|
+
self.offset_size(),
|
|
199
|
+
self.length_size(),
|
|
200
|
+
Some(self.context.filter_registry.as_ref()),
|
|
201
|
+
)?
|
|
202
|
+
.into_iter()
|
|
203
|
+
.map(|attr| {
|
|
204
|
+
let element_count = attr.dataspace.num_elements().ok();
|
|
205
|
+
let decoded_strings = element_count.and_then(|element_count| {
|
|
206
|
+
decoded_vlen_strings_storage(
|
|
207
|
+
&attr.datatype,
|
|
208
|
+
&attr.raw_data,
|
|
209
|
+
self.context.storage.as_ref(),
|
|
210
|
+
self.offset_size(),
|
|
211
|
+
self.length_size(),
|
|
212
|
+
element_count,
|
|
213
|
+
)
|
|
214
|
+
});
|
|
215
|
+
let raw_data = match &attr.datatype {
|
|
216
|
+
crate::messages::datatype::Datatype::VarLen {
|
|
217
|
+
base,
|
|
218
|
+
kind: VarLenKind::String,
|
|
219
|
+
..
|
|
220
|
+
} if base.is_vlen_string_base()
|
|
221
|
+
&& matches!(attr.dataspace.num_elements(), Ok(1)) =>
|
|
222
|
+
{
|
|
223
|
+
resolve_vlen_bytes_storage(
|
|
224
|
+
&attr.raw_data,
|
|
225
|
+
self.context.storage.as_ref(),
|
|
226
|
+
self.offset_size(),
|
|
227
|
+
self.length_size(),
|
|
228
|
+
)
|
|
229
|
+
.unwrap_or_else(|| attr.raw_data.clone())
|
|
230
|
+
}
|
|
231
|
+
_ => attr.raw_data.clone(),
|
|
232
|
+
};
|
|
233
|
+
Attribute {
|
|
234
|
+
name: attr.name,
|
|
235
|
+
datatype: attr.datatype,
|
|
236
|
+
shape: match attr.dataspace.dataspace_type {
|
|
237
|
+
crate::messages::dataspace::DataspaceType::Scalar => vec![],
|
|
238
|
+
crate::messages::dataspace::DataspaceType::Null => vec![0],
|
|
239
|
+
crate::messages::dataspace::DataspaceType::Simple => attr.dataspace.dims,
|
|
240
|
+
},
|
|
241
|
+
raw_data,
|
|
242
|
+
decoded_strings,
|
|
243
|
+
}
|
|
244
|
+
})
|
|
245
|
+
.collect())
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/// Find an attribute by name.
|
|
249
|
+
pub fn attribute(&self, name: &str) -> Result<Attribute> {
|
|
250
|
+
let attrs = self.attributes()?;
|
|
251
|
+
attrs
|
|
252
|
+
.into_iter()
|
|
253
|
+
.find(|a| a.name == name)
|
|
254
|
+
.ok_or_else(|| Error::AttributeNotFound(name.to_string()))
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/// Resolve children from the object header.
|
|
258
|
+
/// Handles both old-style (symbol table) and new-style (link messages) groups.
|
|
259
|
+
fn resolve_children(&self) -> Result<Vec<ChildEntry>> {
|
|
260
|
+
self.resolve_children_with_link_depth(0)
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
fn resolve_child(&self, name: &str) -> Result<Option<ChildEntry>> {
|
|
264
|
+
self.resolve_child_with_link_depth(name, 0)
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
fn resolve_child_with_link_depth(
|
|
268
|
+
&self,
|
|
269
|
+
name: &str,
|
|
270
|
+
link_depth: u32,
|
|
271
|
+
) -> Result<Option<ChildEntry>> {
|
|
272
|
+
let header = self.cached_header(self.address)?;
|
|
273
|
+
|
|
274
|
+
let mut link_info: Option<LinkInfoMessage> = None;
|
|
275
|
+
let mut matching_compact_link: Option<LinkMessage> = None;
|
|
276
|
+
|
|
277
|
+
for msg in &header.messages {
|
|
278
|
+
match msg {
|
|
279
|
+
HdfMessage::SymbolTable(st) => {
|
|
280
|
+
return Ok(self
|
|
281
|
+
.resolve_old_style_group_storage(st)?
|
|
282
|
+
.into_iter()
|
|
283
|
+
.find(|child| child.name == name));
|
|
284
|
+
}
|
|
285
|
+
HdfMessage::Link(link) if link.name == name => {
|
|
286
|
+
matching_compact_link = Some(link.clone());
|
|
287
|
+
}
|
|
288
|
+
HdfMessage::LinkInfo(li) => {
|
|
289
|
+
link_info = Some(li.clone());
|
|
290
|
+
}
|
|
291
|
+
_ => {}
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
if let Some(link) = matching_compact_link {
|
|
296
|
+
if let Some(child) = self.resolve_link_message_target(&link, link_depth)? {
|
|
297
|
+
return Ok(Some(child));
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
if let Some(ref li) = link_info {
|
|
302
|
+
if !Cursor::is_undefined_offset(li.fractal_heap_address, self.offset_size()) {
|
|
303
|
+
return self.resolve_dense_link_storage(li, name, link_depth);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
Ok(None)
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/// Resolve children with a soft-link depth counter to prevent cycles.
|
|
311
|
+
fn resolve_children_with_link_depth(&self, link_depth: u32) -> Result<Vec<ChildEntry>> {
|
|
312
|
+
let header = self.cached_header(self.address)?;
|
|
313
|
+
|
|
314
|
+
let mut children = Vec::new();
|
|
315
|
+
|
|
316
|
+
// Check for old-style groups (symbol table message)
|
|
317
|
+
let mut found_symbol_table = false;
|
|
318
|
+
// Check for new-style groups (link messages)
|
|
319
|
+
let mut link_info: Option<LinkInfoMessage> = None;
|
|
320
|
+
let mut links: Vec<LinkMessage> = Vec::new();
|
|
321
|
+
|
|
322
|
+
for msg in &header.messages {
|
|
323
|
+
match msg {
|
|
324
|
+
HdfMessage::SymbolTable(st) => {
|
|
325
|
+
found_symbol_table = true;
|
|
326
|
+
children = self.resolve_old_style_group_storage(st)?;
|
|
327
|
+
}
|
|
328
|
+
HdfMessage::Link(link) => {
|
|
329
|
+
links.push(link.clone());
|
|
330
|
+
}
|
|
331
|
+
HdfMessage::LinkInfo(li) => {
|
|
332
|
+
link_info = Some(li.clone());
|
|
333
|
+
}
|
|
334
|
+
_ => {}
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
if !found_symbol_table {
|
|
339
|
+
// New-style group: use compact links from header messages
|
|
340
|
+
self.resolve_link_targets(&links, link_depth, &mut children)?;
|
|
341
|
+
|
|
342
|
+
// Dense-link storage can coexist with compact links, so merge both.
|
|
343
|
+
if let Some(ref li) = link_info {
|
|
344
|
+
if !Cursor::is_undefined_offset(li.fractal_heap_address, self.offset_size()) {
|
|
345
|
+
for child in self.resolve_dense_links_storage(li, link_depth)? {
|
|
346
|
+
let is_duplicate = children.iter().any(|existing| {
|
|
347
|
+
existing.name == child.name
|
|
348
|
+
&& existing.location.address == child.location.address
|
|
349
|
+
&& Arc::ptr_eq(&existing.location.context, &child.location.context)
|
|
350
|
+
});
|
|
351
|
+
if !is_duplicate {
|
|
352
|
+
children.push(child);
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
// Storage order is not creation order: libhdf5 places compact link
|
|
359
|
+
// messages wherever the object header has room, and dense links
|
|
360
|
+
// sit in name-hash order. When the group tracks creation order
|
|
361
|
+
// (every link carries one, as in all NetCDF-4 groups), list the
|
|
362
|
+
// children in it, as `H5Literate(H5_INDEX_CRT_ORDER)` and netcdf-c
|
|
363
|
+
// do. A stable sort keeps any equal values in storage order.
|
|
364
|
+
if children.iter().all(|child| child.creation_order.is_some()) {
|
|
365
|
+
children.sort_by_key(|child| child.creation_order);
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
Ok(children)
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/// Resolve link targets (hard and soft), appending to `children`.
|
|
373
|
+
fn resolve_link_targets(
|
|
374
|
+
&self,
|
|
375
|
+
links: &[LinkMessage],
|
|
376
|
+
link_depth: u32,
|
|
377
|
+
children: &mut Vec<ChildEntry>,
|
|
378
|
+
) -> Result<()> {
|
|
379
|
+
for link in links {
|
|
380
|
+
if let Some(child) = self.resolve_link_message_target(link, link_depth)? {
|
|
381
|
+
children.push(child);
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
Ok(())
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
fn resolve_link_message_target(
|
|
388
|
+
&self,
|
|
389
|
+
link: &LinkMessage,
|
|
390
|
+
link_depth: u32,
|
|
391
|
+
) -> Result<Option<ChildEntry>> {
|
|
392
|
+
match &link.target {
|
|
393
|
+
LinkTarget::Hard { address } => Ok(Some(ChildEntry {
|
|
394
|
+
name: link.name.clone(),
|
|
395
|
+
location: self.local_location(*address),
|
|
396
|
+
creation_order: link.creation_order,
|
|
397
|
+
})),
|
|
398
|
+
LinkTarget::Soft { path } => Ok(self
|
|
399
|
+
.resolve_soft_link_depth(path, link_depth)
|
|
400
|
+
.ok()
|
|
401
|
+
.map(|location| ChildEntry {
|
|
402
|
+
name: link.name.clone(),
|
|
403
|
+
location,
|
|
404
|
+
creation_order: link.creation_order,
|
|
405
|
+
})),
|
|
406
|
+
LinkTarget::External { filename, path } => Ok(self
|
|
407
|
+
.resolve_external_link_depth(filename, path, link_depth)?
|
|
408
|
+
.map(|location| ChildEntry {
|
|
409
|
+
name: link.name.clone(),
|
|
410
|
+
location,
|
|
411
|
+
creation_order: link.creation_order,
|
|
412
|
+
})),
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
fn resolve_old_style_group_storage(&self, st: &SymbolTableMessage) -> Result<Vec<ChildEntry>> {
|
|
417
|
+
let heap = LocalHeap::parse_at_storage(
|
|
418
|
+
self.context.storage.as_ref(),
|
|
419
|
+
st.heap_address,
|
|
420
|
+
self.offset_size(),
|
|
421
|
+
self.length_size(),
|
|
422
|
+
)?;
|
|
423
|
+
|
|
424
|
+
let leaves = btree_v1::collect_btree_v1_leaves_storage(
|
|
425
|
+
self.context.storage.as_ref(),
|
|
426
|
+
st.btree_address,
|
|
427
|
+
self.offset_size(),
|
|
428
|
+
self.length_size(),
|
|
429
|
+
None,
|
|
430
|
+
&[],
|
|
431
|
+
None,
|
|
432
|
+
)?;
|
|
433
|
+
|
|
434
|
+
let mut children = Vec::new();
|
|
435
|
+
for (_key, snod_address) in &leaves {
|
|
436
|
+
let header_len = 8 + 2 * usize::from(self.offset_size());
|
|
437
|
+
let prefix = self.context.read_range(*snod_address, header_len)?;
|
|
438
|
+
let mut prefix_cursor = Cursor::new(prefix.as_ref());
|
|
439
|
+
let sig = prefix_cursor.read_bytes(4)?;
|
|
440
|
+
if sig != *b"SNOD" {
|
|
441
|
+
return Err(Error::InvalidData(format!(
|
|
442
|
+
"expected SNOD signature at offset {:#x}",
|
|
443
|
+
snod_address
|
|
444
|
+
)));
|
|
445
|
+
}
|
|
446
|
+
let version = prefix_cursor.read_u8()?;
|
|
447
|
+
if version != 1 {
|
|
448
|
+
return Err(Error::InvalidData(format!(
|
|
449
|
+
"unsupported symbol table node version {}",
|
|
450
|
+
version
|
|
451
|
+
)));
|
|
452
|
+
}
|
|
453
|
+
prefix_cursor.skip(1)?;
|
|
454
|
+
let num_symbols = prefix_cursor.read_u16_le()?;
|
|
455
|
+
let node_len =
|
|
456
|
+
8 + usize::from(num_symbols) * (2 * usize::from(self.offset_size()) + 4 + 4 + 16);
|
|
457
|
+
let bytes = self.context.read_range(*snod_address, node_len)?;
|
|
458
|
+
let mut cursor = Cursor::new(bytes.as_ref());
|
|
459
|
+
let snod = crate::symbol_table::SymbolTableNode::parse(
|
|
460
|
+
&mut cursor,
|
|
461
|
+
self.offset_size(),
|
|
462
|
+
self.length_size(),
|
|
463
|
+
)?;
|
|
464
|
+
|
|
465
|
+
for entry in &snod.entries {
|
|
466
|
+
let name =
|
|
467
|
+
heap.get_string_storage(entry.link_name_offset, self.context.storage.as_ref())?;
|
|
468
|
+
children.push(ChildEntry {
|
|
469
|
+
name,
|
|
470
|
+
location: self.local_location(entry.object_header_address),
|
|
471
|
+
creation_order: None,
|
|
472
|
+
});
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
Ok(children)
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
fn resolve_dense_links_storage(
|
|
480
|
+
&self,
|
|
481
|
+
link_info: &LinkInfoMessage,
|
|
482
|
+
link_depth: u32,
|
|
483
|
+
) -> Result<Vec<ChildEntry>> {
|
|
484
|
+
let heap = FractalHeap::parse_at_storage(
|
|
485
|
+
self.context.storage.as_ref(),
|
|
486
|
+
link_info.fractal_heap_address,
|
|
487
|
+
self.offset_size(),
|
|
488
|
+
self.length_size(),
|
|
489
|
+
)?;
|
|
490
|
+
|
|
491
|
+
let btree_header = btree_v2::BTreeV2Header::parse_at_storage(
|
|
492
|
+
self.context.storage.as_ref(),
|
|
493
|
+
link_info.btree_name_index_address,
|
|
494
|
+
self.offset_size(),
|
|
495
|
+
self.length_size(),
|
|
496
|
+
)?;
|
|
497
|
+
|
|
498
|
+
let records = btree_v2::collect_btree_v2_records_storage(
|
|
499
|
+
self.context.storage.as_ref(),
|
|
500
|
+
&btree_header,
|
|
501
|
+
self.offset_size(),
|
|
502
|
+
self.length_size(),
|
|
503
|
+
None,
|
|
504
|
+
&[],
|
|
505
|
+
None,
|
|
506
|
+
)?;
|
|
507
|
+
|
|
508
|
+
let mut children = Vec::new();
|
|
509
|
+
let mut direct_block_cache = FractalHeapDirectBlockCache::default();
|
|
510
|
+
for record in &records {
|
|
511
|
+
let heap_id = match record {
|
|
512
|
+
btree_v2::BTreeV2Record::LinkNameHash { heap_id, .. }
|
|
513
|
+
| btree_v2::BTreeV2Record::CreationOrder { heap_id, .. } => heap_id,
|
|
514
|
+
_ => continue,
|
|
515
|
+
};
|
|
516
|
+
|
|
517
|
+
let managed_bytes = heap.get_object_storage_cached_with_registry(
|
|
518
|
+
heap_id,
|
|
519
|
+
self.context.storage.as_ref(),
|
|
520
|
+
self.offset_size(),
|
|
521
|
+
self.length_size(),
|
|
522
|
+
&mut direct_block_cache,
|
|
523
|
+
Some(self.context.filter_registry.as_ref()),
|
|
524
|
+
)?;
|
|
525
|
+
|
|
526
|
+
let mut link_cursor = Cursor::new(&managed_bytes);
|
|
527
|
+
let link_msg = link::parse(
|
|
528
|
+
&mut link_cursor,
|
|
529
|
+
self.offset_size(),
|
|
530
|
+
self.length_size(),
|
|
531
|
+
managed_bytes.len(),
|
|
532
|
+
)?;
|
|
533
|
+
|
|
534
|
+
match &link_msg.target {
|
|
535
|
+
LinkTarget::Hard { address } => {
|
|
536
|
+
children.push(ChildEntry {
|
|
537
|
+
name: link_msg.name.clone(),
|
|
538
|
+
location: self.local_location(*address),
|
|
539
|
+
creation_order: link_msg.creation_order,
|
|
540
|
+
});
|
|
541
|
+
}
|
|
542
|
+
LinkTarget::Soft { path } => {
|
|
543
|
+
if let Ok(location) = self.resolve_soft_link_depth(path, link_depth) {
|
|
544
|
+
children.push(ChildEntry {
|
|
545
|
+
name: link_msg.name.clone(),
|
|
546
|
+
location,
|
|
547
|
+
creation_order: link_msg.creation_order,
|
|
548
|
+
});
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
LinkTarget::External { filename, path } => {
|
|
552
|
+
if let Some(location) =
|
|
553
|
+
self.resolve_external_link_depth(filename, path, link_depth)?
|
|
554
|
+
{
|
|
555
|
+
children.push(ChildEntry {
|
|
556
|
+
name: link_msg.name.clone(),
|
|
557
|
+
location,
|
|
558
|
+
creation_order: link_msg.creation_order,
|
|
559
|
+
});
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
Ok(children)
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
fn resolve_dense_link_storage(
|
|
569
|
+
&self,
|
|
570
|
+
link_info: &LinkInfoMessage,
|
|
571
|
+
name: &str,
|
|
572
|
+
link_depth: u32,
|
|
573
|
+
) -> Result<Option<ChildEntry>> {
|
|
574
|
+
let heap = FractalHeap::parse_at_storage(
|
|
575
|
+
self.context.storage.as_ref(),
|
|
576
|
+
link_info.fractal_heap_address,
|
|
577
|
+
self.offset_size(),
|
|
578
|
+
self.length_size(),
|
|
579
|
+
)?;
|
|
580
|
+
|
|
581
|
+
let btree_header = btree_v2::BTreeV2Header::parse_at_storage(
|
|
582
|
+
self.context.storage.as_ref(),
|
|
583
|
+
link_info.btree_name_index_address,
|
|
584
|
+
self.offset_size(),
|
|
585
|
+
self.length_size(),
|
|
586
|
+
)?;
|
|
587
|
+
|
|
588
|
+
let records = btree_v2::collect_btree_v2_link_name_hash_records_storage(
|
|
589
|
+
self.context.storage.as_ref(),
|
|
590
|
+
&btree_header,
|
|
591
|
+
self.offset_size(),
|
|
592
|
+
self.length_size(),
|
|
593
|
+
jenkins_lookup3(name.as_bytes()),
|
|
594
|
+
)?;
|
|
595
|
+
|
|
596
|
+
let mut direct_block_cache = FractalHeapDirectBlockCache::default();
|
|
597
|
+
for record in &records {
|
|
598
|
+
let btree_v2::BTreeV2Record::LinkNameHash { heap_id, .. } = record else {
|
|
599
|
+
continue;
|
|
600
|
+
};
|
|
601
|
+
|
|
602
|
+
let managed_bytes = heap.get_object_storage_cached_with_registry(
|
|
603
|
+
heap_id,
|
|
604
|
+
self.context.storage.as_ref(),
|
|
605
|
+
self.offset_size(),
|
|
606
|
+
self.length_size(),
|
|
607
|
+
&mut direct_block_cache,
|
|
608
|
+
Some(self.context.filter_registry.as_ref()),
|
|
609
|
+
)?;
|
|
610
|
+
|
|
611
|
+
let mut link_cursor = Cursor::new(&managed_bytes);
|
|
612
|
+
let link_msg = link::parse(
|
|
613
|
+
&mut link_cursor,
|
|
614
|
+
self.offset_size(),
|
|
615
|
+
self.length_size(),
|
|
616
|
+
managed_bytes.len(),
|
|
617
|
+
)?;
|
|
618
|
+
if link_msg.name == name {
|
|
619
|
+
return self.resolve_link_message_target(&link_msg, link_depth);
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
Ok(None)
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
pub fn child_name_by_address(&self, address: u64) -> Result<Option<String>> {
|
|
627
|
+
Ok(self
|
|
628
|
+
.resolve_children()?
|
|
629
|
+
.into_iter()
|
|
630
|
+
.find(|child| child.location.address == address)
|
|
631
|
+
.map(|child| child.name))
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
fn child_context(&self, child: &ChildEntry) -> String {
|
|
635
|
+
format!("child '{}' at {:#x}", child.name, child.location.address)
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
fn child_object_kind(&self, child: &ChildEntry) -> Result<ChildObjectKind> {
|
|
639
|
+
let header = self
|
|
640
|
+
.cached_child_header(child)
|
|
641
|
+
.map_err(|err| err.with_context(self.child_context(child)))?;
|
|
642
|
+
|
|
643
|
+
Ok(classify_child_header(header.as_ref()))
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
fn try_open_child_dataset(&self, child: &ChildEntry) -> Result<Option<Dataset>> {
|
|
647
|
+
let header = self
|
|
648
|
+
.cached_child_header(child)
|
|
649
|
+
.map_err(|err| err.with_context(self.child_context(child)))?;
|
|
650
|
+
|
|
651
|
+
if classify_child_header(header.as_ref()) != ChildObjectKind::Dataset {
|
|
652
|
+
return Ok(None);
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
Dataset::from_parsed_header(
|
|
656
|
+
crate::dataset::DatasetParseContext {
|
|
657
|
+
context: child.location.context.clone(),
|
|
658
|
+
},
|
|
659
|
+
child.location.address,
|
|
660
|
+
child.name.clone(),
|
|
661
|
+
header.as_ref(),
|
|
662
|
+
)
|
|
663
|
+
.map(Some)
|
|
664
|
+
.map_err(|err| err.with_context(self.child_context(child)))
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
fn cached_child_header(
|
|
668
|
+
&self,
|
|
669
|
+
child: &ChildEntry,
|
|
670
|
+
) -> Result<Arc<crate::object_header::ObjectHeader>> {
|
|
671
|
+
child
|
|
672
|
+
.location
|
|
673
|
+
.context
|
|
674
|
+
.get_or_parse_header(child.location.address)
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
/// Maximum nesting depth for soft link resolution.
|
|
678
|
+
const MAX_SOFT_LINK_DEPTH: u32 = 16;
|
|
679
|
+
|
|
680
|
+
fn resolve_soft_link_depth(&self, path: &str, depth: u32) -> Result<ObjectLocation> {
|
|
681
|
+
self.resolve_path_location(path, depth, "soft link")
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
fn resolve_external_link_depth(
|
|
685
|
+
&self,
|
|
686
|
+
filename: &str,
|
|
687
|
+
path: &str,
|
|
688
|
+
depth: u32,
|
|
689
|
+
) -> Result<Option<ObjectLocation>> {
|
|
690
|
+
if depth >= Self::MAX_SOFT_LINK_DEPTH {
|
|
691
|
+
return Err(Error::Other(format!(
|
|
692
|
+
"external link resolution exceeded maximum depth ({}) at '{}:{}'",
|
|
693
|
+
Self::MAX_SOFT_LINK_DEPTH,
|
|
694
|
+
filename,
|
|
695
|
+
path,
|
|
696
|
+
)));
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
let Some(resolver) = self.context.external_link_resolver.as_ref() else {
|
|
700
|
+
return Ok(None);
|
|
701
|
+
};
|
|
702
|
+
let Some(file) = resolver.resolve_external_link(filename)? else {
|
|
703
|
+
return Ok(None);
|
|
704
|
+
};
|
|
705
|
+
let root = file.root_group()?;
|
|
706
|
+
Ok(Some(root.resolve_path_location(
|
|
707
|
+
path,
|
|
708
|
+
depth + 1,
|
|
709
|
+
"external link",
|
|
710
|
+
)?))
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
fn resolve_path_location(
|
|
714
|
+
&self,
|
|
715
|
+
path: &str,
|
|
716
|
+
depth: u32,
|
|
717
|
+
link_kind: &str,
|
|
718
|
+
) -> Result<ObjectLocation> {
|
|
719
|
+
if depth >= Self::MAX_SOFT_LINK_DEPTH {
|
|
720
|
+
return Err(Error::Other(format!(
|
|
721
|
+
"{} resolution exceeded maximum depth ({}) — possible cycle at '{}'",
|
|
722
|
+
link_kind,
|
|
723
|
+
Self::MAX_SOFT_LINK_DEPTH,
|
|
724
|
+
path,
|
|
725
|
+
)));
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
let parts: Vec<&str> = path
|
|
729
|
+
.trim_matches('/')
|
|
730
|
+
.split('/')
|
|
731
|
+
.filter(|s| !s.is_empty())
|
|
732
|
+
.collect();
|
|
733
|
+
|
|
734
|
+
if parts.is_empty() {
|
|
735
|
+
return Ok(self.local_location(self.root_address));
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
let start_addr = if path.starts_with('/') {
|
|
739
|
+
self.root_address
|
|
740
|
+
} else {
|
|
741
|
+
self.address
|
|
742
|
+
};
|
|
743
|
+
|
|
744
|
+
let mut current_group = Group::new(
|
|
745
|
+
self.context.clone(),
|
|
746
|
+
start_addr,
|
|
747
|
+
String::new(),
|
|
748
|
+
self.root_address,
|
|
749
|
+
);
|
|
750
|
+
|
|
751
|
+
for &part in &parts[..parts.len() - 1] {
|
|
752
|
+
current_group = current_group.group(part)?;
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
let target_name = parts[parts.len() - 1];
|
|
756
|
+
if let Some(child) = current_group.resolve_child_with_link_depth(target_name, depth + 1)? {
|
|
757
|
+
return Ok(child.location);
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
Err(Error::Other(format!(
|
|
761
|
+
"{} target '{}' not found",
|
|
762
|
+
link_kind, path
|
|
763
|
+
)))
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
fn classify_child_header(header: &crate::object_header::ObjectHeader) -> ChildObjectKind {
|
|
768
|
+
let mut has_dataset_message = false;
|
|
769
|
+
let mut has_attribute_message = false;
|
|
770
|
+
|
|
771
|
+
for msg in &header.messages {
|
|
772
|
+
match msg {
|
|
773
|
+
HdfMessage::SymbolTable(_)
|
|
774
|
+
| HdfMessage::Link(_)
|
|
775
|
+
| HdfMessage::LinkInfo(_)
|
|
776
|
+
| HdfMessage::GroupInfo(_) => return ChildObjectKind::Group,
|
|
777
|
+
HdfMessage::Attribute(_) | HdfMessage::AttributeInfo(_) => {
|
|
778
|
+
has_attribute_message = true;
|
|
779
|
+
}
|
|
780
|
+
HdfMessage::Dataspace(_)
|
|
781
|
+
| HdfMessage::DataLayout(_)
|
|
782
|
+
| HdfMessage::FillValue(_)
|
|
783
|
+
| HdfMessage::FilterPipeline(_) => has_dataset_message = true,
|
|
784
|
+
_ => {}
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
if has_dataset_message {
|
|
789
|
+
ChildObjectKind::Dataset
|
|
790
|
+
} else if has_attribute_message {
|
|
791
|
+
ChildObjectKind::Group
|
|
792
|
+
} else {
|
|
793
|
+
ChildObjectKind::Other
|
|
794
|
+
}
|
|
795
|
+
}
|