@nirs4all/formats-wasm 0.2.11 → 0.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD_PARTY_NOTICES.md +49 -0
- package/nirs4all_formats_wasm_bg.wasm +0 -0
- package/package.json +4 -2
- package/reader-backports/hdf5-reader/Cargo.toml +119 -0
- package/reader-backports/hdf5-reader/LICENSE-APACHE +164 -0
- package/reader-backports/hdf5-reader/LICENSE-MIT +21 -0
- package/reader-backports/hdf5-reader/PROVENANCE.json +80 -0
- package/reader-backports/hdf5-reader/README.md +308 -0
- package/reader-backports/hdf5-reader/UPSTREAM-Cargo.toml +44 -0
- package/reader-backports/hdf5-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/hdf5-reader/src/attribute_api.rs +847 -0
- package/reader-backports/hdf5-reader/src/btree_v1.rs +726 -0
- package/reader-backports/hdf5-reader/src/btree_v2.rs +2068 -0
- package/reader-backports/hdf5-reader/src/cache.rs +354 -0
- package/reader-backports/hdf5-reader/src/checksum.rs +56 -0
- package/reader-backports/hdf5-reader/src/chunk_index.rs +423 -0
- package/reader-backports/hdf5-reader/src/dataset.rs +4733 -0
- package/reader-backports/hdf5-reader/src/datatype_api.rs +497 -0
- package/reader-backports/hdf5-reader/src/error.rs +167 -0
- package/reader-backports/hdf5-reader/src/extensible_array.rs +1886 -0
- package/reader-backports/hdf5-reader/src/filters/deflate.rs +93 -0
- package/reader-backports/hdf5-reader/src/filters/fletcher32.rs +52 -0
- package/reader-backports/hdf5-reader/src/filters/lz4.rs +178 -0
- package/reader-backports/hdf5-reader/src/filters/mod.rs +326 -0
- package/reader-backports/hdf5-reader/src/filters/nbit.rs +586 -0
- package/reader-backports/hdf5-reader/src/filters/scaleoffset.rs +827 -0
- package/reader-backports/hdf5-reader/src/filters/shuffle.rs +99 -0
- package/reader-backports/hdf5-reader/src/fixed_array.rs +763 -0
- package/reader-backports/hdf5-reader/src/fractal_heap.rs +2029 -0
- package/reader-backports/hdf5-reader/src/global_heap.rs +284 -0
- package/reader-backports/hdf5-reader/src/group.rs +795 -0
- package/reader-backports/hdf5-reader/src/io.rs +413 -0
- package/reader-backports/hdf5-reader/src/lib.rs +916 -0
- package/reader-backports/hdf5-reader/src/local_heap.rs +261 -0
- package/reader-backports/hdf5-reader/src/messages/attribute.rs +304 -0
- package/reader-backports/hdf5-reader/src/messages/attribute_info.rs +96 -0
- package/reader-backports/hdf5-reader/src/messages/btree_k.rs +36 -0
- package/reader-backports/hdf5-reader/src/messages/continuation.rs +46 -0
- package/reader-backports/hdf5-reader/src/messages/dataspace.rs +259 -0
- package/reader-backports/hdf5-reader/src/messages/datatype.rs +938 -0
- package/reader-backports/hdf5-reader/src/messages/external_files.rs +50 -0
- package/reader-backports/hdf5-reader/src/messages/fill_value.rs +233 -0
- package/reader-backports/hdf5-reader/src/messages/filter_pipeline.rs +147 -0
- package/reader-backports/hdf5-reader/src/messages/group_info.rs +103 -0
- package/reader-backports/hdf5-reader/src/messages/layout.rs +476 -0
- package/reader-backports/hdf5-reader/src/messages/link.rs +170 -0
- package/reader-backports/hdf5-reader/src/messages/link_info.rs +127 -0
- package/reader-backports/hdf5-reader/src/messages/mod.rs +374 -0
- package/reader-backports/hdf5-reader/src/messages/modification_time.rs +83 -0
- package/reader-backports/hdf5-reader/src/messages/shared.rs +164 -0
- package/reader-backports/hdf5-reader/src/messages/shared_table.rs +67 -0
- package/reader-backports/hdf5-reader/src/messages/symbol_table_msg.rs +51 -0
- package/reader-backports/hdf5-reader/src/object_header.rs +1749 -0
- package/reader-backports/hdf5-reader/src/reference.rs +90 -0
- package/reader-backports/hdf5-reader/src/shared_message_table.rs +425 -0
- package/reader-backports/hdf5-reader/src/storage.rs +538 -0
- package/reader-backports/hdf5-reader/src/superblock.rs +285 -0
- package/reader-backports/hdf5-reader/src/symbol_table.rs +277 -0
- package/reader-backports/netcdf-reader/Cargo.toml +141 -0
- package/reader-backports/netcdf-reader/LICENSE-APACHE +164 -0
- package/reader-backports/netcdf-reader/LICENSE-MIT +21 -0
- package/reader-backports/netcdf-reader/PROVENANCE.json +55 -0
- package/reader-backports/netcdf-reader/README.md +308 -0
- package/reader-backports/netcdf-reader/UPSTREAM-Cargo.toml +48 -0
- package/reader-backports/netcdf-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/netcdf-reader/src/cf/axes.rs +419 -0
- package/reader-backports/netcdf-reader/src/cf/bounds.rs +108 -0
- package/reader-backports/netcdf-reader/src/cf/crs.rs +269 -0
- package/reader-backports/netcdf-reader/src/cf/mod.rs +30 -0
- package/reader-backports/netcdf-reader/src/cf/time.rs +1122 -0
- package/reader-backports/netcdf-reader/src/classic/data.rs +1465 -0
- package/reader-backports/netcdf-reader/src/classic/header.rs +1097 -0
- package/reader-backports/netcdf-reader/src/classic/mod.rs +240 -0
- package/reader-backports/netcdf-reader/src/classic/storage.rs +211 -0
- package/reader-backports/netcdf-reader/src/classic/types.rs +93 -0
- package/reader-backports/netcdf-reader/src/classic/variable.rs +1162 -0
- package/reader-backports/netcdf-reader/src/error.rs +59 -0
- package/reader-backports/netcdf-reader/src/lib.rs +1531 -0
- package/reader-backports/netcdf-reader/src/masked.rs +173 -0
- package/reader-backports/netcdf-reader/src/nc4/attributes.rs +203 -0
- package/reader-backports/netcdf-reader/src/nc4/dimensions.rs +139 -0
- package/reader-backports/netcdf-reader/src/nc4/groups.rs +308 -0
- package/reader-backports/netcdf-reader/src/nc4/mod.rs +572 -0
- package/reader-backports/netcdf-reader/src/nc4/types.rs +309 -0
- package/reader-backports/netcdf-reader/src/nc4/variables.rs +420 -0
- package/reader-backports/netcdf-reader/src/types.rs +18 -0
- package/reader-backports/netcdf-reader/src/unpack.rs +113 -0
- package/reader-backports/netcdf-reader/src/user_defined.rs +569 -0
|
@@ -0,0 +1,476 @@
|
|
|
1
|
+
//! HDF5 Data Layout message (type 0x0008).
|
|
2
|
+
//!
|
|
3
|
+
//! Describes how raw data for a dataset is stored: compact (inline in the
|
|
4
|
+
//! object header), contiguous (a single block in the file), or chunked
|
|
5
|
+
//! (split into fixed-size chunks, indexed by a B-tree).
|
|
6
|
+
|
|
7
|
+
use crate::error::{Error, Result};
|
|
8
|
+
use crate::io::Cursor;
|
|
9
|
+
pub use hdf5_core::{ChunkIndexing, DataLayout, DataLayoutMessage};
|
|
10
|
+
|
|
11
|
+
/// Parse a data layout message.
|
|
12
|
+
pub fn parse(
|
|
13
|
+
cursor: &mut Cursor<'_>,
|
|
14
|
+
offset_size: u8,
|
|
15
|
+
length_size: u8,
|
|
16
|
+
msg_size: usize,
|
|
17
|
+
) -> Result<DataLayoutMessage> {
|
|
18
|
+
let start = cursor.position();
|
|
19
|
+
let version = cursor.read_u8()?;
|
|
20
|
+
|
|
21
|
+
let layout = match version {
|
|
22
|
+
1 | 2 => parse_v1_v2(cursor, offset_size, length_size, version)?,
|
|
23
|
+
3 => parse_v3(cursor, offset_size, length_size)?,
|
|
24
|
+
4 | 5 => parse_v4_v5(cursor, offset_size, length_size, version)?,
|
|
25
|
+
v => return Err(Error::UnsupportedLayoutVersion(v)),
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
let consumed = (cursor.position() - start) as usize;
|
|
29
|
+
if consumed < msg_size {
|
|
30
|
+
cursor.skip(msg_size - consumed)?;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
Ok(DataLayoutMessage { layout })
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// ---------------------------------------------------------------------------
|
|
37
|
+
// Version 1 / 2
|
|
38
|
+
// ---------------------------------------------------------------------------
|
|
39
|
+
|
|
40
|
+
fn parse_v1_v2(
|
|
41
|
+
cursor: &mut Cursor<'_>,
|
|
42
|
+
offset_size: u8,
|
|
43
|
+
_length_size: u8,
|
|
44
|
+
version: u8,
|
|
45
|
+
) -> Result<DataLayout> {
|
|
46
|
+
let dimensionality = cursor.read_u8()?;
|
|
47
|
+
let layout_class = cursor.read_u8()?;
|
|
48
|
+
let _reserved = cursor.read_bytes(if version == 1 { 5 } else { 3 })?;
|
|
49
|
+
|
|
50
|
+
// For v1 there is an optional compact data size field.
|
|
51
|
+
// data_address is only meaningful for contiguous and chunked.
|
|
52
|
+
let data_address = if layout_class != 0 {
|
|
53
|
+
cursor.read_offset(offset_size)?
|
|
54
|
+
} else {
|
|
55
|
+
// For compact, there is no address; skip the offset-sized field.
|
|
56
|
+
cursor.read_offset(offset_size)?
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
// Read dimension sizes. Each is 4 bytes. The number of dimensions:
|
|
60
|
+
// For contiguous: dimensionality values (unused data size).
|
|
61
|
+
// For chunked: (dimensionality-1) chunk dims + 1 element size.
|
|
62
|
+
let mut dim_values = Vec::with_capacity(dimensionality as usize);
|
|
63
|
+
for _ in 0..dimensionality {
|
|
64
|
+
dim_values.push(cursor.read_u32_le()?);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
match layout_class {
|
|
68
|
+
0 => {
|
|
69
|
+
// Compact
|
|
70
|
+
let compact_size = cursor.read_u32_le()? as usize;
|
|
71
|
+
let data = cursor.read_bytes(compact_size)?.to_vec();
|
|
72
|
+
Ok(DataLayout::Compact { data })
|
|
73
|
+
}
|
|
74
|
+
1 => {
|
|
75
|
+
// Contiguous
|
|
76
|
+
// Size is not explicitly stored in v1/v2 for contiguous. The dims
|
|
77
|
+
// encode the logical size but the actual file extent comes from the
|
|
78
|
+
// dataspace * element size. We store the product as size.
|
|
79
|
+
let size = if dim_values.is_empty() {
|
|
80
|
+
0
|
|
81
|
+
} else {
|
|
82
|
+
dim_values.iter().map(|d| *d as u64).product()
|
|
83
|
+
};
|
|
84
|
+
Ok(DataLayout::Contiguous {
|
|
85
|
+
address: data_address,
|
|
86
|
+
size,
|
|
87
|
+
})
|
|
88
|
+
}
|
|
89
|
+
2 => {
|
|
90
|
+
// Chunked — last dimension is the element size
|
|
91
|
+
let (element_size, chunk_dims) = if dim_values.is_empty() {
|
|
92
|
+
(0u32, vec![])
|
|
93
|
+
} else {
|
|
94
|
+
let es = *dim_values.last().unwrap();
|
|
95
|
+
let cd: Vec<u32> = dim_values[..dim_values.len() - 1].to_vec();
|
|
96
|
+
(es, cd)
|
|
97
|
+
};
|
|
98
|
+
Ok(DataLayout::Chunked {
|
|
99
|
+
address: data_address,
|
|
100
|
+
dims: chunk_dims,
|
|
101
|
+
element_size,
|
|
102
|
+
chunk_indexing: None,
|
|
103
|
+
})
|
|
104
|
+
}
|
|
105
|
+
c => Err(Error::UnsupportedLayoutClass(c)),
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
// ---------------------------------------------------------------------------
|
|
110
|
+
// Version 3
|
|
111
|
+
// ---------------------------------------------------------------------------
|
|
112
|
+
|
|
113
|
+
fn parse_v3(cursor: &mut Cursor<'_>, offset_size: u8, length_size: u8) -> Result<DataLayout> {
|
|
114
|
+
let layout_class = cursor.read_u8()?;
|
|
115
|
+
|
|
116
|
+
match layout_class {
|
|
117
|
+
0 => {
|
|
118
|
+
// Compact
|
|
119
|
+
let size = cursor.read_u16_le()? as usize;
|
|
120
|
+
let data = cursor.read_bytes(size)?.to_vec();
|
|
121
|
+
Ok(DataLayout::Compact { data })
|
|
122
|
+
}
|
|
123
|
+
1 => {
|
|
124
|
+
// Contiguous
|
|
125
|
+
let address = cursor.read_offset(offset_size)?;
|
|
126
|
+
let size = cursor.read_length(length_size)?;
|
|
127
|
+
Ok(DataLayout::Contiguous { address, size })
|
|
128
|
+
}
|
|
129
|
+
2 => {
|
|
130
|
+
// Chunked
|
|
131
|
+
let dimensionality = cursor.read_u8()?;
|
|
132
|
+
let address = cursor.read_offset(offset_size)?;
|
|
133
|
+
|
|
134
|
+
// (dimensionality - 1) chunk dims + 1 element size (each 4 bytes)
|
|
135
|
+
let n = dimensionality as usize;
|
|
136
|
+
let mut raw_dims = Vec::with_capacity(n);
|
|
137
|
+
for _ in 0..n {
|
|
138
|
+
raw_dims.push(cursor.read_u32_le()?);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
let (element_size, chunk_dims) = if raw_dims.is_empty() {
|
|
142
|
+
(0, vec![])
|
|
143
|
+
} else {
|
|
144
|
+
let es = *raw_dims.last().unwrap();
|
|
145
|
+
let cd = raw_dims[..raw_dims.len() - 1].to_vec();
|
|
146
|
+
(es, cd)
|
|
147
|
+
};
|
|
148
|
+
|
|
149
|
+
Ok(DataLayout::Chunked {
|
|
150
|
+
address,
|
|
151
|
+
dims: chunk_dims,
|
|
152
|
+
element_size,
|
|
153
|
+
chunk_indexing: None,
|
|
154
|
+
})
|
|
155
|
+
}
|
|
156
|
+
c => Err(Error::UnsupportedLayoutClass(c)),
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// ---------------------------------------------------------------------------
|
|
161
|
+
// Version 4
|
|
162
|
+
// ---------------------------------------------------------------------------
|
|
163
|
+
|
|
164
|
+
/// Parse v4/v5 layout messages.
|
|
165
|
+
///
|
|
166
|
+
/// v4/v5 chunked layouts store only the chunk dimensions here; element size
|
|
167
|
+
/// is derived from the datatype. Filtered chunk-index records switched from
|
|
168
|
+
/// `length_size` to `offset_size` in v5.
|
|
169
|
+
fn parse_v4_v5(
|
|
170
|
+
cursor: &mut Cursor<'_>,
|
|
171
|
+
offset_size: u8,
|
|
172
|
+
length_size: u8,
|
|
173
|
+
version: u8,
|
|
174
|
+
) -> Result<DataLayout> {
|
|
175
|
+
let layout_class = cursor.read_u8()?;
|
|
176
|
+
|
|
177
|
+
match layout_class {
|
|
178
|
+
0 => {
|
|
179
|
+
// Compact
|
|
180
|
+
let size = cursor.read_u16_le()? as usize;
|
|
181
|
+
let data = cursor.read_bytes(size)?.to_vec();
|
|
182
|
+
Ok(DataLayout::Compact { data })
|
|
183
|
+
}
|
|
184
|
+
1 => {
|
|
185
|
+
// Contiguous
|
|
186
|
+
let address = cursor.read_offset(offset_size)?;
|
|
187
|
+
let size = cursor.read_u64_le()?;
|
|
188
|
+
Ok(DataLayout::Contiguous { address, size })
|
|
189
|
+
}
|
|
190
|
+
2 => {
|
|
191
|
+
let start = cursor.clone();
|
|
192
|
+
let direct = parse_v4_v5_chunked(cursor, offset_size, length_size, version, false);
|
|
193
|
+
match direct {
|
|
194
|
+
Ok(layout) => Ok(layout),
|
|
195
|
+
Err(err) if version == 4 && should_retry_v4_chunked_parse(&err) => {
|
|
196
|
+
*cursor = start;
|
|
197
|
+
parse_v4_v5_chunked(cursor, offset_size, length_size, version, true)
|
|
198
|
+
}
|
|
199
|
+
Err(err) => Err(err),
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
c => Err(Error::UnsupportedLayoutClass(c)),
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
fn parse_v4_v5_chunked(
|
|
207
|
+
cursor: &mut Cursor<'_>,
|
|
208
|
+
offset_size: u8,
|
|
209
|
+
length_size: u8,
|
|
210
|
+
version: u8,
|
|
211
|
+
legacy_dim_size_encoding: bool,
|
|
212
|
+
) -> Result<DataLayout> {
|
|
213
|
+
let flags = cursor.read_u8()?;
|
|
214
|
+
let ndims_raw = cursor.read_u8()? as usize;
|
|
215
|
+
let dim_size_enc = cursor.read_u8()?;
|
|
216
|
+
let dim_bytes = if legacy_dim_size_encoding {
|
|
217
|
+
dim_size_enc as usize + 1
|
|
218
|
+
} else {
|
|
219
|
+
dim_size_enc as usize
|
|
220
|
+
};
|
|
221
|
+
|
|
222
|
+
let mut dims = Vec::with_capacity(ndims_raw);
|
|
223
|
+
for _ in 0..ndims_raw {
|
|
224
|
+
let dim = cursor.read_uvar(dim_bytes)?;
|
|
225
|
+
let dim = u32::try_from(dim).map_err(|_| {
|
|
226
|
+
Error::InvalidData(format!("chunk dimension {dim} exceeds u32 capacity"))
|
|
227
|
+
})?;
|
|
228
|
+
dims.push(dim);
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
let index_type = cursor.read_u8()?;
|
|
232
|
+
let chunk_size_len = if version >= 5 {
|
|
233
|
+
offset_size
|
|
234
|
+
} else {
|
|
235
|
+
length_size
|
|
236
|
+
};
|
|
237
|
+
let chunk_indexing = parse_chunk_indexing_v4_v5(cursor, flags, index_type, chunk_size_len)?;
|
|
238
|
+
let address = cursor.read_offset(offset_size)?;
|
|
239
|
+
|
|
240
|
+
Ok(DataLayout::Chunked {
|
|
241
|
+
address,
|
|
242
|
+
dims,
|
|
243
|
+
element_size: 0,
|
|
244
|
+
chunk_indexing: Some(chunk_indexing),
|
|
245
|
+
})
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
fn should_retry_v4_chunked_parse(err: &Error) -> bool {
|
|
249
|
+
match err {
|
|
250
|
+
Error::UnexpectedEof { .. } | Error::UnsupportedChunkIndexType(_) => true,
|
|
251
|
+
Error::InvalidData(msg) => msg.starts_with("unsupported variable integer size:"),
|
|
252
|
+
_ => false,
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/// Parse chunk indexing for v4/v5 layout.
|
|
257
|
+
/// On-disk values: 1=SingleChunk, 2=Implicit, 3=FixedArray, 4=ExtensibleArray, 5=BTreeV2
|
|
258
|
+
fn parse_chunk_indexing_v4_v5(
|
|
259
|
+
cursor: &mut Cursor<'_>,
|
|
260
|
+
flags: u8,
|
|
261
|
+
index_type: u8,
|
|
262
|
+
chunk_size_len: u8,
|
|
263
|
+
) -> Result<ChunkIndexing> {
|
|
264
|
+
match index_type {
|
|
265
|
+
1 => {
|
|
266
|
+
// Single chunk. Flag bit 1 ("single index with filter") marks the
|
|
267
|
+
// inline filtered-chunk size and filter mask; bit 0 is the
|
|
268
|
+
// unrelated "don't filter partial edge chunks" option.
|
|
269
|
+
let idx_flags = if (flags & 0x02) != 0 {
|
|
270
|
+
let filtered_size = cursor.read_u64_le()?;
|
|
271
|
+
let filter_mask = cursor.read_u32_le()?;
|
|
272
|
+
Some((filtered_size, filter_mask))
|
|
273
|
+
} else {
|
|
274
|
+
None
|
|
275
|
+
};
|
|
276
|
+
let (fs, fm) = idx_flags.unwrap_or((0, 0));
|
|
277
|
+
Ok(ChunkIndexing::SingleChunk {
|
|
278
|
+
filtered_size: fs,
|
|
279
|
+
filters: fm,
|
|
280
|
+
})
|
|
281
|
+
}
|
|
282
|
+
2 => Ok(ChunkIndexing::Implicit),
|
|
283
|
+
3 => {
|
|
284
|
+
let page_bits = cursor.read_u8()?;
|
|
285
|
+
Ok(ChunkIndexing::FixedArray {
|
|
286
|
+
page_bits,
|
|
287
|
+
chunk_size_len,
|
|
288
|
+
})
|
|
289
|
+
}
|
|
290
|
+
4 => {
|
|
291
|
+
let max_bits = cursor.read_u8()?;
|
|
292
|
+
let index_bits = cursor.read_u8()?;
|
|
293
|
+
let min_pointers = cursor.read_u8()?;
|
|
294
|
+
let min_elements = cursor.read_u8()?;
|
|
295
|
+
let _max_dblk_page_bits = cursor.read_u8()?;
|
|
296
|
+
Ok(ChunkIndexing::ExtensibleArray {
|
|
297
|
+
max_bits,
|
|
298
|
+
index_bits,
|
|
299
|
+
min_pointers,
|
|
300
|
+
min_elements,
|
|
301
|
+
chunk_size_len,
|
|
302
|
+
})
|
|
303
|
+
}
|
|
304
|
+
5 => {
|
|
305
|
+
cursor.skip(6)?;
|
|
306
|
+
Ok(ChunkIndexing::BTreeV2)
|
|
307
|
+
}
|
|
308
|
+
t => Err(Error::UnsupportedChunkIndexType(t)),
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
#[cfg(test)]
|
|
313
|
+
mod tests {
|
|
314
|
+
use super::*;
|
|
315
|
+
|
|
316
|
+
#[test]
|
|
317
|
+
fn parse_v3_contiguous() {
|
|
318
|
+
let mut data = vec![
|
|
319
|
+
0x03, // version 3
|
|
320
|
+
0x01, // layout class = contiguous
|
|
321
|
+
];
|
|
322
|
+
// address (8 bytes)
|
|
323
|
+
data.extend_from_slice(&0x1000u64.to_le_bytes());
|
|
324
|
+
// size (8 bytes)
|
|
325
|
+
data.extend_from_slice(&4096u64.to_le_bytes());
|
|
326
|
+
|
|
327
|
+
let mut cursor = Cursor::new(&data);
|
|
328
|
+
let msg = parse(&mut cursor, 8, 8, data.len()).unwrap();
|
|
329
|
+
match &msg.layout {
|
|
330
|
+
DataLayout::Contiguous { address, size } => {
|
|
331
|
+
assert_eq!(*address, 0x1000);
|
|
332
|
+
assert_eq!(*size, 4096);
|
|
333
|
+
}
|
|
334
|
+
other => panic!("expected Contiguous, got {:?}", other),
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
#[test]
|
|
339
|
+
fn parse_v3_compact() {
|
|
340
|
+
let mut data = vec![
|
|
341
|
+
0x03, // version 3
|
|
342
|
+
0x00, // layout class = compact
|
|
343
|
+
];
|
|
344
|
+
// compact size = 4
|
|
345
|
+
data.extend_from_slice(&4u16.to_le_bytes());
|
|
346
|
+
// inline data
|
|
347
|
+
data.extend_from_slice(&[0x01, 0x02, 0x03, 0x04]);
|
|
348
|
+
|
|
349
|
+
let mut cursor = Cursor::new(&data);
|
|
350
|
+
let msg = parse(&mut cursor, 8, 8, data.len()).unwrap();
|
|
351
|
+
match &msg.layout {
|
|
352
|
+
DataLayout::Compact { data } => {
|
|
353
|
+
assert_eq!(data, &[0x01, 0x02, 0x03, 0x04]);
|
|
354
|
+
}
|
|
355
|
+
other => panic!("expected Compact, got {:?}", other),
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
#[test]
|
|
360
|
+
fn parse_v3_chunked() {
|
|
361
|
+
let mut data = vec![
|
|
362
|
+
0x03, // version 3
|
|
363
|
+
0x02, // layout class = chunked
|
|
364
|
+
0x03, // dimensionality = 3 (2 chunk dims + 1 element size)
|
|
365
|
+
];
|
|
366
|
+
// address
|
|
367
|
+
data.extend_from_slice(&0x2000u64.to_le_bytes());
|
|
368
|
+
// dim[0] = 256
|
|
369
|
+
data.extend_from_slice(&256u32.to_le_bytes());
|
|
370
|
+
// dim[1] = 128
|
|
371
|
+
data.extend_from_slice(&128u32.to_le_bytes());
|
|
372
|
+
// element size = 4
|
|
373
|
+
data.extend_from_slice(&4u32.to_le_bytes());
|
|
374
|
+
|
|
375
|
+
let mut cursor = Cursor::new(&data);
|
|
376
|
+
let msg = parse(&mut cursor, 8, 8, data.len()).unwrap();
|
|
377
|
+
match &msg.layout {
|
|
378
|
+
DataLayout::Chunked {
|
|
379
|
+
address,
|
|
380
|
+
dims,
|
|
381
|
+
element_size,
|
|
382
|
+
chunk_indexing,
|
|
383
|
+
} => {
|
|
384
|
+
assert_eq!(*address, 0x2000);
|
|
385
|
+
assert_eq!(dims, &[256, 128]);
|
|
386
|
+
assert_eq!(*element_size, 4);
|
|
387
|
+
assert!(chunk_indexing.is_none());
|
|
388
|
+
}
|
|
389
|
+
other => panic!("expected Chunked, got {:?}", other),
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
#[test]
|
|
394
|
+
fn parse_v4_chunked_direct_dim_size_encoding() {
|
|
395
|
+
let mut data = vec![
|
|
396
|
+
0x04, // version 4
|
|
397
|
+
0x02, // layout class = chunked
|
|
398
|
+
0x00, // flags
|
|
399
|
+
0x02, // ndims
|
|
400
|
+
0x04, // 4 bytes per dimension
|
|
401
|
+
];
|
|
402
|
+
data.extend_from_slice(&3u32.to_le_bytes());
|
|
403
|
+
data.extend_from_slice(&5u32.to_le_bytes());
|
|
404
|
+
data.push(0x03); // fixed array indexing
|
|
405
|
+
data.push(0x00); // page bits
|
|
406
|
+
data.extend_from_slice(&0x1122_3344_5566_7788u64.to_le_bytes());
|
|
407
|
+
|
|
408
|
+
let mut cursor = Cursor::new(&data);
|
|
409
|
+
let msg = parse(&mut cursor, 8, 8, data.len()).unwrap();
|
|
410
|
+
match &msg.layout {
|
|
411
|
+
DataLayout::Chunked {
|
|
412
|
+
address,
|
|
413
|
+
dims,
|
|
414
|
+
element_size,
|
|
415
|
+
chunk_indexing,
|
|
416
|
+
} => {
|
|
417
|
+
assert_eq!(*address, 0x1122_3344_5566_7788);
|
|
418
|
+
assert_eq!(dims, &[3, 5]);
|
|
419
|
+
assert_eq!(*element_size, 0);
|
|
420
|
+
match chunk_indexing {
|
|
421
|
+
Some(ChunkIndexing::FixedArray {
|
|
422
|
+
page_bits,
|
|
423
|
+
chunk_size_len,
|
|
424
|
+
}) => {
|
|
425
|
+
assert_eq!(*page_bits, 0);
|
|
426
|
+
assert_eq!(*chunk_size_len, 8);
|
|
427
|
+
}
|
|
428
|
+
other => panic!("expected FixedArray indexing, got {:?}", other),
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
other => panic!("expected Chunked, got {:?}", other),
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
#[test]
|
|
436
|
+
fn parse_v4_chunked_legacy_dim_size_encoding() {
|
|
437
|
+
let mut data = vec![
|
|
438
|
+
0x04, // version 4
|
|
439
|
+
0x02, // layout class = chunked
|
|
440
|
+
0x00, // flags
|
|
441
|
+
0x02, // ndims
|
|
442
|
+
0x03, // legacy encoding: 4 bytes per dimension stored as 3
|
|
443
|
+
];
|
|
444
|
+
data.extend_from_slice(&3u32.to_le_bytes());
|
|
445
|
+
data.extend_from_slice(&5u32.to_le_bytes());
|
|
446
|
+
data.push(0x03); // fixed array indexing
|
|
447
|
+
data.push(0x00); // page bits
|
|
448
|
+
data.extend_from_slice(&0x8877_6655_4433_2211u64.to_le_bytes());
|
|
449
|
+
|
|
450
|
+
let mut cursor = Cursor::new(&data);
|
|
451
|
+
let msg = parse(&mut cursor, 8, 8, data.len()).unwrap();
|
|
452
|
+
match &msg.layout {
|
|
453
|
+
DataLayout::Chunked {
|
|
454
|
+
address,
|
|
455
|
+
dims,
|
|
456
|
+
element_size,
|
|
457
|
+
chunk_indexing,
|
|
458
|
+
} => {
|
|
459
|
+
assert_eq!(*address, 0x8877_6655_4433_2211);
|
|
460
|
+
assert_eq!(dims, &[3, 5]);
|
|
461
|
+
assert_eq!(*element_size, 0);
|
|
462
|
+
match chunk_indexing {
|
|
463
|
+
Some(ChunkIndexing::FixedArray {
|
|
464
|
+
page_bits,
|
|
465
|
+
chunk_size_len,
|
|
466
|
+
}) => {
|
|
467
|
+
assert_eq!(*page_bits, 0);
|
|
468
|
+
assert_eq!(*chunk_size_len, 8);
|
|
469
|
+
}
|
|
470
|
+
other => panic!("expected FixedArray indexing, got {:?}", other),
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
other => panic!("expected Chunked, got {:?}", other),
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
}
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
//! HDF5 Link message (type 0x0006).
|
|
2
|
+
//!
|
|
3
|
+
//! Used in v2+ object headers to describe group membership. Each link
|
|
4
|
+
//! message names one child and specifies its target (hard link to an
|
|
5
|
+
//! object header address, soft link path, or external file + path).
|
|
6
|
+
|
|
7
|
+
use crate::error::{Error, Result};
|
|
8
|
+
use crate::io::Cursor;
|
|
9
|
+
pub use hdf5_core::{LinkMessage, LinkTarget};
|
|
10
|
+
|
|
11
|
+
/// Parse a link message (version 1).
|
|
12
|
+
pub fn parse(
|
|
13
|
+
cursor: &mut Cursor<'_>,
|
|
14
|
+
offset_size: u8,
|
|
15
|
+
_length_size: u8,
|
|
16
|
+
msg_size: usize,
|
|
17
|
+
) -> Result<LinkMessage> {
|
|
18
|
+
let start = cursor.position();
|
|
19
|
+
let version = cursor.read_u8()?;
|
|
20
|
+
|
|
21
|
+
if version != 1 {
|
|
22
|
+
return Err(Error::UnsupportedLinkVersion(version));
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
let flags = cursor.read_u8()?;
|
|
26
|
+
|
|
27
|
+
// Bit 3: link type field present
|
|
28
|
+
let link_type = if (flags & 0x08) != 0 {
|
|
29
|
+
cursor.read_u8()?
|
|
30
|
+
} else {
|
|
31
|
+
0 // default = hard link
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
// Bit 2: creation order present
|
|
35
|
+
let creation_order = if (flags & 0x04) != 0 {
|
|
36
|
+
Some(cursor.read_u64_le()?)
|
|
37
|
+
} else {
|
|
38
|
+
None
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
// Bit 4: link name encoding (0 = ASCII, 1 = UTF-8)
|
|
42
|
+
// We handle both the same way since our strings are already UTF-8.
|
|
43
|
+
let _name_encoding = if (flags & 0x10) != 0 {
|
|
44
|
+
cursor.read_u8()?
|
|
45
|
+
} else {
|
|
46
|
+
0
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
// Name length — size depends on bits 0-1 of flags
|
|
50
|
+
let name_len_size = 1 << (flags & 0x03);
|
|
51
|
+
let name_len = cursor.read_uvar(name_len_size)? as usize;
|
|
52
|
+
|
|
53
|
+
let name = cursor.read_fixed_string(name_len)?;
|
|
54
|
+
|
|
55
|
+
// Link target
|
|
56
|
+
let target = match link_type {
|
|
57
|
+
0 => {
|
|
58
|
+
// Hard link
|
|
59
|
+
let address = cursor.read_offset(offset_size)?;
|
|
60
|
+
LinkTarget::Hard { address }
|
|
61
|
+
}
|
|
62
|
+
1 => {
|
|
63
|
+
// Soft link
|
|
64
|
+
let path_len = cursor.read_u16_le()? as usize;
|
|
65
|
+
let path = cursor.read_fixed_string(path_len)?;
|
|
66
|
+
LinkTarget::Soft { path }
|
|
67
|
+
}
|
|
68
|
+
64 => {
|
|
69
|
+
// External link
|
|
70
|
+
let info_len = cursor.read_u16_le()? as usize;
|
|
71
|
+
let info_start = cursor.position();
|
|
72
|
+
// Version + flags of the external link info
|
|
73
|
+
let _ext_flags = cursor.read_u8()?;
|
|
74
|
+
let filename = cursor.read_null_terminated_string()?;
|
|
75
|
+
let path = cursor.read_null_terminated_string()?;
|
|
76
|
+
// Skip any remaining bytes in the info block
|
|
77
|
+
let info_consumed = (cursor.position() - info_start) as usize;
|
|
78
|
+
if info_consumed < info_len {
|
|
79
|
+
cursor.skip(info_len - info_consumed)?;
|
|
80
|
+
}
|
|
81
|
+
LinkTarget::External { filename, path }
|
|
82
|
+
}
|
|
83
|
+
t => return Err(Error::UnsupportedLinkType(t)),
|
|
84
|
+
};
|
|
85
|
+
|
|
86
|
+
let consumed = (cursor.position() - start) as usize;
|
|
87
|
+
if consumed < msg_size {
|
|
88
|
+
cursor.skip(msg_size - consumed)?;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
Ok(LinkMessage {
|
|
92
|
+
name,
|
|
93
|
+
target,
|
|
94
|
+
creation_order,
|
|
95
|
+
})
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
#[cfg(test)]
|
|
99
|
+
mod tests {
|
|
100
|
+
use super::*;
|
|
101
|
+
|
|
102
|
+
#[test]
|
|
103
|
+
fn parse_hard_link() {
|
|
104
|
+
let mut data = vec![
|
|
105
|
+
0x01, // version 1
|
|
106
|
+
0x00, // flags: name_len_size=1 byte, no extras
|
|
107
|
+
];
|
|
108
|
+
// name length (1 byte) = 5
|
|
109
|
+
data.push(0x05);
|
|
110
|
+
// name = "hello"
|
|
111
|
+
data.extend_from_slice(b"hello");
|
|
112
|
+
// hard link address (8 bytes)
|
|
113
|
+
data.extend_from_slice(&0x3000u64.to_le_bytes());
|
|
114
|
+
|
|
115
|
+
let mut cursor = Cursor::new(&data);
|
|
116
|
+
let msg = parse(&mut cursor, 8, 8, data.len()).unwrap();
|
|
117
|
+
assert_eq!(msg.name, "hello");
|
|
118
|
+
assert!(msg.creation_order.is_none());
|
|
119
|
+
match &msg.target {
|
|
120
|
+
LinkTarget::Hard { address } => assert_eq!(*address, 0x3000),
|
|
121
|
+
other => panic!("expected Hard, got {:?}", other),
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
#[test]
|
|
126
|
+
fn parse_soft_link() {
|
|
127
|
+
let mut data = vec![
|
|
128
|
+
0x01, // version 1
|
|
129
|
+
0x08, // flags: bit 3 = link type present, name_len_size=1
|
|
130
|
+
];
|
|
131
|
+
// link type = 1 (soft)
|
|
132
|
+
data.push(0x01);
|
|
133
|
+
// name length = 3
|
|
134
|
+
data.push(0x03);
|
|
135
|
+
// name = "lnk"
|
|
136
|
+
data.extend_from_slice(b"lnk");
|
|
137
|
+
// soft link: path length (u16) + path
|
|
138
|
+
data.extend_from_slice(&7u16.to_le_bytes());
|
|
139
|
+
data.extend_from_slice(b"/a/b/c\0");
|
|
140
|
+
|
|
141
|
+
let mut cursor = Cursor::new(&data);
|
|
142
|
+
let msg = parse(&mut cursor, 8, 8, data.len()).unwrap();
|
|
143
|
+
assert_eq!(msg.name, "lnk");
|
|
144
|
+
match &msg.target {
|
|
145
|
+
LinkTarget::Soft { path } => assert_eq!(path, "/a/b/c"),
|
|
146
|
+
other => panic!("expected Soft, got {:?}", other),
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
#[test]
|
|
151
|
+
fn parse_hard_link_with_creation_order() {
|
|
152
|
+
let mut data = vec![
|
|
153
|
+
0x01, // version 1
|
|
154
|
+
0x04, // flags: bit 2 = creation order present
|
|
155
|
+
];
|
|
156
|
+
// creation order (u64) = 42
|
|
157
|
+
data.extend_from_slice(&42u64.to_le_bytes());
|
|
158
|
+
// name length = 1
|
|
159
|
+
data.push(0x01);
|
|
160
|
+
// name = "x"
|
|
161
|
+
data.push(b'x');
|
|
162
|
+
// hard link address
|
|
163
|
+
data.extend_from_slice(&0x5000u64.to_le_bytes());
|
|
164
|
+
|
|
165
|
+
let mut cursor = Cursor::new(&data);
|
|
166
|
+
let msg = parse(&mut cursor, 8, 8, data.len()).unwrap();
|
|
167
|
+
assert_eq!(msg.name, "x");
|
|
168
|
+
assert_eq!(msg.creation_order, Some(42));
|
|
169
|
+
}
|
|
170
|
+
}
|