@nirs4all/formats-wasm 0.2.11 → 0.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD_PARTY_NOTICES.md +49 -0
- package/nirs4all_formats_wasm_bg.wasm +0 -0
- package/package.json +4 -2
- package/reader-backports/hdf5-reader/Cargo.toml +119 -0
- package/reader-backports/hdf5-reader/LICENSE-APACHE +164 -0
- package/reader-backports/hdf5-reader/LICENSE-MIT +21 -0
- package/reader-backports/hdf5-reader/PROVENANCE.json +80 -0
- package/reader-backports/hdf5-reader/README.md +308 -0
- package/reader-backports/hdf5-reader/UPSTREAM-Cargo.toml +44 -0
- package/reader-backports/hdf5-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/hdf5-reader/src/attribute_api.rs +847 -0
- package/reader-backports/hdf5-reader/src/btree_v1.rs +726 -0
- package/reader-backports/hdf5-reader/src/btree_v2.rs +2068 -0
- package/reader-backports/hdf5-reader/src/cache.rs +354 -0
- package/reader-backports/hdf5-reader/src/checksum.rs +56 -0
- package/reader-backports/hdf5-reader/src/chunk_index.rs +423 -0
- package/reader-backports/hdf5-reader/src/dataset.rs +4733 -0
- package/reader-backports/hdf5-reader/src/datatype_api.rs +497 -0
- package/reader-backports/hdf5-reader/src/error.rs +167 -0
- package/reader-backports/hdf5-reader/src/extensible_array.rs +1886 -0
- package/reader-backports/hdf5-reader/src/filters/deflate.rs +93 -0
- package/reader-backports/hdf5-reader/src/filters/fletcher32.rs +52 -0
- package/reader-backports/hdf5-reader/src/filters/lz4.rs +178 -0
- package/reader-backports/hdf5-reader/src/filters/mod.rs +326 -0
- package/reader-backports/hdf5-reader/src/filters/nbit.rs +586 -0
- package/reader-backports/hdf5-reader/src/filters/scaleoffset.rs +827 -0
- package/reader-backports/hdf5-reader/src/filters/shuffle.rs +99 -0
- package/reader-backports/hdf5-reader/src/fixed_array.rs +763 -0
- package/reader-backports/hdf5-reader/src/fractal_heap.rs +2029 -0
- package/reader-backports/hdf5-reader/src/global_heap.rs +284 -0
- package/reader-backports/hdf5-reader/src/group.rs +795 -0
- package/reader-backports/hdf5-reader/src/io.rs +413 -0
- package/reader-backports/hdf5-reader/src/lib.rs +916 -0
- package/reader-backports/hdf5-reader/src/local_heap.rs +261 -0
- package/reader-backports/hdf5-reader/src/messages/attribute.rs +304 -0
- package/reader-backports/hdf5-reader/src/messages/attribute_info.rs +96 -0
- package/reader-backports/hdf5-reader/src/messages/btree_k.rs +36 -0
- package/reader-backports/hdf5-reader/src/messages/continuation.rs +46 -0
- package/reader-backports/hdf5-reader/src/messages/dataspace.rs +259 -0
- package/reader-backports/hdf5-reader/src/messages/datatype.rs +938 -0
- package/reader-backports/hdf5-reader/src/messages/external_files.rs +50 -0
- package/reader-backports/hdf5-reader/src/messages/fill_value.rs +233 -0
- package/reader-backports/hdf5-reader/src/messages/filter_pipeline.rs +147 -0
- package/reader-backports/hdf5-reader/src/messages/group_info.rs +103 -0
- package/reader-backports/hdf5-reader/src/messages/layout.rs +476 -0
- package/reader-backports/hdf5-reader/src/messages/link.rs +170 -0
- package/reader-backports/hdf5-reader/src/messages/link_info.rs +127 -0
- package/reader-backports/hdf5-reader/src/messages/mod.rs +374 -0
- package/reader-backports/hdf5-reader/src/messages/modification_time.rs +83 -0
- package/reader-backports/hdf5-reader/src/messages/shared.rs +164 -0
- package/reader-backports/hdf5-reader/src/messages/shared_table.rs +67 -0
- package/reader-backports/hdf5-reader/src/messages/symbol_table_msg.rs +51 -0
- package/reader-backports/hdf5-reader/src/object_header.rs +1749 -0
- package/reader-backports/hdf5-reader/src/reference.rs +90 -0
- package/reader-backports/hdf5-reader/src/shared_message_table.rs +425 -0
- package/reader-backports/hdf5-reader/src/storage.rs +538 -0
- package/reader-backports/hdf5-reader/src/superblock.rs +285 -0
- package/reader-backports/hdf5-reader/src/symbol_table.rs +277 -0
- package/reader-backports/netcdf-reader/Cargo.toml +141 -0
- package/reader-backports/netcdf-reader/LICENSE-APACHE +164 -0
- package/reader-backports/netcdf-reader/LICENSE-MIT +21 -0
- package/reader-backports/netcdf-reader/PROVENANCE.json +55 -0
- package/reader-backports/netcdf-reader/README.md +308 -0
- package/reader-backports/netcdf-reader/UPSTREAM-Cargo.toml +48 -0
- package/reader-backports/netcdf-reader/UPSTREAM-VCS.json +6 -0
- package/reader-backports/netcdf-reader/src/cf/axes.rs +419 -0
- package/reader-backports/netcdf-reader/src/cf/bounds.rs +108 -0
- package/reader-backports/netcdf-reader/src/cf/crs.rs +269 -0
- package/reader-backports/netcdf-reader/src/cf/mod.rs +30 -0
- package/reader-backports/netcdf-reader/src/cf/time.rs +1122 -0
- package/reader-backports/netcdf-reader/src/classic/data.rs +1465 -0
- package/reader-backports/netcdf-reader/src/classic/header.rs +1097 -0
- package/reader-backports/netcdf-reader/src/classic/mod.rs +240 -0
- package/reader-backports/netcdf-reader/src/classic/storage.rs +211 -0
- package/reader-backports/netcdf-reader/src/classic/types.rs +93 -0
- package/reader-backports/netcdf-reader/src/classic/variable.rs +1162 -0
- package/reader-backports/netcdf-reader/src/error.rs +59 -0
- package/reader-backports/netcdf-reader/src/lib.rs +1531 -0
- package/reader-backports/netcdf-reader/src/masked.rs +173 -0
- package/reader-backports/netcdf-reader/src/nc4/attributes.rs +203 -0
- package/reader-backports/netcdf-reader/src/nc4/dimensions.rs +139 -0
- package/reader-backports/netcdf-reader/src/nc4/groups.rs +308 -0
- package/reader-backports/netcdf-reader/src/nc4/mod.rs +572 -0
- package/reader-backports/netcdf-reader/src/nc4/types.rs +309 -0
- package/reader-backports/netcdf-reader/src/nc4/variables.rs +420 -0
- package/reader-backports/netcdf-reader/src/types.rs +18 -0
- package/reader-backports/netcdf-reader/src/unpack.rs +113 -0
- package/reader-backports/netcdf-reader/src/user_defined.rs +569 -0
|
@@ -0,0 +1,2029 @@
|
|
|
1
|
+
//! HDF5 Fractal Heap (FRHP).
|
|
2
|
+
//!
|
|
3
|
+
//! Fractal heaps are the storage mechanism for link messages and attribute
|
|
4
|
+
//! messages in new-style (v2) groups and datasets. They use a doubling-table
|
|
5
|
+
//! scheme with direct and indirect blocks. Objects are addressed by a
|
|
6
|
+
//! heap ID that encodes the offset and length within the heap.
|
|
7
|
+
//!
|
|
8
|
+
//! This module parses the heap header and provides object extraction for
|
|
9
|
+
//! managed, tiny, and unfiltered huge object IDs.
|
|
10
|
+
|
|
11
|
+
use std::collections::{HashMap, HashSet};
|
|
12
|
+
use std::sync::Arc;
|
|
13
|
+
|
|
14
|
+
use crate::checksum::jenkins_lookup3;
|
|
15
|
+
use crate::error::{Error, Result};
|
|
16
|
+
use crate::filters::{self, FilterRegistry};
|
|
17
|
+
use crate::io::Cursor;
|
|
18
|
+
use crate::messages::filter_pipeline::FilterPipelineMessage;
|
|
19
|
+
use crate::storage::Storage;
|
|
20
|
+
|
|
21
|
+
/// Signature bytes for a fractal heap header: ASCII `FRHP`.
|
|
22
|
+
const FRHP_SIGNATURE: [u8; 4] = *b"FRHP";
|
|
23
|
+
|
|
24
|
+
/// Signature bytes for a direct block: ASCII `FHDB`.
|
|
25
|
+
const _FHDB_SIGNATURE: [u8; 4] = *b"FHDB";
|
|
26
|
+
|
|
27
|
+
/// Signature bytes for an indirect block: ASCII `FHIB`.
|
|
28
|
+
const FHIB_SIGNATURE: [u8; 4] = *b"FHIB";
|
|
29
|
+
const MAX_FRACTAL_HEAP_INDIRECT_DEPTH: usize = 64;
|
|
30
|
+
const MAX_FRACTAL_HEAP_INDIRECT_ROWS: u16 = 64;
|
|
31
|
+
|
|
32
|
+
/// Parsed fractal heap header.
|
|
33
|
+
#[derive(Debug, Clone)]
|
|
34
|
+
pub struct FractalHeap {
|
|
35
|
+
/// Size in bytes of heap IDs used to reference objects.
|
|
36
|
+
pub heap_id_len: u16,
|
|
37
|
+
/// Size in bytes of I/O filter info (0 if none).
|
|
38
|
+
pub io_filters_len: u16,
|
|
39
|
+
/// Heap status flags.
|
|
40
|
+
pub flags: u8,
|
|
41
|
+
/// Maximum size of a managed object (larger objects become "huge").
|
|
42
|
+
pub max_managed_object_size: u64,
|
|
43
|
+
/// Next huge object ID to assign.
|
|
44
|
+
pub next_huge_id: u64,
|
|
45
|
+
/// Address of the B-tree v2 used for huge objects.
|
|
46
|
+
pub btree_huge_objects_address: u64,
|
|
47
|
+
/// Address of the free-space manager for managed blocks.
|
|
48
|
+
pub free_space_managed_address: u64,
|
|
49
|
+
/// Total managed space in bytes.
|
|
50
|
+
pub managed_space_amount: u64,
|
|
51
|
+
/// Total managed allocated space in bytes.
|
|
52
|
+
pub managed_alloc_amount: u64,
|
|
53
|
+
/// Iterator offset for managed free-space.
|
|
54
|
+
pub managed_iter_offset: u64,
|
|
55
|
+
/// Number of managed objects.
|
|
56
|
+
pub managed_objects_count: u64,
|
|
57
|
+
/// Total size of huge objects in bytes.
|
|
58
|
+
pub huge_objects_size: u64,
|
|
59
|
+
/// Number of huge objects.
|
|
60
|
+
pub huge_objects_count: u64,
|
|
61
|
+
/// Total size of tiny objects in bytes.
|
|
62
|
+
pub tiny_objects_size: u64,
|
|
63
|
+
/// Number of tiny objects.
|
|
64
|
+
pub tiny_objects_count: u64,
|
|
65
|
+
/// Width of the doubling table (number of direct blocks per row).
|
|
66
|
+
pub table_width: u16,
|
|
67
|
+
/// Size in bytes of the starting (smallest) direct block.
|
|
68
|
+
pub starting_block_size: u64,
|
|
69
|
+
/// Maximum direct block size before switching to indirect blocks.
|
|
70
|
+
pub max_direct_block_size: u64,
|
|
71
|
+
/// Log2 of the maximum managed heap size (used for heap ID encoding).
|
|
72
|
+
pub max_heap_size: u16,
|
|
73
|
+
/// Starting row of the root indirect block (for doubling table).
|
|
74
|
+
pub starting_row_root_indirect: u16,
|
|
75
|
+
/// Address of the root block (direct or indirect).
|
|
76
|
+
pub root_block_address: u64,
|
|
77
|
+
/// Current number of rows in the root indirect block.
|
|
78
|
+
pub current_rows_in_root_indirect: u16,
|
|
79
|
+
/// Filtered root direct block size (present only when io_filters_len > 0).
|
|
80
|
+
pub io_filter_size: Option<u64>,
|
|
81
|
+
/// Filter mask for root direct block (present only when io_filters_len > 0).
|
|
82
|
+
pub io_filter_mask: Option<u32>,
|
|
83
|
+
/// Encoded filter pipeline for heap blocks/huge objects.
|
|
84
|
+
pub io_filter_info: Vec<u8>,
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/// Cache of verified direct blocks for repeated object lookups in one heap.
|
|
88
|
+
#[derive(Debug, Default)]
|
|
89
|
+
pub struct FractalHeapDirectBlockCache {
|
|
90
|
+
blocks: HashMap<DirectBlockCacheKey, Arc<Vec<u8>>>,
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
type DirectBlockCacheKey = (u64, u64, Option<u64>, u32);
|
|
94
|
+
|
|
95
|
+
#[derive(Debug, Clone, Copy)]
|
|
96
|
+
struct DirectBlockLocation {
|
|
97
|
+
address: u64,
|
|
98
|
+
block_offset_in_heap: u64,
|
|
99
|
+
block_size: u64,
|
|
100
|
+
filtered_size: Option<u64>,
|
|
101
|
+
filter_mask: u32,
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
#[derive(Debug, Clone, Copy)]
|
|
105
|
+
struct HugeObjectLocation {
|
|
106
|
+
address: u64,
|
|
107
|
+
disk_length: u64,
|
|
108
|
+
filter_mask: u32,
|
|
109
|
+
memory_length: Option<u64>,
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
impl FractalHeapDirectBlockCache {
|
|
113
|
+
fn get_verified_block_storage(
|
|
114
|
+
&mut self,
|
|
115
|
+
heap: &FractalHeap,
|
|
116
|
+
location: DirectBlockLocation,
|
|
117
|
+
storage: &dyn Storage,
|
|
118
|
+
offset_size: u8,
|
|
119
|
+
filter_registry: Option<&FilterRegistry>,
|
|
120
|
+
) -> Result<Arc<Vec<u8>>> {
|
|
121
|
+
let key = (
|
|
122
|
+
location.address,
|
|
123
|
+
location.block_size,
|
|
124
|
+
location.filtered_size,
|
|
125
|
+
location.filter_mask,
|
|
126
|
+
);
|
|
127
|
+
if let Some(block) = self.blocks.get(&key) {
|
|
128
|
+
return Ok(block.clone());
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
let block =
|
|
132
|
+
heap.load_direct_block_storage(location, storage, offset_size, filter_registry)?;
|
|
133
|
+
heap.verify_direct_block_bytes(&block, offset_size)?;
|
|
134
|
+
let block = Arc::new(block);
|
|
135
|
+
self.blocks.insert(key, block.clone());
|
|
136
|
+
Ok(block)
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
impl FractalHeap {
|
|
141
|
+
/// Parse a fractal heap header at the current cursor position.
|
|
142
|
+
///
|
|
143
|
+
/// Format:
|
|
144
|
+
/// - Signature: `FRHP` (4 bytes)
|
|
145
|
+
/// - Version: 0 (1 byte)
|
|
146
|
+
/// - Heap ID length (u16 LE)
|
|
147
|
+
/// - I/O filters encoded length (u16 LE)
|
|
148
|
+
/// - Flags (u8)
|
|
149
|
+
/// - Max managed object size (u32 LE)
|
|
150
|
+
/// - Next huge object ID (`length_size` bytes)
|
|
151
|
+
/// - B-tree address for huge objects (`offset_size` bytes)
|
|
152
|
+
/// - Managed free-space amount (`length_size` bytes)
|
|
153
|
+
/// - Free-space manager address (`offset_size` bytes)
|
|
154
|
+
/// - Managed space amount (`length_size` bytes)
|
|
155
|
+
/// - Managed alloc amount (`length_size` bytes)
|
|
156
|
+
/// - Managed free-space iterator offset (`length_size` bytes)
|
|
157
|
+
/// - Managed objects count (`length_size` bytes)
|
|
158
|
+
/// - Huge objects size (`length_size` bytes)
|
|
159
|
+
/// - Huge objects count (`length_size` bytes)
|
|
160
|
+
/// - Tiny objects size (`length_size` bytes)
|
|
161
|
+
/// - Tiny objects count (`length_size` bytes)
|
|
162
|
+
/// - Table width (u16 LE)
|
|
163
|
+
/// - Starting block size (`length_size` bytes)
|
|
164
|
+
/// - Maximum direct block size (`length_size` bytes)
|
|
165
|
+
/// - Max heap size (u16 LE)
|
|
166
|
+
/// - Starting row of root indirect block (u16 LE)
|
|
167
|
+
/// - Root block address (`offset_size` bytes)
|
|
168
|
+
/// - Current rows in root indirect block (u16 LE)
|
|
169
|
+
/// - If io_filters_len > 0: filtered root direct block size (`length_size`), filter mask (u32 LE)
|
|
170
|
+
/// - Checksum (u32 LE)
|
|
171
|
+
pub fn parse(cursor: &mut Cursor, offset_size: u8, length_size: u8) -> Result<Self> {
|
|
172
|
+
let start = cursor.position();
|
|
173
|
+
|
|
174
|
+
let sig = cursor.read_bytes(4)?;
|
|
175
|
+
if sig != FRHP_SIGNATURE {
|
|
176
|
+
return Err(Error::InvalidFractalHeapSignature);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
let version = cursor.read_u8()?;
|
|
180
|
+
if version != 0 {
|
|
181
|
+
return Err(Error::UnsupportedFractalHeapVersion(version));
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
let heap_id_len = cursor.read_u16_le()?;
|
|
185
|
+
let io_filters_len = cursor.read_u16_le()?;
|
|
186
|
+
let flags = cursor.read_u8()?;
|
|
187
|
+
|
|
188
|
+
let max_managed_object_size = cursor.read_u32_le()? as u64;
|
|
189
|
+
let next_huge_id = cursor.read_length(length_size)?;
|
|
190
|
+
let btree_huge_objects_address = cursor.read_offset(offset_size)?;
|
|
191
|
+
let _managed_free_space_amount = cursor.read_length(length_size)?;
|
|
192
|
+
let free_space_managed_address = cursor.read_offset(offset_size)?;
|
|
193
|
+
let managed_space_amount = cursor.read_length(length_size)?;
|
|
194
|
+
let managed_alloc_amount = cursor.read_length(length_size)?;
|
|
195
|
+
let managed_iter_offset = cursor.read_length(length_size)?;
|
|
196
|
+
let managed_objects_count = cursor.read_length(length_size)?;
|
|
197
|
+
let huge_objects_size = cursor.read_length(length_size)?;
|
|
198
|
+
let huge_objects_count = cursor.read_length(length_size)?;
|
|
199
|
+
let tiny_objects_size = cursor.read_length(length_size)?;
|
|
200
|
+
let tiny_objects_count = cursor.read_length(length_size)?;
|
|
201
|
+
|
|
202
|
+
let table_width = cursor.read_u16_le()?;
|
|
203
|
+
let starting_block_size = cursor.read_length(length_size)?;
|
|
204
|
+
let max_direct_block_size = cursor.read_length(length_size)?;
|
|
205
|
+
let max_heap_size = cursor.read_u16_le()?;
|
|
206
|
+
let starting_row_root_indirect = cursor.read_u16_le()?;
|
|
207
|
+
let root_block_address = cursor.read_offset(offset_size)?;
|
|
208
|
+
let current_rows_in_root_indirect = cursor.read_u16_le()?;
|
|
209
|
+
|
|
210
|
+
let (io_filter_size, io_filter_mask) = if io_filters_len > 0 {
|
|
211
|
+
let size = cursor.read_length(length_size)?;
|
|
212
|
+
let mask = cursor.read_u32_le()?;
|
|
213
|
+
(Some(size), Some(mask))
|
|
214
|
+
} else {
|
|
215
|
+
(None, None)
|
|
216
|
+
};
|
|
217
|
+
let io_filter_info = if io_filters_len > 0 {
|
|
218
|
+
cursor.read_bytes(usize::from(io_filters_len))?.to_vec()
|
|
219
|
+
} else {
|
|
220
|
+
Vec::new()
|
|
221
|
+
};
|
|
222
|
+
|
|
223
|
+
// Verify checksum.
|
|
224
|
+
let checksum_end = cursor.position();
|
|
225
|
+
let stored_checksum = cursor.read_u32_le()?;
|
|
226
|
+
let computed = jenkins_lookup3(&cursor.data()[start as usize..checksum_end as usize]);
|
|
227
|
+
if computed != stored_checksum {
|
|
228
|
+
return Err(Error::ChecksumMismatch {
|
|
229
|
+
expected: stored_checksum,
|
|
230
|
+
actual: computed,
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
Ok(FractalHeap {
|
|
235
|
+
heap_id_len,
|
|
236
|
+
io_filters_len,
|
|
237
|
+
flags,
|
|
238
|
+
max_managed_object_size,
|
|
239
|
+
next_huge_id,
|
|
240
|
+
btree_huge_objects_address,
|
|
241
|
+
free_space_managed_address,
|
|
242
|
+
managed_space_amount,
|
|
243
|
+
managed_alloc_amount,
|
|
244
|
+
managed_iter_offset,
|
|
245
|
+
managed_objects_count,
|
|
246
|
+
huge_objects_size,
|
|
247
|
+
huge_objects_count,
|
|
248
|
+
tiny_objects_size,
|
|
249
|
+
tiny_objects_count,
|
|
250
|
+
table_width,
|
|
251
|
+
starting_block_size,
|
|
252
|
+
max_direct_block_size,
|
|
253
|
+
max_heap_size,
|
|
254
|
+
starting_row_root_indirect,
|
|
255
|
+
root_block_address,
|
|
256
|
+
current_rows_in_root_indirect,
|
|
257
|
+
io_filter_size,
|
|
258
|
+
io_filter_mask,
|
|
259
|
+
io_filter_info,
|
|
260
|
+
})
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/// Parse a fractal heap header from random-access storage.
|
|
264
|
+
pub fn parse_at_storage(
|
|
265
|
+
storage: &dyn Storage,
|
|
266
|
+
address: u64,
|
|
267
|
+
offset_size: u8,
|
|
268
|
+
length_size: u8,
|
|
269
|
+
) -> Result<Self> {
|
|
270
|
+
let max_header_len = 256usize;
|
|
271
|
+
let available = storage.len().saturating_sub(address);
|
|
272
|
+
let len = usize::try_from(available.min(max_header_len as u64)).map_err(|_| {
|
|
273
|
+
Error::InvalidData("fractal heap header exceeds platform usize capacity".into())
|
|
274
|
+
})?;
|
|
275
|
+
let bytes = storage.read_range(address, len)?;
|
|
276
|
+
let mut cursor = Cursor::new(bytes.as_ref());
|
|
277
|
+
Self::parse(&mut cursor, offset_size, length_size)
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/// Extract any fractal heap object given a heap ID.
|
|
281
|
+
pub fn get_object(
|
|
282
|
+
&self,
|
|
283
|
+
heap_id: &[u8],
|
|
284
|
+
file_data: &[u8],
|
|
285
|
+
offset_size: u8,
|
|
286
|
+
length_size: u8,
|
|
287
|
+
) -> Result<Vec<u8>> {
|
|
288
|
+
self.get_object_with_registry(heap_id, file_data, offset_size, length_size, None)
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/// Extract any fractal heap object using a caller-provided filter registry
|
|
292
|
+
/// for filtered managed and huge objects.
|
|
293
|
+
pub fn get_object_with_registry(
|
|
294
|
+
&self,
|
|
295
|
+
heap_id: &[u8],
|
|
296
|
+
file_data: &[u8],
|
|
297
|
+
offset_size: u8,
|
|
298
|
+
length_size: u8,
|
|
299
|
+
filter_registry: Option<&FilterRegistry>,
|
|
300
|
+
) -> Result<Vec<u8>> {
|
|
301
|
+
match self.heap_id_kind(heap_id)? {
|
|
302
|
+
HeapIdKind::Managed => self.get_managed_object_impl(
|
|
303
|
+
heap_id,
|
|
304
|
+
file_data,
|
|
305
|
+
offset_size,
|
|
306
|
+
length_size,
|
|
307
|
+
filter_registry,
|
|
308
|
+
),
|
|
309
|
+
HeapIdKind::Huge => self.get_huge_object(
|
|
310
|
+
heap_id,
|
|
311
|
+
file_data,
|
|
312
|
+
offset_size,
|
|
313
|
+
length_size,
|
|
314
|
+
filter_registry,
|
|
315
|
+
),
|
|
316
|
+
HeapIdKind::Tiny => self.decode_tiny_object(heap_id),
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/// Extract any fractal heap object from random-access storage.
|
|
321
|
+
pub fn get_object_storage(
|
|
322
|
+
&self,
|
|
323
|
+
heap_id: &[u8],
|
|
324
|
+
storage: &dyn Storage,
|
|
325
|
+
offset_size: u8,
|
|
326
|
+
length_size: u8,
|
|
327
|
+
) -> Result<Vec<u8>> {
|
|
328
|
+
self.get_object_storage_with_registry(heap_id, storage, offset_size, length_size, None)
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/// Extract any fractal heap object from random-access storage using a
|
|
332
|
+
/// caller-provided filter registry for filtered managed and huge objects.
|
|
333
|
+
pub fn get_object_storage_with_registry(
|
|
334
|
+
&self,
|
|
335
|
+
heap_id: &[u8],
|
|
336
|
+
storage: &dyn Storage,
|
|
337
|
+
offset_size: u8,
|
|
338
|
+
length_size: u8,
|
|
339
|
+
filter_registry: Option<&FilterRegistry>,
|
|
340
|
+
) -> Result<Vec<u8>> {
|
|
341
|
+
let mut cache = FractalHeapDirectBlockCache::default();
|
|
342
|
+
self.get_object_storage_cached_with_registry(
|
|
343
|
+
heap_id,
|
|
344
|
+
storage,
|
|
345
|
+
offset_size,
|
|
346
|
+
length_size,
|
|
347
|
+
&mut cache,
|
|
348
|
+
filter_registry,
|
|
349
|
+
)
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/// Extract any fractal heap object from storage, reusing verified direct
|
|
353
|
+
/// blocks across managed object lookups.
|
|
354
|
+
pub fn get_object_storage_cached(
|
|
355
|
+
&self,
|
|
356
|
+
heap_id: &[u8],
|
|
357
|
+
storage: &dyn Storage,
|
|
358
|
+
offset_size: u8,
|
|
359
|
+
length_size: u8,
|
|
360
|
+
direct_block_cache: &mut FractalHeapDirectBlockCache,
|
|
361
|
+
) -> Result<Vec<u8>> {
|
|
362
|
+
self.get_object_storage_cached_with_registry(
|
|
363
|
+
heap_id,
|
|
364
|
+
storage,
|
|
365
|
+
offset_size,
|
|
366
|
+
length_size,
|
|
367
|
+
direct_block_cache,
|
|
368
|
+
None,
|
|
369
|
+
)
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/// Extract any fractal heap object from storage while reusing verified
|
|
373
|
+
/// direct blocks and a caller-provided filter registry.
|
|
374
|
+
pub fn get_object_storage_cached_with_registry(
|
|
375
|
+
&self,
|
|
376
|
+
heap_id: &[u8],
|
|
377
|
+
storage: &dyn Storage,
|
|
378
|
+
offset_size: u8,
|
|
379
|
+
length_size: u8,
|
|
380
|
+
direct_block_cache: &mut FractalHeapDirectBlockCache,
|
|
381
|
+
filter_registry: Option<&FilterRegistry>,
|
|
382
|
+
) -> Result<Vec<u8>> {
|
|
383
|
+
match self.heap_id_kind(heap_id)? {
|
|
384
|
+
HeapIdKind::Managed => self.get_managed_object_storage_cached_impl(
|
|
385
|
+
heap_id,
|
|
386
|
+
storage,
|
|
387
|
+
offset_size,
|
|
388
|
+
length_size,
|
|
389
|
+
direct_block_cache,
|
|
390
|
+
filter_registry,
|
|
391
|
+
),
|
|
392
|
+
HeapIdKind::Huge => self.get_huge_object_storage(
|
|
393
|
+
heap_id,
|
|
394
|
+
storage,
|
|
395
|
+
offset_size,
|
|
396
|
+
length_size,
|
|
397
|
+
filter_registry,
|
|
398
|
+
),
|
|
399
|
+
HeapIdKind::Tiny => self.decode_tiny_object(heap_id),
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
/// Extract a fractal heap object. Kept for existing callers; now also
|
|
404
|
+
/// handles tiny and huge IDs.
|
|
405
|
+
pub fn get_managed_object(
|
|
406
|
+
&self,
|
|
407
|
+
heap_id: &[u8],
|
|
408
|
+
file_data: &[u8],
|
|
409
|
+
offset_size: u8,
|
|
410
|
+
length_size: u8,
|
|
411
|
+
) -> Result<Vec<u8>> {
|
|
412
|
+
self.get_object_with_registry(heap_id, file_data, offset_size, length_size, None)
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
/// Extract a fractal heap object from random-access storage. Kept for
|
|
416
|
+
/// existing callers; now also handles tiny and huge IDs.
|
|
417
|
+
pub fn get_managed_object_storage(
|
|
418
|
+
&self,
|
|
419
|
+
heap_id: &[u8],
|
|
420
|
+
storage: &dyn Storage,
|
|
421
|
+
offset_size: u8,
|
|
422
|
+
length_size: u8,
|
|
423
|
+
) -> Result<Vec<u8>> {
|
|
424
|
+
self.get_object_storage_with_registry(heap_id, storage, offset_size, length_size, None)
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
fn get_managed_object_impl(
|
|
428
|
+
&self,
|
|
429
|
+
heap_id: &[u8],
|
|
430
|
+
file_data: &[u8],
|
|
431
|
+
offset_size: u8,
|
|
432
|
+
length_size: u8,
|
|
433
|
+
filter_registry: Option<&FilterRegistry>,
|
|
434
|
+
) -> Result<Vec<u8>> {
|
|
435
|
+
let (heap_offset, obj_length) = self.decode_managed_heap_id(heap_id)?;
|
|
436
|
+
|
|
437
|
+
if obj_length == 0 {
|
|
438
|
+
return Ok(Vec::new());
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
let location = self.find_direct_block(heap_offset, file_data, offset_size, length_size)?;
|
|
442
|
+
let block = self.load_direct_block(file_data, location, filter_registry)?;
|
|
443
|
+
self.verify_direct_block_bytes(&block, offset_size)?;
|
|
444
|
+
let offset_in_block = heap_offset
|
|
445
|
+
.checked_sub(location.block_offset_in_heap)
|
|
446
|
+
.ok_or_else(|| {
|
|
447
|
+
Error::InvalidData("fractal heap object precedes direct block".into())
|
|
448
|
+
})?;
|
|
449
|
+
let data_start = usize::try_from(offset_in_block)
|
|
450
|
+
.map_err(|_| Error::InvalidData("fractal heap object offset exceeds usize".into()))?;
|
|
451
|
+
let len = usize::try_from(obj_length).map_err(|_| {
|
|
452
|
+
Error::InvalidData("fractal heap object exceeds platform usize capacity".into())
|
|
453
|
+
})?;
|
|
454
|
+
let data_end = data_start
|
|
455
|
+
.checked_add(len)
|
|
456
|
+
.ok_or(Error::OffsetOutOfBounds(location.address))?;
|
|
457
|
+
|
|
458
|
+
if data_end > block.len() {
|
|
459
|
+
return Err(Error::UnexpectedEof {
|
|
460
|
+
offset: location
|
|
461
|
+
.address
|
|
462
|
+
.checked_add(offset_in_block)
|
|
463
|
+
.ok_or(Error::OffsetOutOfBounds(location.address))?,
|
|
464
|
+
needed: obj_length,
|
|
465
|
+
available: block.len().saturating_sub(data_start) as u64,
|
|
466
|
+
});
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
Ok(block[data_start..data_end].to_vec())
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
fn get_managed_object_storage_cached_impl(
|
|
473
|
+
&self,
|
|
474
|
+
heap_id: &[u8],
|
|
475
|
+
storage: &dyn Storage,
|
|
476
|
+
offset_size: u8,
|
|
477
|
+
length_size: u8,
|
|
478
|
+
direct_block_cache: &mut FractalHeapDirectBlockCache,
|
|
479
|
+
filter_registry: Option<&FilterRegistry>,
|
|
480
|
+
) -> Result<Vec<u8>> {
|
|
481
|
+
let (heap_offset, obj_length) = self.decode_managed_heap_id(heap_id)?;
|
|
482
|
+
if obj_length == 0 {
|
|
483
|
+
return Ok(Vec::new());
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
let location =
|
|
487
|
+
self.find_direct_block_storage(heap_offset, storage, offset_size, length_size)?;
|
|
488
|
+
let block = direct_block_cache.get_verified_block_storage(
|
|
489
|
+
self,
|
|
490
|
+
location,
|
|
491
|
+
storage,
|
|
492
|
+
offset_size,
|
|
493
|
+
filter_registry,
|
|
494
|
+
)?;
|
|
495
|
+
let offset_in_block = heap_offset
|
|
496
|
+
.checked_sub(location.block_offset_in_heap)
|
|
497
|
+
.ok_or_else(|| {
|
|
498
|
+
Error::InvalidData("fractal heap object precedes direct block".into())
|
|
499
|
+
})?;
|
|
500
|
+
let start = usize::try_from(offset_in_block)
|
|
501
|
+
.map_err(|_| Error::InvalidData("fractal heap object offset exceeds usize".into()))?;
|
|
502
|
+
let len = usize::try_from(obj_length).map_err(|_| {
|
|
503
|
+
Error::InvalidData("fractal heap object exceeds platform usize capacity".into())
|
|
504
|
+
})?;
|
|
505
|
+
let end = start
|
|
506
|
+
.checked_add(len)
|
|
507
|
+
.ok_or(Error::OffsetOutOfBounds(location.address))?;
|
|
508
|
+
if end > block.len() {
|
|
509
|
+
let data_start = location
|
|
510
|
+
.address
|
|
511
|
+
.checked_add(offset_in_block)
|
|
512
|
+
.ok_or(Error::OffsetOutOfBounds(location.address))?;
|
|
513
|
+
return Err(Error::UnexpectedEof {
|
|
514
|
+
offset: data_start,
|
|
515
|
+
needed: obj_length,
|
|
516
|
+
available: block.len().saturating_sub(start) as u64,
|
|
517
|
+
});
|
|
518
|
+
}
|
|
519
|
+
Ok(block[start..end].to_vec())
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
fn get_huge_object(
|
|
523
|
+
&self,
|
|
524
|
+
heap_id: &[u8],
|
|
525
|
+
file_data: &[u8],
|
|
526
|
+
offset_size: u8,
|
|
527
|
+
length_size: u8,
|
|
528
|
+
filter_registry: Option<&FilterRegistry>,
|
|
529
|
+
) -> Result<Vec<u8>> {
|
|
530
|
+
let location = self.resolve_huge_object_location(
|
|
531
|
+
heap_id,
|
|
532
|
+
Some(file_data),
|
|
533
|
+
None,
|
|
534
|
+
offset_size,
|
|
535
|
+
length_size,
|
|
536
|
+
)?;
|
|
537
|
+
let start = usize::try_from(location.address)
|
|
538
|
+
.map_err(|_| Error::OffsetOutOfBounds(location.address))?;
|
|
539
|
+
let len = usize::try_from(location.disk_length).map_err(|_| {
|
|
540
|
+
Error::InvalidData("huge fractal heap object exceeds platform usize capacity".into())
|
|
541
|
+
})?;
|
|
542
|
+
let end = start
|
|
543
|
+
.checked_add(len)
|
|
544
|
+
.ok_or(Error::OffsetOutOfBounds(location.address))?;
|
|
545
|
+
if end > file_data.len() {
|
|
546
|
+
return Err(Error::UnexpectedEof {
|
|
547
|
+
offset: location.address,
|
|
548
|
+
needed: location.disk_length,
|
|
549
|
+
available: file_data.len().saturating_sub(start) as u64,
|
|
550
|
+
});
|
|
551
|
+
}
|
|
552
|
+
self.decode_huge_object_bytes(&file_data[start..end], location, filter_registry)
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
fn get_huge_object_storage(
|
|
556
|
+
&self,
|
|
557
|
+
heap_id: &[u8],
|
|
558
|
+
storage: &dyn Storage,
|
|
559
|
+
offset_size: u8,
|
|
560
|
+
length_size: u8,
|
|
561
|
+
filter_registry: Option<&FilterRegistry>,
|
|
562
|
+
) -> Result<Vec<u8>> {
|
|
563
|
+
let location = self.resolve_huge_object_location(
|
|
564
|
+
heap_id,
|
|
565
|
+
None,
|
|
566
|
+
Some(storage),
|
|
567
|
+
offset_size,
|
|
568
|
+
length_size,
|
|
569
|
+
)?;
|
|
570
|
+
let len = usize::try_from(location.disk_length).map_err(|_| {
|
|
571
|
+
Error::InvalidData("huge fractal heap object exceeds platform usize capacity".into())
|
|
572
|
+
})?;
|
|
573
|
+
let bytes = storage.read_range(location.address, len)?;
|
|
574
|
+
self.decode_huge_object_bytes(bytes.as_ref(), location, filter_registry)
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
fn decode_huge_object_bytes(
|
|
578
|
+
&self,
|
|
579
|
+
bytes: &[u8],
|
|
580
|
+
location: HugeObjectLocation,
|
|
581
|
+
filter_registry: Option<&FilterRegistry>,
|
|
582
|
+
) -> Result<Vec<u8>> {
|
|
583
|
+
if let Some(memory_length) = location.memory_length {
|
|
584
|
+
self.apply_heap_filters(
|
|
585
|
+
bytes,
|
|
586
|
+
location.filter_mask,
|
|
587
|
+
memory_length,
|
|
588
|
+
"filtered fractal heap huge object",
|
|
589
|
+
filter_registry,
|
|
590
|
+
)
|
|
591
|
+
} else {
|
|
592
|
+
Ok(bytes.to_vec())
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
fn load_direct_block(
|
|
597
|
+
&self,
|
|
598
|
+
file_data: &[u8],
|
|
599
|
+
location: DirectBlockLocation,
|
|
600
|
+
filter_registry: Option<&FilterRegistry>,
|
|
601
|
+
) -> Result<Vec<u8>> {
|
|
602
|
+
let read_len = location.filtered_size.unwrap_or(location.block_size);
|
|
603
|
+
let start = usize::try_from(location.address)
|
|
604
|
+
.map_err(|_| Error::OffsetOutOfBounds(location.address))?;
|
|
605
|
+
let len = usize::try_from(read_len).map_err(|_| {
|
|
606
|
+
Error::InvalidData("fractal heap direct block size exceeds platform usize".into())
|
|
607
|
+
})?;
|
|
608
|
+
let end = start
|
|
609
|
+
.checked_add(len)
|
|
610
|
+
.ok_or(Error::OffsetOutOfBounds(location.address))?;
|
|
611
|
+
if end > file_data.len() {
|
|
612
|
+
return Err(Error::UnexpectedEof {
|
|
613
|
+
offset: location.address,
|
|
614
|
+
needed: read_len,
|
|
615
|
+
available: file_data.len().saturating_sub(start) as u64,
|
|
616
|
+
});
|
|
617
|
+
}
|
|
618
|
+
let block = if location.filtered_size.is_some() {
|
|
619
|
+
self.apply_heap_filters(
|
|
620
|
+
&file_data[start..end],
|
|
621
|
+
location.filter_mask,
|
|
622
|
+
location.block_size,
|
|
623
|
+
"filtered fractal heap direct block",
|
|
624
|
+
filter_registry,
|
|
625
|
+
)?
|
|
626
|
+
} else {
|
|
627
|
+
file_data[start..end].to_vec()
|
|
628
|
+
};
|
|
629
|
+
let expected = usize::try_from(location.block_size).map_err(|_| {
|
|
630
|
+
Error::InvalidData("fractal heap direct block size exceeds platform usize".into())
|
|
631
|
+
})?;
|
|
632
|
+
if block.len() != expected {
|
|
633
|
+
return Err(Error::InvalidData(format!(
|
|
634
|
+
"fractal heap direct block has {} bytes, expected {} bytes",
|
|
635
|
+
block.len(),
|
|
636
|
+
expected
|
|
637
|
+
)));
|
|
638
|
+
}
|
|
639
|
+
Ok(block)
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
fn load_direct_block_storage(
|
|
643
|
+
&self,
|
|
644
|
+
location: DirectBlockLocation,
|
|
645
|
+
storage: &dyn Storage,
|
|
646
|
+
_offset_size: u8,
|
|
647
|
+
filter_registry: Option<&FilterRegistry>,
|
|
648
|
+
) -> Result<Vec<u8>> {
|
|
649
|
+
let read_len = location.filtered_size.unwrap_or(location.block_size);
|
|
650
|
+
let len = usize::try_from(read_len).map_err(|_| {
|
|
651
|
+
Error::InvalidData("fractal heap direct block size exceeds platform usize".into())
|
|
652
|
+
})?;
|
|
653
|
+
let bytes = storage.read_range(location.address, len)?;
|
|
654
|
+
let block = if location.filtered_size.is_some() {
|
|
655
|
+
self.apply_heap_filters(
|
|
656
|
+
bytes.as_ref(),
|
|
657
|
+
location.filter_mask,
|
|
658
|
+
location.block_size,
|
|
659
|
+
"filtered fractal heap direct block",
|
|
660
|
+
filter_registry,
|
|
661
|
+
)?
|
|
662
|
+
} else {
|
|
663
|
+
bytes.to_vec()
|
|
664
|
+
};
|
|
665
|
+
let expected = usize::try_from(location.block_size).map_err(|_| {
|
|
666
|
+
Error::InvalidData("fractal heap direct block size exceeds platform usize".into())
|
|
667
|
+
})?;
|
|
668
|
+
if block.len() != expected {
|
|
669
|
+
return Err(Error::InvalidData(format!(
|
|
670
|
+
"fractal heap direct block has {} bytes, expected {} bytes",
|
|
671
|
+
block.len(),
|
|
672
|
+
expected
|
|
673
|
+
)));
|
|
674
|
+
}
|
|
675
|
+
Ok(block)
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
fn apply_heap_filters(
|
|
679
|
+
&self,
|
|
680
|
+
bytes: &[u8],
|
|
681
|
+
filter_mask: u32,
|
|
682
|
+
expected_len: u64,
|
|
683
|
+
context: &str,
|
|
684
|
+
filter_registry: Option<&FilterRegistry>,
|
|
685
|
+
) -> Result<Vec<u8>> {
|
|
686
|
+
let pipeline = self.filter_pipeline()?;
|
|
687
|
+
let expected = usize::try_from(expected_len).map_err(|_| {
|
|
688
|
+
Error::InvalidData(format!("{context} size exceeds platform usize capacity"))
|
|
689
|
+
})?;
|
|
690
|
+
let filter_output_limit = expected.checked_add(1).ok_or_else(|| {
|
|
691
|
+
Error::InvalidData(format!(
|
|
692
|
+
"{context} filter output limit exceeds platform usize capacity"
|
|
693
|
+
))
|
|
694
|
+
})?;
|
|
695
|
+
let decoded = filters::apply_pipeline_with_limit(
|
|
696
|
+
bytes,
|
|
697
|
+
&pipeline.filters,
|
|
698
|
+
filter_mask,
|
|
699
|
+
1,
|
|
700
|
+
filter_registry,
|
|
701
|
+
Some(filter_output_limit),
|
|
702
|
+
)?;
|
|
703
|
+
if decoded.len() != expected {
|
|
704
|
+
return Err(Error::InvalidData(format!(
|
|
705
|
+
"{context} decoded to {} bytes, expected {} bytes",
|
|
706
|
+
decoded.len(),
|
|
707
|
+
expected
|
|
708
|
+
)));
|
|
709
|
+
}
|
|
710
|
+
Ok(decoded)
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
fn filter_pipeline(&self) -> Result<FilterPipelineMessage> {
|
|
714
|
+
if self.io_filters_len == 0 {
|
|
715
|
+
return Err(Error::InvalidData(
|
|
716
|
+
"fractal heap object is marked filtered but the heap has no filter pipeline".into(),
|
|
717
|
+
));
|
|
718
|
+
}
|
|
719
|
+
let mut cursor = Cursor::new(&self.io_filter_info);
|
|
720
|
+
crate::messages::filter_pipeline::parse(&mut cursor, 0, 0, self.io_filter_info.len())
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
fn resolve_huge_object_location(
|
|
724
|
+
&self,
|
|
725
|
+
heap_id: &[u8],
|
|
726
|
+
file_data: Option<&[u8]>,
|
|
727
|
+
storage: Option<&dyn Storage>,
|
|
728
|
+
offset_size: u8,
|
|
729
|
+
length_size: u8,
|
|
730
|
+
) -> Result<HugeObjectLocation> {
|
|
731
|
+
let direct_unfiltered_len = 1 + usize::from(offset_size) + usize::from(length_size);
|
|
732
|
+
let direct_filtered_len = direct_unfiltered_len + 4 + usize::from(length_size);
|
|
733
|
+
|
|
734
|
+
if self.io_filters_len > 0 && heap_id.len() >= direct_filtered_len {
|
|
735
|
+
let mut cursor = Cursor::new(&heap_id[1..]);
|
|
736
|
+
let address = cursor.read_offset(offset_size)?;
|
|
737
|
+
let disk_length = cursor.read_length(length_size)?;
|
|
738
|
+
let filter_mask = cursor.read_u32_le()?;
|
|
739
|
+
let memory_length = cursor.read_length(length_size)?;
|
|
740
|
+
return Ok(HugeObjectLocation {
|
|
741
|
+
address,
|
|
742
|
+
disk_length,
|
|
743
|
+
filter_mask,
|
|
744
|
+
memory_length: Some(memory_length),
|
|
745
|
+
});
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
if self.io_filters_len == 0 && heap_id.len() >= direct_unfiltered_len {
|
|
749
|
+
let mut cursor = Cursor::new(&heap_id[1..]);
|
|
750
|
+
let address = cursor.read_offset(offset_size)?;
|
|
751
|
+
let disk_length = cursor.read_length(length_size)?;
|
|
752
|
+
return Ok(HugeObjectLocation {
|
|
753
|
+
address,
|
|
754
|
+
disk_length,
|
|
755
|
+
filter_mask: 0,
|
|
756
|
+
memory_length: None,
|
|
757
|
+
});
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
// HDF5 (H5HFhuge.c) keys an indirect huge object by
|
|
761
|
+
// min(id_len - 1, sizeof(hsize_t)) bytes.
|
|
762
|
+
let key_len = (heap_id.len().saturating_sub(1)).min(8);
|
|
763
|
+
if key_len == 0 {
|
|
764
|
+
return Err(Error::InvalidData(
|
|
765
|
+
"huge fractal heap ID is too short".into(),
|
|
766
|
+
));
|
|
767
|
+
}
|
|
768
|
+
if Cursor::is_undefined_offset(self.btree_huge_objects_address, offset_size) {
|
|
769
|
+
return Err(Error::UndefinedAddress);
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
let mut key_cursor = Cursor::new(&heap_id[1..]);
|
|
773
|
+
let object_id = key_cursor.read_uvar(key_len)?;
|
|
774
|
+
|
|
775
|
+
let header = if let Some(storage) = storage {
|
|
776
|
+
crate::btree_v2::BTreeV2Header::parse_at_storage(
|
|
777
|
+
storage,
|
|
778
|
+
self.btree_huge_objects_address,
|
|
779
|
+
offset_size,
|
|
780
|
+
length_size,
|
|
781
|
+
)?
|
|
782
|
+
} else {
|
|
783
|
+
let data = file_data.expect("file_data must exist when storage is None");
|
|
784
|
+
let mut cursor = Cursor::new(data);
|
|
785
|
+
cursor.set_position(self.btree_huge_objects_address);
|
|
786
|
+
crate::btree_v2::BTreeV2Header::parse(&mut cursor, offset_size, length_size)?
|
|
787
|
+
};
|
|
788
|
+
|
|
789
|
+
let records = if let Some(storage) = storage {
|
|
790
|
+
crate::btree_v2::collect_btree_v2_records_storage(
|
|
791
|
+
storage,
|
|
792
|
+
&header,
|
|
793
|
+
offset_size,
|
|
794
|
+
length_size,
|
|
795
|
+
None,
|
|
796
|
+
&[],
|
|
797
|
+
None,
|
|
798
|
+
)?
|
|
799
|
+
} else {
|
|
800
|
+
crate::btree_v2::collect_btree_v2_records(
|
|
801
|
+
file_data.expect("file_data must exist when storage is None"),
|
|
802
|
+
&header,
|
|
803
|
+
offset_size,
|
|
804
|
+
length_size,
|
|
805
|
+
None,
|
|
806
|
+
&[],
|
|
807
|
+
None,
|
|
808
|
+
)?
|
|
809
|
+
};
|
|
810
|
+
|
|
811
|
+
for record in records {
|
|
812
|
+
match record {
|
|
813
|
+
crate::btree_v2::BTreeV2Record::HugeIndirectNonFiltered {
|
|
814
|
+
address,
|
|
815
|
+
length,
|
|
816
|
+
object_id: record_id,
|
|
817
|
+
} if record_id == object_id => {
|
|
818
|
+
return Ok(HugeObjectLocation {
|
|
819
|
+
address,
|
|
820
|
+
disk_length: length,
|
|
821
|
+
filter_mask: 0,
|
|
822
|
+
memory_length: None,
|
|
823
|
+
})
|
|
824
|
+
}
|
|
825
|
+
crate::btree_v2::BTreeV2Record::HugeIndirectFiltered {
|
|
826
|
+
object_id: record_id,
|
|
827
|
+
address,
|
|
828
|
+
filtered_length,
|
|
829
|
+
filter_mask,
|
|
830
|
+
memory_length,
|
|
831
|
+
} if record_id == object_id => {
|
|
832
|
+
return Ok(HugeObjectLocation {
|
|
833
|
+
address,
|
|
834
|
+
disk_length: filtered_length,
|
|
835
|
+
filter_mask,
|
|
836
|
+
memory_length: Some(memory_length),
|
|
837
|
+
});
|
|
838
|
+
}
|
|
839
|
+
_ => {}
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
Err(Error::InvalidData(format!(
|
|
844
|
+
"huge fractal heap object ID {} not found",
|
|
845
|
+
object_id
|
|
846
|
+
)))
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
fn decode_tiny_object(&self, heap_id: &[u8]) -> Result<Vec<u8>> {
|
|
850
|
+
let extended = self.heap_id_len > 18;
|
|
851
|
+
let (data_start, len) = if extended {
|
|
852
|
+
if heap_id.len() < 2 {
|
|
853
|
+
return Err(Error::InvalidData(
|
|
854
|
+
"extended tiny heap ID is too short".into(),
|
|
855
|
+
));
|
|
856
|
+
}
|
|
857
|
+
let encoded = (u16::from(heap_id[0] & 0x0F) << 8) | u16::from(heap_id[1]);
|
|
858
|
+
(2usize, usize::from(encoded) + 1)
|
|
859
|
+
} else {
|
|
860
|
+
(1usize, usize::from(heap_id[0] & 0x0F) + 1)
|
|
861
|
+
};
|
|
862
|
+
let data_end = data_start
|
|
863
|
+
.checked_add(len)
|
|
864
|
+
.ok_or_else(|| Error::InvalidData("tiny heap object length overflows".into()))?;
|
|
865
|
+
if data_end > heap_id.len() {
|
|
866
|
+
return Err(Error::InvalidData(format!(
|
|
867
|
+
"tiny heap object needs {} bytes, heap ID has {}",
|
|
868
|
+
data_end,
|
|
869
|
+
heap_id.len()
|
|
870
|
+
)));
|
|
871
|
+
}
|
|
872
|
+
Ok(heap_id[data_start..data_end].to_vec())
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
fn heap_id_kind(&self, heap_id: &[u8]) -> Result<HeapIdKind> {
|
|
876
|
+
if heap_id.is_empty() {
|
|
877
|
+
return Err(Error::InvalidData("empty fractal heap ID".into()));
|
|
878
|
+
}
|
|
879
|
+
let version = heap_id[0] >> 6;
|
|
880
|
+
if version != 0 {
|
|
881
|
+
return Err(Error::InvalidData(format!(
|
|
882
|
+
"unsupported fractal heap ID version {}",
|
|
883
|
+
version
|
|
884
|
+
)));
|
|
885
|
+
}
|
|
886
|
+
match (heap_id[0] >> 4) & 0x03 {
|
|
887
|
+
0 => Ok(HeapIdKind::Managed),
|
|
888
|
+
1 => Ok(HeapIdKind::Huge),
|
|
889
|
+
2 => Ok(HeapIdKind::Tiny),
|
|
890
|
+
other => Err(Error::InvalidData(format!(
|
|
891
|
+
"unknown fractal heap ID type {}",
|
|
892
|
+
other
|
|
893
|
+
))),
|
|
894
|
+
}
|
|
895
|
+
}
|
|
896
|
+
|
|
897
|
+
fn decode_managed_heap_id(&self, heap_id: &[u8]) -> Result<(u64, u64)> {
|
|
898
|
+
let (offset_bytes, length_bytes) = self.managed_id_widths();
|
|
899
|
+
let needed = 1 + offset_bytes + length_bytes;
|
|
900
|
+
if heap_id.len() < needed {
|
|
901
|
+
return Err(Error::InvalidData(format!(
|
|
902
|
+
"managed fractal heap ID too short: need {} bytes, have {}",
|
|
903
|
+
needed,
|
|
904
|
+
heap_id.len()
|
|
905
|
+
)));
|
|
906
|
+
}
|
|
907
|
+
let mut cursor = Cursor::new(&heap_id[1..needed]);
|
|
908
|
+
let heap_offset = cursor.read_uvar(offset_bytes)?;
|
|
909
|
+
let obj_length = cursor.read_uvar(length_bytes)?;
|
|
910
|
+
Ok((heap_offset, obj_length))
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
fn managed_id_widths(&self) -> (usize, usize) {
|
|
914
|
+
let offset_bytes = usize::from(self.max_heap_size).div_ceil(8).max(1);
|
|
915
|
+
let max_len = self.max_direct_block_size.min(self.max_managed_object_size);
|
|
916
|
+
let length_bytes = bytes_needed_to_encode(max_len).max(1);
|
|
917
|
+
(offset_bytes, length_bytes)
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
/// Find the direct block containing a given heap offset.
|
|
921
|
+
///
|
|
922
|
+
/// Returns (block_file_address, block_offset_within_heap, block_size).
|
|
923
|
+
fn find_direct_block(
|
|
924
|
+
&self,
|
|
925
|
+
heap_offset: u64,
|
|
926
|
+
file_data: &[u8],
|
|
927
|
+
offset_size: u8,
|
|
928
|
+
length_size: u8,
|
|
929
|
+
) -> Result<DirectBlockLocation> {
|
|
930
|
+
if Cursor::is_undefined_offset(self.root_block_address, offset_size) {
|
|
931
|
+
return Err(Error::UndefinedAddress);
|
|
932
|
+
}
|
|
933
|
+
|
|
934
|
+
if self.current_rows_in_root_indirect == 0 {
|
|
935
|
+
// Root block is a direct block.
|
|
936
|
+
// The entire managed space is in this one block.
|
|
937
|
+
Ok(DirectBlockLocation {
|
|
938
|
+
address: self.root_block_address,
|
|
939
|
+
block_offset_in_heap: 0,
|
|
940
|
+
block_size: self.starting_block_size,
|
|
941
|
+
filtered_size: self.io_filter_size,
|
|
942
|
+
filter_mask: self.io_filter_mask.unwrap_or(0),
|
|
943
|
+
})
|
|
944
|
+
} else {
|
|
945
|
+
// Root block is an indirect block — traverse the doubling table.
|
|
946
|
+
let mut visited = HashSet::new();
|
|
947
|
+
self.find_direct_block_via_indirect(
|
|
948
|
+
self.root_block_address,
|
|
949
|
+
heap_offset,
|
|
950
|
+
file_data,
|
|
951
|
+
offset_size,
|
|
952
|
+
length_size,
|
|
953
|
+
self.current_rows_in_root_indirect,
|
|
954
|
+
MAX_FRACTAL_HEAP_INDIRECT_DEPTH,
|
|
955
|
+
&mut visited,
|
|
956
|
+
)
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
fn find_direct_block_storage(
|
|
961
|
+
&self,
|
|
962
|
+
heap_offset: u64,
|
|
963
|
+
storage: &dyn Storage,
|
|
964
|
+
offset_size: u8,
|
|
965
|
+
length_size: u8,
|
|
966
|
+
) -> Result<DirectBlockLocation> {
|
|
967
|
+
if Cursor::is_undefined_offset(self.root_block_address, offset_size) {
|
|
968
|
+
return Err(Error::UndefinedAddress);
|
|
969
|
+
}
|
|
970
|
+
|
|
971
|
+
if self.current_rows_in_root_indirect == 0 {
|
|
972
|
+
Ok(DirectBlockLocation {
|
|
973
|
+
address: self.root_block_address,
|
|
974
|
+
block_offset_in_heap: 0,
|
|
975
|
+
block_size: self.starting_block_size,
|
|
976
|
+
filtered_size: self.io_filter_size,
|
|
977
|
+
filter_mask: self.io_filter_mask.unwrap_or(0),
|
|
978
|
+
})
|
|
979
|
+
} else {
|
|
980
|
+
let mut visited = HashSet::new();
|
|
981
|
+
self.find_direct_block_via_indirect_storage(
|
|
982
|
+
self.root_block_address,
|
|
983
|
+
heap_offset,
|
|
984
|
+
storage,
|
|
985
|
+
offset_size,
|
|
986
|
+
length_size,
|
|
987
|
+
self.current_rows_in_root_indirect,
|
|
988
|
+
MAX_FRACTAL_HEAP_INDIRECT_DEPTH,
|
|
989
|
+
&mut visited,
|
|
990
|
+
)
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
|
|
994
|
+
/// Traverse an indirect block to find the direct block for a given offset.
|
|
995
|
+
#[allow(clippy::too_many_arguments)]
|
|
996
|
+
fn find_direct_block_via_indirect(
|
|
997
|
+
&self,
|
|
998
|
+
indirect_address: u64,
|
|
999
|
+
heap_offset: u64,
|
|
1000
|
+
file_data: &[u8],
|
|
1001
|
+
offset_size: u8,
|
|
1002
|
+
length_size: u8,
|
|
1003
|
+
nrows: u16,
|
|
1004
|
+
depth_remaining: usize,
|
|
1005
|
+
visited: &mut HashSet<u64>,
|
|
1006
|
+
) -> Result<DirectBlockLocation> {
|
|
1007
|
+
self.enter_indirect_block(indirect_address, nrows, depth_remaining, visited)?;
|
|
1008
|
+
|
|
1009
|
+
// Validate FHIB signature
|
|
1010
|
+
let addr = usize::try_from(indirect_address)
|
|
1011
|
+
.map_err(|_| Error::OffsetOutOfBounds(indirect_address))?;
|
|
1012
|
+
if addr
|
|
1013
|
+
.checked_add(4)
|
|
1014
|
+
.map_or(true, |end| end > file_data.len())
|
|
1015
|
+
{
|
|
1016
|
+
return Err(Error::OffsetOutOfBounds(indirect_address));
|
|
1017
|
+
}
|
|
1018
|
+
if file_data[addr..addr + 4] != FHIB_SIGNATURE {
|
|
1019
|
+
return Err(Error::InvalidData(format!(
|
|
1020
|
+
"expected FHIB signature at offset {:#x}, got {:?}",
|
|
1021
|
+
indirect_address,
|
|
1022
|
+
&file_data[addr..addr + 4]
|
|
1023
|
+
)));
|
|
1024
|
+
}
|
|
1025
|
+
|
|
1026
|
+
// The doubling table has `table_width` entries per row.
|
|
1027
|
+
// Row 0 and 1 have blocks of size `starting_block_size`.
|
|
1028
|
+
// Row r (for r >= 1) has blocks of size `starting_block_size * 2^(r-1)`.
|
|
1029
|
+
//
|
|
1030
|
+
// We iterate through the rows to find which block contains the
|
|
1031
|
+
// target offset, then read the block address from the indirect block.
|
|
1032
|
+
|
|
1033
|
+
let width = self.table_width as u64;
|
|
1034
|
+
let mut running_offset: u64 = 0;
|
|
1035
|
+
|
|
1036
|
+
for row in 0..nrows as u64 {
|
|
1037
|
+
let block_size = self.block_size_for_row_checked(row)?;
|
|
1038
|
+
let is_direct = block_size <= self.max_direct_block_size;
|
|
1039
|
+
|
|
1040
|
+
for col in 0..width {
|
|
1041
|
+
let block_end = running_offset.checked_add(block_size).ok_or_else(|| {
|
|
1042
|
+
Error::InvalidData("fractal heap indirect block offset overflows u64".into())
|
|
1043
|
+
})?;
|
|
1044
|
+
if heap_offset >= running_offset && heap_offset < block_end {
|
|
1045
|
+
// This is the block we want. Read its address from the
|
|
1046
|
+
// indirect block.
|
|
1047
|
+
let entry_index = row * width + col;
|
|
1048
|
+
|
|
1049
|
+
// Indirect block layout: signature(4) + version(1) +
|
|
1050
|
+
// heap_header_addr(offset_size) + block_offset(max_heap_size/8 rounded up)
|
|
1051
|
+
// Then direct-block entries, optionally with filtered size
|
|
1052
|
+
// and mask, followed by child indirect-block addresses.
|
|
1053
|
+
let iblock_header_size =
|
|
1054
|
+
4 + 1 + offset_size as u64 + (self.max_heap_size as u64).div_ceil(8);
|
|
1055
|
+
|
|
1056
|
+
if is_direct {
|
|
1057
|
+
let entry_size = self.direct_block_entry_size(offset_size, length_size);
|
|
1058
|
+
let entry_offset =
|
|
1059
|
+
entry_index.checked_mul(entry_size).ok_or_else(|| {
|
|
1060
|
+
Error::InvalidData(
|
|
1061
|
+
"fractal heap direct entry offset overflows u64".into(),
|
|
1062
|
+
)
|
|
1063
|
+
})?;
|
|
1064
|
+
let entry_pos = indirect_address
|
|
1065
|
+
.checked_add(iblock_header_size)
|
|
1066
|
+
.and_then(|pos| pos.checked_add(entry_offset))
|
|
1067
|
+
.ok_or_else(|| {
|
|
1068
|
+
Error::InvalidData(
|
|
1069
|
+
"fractal heap direct entry address overflows u64".into(),
|
|
1070
|
+
)
|
|
1071
|
+
})?;
|
|
1072
|
+
let entry_len = usize::try_from(entry_size).map_err(|_| {
|
|
1073
|
+
Error::InvalidData(
|
|
1074
|
+
"fractal heap direct entry size exceeds platform usize".into(),
|
|
1075
|
+
)
|
|
1076
|
+
})?;
|
|
1077
|
+
let entry_pos_usize = usize::try_from(entry_pos)
|
|
1078
|
+
.map_err(|_| Error::OffsetOutOfBounds(entry_pos))?;
|
|
1079
|
+
if entry_pos_usize
|
|
1080
|
+
.checked_add(entry_len)
|
|
1081
|
+
.map_or(true, |end| end > file_data.len())
|
|
1082
|
+
{
|
|
1083
|
+
return Err(Error::OffsetOutOfBounds(entry_pos));
|
|
1084
|
+
}
|
|
1085
|
+
let mut cursor = Cursor::new(file_data);
|
|
1086
|
+
cursor.set_position(entry_pos);
|
|
1087
|
+
let block_address = cursor.read_offset(offset_size)?;
|
|
1088
|
+
if Cursor::is_undefined_offset(block_address, offset_size) {
|
|
1089
|
+
return Err(Error::UndefinedAddress);
|
|
1090
|
+
}
|
|
1091
|
+
let (filtered_size, filter_mask) = if self.io_filters_len > 0 {
|
|
1092
|
+
(
|
|
1093
|
+
Some(cursor.read_length(length_size)?),
|
|
1094
|
+
cursor.read_u32_le()?,
|
|
1095
|
+
)
|
|
1096
|
+
} else {
|
|
1097
|
+
(None, 0)
|
|
1098
|
+
};
|
|
1099
|
+
return Ok(DirectBlockLocation {
|
|
1100
|
+
address: block_address,
|
|
1101
|
+
block_offset_in_heap: running_offset,
|
|
1102
|
+
block_size,
|
|
1103
|
+
filtered_size,
|
|
1104
|
+
filter_mask,
|
|
1105
|
+
});
|
|
1106
|
+
} else {
|
|
1107
|
+
// Need to recurse into a sub-indirect block.
|
|
1108
|
+
let direct_count = self
|
|
1109
|
+
.max_direct_block_rows_checked()?
|
|
1110
|
+
.checked_mul(u64::from(self.table_width))
|
|
1111
|
+
.ok_or_else(|| {
|
|
1112
|
+
Error::InvalidData(
|
|
1113
|
+
"fractal heap direct entry count overflows u64".into(),
|
|
1114
|
+
)
|
|
1115
|
+
})?;
|
|
1116
|
+
let indirect_index =
|
|
1117
|
+
entry_index.checked_sub(direct_count).ok_or_else(|| {
|
|
1118
|
+
Error::InvalidData(
|
|
1119
|
+
"fractal heap indirect entry precedes direct entries".into(),
|
|
1120
|
+
)
|
|
1121
|
+
})?;
|
|
1122
|
+
let direct_entry_bytes = direct_count
|
|
1123
|
+
.checked_mul(self.direct_block_entry_size(offset_size, length_size))
|
|
1124
|
+
.ok_or_else(|| {
|
|
1125
|
+
Error::InvalidData(
|
|
1126
|
+
"fractal heap direct entry table size overflows u64".into(),
|
|
1127
|
+
)
|
|
1128
|
+
})?;
|
|
1129
|
+
let indirect_entry_offset = indirect_index
|
|
1130
|
+
.checked_mul(u64::from(offset_size))
|
|
1131
|
+
.ok_or_else(|| {
|
|
1132
|
+
Error::InvalidData(
|
|
1133
|
+
"fractal heap indirect entry offset overflows u64".into(),
|
|
1134
|
+
)
|
|
1135
|
+
})?;
|
|
1136
|
+
let entry_pos = indirect_address
|
|
1137
|
+
.checked_add(iblock_header_size)
|
|
1138
|
+
.and_then(|pos| pos.checked_add(direct_entry_bytes))
|
|
1139
|
+
.and_then(|pos| pos.checked_add(indirect_entry_offset))
|
|
1140
|
+
.ok_or_else(|| {
|
|
1141
|
+
Error::InvalidData(
|
|
1142
|
+
"fractal heap indirect entry address overflows u64".into(),
|
|
1143
|
+
)
|
|
1144
|
+
})?;
|
|
1145
|
+
let entry_pos_usize = usize::try_from(entry_pos)
|
|
1146
|
+
.map_err(|_| Error::OffsetOutOfBounds(entry_pos))?;
|
|
1147
|
+
if entry_pos_usize
|
|
1148
|
+
.checked_add(usize::from(offset_size))
|
|
1149
|
+
.map_or(true, |end| end > file_data.len())
|
|
1150
|
+
{
|
|
1151
|
+
return Err(Error::OffsetOutOfBounds(entry_pos));
|
|
1152
|
+
}
|
|
1153
|
+
let mut cursor = Cursor::new(file_data);
|
|
1154
|
+
cursor.set_position(entry_pos);
|
|
1155
|
+
let block_address = cursor.read_offset(offset_size)?;
|
|
1156
|
+
if Cursor::is_undefined_offset(block_address, offset_size) {
|
|
1157
|
+
return Err(Error::UndefinedAddress);
|
|
1158
|
+
}
|
|
1159
|
+
// Determine how many rows the sub-indirect has.
|
|
1160
|
+
let sub_rows = self.rows_for_block_size_checked(block_size)?;
|
|
1161
|
+
return self.find_direct_block_via_indirect(
|
|
1162
|
+
block_address,
|
|
1163
|
+
heap_offset - running_offset,
|
|
1164
|
+
file_data,
|
|
1165
|
+
offset_size,
|
|
1166
|
+
length_size,
|
|
1167
|
+
sub_rows,
|
|
1168
|
+
depth_remaining - 1,
|
|
1169
|
+
visited,
|
|
1170
|
+
);
|
|
1171
|
+
}
|
|
1172
|
+
}
|
|
1173
|
+
running_offset = block_end;
|
|
1174
|
+
}
|
|
1175
|
+
}
|
|
1176
|
+
|
|
1177
|
+
Err(Error::InvalidData(format!(
|
|
1178
|
+
"fractal heap offset {} not found in doubling table",
|
|
1179
|
+
heap_offset
|
|
1180
|
+
)))
|
|
1181
|
+
}
|
|
1182
|
+
|
|
1183
|
+
#[allow(clippy::too_many_arguments)]
|
|
1184
|
+
fn find_direct_block_via_indirect_storage(
|
|
1185
|
+
&self,
|
|
1186
|
+
indirect_address: u64,
|
|
1187
|
+
heap_offset: u64,
|
|
1188
|
+
storage: &dyn Storage,
|
|
1189
|
+
offset_size: u8,
|
|
1190
|
+
length_size: u8,
|
|
1191
|
+
nrows: u16,
|
|
1192
|
+
depth_remaining: usize,
|
|
1193
|
+
visited: &mut HashSet<u64>,
|
|
1194
|
+
) -> Result<DirectBlockLocation> {
|
|
1195
|
+
self.enter_indirect_block(indirect_address, nrows, depth_remaining, visited)?;
|
|
1196
|
+
|
|
1197
|
+
let sig = storage.read_range(indirect_address, 4)?;
|
|
1198
|
+
if sig.as_ref() != FHIB_SIGNATURE {
|
|
1199
|
+
return Err(Error::InvalidData(format!(
|
|
1200
|
+
"expected FHIB signature at offset {:#x}, got {:?}",
|
|
1201
|
+
indirect_address,
|
|
1202
|
+
sig.as_ref()
|
|
1203
|
+
)));
|
|
1204
|
+
}
|
|
1205
|
+
|
|
1206
|
+
let width = self.table_width as u64;
|
|
1207
|
+
let mut running_offset = 0u64;
|
|
1208
|
+
|
|
1209
|
+
for row in 0..u64::from(nrows) {
|
|
1210
|
+
let block_size = self.block_size_for_row_checked(row)?;
|
|
1211
|
+
let is_direct = block_size <= self.max_direct_block_size;
|
|
1212
|
+
|
|
1213
|
+
for col in 0..width {
|
|
1214
|
+
let block_end = running_offset.checked_add(block_size).ok_or_else(|| {
|
|
1215
|
+
Error::InvalidData("fractal heap indirect block offset overflows u64".into())
|
|
1216
|
+
})?;
|
|
1217
|
+
if heap_offset >= running_offset && heap_offset < block_end {
|
|
1218
|
+
let entry_index = row * width + col;
|
|
1219
|
+
let iblock_header_size = 4
|
|
1220
|
+
+ 1
|
|
1221
|
+
+ u64::from(offset_size)
|
|
1222
|
+
+ (u64::from(self.max_heap_size)).div_ceil(8);
|
|
1223
|
+
|
|
1224
|
+
if is_direct {
|
|
1225
|
+
let entry_size = self.direct_block_entry_size(offset_size, length_size);
|
|
1226
|
+
let entry_offset =
|
|
1227
|
+
entry_index.checked_mul(entry_size).ok_or_else(|| {
|
|
1228
|
+
Error::InvalidData(
|
|
1229
|
+
"fractal heap direct entry offset overflows u64".into(),
|
|
1230
|
+
)
|
|
1231
|
+
})?;
|
|
1232
|
+
let entry_pos = indirect_address
|
|
1233
|
+
.checked_add(iblock_header_size)
|
|
1234
|
+
.and_then(|pos| pos.checked_add(entry_offset))
|
|
1235
|
+
.ok_or_else(|| {
|
|
1236
|
+
Error::InvalidData(
|
|
1237
|
+
"fractal heap direct entry address overflows u64".into(),
|
|
1238
|
+
)
|
|
1239
|
+
})?;
|
|
1240
|
+
let entry_len = usize::try_from(entry_size).map_err(|_| {
|
|
1241
|
+
Error::InvalidData(
|
|
1242
|
+
"fractal heap direct entry size exceeds platform usize".into(),
|
|
1243
|
+
)
|
|
1244
|
+
})?;
|
|
1245
|
+
let entry = storage.read_range(entry_pos, entry_len)?;
|
|
1246
|
+
let mut cursor = Cursor::new(entry.as_ref());
|
|
1247
|
+
let block_address = cursor.read_offset(offset_size)?;
|
|
1248
|
+
if Cursor::is_undefined_offset(block_address, offset_size) {
|
|
1249
|
+
return Err(Error::UndefinedAddress);
|
|
1250
|
+
}
|
|
1251
|
+
let (filtered_size, filter_mask) = if self.io_filters_len > 0 {
|
|
1252
|
+
(
|
|
1253
|
+
Some(cursor.read_length(length_size)?),
|
|
1254
|
+
cursor.read_u32_le()?,
|
|
1255
|
+
)
|
|
1256
|
+
} else {
|
|
1257
|
+
(None, 0)
|
|
1258
|
+
};
|
|
1259
|
+
return Ok(DirectBlockLocation {
|
|
1260
|
+
address: block_address,
|
|
1261
|
+
block_offset_in_heap: running_offset,
|
|
1262
|
+
block_size,
|
|
1263
|
+
filtered_size,
|
|
1264
|
+
filter_mask,
|
|
1265
|
+
});
|
|
1266
|
+
}
|
|
1267
|
+
|
|
1268
|
+
let direct_count = self
|
|
1269
|
+
.max_direct_block_rows_checked()?
|
|
1270
|
+
.checked_mul(u64::from(self.table_width))
|
|
1271
|
+
.ok_or_else(|| {
|
|
1272
|
+
Error::InvalidData(
|
|
1273
|
+
"fractal heap direct entry count overflows u64".into(),
|
|
1274
|
+
)
|
|
1275
|
+
})?;
|
|
1276
|
+
let indirect_index =
|
|
1277
|
+
entry_index.checked_sub(direct_count).ok_or_else(|| {
|
|
1278
|
+
Error::InvalidData(
|
|
1279
|
+
"fractal heap indirect entry precedes direct entries".into(),
|
|
1280
|
+
)
|
|
1281
|
+
})?;
|
|
1282
|
+
let direct_entry_bytes = direct_count
|
|
1283
|
+
.checked_mul(self.direct_block_entry_size(offset_size, length_size))
|
|
1284
|
+
.ok_or_else(|| {
|
|
1285
|
+
Error::InvalidData(
|
|
1286
|
+
"fractal heap direct entry table size overflows u64".into(),
|
|
1287
|
+
)
|
|
1288
|
+
})?;
|
|
1289
|
+
let indirect_entry_offset = indirect_index
|
|
1290
|
+
.checked_mul(u64::from(offset_size))
|
|
1291
|
+
.ok_or_else(|| {
|
|
1292
|
+
Error::InvalidData(
|
|
1293
|
+
"fractal heap indirect entry offset overflows u64".into(),
|
|
1294
|
+
)
|
|
1295
|
+
})?;
|
|
1296
|
+
let entry_addr_pos = indirect_address
|
|
1297
|
+
.checked_add(iblock_header_size)
|
|
1298
|
+
.and_then(|pos| pos.checked_add(direct_entry_bytes))
|
|
1299
|
+
.and_then(|pos| pos.checked_add(indirect_entry_offset))
|
|
1300
|
+
.ok_or_else(|| {
|
|
1301
|
+
Error::InvalidData(
|
|
1302
|
+
"fractal heap indirect entry address overflows u64".into(),
|
|
1303
|
+
)
|
|
1304
|
+
})?;
|
|
1305
|
+
let entry = storage.read_range(entry_addr_pos, usize::from(offset_size))?;
|
|
1306
|
+
let mut cursor = Cursor::new(entry.as_ref());
|
|
1307
|
+
let block_address = cursor.read_offset(offset_size)?;
|
|
1308
|
+
if Cursor::is_undefined_offset(block_address, offset_size) {
|
|
1309
|
+
return Err(Error::UndefinedAddress);
|
|
1310
|
+
}
|
|
1311
|
+
|
|
1312
|
+
let sub_rows = self.rows_for_block_size_checked(block_size)?;
|
|
1313
|
+
return self.find_direct_block_via_indirect_storage(
|
|
1314
|
+
block_address,
|
|
1315
|
+
heap_offset - running_offset,
|
|
1316
|
+
storage,
|
|
1317
|
+
offset_size,
|
|
1318
|
+
length_size,
|
|
1319
|
+
sub_rows,
|
|
1320
|
+
depth_remaining - 1,
|
|
1321
|
+
visited,
|
|
1322
|
+
);
|
|
1323
|
+
}
|
|
1324
|
+
running_offset = block_end;
|
|
1325
|
+
}
|
|
1326
|
+
}
|
|
1327
|
+
|
|
1328
|
+
Err(Error::InvalidData(format!(
|
|
1329
|
+
"fractal heap offset {} not found in doubling table",
|
|
1330
|
+
heap_offset
|
|
1331
|
+
)))
|
|
1332
|
+
}
|
|
1333
|
+
|
|
1334
|
+
fn enter_indirect_block(
|
|
1335
|
+
&self,
|
|
1336
|
+
indirect_address: u64,
|
|
1337
|
+
nrows: u16,
|
|
1338
|
+
depth_remaining: usize,
|
|
1339
|
+
visited: &mut HashSet<u64>,
|
|
1340
|
+
) -> Result<()> {
|
|
1341
|
+
if depth_remaining == 0 {
|
|
1342
|
+
return Err(Error::InvalidData(
|
|
1343
|
+
"fractal heap indirect traversal exceeded recursion limit".into(),
|
|
1344
|
+
));
|
|
1345
|
+
}
|
|
1346
|
+
if nrows > MAX_FRACTAL_HEAP_INDIRECT_ROWS {
|
|
1347
|
+
return Err(Error::InvalidData(format!(
|
|
1348
|
+
"fractal heap indirect block has {} rows, limit is {}",
|
|
1349
|
+
nrows, MAX_FRACTAL_HEAP_INDIRECT_ROWS
|
|
1350
|
+
)));
|
|
1351
|
+
}
|
|
1352
|
+
if !visited.insert(indirect_address) {
|
|
1353
|
+
return Err(Error::InvalidData(format!(
|
|
1354
|
+
"fractal heap indirect traversal revisits block at offset {:#x}",
|
|
1355
|
+
indirect_address
|
|
1356
|
+
)));
|
|
1357
|
+
}
|
|
1358
|
+
Ok(())
|
|
1359
|
+
}
|
|
1360
|
+
|
|
1361
|
+
/// Compute the block size for a given row in the doubling table.
|
|
1362
|
+
fn block_size_for_row_checked(&self, row: u64) -> Result<u64> {
|
|
1363
|
+
if row == 0 {
|
|
1364
|
+
Ok(self.starting_block_size)
|
|
1365
|
+
} else {
|
|
1366
|
+
let shift = u32::try_from(row - 1)
|
|
1367
|
+
.map_err(|_| Error::InvalidData("fractal heap indirect row exceeds u32".into()))?;
|
|
1368
|
+
let factor = 1u64.checked_shl(shift).ok_or_else(|| {
|
|
1369
|
+
Error::InvalidData("fractal heap indirect row shift overflows u64".into())
|
|
1370
|
+
})?;
|
|
1371
|
+
self.starting_block_size.checked_mul(factor).ok_or_else(|| {
|
|
1372
|
+
Error::InvalidData("fractal heap indirect block size overflows u64".into())
|
|
1373
|
+
})
|
|
1374
|
+
}
|
|
1375
|
+
}
|
|
1376
|
+
|
|
1377
|
+
/// Compute how many rows of the doubling table fit in a block of the
|
|
1378
|
+
/// given total size.
|
|
1379
|
+
fn rows_for_block_size_checked(&self, total_size: u64) -> Result<u16> {
|
|
1380
|
+
let mut rows = 0u16;
|
|
1381
|
+
let mut accum = 0u64;
|
|
1382
|
+
let width = u64::from(self.table_width);
|
|
1383
|
+
loop {
|
|
1384
|
+
if rows >= MAX_FRACTAL_HEAP_INDIRECT_ROWS {
|
|
1385
|
+
break;
|
|
1386
|
+
}
|
|
1387
|
+
let bs = self.block_size_for_row_checked(u64::from(rows))?;
|
|
1388
|
+
let row_total = bs.checked_mul(width).ok_or_else(|| {
|
|
1389
|
+
Error::InvalidData("fractal heap indirect row size overflows u64".into())
|
|
1390
|
+
})?;
|
|
1391
|
+
let next = accum.checked_add(row_total).ok_or_else(|| {
|
|
1392
|
+
Error::InvalidData("fractal heap indirect table size overflows u64".into())
|
|
1393
|
+
})?;
|
|
1394
|
+
if next > total_size {
|
|
1395
|
+
break;
|
|
1396
|
+
}
|
|
1397
|
+
accum = next;
|
|
1398
|
+
rows = rows.checked_add(1).ok_or_else(|| {
|
|
1399
|
+
Error::InvalidData("fractal heap indirect row count overflows u16".into())
|
|
1400
|
+
})?;
|
|
1401
|
+
}
|
|
1402
|
+
Ok(rows)
|
|
1403
|
+
}
|
|
1404
|
+
|
|
1405
|
+
fn max_direct_block_rows_checked(&self) -> Result<u64> {
|
|
1406
|
+
let mut rows = 0u64;
|
|
1407
|
+
loop {
|
|
1408
|
+
if rows >= u64::from(MAX_FRACTAL_HEAP_INDIRECT_ROWS) {
|
|
1409
|
+
break;
|
|
1410
|
+
}
|
|
1411
|
+
if self.block_size_for_row_checked(rows)? > self.max_direct_block_size {
|
|
1412
|
+
break;
|
|
1413
|
+
}
|
|
1414
|
+
rows += 1;
|
|
1415
|
+
}
|
|
1416
|
+
Ok(rows)
|
|
1417
|
+
}
|
|
1418
|
+
|
|
1419
|
+
fn direct_block_entry_size(&self, offset_size: u8, length_size: u8) -> u64 {
|
|
1420
|
+
let mut size = u64::from(offset_size);
|
|
1421
|
+
if self.io_filters_len > 0 {
|
|
1422
|
+
size += u64::from(length_size) + 4;
|
|
1423
|
+
}
|
|
1424
|
+
size
|
|
1425
|
+
}
|
|
1426
|
+
|
|
1427
|
+
/// Size in bytes of an unfiltered direct block header.
|
|
1428
|
+
fn direct_block_header_size(&self, offset_size: u8) -> usize {
|
|
1429
|
+
// Signature(4) + Version(1) + Heap header address(offset_size) +
|
|
1430
|
+
// Block offset within heap (max_heap_size bits, rounded up to bytes) +
|
|
1431
|
+
// optional Checksum(4).
|
|
1432
|
+
let offset_bytes = (self.max_heap_size as usize).div_ceil(8);
|
|
1433
|
+
let checksum_bytes = if self.direct_blocks_are_checksummed() {
|
|
1434
|
+
4
|
|
1435
|
+
} else {
|
|
1436
|
+
0
|
|
1437
|
+
};
|
|
1438
|
+
4 + 1 + offset_size as usize + offset_bytes + checksum_bytes
|
|
1439
|
+
}
|
|
1440
|
+
|
|
1441
|
+
fn direct_blocks_are_checksummed(&self) -> bool {
|
|
1442
|
+
self.flags & 0x02 != 0
|
|
1443
|
+
}
|
|
1444
|
+
|
|
1445
|
+
fn direct_block_checksum_pos(&self, offset_size: u8) -> Option<usize> {
|
|
1446
|
+
if self.direct_blocks_are_checksummed() {
|
|
1447
|
+
Some(self.direct_block_header_size(offset_size) - 4)
|
|
1448
|
+
} else {
|
|
1449
|
+
None
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
1452
|
+
|
|
1453
|
+
fn verify_direct_block_bytes(&self, block: &[u8], offset_size: u8) -> Result<()> {
|
|
1454
|
+
if block.len() < self.direct_block_header_size(offset_size) {
|
|
1455
|
+
return Err(Error::InvalidData(format!(
|
|
1456
|
+
"fractal heap direct block has {} bytes, expected at least {}",
|
|
1457
|
+
block.len(),
|
|
1458
|
+
self.direct_block_header_size(offset_size)
|
|
1459
|
+
)));
|
|
1460
|
+
}
|
|
1461
|
+
if block[..4] != _FHDB_SIGNATURE {
|
|
1462
|
+
return Err(Error::InvalidData(
|
|
1463
|
+
"invalid fractal heap direct block signature".into(),
|
|
1464
|
+
));
|
|
1465
|
+
}
|
|
1466
|
+
let version = block[4];
|
|
1467
|
+
if version != 0 {
|
|
1468
|
+
return Err(Error::UnsupportedFractalHeapVersion(version));
|
|
1469
|
+
}
|
|
1470
|
+
if let Some(checksum_pos) = self.direct_block_checksum_pos(offset_size) {
|
|
1471
|
+
let stored_checksum = u32::from_le_bytes(
|
|
1472
|
+
block[checksum_pos..checksum_pos + 4]
|
|
1473
|
+
.try_into()
|
|
1474
|
+
.expect("direct block checksum slice has four bytes"),
|
|
1475
|
+
);
|
|
1476
|
+
let mut checksum_data = block.to_vec();
|
|
1477
|
+
checksum_data[checksum_pos..checksum_pos + 4].fill(0);
|
|
1478
|
+
let computed = jenkins_lookup3(&checksum_data);
|
|
1479
|
+
if computed != stored_checksum {
|
|
1480
|
+
return Err(Error::ChecksumMismatch {
|
|
1481
|
+
expected: stored_checksum,
|
|
1482
|
+
actual: computed,
|
|
1483
|
+
});
|
|
1484
|
+
}
|
|
1485
|
+
}
|
|
1486
|
+
Ok(())
|
|
1487
|
+
}
|
|
1488
|
+
}
|
|
1489
|
+
|
|
1490
|
+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
1491
|
+
enum HeapIdKind {
|
|
1492
|
+
Managed,
|
|
1493
|
+
Huge,
|
|
1494
|
+
Tiny,
|
|
1495
|
+
}
|
|
1496
|
+
|
|
1497
|
+
fn bytes_needed_to_encode(value: u64) -> usize {
|
|
1498
|
+
if value <= u8::MAX as u64 {
|
|
1499
|
+
1
|
|
1500
|
+
} else if value <= u16::MAX as u64 {
|
|
1501
|
+
2
|
|
1502
|
+
} else if value <= 0x00FF_FFFF {
|
|
1503
|
+
3
|
|
1504
|
+
} else if value <= u32::MAX as u64 {
|
|
1505
|
+
4
|
|
1506
|
+
} else if value <= 0x00FF_FFFF_FFFF {
|
|
1507
|
+
5
|
|
1508
|
+
} else if value <= 0x0000_FFFF_FFFF_FFFF {
|
|
1509
|
+
6
|
|
1510
|
+
} else if value <= 0x00FF_FFFF_FFFF_FFFF {
|
|
1511
|
+
7
|
|
1512
|
+
} else {
|
|
1513
|
+
8
|
|
1514
|
+
}
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1517
|
+
#[cfg(test)]
|
|
1518
|
+
mod tests {
|
|
1519
|
+
use super::*;
|
|
1520
|
+
use crate::storage::{Storage, StorageBuffer};
|
|
1521
|
+
use flate2::write::ZlibEncoder;
|
|
1522
|
+
use flate2::Compression;
|
|
1523
|
+
use std::io::Write;
|
|
1524
|
+
use std::sync::{Arc, Mutex};
|
|
1525
|
+
|
|
1526
|
+
fn base_heap() -> FractalHeap {
|
|
1527
|
+
FractalHeap {
|
|
1528
|
+
heap_id_len: 8,
|
|
1529
|
+
io_filters_len: 0,
|
|
1530
|
+
flags: 0x02,
|
|
1531
|
+
max_managed_object_size: 128,
|
|
1532
|
+
next_huge_id: 0,
|
|
1533
|
+
btree_huge_objects_address: u64::MAX,
|
|
1534
|
+
free_space_managed_address: 0,
|
|
1535
|
+
managed_space_amount: 0,
|
|
1536
|
+
managed_alloc_amount: 0,
|
|
1537
|
+
managed_iter_offset: 0,
|
|
1538
|
+
managed_objects_count: 0,
|
|
1539
|
+
huge_objects_size: 0,
|
|
1540
|
+
huge_objects_count: 0,
|
|
1541
|
+
tiny_objects_size: 0,
|
|
1542
|
+
tiny_objects_count: 0,
|
|
1543
|
+
table_width: 4,
|
|
1544
|
+
starting_block_size: 256,
|
|
1545
|
+
max_direct_block_size: 4096,
|
|
1546
|
+
max_heap_size: 16,
|
|
1547
|
+
starting_row_root_indirect: 0,
|
|
1548
|
+
root_block_address: 0,
|
|
1549
|
+
current_rows_in_root_indirect: 0,
|
|
1550
|
+
io_filter_size: None,
|
|
1551
|
+
io_filter_mask: None,
|
|
1552
|
+
io_filter_info: Vec::new(),
|
|
1553
|
+
}
|
|
1554
|
+
}
|
|
1555
|
+
|
|
1556
|
+
fn deflate_filter_info() -> Vec<u8> {
|
|
1557
|
+
let mut data = vec![0x02, 0x01];
|
|
1558
|
+
data.extend_from_slice(&1u16.to_le_bytes());
|
|
1559
|
+
data.extend_from_slice(&0u16.to_le_bytes());
|
|
1560
|
+
data.extend_from_slice(&1u16.to_le_bytes());
|
|
1561
|
+
data.extend_from_slice(&6u32.to_le_bytes());
|
|
1562
|
+
data
|
|
1563
|
+
}
|
|
1564
|
+
|
|
1565
|
+
fn zlib_compress(bytes: &[u8]) -> Vec<u8> {
|
|
1566
|
+
let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
|
|
1567
|
+
encoder.write_all(bytes).unwrap();
|
|
1568
|
+
encoder.finish().unwrap()
|
|
1569
|
+
}
|
|
1570
|
+
|
|
1571
|
+
fn filtered_heap_with_info(filter_info: Vec<u8>) -> FractalHeap {
|
|
1572
|
+
let mut heap = base_heap();
|
|
1573
|
+
heap.io_filters_len = filter_info.len() as u16;
|
|
1574
|
+
heap.io_filter_info = filter_info;
|
|
1575
|
+
heap
|
|
1576
|
+
}
|
|
1577
|
+
|
|
1578
|
+
fn direct_block_with_object(heap: &FractalHeap, block_size: usize, obj: &[u8]) -> Vec<u8> {
|
|
1579
|
+
let offset_size = 8;
|
|
1580
|
+
let mut block = vec![0u8; block_size];
|
|
1581
|
+
block[0..4].copy_from_slice(b"FHDB");
|
|
1582
|
+
block[4] = 0;
|
|
1583
|
+
let obj_offset = heap.direct_block_header_size(offset_size);
|
|
1584
|
+
block[obj_offset..obj_offset + obj.len()].copy_from_slice(obj);
|
|
1585
|
+
if let Some(checksum_pos) = heap.direct_block_checksum_pos(offset_size) {
|
|
1586
|
+
let mut checksum_data = block.clone();
|
|
1587
|
+
checksum_data[checksum_pos..checksum_pos + 4].fill(0);
|
|
1588
|
+
let checksum = jenkins_lookup3(&checksum_data);
|
|
1589
|
+
block[checksum_pos..checksum_pos + 4].copy_from_slice(&checksum.to_le_bytes());
|
|
1590
|
+
}
|
|
1591
|
+
block
|
|
1592
|
+
}
|
|
1593
|
+
|
|
1594
|
+
struct CountingStorage {
|
|
1595
|
+
data: Vec<u8>,
|
|
1596
|
+
reads: Arc<Mutex<Vec<(u64, usize)>>>,
|
|
1597
|
+
}
|
|
1598
|
+
|
|
1599
|
+
impl Storage for CountingStorage {
|
|
1600
|
+
fn len(&self) -> u64 {
|
|
1601
|
+
self.data.len() as u64
|
|
1602
|
+
}
|
|
1603
|
+
|
|
1604
|
+
fn read_range(&self, offset: u64, len: usize) -> Result<StorageBuffer> {
|
|
1605
|
+
self.reads.lock().unwrap().push((offset, len));
|
|
1606
|
+
let start = usize::try_from(offset).map_err(|_| Error::OffsetOutOfBounds(offset))?;
|
|
1607
|
+
let end = start
|
|
1608
|
+
.checked_add(len)
|
|
1609
|
+
.ok_or(Error::OffsetOutOfBounds(offset))?;
|
|
1610
|
+
if end > self.data.len() {
|
|
1611
|
+
return Err(Error::UnexpectedEof {
|
|
1612
|
+
offset,
|
|
1613
|
+
needed: len as u64,
|
|
1614
|
+
available: self.len().saturating_sub(offset),
|
|
1615
|
+
});
|
|
1616
|
+
}
|
|
1617
|
+
Ok(StorageBuffer::from_vec(self.data[start..end].to_vec()))
|
|
1618
|
+
}
|
|
1619
|
+
}
|
|
1620
|
+
|
|
1621
|
+
#[test]
|
|
1622
|
+
fn block_size_for_row_scales_by_table_width() {
|
|
1623
|
+
let heap = FractalHeap {
|
|
1624
|
+
heap_id_len: 8,
|
|
1625
|
+
io_filters_len: 0,
|
|
1626
|
+
flags: 0x02,
|
|
1627
|
+
max_managed_object_size: 0,
|
|
1628
|
+
next_huge_id: 0,
|
|
1629
|
+
btree_huge_objects_address: 0,
|
|
1630
|
+
free_space_managed_address: 0,
|
|
1631
|
+
managed_space_amount: 0,
|
|
1632
|
+
managed_alloc_amount: 0,
|
|
1633
|
+
managed_iter_offset: 0,
|
|
1634
|
+
managed_objects_count: 0,
|
|
1635
|
+
huge_objects_size: 0,
|
|
1636
|
+
huge_objects_count: 0,
|
|
1637
|
+
tiny_objects_size: 0,
|
|
1638
|
+
tiny_objects_count: 0,
|
|
1639
|
+
table_width: 4,
|
|
1640
|
+
starting_block_size: 256,
|
|
1641
|
+
max_direct_block_size: 4096,
|
|
1642
|
+
max_heap_size: 16,
|
|
1643
|
+
starting_row_root_indirect: 0,
|
|
1644
|
+
root_block_address: 0,
|
|
1645
|
+
current_rows_in_root_indirect: 0,
|
|
1646
|
+
io_filter_size: None,
|
|
1647
|
+
io_filter_mask: None,
|
|
1648
|
+
io_filter_info: Vec::new(),
|
|
1649
|
+
};
|
|
1650
|
+
|
|
1651
|
+
assert_eq!(heap.block_size_for_row_checked(0).unwrap(), 256);
|
|
1652
|
+
assert_eq!(heap.block_size_for_row_checked(1).unwrap(), 256); // 256 * 2^0
|
|
1653
|
+
assert_eq!(heap.block_size_for_row_checked(2).unwrap(), 512); // 256 * 2^1
|
|
1654
|
+
assert_eq!(heap.block_size_for_row_checked(3).unwrap(), 1024); // 256 * 2^2
|
|
1655
|
+
}
|
|
1656
|
+
|
|
1657
|
+
#[test]
|
|
1658
|
+
fn find_direct_block_rejects_indirect_row_count_above_limit() {
|
|
1659
|
+
let mut heap = base_heap();
|
|
1660
|
+
heap.root_block_address = 0;
|
|
1661
|
+
heap.current_rows_in_root_indirect = MAX_FRACTAL_HEAP_INDIRECT_ROWS + 1;
|
|
1662
|
+
|
|
1663
|
+
let err = heap.find_direct_block(0, &[], 8, 8).unwrap_err();
|
|
1664
|
+
assert!(
|
|
1665
|
+
matches!(err, Error::InvalidData(ref message) if message.contains("limit")),
|
|
1666
|
+
"{err:?}"
|
|
1667
|
+
);
|
|
1668
|
+
|
|
1669
|
+
let storage = crate::storage::BytesStorage::new(Vec::new());
|
|
1670
|
+
let err = heap
|
|
1671
|
+
.find_direct_block_storage(0, &storage, 8, 8)
|
|
1672
|
+
.unwrap_err();
|
|
1673
|
+
assert!(
|
|
1674
|
+
matches!(err, Error::InvalidData(ref message) if message.contains("limit")),
|
|
1675
|
+
"{err:?}"
|
|
1676
|
+
);
|
|
1677
|
+
}
|
|
1678
|
+
|
|
1679
|
+
#[test]
|
|
1680
|
+
fn find_direct_block_rejects_cyclic_indirect_block() {
|
|
1681
|
+
let mut heap = base_heap();
|
|
1682
|
+
let indirect_address = 512u64;
|
|
1683
|
+
heap.table_width = 1;
|
|
1684
|
+
heap.starting_block_size = 256;
|
|
1685
|
+
heap.max_direct_block_size = 256;
|
|
1686
|
+
heap.max_heap_size = 64;
|
|
1687
|
+
heap.root_block_address = indirect_address;
|
|
1688
|
+
heap.current_rows_in_root_indirect = 3;
|
|
1689
|
+
|
|
1690
|
+
let offset_size = 8u8;
|
|
1691
|
+
let length_size = 8u8;
|
|
1692
|
+
let offset_bytes = usize::from(heap.max_heap_size).div_ceil(8);
|
|
1693
|
+
let iblock_header_size = 4 + 1 + usize::from(offset_size) + offset_bytes;
|
|
1694
|
+
let direct_count = usize::try_from(
|
|
1695
|
+
heap.max_direct_block_rows_checked().unwrap() * u64::from(heap.table_width),
|
|
1696
|
+
)
|
|
1697
|
+
.unwrap();
|
|
1698
|
+
let direct_entry_size =
|
|
1699
|
+
usize::try_from(heap.direct_block_entry_size(offset_size, length_size)).unwrap();
|
|
1700
|
+
let indirect_entry_pos = iblock_header_size + direct_count * direct_entry_size;
|
|
1701
|
+
let mut indirect = vec![0u8; indirect_entry_pos + usize::from(offset_size)];
|
|
1702
|
+
indirect[0..4].copy_from_slice(b"FHIB");
|
|
1703
|
+
indirect[4] = 0;
|
|
1704
|
+
indirect[indirect_entry_pos..indirect_entry_pos + usize::from(offset_size)]
|
|
1705
|
+
.copy_from_slice(&indirect_address.to_le_bytes());
|
|
1706
|
+
|
|
1707
|
+
let mut file_data = vec![0u8; indirect_address as usize + indirect.len()];
|
|
1708
|
+
file_data[indirect_address as usize..indirect_address as usize + indirect.len()]
|
|
1709
|
+
.copy_from_slice(&indirect);
|
|
1710
|
+
|
|
1711
|
+
let err = heap
|
|
1712
|
+
.find_direct_block(600, &file_data, offset_size, length_size)
|
|
1713
|
+
.unwrap_err();
|
|
1714
|
+
assert!(
|
|
1715
|
+
matches!(err, Error::InvalidData(ref message) if message.contains("revisits block")),
|
|
1716
|
+
"{err:?}"
|
|
1717
|
+
);
|
|
1718
|
+
|
|
1719
|
+
let storage = crate::storage::BytesStorage::new(file_data);
|
|
1720
|
+
let err = heap
|
|
1721
|
+
.find_direct_block_storage(600, &storage, offset_size, length_size)
|
|
1722
|
+
.unwrap_err();
|
|
1723
|
+
assert!(
|
|
1724
|
+
matches!(err, Error::InvalidData(ref message) if message.contains("revisits block")),
|
|
1725
|
+
"{err:?}"
|
|
1726
|
+
);
|
|
1727
|
+
}
|
|
1728
|
+
|
|
1729
|
+
#[test]
|
|
1730
|
+
fn get_tiny_object() {
|
|
1731
|
+
let heap = base_heap();
|
|
1732
|
+
let heap_id = [0x20 | 3, b't', b'i', b'n', b'y'];
|
|
1733
|
+
let result = heap.get_object(&heap_id, &[], 8, 8).unwrap();
|
|
1734
|
+
assert_eq!(result, b"tiny");
|
|
1735
|
+
}
|
|
1736
|
+
|
|
1737
|
+
#[test]
|
|
1738
|
+
fn get_huge_direct_object() {
|
|
1739
|
+
let heap = base_heap();
|
|
1740
|
+
let mut file_data = vec![0u8; 128];
|
|
1741
|
+
file_data[64..68].copy_from_slice(b"huge");
|
|
1742
|
+
|
|
1743
|
+
let mut heap_id = Vec::new();
|
|
1744
|
+
heap_id.push(0x10);
|
|
1745
|
+
heap_id.extend_from_slice(&64u64.to_le_bytes());
|
|
1746
|
+
heap_id.extend_from_slice(&4u64.to_le_bytes());
|
|
1747
|
+
|
|
1748
|
+
let result = heap.get_object(&heap_id, &file_data, 8, 8).unwrap();
|
|
1749
|
+
assert_eq!(result, b"huge");
|
|
1750
|
+
}
|
|
1751
|
+
|
|
1752
|
+
#[test]
|
|
1753
|
+
fn get_filtered_huge_direct_object() {
|
|
1754
|
+
let heap = filtered_heap_with_info(deflate_filter_info());
|
|
1755
|
+
let payload = b"filtered huge payload";
|
|
1756
|
+
let compressed = zlib_compress(payload);
|
|
1757
|
+
let address = 64u64;
|
|
1758
|
+
let mut file_data = vec![0u8; address as usize + compressed.len()];
|
|
1759
|
+
file_data[address as usize..].copy_from_slice(&compressed);
|
|
1760
|
+
|
|
1761
|
+
let mut heap_id = Vec::new();
|
|
1762
|
+
heap_id.push(0x10);
|
|
1763
|
+
heap_id.extend_from_slice(&address.to_le_bytes());
|
|
1764
|
+
heap_id.extend_from_slice(&(compressed.len() as u64).to_le_bytes());
|
|
1765
|
+
heap_id.extend_from_slice(&0u32.to_le_bytes());
|
|
1766
|
+
heap_id.extend_from_slice(&(payload.len() as u64).to_le_bytes());
|
|
1767
|
+
|
|
1768
|
+
let result = heap.get_object(&heap_id, &file_data, 8, 8).unwrap();
|
|
1769
|
+
assert_eq!(result, payload);
|
|
1770
|
+
}
|
|
1771
|
+
|
|
1772
|
+
#[test]
|
|
1773
|
+
fn get_filtered_managed_object_direct_root() {
|
|
1774
|
+
let mut heap = filtered_heap_with_info(deflate_filter_info());
|
|
1775
|
+
let block_address = 1000u64;
|
|
1776
|
+
let obj_data = b"filtered managed";
|
|
1777
|
+
let block = direct_block_with_object(&heap, heap.starting_block_size as usize, obj_data);
|
|
1778
|
+
let compressed = zlib_compress(&block);
|
|
1779
|
+
heap.root_block_address = block_address;
|
|
1780
|
+
heap.io_filter_size = Some(compressed.len() as u64);
|
|
1781
|
+
heap.io_filter_mask = Some(0);
|
|
1782
|
+
|
|
1783
|
+
let file_size = block_address as usize + compressed.len();
|
|
1784
|
+
let mut file_data = vec![0u8; file_size];
|
|
1785
|
+
file_data[block_address as usize..].copy_from_slice(&compressed);
|
|
1786
|
+
|
|
1787
|
+
let obj_offset = heap.direct_block_header_size(8) as u16;
|
|
1788
|
+
let mut heap_id = vec![0x00];
|
|
1789
|
+
heap_id.extend_from_slice(&obj_offset.to_le_bytes());
|
|
1790
|
+
heap_id.push(obj_data.len() as u8);
|
|
1791
|
+
|
|
1792
|
+
let result = heap.get_object(&heap_id, &file_data, 8, 8).unwrap();
|
|
1793
|
+
assert_eq!(result, obj_data);
|
|
1794
|
+
}
|
|
1795
|
+
|
|
1796
|
+
#[test]
|
|
1797
|
+
fn get_filtered_managed_object_from_indirect_block() {
|
|
1798
|
+
let mut heap = filtered_heap_with_info(deflate_filter_info());
|
|
1799
|
+
let indirect_address = 512u64;
|
|
1800
|
+
let block_address = 1000u64;
|
|
1801
|
+
let obj_data = b"filtered child";
|
|
1802
|
+
let block = direct_block_with_object(&heap, heap.starting_block_size as usize, obj_data);
|
|
1803
|
+
let compressed = zlib_compress(&block);
|
|
1804
|
+
|
|
1805
|
+
heap.root_block_address = indirect_address;
|
|
1806
|
+
heap.current_rows_in_root_indirect = 1;
|
|
1807
|
+
|
|
1808
|
+
let offset_bytes = usize::from(heap.max_heap_size).div_ceil(8);
|
|
1809
|
+
let iblock_header_size = 4 + 1 + 8 + offset_bytes;
|
|
1810
|
+
let direct_entry_size = 8 + 8 + 4;
|
|
1811
|
+
let mut indirect = vec![0u8; iblock_header_size + direct_entry_size * 4];
|
|
1812
|
+
indirect[0..4].copy_from_slice(b"FHIB");
|
|
1813
|
+
indirect[4] = 0;
|
|
1814
|
+
let entry_pos = iblock_header_size;
|
|
1815
|
+
indirect[entry_pos..entry_pos + 8].copy_from_slice(&block_address.to_le_bytes());
|
|
1816
|
+
indirect[entry_pos + 8..entry_pos + 16]
|
|
1817
|
+
.copy_from_slice(&(compressed.len() as u64).to_le_bytes());
|
|
1818
|
+
indirect[entry_pos + 16..entry_pos + 20].copy_from_slice(&0u32.to_le_bytes());
|
|
1819
|
+
|
|
1820
|
+
let file_size = block_address as usize + compressed.len();
|
|
1821
|
+
let mut file_data = vec![0u8; file_size];
|
|
1822
|
+
file_data[indirect_address as usize..indirect_address as usize + indirect.len()]
|
|
1823
|
+
.copy_from_slice(&indirect);
|
|
1824
|
+
file_data[block_address as usize..].copy_from_slice(&compressed);
|
|
1825
|
+
let storage = crate::storage::BytesStorage::new(file_data);
|
|
1826
|
+
|
|
1827
|
+
let obj_offset = heap.direct_block_header_size(8) as u16;
|
|
1828
|
+
let mut heap_id = vec![0x00];
|
|
1829
|
+
heap_id.extend_from_slice(&obj_offset.to_le_bytes());
|
|
1830
|
+
heap_id.push(obj_data.len() as u8);
|
|
1831
|
+
|
|
1832
|
+
let result = heap.get_object_storage(&heap_id, &storage, 8, 8).unwrap();
|
|
1833
|
+
assert_eq!(result, obj_data);
|
|
1834
|
+
}
|
|
1835
|
+
|
|
1836
|
+
#[test]
|
|
1837
|
+
fn direct_block_header_size_includes_optional_fields() {
|
|
1838
|
+
let heap = FractalHeap {
|
|
1839
|
+
heap_id_len: 8,
|
|
1840
|
+
io_filters_len: 0,
|
|
1841
|
+
flags: 0x02,
|
|
1842
|
+
max_managed_object_size: 0,
|
|
1843
|
+
next_huge_id: 0,
|
|
1844
|
+
btree_huge_objects_address: 0,
|
|
1845
|
+
free_space_managed_address: 0,
|
|
1846
|
+
managed_space_amount: 0,
|
|
1847
|
+
managed_alloc_amount: 0,
|
|
1848
|
+
managed_iter_offset: 0,
|
|
1849
|
+
managed_objects_count: 0,
|
|
1850
|
+
huge_objects_size: 0,
|
|
1851
|
+
huge_objects_count: 0,
|
|
1852
|
+
tiny_objects_size: 0,
|
|
1853
|
+
tiny_objects_count: 0,
|
|
1854
|
+
table_width: 4,
|
|
1855
|
+
starting_block_size: 256,
|
|
1856
|
+
max_direct_block_size: 4096,
|
|
1857
|
+
max_heap_size: 16,
|
|
1858
|
+
starting_row_root_indirect: 0,
|
|
1859
|
+
root_block_address: 0,
|
|
1860
|
+
current_rows_in_root_indirect: 0,
|
|
1861
|
+
io_filter_size: None,
|
|
1862
|
+
io_filter_mask: None,
|
|
1863
|
+
io_filter_info: Vec::new(),
|
|
1864
|
+
};
|
|
1865
|
+
|
|
1866
|
+
// sig(4) + ver(1) + addr(8) + offset_bytes(2) + checksum(4) = 19
|
|
1867
|
+
assert_eq!(heap.direct_block_header_size(8), 19);
|
|
1868
|
+
|
|
1869
|
+
// With 4-byte offsets: sig(4) + ver(1) + addr(4) + offset_bytes(2) + checksum(4) = 15
|
|
1870
|
+
assert_eq!(heap.direct_block_header_size(4), 15);
|
|
1871
|
+
}
|
|
1872
|
+
|
|
1873
|
+
#[test]
|
|
1874
|
+
fn get_managed_object_direct_root() {
|
|
1875
|
+
// Set up a fractal heap where the root is a direct block.
|
|
1876
|
+
let offset_size: u8 = 8;
|
|
1877
|
+
let max_heap_size: u16 = 16;
|
|
1878
|
+
let starting_block_size: u64 = 256;
|
|
1879
|
+
|
|
1880
|
+
// Direct block header size: sig(4) + ver(1) + addr(8) + offset_bytes(2) + checksum(4) = 19
|
|
1881
|
+
// (no I/O filters => checksum present)
|
|
1882
|
+
let db_header_size = 19usize;
|
|
1883
|
+
|
|
1884
|
+
// Place the direct block at file offset 1000.
|
|
1885
|
+
let block_address: u64 = 1000;
|
|
1886
|
+
|
|
1887
|
+
let heap = FractalHeap {
|
|
1888
|
+
heap_id_len: 8,
|
|
1889
|
+
io_filters_len: 0,
|
|
1890
|
+
flags: 0x02,
|
|
1891
|
+
max_managed_object_size: 128,
|
|
1892
|
+
next_huge_id: 0,
|
|
1893
|
+
btree_huge_objects_address: u64::MAX,
|
|
1894
|
+
free_space_managed_address: 0,
|
|
1895
|
+
managed_space_amount: starting_block_size,
|
|
1896
|
+
managed_alloc_amount: starting_block_size,
|
|
1897
|
+
managed_iter_offset: 0,
|
|
1898
|
+
managed_objects_count: 1,
|
|
1899
|
+
huge_objects_size: 0,
|
|
1900
|
+
huge_objects_count: 0,
|
|
1901
|
+
tiny_objects_size: 0,
|
|
1902
|
+
tiny_objects_count: 0,
|
|
1903
|
+
table_width: 4,
|
|
1904
|
+
starting_block_size,
|
|
1905
|
+
max_direct_block_size: 4096,
|
|
1906
|
+
max_heap_size,
|
|
1907
|
+
starting_row_root_indirect: 0,
|
|
1908
|
+
root_block_address: block_address,
|
|
1909
|
+
current_rows_in_root_indirect: 0,
|
|
1910
|
+
io_filter_size: None,
|
|
1911
|
+
io_filter_mask: None,
|
|
1912
|
+
io_filter_info: Vec::new(),
|
|
1913
|
+
};
|
|
1914
|
+
|
|
1915
|
+
// Build file data with the direct block.
|
|
1916
|
+
let file_size = block_address as usize + starting_block_size as usize + 100;
|
|
1917
|
+
let mut file_data = vec![0u8; file_size];
|
|
1918
|
+
|
|
1919
|
+
// Write direct block header at block_address.
|
|
1920
|
+
let ba = block_address as usize;
|
|
1921
|
+
file_data[ba..ba + 4].copy_from_slice(b"FHDB");
|
|
1922
|
+
file_data[ba + 4] = 0; // version
|
|
1923
|
+
// heap header address (8 bytes) — doesn't matter for this test
|
|
1924
|
+
// block offset (2 bytes) — 0
|
|
1925
|
+
|
|
1926
|
+
// Write object data at its direct-block offset, after the block header.
|
|
1927
|
+
let obj_data = b"test object data";
|
|
1928
|
+
let obj_start = ba + db_header_size;
|
|
1929
|
+
file_data[obj_start..obj_start + obj_data.len()].copy_from_slice(obj_data);
|
|
1930
|
+
let checksum_pos = ba + db_header_size - 4;
|
|
1931
|
+
let mut checksum_data = file_data[ba..ba + starting_block_size as usize].to_vec();
|
|
1932
|
+
checksum_data[checksum_pos - ba..checksum_pos - ba + 4].fill(0);
|
|
1933
|
+
let checksum = jenkins_lookup3(&checksum_data);
|
|
1934
|
+
file_data[checksum_pos..checksum_pos + 4].copy_from_slice(&checksum.to_le_bytes());
|
|
1935
|
+
|
|
1936
|
+
// Build heap ID for managed object at its direct-block offset, length=16.
|
|
1937
|
+
// Type nibble = 0, offset = direct block header size (16 bits), length = 16.
|
|
1938
|
+
let heap_id = [0x00, db_header_size as u8, 0x00, 0x10];
|
|
1939
|
+
|
|
1940
|
+
let result = heap
|
|
1941
|
+
.get_managed_object(&heap_id, &file_data, offset_size, 8)
|
|
1942
|
+
.unwrap();
|
|
1943
|
+
assert_eq!(result, obj_data);
|
|
1944
|
+
}
|
|
1945
|
+
|
|
1946
|
+
#[test]
|
|
1947
|
+
fn get_object_storage_cached_reads_direct_block_once() {
|
|
1948
|
+
let offset_size: u8 = 8;
|
|
1949
|
+
let max_heap_size: u16 = 16;
|
|
1950
|
+
let starting_block_size: u64 = 256;
|
|
1951
|
+
let db_header_size = 19usize;
|
|
1952
|
+
let block_address: u64 = 1000;
|
|
1953
|
+
|
|
1954
|
+
let heap = FractalHeap {
|
|
1955
|
+
heap_id_len: 8,
|
|
1956
|
+
io_filters_len: 0,
|
|
1957
|
+
flags: 0x02,
|
|
1958
|
+
max_managed_object_size: 128,
|
|
1959
|
+
next_huge_id: 0,
|
|
1960
|
+
btree_huge_objects_address: u64::MAX,
|
|
1961
|
+
free_space_managed_address: 0,
|
|
1962
|
+
managed_space_amount: starting_block_size,
|
|
1963
|
+
managed_alloc_amount: starting_block_size,
|
|
1964
|
+
managed_iter_offset: 0,
|
|
1965
|
+
managed_objects_count: 2,
|
|
1966
|
+
huge_objects_size: 0,
|
|
1967
|
+
huge_objects_count: 0,
|
|
1968
|
+
tiny_objects_size: 0,
|
|
1969
|
+
tiny_objects_count: 0,
|
|
1970
|
+
table_width: 4,
|
|
1971
|
+
starting_block_size,
|
|
1972
|
+
max_direct_block_size: 4096,
|
|
1973
|
+
max_heap_size,
|
|
1974
|
+
starting_row_root_indirect: 0,
|
|
1975
|
+
root_block_address: block_address,
|
|
1976
|
+
current_rows_in_root_indirect: 0,
|
|
1977
|
+
io_filter_size: None,
|
|
1978
|
+
io_filter_mask: None,
|
|
1979
|
+
io_filter_info: Vec::new(),
|
|
1980
|
+
};
|
|
1981
|
+
|
|
1982
|
+
let file_size = block_address as usize + starting_block_size as usize;
|
|
1983
|
+
let mut file_data = vec![0u8; file_size];
|
|
1984
|
+
let ba = block_address as usize;
|
|
1985
|
+
file_data[ba..ba + 4].copy_from_slice(b"FHDB");
|
|
1986
|
+
file_data[ba + 4] = 0;
|
|
1987
|
+
|
|
1988
|
+
let obj1_offset = db_header_size;
|
|
1989
|
+
let obj2_offset = db_header_size + 16;
|
|
1990
|
+
file_data[ba + obj1_offset..ba + obj1_offset + 4].copy_from_slice(b"one!");
|
|
1991
|
+
file_data[ba + obj2_offset..ba + obj2_offset + 4].copy_from_slice(b"two!");
|
|
1992
|
+
|
|
1993
|
+
let checksum_pos = ba + db_header_size - 4;
|
|
1994
|
+
let mut checksum_data = file_data[ba..ba + starting_block_size as usize].to_vec();
|
|
1995
|
+
checksum_data[checksum_pos - ba..checksum_pos - ba + 4].fill(0);
|
|
1996
|
+
let checksum = jenkins_lookup3(&checksum_data);
|
|
1997
|
+
file_data[checksum_pos..checksum_pos + 4].copy_from_slice(&checksum.to_le_bytes());
|
|
1998
|
+
|
|
1999
|
+
let reads = Arc::new(Mutex::new(Vec::new()));
|
|
2000
|
+
let storage = CountingStorage {
|
|
2001
|
+
data: file_data,
|
|
2002
|
+
reads: reads.clone(),
|
|
2003
|
+
};
|
|
2004
|
+
let mut cache = FractalHeapDirectBlockCache::default();
|
|
2005
|
+
let heap_id1 = [0x00, obj1_offset as u8, 0x00, 0x04];
|
|
2006
|
+
let heap_id2 = [0x00, obj2_offset as u8, 0x00, 0x04];
|
|
2007
|
+
|
|
2008
|
+
assert_eq!(
|
|
2009
|
+
heap.get_object_storage_cached(&heap_id1, &storage, offset_size, 8, &mut cache)
|
|
2010
|
+
.unwrap(),
|
|
2011
|
+
b"one!"
|
|
2012
|
+
);
|
|
2013
|
+
assert_eq!(
|
|
2014
|
+
heap.get_object_storage_cached(&heap_id2, &storage, offset_size, 8, &mut cache)
|
|
2015
|
+
.unwrap(),
|
|
2016
|
+
b"two!"
|
|
2017
|
+
);
|
|
2018
|
+
|
|
2019
|
+
let direct_block_reads = reads
|
|
2020
|
+
.lock()
|
|
2021
|
+
.unwrap()
|
|
2022
|
+
.iter()
|
|
2023
|
+
.filter(|&&(offset, len)| {
|
|
2024
|
+
offset == block_address && len == starting_block_size as usize
|
|
2025
|
+
})
|
|
2026
|
+
.count();
|
|
2027
|
+
assert_eq!(direct_block_reads, 1);
|
|
2028
|
+
}
|
|
2029
|
+
}
|