@wasm-oj/server 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +6 -0
  3. package/THIRD_PARTY_NOTICES.md +302 -0
  4. package/crates/runtime-core/Cargo.lock +5099 -0
  5. package/crates/runtime-core/Cargo.toml +66 -0
  6. package/crates/runtime-core/README.md +47 -0
  7. package/crates/runtime-core/src/bin/wasm-oj-compiler.rs +418 -0
  8. package/crates/runtime-core/src/bin/wasm-oj-runner.rs +294 -0
  9. package/crates/runtime-core/src/capabilities.rs +118 -0
  10. package/crates/runtime-core/src/compiler.rs +658 -0
  11. package/crates/runtime-core/src/contract.rs +5 -0
  12. package/crates/runtime-core/src/deterministic.rs +1051 -0
  13. package/crates/runtime-core/src/error.rs +58 -0
  14. package/crates/runtime-core/src/filesystem.rs +547 -0
  15. package/crates/runtime-core/src/filesystem_quota.rs +167 -0
  16. package/crates/runtime-core/src/go_compiler_session.rs +297 -0
  17. package/crates/runtime-core/src/interactive.rs +1019 -0
  18. package/crates/runtime-core/src/judge_package.rs +1539 -0
  19. package/crates/runtime-core/src/lib.rs +98 -0
  20. package/crates/runtime-core/src/memory.rs +84 -0
  21. package/crates/runtime-core/src/meter.rs +549 -0
  22. package/crates/runtime-core/src/module_imports.rs +149 -0
  23. package/crates/runtime-core/src/module_policy.rs +714 -0
  24. package/crates/runtime-core/src/output.rs +204 -0
  25. package/crates/runtime-core/src/run/mod.rs +208 -0
  26. package/crates/runtime-core/src/run/native.rs +260 -0
  27. package/crates/runtime-core/src/run/web.rs +229 -0
  28. package/crates/runtime-core/src/run/web_runtime.rs +109 -0
  29. package/crates/runtime-core/src/types.rs +268 -0
  30. package/crates/runtime-core/src/web.rs +83 -0
  31. package/dist/chunks/go-toolchain-Dbt-lp2L.js +426 -0
  32. package/dist/chunks/java-toolchain-DajoRCHu.js +44 -0
  33. package/dist/chunks/python-toolchain-Dx834o2A.js +4 -0
  34. package/dist/chunks/rust-toolchain-CJ3sMxPE.js +252 -0
  35. package/dist/chunks/toolchains-C6KuA1yM.js +224 -0
  36. package/dist/go-stage.mjs +193 -0
  37. package/dist/index.d.ts +221 -0
  38. package/dist/index.js +4393 -0
  39. package/dist/java-stage.mjs +111 -0
  40. package/dist/python-stage.mjs +90 -0
  41. package/dist/rustc-stage.mjs +301 -0
  42. package/dist/server-build-stage.mjs +2564 -0
  43. package/dist/server-runner-stage.mjs +155 -0
  44. package/licenses/fflate-MIT.txt +21 -0
  45. package/licenses/runtime-core-dependencies.html +6253 -0
  46. package/licenses/runtime-core-dependencies.json +3041 -0
  47. package/licenses/wasmer-sdk-MIT.txt +21 -0
  48. package/licenses/wasmer-sdk-dependencies.html +6901 -0
  49. package/licenses/wasmer-sdk-dependencies.json +3013 -0
  50. package/package.json +70 -0
  51. package/rust-toolchain.toml +5 -0
  52. package/testdata/wojjdg02-v2-text.hex +1 -0
  53. package/vendor/shared-buffer/Cargo.toml +22 -0
  54. package/vendor/shared-buffer/LICENSE_APACHE.md +176 -0
  55. package/vendor/shared-buffer/LICENSE_MIT.md +25 -0
  56. package/vendor/shared-buffer/README.md +34 -0
  57. package/vendor/shared-buffer/src/lib.rs +58 -0
  58. package/vendor/shared-buffer/src/mmap.rs +250 -0
  59. package/vendor/shared-buffer/src/owned.rs +389 -0
  60. package/vendor/virtual-fs/Cargo.toml +181 -0
  61. package/vendor/virtual-fs/LICENSE +25 -0
  62. package/vendor/virtual-fs/src/arc_box_file.rs +142 -0
  63. package/vendor/virtual-fs/src/arc_file.rs +182 -0
  64. package/vendor/virtual-fs/src/arc_fs.rs +68 -0
  65. package/vendor/virtual-fs/src/buffer_file.rs +103 -0
  66. package/vendor/virtual-fs/src/builder.rs +232 -0
  67. package/vendor/virtual-fs/src/combine_file.rs +101 -0
  68. package/vendor/virtual-fs/src/cow_file.rs +345 -0
  69. package/vendor/virtual-fs/src/dual_write_file.rs +113 -0
  70. package/vendor/virtual-fs/src/empty_fs.rs +81 -0
  71. package/vendor/virtual-fs/src/filesystems.rs +108 -0
  72. package/vendor/virtual-fs/src/host_fs.rs +1390 -0
  73. package/vendor/virtual-fs/src/lib.rs +782 -0
  74. package/vendor/virtual-fs/src/limiter.rs +252 -0
  75. package/vendor/virtual-fs/src/mem_fs/file.rs +1799 -0
  76. package/vendor/virtual-fs/src/mem_fs/file_opener.rs +941 -0
  77. package/vendor/virtual-fs/src/mem_fs/filesystem.rs +2134 -0
  78. package/vendor/virtual-fs/src/mem_fs/mod.rs +245 -0
  79. package/vendor/virtual-fs/src/mem_fs/offloaded_file.rs +474 -0
  80. package/vendor/virtual-fs/src/mem_fs/stdio.rs +318 -0
  81. package/vendor/virtual-fs/src/mount_fs.rs +2225 -0
  82. package/vendor/virtual-fs/src/null_file.rs +87 -0
  83. package/vendor/virtual-fs/src/ops.rs +364 -0
  84. package/vendor/virtual-fs/src/overlay_fs.rs +2216 -0
  85. package/vendor/virtual-fs/src/passthru_fs.rs +119 -0
  86. package/vendor/virtual-fs/src/pipe.rs +603 -0
  87. package/vendor/virtual-fs/src/random_file.rs +88 -0
  88. package/vendor/virtual-fs/src/special_file.rs +108 -0
  89. package/vendor/virtual-fs/src/static_file.rs +133 -0
  90. package/vendor/virtual-fs/src/static_fs.rs +460 -0
  91. package/vendor/virtual-fs/src/tmp_fs.rs +95 -0
  92. package/vendor/virtual-fs/src/trace_fs.rs +258 -0
  93. package/vendor/virtual-fs/src/webc_volume_fs.rs +829 -0
  94. package/vendor/virtual-fs/src/zero_file.rs +90 -0
@@ -0,0 +1,245 @@
1
+ mod file;
2
+ mod file_opener;
3
+ mod filesystem;
4
+ mod offloaded_file;
5
+ mod stdio;
6
+
7
+ use file::{File, FileHandle, ReadOnlyFile};
8
+ pub use filesystem::FileSystem;
9
+ pub use offloaded_file::OffloadBackingStore;
10
+ #[cfg(not(feature = "js"))]
11
+ use std::time::{SystemTime, UNIX_EPOCH};
12
+ pub use stdio::{Stderr, Stdin, Stdout};
13
+ #[cfg(feature = "js")]
14
+ pub use web_time::{SystemTime, UNIX_EPOCH};
15
+
16
+ use crate::Metadata;
17
+ use std::{
18
+ ffi::{OsStr, OsString},
19
+ path::PathBuf,
20
+ sync::{
21
+ Arc, Mutex,
22
+ atomic::{AtomicBool, AtomicUsize, Ordering},
23
+ },
24
+ };
25
+
26
+ use self::offloaded_file::OffloadedFile;
27
+
28
+ type Inode = usize;
29
+ const ROOT_INODE: Inode = 0;
30
+
31
+ #[derive(Debug, Default)]
32
+ struct FileLifecycle {
33
+ open_handles: AtomicUsize,
34
+ unlinked: AtomicBool,
35
+ }
36
+
37
+ impl FileLifecycle {
38
+ fn opened(&self) {
39
+ self.open_handles.fetch_add(1, Ordering::AcqRel);
40
+ }
41
+
42
+ fn closed(&self) -> usize {
43
+ let previous = self
44
+ .open_handles
45
+ .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| {
46
+ count.checked_sub(1)
47
+ })
48
+ .expect("FileLifecycle::closed called with zero open handles");
49
+ previous - 1
50
+ }
51
+
52
+ fn open_handle_count(&self) -> usize {
53
+ self.open_handles.load(Ordering::Acquire)
54
+ }
55
+
56
+ fn mark_unlinked(&self) {
57
+ self.unlinked.store(true, Ordering::Release);
58
+ }
59
+
60
+ fn is_unlinked(&self) -> bool {
61
+ self.unlinked.load(Ordering::Acquire)
62
+ }
63
+ }
64
+
65
+ #[derive(Debug)]
66
+ struct FileNode {
67
+ inode: Inode,
68
+ name: OsString,
69
+ file: File,
70
+ metadata: Metadata,
71
+ lifecycle: Arc<FileLifecycle>,
72
+ }
73
+
74
+ #[derive(Debug)]
75
+ struct ReadOnlyFileNode {
76
+ inode: Inode,
77
+ name: OsString,
78
+ file: ReadOnlyFile,
79
+ metadata: Metadata,
80
+ lifecycle: Arc<FileLifecycle>,
81
+ }
82
+
83
+ #[derive(Debug)]
84
+ struct OffloadedFileNode {
85
+ inode: Inode,
86
+ name: OsString,
87
+ file: OffloadedFile,
88
+ metadata: Metadata,
89
+ lifecycle: Arc<FileLifecycle>,
90
+ }
91
+
92
+ #[derive(Debug)]
93
+ struct ArcFileNode {
94
+ inode: Inode,
95
+ name: OsString,
96
+ fs: Arc<dyn crate::FileSystem + Send + Sync>,
97
+ path: PathBuf,
98
+ metadata: Metadata,
99
+ lifecycle: Arc<FileLifecycle>,
100
+ }
101
+
102
+ // FIXME: this is broken!!! A `VirtualFile` stores its own offset,
103
+ // so a file stored this way can only be read once!
104
+ #[derive(Debug)]
105
+ struct CustomFileNode {
106
+ inode: Inode,
107
+ name: OsString,
108
+ file: Mutex<Box<dyn crate::VirtualFile + Send + Sync>>,
109
+ metadata: Metadata,
110
+ lifecycle: Arc<FileLifecycle>,
111
+ }
112
+
113
+ #[derive(Debug)]
114
+ struct DirectoryNode {
115
+ inode: Inode,
116
+ name: OsString,
117
+ children: Vec<Inode>,
118
+ metadata: Metadata,
119
+ }
120
+
121
+ #[derive(Debug)]
122
+ struct ArcDirectoryNode {
123
+ inode: Inode,
124
+ name: OsString,
125
+ fs: Arc<dyn crate::FileSystem + Send + Sync>,
126
+ path: PathBuf,
127
+ metadata: Metadata,
128
+ }
129
+
130
+ #[derive(Debug)]
131
+ struct SymlinkNode {
132
+ inode: Inode,
133
+ name: OsString,
134
+ target: PathBuf,
135
+ metadata: Metadata,
136
+ }
137
+
138
+ #[derive(Debug)]
139
+ enum Node {
140
+ File(FileNode),
141
+ OffloadedFile(OffloadedFileNode),
142
+ ReadOnlyFile(ReadOnlyFileNode),
143
+ ArcFile(ArcFileNode),
144
+ CustomFile(CustomFileNode),
145
+ Symlink(SymlinkNode),
146
+ Directory(DirectoryNode),
147
+ ArcDirectory(ArcDirectoryNode),
148
+ }
149
+
150
+ impl Node {
151
+ fn inode(&self) -> Inode {
152
+ *match self {
153
+ Self::File(FileNode { inode, .. }) => inode,
154
+ Self::OffloadedFile(OffloadedFileNode { inode, .. }) => inode,
155
+ Self::ReadOnlyFile(ReadOnlyFileNode { inode, .. }) => inode,
156
+ Self::ArcFile(ArcFileNode { inode, .. }) => inode,
157
+ Self::CustomFile(CustomFileNode { inode, .. }) => inode,
158
+ Self::Symlink(SymlinkNode { inode, .. }) => inode,
159
+ Self::Directory(DirectoryNode { inode, .. }) => inode,
160
+ Self::ArcDirectory(ArcDirectoryNode { inode, .. }) => inode,
161
+ }
162
+ }
163
+
164
+ fn name(&self) -> &OsStr {
165
+ match self {
166
+ Self::File(FileNode { name, .. }) => name.as_os_str(),
167
+ Self::OffloadedFile(OffloadedFileNode { name, .. }) => name.as_os_str(),
168
+ Self::ReadOnlyFile(ReadOnlyFileNode { name, .. }) => name.as_os_str(),
169
+ Self::ArcFile(ArcFileNode { name, .. }) => name.as_os_str(),
170
+ Self::CustomFile(CustomFileNode { name, .. }) => name.as_os_str(),
171
+ Self::Symlink(SymlinkNode { name, .. }) => name.as_os_str(),
172
+ Self::Directory(DirectoryNode { name, .. }) => name.as_os_str(),
173
+ Self::ArcDirectory(ArcDirectoryNode { name, .. }) => name.as_os_str(),
174
+ }
175
+ }
176
+
177
+ fn metadata(&self) -> &Metadata {
178
+ match self {
179
+ Self::File(FileNode { metadata, .. }) => metadata,
180
+ Self::OffloadedFile(OffloadedFileNode { metadata, .. }) => metadata,
181
+ Self::ReadOnlyFile(ReadOnlyFileNode { metadata, .. }) => metadata,
182
+ Self::ArcFile(ArcFileNode { metadata, .. }) => metadata,
183
+ Self::CustomFile(CustomFileNode { metadata, .. }) => metadata,
184
+ Self::Symlink(SymlinkNode { metadata, .. }) => metadata,
185
+ Self::Directory(DirectoryNode { metadata, .. }) => metadata,
186
+ Self::ArcDirectory(ArcDirectoryNode { metadata, .. }) => metadata,
187
+ }
188
+ }
189
+
190
+ fn file_lifecycle(&self) -> Option<&Arc<FileLifecycle>> {
191
+ match self {
192
+ Self::File(FileNode { lifecycle, .. })
193
+ | Self::OffloadedFile(OffloadedFileNode { lifecycle, .. })
194
+ | Self::ReadOnlyFile(ReadOnlyFileNode { lifecycle, .. })
195
+ | Self::ArcFile(ArcFileNode { lifecycle, .. })
196
+ | Self::CustomFile(CustomFileNode { lifecycle, .. }) => Some(lifecycle),
197
+ _ => None,
198
+ }
199
+ }
200
+
201
+ fn metadata_mut(&mut self) -> &mut Metadata {
202
+ match self {
203
+ Self::File(FileNode { metadata, .. }) => metadata,
204
+ Self::OffloadedFile(OffloadedFileNode { metadata, .. }) => metadata,
205
+ Self::ReadOnlyFile(ReadOnlyFileNode { metadata, .. }) => metadata,
206
+ Self::ArcFile(ArcFileNode { metadata, .. }) => metadata,
207
+ Self::CustomFile(CustomFileNode { metadata, .. }) => metadata,
208
+ Self::Symlink(SymlinkNode { metadata, .. }) => metadata,
209
+ Self::Directory(DirectoryNode { metadata, .. }) => metadata,
210
+ Self::ArcDirectory(ArcDirectoryNode { metadata, .. }) => metadata,
211
+ }
212
+ }
213
+
214
+ fn set_name(&mut self, new_name: OsString) {
215
+ match self {
216
+ Self::File(FileNode { name, .. }) => *name = new_name,
217
+ Self::OffloadedFile(OffloadedFileNode { name, .. }) => *name = new_name,
218
+ Self::ReadOnlyFile(ReadOnlyFileNode { name, .. }) => *name = new_name,
219
+ Self::ArcFile(ArcFileNode { name, .. }) => *name = new_name,
220
+ Self::CustomFile(CustomFileNode { name, .. }) => *name = new_name,
221
+ Self::Symlink(SymlinkNode { name, .. }) => *name = new_name,
222
+ Self::Directory(DirectoryNode { name, .. }) => *name = new_name,
223
+ Self::ArcDirectory(ArcDirectoryNode { name, .. }) => *name = new_name,
224
+ }
225
+ }
226
+ }
227
+
228
+ #[derive(Clone, Copy, Debug, Default)]
229
+ pub(super) enum MetadataClock {
230
+ #[default]
231
+ System,
232
+ Fixed(u64),
233
+ }
234
+
235
+ impl MetadataClock {
236
+ fn now(self) -> u64 {
237
+ match self {
238
+ Self::System => SystemTime::now()
239
+ .duration_since(UNIX_EPOCH)
240
+ .expect("system time must not precede the Unix epoch")
241
+ .as_nanos() as u64,
242
+ Self::Fixed(timestamp) => timestamp,
243
+ }
244
+ }
245
+ }
@@ -0,0 +1,474 @@
1
+ use bytes::Bytes;
2
+ use shared_buffer::OwnedBuffer;
3
+ use std::{
4
+ cmp,
5
+ fs::File,
6
+ io,
7
+ ops::Range,
8
+ sync::{Arc, Mutex, MutexGuard},
9
+ };
10
+
11
+ use crate::limiter::DynFsMemoryLimiter;
12
+
13
+ #[derive(Debug)]
14
+ pub enum FileExtent {
15
+ MmapOffload { offset: u64, size: u64 },
16
+ RepeatingBytes { value: u8, cnt: u64 },
17
+ InMemory { data: Bytes },
18
+ }
19
+
20
+ impl FileExtent {
21
+ pub fn size(&self) -> u64 {
22
+ match self {
23
+ FileExtent::MmapOffload { size, .. } => *size,
24
+ FileExtent::RepeatingBytes { cnt, .. } => *cnt,
25
+ FileExtent::InMemory { data } => data.len() as u64,
26
+ }
27
+ }
28
+
29
+ pub fn resize(&mut self, new_size: u64) {
30
+ match self {
31
+ FileExtent::MmapOffload { size, .. } => *size = new_size.min(*size),
32
+ FileExtent::RepeatingBytes { cnt, .. } => *cnt = new_size,
33
+ FileExtent::InMemory { data } => {
34
+ *data = data.slice(..(new_size as usize));
35
+ }
36
+ }
37
+ }
38
+ }
39
+
40
+ #[derive(Debug)]
41
+ struct OffloadBackingStoreState {
42
+ mmap_file: Option<File>,
43
+ mmap_offload: OwnedBuffer,
44
+ }
45
+
46
+ impl OffloadBackingStoreState {
47
+ fn get_slice(&mut self, range: Range<u64>) -> io::Result<&[u8]> {
48
+ let offset = range.start;
49
+ let size = match range.end {
50
+ u64::MAX => {
51
+ let len = self.mmap_offload.len() as u64;
52
+ if len < offset {
53
+ tracing::trace!("range out of bounds {} vs {}", len, offset);
54
+ return Err(io::ErrorKind::UnexpectedEof.into());
55
+ }
56
+ len - offset
57
+ }
58
+ end => end - offset,
59
+ };
60
+
61
+ let end = offset + size;
62
+ if end > self.mmap_offload.len() as u64 {
63
+ let mmap_file = match self.mmap_file.as_ref() {
64
+ Some(a) => a,
65
+ None => {
66
+ tracing::trace!(
67
+ "mmap buffer out of bounds and no mmap file to reload {} vs {}",
68
+ end,
69
+ self.mmap_offload.len()
70
+ );
71
+ return Err(io::ErrorKind::UnexpectedEof.into());
72
+ }
73
+ };
74
+ self.mmap_offload = OwnedBuffer::from_file(mmap_file).map_err(io::Error::other)?;
75
+ if end > self.mmap_offload.len() as u64 {
76
+ tracing::trace!(
77
+ "mmap buffer out of bounds {} vs {} for {:?}",
78
+ end,
79
+ self.mmap_offload.len(),
80
+ range
81
+ );
82
+ return Err(io::ErrorKind::UnexpectedEof.into());
83
+ }
84
+ }
85
+ Ok(&self.mmap_offload[offset as usize..end as usize])
86
+ }
87
+ }
88
+
89
+ #[derive(Debug, Clone)]
90
+ pub struct OffloadBackingStore {
91
+ state: Arc<Mutex<OffloadBackingStoreState>>,
92
+ }
93
+
94
+ impl OffloadBackingStore {
95
+ pub fn new(mmap_offload: OwnedBuffer, mmap_file: Option<File>) -> Self {
96
+ Self {
97
+ state: Arc::new(Mutex::new(OffloadBackingStoreState {
98
+ mmap_file,
99
+ mmap_offload,
100
+ })),
101
+ }
102
+ }
103
+
104
+ pub fn from_file(file: &File) -> Self {
105
+ let file = file.try_clone().unwrap();
106
+ let buffer = OwnedBuffer::from_file(&file).unwrap();
107
+ Self::new(buffer, Some(file))
108
+ }
109
+
110
+ pub fn from_buffer(buffer: OwnedBuffer) -> Self {
111
+ Self::new(buffer, None)
112
+ }
113
+
114
+ pub fn owned_buffer(&self) -> OwnedBuffer {
115
+ let guard = self.state.lock().unwrap();
116
+ guard.mmap_offload.clone()
117
+ }
118
+
119
+ fn lock(&self) -> MutexGuard<'_, OffloadBackingStoreState> {
120
+ self.state.lock().unwrap()
121
+ }
122
+ }
123
+
124
+ #[derive(Debug)]
125
+ pub struct OffloadedFile {
126
+ backing: OffloadBackingStore,
127
+ #[allow(dead_code)]
128
+ limiter: Option<DynFsMemoryLimiter>,
129
+ extents: Vec<FileExtent>,
130
+ size: u64,
131
+ }
132
+
133
+ pub enum OffloadWrite<'a> {
134
+ MmapOffset { offset: u64, size: u64 },
135
+ Buffer(&'a [u8]),
136
+ }
137
+
138
+ impl OffloadWrite<'_> {
139
+ fn len(&self) -> usize {
140
+ match self {
141
+ OffloadWrite::MmapOffset { size, .. } => *size as usize,
142
+ OffloadWrite::Buffer(data) => data.len(),
143
+ }
144
+ }
145
+ }
146
+
147
+ impl OffloadedFile {
148
+ pub fn new(limiter: Option<DynFsMemoryLimiter>, backing: OffloadBackingStore) -> Self {
149
+ Self {
150
+ backing,
151
+ limiter,
152
+ extents: Vec::new(),
153
+ size: 0,
154
+ }
155
+ }
156
+
157
+ pub fn seek(&self, position: io::SeekFrom, cursor: &mut u64) -> io::Result<u64> {
158
+ let to_err = |_| io::ErrorKind::InvalidInput;
159
+
160
+ // Calculate the next cursor.
161
+ let next_cursor: i64 = match position {
162
+ io::SeekFrom::Start(offset) => offset.try_into().map_err(to_err)?,
163
+ io::SeekFrom::End(offset) => self.len() as i64 + offset,
164
+ io::SeekFrom::Current(offset) => {
165
+ TryInto::<i64>::try_into(*cursor).map_err(to_err)? + offset
166
+ }
167
+ };
168
+
169
+ // It's an error to seek before byte 0.
170
+ if next_cursor < 0 {
171
+ return Err(io::Error::new(
172
+ io::ErrorKind::InvalidInput,
173
+ "seeking before the byte 0",
174
+ ));
175
+ }
176
+
177
+ let next_cursor = next_cursor.try_into().map_err(to_err)?;
178
+ *cursor = next_cursor;
179
+ Ok(*cursor)
180
+ }
181
+
182
+ pub fn read(&self, mut buf: &mut [u8], cursor: &mut u64) -> io::Result<usize> {
183
+ let cursor_start = *cursor;
184
+
185
+ let mut extent_offset = cursor_start;
186
+ let mut extent_index = 0usize;
187
+ while !buf.is_empty() && extent_index < self.extents.len() {
188
+ let extent = &self.extents[extent_index];
189
+
190
+ if extent_offset >= extent.size() {
191
+ extent_offset -= extent.size();
192
+ extent_index += 1;
193
+ continue;
194
+ }
195
+
196
+ let read = match extent {
197
+ FileExtent::MmapOffload {
198
+ offset: mmap_offset,
199
+ size: extent_size,
200
+ } => {
201
+ let mut backing = self.backing.lock();
202
+ let mmap_offset_plus_extent = mmap_offset + extent_offset;
203
+ let data = backing.get_slice(
204
+ mmap_offset_plus_extent
205
+ ..(mmap_offset_plus_extent + *extent_size - extent_offset),
206
+ )?;
207
+ let data_len = cmp::min(buf.len(), data.len());
208
+ buf[..data_len].copy_from_slice(&data[..data_len]);
209
+ data_len
210
+ }
211
+ FileExtent::RepeatingBytes { value, cnt } => {
212
+ let cnt = cmp::min(buf.len() as u64, cnt - extent_offset) as usize;
213
+ buf[..cnt].iter_mut().for_each(|d| {
214
+ *d = *value;
215
+ });
216
+ cnt
217
+ }
218
+ FileExtent::InMemory { data } => {
219
+ let data = &data.as_ref()[extent_offset as usize..];
220
+ let data_len = cmp::min(buf.len(), data.len());
221
+ buf[..data_len].copy_from_slice(&data[..data_len]);
222
+ data_len
223
+ }
224
+ };
225
+
226
+ *cursor += read as u64;
227
+ extent_offset = 0;
228
+ extent_index += 1;
229
+ buf = &mut buf[read..];
230
+ }
231
+ Ok((*cursor - cursor_start) as usize)
232
+ }
233
+
234
+ pub fn write(&mut self, data: OffloadWrite<'_>, cursor: &mut u64) -> io::Result<usize> {
235
+ let original_extent_offset = *cursor;
236
+ let mut extent_offset = original_extent_offset;
237
+ let mut data_len = data.len() as u64;
238
+
239
+ // We need to split any extents that are intersecting with the
240
+ // start or end of the new block of data we are about to write
241
+ let mut split_extents = |mut split_at: u64| {
242
+ let mut index = 0usize;
243
+ while split_at > 0 && index < self.extents.len() {
244
+ let extent = &mut self.extents[index];
245
+ if split_at >= extent.size() {
246
+ split_at -= extent.size();
247
+ index += 1;
248
+ continue;
249
+ } else if split_at == 0 {
250
+ break;
251
+ } else {
252
+ let new_extent = match extent {
253
+ FileExtent::MmapOffload {
254
+ offset: other_offset,
255
+ size: other_size,
256
+ } => FileExtent::MmapOffload {
257
+ offset: *other_offset + split_at,
258
+ size: *other_size - split_at,
259
+ },
260
+ FileExtent::RepeatingBytes {
261
+ value: other_value,
262
+ cnt: other_cnt,
263
+ } => FileExtent::RepeatingBytes {
264
+ value: *other_value,
265
+ cnt: *other_cnt - split_at,
266
+ },
267
+ FileExtent::InMemory { data: other_data } => FileExtent::InMemory {
268
+ data: other_data.slice((split_at as usize)..),
269
+ },
270
+ };
271
+ extent.resize(split_at);
272
+ self.extents.insert(index + 1, new_extent);
273
+ break;
274
+ }
275
+ }
276
+ };
277
+
278
+ // If the extent is below the actual size of the file then we need to split it
279
+ let mut index = if extent_offset < self.size {
280
+ split_extents(extent_offset);
281
+ split_extents(extent_offset + data_len);
282
+
283
+ // Now we delete all the extents that exist between the
284
+ // range that we are about to insert
285
+ let mut index = 0usize;
286
+ while index < self.extents.len() {
287
+ let extent = &self.extents[index];
288
+ if extent_offset >= extent.size() {
289
+ extent_offset -= extent.size();
290
+ index += 1;
291
+ continue;
292
+ } else {
293
+ break;
294
+ }
295
+ }
296
+ while index < self.extents.len() {
297
+ let extent = &self.extents[index];
298
+ if data_len < extent.size() {
299
+ break;
300
+ }
301
+ data_len -= extent.size();
302
+ self.extents.remove(index);
303
+ }
304
+ index
305
+ } else {
306
+ self.extents.len()
307
+ };
308
+
309
+ // If we have a gap that needs to be filled then do so
310
+ if extent_offset > self.size {
311
+ self.extents.insert(
312
+ index,
313
+ FileExtent::RepeatingBytes {
314
+ value: 0,
315
+ cnt: extent_offset - self.size,
316
+ },
317
+ );
318
+ self.size = extent_offset;
319
+ index += 1;
320
+ }
321
+
322
+ // Insert the extent into the buffer
323
+ match data {
324
+ OffloadWrite::MmapOffset { offset, size } => {
325
+ self.extents
326
+ .insert(index, FileExtent::MmapOffload { offset, size });
327
+ }
328
+ OffloadWrite::Buffer(data) => {
329
+ // Finally we add the new extent
330
+ let data_start = data.as_ptr() as u64;
331
+ let data_end = data_start + data.len() as u64;
332
+ let mut backing = self.backing.lock();
333
+ let backing_data = backing.get_slice(0u64..u64::MAX)?;
334
+
335
+ let mmap_start = backing_data.as_ptr() as u64;
336
+ let mmap_end = mmap_start + backing_data.len() as u64;
337
+
338
+ // If the data is within the mmap buffer then we use a extent range
339
+ // to represent the data, otherwise we fall back on copying the data
340
+ let new_extent = if data_start >= mmap_start && data_end <= mmap_end {
341
+ FileExtent::MmapOffload {
342
+ offset: data_start - mmap_start,
343
+ size: data_end - data_start,
344
+ }
345
+ } else {
346
+ FileExtent::InMemory {
347
+ data: data.to_vec().into(),
348
+ }
349
+ };
350
+ self.extents.insert(index, new_extent);
351
+ }
352
+ }
353
+ self.size = self.size.max(original_extent_offset + data.len() as u64);
354
+
355
+ // Update the cursor
356
+ *cursor += data.len() as u64;
357
+ Ok(data.len())
358
+ }
359
+
360
+ pub fn flush(&mut self) -> io::Result<()> {
361
+ Ok(())
362
+ }
363
+
364
+ pub fn resize(&mut self, new_len: u64, value: u8) {
365
+ let mut cur_len = self.len();
366
+ if new_len > cur_len {
367
+ self.extents.push(FileExtent::RepeatingBytes {
368
+ value,
369
+ cnt: new_len - cur_len,
370
+ });
371
+ }
372
+ while cur_len > new_len && !self.extents.is_empty() {
373
+ let extent: &mut FileExtent = self.extents.last_mut().unwrap();
374
+ let diff = extent.size().min(cur_len - new_len);
375
+ extent.resize(extent.size() - diff);
376
+ cur_len -= diff;
377
+ if extent.size() == 0 {
378
+ self.extents.pop();
379
+ }
380
+ }
381
+ self.size = new_len;
382
+ }
383
+
384
+ pub fn len(&self) -> u64 {
385
+ self.size
386
+ }
387
+
388
+ pub fn truncate(&mut self) {
389
+ self.extents.clear();
390
+ self.size = 0;
391
+ }
392
+ }
393
+
394
+ #[cfg(test)]
395
+ mod tests {
396
+ use super::*;
397
+ use std::io;
398
+
399
+ #[test]
400
+ #[tracing_test::traced_test]
401
+ pub fn test_offload_file() -> io::Result<()> {
402
+ let buffer = OwnedBuffer::from_bytes(std::iter::repeat_n(12u8, 100).collect::<Vec<_>>());
403
+ let test_data2 = buffer.clone();
404
+
405
+ let backing = OffloadBackingStore::new(buffer, None);
406
+ let mut file = OffloadedFile::new(None, backing);
407
+
408
+ let mut cursor = 0u64;
409
+ let test_data = std::iter::repeat_n(56u8, 100).collect::<Vec<_>>();
410
+ file.write(OffloadWrite::Buffer(&test_data), &mut cursor)?;
411
+
412
+ assert_eq!(file.len(), 100);
413
+
414
+ cursor = 0;
415
+ let mut result = std::iter::repeat_n(0u8, 100).collect::<Vec<_>>();
416
+ file.read(&mut result, &mut cursor)?;
417
+ assert_eq!(&result, &std::iter::repeat_n(56u8, 100).collect::<Vec<_>>());
418
+
419
+ cursor = 50;
420
+ file.write(OffloadWrite::Buffer(&test_data2), &mut cursor)?;
421
+
422
+ assert_eq!(file.len(), 150);
423
+
424
+ cursor = 0;
425
+ let mut result = std::iter::repeat_n(0u8, 150).collect::<Vec<_>>();
426
+ file.read(&mut result, &mut cursor)?;
427
+ assert_eq!(
428
+ &result,
429
+ &std::iter::repeat_n(56u8, 50)
430
+ .chain(std::iter::repeat_n(12u8, 100))
431
+ .collect::<Vec<_>>()
432
+ );
433
+
434
+ file.resize(200, 99u8);
435
+ assert_eq!(file.len(), 200);
436
+
437
+ cursor = 0;
438
+ let mut result = std::iter::repeat_n(0u8, 200).collect::<Vec<_>>();
439
+ file.read(&mut result, &mut cursor)?;
440
+ assert_eq!(
441
+ &result,
442
+ &std::iter::repeat_n(56u8, 50)
443
+ .chain(std::iter::repeat_n(12u8, 100))
444
+ .chain(std::iter::repeat_n(99u8, 50))
445
+ .collect::<Vec<_>>()
446
+ );
447
+
448
+ file.resize(33, 1u8);
449
+
450
+ cursor = 0;
451
+ let mut result = std::iter::repeat_n(0u8, 33).collect::<Vec<_>>();
452
+ file.read(&mut result, &mut cursor)?;
453
+ assert_eq!(&result, &std::iter::repeat_n(56u8, 33).collect::<Vec<_>>());
454
+
455
+ let mut cursor = 10u64;
456
+ let test_data = std::iter::repeat_n(74u8, 10).collect::<Vec<_>>();
457
+ file.write(OffloadWrite::Buffer(&test_data), &mut cursor)?;
458
+
459
+ assert_eq!(file.len(), 33);
460
+
461
+ cursor = 0;
462
+ let mut result = std::iter::repeat_n(0u8, 33).collect::<Vec<_>>();
463
+ file.read(&mut result, &mut cursor)?;
464
+ assert_eq!(
465
+ &result,
466
+ &std::iter::repeat_n(56u8, 10)
467
+ .chain(std::iter::repeat_n(74u8, 10))
468
+ .chain(std::iter::repeat_n(56u8, 13))
469
+ .collect::<Vec<_>>()
470
+ );
471
+
472
+ Ok(())
473
+ }
474
+ }