@wasm-oj/server 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +6 -0
- package/THIRD_PARTY_NOTICES.md +302 -0
- package/crates/runtime-core/Cargo.lock +5099 -0
- package/crates/runtime-core/Cargo.toml +66 -0
- package/crates/runtime-core/README.md +47 -0
- package/crates/runtime-core/src/bin/wasm-oj-compiler.rs +418 -0
- package/crates/runtime-core/src/bin/wasm-oj-runner.rs +294 -0
- package/crates/runtime-core/src/capabilities.rs +118 -0
- package/crates/runtime-core/src/compiler.rs +658 -0
- package/crates/runtime-core/src/contract.rs +5 -0
- package/crates/runtime-core/src/deterministic.rs +1051 -0
- package/crates/runtime-core/src/error.rs +58 -0
- package/crates/runtime-core/src/filesystem.rs +547 -0
- package/crates/runtime-core/src/filesystem_quota.rs +167 -0
- package/crates/runtime-core/src/go_compiler_session.rs +297 -0
- package/crates/runtime-core/src/interactive.rs +1019 -0
- package/crates/runtime-core/src/judge_package.rs +1539 -0
- package/crates/runtime-core/src/lib.rs +98 -0
- package/crates/runtime-core/src/memory.rs +84 -0
- package/crates/runtime-core/src/meter.rs +549 -0
- package/crates/runtime-core/src/module_imports.rs +149 -0
- package/crates/runtime-core/src/module_policy.rs +714 -0
- package/crates/runtime-core/src/output.rs +204 -0
- package/crates/runtime-core/src/run/mod.rs +208 -0
- package/crates/runtime-core/src/run/native.rs +260 -0
- package/crates/runtime-core/src/run/web.rs +229 -0
- package/crates/runtime-core/src/run/web_runtime.rs +109 -0
- package/crates/runtime-core/src/types.rs +268 -0
- package/crates/runtime-core/src/web.rs +83 -0
- package/dist/chunks/go-toolchain-Dbt-lp2L.js +426 -0
- package/dist/chunks/java-toolchain-DajoRCHu.js +44 -0
- package/dist/chunks/python-toolchain-Dx834o2A.js +4 -0
- package/dist/chunks/rust-toolchain-CJ3sMxPE.js +252 -0
- package/dist/chunks/toolchains-C6KuA1yM.js +224 -0
- package/dist/go-stage.mjs +193 -0
- package/dist/index.d.ts +221 -0
- package/dist/index.js +4393 -0
- package/dist/java-stage.mjs +111 -0
- package/dist/python-stage.mjs +90 -0
- package/dist/rustc-stage.mjs +301 -0
- package/dist/server-build-stage.mjs +2564 -0
- package/dist/server-runner-stage.mjs +155 -0
- package/licenses/fflate-MIT.txt +21 -0
- package/licenses/runtime-core-dependencies.html +6253 -0
- package/licenses/runtime-core-dependencies.json +3041 -0
- package/licenses/wasmer-sdk-MIT.txt +21 -0
- package/licenses/wasmer-sdk-dependencies.html +6901 -0
- package/licenses/wasmer-sdk-dependencies.json +3013 -0
- package/package.json +70 -0
- package/rust-toolchain.toml +5 -0
- package/testdata/wojjdg02-v2-text.hex +1 -0
- package/vendor/shared-buffer/Cargo.toml +22 -0
- package/vendor/shared-buffer/LICENSE_APACHE.md +176 -0
- package/vendor/shared-buffer/LICENSE_MIT.md +25 -0
- package/vendor/shared-buffer/README.md +34 -0
- package/vendor/shared-buffer/src/lib.rs +58 -0
- package/vendor/shared-buffer/src/mmap.rs +250 -0
- package/vendor/shared-buffer/src/owned.rs +389 -0
- package/vendor/virtual-fs/Cargo.toml +181 -0
- package/vendor/virtual-fs/LICENSE +25 -0
- package/vendor/virtual-fs/src/arc_box_file.rs +142 -0
- package/vendor/virtual-fs/src/arc_file.rs +182 -0
- package/vendor/virtual-fs/src/arc_fs.rs +68 -0
- package/vendor/virtual-fs/src/buffer_file.rs +103 -0
- package/vendor/virtual-fs/src/builder.rs +232 -0
- package/vendor/virtual-fs/src/combine_file.rs +101 -0
- package/vendor/virtual-fs/src/cow_file.rs +345 -0
- package/vendor/virtual-fs/src/dual_write_file.rs +113 -0
- package/vendor/virtual-fs/src/empty_fs.rs +81 -0
- package/vendor/virtual-fs/src/filesystems.rs +108 -0
- package/vendor/virtual-fs/src/host_fs.rs +1390 -0
- package/vendor/virtual-fs/src/lib.rs +782 -0
- package/vendor/virtual-fs/src/limiter.rs +252 -0
- package/vendor/virtual-fs/src/mem_fs/file.rs +1799 -0
- package/vendor/virtual-fs/src/mem_fs/file_opener.rs +941 -0
- package/vendor/virtual-fs/src/mem_fs/filesystem.rs +2134 -0
- package/vendor/virtual-fs/src/mem_fs/mod.rs +245 -0
- package/vendor/virtual-fs/src/mem_fs/offloaded_file.rs +474 -0
- package/vendor/virtual-fs/src/mem_fs/stdio.rs +318 -0
- package/vendor/virtual-fs/src/mount_fs.rs +2225 -0
- package/vendor/virtual-fs/src/null_file.rs +87 -0
- package/vendor/virtual-fs/src/ops.rs +364 -0
- package/vendor/virtual-fs/src/overlay_fs.rs +2216 -0
- package/vendor/virtual-fs/src/passthru_fs.rs +119 -0
- package/vendor/virtual-fs/src/pipe.rs +603 -0
- package/vendor/virtual-fs/src/random_file.rs +88 -0
- package/vendor/virtual-fs/src/special_file.rs +108 -0
- package/vendor/virtual-fs/src/static_file.rs +133 -0
- package/vendor/virtual-fs/src/static_fs.rs +460 -0
- package/vendor/virtual-fs/src/tmp_fs.rs +95 -0
- package/vendor/virtual-fs/src/trace_fs.rs +258 -0
- package/vendor/virtual-fs/src/webc_volume_fs.rs +829 -0
- package/vendor/virtual-fs/src/zero_file.rs +90 -0
|
@@ -0,0 +1,658 @@
|
|
|
1
|
+
use crate::capabilities::attach_capability_denials;
|
|
2
|
+
use crate::deterministic::{COMPILER_DETERMINISM, attach_deterministic_imports};
|
|
3
|
+
use crate::filesystem::{
|
|
4
|
+
RuntimeProjectFilesystem, compiler_project_files, compiler_project_files_with_base,
|
|
5
|
+
is_normalized_guest_path, read_files,
|
|
6
|
+
};
|
|
7
|
+
use crate::module_imports::attach_imported_memory;
|
|
8
|
+
use crate::module_policy::{DEFERRED_START_EXPORT, defer_start_section, validate_memory_limit};
|
|
9
|
+
use crate::output::{OutputBudget, OutputCapture};
|
|
10
|
+
use crate::{
|
|
11
|
+
CompilePipelineResult, CompileRequest, CompileResponse, CompileResult, CompilerToolchainConfig,
|
|
12
|
+
ExecutionTermination, RunError, RunFailure,
|
|
13
|
+
};
|
|
14
|
+
use std::collections::HashMap;
|
|
15
|
+
use std::io::Write;
|
|
16
|
+
use std::sync::{Arc, Mutex};
|
|
17
|
+
use wasmer::{AsStoreMut, Engine, Imports, Instance, Memory, Module, Store};
|
|
18
|
+
use wasmer_types::StoreId;
|
|
19
|
+
use wasmer_wasix::bin_factory::{BinaryPackage, spawn_exec};
|
|
20
|
+
use wasmer_wasix::runtime::module_cache::{self, ModuleCache};
|
|
21
|
+
use wasmer_wasix::runtime::package_loader::BuiltinPackageLoader;
|
|
22
|
+
use wasmer_wasix::{
|
|
23
|
+
Pipe, PluggableRuntime, Runtime, WasiEnv, WasiError, WasiModuleInstanceHandles,
|
|
24
|
+
WasiModuleTreeHandles, wasmer_wasix_types,
|
|
25
|
+
};
|
|
26
|
+
use webc::Container;
|
|
27
|
+
|
|
28
|
+
/// A content-verified compiler package reusable across isolated invocations.
|
|
29
|
+
///
|
|
30
|
+
/// Package command metadata remains authoritative for argv, imported memory,
|
|
31
|
+
/// and child commands. Each `compile` call creates a fresh process tree,
|
|
32
|
+
/// deterministic import state, and in-memory filesystem.
|
|
33
|
+
pub struct CompilerToolchain {
|
|
34
|
+
engine: Engine,
|
|
35
|
+
package: BinaryPackage,
|
|
36
|
+
/// Compiled-module cache shared by every compile lifecycle.
|
|
37
|
+
///
|
|
38
|
+
/// Modules are immutable once compiled, so sharing the cache keeps the
|
|
39
|
+
/// per-request isolation contract (fresh store, instance, WASI state, and
|
|
40
|
+
/// filesystem per compile) while avoiding a full atom recompilation on
|
|
41
|
+
/// every request.
|
|
42
|
+
module_cache: Arc<dyn ModuleCache + Send + Sync>,
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
impl CompilerToolchain {
|
|
46
|
+
pub fn new(config: CompilerToolchainConfig) -> Result<Self, RunError> {
|
|
47
|
+
validate_toolchain_config(&config)?;
|
|
48
|
+
with_native_runtime(|| {
|
|
49
|
+
let version = webc::detect(&config.package[..]).map_err(|error| {
|
|
50
|
+
RunError::Compile(format!("failed to detect compiler package: {error}"))
|
|
51
|
+
})?;
|
|
52
|
+
let container = Container::from_bytes_and_version(config.package.into(), version)
|
|
53
|
+
.map_err(|error| {
|
|
54
|
+
RunError::Compile(format!("failed to parse compiler package: {error}"))
|
|
55
|
+
})?;
|
|
56
|
+
for (name, command) in &container.manifest().commands {
|
|
57
|
+
let atom = command
|
|
58
|
+
.atom()
|
|
59
|
+
.map_err(|error| {
|
|
60
|
+
RunError::Compile(format!(
|
|
61
|
+
"failed to inspect compiler command '{name}': {error}"
|
|
62
|
+
))
|
|
63
|
+
})?
|
|
64
|
+
.ok_or_else(|| {
|
|
65
|
+
RunError::Compile(format!(
|
|
66
|
+
"compiler command '{name}' has no atom annotation"
|
|
67
|
+
))
|
|
68
|
+
})?;
|
|
69
|
+
let bytes = container.get_atom(&atom.name).ok_or_else(|| {
|
|
70
|
+
RunError::Compile(format!(
|
|
71
|
+
"compiler command '{name}' references missing atom '{}'",
|
|
72
|
+
atom.name
|
|
73
|
+
))
|
|
74
|
+
})?;
|
|
75
|
+
validate_memory_limit(bytes.as_ref(), config.memory_limit_bytes)
|
|
76
|
+
.map_err(RunError::Compile)?;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
let engine = compiler_engine(config.memory_limit_bytes)?;
|
|
80
|
+
let module_cache: Arc<dyn ModuleCache + Send + Sync> =
|
|
81
|
+
Arc::new(module_cache::in_memory());
|
|
82
|
+
let mut loader_runtime = compiler_runtime_base(engine.clone());
|
|
83
|
+
loader_runtime.set_package_loader(BuiltinPackageLoader::new());
|
|
84
|
+
loader_runtime.module_cache = module_cache.clone();
|
|
85
|
+
let loader_runtime: Arc<dyn Runtime + Send + Sync> = Arc::new(loader_runtime);
|
|
86
|
+
let package = futures::executor::block_on(BinaryPackage::from_webc(
|
|
87
|
+
&container,
|
|
88
|
+
loader_runtime.as_ref(),
|
|
89
|
+
))
|
|
90
|
+
.map_err(|error| {
|
|
91
|
+
RunError::Compile(format!("failed to load compiler package: {error}"))
|
|
92
|
+
})?;
|
|
93
|
+
if package.commands.is_empty() {
|
|
94
|
+
return Err(RunError::Compile(
|
|
95
|
+
"compiler package exposes no commands".to_string(),
|
|
96
|
+
));
|
|
97
|
+
}
|
|
98
|
+
Ok(Self {
|
|
99
|
+
engine,
|
|
100
|
+
package,
|
|
101
|
+
module_cache,
|
|
102
|
+
})
|
|
103
|
+
})
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
pub async fn compile(&self, request: CompileRequest) -> Result<CompileResult, RunError> {
|
|
107
|
+
validate_compile_request(&request, &self.package)?;
|
|
108
|
+
crate::run::validate_mounted_files(&request.files, "compiler")?;
|
|
109
|
+
let project_filesystem = compiler_project_files(&request.files, &request.output_paths)?;
|
|
110
|
+
self.compile_in_runtime(request, project_filesystem).await
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
pub async fn compile_pipeline(
|
|
114
|
+
&self,
|
|
115
|
+
files: std::collections::BTreeMap<String, serde_bytes::ByteBuf>,
|
|
116
|
+
stages: Vec<CompileRequest>,
|
|
117
|
+
) -> Result<CompilePipelineResult, RunError> {
|
|
118
|
+
self.validate_pipeline(&files, &stages)?;
|
|
119
|
+
let output_paths = pipeline_output_paths(&stages);
|
|
120
|
+
let filesystem = compiler_project_files(&files, &output_paths)?;
|
|
121
|
+
self.compile_pipeline_in_filesystem(stages, filesystem)
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/// Compile a source snapshot over a session-owned immutable filesystem.
|
|
125
|
+
///
|
|
126
|
+
/// The immutable base is referenced by the per-build copy-on-write overlay;
|
|
127
|
+
/// its bytes are not copied into each pipeline request or mutable project
|
|
128
|
+
/// filesystem.
|
|
129
|
+
pub async fn compile_pipeline_with_base(
|
|
130
|
+
&self,
|
|
131
|
+
files: std::collections::BTreeMap<String, serde_bytes::ByteBuf>,
|
|
132
|
+
stages: Vec<CompileRequest>,
|
|
133
|
+
immutable: Arc<dyn virtual_fs::FileSystem + Send + Sync>,
|
|
134
|
+
) -> Result<CompilePipelineResult, RunError> {
|
|
135
|
+
self.validate_pipeline(&files, &stages)?;
|
|
136
|
+
let output_paths = pipeline_output_paths(&stages);
|
|
137
|
+
let filesystem = compiler_project_files_with_base(&files, &output_paths, immutable)?;
|
|
138
|
+
self.compile_pipeline_in_filesystem(stages, filesystem)
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
fn validate_pipeline(
|
|
142
|
+
&self,
|
|
143
|
+
files: &std::collections::BTreeMap<String, serde_bytes::ByteBuf>,
|
|
144
|
+
stages: &[CompileRequest],
|
|
145
|
+
) -> Result<(), RunError> {
|
|
146
|
+
if stages.is_empty() {
|
|
147
|
+
return Err(RunError::InvalidRequest(
|
|
148
|
+
"a compiler pipeline requires at least one stage".to_string(),
|
|
149
|
+
));
|
|
150
|
+
}
|
|
151
|
+
crate::run::validate_mounted_files(files, "compiler")?;
|
|
152
|
+
for stage in stages {
|
|
153
|
+
validate_compile_request(stage, &self.package)?;
|
|
154
|
+
if !stage.files.is_empty() {
|
|
155
|
+
return Err(RunError::InvalidRequest(
|
|
156
|
+
"compiler pipeline stage files must be declared in the shared file set"
|
|
157
|
+
.to_string(),
|
|
158
|
+
));
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
Ok(())
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
fn compile_pipeline_in_filesystem(
|
|
165
|
+
&self,
|
|
166
|
+
stages: Vec<CompileRequest>,
|
|
167
|
+
filesystem: RuntimeProjectFilesystem,
|
|
168
|
+
) -> Result<CompilePipelineResult, RunError> {
|
|
169
|
+
let mut results = Vec::with_capacity(stages.len());
|
|
170
|
+
for stage in stages {
|
|
171
|
+
let result = self.compile_direct(stage, &filesystem)?;
|
|
172
|
+
let succeeded = result.code == 0;
|
|
173
|
+
results.push(result);
|
|
174
|
+
if !succeeded {
|
|
175
|
+
break;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
Ok(CompilePipelineResult { stages: results })
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
pub async fn compile_response(&self, request: CompileRequest) -> CompileResponse {
|
|
182
|
+
match self.compile(request).await {
|
|
183
|
+
Ok(result) => CompileResponse {
|
|
184
|
+
ok: true,
|
|
185
|
+
result: Some(result),
|
|
186
|
+
error: None,
|
|
187
|
+
},
|
|
188
|
+
Err(error) => CompileResponse {
|
|
189
|
+
ok: false,
|
|
190
|
+
result: None,
|
|
191
|
+
error: Some(RunFailure {
|
|
192
|
+
code: error.code(),
|
|
193
|
+
message: error.to_string(),
|
|
194
|
+
}),
|
|
195
|
+
},
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
async fn compile_in_runtime(
|
|
200
|
+
&self,
|
|
201
|
+
request: CompileRequest,
|
|
202
|
+
project_filesystem: RuntimeProjectFilesystem,
|
|
203
|
+
) -> Result<CompileResult, RunError> {
|
|
204
|
+
let filesystem = project_filesystem.filesystem();
|
|
205
|
+
let (mut stdin_writer, stdin_reader) = Pipe::channel();
|
|
206
|
+
stdin_writer
|
|
207
|
+
.write_all(&request.stdin)
|
|
208
|
+
.map_err(|error| RunError::Io(error.to_string()))?;
|
|
209
|
+
drop(stdin_writer);
|
|
210
|
+
|
|
211
|
+
let output_limit = usize::try_from(request.output_limit_bytes)
|
|
212
|
+
.map_err(|_| RunError::InvalidRequest("output limit exceeds host range".to_string()))?;
|
|
213
|
+
let output_budget = OutputBudget::new(output_limit);
|
|
214
|
+
let (stdout_capture, stdout_file) = OutputCapture::new(output_budget.clone(), 1);
|
|
215
|
+
let (stderr_capture, stderr_file) = OutputCapture::new(output_budget, 2);
|
|
216
|
+
let runtime = compiler_runtime(
|
|
217
|
+
self.engine.clone(),
|
|
218
|
+
&COMPILER_DETERMINISM,
|
|
219
|
+
self.module_cache.clone(),
|
|
220
|
+
)?;
|
|
221
|
+
|
|
222
|
+
let mut builder = WasiEnv::builder(&request.command)
|
|
223
|
+
.runtime(runtime.clone())
|
|
224
|
+
.args(request.args.clone())
|
|
225
|
+
.envs(request.env.clone())
|
|
226
|
+
.stdin(Box::new(stdin_reader))
|
|
227
|
+
.stdout(Box::new(stdout_file))
|
|
228
|
+
.stderr(Box::new(stderr_file))
|
|
229
|
+
.fs(filesystem.clone())
|
|
230
|
+
.use_webc(self.package.clone());
|
|
231
|
+
builder
|
|
232
|
+
.add_preopen_build(|directory| {
|
|
233
|
+
directory.directory("/").read(true).write(true).create(true)
|
|
234
|
+
})
|
|
235
|
+
.map_err(|error| {
|
|
236
|
+
RunError::InvalidRequest(format!("failed to preopen compiler filesystem: {error}"))
|
|
237
|
+
})?;
|
|
238
|
+
if let Some(cwd) = &request.cwd {
|
|
239
|
+
builder.set_current_dir(cwd);
|
|
240
|
+
}
|
|
241
|
+
let env = builder.build().map_err(|error| {
|
|
242
|
+
RunError::Compile(format!(
|
|
243
|
+
"failed to build compiler WASI environment: {error}"
|
|
244
|
+
))
|
|
245
|
+
})?;
|
|
246
|
+
let command = self.package.get_command(&request.command).ok_or_else(|| {
|
|
247
|
+
RunError::InvalidRequest(format!(
|
|
248
|
+
"compiler package does not expose command '{}'",
|
|
249
|
+
request.command
|
|
250
|
+
))
|
|
251
|
+
})?;
|
|
252
|
+
env.prepare_spawn(command);
|
|
253
|
+
let mut handle = spawn_exec(self.package.clone(), &request.command, env, &runtime)
|
|
254
|
+
.await
|
|
255
|
+
.map_err(|error| {
|
|
256
|
+
RunError::Compile(format!("failed to start compiler command: {error}"))
|
|
257
|
+
})?;
|
|
258
|
+
let exit_code = handle
|
|
259
|
+
.wait_finished()
|
|
260
|
+
.await
|
|
261
|
+
.map_err(|error| RunError::Runtime(format!("compiler command failed: {error}")))?;
|
|
262
|
+
|
|
263
|
+
let stdout = stdout_capture.bytes();
|
|
264
|
+
let stderr = stderr_capture.bytes();
|
|
265
|
+
let output_exceeded = stdout_capture.exceeded() || stderr_capture.exceeded();
|
|
266
|
+
let output_files = read_files(&filesystem, &request.output_paths)?;
|
|
267
|
+
let (code, termination) = if project_filesystem.quota_exceeded() {
|
|
268
|
+
(137, ExecutionTermination::FilesystemLimit)
|
|
269
|
+
} else if output_exceeded {
|
|
270
|
+
(137, ExecutionTermination::OutputLimit)
|
|
271
|
+
} else {
|
|
272
|
+
(exit_code.raw(), ExecutionTermination::Exited)
|
|
273
|
+
};
|
|
274
|
+
Ok(CompileResult {
|
|
275
|
+
code,
|
|
276
|
+
stdout,
|
|
277
|
+
stderr,
|
|
278
|
+
output_files,
|
|
279
|
+
termination,
|
|
280
|
+
})
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/// Execute one compiler stage directly in a fresh Wasmer store.
|
|
284
|
+
///
|
|
285
|
+
/// Compiler pipelines deliberately share only their virtual filesystem.
|
|
286
|
+
/// Every stage still receives a new store, WASI environment, deterministic
|
|
287
|
+
/// host state, and module instance. Direct instantiation also keeps browser
|
|
288
|
+
/// and native behavior aligned for toolchains that perform substantial
|
|
289
|
+
/// random-access file I/O, such as the standard Go compiler and linker.
|
|
290
|
+
fn compile_direct(
|
|
291
|
+
&self,
|
|
292
|
+
request: CompileRequest,
|
|
293
|
+
project_filesystem: &RuntimeProjectFilesystem,
|
|
294
|
+
) -> Result<CompileResult, RunError> {
|
|
295
|
+
let filesystem = project_filesystem.filesystem();
|
|
296
|
+
let command = self.package.get_command(&request.command).ok_or_else(|| {
|
|
297
|
+
RunError::InvalidRequest(format!(
|
|
298
|
+
"compiler package does not expose command '{}'",
|
|
299
|
+
request.command
|
|
300
|
+
))
|
|
301
|
+
})?;
|
|
302
|
+
let executable =
|
|
303
|
+
defer_start_section(command.atom_ref().as_ref()).map_err(RunError::Compile)?;
|
|
304
|
+
let mut store = Store::new(self.engine.clone());
|
|
305
|
+
let module = Module::new(&store, &executable.wasm).map_err(|error| {
|
|
306
|
+
RunError::Compile(format!(
|
|
307
|
+
"failed to compile compiler command '{}': {error}",
|
|
308
|
+
request.command
|
|
309
|
+
))
|
|
310
|
+
})?;
|
|
311
|
+
|
|
312
|
+
let (mut stdin_writer, stdin_reader) = Pipe::channel();
|
|
313
|
+
stdin_writer
|
|
314
|
+
.write_all(&request.stdin)
|
|
315
|
+
.map_err(|error| RunError::Io(error.to_string()))?;
|
|
316
|
+
drop(stdin_writer);
|
|
317
|
+
|
|
318
|
+
let output_limit = usize::try_from(request.output_limit_bytes)
|
|
319
|
+
.map_err(|_| RunError::InvalidRequest("output limit exceeds host range".to_string()))?;
|
|
320
|
+
let output_budget = OutputBudget::new(output_limit);
|
|
321
|
+
let (stdout_capture, stdout_file) = OutputCapture::new(output_budget.clone(), 1);
|
|
322
|
+
let (stderr_capture, stderr_file) = OutputCapture::new(output_budget, 2);
|
|
323
|
+
let runtime = compiler_runtime(
|
|
324
|
+
self.engine.clone(),
|
|
325
|
+
&COMPILER_DETERMINISM,
|
|
326
|
+
self.module_cache.clone(),
|
|
327
|
+
)?;
|
|
328
|
+
|
|
329
|
+
let mut builder = WasiEnv::builder(&request.command)
|
|
330
|
+
.runtime(runtime)
|
|
331
|
+
.args(request.args.clone())
|
|
332
|
+
.envs(request.env.clone())
|
|
333
|
+
.stdin(Box::new(stdin_reader))
|
|
334
|
+
.stdout(Box::new(stdout_file))
|
|
335
|
+
.stderr(Box::new(stderr_file))
|
|
336
|
+
.fs(filesystem.clone());
|
|
337
|
+
builder
|
|
338
|
+
.add_preopen_build(|directory| {
|
|
339
|
+
directory.directory("/").read(true).write(true).create(true)
|
|
340
|
+
})
|
|
341
|
+
.map_err(|error| {
|
|
342
|
+
RunError::InvalidRequest(format!("failed to preopen compiler filesystem: {error}"))
|
|
343
|
+
})?;
|
|
344
|
+
if let Some(cwd) = &request.cwd {
|
|
345
|
+
builder.set_current_dir(cwd);
|
|
346
|
+
}
|
|
347
|
+
let mut sandbox = builder.finalize(&mut store).map_err(|error| {
|
|
348
|
+
RunError::Compile(format!(
|
|
349
|
+
"failed to finalize compiler WASI environment: {error}"
|
|
350
|
+
))
|
|
351
|
+
})?;
|
|
352
|
+
let mut imports = sandbox
|
|
353
|
+
.import_object_for_all_wasi_versions(&mut store, &module)
|
|
354
|
+
.map_err(|error| {
|
|
355
|
+
RunError::Compile(format!("failed to create compiler WASI imports: {error}"))
|
|
356
|
+
})?;
|
|
357
|
+
let memory_slot: Arc<Mutex<Option<Memory>>> = Arc::new(Mutex::new(None));
|
|
358
|
+
attach_deterministic_imports(
|
|
359
|
+
&mut store,
|
|
360
|
+
&mut imports,
|
|
361
|
+
memory_slot.clone(),
|
|
362
|
+
&COMPILER_DETERMINISM,
|
|
363
|
+
crate::deterministic::VirtualClock::unbounded(&COMPILER_DETERMINISM),
|
|
364
|
+
0,
|
|
365
|
+
);
|
|
366
|
+
attach_capability_denials(&mut store, &module, &mut imports).map_err(RunError::Compile)?;
|
|
367
|
+
let imported_memory =
|
|
368
|
+
attach_imported_memory(&mut store, &module, &mut imports).map_err(RunError::Compile)?;
|
|
369
|
+
let instance = Instance::new(&mut store, &module, &imports).map_err(|error| {
|
|
370
|
+
RunError::Compile(format!(
|
|
371
|
+
"failed to instantiate compiler command '{}': {error}",
|
|
372
|
+
request.command
|
|
373
|
+
))
|
|
374
|
+
})?;
|
|
375
|
+
let guest_memory = instance
|
|
376
|
+
.exports
|
|
377
|
+
.get_memory("memory")
|
|
378
|
+
.cloned()
|
|
379
|
+
.ok()
|
|
380
|
+
.or(imported_memory)
|
|
381
|
+
.ok_or_else(|| {
|
|
382
|
+
RunError::Compile(format!(
|
|
383
|
+
"compiler command '{}' has no guest linear memory",
|
|
384
|
+
request.command
|
|
385
|
+
))
|
|
386
|
+
})?;
|
|
387
|
+
*memory_slot
|
|
388
|
+
.lock()
|
|
389
|
+
.map_err(|error| RunError::Runtime(error.to_string()))? = Some(guest_memory.clone());
|
|
390
|
+
let handles = WasiModuleTreeHandles::Static(WasiModuleInstanceHandles::new(
|
|
391
|
+
guest_memory,
|
|
392
|
+
&store,
|
|
393
|
+
instance.clone(),
|
|
394
|
+
None,
|
|
395
|
+
));
|
|
396
|
+
sandbox
|
|
397
|
+
.initialize_handles_and_layout(&mut store, instance.clone(), handles, None, true)
|
|
398
|
+
.map_err(|error| {
|
|
399
|
+
RunError::Compile(format!("failed to initialize compiler instance: {error}"))
|
|
400
|
+
})?;
|
|
401
|
+
let start = instance.exports.get_function("_start").map_err(|error| {
|
|
402
|
+
RunError::Compile(format!(
|
|
403
|
+
"compiler command '{}' has no _start function: {error}",
|
|
404
|
+
request.command
|
|
405
|
+
))
|
|
406
|
+
})?;
|
|
407
|
+
let execution = if executable.has_deferred_start {
|
|
408
|
+
let initializer = instance
|
|
409
|
+
.exports
|
|
410
|
+
.get_function(DEFERRED_START_EXPORT)
|
|
411
|
+
.map_err(|error| {
|
|
412
|
+
RunError::Runtime(format!(
|
|
413
|
+
"compiler deferred start function is unavailable: {error}"
|
|
414
|
+
))
|
|
415
|
+
})?;
|
|
416
|
+
match initializer.call(&mut store, &[]) {
|
|
417
|
+
Ok(_) => start.call(&mut store, &[]),
|
|
418
|
+
Err(error) => Err(error),
|
|
419
|
+
}
|
|
420
|
+
} else {
|
|
421
|
+
start.call(&mut store, &[])
|
|
422
|
+
};
|
|
423
|
+
|
|
424
|
+
let mut code = 0;
|
|
425
|
+
if let Err(error) = execution {
|
|
426
|
+
if let Some(wasi_error) = crate::wasi_error(&error) {
|
|
427
|
+
match wasi_error {
|
|
428
|
+
WasiError::Exit(exit) => {
|
|
429
|
+
let errno: wasmer_wasix_types::wasi::Errno = (*exit).into();
|
|
430
|
+
code = errno as i32;
|
|
431
|
+
}
|
|
432
|
+
WasiError::UnknownWasiVersion => {
|
|
433
|
+
return Err(RunError::WasiUnsupported(
|
|
434
|
+
"unknown compiler WASI version".to_string(),
|
|
435
|
+
));
|
|
436
|
+
}
|
|
437
|
+
WasiError::ThreadExit => {
|
|
438
|
+
return Err(RunError::WasiUnsupported(
|
|
439
|
+
"compiler thread exit".to_string(),
|
|
440
|
+
));
|
|
441
|
+
}
|
|
442
|
+
WasiError::DeepSleep(_) => {
|
|
443
|
+
return Err(RunError::WasiUnsupported("compiler deep sleep".to_string()));
|
|
444
|
+
}
|
|
445
|
+
WasiError::DlSymbolResolutionFailed(symbol) => {
|
|
446
|
+
return Err(RunError::WasiUnsupported(format!(
|
|
447
|
+
"compiler unresolved symbol {symbol}"
|
|
448
|
+
)));
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
} else {
|
|
452
|
+
return Err(RunError::Runtime(format!(
|
|
453
|
+
"compiler command '{}' trapped: {error}",
|
|
454
|
+
request.command
|
|
455
|
+
)));
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
sandbox.on_exit(
|
|
459
|
+
&mut store,
|
|
460
|
+
Some(wasmer_wasix_types::wasi::Errno::Success.into()),
|
|
461
|
+
);
|
|
462
|
+
|
|
463
|
+
let stdout = stdout_capture.bytes();
|
|
464
|
+
let stderr = stderr_capture.bytes();
|
|
465
|
+
let output_exceeded = stdout_capture.exceeded() || stderr_capture.exceeded();
|
|
466
|
+
let output_files = read_files(&filesystem, &request.output_paths)?;
|
|
467
|
+
let (code, termination) = if project_filesystem.quota_exceeded() {
|
|
468
|
+
(137, ExecutionTermination::FilesystemLimit)
|
|
469
|
+
} else if output_exceeded {
|
|
470
|
+
(137, ExecutionTermination::OutputLimit)
|
|
471
|
+
} else {
|
|
472
|
+
(code, ExecutionTermination::Exited)
|
|
473
|
+
};
|
|
474
|
+
Ok(CompileResult {
|
|
475
|
+
code,
|
|
476
|
+
stdout,
|
|
477
|
+
stderr,
|
|
478
|
+
output_files,
|
|
479
|
+
termination,
|
|
480
|
+
})
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
fn pipeline_output_paths(stages: &[CompileRequest]) -> Vec<String> {
|
|
485
|
+
stages
|
|
486
|
+
.iter()
|
|
487
|
+
.flat_map(|stage| stage.output_paths.iter().cloned())
|
|
488
|
+
.collect()
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
fn compiler_runtime(
|
|
492
|
+
engine: Engine,
|
|
493
|
+
determinism: &crate::DeterminismConfig,
|
|
494
|
+
module_cache: Arc<dyn ModuleCache + Send + Sync>,
|
|
495
|
+
) -> Result<Arc<dyn Runtime + Send + Sync>, RunError> {
|
|
496
|
+
type MemorySlot = Arc<Mutex<Option<Memory>>>;
|
|
497
|
+
let pending: Arc<Mutex<HashMap<StoreId, MemorySlot>>> = Arc::new(Mutex::new(HashMap::new()));
|
|
498
|
+
let imports_pending = pending.clone();
|
|
499
|
+
let instance_pending = pending;
|
|
500
|
+
let config = determinism.clone();
|
|
501
|
+
let mut runtime = compiler_runtime_base(engine);
|
|
502
|
+
runtime.module_cache = module_cache;
|
|
503
|
+
runtime.with_additional_imports(move |_module, store| {
|
|
504
|
+
let store_id = store.objects_mut().id();
|
|
505
|
+
let memory = Arc::new(Mutex::new(None));
|
|
506
|
+
let mut pending = imports_pending
|
|
507
|
+
.lock()
|
|
508
|
+
.map_err(|error| std::io::Error::other(error.to_string()))?;
|
|
509
|
+
if pending.insert(store_id, memory.clone()).is_some() {
|
|
510
|
+
return Err(std::io::Error::other(
|
|
511
|
+
"deterministic import state already exists for compiler store",
|
|
512
|
+
)
|
|
513
|
+
.into());
|
|
514
|
+
}
|
|
515
|
+
drop(pending);
|
|
516
|
+
let mut imports = Imports::new();
|
|
517
|
+
attach_deterministic_imports(
|
|
518
|
+
store,
|
|
519
|
+
&mut imports,
|
|
520
|
+
memory,
|
|
521
|
+
&config,
|
|
522
|
+
crate::deterministic::VirtualClock::unbounded(&config),
|
|
523
|
+
0,
|
|
524
|
+
);
|
|
525
|
+
Ok(imports)
|
|
526
|
+
});
|
|
527
|
+
runtime.with_instance_setup(move |_module, store, instance, imported_memory| {
|
|
528
|
+
let store_id = store.objects_mut().id();
|
|
529
|
+
let memory_slot = instance_pending
|
|
530
|
+
.lock()
|
|
531
|
+
.map_err(|error| std::io::Error::other(error.to_string()))?
|
|
532
|
+
.remove(&store_id)
|
|
533
|
+
.ok_or_else(|| {
|
|
534
|
+
std::io::Error::other("compiler instance has no deterministic import state")
|
|
535
|
+
})?;
|
|
536
|
+
let memory = instance
|
|
537
|
+
.exports
|
|
538
|
+
.get_memory("memory")
|
|
539
|
+
.cloned()
|
|
540
|
+
.ok()
|
|
541
|
+
.or_else(|| imported_memory.cloned())
|
|
542
|
+
.ok_or_else(|| std::io::Error::other("compiler instance has no linear memory"))?;
|
|
543
|
+
*memory_slot
|
|
544
|
+
.lock()
|
|
545
|
+
.map_err(|error| std::io::Error::other(error.to_string()))? = Some(memory);
|
|
546
|
+
Ok(())
|
|
547
|
+
});
|
|
548
|
+
Ok(Arc::new(runtime))
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
fn validate_toolchain_config(config: &CompilerToolchainConfig) -> Result<(), RunError> {
|
|
552
|
+
if config.package.is_empty() {
|
|
553
|
+
return Err(RunError::InvalidRequest(
|
|
554
|
+
"compiler package must not be empty".to_string(),
|
|
555
|
+
));
|
|
556
|
+
}
|
|
557
|
+
if config.memory_limit_bytes == 0 || !config.memory_limit_bytes.is_multiple_of(65_536) {
|
|
558
|
+
return Err(RunError::InvalidRequest(
|
|
559
|
+
"memoryLimitBytes must be a positive multiple of 64 KiB".to_string(),
|
|
560
|
+
));
|
|
561
|
+
}
|
|
562
|
+
Ok(())
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
fn validate_compile_request(
|
|
566
|
+
request: &CompileRequest,
|
|
567
|
+
package: &BinaryPackage,
|
|
568
|
+
) -> Result<(), RunError> {
|
|
569
|
+
if request.command.is_empty() || package.get_command(&request.command).is_none() {
|
|
570
|
+
return Err(RunError::InvalidRequest(format!(
|
|
571
|
+
"compiler package does not expose command '{}'",
|
|
572
|
+
request.command
|
|
573
|
+
)));
|
|
574
|
+
}
|
|
575
|
+
if request.output_limit_bytes == 0 || usize::try_from(request.output_limit_bytes).is_err() {
|
|
576
|
+
return Err(RunError::InvalidRequest(
|
|
577
|
+
"outputLimitBytes is not representable on this host".to_string(),
|
|
578
|
+
));
|
|
579
|
+
}
|
|
580
|
+
if let Some(cwd) = &request.cwd
|
|
581
|
+
&& !is_normalized_guest_path(cwd)
|
|
582
|
+
{
|
|
583
|
+
return Err(RunError::InvalidRequest(
|
|
584
|
+
"cwd must be an absolute normalized guest path".to_string(),
|
|
585
|
+
));
|
|
586
|
+
}
|
|
587
|
+
for output_path in &request.output_paths {
|
|
588
|
+
if !is_normalized_guest_path(output_path) {
|
|
589
|
+
return Err(RunError::InvalidRequest(format!(
|
|
590
|
+
"output path must be absolute and normalized: {output_path}"
|
|
591
|
+
)));
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
Ok(())
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
#[cfg(target_arch = "wasm32")]
|
|
598
|
+
fn compiler_runtime_base(engine: Engine) -> PluggableRuntime {
|
|
599
|
+
let tasks: Arc<dyn wasmer_wasix::runtime::task_manager::VirtualTaskManager> =
|
|
600
|
+
Arc::new(crate::run::web_runtime::WebTaskManager);
|
|
601
|
+
let mut runtime = PluggableRuntime::new(tasks);
|
|
602
|
+
runtime.set_engine(engine);
|
|
603
|
+
runtime
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
#[cfg(not(target_arch = "wasm32"))]
|
|
607
|
+
fn compiler_runtime_base(engine: Engine) -> PluggableRuntime {
|
|
608
|
+
use wasmer_wasix::runtime::task_manager::tokio::TokioTaskManager;
|
|
609
|
+
|
|
610
|
+
let tasks: Arc<dyn wasmer_wasix::runtime::task_manager::VirtualTaskManager> =
|
|
611
|
+
Arc::new(TokioTaskManager::default());
|
|
612
|
+
let mut runtime = PluggableRuntime::new(tasks);
|
|
613
|
+
runtime.set_engine(engine);
|
|
614
|
+
runtime
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
#[cfg(target_arch = "wasm32")]
|
|
618
|
+
fn compiler_engine(_memory_limit_bytes: u64) -> Result<Engine, RunError> {
|
|
619
|
+
Ok(Engine::default())
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
#[cfg(not(target_arch = "wasm32"))]
|
|
623
|
+
fn compiler_engine(memory_limit_bytes: u64) -> Result<Engine, RunError> {
|
|
624
|
+
use crate::memory::LimitingTunables;
|
|
625
|
+
use wasmer::Pages;
|
|
626
|
+
use wasmer::sys::{BaseTunables, Cranelift, NativeEngineExt, Target};
|
|
627
|
+
|
|
628
|
+
let pages = u32::try_from(memory_limit_bytes / 65_536).map_err(|_| {
|
|
629
|
+
RunError::InvalidRequest("memory limit exceeds Wasmer page range".to_string())
|
|
630
|
+
})?;
|
|
631
|
+
let base = BaseTunables::for_target(&Target::default());
|
|
632
|
+
let mut engine: Engine = Cranelift::default().into();
|
|
633
|
+
engine.set_tunables(LimitingTunables::new(base, Pages(pages)));
|
|
634
|
+
Ok(engine)
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
#[cfg(target_arch = "wasm32")]
|
|
638
|
+
fn with_native_runtime<T>(operation: impl FnOnce() -> Result<T, RunError>) -> Result<T, RunError> {
|
|
639
|
+
operation()
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
#[cfg(not(target_arch = "wasm32"))]
|
|
643
|
+
fn with_native_runtime<T>(operation: impl FnOnce() -> Result<T, RunError>) -> Result<T, RunError> {
|
|
644
|
+
let runtime = if tokio::runtime::Handle::try_current().is_err() {
|
|
645
|
+
Some(
|
|
646
|
+
tokio::runtime::Builder::new_current_thread()
|
|
647
|
+
.enable_all()
|
|
648
|
+
.build()
|
|
649
|
+
.map_err(|error| {
|
|
650
|
+
RunError::Runtime(format!("failed to initialize Tokio: {error}"))
|
|
651
|
+
})?,
|
|
652
|
+
)
|
|
653
|
+
} else {
|
|
654
|
+
None
|
|
655
|
+
};
|
|
656
|
+
let _runtime_guard = runtime.as_ref().map(tokio::runtime::Runtime::enter);
|
|
657
|
+
operation()
|
|
658
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
pub const WASM_OJ_CONTRACT_VERSION: u32 = 2;
|
|
2
|
+
pub const WASM_OJ_CONTRACT_ID: &str = "wasm-oj-v2";
|
|
3
|
+
pub const WASM_OJ_COMPILE_BATCH_SCHEMA: &str = "wasm-oj-v2/compile-batch";
|
|
4
|
+
pub const WASM_OJ_INTERACTIVE_REQUEST_SCHEMA: &str = "wasm-oj-v2/interactive-request";
|
|
5
|
+
pub const WASM_OJ_RUN_REQUEST_SCHEMA: &str = "wasm-oj-v2/run-request";
|