kobako 0.27.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.release-please-manifest.json +1 -1
- data/CHANGELOG.md +241 -0
- data/Cargo.lock +132 -126
- data/README.md +240 -147
- data/ROADMAP.md +28 -22
- data/SECURITY.md +2 -2
- data/crates/kobako-runtime/CHANGELOG.md +117 -0
- data/crates/kobako-runtime/Cargo.toml +27 -6
- data/crates/kobako-runtime/README.md +16 -10
- data/crates/kobako-runtime/src/dispatch.rs +8 -0
- data/crates/kobako-runtime/src/error.rs +8 -9
- data/crates/kobako-runtime/src/filesystem/directory/spelling.rs +194 -0
- data/crates/kobako-runtime/src/filesystem/directory.rs +1082 -0
- data/crates/kobako-runtime/src/filesystem/memory.rs +1100 -0
- data/crates/kobako-runtime/src/filesystem/overlay/exclusive.rs +88 -0
- data/crates/kobako-runtime/src/filesystem/overlay/stack.rs +808 -0
- data/crates/kobako-runtime/src/filesystem/overlay.rs +1955 -0
- data/crates/kobako-runtime/src/filesystem/path.rs +60 -0
- data/crates/kobako-runtime/src/filesystem/temporary.rs +167 -0
- data/crates/kobako-runtime/src/filesystem.rs +392 -0
- data/crates/kobako-runtime/src/grant.rs +20 -0
- data/crates/kobako-runtime/src/lib.rs +12 -2
- data/crates/kobako-runtime/src/mount.rs +178 -0
- data/crates/kobako-runtime/src/profile.rs +4 -4
- data/crates/kobako-runtime/src/runtime.rs +43 -17
- data/crates/kobako-runtime/src/snapshot.rs +33 -15
- data/crates/kobako-runtime/src/yielder.rs +14 -6
- data/crates/kobako-transport/CHANGELOG.md +45 -0
- data/crates/kobako-transport/Cargo.toml +1 -1
- data/crates/kobako-transport/README.md +21 -12
- data/crates/kobako-transport/src/abi.rs +11 -14
- data/crates/kobako-transport/src/envelope/bytes.rs +18 -6
- data/crates/kobako-transport/src/envelope/compile_reply.rs +139 -0
- data/crates/kobako-transport/src/envelope/fault.rs +2 -1
- data/crates/kobako-transport/src/envelope/outcome.rs +17 -16
- data/crates/kobako-transport/src/envelope/preamble.rs +513 -0
- data/crates/kobako-transport/src/envelope/reply.rs +103 -61
- data/crates/kobako-transport/src/envelope/request.rs +305 -0
- data/crates/kobako-transport/src/envelope/run.rs +1 -2
- data/crates/kobako-transport/src/envelope/tag.rs +49 -0
- data/crates/kobako-transport/src/envelope/unsupported.rs +105 -0
- data/crates/kobako-transport/src/envelope.rs +19 -12
- data/crates/kobako-transport/src/lib.rs +0 -2
- data/crates/kobako-wasmtime/CHANGELOG.md +163 -0
- data/crates/kobako-wasmtime/Cargo.toml +23 -16
- data/crates/kobako-wasmtime/README.md +6 -4
- data/crates/kobako-wasmtime/src/abi.rs +67 -36
- data/crates/kobako-wasmtime/src/ambient.rs +64 -5
- data/crates/kobako-wasmtime/src/cache.rs +204 -127
- data/crates/kobako-wasmtime/src/capture.rs +4 -11
- data/crates/kobako-wasmtime/src/config.rs +69 -15
- data/crates/kobako-wasmtime/src/descriptors.rs +167 -0
- data/crates/kobako-wasmtime/src/dispatch.rs +95 -87
- data/crates/kobako-wasmtime/src/driver.rs +174 -125
- data/crates/kobako-wasmtime/src/exports.rs +88 -34
- data/crates/kobako-wasmtime/src/guest_mem.rs +151 -138
- data/crates/kobako-wasmtime/src/imports.rs +46 -0
- data/crates/kobako-wasmtime/src/instance_pre.rs +19 -58
- data/crates/kobako-wasmtime/src/invocation.rs +84 -412
- data/crates/kobako-wasmtime/src/io.rs +89 -0
- data/crates/kobako-wasmtime/src/lib.rs +7 -24
- data/crates/kobako-wasmtime/src/limiter.rs +253 -0
- data/crates/kobako-wasmtime/src/preview1/args.rs +60 -0
- data/crates/kobako-wasmtime/src/preview1/beneath.rs +188 -0
- data/crates/kobako-wasmtime/src/preview1/differential/mount.rs +609 -0
- data/crates/kobako-wasmtime/src/preview1/differential.rs +372 -0
- data/crates/kobako-wasmtime/src/preview1/errno.rs +267 -0
- data/crates/kobako-wasmtime/src/preview1/fd.rs +672 -0
- data/crates/kobako-wasmtime/src/preview1/memory.rs +205 -0
- data/crates/kobako-wasmtime/src/preview1/mount_view.rs +2499 -0
- data/crates/kobako-wasmtime/src/preview1/path.rs +421 -0
- data/crates/kobako-wasmtime/src/preview1/poll.rs +333 -0
- data/crates/kobako-wasmtime/src/preview1/sock.rs +30 -0
- data/crates/kobako-wasmtime/src/preview1.rs +461 -0
- data/crates/kobako-wasmtime/src/trap.rs +66 -59
- data/crates/kobako-wasmtime/src/wasi.rs +317 -0
- data/crates/kobako-wasmtime/tests/compile_posture.rs +43 -0
- data/crates/kobako-wasmtime/tests/guest_file.rs +1054 -0
- data/crates/kobako-wasmtime/tests/preamble_flags.rs +70 -0
- data/crates/kobako-wasmtime/tests/snippet_compile.rs +95 -0
- data/crates/kobako-wasmtime/tests/support/mod.rs +30 -0
- data/crates/kobako-wasmtime/tests/unanswered_call.rs +86 -0
- data/crates/kobako-wasmtime/tests/unsupported_call.rs +51 -0
- data/data/kobako.wasm +0 -0
- data/ext/kobako/Cargo.toml +4 -2
- data/ext/kobako/src/runtime/bridge.rs +177 -113
- data/ext/kobako/src/runtime/errors.rs +26 -32
- data/ext/kobako/src/runtime/escape.rs +53 -0
- data/ext/kobako/src/runtime/filesystem/grants.rs +91 -0
- data/ext/kobako/src/runtime/filesystem/ruby.rs +400 -0
- data/ext/kobako/src/runtime/filesystem.rs +236 -0
- data/ext/kobako/src/runtime/gvl.rs +11 -13
- data/ext/kobako/src/runtime/pinned_value.rs +31 -0
- data/ext/kobako/src/runtime/snapshot.rs +127 -0
- data/ext/kobako/src/runtime.rs +190 -312
- data/lib/kobako/capture.rb +1 -4
- data/lib/kobako/catalog/extensions.rb +21 -59
- data/lib/kobako/catalog/flags.rb +29 -0
- data/lib/kobako/catalog/grants.rb +30 -0
- data/lib/kobako/catalog/handles.rb +7 -54
- data/lib/kobako/catalog/mount.rb +27 -0
- data/lib/kobako/catalog/services.rb +17 -60
- data/lib/kobako/catalog/snippets.rb +41 -69
- data/lib/kobako/catalog.rb +13 -10
- data/lib/kobako/codec/decoder.rb +10 -28
- data/lib/kobako/codec/encoder.rb +3 -22
- data/lib/kobako/codec/error.rb +6 -27
- data/lib/kobako/codec/ext_types.rb +72 -94
- data/lib/kobako/codec/handle_walk.rb +65 -130
- data/lib/kobako/codec/nesting.rb +3 -8
- data/lib/kobako/codec/state.rb +21 -11
- data/lib/kobako/codec/utils.rb +6 -34
- data/lib/kobako/codec/value_budget.rb +86 -0
- data/lib/kobako/codec.rb +10 -22
- data/lib/kobako/constant_name.rb +8 -0
- data/lib/kobako/context.rb +83 -117
- data/lib/kobako/errors.rb +57 -132
- data/lib/kobako/execution.rb +16 -21
- data/lib/kobako/extension.rb +23 -40
- data/lib/kobako/filesystem.rb +38 -0
- data/lib/kobako/handle.rb +7 -12
- data/lib/kobako/outcome.rb +51 -80
- data/lib/kobako/payload/arguments.rb +1 -16
- data/lib/kobako/payload.rb +4 -12
- data/lib/kobako/pool.rb +25 -55
- data/lib/kobako/runtime.rb +7 -21
- data/lib/kobako/sandbox.rb +128 -137
- data/lib/kobako/sandbox_options.rb +71 -90
- data/lib/kobako/snippet/bytecode.rb +11 -0
- data/lib/kobako/snippet/source.rb +1 -16
- data/lib/kobako/snippet.rb +2 -10
- data/lib/kobako/transport/block_failure.rb +35 -0
- data/lib/kobako/transport/call.rb +1 -14
- data/lib/kobako/transport/dispatcher.rb +127 -206
- data/lib/kobako/transport/error.rb +6 -20
- data/lib/kobako/transport/exposure.rb +10 -17
- data/lib/kobako/transport/reflection.rb +108 -63
- data/lib/kobako/transport/run.rb +17 -71
- data/lib/kobako/transport/yielder.rb +20 -94
- data/lib/kobako/transport.rb +4 -15
- data/lib/kobako/unresolved.rb +4 -11
- data/lib/kobako/usage.rb +2 -6
- data/lib/kobako/version.rb +1 -1
- data/lib/kobako.rb +1 -0
- data/sig/kobako/catalog/extensions.rbs +2 -2
- data/sig/kobako/catalog/flags.rbs +13 -0
- data/sig/kobako/catalog/grants.rbs +13 -0
- data/sig/kobako/catalog/mount.rbs +28 -0
- data/sig/kobako/catalog/services.rbs +3 -5
- data/sig/kobako/catalog/snippets.rbs +5 -5
- data/sig/kobako/catalog.rbs +1 -0
- data/sig/kobako/codec/ext_types.rbs +10 -8
- data/sig/kobako/codec/handle_walk.rbs +9 -7
- data/sig/kobako/codec/state.rbs +7 -1
- data/sig/kobako/codec/utils.rbs +2 -2
- data/sig/kobako/codec/value_budget.rbs +23 -0
- data/sig/kobako/codec.rbs +4 -2
- data/sig/kobako/constant_name.rbs +3 -0
- data/sig/kobako/context.rbs +15 -8
- data/sig/kobako/errors.rbs +10 -4
- data/sig/kobako/filesystem.rbs +88 -0
- data/sig/kobako/guest_yielder.rbs +9 -0
- data/sig/kobako/outcome.rbs +6 -8
- data/sig/kobako/pool.rbs +1 -1
- data/sig/kobako/runtime.rbs +21 -45
- data/sig/kobako/sandbox.rbs +11 -1
- data/sig/kobako/sandbox_options.rbs +25 -7
- data/sig/kobako/snapshot.rbs +20 -0
- data/sig/kobako/snippet/bytecode.rbs +12 -0
- data/sig/kobako/transport/block_failure.rbs +13 -0
- data/sig/kobako/transport/dispatcher.rbs +15 -19
- data/sig/kobako/transport/exposure.rbs +1 -1
- data/sig/kobako/transport/reflection.rbs +19 -4
- data/sig/kobako/transport/run.rbs +1 -3
- data/sig/kobako/transport/yielder.rbs +5 -9
- data/sig/kobako/transport.rbs +9 -8
- metadata +75 -10
- data/crates/kobako-transport/src/envelope/invocation_frames.rs +0 -241
- data/crates/kobako-wasmtime/src/frames.rs +0 -273
- data/lib/kobako/snippet/binary.rb +0 -26
- data/rust-toolchain.toml +0 -15
- data/sig/kobako/snippet/binary.rbs +0 -12
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
//!
|
|
2
|
-
//! that separates the hermetic rung from permissive
|
|
3
|
-
//!
|
|
1
|
+
//! The guest's time and entropy at the WASI layer — the one ambient
|
|
2
|
+
//! authority that separates the hermetic rung from permissive. `Wasi::new`
|
|
3
|
+
//! wires the frozen sources on `Hermetic` and the live clock on
|
|
4
|
+
//! `Permissive`; `Clocks` is what the guest's waits pass on under either.
|
|
4
5
|
//!
|
|
5
6
|
//! `wasmtime-wasi`'s `WasiCtxBuilder` defaults the guest's `wasi:clocks` to
|
|
6
7
|
//! the host wall / monotonic clock and `wasi:random` to a fresh per-context
|
|
@@ -8,7 +9,7 @@
|
|
|
8
9
|
//! (`build_config/wasi.rb`), but a future libc-backed gem would silently
|
|
9
10
|
//! obtain real time and host entropy — a covert timing channel and a
|
|
10
11
|
//! nondeterminism source the hermetic posture deliberately excludes
|
|
11
|
-
//! (docs/security
|
|
12
|
+
//! (docs/guides/security.md).
|
|
12
13
|
//! Pinning the clocks to the Unix epoch and the RNG to a constant stream
|
|
13
14
|
//! makes that denial a property of the host, not merely of the gem allowlist.
|
|
14
15
|
//!
|
|
@@ -16,7 +17,7 @@
|
|
|
16
17
|
//! wasmtime epoch interruption against a host `Instant`, never the guest's
|
|
17
18
|
//! frozen `wasi:clocks/monotonic-clock`.
|
|
18
19
|
|
|
19
|
-
use std::time::Duration;
|
|
20
|
+
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
|
20
21
|
|
|
21
22
|
use wasmtime_wasi::random::Deterministic;
|
|
22
23
|
use wasmtime_wasi::{HostMonotonicClock, HostWallClock};
|
|
@@ -53,6 +54,64 @@ pub(crate) fn deterministic_rng() -> Deterministic {
|
|
|
53
54
|
Deterministic::new(vec![0])
|
|
54
55
|
}
|
|
55
56
|
|
|
57
|
+
/// The host's monotonic clock read from a fixed origin. The guest's
|
|
58
|
+
/// `clock_time_get` and a wait it subscribes to share it, so both agree on
|
|
59
|
+
/// what an instant means.
|
|
60
|
+
#[derive(Clone, Copy)]
|
|
61
|
+
pub(crate) struct LiveMonotonicClock {
|
|
62
|
+
origin: Instant,
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
impl LiveMonotonicClock {
|
|
66
|
+
pub(crate) fn new() -> Self {
|
|
67
|
+
Self {
|
|
68
|
+
origin: Instant::now(),
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
impl HostMonotonicClock for LiveMonotonicClock {
|
|
74
|
+
fn resolution(&self) -> u64 {
|
|
75
|
+
1
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
fn now(&self) -> u64 {
|
|
79
|
+
u64::try_from(self.origin.elapsed().as_nanos()).unwrap_or(u64::MAX)
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/// The clocks the guest observes. A wait it subscribes to passes on the
|
|
84
|
+
/// same clocks: frozen ones never advance, so the wait ends at once.
|
|
85
|
+
#[derive(Clone, Copy)]
|
|
86
|
+
pub(crate) enum Clocks {
|
|
87
|
+
Frozen,
|
|
88
|
+
Live(LiveMonotonicClock),
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
impl Clocks {
|
|
92
|
+
pub(crate) fn monotonic_now(&self) -> u64 {
|
|
93
|
+
match self {
|
|
94
|
+
Self::Frozen => FrozenMonotonicClock.now(),
|
|
95
|
+
Self::Live(clock) => clock.now(),
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
pub(crate) fn wall_now(&self) -> u64 {
|
|
100
|
+
let since_epoch = match self {
|
|
101
|
+
Self::Frozen => FrozenWallClock.now(),
|
|
102
|
+
Self::Live(_) => SystemTime::now()
|
|
103
|
+
.duration_since(UNIX_EPOCH)
|
|
104
|
+
.unwrap_or(Duration::ZERO),
|
|
105
|
+
};
|
|
106
|
+
u64::try_from(since_epoch.as_nanos()).unwrap_or(u64::MAX)
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/// Whether a wait on these clocks takes host time.
|
|
110
|
+
pub(crate) fn advances(&self) -> bool {
|
|
111
|
+
matches!(self, Self::Live(_))
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
56
115
|
#[cfg(test)]
|
|
57
116
|
mod tests {
|
|
58
117
|
//! Pin the frozen-clock contract: the guest must observe no real wall
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
//! Process-wide caches for the wasmtime `Engine` and compiled
|
|
2
2
|
//! `Module`, plus the on-disk compiled-artifact cache.
|
|
3
3
|
//!
|
|
4
|
-
//!
|
|
5
|
-
//!
|
|
4
|
+
//! Wasm engine types stay hidden from the Host App and downstream
|
|
5
|
+
//! gems. To amortise Engine creation and
|
|
6
6
|
//! Module JIT compilation across multiple sandbox constructions, the
|
|
7
7
|
//! driver keeps a process-scope shared Engine and a per-path Module
|
|
8
8
|
//! cache. Both are transparent to frontends, which construct a
|
|
@@ -10,21 +10,22 @@
|
|
|
10
10
|
//!
|
|
11
11
|
//! Across processes, the Cranelift compile cost is amortised by a
|
|
12
12
|
//! best-effort `.cwasm` disk cache keyed by the SHA-256 of the Guest
|
|
13
|
-
//! Binary bytes; every cache failure
|
|
14
|
-
//! back to in-process compilation.
|
|
13
|
+
//! Binary bytes and the engine that compiles them; every cache failure
|
|
14
|
+
//! falls back to in-process compilation.
|
|
15
15
|
//!
|
|
16
|
-
//! Concurrency:
|
|
17
|
-
//!
|
|
18
|
-
//!
|
|
19
|
-
//!
|
|
16
|
+
//! Concurrency: the Mutex is held only for the HashMap lookup and insert,
|
|
17
|
+
//! never across a compile, so threads building Drivers at once do not
|
|
18
|
+
//! wait on each other. Two concurrent misses on one path both compile,
|
|
19
|
+
//! and the later insert wins with an equivalent Module.
|
|
20
20
|
|
|
21
21
|
use std::collections::HashMap;
|
|
22
22
|
use std::fmt::Write as _;
|
|
23
23
|
use std::fs;
|
|
24
|
+
use std::hash::{DefaultHasher, Hash, Hasher};
|
|
24
25
|
use std::path::{Path, PathBuf};
|
|
25
26
|
use std::sync::{Mutex, OnceLock};
|
|
26
27
|
use std::thread;
|
|
27
|
-
use std::time::
|
|
28
|
+
use std::time::Duration;
|
|
28
29
|
|
|
29
30
|
use sha2::{Digest, Sha256};
|
|
30
31
|
use wasmtime::{Config as WtConfig, Engine as WtEngine, Module as WtModule};
|
|
@@ -43,21 +44,10 @@ static MODULE_CACHE: OnceLock<Mutex<HashMap<PathBuf, WtModule>>> = OnceLock::new
|
|
|
43
44
|
/// 10 ms across the whole process).
|
|
44
45
|
const EPOCH_TICK: Duration = Duration::from_millis(10);
|
|
45
46
|
|
|
46
|
-
///
|
|
47
|
-
///
|
|
48
|
-
///
|
|
49
|
-
///
|
|
50
|
-
/// can be loaded. The mruby wasi build config uses
|
|
51
|
-
/// `-mllvm -wasm-use-legacy-eh=false`, which generates new-style
|
|
52
|
-
/// exception handling instructions in the wasm32 object files;
|
|
53
|
-
/// wasmtime must have the proposal enabled to parse and JIT those
|
|
54
|
-
/// instructions.
|
|
55
|
-
///
|
|
56
|
-
/// Also enables `epoch_interruption(true)` so every Store can install an
|
|
57
|
-
/// `epoch_deadline_callback` for the per-run wall-clock cap
|
|
58
|
-
/// cap. The first call spawns the process-singleton ticker
|
|
59
|
-
/// thread that drives `engine.increment_epoch()` at `EPOCH_TICK`
|
|
60
|
-
/// cadence; subsequent calls reuse the same engine and ticker.
|
|
47
|
+
/// The exceptions proposal is on because the mruby wasi build emits
|
|
48
|
+
/// new-style exception handling (`-mllvm -wasm-use-legacy-eh=false`) for
|
|
49
|
+
/// its setjmp path. Epoch interruption backs the wall-clock cap, driven by
|
|
50
|
+
/// one process-wide ticker thread.
|
|
61
51
|
pub(crate) fn shared_engine() -> Result<&'static WtEngine, SetupError> {
|
|
62
52
|
if let Some(engine) = SHARED_ENGINE.get() {
|
|
63
53
|
return Ok(engine);
|
|
@@ -72,11 +62,6 @@ pub(crate) fn shared_engine() -> Result<&'static WtEngine, SetupError> {
|
|
|
72
62
|
Ok(engine)
|
|
73
63
|
}
|
|
74
64
|
|
|
75
|
-
/// Spawn the process-singleton epoch ticker. The thread holds a clone of
|
|
76
|
-
/// the shared Engine (`wasmtime::Engine` is reference-counted internally)
|
|
77
|
-
/// and ticks the epoch counter at `EPOCH_TICK` cadence. Idempotent
|
|
78
|
-
/// across reentrant calls to `shared_engine` because `OnceLock`
|
|
79
|
-
/// gates the spawn.
|
|
80
65
|
fn spawn_epoch_ticker(engine: WtEngine) {
|
|
81
66
|
static TICKER_SPAWNED: OnceLock<()> = OnceLock::new();
|
|
82
67
|
TICKER_SPAWNED.get_or_init(|| {
|
|
@@ -90,11 +75,8 @@ fn spawn_epoch_ticker(engine: WtEngine) {
|
|
|
90
75
|
});
|
|
91
76
|
}
|
|
92
77
|
|
|
93
|
-
///
|
|
94
|
-
///
|
|
95
|
-
/// (boundary → `Kobako::ModuleNotBuiltError`) when the file is missing —
|
|
96
|
-
/// the headline error for the common pre-build state on a fresh clone
|
|
97
|
-
/// before `rake compile`.
|
|
78
|
+
/// A missing file answers `SetupError::ModuleNotBuilt`, the common state of
|
|
79
|
+
/// a fresh clone before `rake compile`.
|
|
98
80
|
pub(crate) fn cached_module(path: &Path) -> Result<WtModule, SetupError> {
|
|
99
81
|
let cache = MODULE_CACHE.get_or_init(|| Mutex::new(HashMap::new()));
|
|
100
82
|
|
|
@@ -109,7 +91,7 @@ pub(crate) fn cached_module(path: &Path) -> Result<WtModule, SetupError> {
|
|
|
109
91
|
|
|
110
92
|
if !path.exists() {
|
|
111
93
|
return Err(SetupError::ModuleNotBuilt(format!(
|
|
112
|
-
"Sandbox runtime not found at {}
|
|
94
|
+
"Sandbox runtime not found at {}",
|
|
113
95
|
path.display()
|
|
114
96
|
)));
|
|
115
97
|
}
|
|
@@ -121,7 +103,7 @@ pub(crate) fn cached_module(path: &Path) -> Result<WtModule, SetupError> {
|
|
|
121
103
|
))
|
|
122
104
|
})?;
|
|
123
105
|
let engine = shared_engine()?;
|
|
124
|
-
let artifact = artifact_path(&bytes);
|
|
106
|
+
let artifact = artifact_path(engine, &bytes);
|
|
125
107
|
let module = match artifact.as_deref().and_then(|p| load_artifact(engine, p)) {
|
|
126
108
|
Some(module) => module,
|
|
127
109
|
None => {
|
|
@@ -143,81 +125,120 @@ pub(crate) fn cached_module(path: &Path) -> Result<WtModule, SetupError> {
|
|
|
143
125
|
/// Retention window for unused cache entries. A hit refreshes the
|
|
144
126
|
/// artifact's mtime, so only entries no process has loaded for the
|
|
145
127
|
/// whole window are removed by `prune_stale`.
|
|
128
|
+
#[cfg(unix)]
|
|
146
129
|
const ARTIFACT_TTL: Duration = Duration::from_secs(30 * 24 * 60 * 60);
|
|
147
130
|
|
|
148
|
-
///
|
|
149
|
-
///
|
|
150
|
-
///
|
|
151
|
-
///
|
|
152
|
-
///
|
|
153
|
-
///
|
|
154
|
-
|
|
155
|
-
/// entry. wasmtime
|
|
156
|
-
/// itself rejects an artifact produced by an incompatible wasmtime
|
|
157
|
-
/// version or Config at deserialize time. Returns `None` when no home
|
|
158
|
-
/// directory is available — the caller then just compiles in-process.
|
|
159
|
-
fn artifact_path(wasm_bytes: &[u8]) -> Option<PathBuf> {
|
|
131
|
+
/// Content addressing makes a rebuilt Guest Binary a new entry rather than
|
|
132
|
+
/// an invalidation problem. The engine segment names the wasmtime release
|
|
133
|
+
/// and Config that compiled the artifact, because wasmtime's own
|
|
134
|
+
/// deserialize check compares only the major version: without it, an
|
|
135
|
+
/// artifact from a release with a since-fixed miscompile would load
|
|
136
|
+
/// under the patched one.
|
|
137
|
+
fn artifact_path(engine: &WtEngine, wasm_bytes: &[u8]) -> Option<PathBuf> {
|
|
160
138
|
let base = std::env::var_os("XDG_CACHE_HOME")
|
|
161
139
|
.map(PathBuf::from)
|
|
162
140
|
.or_else(|| std::env::var_os("HOME").map(|home| PathBuf::from(home).join(".cache")))?;
|
|
163
141
|
let digest = Sha256::digest(wasm_bytes);
|
|
164
|
-
let mut name = String::with_capacity(
|
|
142
|
+
let mut name = String::with_capacity(96);
|
|
165
143
|
for byte in digest {
|
|
166
144
|
let _ = write!(name, "{:02x}", byte);
|
|
167
145
|
}
|
|
168
|
-
let _ = write!(name, "-{}.cwasm",
|
|
146
|
+
let _ = write!(name, "-{:016x}.cwasm", engine_key(engine));
|
|
169
147
|
Some(base.join("kobako").join(name))
|
|
170
148
|
}
|
|
171
149
|
|
|
172
|
-
///
|
|
173
|
-
///
|
|
174
|
-
///
|
|
175
|
-
|
|
176
|
-
|
|
150
|
+
/// wasmtime's hash of everything an artifact's compatibility depends on,
|
|
151
|
+
/// its full release version included. `DefaultHasher::new` is unseeded,
|
|
152
|
+
/// so every process of one build derives the same key.
|
|
153
|
+
fn engine_key(engine: &WtEngine) -> u64 {
|
|
154
|
+
let mut hasher = DefaultHasher::new();
|
|
155
|
+
engine.precompile_compatibility_hash().hash(&mut hasher);
|
|
156
|
+
hasher.finish()
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/// A hit refreshes the file's mtime, so the retention window measures time
|
|
160
|
+
/// since last use rather than since creation.
|
|
161
|
+
#[cfg(unix)]
|
|
177
162
|
fn load_artifact(engine: &WtEngine, artifact: &Path) -> Option<WtModule> {
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
//
|
|
184
|
-
//
|
|
185
|
-
//
|
|
186
|
-
// the
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
.open(artifact)
|
|
191
|
-
.and_then(|f| f.set_modified(SystemTime::now()));
|
|
192
|
-
Some(module)
|
|
163
|
+
use rustix::fs::{openat, Mode, OFlags};
|
|
164
|
+
let dir = open_private_dir(artifact.parent()?)?;
|
|
165
|
+
let flags = OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
|
|
166
|
+
let file = fs::File::from(openat(&dir, artifact.file_name()?, flags, Mode::empty()).ok()?);
|
|
167
|
+
let _ = file.set_modified(std::time::SystemTime::now());
|
|
168
|
+
// SAFETY: `Module::deserialize_open_file` trusts the artifact bytes
|
|
169
|
+
// and needs them unchanged while the Module lives. The file was
|
|
170
|
+
// opened through the directory `open_private_dir` checked, which only
|
|
171
|
+
// the current user can write, so it is an artifact this module
|
|
172
|
+
// stored, addressed by the content hash of the Guest Binary being
|
|
173
|
+
// constructed — it carries exactly that binary's trust.
|
|
174
|
+
unsafe { WtModule::deserialize_open_file(engine, file) }.ok()
|
|
193
175
|
}
|
|
194
176
|
|
|
195
|
-
///
|
|
196
|
-
///
|
|
197
|
-
///
|
|
198
|
-
///
|
|
199
|
-
/// cache
|
|
200
|
-
/// rebuilds.
|
|
177
|
+
/// The temp-file and rename pair keeps concurrent processes from observing
|
|
178
|
+
/// a partial write, and syncing the bytes before the rename keeps a crash
|
|
179
|
+
/// from leaving the artifact's name over blocks never written, which the
|
|
180
|
+
/// unsafe deserialize would trust. Failures are swallowed because the
|
|
181
|
+
/// cache is only an optimisation, and pruning on write keeps the
|
|
182
|
+
/// directory bounded across rebuilds.
|
|
183
|
+
#[cfg(unix)]
|
|
201
184
|
fn store_artifact(module: &WtModule, artifact: &Path) {
|
|
185
|
+
use rustix::fs::{fsync, renameat};
|
|
186
|
+
use std::io::Write as _;
|
|
187
|
+
let (Some(dir), Some(name)) = (artifact.parent(), artifact.file_name()) else {
|
|
188
|
+
return;
|
|
189
|
+
};
|
|
202
190
|
let Ok(bytes) = module.serialize() else {
|
|
203
191
|
return;
|
|
204
192
|
};
|
|
205
|
-
|
|
206
|
-
if create_cache_dir(dir).is_err() || !dir_is_private(dir) {
|
|
193
|
+
if create_cache_dir(dir).is_err() {
|
|
207
194
|
return;
|
|
208
195
|
}
|
|
209
|
-
let
|
|
210
|
-
|
|
196
|
+
let Some(dir) = open_private_dir(dir) else {
|
|
197
|
+
return;
|
|
198
|
+
};
|
|
199
|
+
let Some((tmp, mut file)) = create_temp(&dir, name) else {
|
|
200
|
+
return;
|
|
201
|
+
};
|
|
202
|
+
if file
|
|
203
|
+
.write_all(&bytes)
|
|
204
|
+
.and_then(|()| file.sync_all())
|
|
205
|
+
.is_err()
|
|
206
|
+
{
|
|
211
207
|
return;
|
|
212
208
|
}
|
|
213
|
-
if
|
|
214
|
-
|
|
209
|
+
if renameat(&dir, &tmp, &dir, name).is_ok() {
|
|
210
|
+
let _ = fsync(&dir);
|
|
211
|
+
prune_stale(&dir, name);
|
|
215
212
|
}
|
|
216
213
|
}
|
|
217
214
|
|
|
218
|
-
///
|
|
219
|
-
///
|
|
220
|
-
///
|
|
215
|
+
/// A temp file no other writer shares: two threads of one process storing
|
|
216
|
+
/// the same artifact at once would otherwise write into one file, and a
|
|
217
|
+
/// reader could load the interleaved bytes the unsafe deserialize trusts.
|
|
218
|
+
#[cfg(unix)]
|
|
219
|
+
fn create_temp(dir: &rustix::fd::OwnedFd, name: &std::ffi::OsStr) -> Option<(PathBuf, fs::File)> {
|
|
220
|
+
use rustix::fs::{openat, Mode, OFlags};
|
|
221
|
+
use std::sync::atomic::{AtomicU64, Ordering};
|
|
222
|
+
static NEXT_TEMP: AtomicU64 = AtomicU64::new(0);
|
|
223
|
+
let serial = NEXT_TEMP.fetch_add(1, Ordering::Relaxed);
|
|
224
|
+
let tmp = Path::new(name).with_extension(format!("tmp{}.{serial}", std::process::id()));
|
|
225
|
+
let flags = OFlags::WRONLY | OFlags::CREATE | OFlags::EXCL | OFlags::NOFOLLOW | OFlags::CLOEXEC;
|
|
226
|
+
let fd = openat(dir, &tmp, flags, Mode::RUSR | Mode::WUSR).ok()?;
|
|
227
|
+
Some((tmp, fs::File::from(fd)))
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/// The disk cache trusts its directory's owner, which only Unix reports,
|
|
231
|
+
/// so elsewhere every process compiles the Guest Binary itself.
|
|
232
|
+
#[cfg(not(unix))]
|
|
233
|
+
fn load_artifact(_engine: &WtEngine, _artifact: &Path) -> Option<WtModule> {
|
|
234
|
+
None
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
#[cfg(not(unix))]
|
|
238
|
+
fn store_artifact(_module: &WtModule, _artifact: &Path) {}
|
|
239
|
+
|
|
240
|
+
/// Create the cache directory owner-only (`0700`) so no other local user
|
|
241
|
+
/// can plant an artifact the unsafe deserialize would trust.
|
|
221
242
|
#[cfg(unix)]
|
|
222
243
|
fn create_cache_dir(dir: &Path) -> std::io::Result<()> {
|
|
223
244
|
use std::os::unix::fs::DirBuilderExt;
|
|
@@ -227,61 +248,117 @@ fn create_cache_dir(dir: &Path) -> std::io::Result<()> {
|
|
|
227
248
|
.create(dir)
|
|
228
249
|
}
|
|
229
250
|
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
///
|
|
236
|
-
/// deserialize relies on: owned by the current effective user and
|
|
237
|
-
/// writable by no one else. A pre-existing directory another user owns
|
|
238
|
-
/// or can write to — e.g. under a shared `XDG_CACHE_HOME` — fails here
|
|
239
|
-
/// and both disk-cache tiers are skipped.
|
|
251
|
+
/// The cache directory as one descriptor, once it upholds the trust the
|
|
252
|
+
/// unsafe deserialize relies on: a directory rather than a link in its
|
|
253
|
+
/// place, owned by the current effective user and writable by no one
|
|
254
|
+
/// else. Every later step goes through the descriptor, so a path swapped
|
|
255
|
+
/// after the check reaches nothing. One that fails — another user's,
|
|
256
|
+
/// under a shared `XDG_CACHE_HOME`, say — skips both disk-cache tiers.
|
|
240
257
|
#[cfg(unix)]
|
|
241
|
-
fn
|
|
242
|
-
use
|
|
243
|
-
let
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
#[cfg(not(unix))]
|
|
251
|
-
fn dir_is_private(_dir: &Path) -> bool {
|
|
252
|
-
true
|
|
258
|
+
fn open_private_dir(dir: &Path) -> Option<rustix::fd::OwnedFd> {
|
|
259
|
+
use rustix::fs::{fstat, open, Mode, OFlags};
|
|
260
|
+
let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
|
|
261
|
+
let fd = open(dir, flags, Mode::empty()).ok()?;
|
|
262
|
+
let stat = fstat(&fd).ok()?;
|
|
263
|
+
let owned = stat.st_uid == rustix::process::geteuid().as_raw();
|
|
264
|
+
(owned && stat.st_mode & 0o022 == 0).then_some(fd)
|
|
253
265
|
}
|
|
254
266
|
|
|
255
|
-
///
|
|
256
|
-
///
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
267
|
+
/// Live temp files are seconds old and never qualify; foreign file names
|
|
268
|
+
/// are left untouched.
|
|
269
|
+
#[cfg(unix)]
|
|
270
|
+
fn prune_stale(dir: &rustix::fd::OwnedFd, keep: &std::ffi::OsStr) {
|
|
271
|
+
use rustix::fs::{statat, unlinkat, AtFlags, Dir};
|
|
272
|
+
use std::os::unix::ffi::OsStrExt;
|
|
273
|
+
use std::time::{SystemTime, UNIX_EPOCH};
|
|
274
|
+
let Ok(mut entries) = Dir::read_from(dir) else {
|
|
261
275
|
return;
|
|
262
276
|
};
|
|
263
|
-
|
|
264
|
-
let
|
|
265
|
-
if
|
|
277
|
+
while let Some(Ok(entry)) = entries.read() {
|
|
278
|
+
let name = entry.file_name();
|
|
279
|
+
if name.to_bytes() == keep.as_bytes() || !name.to_str().is_ok_and(cache_entry_name) {
|
|
266
280
|
continue;
|
|
267
281
|
}
|
|
268
|
-
let stale =
|
|
269
|
-
.metadata()
|
|
270
|
-
.and_then(|meta| meta.modified())
|
|
282
|
+
let stale = statat(dir, name, AtFlags::SYMLINK_NOFOLLOW)
|
|
271
283
|
.ok()
|
|
272
|
-
.and_then(|
|
|
284
|
+
.and_then(|stat| u64::try_from(stat.st_mtime).ok())
|
|
285
|
+
.and_then(|secs| {
|
|
286
|
+
SystemTime::now()
|
|
287
|
+
.duration_since(UNIX_EPOCH + Duration::from_secs(secs))
|
|
288
|
+
.ok()
|
|
289
|
+
})
|
|
273
290
|
.is_some_and(|age| age > ARTIFACT_TTL);
|
|
274
291
|
if stale {
|
|
275
|
-
let _ =
|
|
292
|
+
let _ = unlinkat(dir, name, AtFlags::empty());
|
|
276
293
|
}
|
|
277
294
|
}
|
|
278
295
|
}
|
|
279
296
|
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
fn cache_entry_name(path: &Path) -> bool {
|
|
283
|
-
let Some(name) = path.file_name().and_then(|n| n.to_str()) else {
|
|
284
|
-
return false;
|
|
285
|
-
};
|
|
297
|
+
#[cfg(unix)]
|
|
298
|
+
fn cache_entry_name(name: &str) -> bool {
|
|
286
299
|
name.ends_with(".cwasm") || name.contains(".tmp")
|
|
287
300
|
}
|
|
301
|
+
|
|
302
|
+
#[cfg(test)]
|
|
303
|
+
mod tests {
|
|
304
|
+
use super::*;
|
|
305
|
+
use wasmtime::ModuleVersionStrategy;
|
|
306
|
+
|
|
307
|
+
fn engine_with(version: &str) -> WtEngine {
|
|
308
|
+
let mut config = WtConfig::new();
|
|
309
|
+
config
|
|
310
|
+
.module_version(ModuleVersionStrategy::Custom(version.into()))
|
|
311
|
+
.expect("a short version string is accepted");
|
|
312
|
+
WtEngine::new(&config).expect("engine")
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
fn entry_name(engine: &WtEngine) -> PathBuf {
|
|
316
|
+
let path = artifact_path(engine, b"\0asm").expect("a cache root");
|
|
317
|
+
PathBuf::from(path.file_name().expect("an entry name"))
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
// A custom module version stands in for another wasmtime release:
|
|
321
|
+
// wasmtime's compatibility hash covers its full version the same way.
|
|
322
|
+
// @behavior RT-108
|
|
323
|
+
#[test]
|
|
324
|
+
fn an_artifact_from_another_engine_release_is_never_the_entry_read() {
|
|
325
|
+
assert_ne!(
|
|
326
|
+
entry_name(&engine_with("49.0.3")),
|
|
327
|
+
entry_name(&engine_with("49.0.4")),
|
|
328
|
+
"a Guest Binary cached by one engine release must not be the entry another release reads"
|
|
329
|
+
);
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
// Two writers in one process take temp files in turn; each must get a
|
|
333
|
+
// file of its own rather than reopen the other's.
|
|
334
|
+
// @behavior RT-113
|
|
335
|
+
#[cfg(unix)]
|
|
336
|
+
#[test]
|
|
337
|
+
fn two_writers_of_one_artifact_never_share_a_temp_file() {
|
|
338
|
+
let root = std::env::temp_dir().join(format!("kobako-temp-{}", std::process::id()));
|
|
339
|
+
create_cache_dir(&root).expect("a scratch directory");
|
|
340
|
+
let dir = open_private_dir(&root).expect("a private directory");
|
|
341
|
+
let name = std::ffi::OsStr::new("entry.cwasm");
|
|
342
|
+
let first = create_temp(&dir, name).map(|(path, _)| path);
|
|
343
|
+
let second = create_temp(&dir, name).map(|(path, _)| path);
|
|
344
|
+
fs::remove_dir_all(&root).expect("the scratch directory removed");
|
|
345
|
+
let (first, second) = (
|
|
346
|
+
first.expect("a temp file"),
|
|
347
|
+
second.expect("a second temp file"),
|
|
348
|
+
);
|
|
349
|
+
assert_ne!(
|
|
350
|
+
first, second,
|
|
351
|
+
"two stores of one artifact in one process must each write a temp file of their own"
|
|
352
|
+
);
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
// @behavior RT-108
|
|
356
|
+
#[test]
|
|
357
|
+
fn engines_of_one_release_and_config_share_the_entry() {
|
|
358
|
+
assert_eq!(
|
|
359
|
+
entry_name(&engine_with("49.0.4")),
|
|
360
|
+
entry_name(&engine_with("49.0.4")),
|
|
361
|
+
"two engines of one release and Config must read the entry either of them cached"
|
|
362
|
+
);
|
|
363
|
+
}
|
|
364
|
+
}
|
|
@@ -7,10 +7,8 @@
|
|
|
7
7
|
//! channel) so a regression that only breaks one channel cannot sneak
|
|
8
8
|
//! through the test that pins them.
|
|
9
9
|
|
|
10
|
-
///
|
|
11
|
-
///
|
|
12
|
-
/// "wrote exactly cap" and "exceeded cap" cases stay distinguishable;
|
|
13
|
-
/// `usize::MAX` when the channel is uncapped.
|
|
10
|
+
/// One byte past the cap, so "wrote exactly cap" and "exceeded cap" stay
|
|
11
|
+
/// distinguishable.
|
|
14
12
|
pub(crate) fn pipe_capacity(cap: Option<usize>) -> usize {
|
|
15
13
|
match cap {
|
|
16
14
|
Some(c) => c.saturating_add(1),
|
|
@@ -18,13 +16,8 @@ pub(crate) fn pipe_capacity(cap: Option<usize>) -> usize {
|
|
|
18
16
|
}
|
|
19
17
|
}
|
|
20
18
|
|
|
21
|
-
///
|
|
22
|
-
///
|
|
23
|
-
/// costs one copy out of the pipe, not two), return the bytes Ruby
|
|
24
|
-
/// should observe plus the truncation flag. `truncated` is `true` only
|
|
25
|
-
/// when the snapshot strictly exceeded the cap — this is the "wrote
|
|
26
|
-
/// `cap + 1` bytes into a `cap + 1`-sized pipe" case; "wrote exactly
|
|
27
|
-
/// `cap` bytes" stays `false`.
|
|
19
|
+
/// Takes the snapshot by value and truncates in place, so the readback
|
|
20
|
+
/// costs one copy out of the pipe rather than two.
|
|
28
21
|
pub(crate) fn clip_capture(mut raw: Vec<u8>, cap: Option<usize>) -> (Vec<u8>, bool) {
|
|
29
22
|
match cap {
|
|
30
23
|
Some(c) if raw.len() > c => {
|