kobako 0.28.0 → 0.29.0

data/crates/kobako-wasmtime/src/cache.rs

@@ -10,8 +10,8 @@
 //!
 //! Across processes, the Cranelift compile cost is amortised by a
 //! best-effort `.cwasm` disk cache keyed by the SHA-256 of the Guest
-//! Binary bytes; every cache failure falls
-//! back to in-process compilation.
+//! Binary bytes and the engine that compiles them; every cache failure
+//! falls back to in-process compilation.
 //!
 //! Concurrency: the Mutex is held only for the HashMap lookup and insert,
 //! never across a compile, so threads building Drivers at once do not
@@ -21,10 +21,11 @@
 use std::collections::HashMap;
 use std::fmt::Write as _;
 use std::fs;
+use std::hash::{DefaultHasher, Hash, Hasher};
 use std::path::{Path, PathBuf};
 use std::sync::{Mutex, OnceLock};
 use std::thread;
-use std::time::{Duration, SystemTime};
+use std::time::Duration;
 
 use sha2::{Digest, Sha256};
 use wasmtime::{Config as WtConfig, Engine as WtEngine, Module as WtModule};
@@ -102,7 +103,7 @@ pub(crate) fn cached_module(path: &Path) -> Result<WtModule, SetupError> {
         ))
     })?;
     let engine = shared_engine()?;
-    let artifact = artifact_path(&bytes);
+    let artifact = artifact_path(engine, &bytes);
     let module = match artifact.as_deref().and_then(|p| load_artifact(engine, p)) {
         Some(module) => module,
         None => {
@@ -124,70 +125,120 @@ pub(crate) fn cached_module(path: &Path) -> Result<WtModule, SetupError> {
 /// Retention window for unused cache entries. A hit refreshes the
 /// artifact's mtime, so only entries no process has loaded for the
 /// whole window are removed by `prune_stale`.
+#[cfg(unix)]
 const ARTIFACT_TTL: Duration = Duration::from_secs(30 * 24 * 60 * 60);
 
 /// Content addressing makes a rebuilt Guest Binary a new entry rather than
-/// an invalidation problem; the crate-version segment keeps two installed
-/// kobako-wasmtime versions, each pinning its own wasmtime, from thrashing
-/// one key. wasmtime itself rejects an artifact from an incompatible
-/// version or Config at deserialize time.
-fn artifact_path(wasm_bytes: &[u8]) -> Option<PathBuf> {
+/// an invalidation problem. The engine segment names the wasmtime release
+/// and Config that compiled the artifact, because wasmtime's own
+/// deserialize check compares only the major version: without it, an
+/// artifact from a release with a since-fixed miscompile would load
+/// under the patched one.
+fn artifact_path(engine: &WtEngine, wasm_bytes: &[u8]) -> Option<PathBuf> {
     let base = std::env::var_os("XDG_CACHE_HOME")
         .map(PathBuf::from)
         .or_else(|| std::env::var_os("HOME").map(|home| PathBuf::from(home).join(".cache")))?;
     let digest = Sha256::digest(wasm_bytes);
-    let mut name = String::with_capacity(80);
+    let mut name = String::with_capacity(96);
     for byte in digest {
         let _ = write!(name, "{:02x}", byte);
     }
-    let _ = write!(name, "-{}.cwasm", env!("CARGO_PKG_VERSION"));
+    let _ = write!(name, "-{:016x}.cwasm", engine_key(engine));
     Some(base.join("kobako").join(name))
 }
 
+/// wasmtime's hash of everything an artifact's compatibility depends on,
+/// its full release version included. `DefaultHasher::new` is unseeded,
+/// so every process of one build derives the same key.
+fn engine_key(engine: &WtEngine) -> u64 {
+    let mut hasher = DefaultHasher::new();
+    engine.precompile_compatibility_hash().hash(&mut hasher);
+    hasher.finish()
+}
+
 /// A hit refreshes the file's mtime, so the retention window measures time
 /// since last use rather than since creation.
+#[cfg(unix)]
 fn load_artifact(engine: &WtEngine, artifact: &Path) -> Option<WtModule> {
-    if !artifact.exists() || !artifact.parent().is_some_and(dir_is_private) {
-        return None;
-    }
-    // SAFETY: `Module::deserialize_file` trusts the artifact bytes.
-    // `dir_is_private` just verified the cache directory is owned by
-    // the current user and writable by no one else, so only files this
-    // module wrote are loaded, addressed by the content hash of the
-    // Guest Binary being constructed — the artifact carries exactly
-    // the trust of `data/kobako.wasm`.
-    let module = unsafe { WtModule::deserialize_file(engine, artifact) }.ok()?;
-    let _ = fs::File::options()
-        .append(true)
-        .open(artifact)
-        .and_then(|f| f.set_modified(SystemTime::now()));
-    Some(module)
+    use rustix::fs::{openat, Mode, OFlags};
+    let dir = open_private_dir(artifact.parent()?)?;
+    let flags = OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
+    let file = fs::File::from(openat(&dir, artifact.file_name()?, flags, Mode::empty()).ok()?);
+    let _ = file.set_modified(std::time::SystemTime::now());
+    // SAFETY: `Module::deserialize_open_file` trusts the artifact bytes
+    // and needs them unchanged while the Module lives. The file was
+    // opened through the directory `open_private_dir` checked, which only
+    // the current user can write, so it is an artifact this module
+    // stored, addressed by the content hash of the Guest Binary being
+    // constructed — it carries exactly that binary's trust.
+    unsafe { WtModule::deserialize_open_file(engine, file) }.ok()
 }
 
 /// The temp-file and rename pair keeps concurrent processes from observing
-/// a partial write. Failures are swallowed because the cache is only an
-/// optimisation, and pruning on write keeps the directory bounded across
-/// rebuilds.
+/// a partial write, and syncing the bytes before the rename keeps a crash
+/// from leaving the artifact's name over blocks never written, which the
+/// unsafe deserialize would trust. Failures are swallowed because the
+/// cache is only an optimisation, and pruning on write keeps the
+/// directory bounded across rebuilds.
+#[cfg(unix)]
 fn store_artifact(module: &WtModule, artifact: &Path) {
+    use rustix::fs::{fsync, renameat};
+    use std::io::Write as _;
+    let (Some(dir), Some(name)) = (artifact.parent(), artifact.file_name()) else {
+        return;
+    };
     let Ok(bytes) = module.serialize() else {
         return;
     };
-    let Some(dir) = artifact.parent() else { return };
-    if create_cache_dir(dir).is_err() || !dir_is_private(dir) {
+    if create_cache_dir(dir).is_err() {
         return;
     }
-    let tmp = artifact.with_extension(format!("tmp{}", std::process::id()));
-    if fs::write(&tmp, bytes).is_err() {
+    let Some(dir) = open_private_dir(dir) else {
+        return;
+    };
+    let Some((tmp, mut file)) = create_temp(&dir, name) else {
+        return;
+    };
+    if file
+        .write_all(&bytes)
+        .and_then(|()| file.sync_all())
+        .is_err()
+    {
         return;
     }
-    if fs::rename(&tmp, artifact).is_ok() {
-        prune_stale(dir, artifact);
+    if renameat(&dir, &tmp, &dir, name).is_ok() {
+        let _ = fsync(&dir);
+        prune_stale(&dir, name);
     }
 }
 
-/// Create the cache directory owner-only (`0700`) on Unix so no other
-/// local user can plant an artifact the unsafe deserialize would
-/// trust; elsewhere fall back to default permissions.
+/// A temp file no other writer shares: two threads of one process storing
+/// the same artifact at once would otherwise write into one file, and a
+/// reader could load the interleaved bytes the unsafe deserialize trusts.
+#[cfg(unix)]
+fn create_temp(dir: &rustix::fd::OwnedFd, name: &std::ffi::OsStr) -> Option<(PathBuf, fs::File)> {
+    use rustix::fs::{openat, Mode, OFlags};
+    use std::sync::atomic::{AtomicU64, Ordering};
+    static NEXT_TEMP: AtomicU64 = AtomicU64::new(0);
+    let serial = NEXT_TEMP.fetch_add(1, Ordering::Relaxed);
+    let tmp = Path::new(name).with_extension(format!("tmp{}.{serial}", std::process::id()));
+    let flags = OFlags::WRONLY | OFlags::CREATE | OFlags::EXCL | OFlags::NOFOLLOW | OFlags::CLOEXEC;
+    let fd = openat(dir, &tmp, flags, Mode::RUSR | Mode::WUSR).ok()?;
+    Some((tmp, fs::File::from(fd)))
+}
+
+/// The disk cache trusts its directory's owner, which only Unix reports,
+/// so elsewhere every process compiles the Guest Binary itself.
+#[cfg(not(unix))]
+fn load_artifact(_engine: &WtEngine, _artifact: &Path) -> Option<WtModule> {
+    None
+}
+
+#[cfg(not(unix))]
+fn store_artifact(_module: &WtModule, _artifact: &Path) {}
+
+/// Create the cache directory owner-only (`0700`) so no other local user
+/// can plant an artifact the unsafe deserialize would trust.
 #[cfg(unix)]
 fn create_cache_dir(dir: &Path) -> std::io::Result<()> {
     use std::os::unix::fs::DirBuilderExt;
@@ -197,57 +248,117 @@ fn create_cache_dir(dir: &Path) -> std::io::Result<()> {
         .create(dir)
 }
 
-#[cfg(not(unix))]
-fn create_cache_dir(dir: &Path) -> std::io::Result<()> {
-    fs::create_dir_all(dir)
-}
-
-/// Returns whether the cache directory upholds the trust the unsafe
-/// deserialize relies on: owned by the current effective user and
-/// writable by no one else. A pre-existing directory another user owns
-/// or can write to — e.g. under a shared `XDG_CACHE_HOME` — fails here
-/// and both disk-cache tiers are skipped.
+/// The cache directory as one descriptor, once it upholds the trust the
+/// unsafe deserialize relies on: a directory rather than a link in its
+/// place, owned by the current effective user and writable by no one
+/// else. Every later step goes through the descriptor, so a path swapped
+/// after the check reaches nothing. One that fails — another user's,
+/// under a shared `XDG_CACHE_HOME`, say — skips both disk-cache tiers.
 #[cfg(unix)]
-fn dir_is_private(dir: &Path) -> bool {
-    use std::os::unix::fs::MetadataExt;
-    let Ok(meta) = fs::metadata(dir) else {
-        return false;
-    };
-    // SAFETY: `geteuid` reads process state and has no preconditions.
-    meta.uid() == unsafe { libc::geteuid() } && meta.mode() & 0o022 == 0
-}
-
-#[cfg(not(unix))]
-fn dir_is_private(_dir: &Path) -> bool {
-    true
+fn open_private_dir(dir: &Path) -> Option<rustix::fd::OwnedFd> {
+    use rustix::fs::{fstat, open, Mode, OFlags};
+    let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
+    let fd = open(dir, flags, Mode::empty()).ok()?;
+    let stat = fstat(&fd).ok()?;
+    let owned = stat.st_uid == rustix::process::geteuid().as_raw();
+    (owned && stat.st_mode & 0o022 == 0).then_some(fd)
 }
 
 /// Live temp files are seconds old and never qualify; foreign file names
 /// are left untouched.
-fn prune_stale(dir: &Path, keep: &Path) {
-    let Ok(entries) = fs::read_dir(dir) else {
+#[cfg(unix)]
+fn prune_stale(dir: &rustix::fd::OwnedFd, keep: &std::ffi::OsStr) {
+    use rustix::fs::{statat, unlinkat, AtFlags, Dir};
+    use std::os::unix::ffi::OsStrExt;
+    use std::time::{SystemTime, UNIX_EPOCH};
+    let Ok(mut entries) = Dir::read_from(dir) else {
         return;
     };
-    for entry in entries.flatten() {
-        let path = entry.path();
-        if path == keep || !cache_entry_name(&path) {
+    while let Some(Ok(entry)) = entries.read() {
+        let name = entry.file_name();
+        if name.to_bytes() == keep.as_bytes() || !name.to_str().is_ok_and(cache_entry_name) {
             continue;
         }
-        let stale = entry
-            .metadata()
-            .and_then(|meta| meta.modified())
+        let stale = statat(dir, name, AtFlags::SYMLINK_NOFOLLOW)
             .ok()
-            .and_then(|mtime| mtime.elapsed().ok())
+            .and_then(|stat| u64::try_from(stat.st_mtime).ok())
+            .and_then(|secs| {
+                SystemTime::now()
+                    .duration_since(UNIX_EPOCH + Duration::from_secs(secs))
+                    .ok()
+            })
             .is_some_and(|age| age > ARTIFACT_TTL);
         if stale {
-            let _ = fs::remove_file(&path);
+            let _ = unlinkat(dir, name, AtFlags::empty());
         }
     }
 }
 
-fn cache_entry_name(path: &Path) -> bool {
-    let Some(name) = path.file_name().and_then(|n| n.to_str()) else {
-        return false;
-    };
+#[cfg(unix)]
+fn cache_entry_name(name: &str) -> bool {
     name.ends_with(".cwasm") || name.contains(".tmp")
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use wasmtime::ModuleVersionStrategy;
+
+    fn engine_with(version: &str) -> WtEngine {
+        let mut config = WtConfig::new();
+        config
+            .module_version(ModuleVersionStrategy::Custom(version.into()))
+            .expect("a short version string is accepted");
+        WtEngine::new(&config).expect("engine")
+    }
+
+    fn entry_name(engine: &WtEngine) -> PathBuf {
+        let path = artifact_path(engine, b"\0asm").expect("a cache root");
+        PathBuf::from(path.file_name().expect("an entry name"))
+    }
+
+    // A custom module version stands in for another wasmtime release:
+    // wasmtime's compatibility hash covers its full version the same way.
+    // @behavior RT-108
+    #[test]
+    fn an_artifact_from_another_engine_release_is_never_the_entry_read() {
+        assert_ne!(
+            entry_name(&engine_with("49.0.3")),
+            entry_name(&engine_with("49.0.4")),
+            "a Guest Binary cached by one engine release must not be the entry another release reads"
+        );
+    }
+
+    // Two writers in one process take temp files in turn; each must get a
+    // file of its own rather than reopen the other's.
+    // @behavior RT-113
+    #[cfg(unix)]
+    #[test]
+    fn two_writers_of_one_artifact_never_share_a_temp_file() {
+        let root = std::env::temp_dir().join(format!("kobako-temp-{}", std::process::id()));
+        create_cache_dir(&root).expect("a scratch directory");
+        let dir = open_private_dir(&root).expect("a private directory");
+        let name = std::ffi::OsStr::new("entry.cwasm");
+        let first = create_temp(&dir, name).map(|(path, _)| path);
+        let second = create_temp(&dir, name).map(|(path, _)| path);
+        fs::remove_dir_all(&root).expect("the scratch directory removed");
+        let (first, second) = (
+            first.expect("a temp file"),
+            second.expect("a second temp file"),
+        );
+        assert_ne!(
+            first, second,
+            "two stores of one artifact in one process must each write a temp file of their own"
+        );
+    }
+
+    // @behavior RT-108
+    #[test]
+    fn engines_of_one_release_and_config_share_the_entry() {
+        assert_eq!(
+            entry_name(&engine_with("49.0.4")),
+            entry_name(&engine_with("49.0.4")),
+            "two engines of one release and Config must read the entry either of them cached"
+        );
+    }
+}

data/crates/kobako-wasmtime/src/config.rs

@@ -1,4 +1,4 @@
-//! Per-`Driver` execution configuration.
+//! The caps a `Driver` applies to every invocation.
 //!
 //! Every cap a frontend forwards into `Driver::new`. A plain value carrier
 //! owned by the `Driver` — distinct from the process-wide engine/module
@@ -17,10 +17,14 @@ use kobako_runtime::profile::Profile;
 
 /// The caps and requested isolation profile for one `Driver`. `None` on
 /// any cap field disables that cap.
+#[non_exhaustive]
 pub struct Config {
     /// Wall-clock cap for one guest `#eval` / `#run`; the invocation traps
     /// once it passes.
     pub timeout: Option<Duration>,
+    /// Wall-clock cap for setting the Guest Binary up the first time its
+    /// path is loaded; past it, construction fails.
+    pub setup_timeout: Option<Duration>,
     /// Guest linear-memory cap, in bytes, charged to what one invocation
     /// grows beyond the image it starts from.
     pub memory_limit: Option<usize>,
@@ -29,9 +33,62 @@ pub struct Config {
     pub stdout_limit: Option<usize>,
     /// Byte cap for guest stderr capture, the same as `stdout_limit`.
     pub stderr_limit: Option<usize>,
+    /// Descriptors one invocation may hold open, the standard ones and
+    /// each Mount's preopen included; an open past it answers `MFILE`.
+    pub open_file_limit: Option<usize>,
     /// Isolation posture the frontend requested. `Hermetic` freezes
     /// ambient time and entropy, `Permissive` leaves the live WASI
     /// sources, and `Driver::profile` declares it back as the built
     /// posture.
     pub profile: Profile,
 }
+
+impl Default for Config {
+    /// The caps every frontend defaults to: a 60-second deadline, 10
+    /// seconds to set the Guest Binary up, 4 MiB for memory, 1 MiB for
+    /// each output channel, 1024 open files, and the hermetic floor.
+    fn default() -> Self {
+        Config {
+            timeout: Some(Duration::from_secs(60)),
+            setup_timeout: Some(Duration::from_secs(10)),
+            memory_limit: Some(4 << 20),
+            stdout_limit: Some(1 << 20),
+            stderr_limit: Some(1 << 20),
+            open_file_limit: Some(1024),
+            profile: Profile::Hermetic,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    // Literal values, so a drift from the defaults the SDK's Options and
+    // the Ruby frontend apply fails here.
+    #[test]
+    fn a_default_config_carries_every_frontends_default_caps() {
+        let config = Config::default();
+        assert_eq!(
+            (
+                config.timeout,
+                config.setup_timeout,
+                config.memory_limit,
+                config.stdout_limit,
+                config.stderr_limit,
+                config.open_file_limit,
+                config.profile
+            ),
+            (
+                Some(Duration::from_secs(60)),
+                Some(Duration::from_secs(10)),
+                Some(4 * 1024 * 1024),
+                Some(1024 * 1024),
+                Some(1024 * 1024),
+                Some(1024),
+                Profile::Hermetic
+            ),
+            "Config::default must carry the caps every frontend defaults to"
+        );
+    }
+}

data/crates/kobako-wasmtime/src/descriptors.rs

@@ -0,0 +1,167 @@
+//! The guest's descriptor table: what each preview1 file descriptor
+//! stands for within one invocation.
+//!
+//! kobako answers every preview1 call that reads this table, so the guest
+//! sees one table and no call falls through to a second one that
+//! disagrees. Numbering follows wasmtime-wasi's: a closed number is reused
+//! before a fresh one is taken.
+
+use std::collections::{BTreeMap, BTreeSet};
+use std::sync::Arc;
+
+use kobako_runtime::filesystem::{DirEntry, File, Hold, Metadata};
+use kobako_runtime::mount::Mount;
+use wasmtime_wasi::p2::pipe::{MemoryInputPipe, MemoryOutputPipe};
+
+/// What one descriptor stands for.
+pub(crate) enum Descriptor {
+    /// Descriptor 0, carrying the Preamble.
+    Stdin(MemoryInputPipe),
+    /// Descriptor 1 or 2, a capture pipe the guest writes a Channel into.
+    Channel(MemoryOutputPipe),
+    /// A directory beneath a Mount; a preopen is the Mount's own root.
+    Directory(OpenDirectory),
+    File(OpenFile),
+}
+
+/// What a directory is, apart from where it stands: the device and inode
+/// numbers its Filesystem names it by.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) struct Identity {
+    dev: u64,
+    ino: u64,
+}
+
+impl Identity {
+    /// The identity `metadata` names, or `None` where its inode of zero
+    /// reports none.
+    pub(crate) fn of(metadata: &Metadata) -> Option<Self> {
+        (metadata.ino != 0).then_some(Self {
+            dev: metadata.dev,
+            ino: metadata.ino,
+        })
+    }
+}
+
+pub(crate) struct OpenDirectory {
+    pub(crate) mount: Arc<Mount>,
+    /// The directory's path relative to the mount's root.
+    pub(crate) path: String,
+    /// The identity the directory had when it was opened, so one made in
+    /// its place is not taken for it; without one it follows its path.
+    pub(crate) identity: Option<Identity>,
+    /// What keeps that identity from passing to a directory made in this
+    /// one's place while the descriptor is open.
+    pub(crate) _hold: Option<Hold>,
+    pub(crate) preopen: bool,
+    /// The entries a listing began with, held until it reaches its end, so
+    /// each later call reads on without listing the directory again.
+    pub(crate) listing: Option<Vec<DirEntry>>,
+}
+
+pub(crate) struct OpenFile {
+    pub(crate) file: Box<dyn File>,
+    pub(crate) position: u64,
+    pub(crate) append: bool,
+    pub(crate) readable: bool,
+    pub(crate) writable: bool,
+    /// The Mount it was opened beneath, whose permission to write answers a
+    /// wait to write and a resize, as wasmtime-wasi answers both by its
+    /// preopen's permissions.
+    pub(crate) mount: Arc<Mount>,
+}
+
+pub(crate) struct Descriptors {
+    used: BTreeMap<u32, Descriptor>,
+    free: BTreeSet<u32>,
+    limit: Option<usize>,
+}
+
+impl Descriptors {
+    /// The table every invocation starts from: stdin, then the two
+    /// Channels, then one preopen per Mount from descriptor 3. It holds at
+    /// most `limit` descriptors, those included, as a process's
+    /// `RLIMIT_NOFILE` counts every one it holds.
+    pub(crate) fn new(
+        stdin: MemoryInputPipe,
+        stdout: MemoryOutputPipe,
+        stderr: MemoryOutputPipe,
+        mounts: Vec<Mount>,
+        limit: Option<usize>,
+    ) -> Self {
+        let mut used = BTreeMap::from([
+            (0, Descriptor::Stdin(stdin)),
+            (1, Descriptor::Channel(stdout)),
+            (2, Descriptor::Channel(stderr)),
+        ]);
+        for (fd, mount) in (3..).zip(mounts) {
+            // A root the Filesystem cannot stat or hold reports no
+            // identity, and the descriptor follows its path alone.
+            let filesystem = mount.filesystem();
+            let identity = filesystem
+                .metadata("")
+                .ok()
+                .and_then(|root| Identity::of(&root));
+            let hold = filesystem.hold_directory("").ok().flatten();
+            used.insert(
+                fd,
+                Descriptor::Directory(OpenDirectory {
+                    mount: Arc::new(mount),
+                    path: String::new(),
+                    identity,
+                    _hold: hold,
+                    preopen: true,
+                    listing: None,
+                }),
+            );
+        }
+        Self {
+            used,
+            free: BTreeSet::new(),
+            limit,
+        }
+    }
+
+    pub(crate) fn get(&self, fd: u32) -> Option<&Descriptor> {
+        self.used.get(&fd)
+    }
+
+    pub(crate) fn get_mut(&mut self, fd: u32) -> Option<&mut Descriptor> {
+        self.used.get_mut(&fd)
+    }
+
+    /// Take `fd` out of the table, freeing its number for reuse.
+    pub(crate) fn remove(&mut self, fd: u32) -> Option<Descriptor> {
+        let descriptor = self.used.remove(&fd)?;
+        self.free.insert(fd);
+        Some(descriptor)
+    }
+
+    /// Place `descriptor` at `fd`, a number the caller has just freed.
+    pub(crate) fn place(&mut self, fd: u32, descriptor: Descriptor) {
+        self.free.remove(&fd);
+        self.used.insert(fd, descriptor);
+    }
+
+    /// Whether the table holds as many descriptors as it may.
+    pub(crate) fn is_full(&self) -> bool {
+        self.limit.is_some_and(|limit| self.used.len() >= limit)
+    }
+
+    /// Add `descriptor` under the highest freed number, or the one past
+    /// the highest in use; `None` when the table is full.
+    pub(crate) fn push(&mut self, descriptor: Descriptor) -> Option<u32> {
+        if self.is_full() {
+            return None;
+        }
+        let fd = match self.free.pop_last() {
+            Some(fd) => fd,
+            None => match self.used.last_key_value() {
+                Some((last, _)) => last.checked_add(1)?,
+                None => 0,
+            },
+        };
+        self.used.insert(fd, descriptor);
+        Some(fd)
+    }
+}