@alwith/module-computer 0.0.0-stage → 0.1.0

package/rust/src/lib.rs

@@ -0,0 +1,408 @@
+//! Application-owned Cua Driver lifecycle. No agent, UI, or conversation storage.
+pub mod gateway;
+pub mod permissions;
+pub mod preview;
+mod wire;
+
+use serde::Serialize;
+use serde_json::json;
+use std::{
+    path::PathBuf,
+    process::{Child, ChildStdin, Command, Stdio},
+    sync::Mutex,
+    time::{Duration, Instant},
+};
+use tempfile::TempDir;
+
+pub const DRIVER_VERSION: &str = "0.36.0";
+const START_TIMEOUT: Duration = Duration::from_secs(12);
+
+#[derive(Debug, Clone, Serialize, specta::Type)]
+#[serde(rename_all = "camelCase")]
+pub struct Status {
+    pub permissions: permissions::Permissions,
+    pub available: bool,
+    pub session_id: Option<String>,
+    pub ready: bool,
+    pub version: String,
+}
+
+#[derive(Debug, Clone, Serialize, specta::Type)]
+pub struct Environment {
+    pub name: String,
+    pub value: String,
+}
+
+#[derive(Debug, Clone, Serialize, specta::Type)]
+pub struct Connection {
+    pub generation: String,
+    pub command: String,
+    pub args: Vec<String>,
+    pub env: Vec<Environment>,
+}
+
+struct Running {
+    session: String,
+    generation: String,
+    endpoint: String,
+    child: Child,
+    liveness: Option<ChildStdin>,
+    _directory: TempDir,
+    ready: bool,
+    revoked: bool,
+}
+
+pub struct ComputerHost {
+    binary: PathBuf,
+    bundle_id: String,
+    cursor_theme: Option<(String, Vec<u8>)>,
+    operations: tokio::sync::Mutex<()>,
+    running: Mutex<Option<Running>>,
+    closed: std::sync::atomic::AtomicBool,
+}
+
+impl ComputerHost {
+    pub fn new(binary: PathBuf, bundle_id: String) -> Self {
+        Self {
+            binary,
+            bundle_id,
+            cursor_theme: None,
+            operations: tokio::sync::Mutex::new(()),
+            running: Mutex::new(None),
+            closed: std::sync::atomic::AtomicBool::new(false),
+        }
+    }
+
+    pub fn with_cursor_theme(mut self, id: String, bytes: Vec<u8>) -> Self {
+        self.cursor_theme = Some((id, bytes));
+        self
+    }
+
+    pub fn status(&self) -> Result<Status, String> {
+        let mut slot = self.running.lock().unwrap();
+        if let Some(running) = slot.as_mut()
+            && running.child.try_wait().map_err(|e| e.to_string())?.is_some()
+        {
+            *slot = None;
+        }
+        Ok(Status {
+            permissions: permissions::status(),
+            available: self.binary.is_file(),
+            session_id: slot.as_ref().map(|r| r.session.clone()),
+            ready: slot.as_ref().is_some_and(|r| r.ready && !r.revoked),
+            version: DRIVER_VERSION.into(),
+        })
+    }
+
+    pub fn connection(&self, session: &str) -> Result<Option<Connection>, String> {
+        let status = self.status()?;
+        if status.session_id.as_deref() != Some(session) || !status.ready {
+            return Ok(None);
+        }
+        if !status.permissions.accessibility || !status.permissions.screen_capture {
+            return Err("Computer permissions were revoked; stop computer use and grant permissions again".into());
+        }
+        let slot = self.running.lock().unwrap();
+        Ok(slot.as_ref().filter(|r| r.session == session && r.ready && !r.revoked).map(|r| Connection {
+            generation: r.generation.clone(),
+            command: self.binary.to_string_lossy().into(),
+            args: vec![
+                "mcp".into(),
+                "--embedded".into(),
+                "--socket".into(),
+                r.endpoint.clone(),
+                "--host-bundle-id".into(),
+                self.bundle_id.clone(),
+            ],
+            env: vec![
+                Environment { name: "CUA_DRIVER_RS_TELEMETRY_ENABLED".into(), value: "0".into() },
+                Environment { name: "CUA_TELEMETRY_ENABLED".into(), value: "0".into() },
+                Environment {
+                    name: "CUA_DRIVER_RS_HOME".into(),
+                    value: r._directory.path().join("state").to_string_lossy().into_owned(),
+                },
+            ],
+        }))
+    }
+
+    pub async fn start(&self, session: String) -> Result<Status, String> {
+        if session.is_empty() || session.len() > 256 {
+            return Err("Invalid computer session identity".into());
+        }
+        let _operation = self.operations.lock().await;
+        if self.closed.load(std::sync::atomic::Ordering::Acquire) {
+            return Err("Computer host is shutting down".into());
+        }
+        let status = self.status()?;
+        if !status.permissions.supported {
+            return Err("Computer use currently supports macOS and Windows".into());
+        }
+        if !status.permissions.accessibility || !status.permissions.screen_capture {
+            return Err("Grant Accessibility and Screen Recording to the host application first".into());
+        }
+        if let Some(owner) = status.session_id {
+            if owner == session && status.ready {
+                return self.status();
+            }
+            return Err("Another computer session is active or stopping".into());
+        }
+        if !self.binary.is_absolute() || !self.binary.is_file() {
+            return Err("Bundled Cua Driver is missing; run staging or reinstall the application".into());
+        }
+        let directory = tempfile::Builder::new().prefix("alwith-cu-").tempdir().map_err(|e| e.to_string())?;
+        let generation = uuid::Uuid::new_v4().to_string();
+        #[cfg(unix)]
+        let endpoint = directory.path().join("driver.sock").to_string_lossy().into_owned();
+        #[cfg(windows)]
+        let endpoint = format!(r"\\.\pipe\alwith-computer-{generation}");
+        let mut command = Command::new(&self.binary);
+        command
+            .args([
+                "serve",
+                "--embedded",
+                "--parent-liveness-stdio",
+                "--no-permissions-gate",
+                "--socket",
+                &endpoint,
+                "--host-bundle-id",
+                &self.bundle_id,
+                "--permission-mode",
+                "standard",
+            ])
+            .env_clear()
+            .stdin(Stdio::piped())
+            .stdout(Stdio::null())
+            .stderr(Stdio::null());
+        for (key, value) in std::env::vars_os() {
+            if inherited_environment(&key.to_string_lossy()) {
+                command.env(key, value);
+            }
+        }
+        command
+            .env("CUA_DRIVER_EMBEDDED_HOST_PID", std::process::id().to_string())
+            .env("CUA_DRIVER_RS_HOME", directory.path().join("state"))
+            .env("CUA_DRIVER_RS_TELEMETRY_ENABLED", "0")
+            .env("CUA_TELEMETRY_ENABLED", "0")
+            .env("CUA_DRIVER_RS_UPDATE_CHECK", "0");
+        #[cfg(windows)]
+        {
+            use std::os::windows::process::CommandExt;
+            command.creation_flags(0x08000000);
+        }
+        if let Some((id, bytes)) = &self.cursor_theme {
+            if id.is_empty() || !id.bytes().all(|c| c.is_ascii_alphanumeric() || matches!(c, b'.' | b'-' | b'_')) {
+                return Err("Invalid bundled cursor theme identity".into());
+            }
+            let themes = directory.path().join("cursor-themes");
+            std::fs::create_dir_all(&themes).map_err(|e| e.to_string())?;
+            std::fs::write(themes.join(format!("{id}.cua-theme")), bytes).map_err(|e| e.to_string())?;
+            command.args(["--cursor-theme", id]).env("CUA_DRIVER_CURSOR_THEME_DIR", themes);
+        }
+        let mut child = command.spawn().map_err(|e| format!("Cannot start Cua Driver: {e}"))?;
+        let pid = child.id();
+        let liveness = child.stdin.take();
+        *self.running.lock().unwrap() = Some(Running {
+            session,
+            generation,
+            endpoint: endpoint.clone(),
+            child,
+            liveness,
+            _directory: directory,
+            ready: false,
+            revoked: false,
+        });
+        let started = Instant::now();
+        let result = loop {
+            {
+                let mut slot = self.running.lock().unwrap();
+                let running = slot.as_mut().ok_or("Computer driver exited during startup")?;
+                if running.revoked || self.closed.load(std::sync::atomic::Ordering::Acquire) {
+                    break Err("Computer startup cancelled".into());
+                }
+                match running.child.try_wait() {
+                    Ok(Some(exit)) => break Err(format!("Cua Driver exited during startup: {exit}")),
+                    Err(error) => break Err(error.to_string()),
+                    Ok(None) => {}
+                }
+            }
+            if let Ok(Ok(metadata)) = tokio::time::timeout(Duration::from_millis(300), wire::metadata(&endpoint)).await
+            {
+                break validate_metadata(&metadata, pid, &self.bundle_id);
+            }
+            if started.elapsed() >= START_TIMEOUT {
+                break Err("Cua Driver did not become ready in time".into());
+            }
+            tokio::time::sleep(Duration::from_millis(60)).await;
+        };
+        if let Err(error) = result {
+            let cleanup = self.drain().await;
+            return Err(match cleanup {
+                Ok(()) => error,
+                Err(cleanup) => format!("{error}; cleanup: {cleanup}"),
+            });
+        }
+        let revoked = {
+            let mut slot = self.running.lock().unwrap();
+            let running = slot.as_mut().ok_or("Computer driver exited during startup")?;
+            let revoked = running.revoked || self.closed.load(std::sync::atomic::Ordering::Acquire);
+            if !revoked {
+                running.ready = true;
+            }
+            revoked
+        };
+        if revoked {
+            self.drain().await?;
+            return Err("Computer startup cancelled".into());
+        }
+        self.status()
+    }
+
+    /// Revoke before waiting for startup/cleanup, and never stop a different owner.
+    pub async fn stop(&self, session: &str) -> Result<(), String> {
+        let generation = {
+            let mut slot = self.running.lock().unwrap();
+            let Some(running) = slot.as_mut().filter(|r| r.session == session) else {
+                return Ok(());
+            };
+            running.revoked = true;
+            running.generation.clone()
+        };
+        let _operation = self.operations.lock().await;
+        let matches =
+            self.running.lock().unwrap().as_ref().is_some_and(|r| r.session == session && r.generation == generation);
+        if matches {
+            self.drain().await?;
+        }
+        Ok(())
+    }
+
+    pub async fn shutdown(&self) -> Result<(), String> {
+        self.closed.store(true, std::sync::atomic::Ordering::Release);
+        let _operation = self.operations.lock().await;
+        self.drain().await
+    }
+
+    pub async fn stop_current(&self) -> Result<(), String> {
+        let owner = self.running.lock().unwrap().as_ref().map(|r| r.session.clone());
+        if let Some(owner) = owner {
+            self.stop(&owner).await?;
+        }
+        Ok(())
+    }
+
+    async fn drain(&self) -> Result<(), String> {
+        let endpoint = {
+            let mut slot = self.running.lock().unwrap();
+            let Some(running) = slot.as_mut() else {
+                return Ok(());
+            };
+            running.revoked = true;
+            running.liveness.take();
+            running.endpoint.clone()
+        };
+        let _ =
+            tokio::time::timeout(Duration::from_millis(300), wire::request(&endpoint, json!({"method":"shutdown"})))
+                .await;
+        let started = Instant::now();
+        loop {
+            {
+                let mut slot = self.running.lock().unwrap();
+                let Some(running) = slot.as_mut() else {
+                    return Ok(());
+                };
+                if running.child.try_wait().map_err(|e| e.to_string())?.is_some() {
+                    *slot = None;
+                    return Ok(());
+                }
+                if started.elapsed() > Duration::from_secs(2) {
+                    running.child.kill().map_err(|e| format!("Cannot stop Cua Driver: {e}"))?;
+                    running.child.wait().map_err(|e| e.to_string())?;
+                    *slot = None;
+                    return Ok(());
+                }
+            }
+            tokio::time::sleep(Duration::from_millis(30)).await;
+        }
+    }
+
+    pub fn kill_on_exit(&self) {
+        self.closed.store(true, std::sync::atomic::Ordering::Release);
+        if let Some(mut running) = self.running.lock().unwrap().take() {
+            running.liveness.take();
+            let _ = running.child.kill();
+            let _ = running.child.wait();
+        }
+    }
+}
+
+impl Drop for ComputerHost {
+    fn drop(&mut self) {
+        self.kill_on_exit();
+    }
+}
+
+fn inherited_environment(name: &str) -> bool {
+    matches!(
+        name.to_ascii_uppercase().as_str(),
+        "PATH"
+            | "HOME"
+            | "USER"
+            | "LOGNAME"
+            | "TMPDIR"
+            | "TMP"
+            | "TEMP"
+            | "LANG"
+            | "USERPROFILE"
+            | "USERNAME"
+            | "HOMEDRIVE"
+            | "HOMEPATH"
+            | "SYSTEMROOT"
+            | "WINDIR"
+            | "COMSPEC"
+            | "PATHEXT"
+            | "APPDATA"
+            | "LOCALAPPDATA"
+            | "PROGRAMDATA"
+    ) || name.starts_with("LC_")
+}
+
+fn validate_metadata(value: &serde_json::Value, pid: u32, bundle: &str) -> Result<(), String> {
+    if value["driver_version"] != DRIVER_VERSION
+        || value["contract_version"] != "0.8.0"
+        || value["pid"] != pid
+        || value["embedded"] != true
+        || value["host_bundle_id"] != bundle
+    {
+        return Err("Cua Driver identity or protocol version mismatch".into());
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[test]
+    fn rejects_wrong_driver_identity_and_untrusted_environment() {
+        let metadata = json!({"driver_version":DRIVER_VERSION,"contract_version":"0.8.0","pid":42,"embedded":true,"host_bundle_id":"ai.test"});
+        assert!(validate_metadata(&metadata, 42, "ai.test").is_ok());
+        assert!(validate_metadata(&metadata, 43, "ai.test").is_err());
+        assert!(validate_metadata(&metadata, 42, "other").is_err());
+        for name in [
+            "CUA_DRIVER_PERMISSION_MODE",
+            "CUA_DRIVER_DANGEROUSLY_BYPASS_APPROVALS",
+            "OPENAI_API_KEY",
+            "DYLD_INSERT_LIBRARIES",
+        ] {
+            assert!(!inherited_environment(name));
+        }
+        assert!(inherited_environment("SystemRoot"));
+    }
+    #[tokio::test]
+    async fn stopping_an_unknown_session_never_starts_a_process() {
+        let host = ComputerHost::new(PathBuf::from("/missing/cua-driver"), "test".into());
+        host.stop("unknown").await.unwrap();
+        assert!(host.connection("unknown").unwrap().is_none());
+        host.shutdown().await.unwrap();
+        assert!(host.start("new".into()).await.unwrap_err().contains("shutting down"));
+    }
+}

package/rust/src/permissions.rs

@@ -0,0 +1,58 @@
+use serde::Serialize;
+
+#[derive(Debug, Clone, Serialize, specta::Type)]
+#[serde(rename_all = "camelCase")]
+pub struct Permissions {
+    pub supported: bool,
+    pub accessibility: bool,
+    pub screen_capture: bool,
+}
+
+pub fn status() -> Permissions {
+    #[cfg(target_os = "macos")]
+    // SAFETY: these system preflight functions take no pointers and do not prompt.
+    unsafe {
+        Permissions {
+            supported: true,
+            accessibility: AXIsProcessTrusted(),
+            screen_capture: CGPreflightScreenCaptureAccess(),
+        }
+    }
+    #[cfg(target_os = "windows")]
+    {
+        Permissions { supported: true, accessibility: true, screen_capture: true }
+    }
+    #[cfg(not(any(target_os = "macos", target_os = "windows")))]
+    {
+        Permissions { supported: false, accessibility: false, screen_capture: false }
+    }
+}
+
+pub fn request() -> Permissions {
+    #[cfg(target_os = "macos")]
+    {
+        use core_foundation::{base::TCFType, boolean::CFBoolean, dictionary::CFDictionary, string::CFString};
+        let options =
+            CFDictionary::from_CFType_pairs(&[(CFString::new("AXTrustedCheckOptionPrompt"), CFBoolean::true_value())]);
+        // SAFETY: the dictionary lives throughout the synchronous system call.
+        unsafe {
+            AXIsProcessTrustedWithOptions(options.as_concrete_TypeRef().cast());
+            CGRequestScreenCaptureAccess();
+        }
+    }
+    status()
+}
+
+#[cfg(target_os = "macos")]
+#[link(name = "ApplicationServices", kind = "framework")]
+unsafe extern "C" {
+    fn AXIsProcessTrusted() -> bool;
+    fn AXIsProcessTrustedWithOptions(options: *const std::ffi::c_void) -> bool;
+}
+
+#[cfg(target_os = "macos")]
+#[link(name = "CoreGraphics", kind = "framework")]
+unsafe extern "C" {
+    fn CGPreflightScreenCaptureAccess() -> bool;
+    fn CGRequestScreenCaptureAccess() -> bool;
+}

package/rust/src/preview.rs

@@ -0,0 +1,82 @@
+//! Independent, bounded window capture; never touches the driver's grounding caches.
+use crate::gateway::Target;
+#[derive(Clone, Debug, serde::Serialize, specta::Type)]
+#[serde(rename_all = "camelCase")]
+pub struct Frame {
+    pub image: String,
+    pub title: String,
+    pub app_name: String,
+    pub cursor: Option<CursorPosition>,
+}
+#[derive(Clone, Debug, serde::Serialize, specta::Type)]
+pub struct CursorPosition {
+    pub x: f64,
+    pub y: f64,
+}
+
+pub fn normalize_cursor(cursor: CursorPosition, x: f64, y: f64, width: f64, height: f64) -> Option<CursorPosition> {
+    if width <= 0.0 || height <= 0.0 {
+        return None;
+    }
+    let x = (cursor.x - x) / width;
+    let y = (cursor.y - y) / height;
+    (x.is_finite() && y.is_finite() && (0.0..=1.0).contains(&x) && (0.0..=1.0).contains(&y))
+        .then_some(CursorPosition { x, y })
+}
+#[cfg(any(target_os = "macos", target_os = "windows"))]
+pub fn capture(target: &Target, cursor: Option<CursorPosition>) -> Result<Frame, String> {
+    use base64::Engine;
+    if !crate::permissions::status().screen_capture {
+        return Err("Screen capture permission is unavailable".into());
+    }
+    let windows = xcap::Window::all().map_err(|e| e.to_string())?;
+    let window = windows
+        .into_iter()
+        .find(|w| w.id().ok() == Some(target.window_id) && w.pid().ok() == Some(target.pid))
+        .ok_or("Target window is no longer available")?;
+    if window.is_minimized().map_err(|e| e.to_string())? {
+        return Err("Target window is minimized".into());
+    }
+    let title = window.title().map_err(|e| e.to_string())?;
+    let app_name = window.app_name().map_err(|e| e.to_string())?;
+    let cursor = cursor.and_then(|point| {
+        normalize_cursor(
+            point,
+            window.x().ok()? as f64,
+            window.y().ok()? as f64,
+            window.width().ok()? as f64,
+            window.height().ok()? as f64,
+        )
+    });
+    let pixels = window.capture_image().map_err(|e| e.to_string())?;
+    let image = image::DynamicImage::ImageRgba8(pixels).thumbnail(960, 600).to_rgb8();
+    let mut bytes = Vec::new();
+    image::codecs::jpeg::JpegEncoder::new_with_quality(&mut bytes, 65)
+        .encode_image(&image)
+        .map_err(|e| e.to_string())?;
+    if bytes.len() > 1024 * 1024 {
+        return Err("Preview frame exceeds size limit".into());
+    }
+    Ok(Frame {
+        image: format!("data:image/jpeg;base64,{}", base64::engine::general_purpose::STANDARD.encode(bytes)),
+        title,
+        app_name,
+        cursor,
+    })
+}
+#[cfg(not(any(target_os = "macos", target_os = "windows")))]
+pub fn capture(_: &Target, _: Option<CursorPosition>) -> Result<Frame, String> {
+    Err("Window preview is unavailable on this platform".into())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[test]
+    fn cursor_coordinates_follow_window_geometry_and_ignore_other_displays() {
+        let p = normalize_cursor(CursorPosition { x: -900.0, y: 150.0 }, -1000.0, 50.0, 200.0, 200.0).unwrap();
+        assert_eq!((p.x, p.y), (0.5, 0.5));
+        assert!(normalize_cursor(CursorPosition { x: 100.0, y: 150.0 }, -1000.0, 50.0, 200.0, 200.0).is_none());
+        assert!(normalize_cursor(CursorPosition { x: f64::NAN, y: 0.0 }, 0.0, 0.0, 200.0, 200.0).is_none());
+    }
+}

package/rust/src/wire.rs

@@ -0,0 +1,48 @@
+use serde_json::{Value, json};
+use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, BufReader};
+
+pub async fn request(endpoint: &str, value: Value) -> Result<Value, String> {
+    #[cfg(unix)]
+    let stream = tokio::net::UnixStream::connect(endpoint).await.map_err(|e| e.to_string())?;
+    #[cfg(windows)]
+    let stream = tokio::net::windows::named_pipe::ClientOptions::new().open(endpoint).map_err(|e| e.to_string())?;
+    exchange(stream, value).await
+}
+
+async fn exchange<S: AsyncRead + AsyncWrite + Unpin>(mut stream: S, value: Value) -> Result<Value, String> {
+    let bytes = serde_json::to_vec(&value).map_err(|e| e.to_string())?;
+    stream.write_all(&bytes).await.map_err(|e| e.to_string())?;
+    stream.write_all(b"\n").await.map_err(|e| e.to_string())?;
+    let mut line = String::new();
+    BufReader::new(stream).take(65537).read_line(&mut line).await.map_err(|e| e.to_string())?;
+    if line.len() > 65536 || !line.ends_with('\n') {
+        return Err("Invalid Driver control response framing".into());
+    }
+    let response: Value = serde_json::from_str(&line).map_err(|e| e.to_string())?;
+    if response["ok"] != true {
+        return Err(response["error"].as_str().unwrap_or("Driver control request failed").into());
+    }
+    response.get("result").cloned().ok_or_else(|| "Driver response omitted result".into())
+}
+
+pub async fn metadata(endpoint: &str) -> Result<Value, String> {
+    request(endpoint, json!({"method":"metadata"})).await
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[tokio::test]
+    async fn rejects_truncated_and_failed_control_responses() {
+        for reply in ["{\"ok\":true,\"result\":{}}", "{\"ok\":false,\"error\":\"denied\"}\n", "{\"ok\":true}\n"] {
+            let (client, mut server) = tokio::io::duplex(1024);
+            let reply = reply.to_owned();
+            tokio::spawn(async move {
+                let mut input = [0u8; 64];
+                let _ = server.read(&mut input).await;
+                server.write_all(reply.as_bytes()).await.unwrap();
+            });
+            assert!(exchange(client, json!({"method":"metadata"})).await.is_err());
+        }
+    }
+}