@pheem49/mint 1.16.0 → 1.17.0

package/crates/mint-core/src/browser/lifecycle.rs

@@ -43,7 +43,7 @@ pub fn enable_browser_tools(config: &mut MintConfig) -> bool {
     changed
 }
 
-pub async fn spawn_automation_browser_with_url(
+async fn spawn_browser_process(
     config: &MintConfig,
     initial_url: Option<&str>,
 ) -> Result<(), String> {
@@ -54,13 +54,6 @@ pub async fn spawn_automation_browser_with_url(
         .unwrap_or("http://127.0.0.1:9222/json/list");
 
     if fetch_pages_endpoint(endpoint).await.is_ok() {
-        if let Some(url) = initial_url {
-            let trimmed = url.trim();
-            if !trimmed.is_empty() {
-                ensure_page_open(config).await?;
-                super::navigate::navigate(config, trimmed).await?;
-            }
-        }
         return Ok(());
     }
 
@@ -77,51 +70,40 @@ pub async fn spawn_automation_browser_with_url(
         .and_then(Value::as_str)
         .unwrap_or("chromium");
 
-    let profile_dir = dirs::config_dir()
-        .unwrap_or_else(std::env::temp_dir)
-        .join("mint")
-        .join("automation-profile");
-    let profile_arg = format!("--user-data-dir={}", profile_dir.to_string_lossy());
+    let args = build_browser_args(config, endpoint, initial_url)?;
 
-    let mut args = vec![
-        format!(
-            "--remote-debugging-port={}",
-            reqwest::Url::parse(endpoint)
-                .map_err(|e| e.to_string())?
-                .port_or_known_default()
-                .ok_or("Missing debugging port")?
-        ),
-        "--no-first-run".to_owned(),
-        "--no-default-browser-check".to_owned(),
-        profile_arg,
-    ];
-
-    if let Some(url) = initial_url {
-        let trimmed = url.trim();
-        if !trimmed.is_empty() {
-            args.push(trimmed.to_owned());
-        }
-    }
+    let custom_exe = config
+        .extra
+        .get("browserExecutable")
+        .and_then(Value::as_str)
+        .map(str::trim)
+        .filter(|s| !s.is_empty());
 
     let mut spawned = false;
-    let executables: Vec<&str> = if cfg!(target_os = "windows") {
-        vec!["chrome.exe", "chromium.exe", "msedge.exe"]
+    let mut executables: Vec<&str> = Vec::new();
+    if let Some(exe) = custom_exe {
+        executables.push(exe);
+    }
+    if cfg!(target_os = "windows") {
+        executables.extend_from_slice(&["chrome.exe", "chromium.exe", "msedge.exe"]);
     } else if cfg!(target_os = "macos") {
-        vec![
+        executables.extend_from_slice(&[
             "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
             "/Applications/Chromium.app/Contents/MacOS/Chromium",
             "chromium",
             "google-chrome",
-        ]
+        ]);
     } else {
-        vec![
-            "chromium",
-            "google-chrome-stable",
+        executables.extend_from_slice(&[
             "google-chrome",
+            "google-chrome-stable",
+            "chromium",
             "chrome",
             "chromium-browser",
-        ]
-    };
+            "/opt/google/chrome/google-chrome",
+            "/opt/google/chrome/chrome",
+        ]);
+    }
 
     for exe in executables {
         if std::process::Command::new(exe)
@@ -156,12 +138,12 @@ pub async fn spawn_automation_browser_with_url(
 }
 
 pub async fn spawn_automation_browser(config: &MintConfig) -> Result<(), String> {
-    spawn_automation_browser_with_url(config, None).await
+    spawn_browser_process(config, None).await
 }
 
 pub async fn ensure_page_open(config: &MintConfig) -> Result<(), String> {
     if !is_browser_running(config).await {
-        return Err("Browser automation is not running. Please run 'mint auto' first.".to_string());
+        spawn_browser_process(config, None).await?;
     }
     let endpoint = config
         .extra
@@ -189,3 +171,121 @@ pub async fn ensure_page_open(config: &MintConfig) -> Result<(), String> {
     }
     Ok(())
 }
+
+pub async fn spawn_automation_browser_with_url(
+    config: &MintConfig,
+    initial_url: Option<&str>,
+) -> Result<(), String> {
+    spawn_browser_process(config, initial_url).await?;
+    if let Some(url) = initial_url {
+        let trimmed = url.trim();
+        if !trimmed.is_empty() {
+            ensure_page_open(config).await?;
+            super::navigate::navigate(config, trimmed).await?;
+        }
+    }
+    Ok(())
+}
+
+pub(crate) fn build_browser_args(
+    config: &MintConfig,
+    endpoint: &str,
+    initial_url: Option<&str>,
+) -> Result<Vec<String>, String> {
+    let agent_mode = config
+        .extra
+        .get("browserAgentMode")
+        .and_then(Value::as_str)
+        .unwrap_or("headless");
+
+    let profile_dir = dirs::config_dir()
+        .unwrap_or_else(std::env::temp_dir)
+        .join("mint")
+        .join("automation-profile");
+    let profile_arg = format!("--user-data-dir={}", profile_dir.to_string_lossy());
+
+    let mut args = vec![
+        format!(
+            "--remote-debugging-port={}",
+            reqwest::Url::parse(endpoint)
+                .map_err(|e| e.to_string())?
+                .port_or_known_default()
+                .ok_or("Missing debugging port")?
+        ),
+        "--no-first-run".to_owned(),
+        "--no-default-browser-check".to_owned(),
+        profile_arg,
+    ];
+
+    if agent_mode == "headless" {
+        args.push("--headless=new".to_owned());
+        args.push("--disable-gpu".to_owned());
+        if cfg!(target_os = "linux") {
+            args.push("--no-sandbox".to_owned());
+            args.push("--disable-setuid-sandbox".to_owned());
+            args.push("--disable-dev-shm-usage".to_owned());
+        }
+    }
+
+    let width = config
+        .extra
+        .get("browserViewportWidth")
+        .and_then(Value::as_u64)
+        .unwrap_or(1440);
+    let height = config
+        .extra
+        .get("browserViewportHeight")
+        .and_then(Value::as_u64)
+        .unwrap_or(900);
+    args.push(format!("--window-size={width},{height}"));
+
+    if let Some(url) = initial_url {
+        let trimmed = url.trim();
+        if !trimmed.is_empty() {
+            args.push(trimmed.to_owned());
+        }
+    }
+
+    Ok(args)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn build_browser_args_headless_mode() {
+        let config = MintConfig::default();
+        let args = build_browser_args(&config, "http://127.0.0.1:9222/json/list", None).unwrap();
+        assert!(args.contains(&"--headless=new".to_string()));
+        assert!(args.contains(&"--window-size=1440,900".to_string()));
+        assert!(
+            args.iter()
+                .any(|a| a.starts_with("--remote-debugging-port=9222"))
+        );
+    }
+
+    #[test]
+    fn build_browser_args_headed_mode() {
+        let mut config = MintConfig::default();
+        config
+            .extra
+            .insert("browserAgentMode".into(), serde_json::json!("headed"));
+        config
+            .extra
+            .insert("browserViewportWidth".into(), serde_json::json!(1920));
+        config
+            .extra
+            .insert("browserViewportHeight".into(), serde_json::json!(1080));
+
+        let args = build_browser_args(
+            &config,
+            "http://127.0.0.1:9222/json/list",
+            Some("https://example.com"),
+        )
+        .unwrap();
+        assert!(!args.contains(&"--headless=new".to_string()));
+        assert!(args.contains(&"--window-size=1920,1080".to_string()));
+        assert!(args.contains(&"https://example.com".to_string()));
+    }
+}

package/crates/mint-core/src/browser/mod.rs

@@ -20,15 +20,16 @@ mod lifecycle;
 mod logging;
 mod navigate;
 mod overlay;
+pub mod runtime;
 mod session;
+pub mod workflow;
 
 pub(crate) use lifecycle::BROWSER_TOOLS;
 pub use session::{
-    BrowserActionResult, BrowserObservation, BrowserSession, execute_action, run_session, with_tab,
-};
-pub(crate) use session::{
-    finish_evidence as session_finish_evidence, last_state as session_last_state, selected_tab,
+    BrowserActionResult, BrowserObservation, BrowserSession, capture_metadata, execute_action,
+    run_session, with_tab,
 };
+pub(crate) use session::{finish_evidence as session_finish_evidence, selected_tab};
 
 use serde::Serialize;
 

package/crates/mint-core/src/browser/navigate.rs

@@ -116,7 +116,27 @@ pub async fn read_page_text(config: &MintConfig) -> Result<String, String> {
         config,
         "Runtime.evaluate",
         json!({
-            "expression": "document.body ? document.body.innerText.substring(0, 12000) : ''",
+            "expression": r#"(() => {
+                if (!document.body) return '';
+                try {
+                    const clone = document.body.cloneNode(true);
+                    const ignore = [
+                        'script', 'style', 'noscript', 'svg', 'canvas',
+                        '[aria-hidden="true"]',
+                        '.cookie-banner', '#cookie-banner',
+                        '.cookie-notice', '#cookie-notice',
+                        '.gdpr', '#gdpr'
+                    ];
+                    for (const sel of ignore) {
+                        clone.querySelectorAll(sel).forEach(el => el.remove());
+                    }
+                    const text = clone.innerText || '';
+                    const cleaned = text.replace(/[\r\n]{3,}/g, '\n\n').replace(/[ \t]{2,}/g, ' ').trim();
+                    return cleaned.substring(0, 12000);
+                } catch (_) {
+                    return (document.body.innerText || '').substring(0, 12000);
+                }
+            })()"#,
             "returnByValue": true
         }),
     )
@@ -145,22 +165,75 @@ pub async fn read_page_text(config: &MintConfig) -> Result<String, String> {
 /// Capture a PNG screenshot of the current browser page.
 /// Returns base64-encoded PNG string.
 pub async fn screenshot(config: &MintConfig) -> Result<String, String> {
+    capture_page(config, &json!({"mode":"viewport"})).await
+}
+
+pub(super) async fn capture_page(
+    config: &MintConfig,
+    args: &serde_json::Value,
+) -> Result<String, String> {
     log_action("SCREENSHOT", "Capturing screenshot");
     ensure_page_open(config).await?;
 
-    match cdp_call_raw(
+    let mode = args["mode"]
+        .as_str()
+        .filter(|s| !s.is_empty())
+        .unwrap_or("viewport");
+    let mut params =
+        json!({"format":"png","fromSurface":true,"captureBeyondViewport":mode!="viewport"});
+    let geometry = if mode == "fullPage" {
+        let metrics = cdp_call_raw(config, "Page.getLayoutMetrics", json!({})).await?;
+        let size = metrics["result"]
+            .get("cssContentSize")
+            .unwrap_or(&metrics["result"]["contentSize"]);
+        json!({"x":0,"y":0,"width":size["width"],"height":size["height"],"scale":1})
+    } else if mode == "element" {
+        let selector = args["selector"]
+            .as_str()
+            .filter(|s| !s.is_empty())
+            .ok_or("Element capture requires a selector or current elementRef")?;
+        let find = super::interact::selector_to_js_find(selector);
+        let response=cdp_call_raw(config,"Runtime.evaluate",json!({"expression":format!("(() => {{ {find} if(!el || !el.isConnected) throw new Error('stale_reference'); const r=el.getBoundingClientRect(); return {{x:r.x+scrollX,y:r.y+scrollY,width:r.width,height:r.height,scale:1}}; }})()"),"returnByValue":true})).await?;
+        response["result"]["result"]["value"].clone()
+    } else if mode == "viewport" {
+        let response = cdp_call_raw(
+            config,
+            "Runtime.evaluate",
+            json!({"expression":"({width:innerWidth,height:innerHeight})","returnByValue":true}),
+        )
+        .await?;
+        response["result"]["result"]["value"].clone()
+    } else {
+        return Err("Screenshot mode must be viewport, fullPage or element".into());
+    };
+    let dpr = cdp_call_raw(
         config,
-        "Page.captureScreenshot",
-        json!({
-            "format": "png",
-            "fromSurface": true,
-            "captureBeyondViewport": false
-        }),
+        "Runtime.evaluate",
+        json!({"expression":"devicePixelRatio","returnByValue":true}),
     )
-    .await
+    .await?["result"]["result"]["value"]
+        .as_f64()
+        .unwrap_or(1.0);
+    let width = geometry["width"].as_f64().unwrap_or(0.0);
+    let height = geometry["height"].as_f64().unwrap_or(0.0);
+    if !width.is_finite()
+        || !height.is_finite()
+        || width <= 0.0
+        || height <= 0.0
+        || width * height * dpr * dpr > crate::media::tool_images::PIXEL_LIMIT as f64
     {
+        return Err("Screenshot exceeds 32 million pixels or target has no visible area; capture the viewport or one element.".into());
+    }
+    if mode != "viewport" {
+        params["clip"] = geometry;
+    }
+
+    match cdp_call_raw(config, "Page.captureScreenshot", params).await {
         Ok(response) => {
             if let Some(data) = response["result"]["data"].as_str() {
+                if data.len() > crate::media::tool_images::ORIGINAL_LIMIT.div_ceil(3) * 4 + 4 {
+                    return Err("Screenshot exceeds 20 MiB; capture a smaller region.".into());
+                }
                 log_action(
                     "SCREENSHOT_SUCCESS",
                     &format!("Captured {} base64 bytes", data.len()),

package/crates/mint-core/src/browser/runtime.rs

@@ -0,0 +1,284 @@
+//! Owned Chrome runtime shared by the CLI and the conversational agent.
+use crate::MintConfig;
+use serde_json::{Value, json};
+use std::{
+    io::Read,
+    path::PathBuf,
+    process::{Child, Command, Stdio},
+    sync::{Arc, Mutex},
+    time::Duration,
+};
+
+pub struct ManagedBrowser {
+    child: Child,
+    profile: PathBuf,
+    pub endpoint: String,
+    pub command: String,
+    pub headless: bool,
+    diagnostics: Arc<Mutex<String>>,
+}
+
+impl Drop for ManagedBrowser {
+    fn drop(&mut self) {
+        #[cfg(unix)]
+        unsafe {
+            libc::kill(-(self.child.id() as i32), libc::SIGKILL);
+        }
+        #[cfg(windows)]
+        {
+            use std::os::windows::process::CommandExt;
+            let _ = Command::new("taskkill")
+                .args(["/PID", &self.child.id().to_string(), "/T", "/F"])
+                .creation_flags(0x08000000)
+                .output();
+        }
+        let _ = self.child.kill();
+        let _ = self.child.wait();
+        let _ = std::fs::remove_dir_all(&self.profile);
+    }
+}
+
+impl ManagedBrowser {
+    pub fn metadata(&self) -> Value {
+        json!({"mode":if self.headless {"headless"} else {"headed"},"command":self.command,"diagnostics":self.diagnostics.lock().unwrap().as_str()})
+    }
+
+    pub async fn launch(config: &MintConfig) -> Result<Self, String> {
+        let headless =
+            config.extra.get("browserAgentMode").and_then(Value::as_str) != Some("headed");
+        let width = config
+            .extra
+            .get("browserViewportWidth")
+            .and_then(Value::as_u64)
+            .unwrap_or(1440)
+            .clamp(320, 7680);
+        let height = config
+            .extra
+            .get("browserViewportHeight")
+            .and_then(Value::as_u64)
+            .unwrap_or(900)
+            .clamp(240, 4320);
+        let profile =
+            std::env::temp_dir().join(format!("mint-browser-run-{}", uuid::Uuid::new_v4()));
+        std::fs::create_dir_all(&profile).map_err(|e| e.to_string())?;
+        let mut args = vec![
+            "--remote-debugging-port=0".into(),
+            "--remote-debugging-address=127.0.0.1".into(),
+            format!("--user-data-dir={}", profile.display()),
+            format!("--window-size={width},{height}"),
+            "--no-first-run".into(),
+            "--no-default-browser-check".into(),
+            "--disable-background-networking".into(),
+            "--disable-component-update".into(),
+        ];
+        if headless {
+            args.push("--headless=new".into());
+            args.push("--disable-gpu".into());
+            if cfg!(target_os = "linux") {
+                args.push("--no-sandbox".into());
+                args.push("--disable-setuid-sandbox".into());
+                args.push("--disable-dev-shm-usage".into());
+            }
+        }
+        if let Some(extra) = config
+            .extra
+            .get("browserLaunchArgs")
+            .and_then(Value::as_array)
+        {
+            for arg in extra.iter().filter_map(Value::as_str) {
+                if arg.starts_with("--remote-debugging")
+                    || arg.starts_with("--user-data-dir")
+                    || arg.starts_with("--headless")
+                {
+                    let _ = std::fs::remove_dir_all(&profile);
+                    return Err(
+                        "browserLaunchArgs cannot override the managed endpoint, profile or mode"
+                            .into(),
+                    );
+                }
+                args.push(arg.into());
+            }
+        }
+        args.push("about:blank".into());
+        let configured = config
+            .extra
+            .get("browserExecutable")
+            .and_then(Value::as_str)
+            .filter(|s| !s.is_empty())
+            .or_else(|| {
+                config
+                    .extra
+                    .get("automationBrowser")
+                    .and_then(Value::as_str)
+                    .filter(|s| !s.is_empty() && *s != "chromium")
+            });
+        if configured.is_some_and(|exe| exe.to_ascii_lowercase().contains("firefox")) {
+            let _ = std::fs::remove_dir_all(&profile);
+            return Err("Browser automation requires Chrome/Chromium/Edge; set browserExecutable to a Chromium-based browser.".into());
+        }
+        let candidates: Vec<String> = if let Some(exe) = configured {
+            vec![exe.into()]
+        } else {
+            let mut candidates: Vec<String> = if cfg!(target_os = "windows") {
+                vec![
+                    "chrome.exe".into(),
+                    "msedge.exe".into(),
+                    "chromium.exe".into(),
+                ]
+            } else if cfg!(target_os = "macos") {
+                vec![
+                    "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome".into(),
+                    "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge".into(),
+                    "/Applications/Chromium.app/Contents/MacOS/Chromium".into(),
+                    "google-chrome".into(),
+                    "chromium".into(),
+                ]
+            } else {
+                vec![
+                    "google-chrome".into(),
+                    "google-chrome-stable".into(),
+                    "chromium".into(),
+                    "chromium-browser".into(),
+                    "microsoft-edge".into(),
+                    "/opt/google/chrome/google-chrome".into(),
+                    "/opt/google/chrome/chrome".into(),
+                ]
+            };
+            #[cfg(target_os = "windows")]
+            for base in ["PROGRAMFILES", "PROGRAMFILES(X86)", "LOCALAPPDATA"] {
+                if let Some(dir) = std::env::var_os(base) {
+                    for tail in [
+                        "Google/Chrome/Application/chrome.exe",
+                        "Microsoft/Edge/Application/msedge.exe",
+                    ] {
+                        candidates.push(
+                            PathBuf::from(&dir)
+                                .join(tail)
+                                .to_string_lossy()
+                                .into_owned(),
+                        );
+                    }
+                }
+            }
+            candidates.shrink_to_fit();
+            candidates
+        };
+        let diagnostics = Arc::new(Mutex::new(String::new()));
+        let mut launched = None;
+        for exe in candidates {
+            let mut command = Command::new(&exe);
+            command
+                .args(&args)
+                .stdout(Stdio::null())
+                .stderr(Stdio::piped());
+            #[cfg(unix)]
+            {
+                use std::os::unix::process::CommandExt;
+                command.process_group(0);
+            }
+            #[cfg(windows)]
+            {
+                use std::os::windows::process::CommandExt;
+                command.creation_flags(0x08000000);
+            }
+            if let Ok(mut child) = command.spawn() {
+                let mut stderr = child.stderr.take().unwrap();
+                let output = diagnostics.clone();
+                std::thread::spawn(move || {
+                    let mut bytes = [0; 4096];
+                    while let Ok(n) = stderr.read(&mut bytes) {
+                        if n == 0 {
+                            break;
+                        }
+                        let mut text = output.lock().unwrap();
+                        text.push_str(&String::from_utf8_lossy(&bytes[..n]));
+                        if text.len() > 16_384 {
+                            let mut cut = text.len() - 16_384;
+                            while !text.is_char_boundary(cut) {
+                                cut += 1;
+                            }
+                            text.drain(..cut);
+                        }
+                    }
+                });
+                let display = std::iter::once(exe.as_str())
+                    .chain(args.iter().map(String::as_str))
+                    .map(|s| {
+                        shlex::try_quote(s)
+                            .map(|s| s.into_owned())
+                            .unwrap_or_else(|_| "<argument>".into())
+                    })
+                    .collect::<Vec<_>>()
+                    .join(" ");
+                launched = Some((child, display));
+                break;
+            }
+        }
+        let Some((child, command)) = launched else {
+            let _ = std::fs::remove_dir_all(profile);
+            return Err("Chrome/Chromium/Edge not found. Install a Chromium browser or set browserExecutable to its executable path.".into());
+        };
+        let mut runtime = Self {
+            child,
+            profile,
+            endpoint: String::new(),
+            command,
+            headless,
+            diagnostics,
+        };
+        for _ in 0..100 {
+            if let Some(status) = runtime.child.try_wait().map_err(|e| e.to_string())? {
+                return Err(format!(
+                    "Chrome exited during startup ({status}): {}",
+                    runtime.diagnostics.lock().unwrap()
+                ));
+            }
+            if let Ok(port_file) =
+                std::fs::read_to_string(runtime.profile.join("DevToolsActivePort"))
+                && let Some(port) = port_file.lines().next().and_then(|p| p.parse::<u16>().ok())
+            {
+                let endpoint = format!("http://127.0.0.1:{port}/json/list");
+                if super::cdp::fetch_pages_endpoint(&endpoint).await.is_ok() {
+                    runtime.endpoint = endpoint;
+                    super::logging::log_action("LAUNCH", &runtime.command);
+                    return Ok(runtime);
+                }
+            }
+            tokio::time::sleep(Duration::from_millis(100)).await;
+        }
+        Err(format!(
+            "Chrome startup timed out: {}",
+            runtime.diagnostics.lock().unwrap()
+        ))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[tokio::test]
+    #[ignore = "requires local Chrome; launches an isolated real browser"]
+    async fn managed_chrome_owns_its_profile_and_is_cleaned_on_drop() {
+        let mut config = MintConfig::default();
+        config
+            .extra
+            .insert("browserLaunchArgs".into(), json!(["--no-sandbox"]));
+        let runtime = ManagedBrowser::launch(&config).await.unwrap();
+        let profile = runtime.profile.clone();
+        let endpoint = runtime.endpoint.clone();
+        assert!(runtime.command.contains("--headless=new"));
+        assert!(profile.exists());
+        assert!(
+            super::super::cdp::fetch_pages_endpoint(&endpoint)
+                .await
+                .is_ok()
+        );
+        drop(runtime);
+        assert!(!profile.exists());
+        assert!(
+            super::super::cdp::fetch_pages_endpoint(&endpoint)
+                .await
+                .is_err()
+        );
+    }
+}