@pheem49/mint 1.9.0 → 1.10.0

package/crates/mint-core/src/image_gen.rs

@@ -10,8 +10,8 @@ use crate::MintConfig;
 // Public request / response types
 // ─────────────────────────────────────────────────────────────────────────────
 
-/// A request to generate images from any supported provider.
-#[derive(Debug, Clone, Deserialize)]
+/// A request to generate or edit images from any supported provider.
+#[derive(Debug, Clone, Deserialize, Default)]
 #[serde(rename_all = "camelCase")]
 pub struct ImageGenRequest {
     pub prompt: String,
@@ -30,6 +30,15 @@ pub struct ImageGenRequest {
     /// Falls back to `config.image_gen_provider` when omitted.
     #[serde(default)]
     pub provider: Option<String>,
+    /// Optional input image data URI (`data:image/png;base64,...`) for Image-to-Image / Inpainting
+    #[serde(default)]
+    pub image_data_uri: Option<String>,
+    /// Optional mask image data URI for Inpainting
+    #[serde(default)]
+    pub mask_data_uri: Option<String>,
+    /// Optional mode: "generate" | "edit" | "inpaint". Inferred if omitted.
+    #[serde(default)]
+    pub mode: Option<String>,
 }
 
 /// One generated image returned from the model.
@@ -76,6 +85,10 @@ pub enum ImageGenError {
     // ── Generic ──────────────────────────────────────────────────────────────
     #[error("unsupported image generation provider: {0}")]
     UnsupportedProvider(String),
+    #[error(
+        "image-to-image editing is not supported by {0} — please use Replicate, Stability AI, or DALL·E"
+    )]
+    UnsupportedMode(String),
     #[error("API request failed: {0}")]
     Request(#[from] reqwest::Error),
     #[error("the model returned no images for this prompt")]
@@ -129,6 +142,15 @@ async fn call_nanobanana(
     config: &MintConfig,
     request: &ImageGenRequest,
 ) -> Result<ImageGenResponse, ImageGenError> {
+    if request.image_data_uri.is_some()
+        || request.mode.as_deref() == Some("edit")
+        || request.mode.as_deref() == Some("inpaint")
+    {
+        return Err(ImageGenError::UnsupportedMode(
+            "NanoBanana (Gemini Text-to-Image)".to_owned(),
+        ));
+    }
+
     let api_key = if config.api_key.trim().is_empty() {
         std::env::var("GEMINI_API_KEY").unwrap_or_default()
     } else {
@@ -270,6 +292,105 @@ async fn call_dalle(
         return Err(ImageGenError::MissingOpenAiKey);
     }
 
+    if let Some(ref img_data_uri) = request.image_data_uri {
+        let (mime_type, img_bytes) = parse_data_uri(img_data_uri)
+            .ok_or_else(|| ImageGenError::ModelError("invalid base64 image data URI".to_owned()))?;
+
+        let size = aspect_ratio_to_dalle_size(request.aspect_ratio.as_deref());
+        let boundary = format!("MintBoundary{}", uuid_v4_hex());
+
+        let mut text_fields = vec![
+            ("prompt".to_owned(), request.prompt.clone()),
+            ("n".to_owned(), "1".to_owned()),
+            ("size".to_owned(), size.to_owned()),
+            ("response_format".to_owned(), "b64_json".to_owned()),
+            ("model".to_owned(), "dall-e-2".to_owned()),
+        ];
+        if let Some(ref neg) = request.negative_prompt {
+            if !neg.trim().is_empty() {
+                text_fields.push((
+                    "prompt".to_owned(),
+                    format!("{} (avoid: {})", request.prompt, neg),
+                ));
+            }
+        }
+
+        let mut file_fields = vec![(
+            "image",
+            "image.png",
+            mime_type.as_str(),
+            img_bytes.as_slice(),
+        )];
+
+        let mask_bytes_storage;
+        if let Some(ref mask_data_uri) = request.mask_data_uri {
+            if let Some((mask_mime, mask_bytes)) = parse_data_uri(mask_data_uri) {
+                mask_bytes_storage = (mask_mime, mask_bytes);
+                file_fields.push((
+                    "mask",
+                    "mask.png",
+                    mask_bytes_storage.0.as_str(),
+                    mask_bytes_storage.1.as_slice(),
+                ));
+            }
+        }
+
+        let body = build_multipart_binary(&boundary, &text_fields, &file_fields);
+
+        let response: Value = client
+            .post("https://api.openai.com/v1/images/edits")
+            .bearer_auth(&api_key)
+            .header(
+                "content-type",
+                format!("multipart/form-data; boundary={boundary}"),
+            )
+            .body(body)
+            .send()
+            .await?
+            .error_for_status()
+            .map_err(ImageGenError::Request)?
+            .json()
+            .await?;
+
+        if let Some(err) = response.get("error") {
+            let msg = err["message"]
+                .as_str()
+                .unwrap_or("unknown OpenAI API error");
+            return Err(ImageGenError::ModelError(msg.to_owned()));
+        }
+
+        let data = response["data"]
+            .as_array()
+            .ok_or(ImageGenError::UnexpectedResponse)?;
+
+        let images: Vec<GeneratedImage> = data
+            .iter()
+            .filter_map(|item| {
+                let b64 = item["b64_json"].as_str()?;
+                if b64.is_empty() || STANDARD.decode(b64).is_err() {
+                    return None;
+                }
+                let data_uri = format!("data:image/png;base64,{b64}");
+                Some(GeneratedImage {
+                    data_uri,
+                    mime_type: "image/png".to_owned(),
+                })
+            })
+            .collect();
+
+        if images.is_empty() {
+            return Err(ImageGenError::NoImagesReturned);
+        }
+
+        return Ok(ImageGenResponse {
+            images,
+            model: "dall-e-2".to_owned(),
+            provider: "dalle".to_owned(),
+            prompt: request.prompt.clone(),
+            description: None,
+        });
+    }
+
     let model = request
         .model
         .as_deref()
@@ -393,42 +514,98 @@ async fn call_stability(
             }
         });
 
-    // Route: Stable Image Core uses /core; SD3.x models use /sd3
-    let (endpoint, model_param) = if model == "core" {
+    let boundary = format!("MintBoundary{}", uuid_v4_hex());
+
+    let (endpoint, raw_body) = if let Some(ref img_uri) = request.image_data_uri {
+        let (mime_type, img_bytes) = parse_data_uri(img_uri)
+            .ok_or_else(|| ImageGenError::ModelError("invalid base64 image data URI".to_owned()))?;
+
+        let is_inpaint =
+            request.mask_data_uri.is_some() || request.mode.as_deref() == Some("inpaint");
+        let endpoint = if is_inpaint {
+            "https://api.stability.ai/v2beta/stable-image/edit/inpaint".to_owned()
+        } else {
+            "https://api.stability.ai/v2beta/stable-image/generate/sd3".to_owned()
+        };
+
+        let mut body_parts: Vec<(String, String)> = vec![
+            ("prompt".into(), request.prompt.clone()),
+            ("output_format".into(), "png".to_owned()),
+        ];
+        if !is_inpaint && model != "core" {
+            body_parts.push(("model".into(), model.clone()));
+            body_parts.push(("mode".into(), "image-to-image".to_owned()));
+            body_parts.push(("strength".into(), "0.7".to_owned()));
+        }
+
+        if let Some(neg) = &request.negative_prompt {
+            let neg = neg.trim();
+            if !neg.is_empty() {
+                body_parts.push(("negative_prompt".into(), neg.to_owned()));
+            }
+        }
+
+        let mut file_fields = vec![(
+            "image",
+            "input.png",
+            mime_type.as_str(),
+            img_bytes.as_slice(),
+        )];
+        let mask_storage;
+        if let Some(ref mask_uri) = request.mask_data_uri {
+            if let Some((mask_mime, mask_bytes)) = parse_data_uri(mask_uri) {
+                mask_storage = (mask_mime, mask_bytes);
+                file_fields.push((
+                    "mask",
+                    "mask.png",
+                    mask_storage.0.as_str(),
+                    mask_storage.1.as_slice(),
+                ));
+            }
+        }
+
         (
-            "https://api.stability.ai/v2beta/stable-image/generate/core".to_owned(),
-            None,
+            endpoint,
+            build_multipart_binary(&boundary, &body_parts, &file_fields),
         )
     } else {
-        (
-            "https://api.stability.ai/v2beta/stable-image/generate/sd3".to_owned(),
-            Some(model.clone()),
-        )
-    };
-
-    let aspect_ratio = request.aspect_ratio.as_deref().unwrap_or("1:1");
+        // Route: Stable Image Core uses /core; SD3.x models use /sd3
+        let (endpoint, model_param) = if model == "core" {
+            (
+                "https://api.stability.ai/v2beta/stable-image/generate/core".to_owned(),
+                None,
+            )
+        } else {
+            (
+                "https://api.stability.ai/v2beta/stable-image/generate/sd3".to_owned(),
+                Some(model.clone()),
+            )
+        };
 
-    // Build multipart/form-data manually (reqwest multipart feature not enabled).
-    let boundary = format!("MintBoundary{}", uuid_v4_hex());
+        let aspect_ratio = request.aspect_ratio.as_deref().unwrap_or("1:1");
 
-    let mut body_parts: Vec<(String, String)> = vec![
-        ("prompt".into(), request.prompt.clone()),
-        ("aspect_ratio".into(), aspect_ratio.to_owned()),
-        ("output_format".into(), "png".to_owned()),
-    ];
+        let mut body_parts: Vec<(String, String)> = vec![
+            ("prompt".into(), request.prompt.clone()),
+            ("aspect_ratio".into(), aspect_ratio.to_owned()),
+            ("output_format".into(), "png".to_owned()),
+        ];
 
-    if let Some(neg) = &request.negative_prompt {
-        let neg = neg.trim();
-        if !neg.is_empty() {
-            body_parts.push(("negative_prompt".into(), neg.to_owned()));
+        if let Some(neg) = &request.negative_prompt {
+            let neg = neg.trim();
+            if !neg.is_empty() {
+                body_parts.push(("negative_prompt".into(), neg.to_owned()));
+            }
         }
-    }
 
-    if let Some(m) = model_param {
-        body_parts.push(("model".into(), m));
-    }
+        if let Some(m) = model_param {
+            body_parts.push(("model".into(), m));
+        }
 
-    let raw_body = build_multipart_text(&boundary, &body_parts);
+        (
+            endpoint,
+            build_multipart_text(&boundary, &body_parts).into_bytes(),
+        )
+    };
 
     let response = client
         .post(&endpoint)
@@ -604,17 +781,23 @@ async fn call_replicate(
         .map(str::to_owned)
         .unwrap_or_else(|| {
             let m = config.replicate_model.trim();
-            if m.is_empty() {
-                "black-forest-labs/flux-1.1-pro".to_owned()
-            } else {
+            if !m.is_empty() {
                 m.to_owned()
+            } else if request.image_data_uri.is_some() {
+                if request.mask_data_uri.is_some() {
+                    "black-forest-labs/flux-fill-dev".to_owned()
+                } else {
+                    "timbrooks/instruct-pix2pix".to_owned()
+                }
+            } else {
+                "black-forest-labs/flux-1.1-pro".to_owned()
             }
         });
 
     let num_images = request.num_images.unwrap_or(1).clamp(1, 4);
     let aspect_ratio = request.aspect_ratio.as_deref().unwrap_or("1:1");
 
-    // Build input — common fields used by FLUX and SDXL families
+    // Build input — common fields used by FLUX, InstructPix2Pix, and SDXL families
     let mut input = json!({
         "prompt": request.prompt,
         "num_outputs": num_images,
@@ -623,6 +806,13 @@ async fn call_replicate(
         "output_quality": 90
     });
 
+    if let Some(ref img_uri) = request.image_data_uri {
+        input["image"] = json!(img_uri);
+        if let Some(ref mask_uri) = request.mask_data_uri {
+            input["mask"] = json!(mask_uri);
+        }
+    }
+
     if let Some(neg) = &request.negative_prompt {
         let neg = neg.trim();
         if !neg.is_empty() {
@@ -790,6 +980,58 @@ fn build_multipart_text(boundary: &str, fields: &[(String, String)]) -> String {
     body
 }
 
+/// Parse base64 data URI (e.g. data:image/png;base64,...) into (mime_type, bytes).
+fn parse_data_uri(data_uri: &str) -> Option<(String, Vec<u8>)> {
+    if !data_uri.starts_with("data:") {
+        return None;
+    }
+    let parts: Vec<&str> = data_uri.splitn(2, ',').collect();
+    if parts.len() != 2 {
+        return None;
+    }
+    let header = parts[0];
+    let b64 = parts[1];
+
+    let mime_type = header
+        .trim_start_matches("data:")
+        .split(';')
+        .next()
+        .unwrap_or("image/png")
+        .trim()
+        .to_string();
+
+    let bytes = STANDARD.decode(b64.trim()).ok()?;
+    Some((mime_type, bytes))
+}
+
+/// Build multipart body containing both text fields and binary file fields.
+fn build_multipart_binary(
+    boundary: &str,
+    text_fields: &[(String, String)],
+    file_fields: &[(&str, &str, &str, &[u8])],
+) -> Vec<u8> {
+    let mut body = Vec::new();
+    for (name, value) in text_fields {
+        body.extend_from_slice(format!("--{boundary}\r\n").as_bytes());
+        body.extend_from_slice(
+            format!("Content-Disposition: form-data; name=\"{name}\"\r\n\r\n").as_bytes(),
+        );
+        body.extend_from_slice(value.as_bytes());
+        body.extend_from_slice(b"\r\n");
+    }
+    for (param_name, file_name, mime_type, bytes) in file_fields {
+        body.extend_from_slice(format!("--{boundary}\r\n").as_bytes());
+        body.extend_from_slice(
+            format!("Content-Disposition: form-data; name=\"{param_name}\"; filename=\"{file_name}\"\r\n").as_bytes(),
+        );
+        body.extend_from_slice(format!("Content-Type: {mime_type}\r\n\r\n").as_bytes());
+        body.extend_from_slice(bytes);
+        body.extend_from_slice(b"\r\n");
+    }
+    body.extend_from_slice(format!("--{boundary}--\r\n").as_bytes());
+    body
+}
+
 /// Build the final prompt string, optionally embedding aspect ratio and negative
 /// prompt guidance. Used by providers that accept a plain text prompt.
 fn build_prompt(request: &ImageGenRequest) -> String {
@@ -832,6 +1074,9 @@ mod tests {
             num_images: None,
             model: None,
             provider: None,
+            image_data_uri: None,
+            mask_data_uri: None,
+            mode: None,
         }
     }
 
@@ -850,6 +1095,9 @@ mod tests {
             num_images: Some(2),
             model: None,
             provider: None,
+            image_data_uri: None,
+            mask_data_uri: None,
+            mode: None,
         };
         let result = build_prompt(&req);
         assert!(result.contains("a cat"));
@@ -866,6 +1114,9 @@ mod tests {
             num_images: None,
             model: None,
             provider: None,
+            image_data_uri: None,
+            mask_data_uri: None,
+            mode: None,
         };
         assert!(!build_prompt(&req).contains("aspect ratio"));
     }
@@ -892,4 +1143,14 @@ mod tests {
         assert_eq!(aspect_ratio_to_ideogram(Some("1:1")), "ASPECT_1_1");
         assert_eq!(aspect_ratio_to_ideogram(None), "ASPECT_1_1");
     }
+
+    #[test]
+    fn parse_data_uri_valid() {
+        let uri = "data:image/png;base64,aGVsbG8=";
+        let parsed = parse_data_uri(uri);
+        assert!(parsed.is_some());
+        let (mime, bytes) = parsed.unwrap();
+        assert_eq!(mime, "image/png");
+        assert_eq!(bytes, b"hello");
+    }
 }

package/crates/mint-core/src/lib.rs

@@ -20,6 +20,7 @@ pub mod skills;
 pub mod symbols;
 pub mod tasks;
 pub mod tts;
+pub mod video_gen;
 pub mod weather;
 pub mod web_search;
 pub mod workflows;
@@ -31,8 +32,9 @@ pub use chat::{
 };
 
 pub use browser::{
-    BrowserTab, click, is_browser_running, list_tabs, navigate, read_page_text,
-    spawn_automation_browser, type_text,
+    BrowserTab, click, get_element_coordinates, is_browser_running, key_press, list_tabs,
+    mouse_click, mouse_move, navigate, read_page_text, screenshot, spawn_automation_browser,
+    type_text, type_text_native,
 };
 pub use code_tools::{
     AppliedCodeEdit, CodeEdit, CodeEditPreview, CodeEditProposal, CodeFile, CodeInspectionError,
@@ -82,6 +84,7 @@ pub use skills::{SkillError, learn_skill, learned_skills_context};
 pub use symbols::{CodeSymbol, SymbolError, SymbolIndex, build_symbol_index};
 pub use tasks::{Task, TaskError, TaskStore, tasks_path};
 pub use tts::{TtsUrl, google_tts_urls};
+pub use video_gen::{VideoGenError, VideoGenRequest, VideoGenResponse, generate_video};
 pub use weather::{WeatherError, WeatherReport, weather};
 pub use workflows::{WorkflowError, load_workflows, save_workflows, workflows_path};
 pub mod api_server;

package/crates/mint-core/src/mcp.rs

@@ -2,7 +2,10 @@ use std::{
     collections::BTreeMap,
     io::{BufRead, BufReader, Write},
     process::{Child, Command, Stdio},
-    sync::mpsc,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        mpsc,
+    },
     time::{Duration, Instant},
 };
 
@@ -12,7 +15,8 @@ use thiserror::Error;
 
 use crate::{ConfigError, MintConfig, load_config, save_config};
 
-const MCP_TIMEOUT: Duration = Duration::from_secs(10);
+const MCP_TIMEOUT: Duration = Duration::from_secs(30);
+static OAUTH_DETECTED: AtomicBool = AtomicBool::new(false);
 
 #[derive(Debug, Clone, Deserialize, Serialize)]
 pub struct McpServer {
@@ -21,6 +25,8 @@ pub struct McpServer {
     pub args: Vec<String>,
     #[serde(default)]
     pub env: BTreeMap<String, String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub icon: Option<String>,
 }
 
 #[derive(Debug, Error)]
@@ -33,7 +39,9 @@ pub enum McpError {
     InvalidEnvironment,
     #[error("MCP server '{0}' is not configured")]
     MissingServer(String),
-    #[error("MCP tool '{server}/{tool}' is not allowed by policy")]
+    #[error(
+        "MCP tool '{server}/{tool}' is not allowed by policy. To allow it, please run: /mcp allow {server} {tool} (or /mcp allow {server} * to allow all tools on this server)"
+    )]
     NotAllowed { server: String, tool: String },
     #[error("unable to start MCP server '{command}': {source}")]
     Start {
@@ -82,6 +90,7 @@ pub fn add_mcp_server(
             command: command.into(),
             args,
             env: parse_env(env)?,
+            icon: None,
         },
     );
     save_mcp_servers(&mut config, servers)
@@ -186,18 +195,86 @@ fn parse_env(values: Vec<String>) -> Result<BTreeMap<String, String>, McpError>
         .collect()
 }
 
+fn find_url(line: &str) -> Option<String> {
+    let start_idx = line.find("http://").or_else(|| line.find("https://"))?;
+    let rest = &line[start_idx..];
+    let end_idx = rest
+        .find(|c: char| c.is_whitespace() || c == '"' || c == '\'' || c == '<' || c == '>')
+        .unwrap_or(rest.len());
+    let url = rest[..end_idx].to_string();
+
+    let lower = url.to_lowercase();
+    if lower.contains("registry.npmjs.org")
+        || lower.contains("npmjs.com")
+        || lower.contains("github.com/modelcontextprotocol")
+    {
+        return None;
+    }
+
+    if lower.contains("oauth")
+        || lower.contains("auth")
+        || lower.contains("login")
+        || lower.contains("google.com")
+        || lower.contains("authorize")
+    {
+        Some(url)
+    } else {
+        None
+    }
+}
+
+fn open_url_in_browser(url: &str) -> std::io::Result<()> {
+    #[cfg(target_os = "linux")]
+    {
+        Command::new("xdg-open").arg(url).spawn().map(|_| ())
+    }
+    #[cfg(target_os = "macos")]
+    {
+        Command::new("open").arg(url).spawn().map(|_| ())
+    }
+    #[cfg(target_os = "windows")]
+    {
+        Command::new("cmd")
+            .args(&["/C", "start", url])
+            .spawn()
+            .map(|_| ())
+    }
+    #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
+    {
+        let _ = url;
+        Ok(())
+    }
+}
+
 fn start_server(server: &McpServer) -> Result<Child, McpError> {
     let mut process = Command::new(&server.command)
         .args(&server.args)
         .envs(&server.env)
         .stdin(Stdio::piped())
         .stdout(Stdio::piped())
-        .stderr(Stdio::null())
+        .stderr(Stdio::piped())
         .spawn()
         .map_err(|source| McpError::Start {
             command: server.command.clone(),
             source,
         })?;
+
+    if let Some(stderr) = process.stderr.take() {
+        std::thread::spawn(move || {
+            let reader = BufReader::new(stderr);
+            for line in reader.lines().map_while(Result::ok) {
+                if let Some(url) = find_url(&line) {
+                    println!(
+                        "\n\x1b[1;33m[MCP Authorization Needed]\x1b[0m Opening browser to authenticate: {}\n",
+                        url
+                    );
+                    OAUTH_DETECTED.store(true, Ordering::Relaxed);
+                    let _ = open_url_in_browser(&url);
+                }
+            }
+        });
+    }
+
     write_message(
         &mut process,
         &json!({
@@ -224,26 +301,62 @@ fn exchange(process: &mut Child, request: Value) -> Result<Value, McpError> {
     let (sender, receiver) = mpsc::channel();
     std::thread::spawn(move || {
         for line in BufReader::new(stdout).lines().map_while(Result::ok) {
+            if let Some(url) = find_url(&line) {
+                println!(
+                    "\n\x1b[1;33m[MCP Authorization Needed]\x1b[0m Opening browser to authenticate: {}\n",
+                    url
+                );
+                OAUTH_DETECTED.store(true, Ordering::Relaxed);
+                let _ = open_url_in_browser(&url);
+            }
             if let Ok(value) = serde_json::from_str::<Value>(&line) {
                 let _ = sender.send(value);
             }
         }
     });
+
+    let timeout = if OAUTH_DETECTED.load(Ordering::Relaxed) {
+        Duration::from_secs(120)
+    } else {
+        MCP_TIMEOUT
+    };
+
     let started = Instant::now();
-    while started.elapsed() < MCP_TIMEOUT {
+    while started.elapsed() < timeout {
         let response = receiver
-            .recv_timeout(MCP_TIMEOUT.saturating_sub(started.elapsed()))
+            .recv_timeout(timeout.saturating_sub(started.elapsed()))
             .map_err(|_| McpError::Timeout)?;
         if response["id"] == 2 {
+            OAUTH_DETECTED.store(false, Ordering::Relaxed);
             if let Some(error) = response.get("error") {
                 return Err(McpError::Tool(error.clone()));
             }
             return Ok(response.get("result").cloned().unwrap_or(Value::Null));
         }
     }
+    OAUTH_DETECTED.store(false, Ordering::Relaxed);
     Err(McpError::Timeout)
 }
 
+pub fn list_server_tools(config: &MintConfig, server_name: &str) -> Result<Value, McpError> {
+    let servers = configured_mcp_servers(config)?;
+    let server = servers
+        .get(server_name)
+        .ok_or_else(|| McpError::MissingServer(server_name.into()))?;
+    let mut process = start_server(server)?;
+    let result = exchange(
+        &mut process,
+        json!({
+            "jsonrpc": "2.0",
+            "id": 2,
+            "method": "tools/list",
+            "params": {}
+        }),
+    );
+    let _ = process.kill();
+    result
+}
+
 fn write_message(process: &mut Child, message: &Value) -> Result<(), McpError> {
     let stdin = process.stdin.as_mut().ok_or(McpError::MissingStdin)?;
     writeln!(stdin, "{message}").map_err(McpError::Write)?;