@pheem49/mint 1.9.0 → 1.10.0

package/crates/mint-core/src/orchestration.rs

@@ -302,6 +302,7 @@ pub fn build_system_prompt(config: &MintConfig) -> String {
         "note_write",
         "run_plugin",
         "mcp_tool",
+        "mcp_list_tools",
         "run_shell",
         "verify",
         "apply_patch",
@@ -313,6 +314,10 @@ pub fn build_system_prompt(config: &MintConfig) -> String {
         allowed_actions.push("browser_click");
         allowed_actions.push("browser_type");
         allowed_actions.push("browser_read");
+        allowed_actions.push("browser_mouse_move");
+        allowed_actions.push("browser_mouse_click");
+        allowed_actions.push("browser_key_press");
+        allowed_actions.push("browser_screenshot");
     }
 
     allowed_actions.retain(|action| !config.disabled_tools.contains(&action.to_string()));
@@ -354,15 +359,27 @@ pub fn build_system_prompt(config: &MintConfig) -> String {
         input_formats.push("- browser_open: {\"url\":\"https://example.com\"}");
     }
     if allowed_actions.contains(&"browser_click") {
-        input_formats.push("- browser_click: {\"selector\":\"button.submit-btn\"}");
+        input_formats.push("- browser_click: {\"selector\":\"button.submit-btn\"} (CSS selector, or text=Login, contains=Submit, xpath=//button)");
     }
     if allowed_actions.contains(&"browser_type") {
         input_formats
-            .push("- browser_type: {\"selector\":\"input.search-bar\", \"text\":\"search query\"}");
+            .push("- browser_type: {\"selector\":\"input.search-bar\", \"text\":\"search query\"} (CSS selector, or text=Placeholder, contains=Label)");
     }
     if allowed_actions.contains(&"browser_read") {
         input_formats.push("- browser_read: {}");
     }
+    if allowed_actions.contains(&"browser_mouse_move") {
+        input_formats.push("- browser_mouse_move: {\"x\":640,\"y\":360}");
+    }
+    if allowed_actions.contains(&"browser_mouse_click") {
+        input_formats.push("- browser_mouse_click: {\"x\":640,\"y\":360,\"button\":\"left\"}");
+    }
+    if allowed_actions.contains(&"browser_key_press") {
+        input_formats.push("- browser_key_press: {\"key\":\"Enter\"} (keys: Enter,Tab,Escape,Backspace,Delete,ArrowUp,ArrowDown,ArrowLeft,ArrowRight,F1-F12,Space)");
+    }
+    if allowed_actions.contains(&"browser_screenshot") {
+        input_formats.push("- browser_screenshot: {}");
+    }
     if allowed_actions.contains(&"memory_recall") {
         input_formats.push("- memory_recall: {\"query\":\"what did user say about X\"}");
     }
@@ -410,8 +427,19 @@ pub fn build_system_prompt(config: &MintConfig) -> String {
     if allowed_actions.contains(&"run_plugin") {
         input_formats.push("- run_plugin: {\"name\":\"gmail|google_calendar|notion|docker|spotify|obsidian|system_metrics\",\"instruction\":\"instruction string\"}");
     }
+    let mcp_str;
     if allowed_actions.contains(&"mcp_tool") {
-        input_formats.push("- mcp_tool: {\"server\":\"configured-server\",\"tool\":\"tool-name\",\"arguments\":{}}");
+        let servers = crate::mcp::list_mcp_servers()
+            .map(|m| m.keys().cloned().collect::<Vec<String>>().join(", "))
+            .unwrap_or_default();
+        mcp_str = format!(
+            "- mcp_tool: {{\"server\":\"<name>\",\"tool\":\"tool-name\",\"arguments\":{{}}}} (Available configured servers: {})",
+            servers
+        );
+        input_formats.push(&mcp_str);
+    }
+    if allowed_actions.contains(&"mcp_list_tools") {
+        input_formats.push("- mcp_list_tools: {\"server\":\"configured-server\"}");
     }
     if allowed_actions.contains(&"run_shell") {
         input_formats.push(
@@ -447,7 +475,7 @@ pub fn build_system_prompt(config: &MintConfig) -> String {
         );
     }
     if allowed_actions.contains(&"apply_patch") && allowed_actions.contains(&"write_file") {
-        rules.push("3. Use apply_patch for all edits to existing files. write_file is only for creating new files inside the workspace.");
+        rules.push("3. Use apply_patch for all edits to existing files (write_file is only for creating new files). For edits across multiple sections of a file, supply separate small hunks in the 'hunks' array (one hunk per edit location). Keep oldText in each hunk minimal (only 1-3 exact lines needed to pinpoint the edit). NEVER bundle multiple separate edits into a single giant hunk that spans the entire file.");
     }
     if allowed_actions.contains(&"run_shell")
         || allowed_actions.contains(&"write_file")
@@ -468,16 +496,28 @@ pub fn build_system_prompt(config: &MintConfig) -> String {
         rules.push("7a. Use browser_open to navigate the virtual browser to a URL.");
     }
     if allowed_actions.contains(&"browser_click") {
-        rules.push("7b. Use browser_click to click elements on the page.");
+        rules.push("7b. Use browser_click to click elements. Selector can be: CSS selector (button.class, #id, [attr=val]), text=ExactText to find by visible text, contains=PartialText for partial match, or xpath=//expr for XPath. Prefer text= or contains= when the element has visible text but no unique CSS class.");
     }
     if allowed_actions.contains(&"browser_type") {
-        rules.push("7c. Use browser_type to enter text/input into form fields or search boxes.");
+        rules.push("7c. Use browser_type to enter text into form fields or search boxes. Selector supports the same formats as browser_click.");
     }
     if allowed_actions.contains(&"browser_read") {
         rules.push(
             "7d. Use browser_read to read the text content of the active page to analyze it.",
         );
     }
+    if allowed_actions.contains(&"browser_mouse_move") {
+        rules.push("7e. Use browser_mouse_move to move the real mouse cursor to absolute (x,y) coordinates.");
+    }
+    if allowed_actions.contains(&"browser_mouse_click") {
+        rules.push("7f. Use browser_mouse_click to perform a native mouse click at (x,y) coordinates. Use browser_screenshot first to find the target position.");
+    }
+    if allowed_actions.contains(&"browser_key_press") {
+        rules.push("7g. Use browser_key_press to press a keyboard key (e.g. Enter, Tab, Escape, ArrowDown). Use after browser_type or browser_mouse_click to interact with focused elements.");
+    }
+    if allowed_actions.contains(&"browser_screenshot") {
+        rules.push("7h. Use browser_screenshot to capture the current page as a PNG image (base64). Use it to inspect the visual state of the page before deciding where to click.");
+    }
     if allowed_actions.contains(&"memory_recall") {
         rules.push("8. Use memory_recall to search past interactions before asking the user to repeat context.");
     }
@@ -658,6 +698,14 @@ struct AgentInput {
     selector: String,
     #[serde(default)]
     text: String,
+    #[serde(default)]
+    x: Option<f64>,
+    #[serde(default)]
+    y: Option<f64>,
+    #[serde(default)]
+    button: String,
+    #[serde(default)]
+    key: String,
 }
 
 #[derive(Debug, Deserialize, Serialize)]
@@ -772,6 +820,7 @@ pub async fn orchestrate_agent_loop<Approve, Progress, Chunk>(
     task: &str,
     root: &Path,
     image_data_uri: Option<String>,
+    video_data_uri: Option<String>,
     chat_id: Option<&str>,
     agent_id: Option<&str>,
     fast_mode: bool,
@@ -793,15 +842,17 @@ where
         ))
     })?;
     let resolved_task = resolve_github_links(task, config).await;
-    let skills = crate::skills::learned_skills_context(Some(&root)).unwrap_or_default();
-    let mut observation = initial_observation(&resolved_task, &root, &skills);
-    let mut pending_image = image_data_uri;
-
-    let mut system_prompt = build_system_prompt(config);
     let chat_id = chat_id
         .map(str::trim)
         .filter(|chat_id| !chat_id.is_empty())
         .unwrap_or(DEFAULT_CONVERSATION_ID);
+    let skills =
+        crate::skills::learned_skills_context(Some(&root), Some(chat_id)).unwrap_or_default();
+    let mut observation = initial_observation(&resolved_task, &root, &skills);
+    let mut pending_image = image_data_uri;
+    let mut pending_video = video_data_uri;
+
+    let mut system_prompt = build_system_prompt(config);
 
     if let Ok(memory) = MemoryStore::open_default() {
         let mut profile_instructions = String::new();
@@ -886,6 +937,7 @@ where
                 chat_id: Some(chat_id.to_owned()),
                 image_data_uri: pending_image.take(),
                 audio_data_uri: None,
+                video_data_uri: pending_video.take(),
                 document_attachment: None,
                 workspace_path: None,
                 agent_id: None,
@@ -915,6 +967,7 @@ where
                         chat_id: Some(chat_id.to_owned()),
                         image_data_uri: None,
                         audio_data_uri: None,
+                        video_data_uri: None,
                         document_attachment: None,
                         workspace_path: None,
                         agent_id: None,
@@ -1345,6 +1398,54 @@ where
                 .map_err(OrchestrationError::Agent)?;
             Ok(result)
         }
+        "browser_mouse_move" => {
+            let x = input.x.ok_or_else(|| {
+                OrchestrationError::Agent("browser_mouse_move requires 'x'".into())
+            })?;
+            let y = input.y.ok_or_else(|| {
+                OrchestrationError::Agent("browser_mouse_move requires 'y'".into())
+            })?;
+            let result = crate::browser::mouse_move(config, x, y)
+                .await
+                .map_err(OrchestrationError::Agent)?;
+            Ok(result)
+        }
+        "browser_mouse_click" => {
+            let x = input.x.ok_or_else(|| {
+                OrchestrationError::Agent("browser_mouse_click requires 'x'".into())
+            })?;
+            let y = input.y.ok_or_else(|| {
+                OrchestrationError::Agent("browser_mouse_click requires 'y'".into())
+            })?;
+            let button = if input.button.is_empty() {
+                "left"
+            } else {
+                &input.button
+            };
+            let result = crate::browser::mouse_click(config, x, y, button)
+                .await
+                .map_err(OrchestrationError::Agent)?;
+            Ok(result)
+        }
+        "browser_key_press" => {
+            let key = if !input.key.is_empty() {
+                &input.key
+            } else {
+                return Err(OrchestrationError::Agent(
+                    "browser_key_press requires 'key'".into(),
+                ));
+            };
+            let result = crate::browser::key_press(config, key)
+                .await
+                .map_err(OrchestrationError::Agent)?;
+            Ok(result)
+        }
+        "browser_screenshot" => {
+            let data = crate::browser::screenshot(config)
+                .await
+                .map_err(OrchestrationError::Agent)?;
+            Ok(format!("data:image/png;base64,{data}"))
+        }
         "memory_recall" => {
             let query = required(&input.query, "query")?;
             let query_lower = query.to_ascii_lowercase();
@@ -1553,6 +1654,25 @@ where
                 ApprovalOutcome::Intercepted(obs) => Ok(obs),
             }
         }
+        "mcp_list_tools" => {
+            let server = required(&input.server, "server")?;
+            let approved = approve_cb(&AgentApproval::McpTool {
+                server: server.to_owned(),
+                tool: "list_tools".to_owned(),
+                arguments: serde_json::json!({}),
+            })
+            .map_err(OrchestrationError::Agent)?;
+
+            match approved {
+                ApprovalOutcome::Approved => Ok(serde_json::to_string_pretty(
+                    &crate::mcp::list_server_tools(config, server)
+                        .map_err(|e| OrchestrationError::Agent(e.to_string()))?,
+                )
+                .map_err(|e| OrchestrationError::Agent(e.to_string()))?),
+                ApprovalOutcome::Denied => Ok(format!("User denied MCP list tools: {}", server)),
+                ApprovalOutcome::Intercepted(obs) => Ok(obs),
+            }
+        }
         "run_shell" => {
             let command = required(&input.command, "command")?;
             let mode = classify_shell_command(command).mode.as_str().to_owned();
@@ -2260,6 +2380,7 @@ Example response:
         chat_id: None,
         image_data_uri: None,
         audio_data_uri: None,
+        video_data_uri: None,
         document_attachment: None,
         workspace_path: None,
         agent_id: None,
@@ -2320,6 +2441,7 @@ mod tests {
             chat_id: None,
             image_data_uri: None,
             audio_data_uri: None,
+            video_data_uri: None,
             document_attachment: None,
             workspace_path: None,
             agent_id: None,

package/crates/mint-core/src/pictures.rs

@@ -2,6 +2,7 @@ use std::{
     fs,
     io::Cursor,
     path::{Path, PathBuf},
+    process::Command,
     time::{SystemTime, UNIX_EPOCH},
 };
 
@@ -131,6 +132,11 @@ pub fn parse_data_uri(raw: &str) -> Option<(String, &'static str, Vec<u8>)> {
         "image/jpeg" | "image/jpg" => "jpg",
         "image/webp" => "webp",
         "image/gif" => "gif",
+        "video/mp4" => "mp4",
+        "video/webm" => "webm",
+        "video/quicktime" => "mov",
+        "video/x-msvideo" => "avi",
+        "video/x-matroska" => "mkv",
         _ => return None,
     };
     Some((mime_type, extension, STANDARD.decode(encoded).ok()?))
@@ -166,7 +172,13 @@ fn save_chat_images_to_directory(
         let id = filename
             .trim_end_matches(&format!(".{extension}"))
             .to_owned();
-        let thumbnail_path = create_thumbnail_from_bytes(directory, &id, &bytes);
+        let thumbnail_path = if mime_type.starts_with("image/") {
+            create_thumbnail_from_bytes(directory, &id, &bytes)
+        } else if mime_type.starts_with("video/") {
+            extract_video_thumbnail_from_bytes(directory, &id, &bytes)
+        } else {
+            None
+        };
         let entry = PictureEntry {
             id,
             filename,
@@ -221,6 +233,57 @@ fn create_thumbnail_from_bytes(directory: &Path, id: &str, bytes: &[u8]) -> Opti
     Some(path)
 }
 
+fn extract_video_thumbnail_from_bytes(directory: &Path, id: &str, bytes: &[u8]) -> Option<PathBuf> {
+    // Write video to temporary file
+    let temp_video = std::env::temp_dir().join(format!("mint-video-{}.tmp", id));
+    if fs::write(&temp_video, bytes).is_err() {
+        return None;
+    }
+
+    let temp_frame = std::env::temp_dir().join(format!("mint-frame-{}.png", id));
+
+    let res = (|| -> Option<PathBuf> {
+        let temp_video_str = temp_video.to_str()?;
+        let temp_frame_str = temp_frame.to_str()?;
+
+        let output = Command::new("ffmpeg")
+            .args(&["-i", temp_video_str])
+            .args(&["-vf", "select=eq(n\\,0)"])
+            .args(&["-vframes", "1"])
+            .args(&["-y", temp_frame_str])
+            .stderr(std::process::Stdio::null())
+            .stdout(std::process::Stdio::null())
+            .output()
+            .ok()?;
+
+        if !output.status.success() {
+            return None;
+        }
+
+        // Load extracted frame and create thumbnail
+        let frame_bytes = fs::read(&temp_frame).ok()?;
+        let image = image::load_from_memory(&frame_bytes).ok()?;
+        let thumbnail = image.thumbnail(THUMBNAIL_MAX_SIZE, THUMBNAIL_MAX_SIZE);
+
+        let thumbnail_directory = thumbnails_directory(directory);
+        ensure_directory(&thumbnail_directory).ok()?;
+        let path = thumbnail_directory.join(format!("{id}.thumb.png"));
+
+        let mut encoded = Cursor::new(Vec::new());
+        thumbnail
+            .write_to(&mut encoded, image::ImageFormat::Png)
+            .ok()?;
+        fs::write(&path, encoded.into_inner()).ok()?;
+        Some(path)
+    })();
+
+    // Always clean up temp files to avoid leaking in /tmp
+    let _ = fs::remove_file(&temp_video);
+    let _ = fs::remove_file(&temp_frame);
+
+    res
+}
+
 fn read_index(directory: &Path) -> Result<Vec<PictureEntry>, PictureError> {
     let path = directory.join("pictures.json");
     if !path.exists() {

package/crates/mint-core/src/skills.rs

@@ -53,10 +53,38 @@ pub fn learn_skill(path: &Path) -> Result<LearnedSkill, SkillError> {
     Ok(MemoryStore::open_default()?.add_learned_skill(name, &path.to_string_lossy(), &content)?)
 }
 
-pub fn learned_skills_context(workspace_root: Option<&Path>) -> Result<String, SkillError> {
+pub fn load_agent_rules_file(file_path: &Path, list: &mut Vec<LearnedSkill>) {
+    if file_path.is_file() {
+        if let Ok(content) = fs::read_to_string(file_path) {
+            let name = file_path
+                .file_name()
+                .and_then(|n| n.to_str())
+                .unwrap_or("AGENTS.md")
+                .to_string();
+            let description = parse_skill_description(&content)
+                .or_else(|| Some("Workspace instructions and rules".to_string()));
+            list.push(LearnedSkill {
+                id: 0,
+                name,
+                source_path: file_path.to_string_lossy().to_string(),
+                content,
+                created_at: String::new(),
+                description,
+            });
+        }
+    }
+}
+
+pub fn learned_skills_context(
+    workspace_root: Option<&Path>,
+    chat_id: Option<&str>,
+) -> Result<String, SkillError> {
     let mut skills = MemoryStore::open_default()?.learned_skills(20)?;
 
     if let Some(home) = dirs::home_dir() {
+        let global_agents_path = home.join(".gemini").join("config").join("AGENTS.md");
+        load_agent_rules_file(&global_agents_path, &mut skills);
+
         let global_skills_path = home.join(".config").join("mint").join("mint-skills");
         if !global_skills_path.exists() {
             let _ = std::fs::create_dir_all(&global_skills_path);
@@ -65,6 +93,12 @@ pub fn learned_skills_context(workspace_root: Option<&Path>) -> Result<String, S
     }
 
     if let Some(root) = workspace_root {
+        let workspace_agents_path1 = root.join(".agents").join("AGENTS.md");
+        load_agent_rules_file(&workspace_agents_path1, &mut skills);
+
+        let workspace_agents_path2 = root.join("AGENTS.md");
+        load_agent_rules_file(&workspace_agents_path2, &mut skills);
+
         let workspace_skills_path1 = root.join(".agents").join("skills");
         load_skills_from_dir(&workspace_skills_path1, &mut skills);
 
@@ -77,11 +111,53 @@ pub fn learned_skills_context(workspace_root: Option<&Path>) -> Result<String, S
         unique_skills.insert(skill.name.clone(), skill);
     }
 
-    let value = unique_skills
-        .into_values()
-        .map(|skill| format!("Skill: {}\n{}", skill.name, skill.content))
-        .collect::<Vec<_>>()
-        .join("\n\n");
+    let mut has_history = false;
+    if let Some(cid) = chat_id {
+        if let Ok(memory) = MemoryStore::open_default() {
+            if let Ok(interactions) = memory.recent_interactions_for_chat(cid, 1) {
+                has_history = !interactions.is_empty();
+            }
+        }
+    }
+
+    let mut formatted_skills = Vec::new();
+    for skill in unique_skills.into_values() {
+        let mut is_workspace_skill = false;
+        let mut rel_path = String::new();
+        if let Some(root) = workspace_root {
+            let skill_path = Path::new(&skill.source_path);
+            if let Ok(canonical_skill) = skill_path.canonicalize() {
+                if let Ok(canonical_root) = root.canonicalize() {
+                    if canonical_skill.starts_with(&canonical_root) {
+                        if let Ok(relative) = canonical_skill.strip_prefix(&canonical_root) {
+                            is_workspace_skill = true;
+                            rel_path = relative.to_string_lossy().to_string();
+                        }
+                    }
+                }
+            }
+        }
+
+        if is_workspace_skill {
+            let desc = skill
+                .description
+                .as_deref()
+                .unwrap_or("No description provided");
+            let status_msg = if has_history {
+                "READ. You have already read this skill in a previous turn of this conversation. Do NOT call `read_file` to read it again unless you need to re-verify its contents."
+            } else {
+                "UNREAD. You MUST execute the `read_file` tool with the path above to read this skill before starting the task."
+            };
+            formatted_skills.push(format!(
+                "Skill: {}\nDescription: {}\nPath: {}\nStatus: {}",
+                skill.name, desc, rel_path, status_msg
+            ));
+        } else {
+            formatted_skills.push(format!("Skill: {}\n{}", skill.name, skill.content));
+        }
+    }
+
+    let value = formatted_skills.join("\n\n");
     if value.len() <= MAX_CONTEXT_BYTES {
         return Ok(value);
     }