@pheem49/mint 1.8.2 → 1.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/Cargo.lock +55 -3
  2. package/Cargo.toml +2 -1
  3. package/Release_Note.md +297 -107
  4. package/bin/mint +0 -0
  5. package/crates/mint-cli/Cargo.toml +1 -0
  6. package/crates/mint-cli/src/actions.rs +233 -0
  7. package/crates/mint-cli/src/agent.rs +382 -127
  8. package/crates/mint-cli/src/gmail.rs +22 -2
  9. package/crates/mint-cli/src/image.rs +8 -8
  10. package/crates/mint-cli/src/interactive.rs +2888 -0
  11. package/crates/mint-cli/src/main.rs +391 -2506
  12. package/crates/mint-cli/src/markdown.rs +294 -0
  13. package/crates/mint-cli/src/onboard.rs +152 -109
  14. package/crates/mint-cli/src/setup.rs +143 -143
  15. package/crates/mint-cli/src/updater.rs +48 -6
  16. package/crates/mint-core/Cargo.toml +1 -0
  17. package/crates/mint-core/src/agent_loop.rs +1 -0
  18. package/crates/mint-core/src/api_server.rs +431 -65
  19. package/crates/mint-core/src/browser.rs +630 -102
  20. package/crates/mint-core/src/channels.rs +105 -6
  21. package/crates/mint-core/src/chat.rs +174 -11
  22. package/crates/mint-core/src/code_tools.rs +47 -31
  23. package/crates/mint-core/src/config.rs +302 -17
  24. package/crates/mint-core/src/image_gen.rs +299 -38
  25. package/crates/mint-core/src/lib.rs +10 -7
  26. package/crates/mint-core/src/mcp.rs +119 -6
  27. package/crates/mint-core/src/memory.rs +6 -1
  28. package/crates/mint-core/src/orchestration.rs +237 -96
  29. package/crates/mint-core/src/pictures.rs +67 -5
  30. package/crates/mint-core/src/semantic.rs +1 -1
  31. package/crates/mint-core/src/shell.rs +1 -1
  32. package/crates/mint-core/src/skills.rs +128 -25
  33. package/crates/mint-core/src/video_gen.rs +235 -0
  34. package/package.json +8 -2
  35. package/shared-modules/linux-audio/fluidsynth2-static.json +9 -3
  36. package/shared-modules/linux-audio/fluidsynth2.json +8 -2
  37. package/shared-modules/linux-audio/lilv.json +26 -19
  38. package/shared-modules/linux-audio/lv2.json +0 -1
  39. package/shared-modules/luajit/luajit.json +2 -2
  40. package/skills-lock.json +41 -0
  41. package/src/renderer/index-web.html +1 -1
  42. package/src/renderer/index.html +1 -1
  43. package/src/renderer/shared/components/AgentActivityTable.tsx +25 -6
  44. package/src/renderer/shared/constants/slashCommands.ts +27 -0
  45. package/src/renderer/shared/platform.ts +3 -0
  46. package/src/renderer/shared/types.ts +4 -0
  47. package/src/renderer/shared/utils/approval.tsx +86 -7
  48. package/src/renderer/shared/utils/fileIcons.ts +135 -0
  49. package/src/renderer/shared/utils/markdown.tsx +217 -88
  50. package/src/renderer/shared/utils/slashCommandProcessor.ts +239 -0
  51. package/src/renderer/src/components/ChatPanel.tsx +486 -69
  52. package/src/renderer/src/components/DashboardSidebar.tsx +11 -2
  53. package/src/renderer/src/components/ImageStudioPanel.tsx +87 -4
  54. package/src/renderer/src/components/MintDashboard.tsx +195 -7
  55. package/src/renderer/src/components/PicturesLibrary.tsx +156 -15
  56. package/src/renderer/src/components/Settings/AgentsTab.tsx +21 -2
  57. package/src/renderer/src/components/Settings/AutomationTab.tsx +316 -18
  58. package/src/renderer/src/components/Settings/GeneralTab.tsx +795 -413
  59. package/src/renderer/src/components/Settings/PluginsTab.tsx +304 -70
  60. package/src/renderer/src/components/Settings/ThemeTab.tsx +6 -1
  61. package/src/renderer/src/components/SettingsWindow.tsx +106 -3
  62. package/src/renderer/src/components/VeoStudioPanel.tsx +549 -0
  63. package/src/renderer/src/components/WorkspacePanel.tsx +7 -125
  64. package/src/renderer/src/css/image-studio.css +106 -0
  65. package/src/renderer/src/css/settings.css +158 -0
  66. package/src/renderer/src/css/styles.css +368 -1
  67. package/src/renderer/src/css/veo-studio.css +626 -0
  68. package/src/renderer/src/index.css +1 -0
  69. package/src/renderer/src/tauri.ts +144 -17
  70. package/src/renderer/src-web/components/ChatPanel.tsx +385 -93
  71. package/src/renderer/src-web/components/DashboardSidebar.tsx +16 -1
  72. package/src/renderer/src-web/components/ImageStudioPanel.tsx +87 -4
  73. package/src/renderer/src-web/components/MintDashboard.tsx +156 -5
  74. package/src/renderer/src-web/components/PicturesLibrary.tsx +155 -15
  75. package/src/renderer/src-web/components/Settings/AgentsTab.tsx +21 -2
  76. package/src/renderer/src-web/components/Settings/AutomationTab.tsx +316 -18
  77. package/src/renderer/src-web/components/Settings/GeneralTab.tsx +795 -411
  78. package/src/renderer/src-web/components/Settings/PluginsTab.tsx +304 -70
  79. package/src/renderer/src-web/components/Settings/ThemeTab.tsx +6 -1
  80. package/src/renderer/src-web/components/SettingsWindow.tsx +110 -3
  81. package/src/renderer/src-web/components/VeoStudioPanel.tsx +549 -0
  82. package/src/renderer/src-web/css/image-studio.css +106 -0
  83. package/src/renderer/src-web/css/settings.css +158 -0
  84. package/src/renderer/src-web/css/styles.css +369 -1
  85. package/src/renderer/src-web/css/veo-studio.css +650 -0
  86. package/src/renderer/src-web/index.css +1 -0
  87. package/src/renderer/src-web/tauri.ts +135 -16
  88. package/src-tauri/src/desktop.rs +18 -16
  89. package/src-tauri/src/discord_rpc.rs +1 -1
  90. package/src-tauri/src/lib.rs +155 -25
  91. package/src-tauri/src/webhooks.rs +45 -31
  92. package/src-tauri/src/workflows.rs +1 -0
  93. package/src-tauri/tauri.conf.json +1 -1
  94. package/temp_gridgeist/AGENTS.md +8 -0
  95. package/temp_gridgeist/CHANGELOG.md +130 -0
  96. package/temp_gridgeist/LICENSE +21 -0
  97. package/temp_gridgeist/README.md +323 -0
  98. package/temp_gridgeist/README.th.md +325 -0
  99. package/temp_gridgeist/skills/gridgeist/SKILL.md +88 -0
  100. package/temp_gridgeist/skills/gridgeist/agents/openai.yaml +9 -0
  101. package/temp_gridgeist/skills/gridgeist/assets/gridgeist-small.svg +10 -0
  102. package/temp_gridgeist/skills/gridgeist/assets/gridgeist.png +0 -0
  103. package/temp_gridgeist/skills/gridgeist/assets/gridgeist.svg +18 -0
  104. package/temp_gridgeist/skills/gridgeist/references/design-language.md +88 -0
  105. package/temp_gridgeist/skills/gridgeist/references/review-checklist.md +128 -0
  106. package/temp_gridgeist/skills/gridgeist/references/system-contract.md +109 -0
@@ -71,15 +71,14 @@ pub async fn resolve_github_links(message: &str, config: &MintConfig) -> String
71
71
  .collect();
72
72
 
73
73
  let repo_key = format!("{owner}/{repo_clean}");
74
- if resolved_repos.insert(repo_key.clone()) {
75
- if let Ok(summary) =
74
+ if resolved_repos.insert(repo_key.clone())
75
+ && let Ok(summary) =
76
76
  crate::code_tools::fetch_github_repo_summary(owner, &repo_clean).await
77
- {
78
- resolved_msg.push_str(&format!(
77
+ {
78
+ resolved_msg.push_str(&format!(
79
79
  "\n\n--- Auto-Resolved GitHub Metadata for {} ---\n{}\n--------------------------------------------",
80
80
  repo_key, summary
81
81
  ));
82
- }
83
82
  }
84
83
  }
85
84
  }
@@ -210,18 +209,18 @@ fn enrich_request(
210
209
  let mut enriched = request.clone();
211
210
 
212
211
  let mut profile_instructions = String::new();
213
- if let Ok(Some(name)) = memory.get_profile("name") {
214
- if !name.trim().is_empty() {
215
- profile_instructions.push_str(&format!("User Name: {}\n", name.trim()));
216
- }
212
+ if let Ok(Some(name)) = memory.get_profile("name")
213
+ && !name.trim().is_empty()
214
+ {
215
+ profile_instructions.push_str(&format!("User Name: {}\n", name.trim()));
217
216
  }
218
- if let Ok(Some(preferences)) = memory.get_profile("preferences") {
219
- if !preferences.trim().is_empty() {
220
- profile_instructions.push_str(&format!(
221
- "User Preferences & Profile:\n{}\n",
222
- preferences.trim()
223
- ));
224
- }
217
+ if let Ok(Some(preferences)) = memory.get_profile("preferences")
218
+ && !preferences.trim().is_empty()
219
+ {
220
+ profile_instructions.push_str(&format!(
221
+ "User Preferences & Profile:\n{}\n",
222
+ preferences.trim()
223
+ ));
225
224
  }
226
225
 
227
226
  if !profile_instructions.is_empty() {
@@ -239,8 +238,8 @@ fn enrich_request(
239
238
  "\n\n[Active Environment Context]\n\
240
239
  You are running on: {}\n\
241
240
  Using AI Model: {}\n",
242
- config.ai_provider,
243
- config.active_model()
241
+ config.ai_provider,
242
+ config.active_model()
244
243
  );
245
244
  enriched.system_instruction.push_str(&active_model_info);
246
245
 
@@ -303,6 +302,7 @@ pub fn build_system_prompt(config: &MintConfig) -> String {
303
302
  "note_write",
304
303
  "run_plugin",
305
304
  "mcp_tool",
305
+ "mcp_list_tools",
306
306
  "run_shell",
307
307
  "verify",
308
308
  "apply_patch",
@@ -314,6 +314,10 @@ pub fn build_system_prompt(config: &MintConfig) -> String {
314
314
  allowed_actions.push("browser_click");
315
315
  allowed_actions.push("browser_type");
316
316
  allowed_actions.push("browser_read");
317
+ allowed_actions.push("browser_mouse_move");
318
+ allowed_actions.push("browser_mouse_click");
319
+ allowed_actions.push("browser_key_press");
320
+ allowed_actions.push("browser_screenshot");
317
321
  }
318
322
 
319
323
  allowed_actions.retain(|action| !config.disabled_tools.contains(&action.to_string()));
@@ -355,15 +359,27 @@ pub fn build_system_prompt(config: &MintConfig) -> String {
355
359
  input_formats.push("- browser_open: {\"url\":\"https://example.com\"}");
356
360
  }
357
361
  if allowed_actions.contains(&"browser_click") {
358
- input_formats.push("- browser_click: {\"selector\":\"button.submit-btn\"}");
362
+ input_formats.push("- browser_click: {\"selector\":\"button.submit-btn\"} (CSS selector, or text=Login, contains=Submit, xpath=//button)");
359
363
  }
360
364
  if allowed_actions.contains(&"browser_type") {
361
365
  input_formats
362
- .push("- browser_type: {\"selector\":\"input.search-bar\", \"text\":\"search query\"}");
366
+ .push("- browser_type: {\"selector\":\"input.search-bar\", \"text\":\"search query\"} (CSS selector, or text=Placeholder, contains=Label)");
363
367
  }
364
368
  if allowed_actions.contains(&"browser_read") {
365
369
  input_formats.push("- browser_read: {}");
366
370
  }
371
+ if allowed_actions.contains(&"browser_mouse_move") {
372
+ input_formats.push("- browser_mouse_move: {\"x\":640,\"y\":360}");
373
+ }
374
+ if allowed_actions.contains(&"browser_mouse_click") {
375
+ input_formats.push("- browser_mouse_click: {\"x\":640,\"y\":360,\"button\":\"left\"}");
376
+ }
377
+ if allowed_actions.contains(&"browser_key_press") {
378
+ input_formats.push("- browser_key_press: {\"key\":\"Enter\"} (keys: Enter,Tab,Escape,Backspace,Delete,ArrowUp,ArrowDown,ArrowLeft,ArrowRight,F1-F12,Space)");
379
+ }
380
+ if allowed_actions.contains(&"browser_screenshot") {
381
+ input_formats.push("- browser_screenshot: {}");
382
+ }
367
383
  if allowed_actions.contains(&"memory_recall") {
368
384
  input_formats.push("- memory_recall: {\"query\":\"what did user say about X\"}");
369
385
  }
@@ -411,8 +427,19 @@ pub fn build_system_prompt(config: &MintConfig) -> String {
411
427
  if allowed_actions.contains(&"run_plugin") {
412
428
  input_formats.push("- run_plugin: {\"name\":\"gmail|google_calendar|notion|docker|spotify|obsidian|system_metrics\",\"instruction\":\"instruction string\"}");
413
429
  }
430
+ let mcp_str;
414
431
  if allowed_actions.contains(&"mcp_tool") {
415
- input_formats.push("- mcp_tool: {\"server\":\"configured-server\",\"tool\":\"tool-name\",\"arguments\":{}}");
432
+ let servers = crate::mcp::list_mcp_servers()
433
+ .map(|m| m.keys().cloned().collect::<Vec<String>>().join(", "))
434
+ .unwrap_or_default();
435
+ mcp_str = format!(
436
+ "- mcp_tool: {{\"server\":\"<name>\",\"tool\":\"tool-name\",\"arguments\":{{}}}} (Available configured servers: {})",
437
+ servers
438
+ );
439
+ input_formats.push(&mcp_str);
440
+ }
441
+ if allowed_actions.contains(&"mcp_list_tools") {
442
+ input_formats.push("- mcp_list_tools: {\"server\":\"configured-server\"}");
416
443
  }
417
444
  if allowed_actions.contains(&"run_shell") {
418
445
  input_formats.push(
@@ -448,7 +475,7 @@ pub fn build_system_prompt(config: &MintConfig) -> String {
448
475
  );
449
476
  }
450
477
  if allowed_actions.contains(&"apply_patch") && allowed_actions.contains(&"write_file") {
451
- rules.push("3. Use apply_patch for all edits to existing files. write_file is only for creating new files inside the workspace.");
478
+ rules.push("3. Use apply_patch for all edits to existing files (write_file is only for creating new files). For edits across multiple sections of a file, supply separate small hunks in the 'hunks' array (one hunk per edit location). Keep oldText in each hunk minimal (only 1-3 exact lines needed to pinpoint the edit). NEVER bundle multiple separate edits into a single giant hunk that spans the entire file.");
452
479
  }
453
480
  if allowed_actions.contains(&"run_shell")
454
481
  || allowed_actions.contains(&"write_file")
@@ -469,16 +496,28 @@ pub fn build_system_prompt(config: &MintConfig) -> String {
469
496
  rules.push("7a. Use browser_open to navigate the virtual browser to a URL.");
470
497
  }
471
498
  if allowed_actions.contains(&"browser_click") {
472
- rules.push("7b. Use browser_click to click elements on the page.");
499
+ rules.push("7b. Use browser_click to click elements. Selector can be: CSS selector (button.class, #id, [attr=val]), text=ExactText to find by visible text, contains=PartialText for partial match, or xpath=//expr for XPath. Prefer text= or contains= when the element has visible text but no unique CSS class.");
473
500
  }
474
501
  if allowed_actions.contains(&"browser_type") {
475
- rules.push("7c. Use browser_type to enter text/input into form fields or search boxes.");
502
+ rules.push("7c. Use browser_type to enter text into form fields or search boxes. Selector supports the same formats as browser_click.");
476
503
  }
477
504
  if allowed_actions.contains(&"browser_read") {
478
505
  rules.push(
479
506
  "7d. Use browser_read to read the text content of the active page to analyze it.",
480
507
  );
481
508
  }
509
+ if allowed_actions.contains(&"browser_mouse_move") {
510
+ rules.push("7e. Use browser_mouse_move to move the real mouse cursor to absolute (x,y) coordinates.");
511
+ }
512
+ if allowed_actions.contains(&"browser_mouse_click") {
513
+ rules.push("7f. Use browser_mouse_click to perform a native mouse click at (x,y) coordinates. Use browser_screenshot first to find the target position.");
514
+ }
515
+ if allowed_actions.contains(&"browser_key_press") {
516
+ rules.push("7g. Use browser_key_press to press a keyboard key (e.g. Enter, Tab, Escape, ArrowDown). Use after browser_type or browser_mouse_click to interact with focused elements.");
517
+ }
518
+ if allowed_actions.contains(&"browser_screenshot") {
519
+ rules.push("7h. Use browser_screenshot to capture the current page as a PNG image (base64). Use it to inspect the visual state of the page before deciding where to click.");
520
+ }
482
521
  if allowed_actions.contains(&"memory_recall") {
483
522
  rules.push("8. Use memory_recall to search past interactions before asking the user to repeat context.");
484
523
  }
@@ -659,6 +698,14 @@ struct AgentInput {
659
698
  selector: String,
660
699
  #[serde(default)]
661
700
  text: String,
701
+ #[serde(default)]
702
+ x: Option<f64>,
703
+ #[serde(default)]
704
+ y: Option<f64>,
705
+ #[serde(default)]
706
+ button: String,
707
+ #[serde(default)]
708
+ key: String,
662
709
  }
663
710
 
664
711
  #[derive(Debug, Deserialize, Serialize)]
@@ -685,7 +732,8 @@ fn resolve_agent_config(
685
732
  return (config.clone(), "".to_string(), None, None);
686
733
  }
687
734
 
688
- let enabled_agents: Vec<&crate::config::AgentConfig> = config.agents.iter().filter(|a| a.enabled).collect();
735
+ let enabled_agents: Vec<&crate::config::AgentConfig> =
736
+ config.agents.iter().filter(|a| a.enabled).collect();
689
737
  if enabled_agents.is_empty() {
690
738
  return (config.clone(), "".to_string(), None, None);
691
739
  }
@@ -694,7 +742,9 @@ fn resolve_agent_config(
694
742
  enabled_agents.iter().find(|a| a.id == id).copied()
695
743
  } else {
696
744
  // Multi-Agent Pipeline Collaboration (Planner -> Coder -> Reviewer)
697
- let plan_created = trajectory.iter().any(|step| step.contains("- Action: create_plan"));
745
+ let plan_created = trajectory
746
+ .iter()
747
+ .any(|step| step.contains("- Action: create_plan"));
698
748
  if !plan_created {
699
749
  if let Some(planner) = enabled_agents.iter().find(|a| a.id == "planner") {
700
750
  Some(*planner)
@@ -708,7 +758,9 @@ fn resolve_agent_config(
708
758
  });
709
759
  if edits_made {
710
760
  if let Some(last_step) = trajectory.last() {
711
- if last_step.contains("- Action: write_file") || last_step.contains("- Action: apply_patch") {
761
+ if last_step.contains("- Action: write_file")
762
+ || last_step.contains("- Action: apply_patch")
763
+ {
712
764
  if let Some(reviewer) = enabled_agents.iter().find(|a| a.id == "reviewer") {
713
765
  Some(*reviewer)
714
766
  } else {
@@ -741,21 +793,26 @@ fn resolve_agent_config(
741
793
  cfg_clone.local_model_name = agent.model.clone();
742
794
  cfg_clone.ollama_model = agent.model.clone();
743
795
 
744
- if let Some(key) = &agent.api_key {
745
- if !key.trim().is_empty() {
746
- match agent.provider.as_str() {
747
- "gemini" => cfg_clone.api_key = key.clone(),
748
- "openai" => cfg_clone.openai_api_key = key.clone(),
749
- "anthropic" => cfg_clone.anthropic_api_key = key.clone(),
750
- "openrouter" => cfg_clone.openrouter_api_key = key.clone(),
751
- "deepseek" => cfg_clone.deepseek_api_key = key.clone(),
752
- "huggingface" => cfg_clone.hf_api_key = key.clone(),
753
- _ => {}
754
- }
796
+ if let Some(key) = &agent.api_key
797
+ && !key.trim().is_empty()
798
+ {
799
+ match agent.provider.as_str() {
800
+ "gemini" => cfg_clone.api_key = key.clone(),
801
+ "openai" => cfg_clone.openai_api_key = key.clone(),
802
+ "anthropic" => cfg_clone.anthropic_api_key = key.clone(),
803
+ "openrouter" => cfg_clone.openrouter_api_key = key.clone(),
804
+ "deepseek" => cfg_clone.deepseek_api_key = key.clone(),
805
+ "huggingface" => cfg_clone.hf_api_key = key.clone(),
806
+ _ => {}
755
807
  }
756
808
  }
757
809
 
758
- (cfg_clone, agent.system_instruction.clone(), Some(agent.name.clone()), Some(agent.model.clone()))
810
+ (
811
+ cfg_clone,
812
+ agent.system_instruction.clone(),
813
+ Some(agent.name.clone()),
814
+ Some(agent.model.clone()),
815
+ )
759
816
  }
760
817
 
761
818
  pub async fn orchestrate_agent_loop<Approve, Progress, Chunk>(
@@ -763,6 +820,7 @@ pub async fn orchestrate_agent_loop<Approve, Progress, Chunk>(
763
820
  task: &str,
764
821
  root: &Path,
765
822
  image_data_uri: Option<String>,
823
+ video_data_uri: Option<String>,
766
824
  chat_id: Option<&str>,
767
825
  agent_id: Option<&str>,
768
826
  fast_mode: bool,
@@ -784,30 +842,32 @@ where
784
842
  ))
785
843
  })?;
786
844
  let resolved_task = resolve_github_links(task, config).await;
787
- let skills = crate::skills::learned_skills_context(Some(&root)).unwrap_or_default();
788
- let mut observation = initial_observation(&resolved_task, &root, &skills);
789
- let mut pending_image = image_data_uri;
790
-
791
- let mut system_prompt = build_system_prompt(config);
792
845
  let chat_id = chat_id
793
846
  .map(str::trim)
794
847
  .filter(|chat_id| !chat_id.is_empty())
795
848
  .unwrap_or(DEFAULT_CONVERSATION_ID);
849
+ let skills =
850
+ crate::skills::learned_skills_context(Some(&root), Some(chat_id)).unwrap_or_default();
851
+ let mut observation = initial_observation(&resolved_task, &root, &skills);
852
+ let mut pending_image = image_data_uri;
853
+ let mut pending_video = video_data_uri;
854
+
855
+ let mut system_prompt = build_system_prompt(config);
796
856
 
797
857
  if let Ok(memory) = MemoryStore::open_default() {
798
858
  let mut profile_instructions = String::new();
799
- if let Ok(Some(name)) = memory.get_profile("name") {
800
- if !name.trim().is_empty() {
801
- profile_instructions.push_str(&format!("User Name: {}\n", name.trim()));
802
- }
859
+ if let Ok(Some(name)) = memory.get_profile("name")
860
+ && !name.trim().is_empty()
861
+ {
862
+ profile_instructions.push_str(&format!("User Name: {}\n", name.trim()));
803
863
  }
804
- if let Ok(Some(preferences)) = memory.get_profile("preferences") {
805
- if !preferences.trim().is_empty() {
806
- profile_instructions.push_str(&format!(
807
- "User Preferences & Profile:\n{}\n",
808
- preferences.trim()
809
- ));
810
- }
864
+ if let Ok(Some(preferences)) = memory.get_profile("preferences")
865
+ && !preferences.trim().is_empty()
866
+ {
867
+ profile_instructions.push_str(&format!(
868
+ "User Preferences & Profile:\n{}\n",
869
+ preferences.trim()
870
+ ));
811
871
  }
812
872
 
813
873
  if !profile_instructions.is_empty() {
@@ -844,7 +904,8 @@ where
844
904
  let mut trajectory: Vec<String> = Vec::new();
845
905
 
846
906
  for step in 1..=MAX_STEPS {
847
- let (active_config, agent_instruction, active_agent_name, active_model_name) = resolve_agent_config(config, agent_id, &trajectory);
907
+ let (active_config, agent_instruction, active_agent_name, active_model_name) =
908
+ resolve_agent_config(config, agent_id, &trajectory);
848
909
 
849
910
  progress(AgentProgress::Thinking {
850
911
  elapsed_secs: started_at.elapsed().as_secs(),
@@ -858,12 +919,14 @@ where
858
919
  "\n\n[Active Environment Context]\n\
859
920
  You are running on: {}\n\
860
921
  Using AI Model: {}\n",
861
- active_config.ai_provider,
862
- model_name
922
+ active_config.ai_provider, model_name
863
923
  ));
864
924
  }
865
925
  if !agent_instruction.is_empty() {
866
- active_system_prompt.push_str(&format!("\n\nYour Current Role & System Instructions:\n{}", agent_instruction));
926
+ active_system_prompt.push_str(&format!(
927
+ "\n\nYour Current Role & System Instructions:\n{}",
928
+ agent_instruction
929
+ ));
867
930
  }
868
931
 
869
932
  let (response, fallback) = send_chat_with_fallback(
@@ -874,6 +937,7 @@ where
874
937
  chat_id: Some(chat_id.to_owned()),
875
938
  image_data_uri: pending_image.take(),
876
939
  audio_data_uri: None,
940
+ video_data_uri: pending_video.take(),
877
941
  document_attachment: None,
878
942
  workspace_path: None,
879
943
  agent_id: None,
@@ -903,6 +967,7 @@ where
903
967
  chat_id: Some(chat_id.to_owned()),
904
968
  image_data_uri: None,
905
969
  audio_data_uri: None,
970
+ video_data_uri: None,
906
971
  document_attachment: None,
907
972
  workspace_path: None,
908
973
  agent_id: None,
@@ -1261,14 +1326,14 @@ where
1261
1326
  if crate::is_browser_running(config).await {
1262
1327
  let result = crate::browser::navigate(config, url)
1263
1328
  .await
1264
- .map_err(|e| OrchestrationError::Agent(e))?;
1329
+ .map_err(OrchestrationError::Agent)?;
1265
1330
  Ok(result)
1266
1331
  } else {
1267
1332
  let opened = if cfg!(target_os = "macos") {
1268
1333
  std::process::Command::new("open").arg(url).spawn().is_ok()
1269
1334
  } else if cfg!(target_os = "windows") {
1270
1335
  std::process::Command::new("cmd")
1271
- .args(&["/C", "start", url])
1336
+ .args(["/C", "start", url])
1272
1337
  .spawn()
1273
1338
  .is_ok()
1274
1339
  } else {
@@ -1300,7 +1365,7 @@ where
1300
1365
  };
1301
1366
  let result = crate::browser::click(config, selector)
1302
1367
  .await
1303
- .map_err(|e| OrchestrationError::Agent(e))?;
1368
+ .map_err(OrchestrationError::Agent)?;
1304
1369
  Ok(result)
1305
1370
  }
1306
1371
  "browser_type" => {
@@ -1324,15 +1389,63 @@ where
1324
1389
  };
1325
1390
  let result = crate::browser::type_text(config, selector, text)
1326
1391
  .await
1327
- .map_err(|e| OrchestrationError::Agent(e))?;
1392
+ .map_err(OrchestrationError::Agent)?;
1328
1393
  Ok(result)
1329
1394
  }
1330
1395
  "browser_read" => {
1331
1396
  let result = crate::browser::read_page_text(config)
1332
1397
  .await
1333
- .map_err(|e| OrchestrationError::Agent(e))?;
1398
+ .map_err(OrchestrationError::Agent)?;
1334
1399
  Ok(result)
1335
1400
  }
1401
+ "browser_mouse_move" => {
1402
+ let x = input.x.ok_or_else(|| {
1403
+ OrchestrationError::Agent("browser_mouse_move requires 'x'".into())
1404
+ })?;
1405
+ let y = input.y.ok_or_else(|| {
1406
+ OrchestrationError::Agent("browser_mouse_move requires 'y'".into())
1407
+ })?;
1408
+ let result = crate::browser::mouse_move(config, x, y)
1409
+ .await
1410
+ .map_err(OrchestrationError::Agent)?;
1411
+ Ok(result)
1412
+ }
1413
+ "browser_mouse_click" => {
1414
+ let x = input.x.ok_or_else(|| {
1415
+ OrchestrationError::Agent("browser_mouse_click requires 'x'".into())
1416
+ })?;
1417
+ let y = input.y.ok_or_else(|| {
1418
+ OrchestrationError::Agent("browser_mouse_click requires 'y'".into())
1419
+ })?;
1420
+ let button = if input.button.is_empty() {
1421
+ "left"
1422
+ } else {
1423
+ &input.button
1424
+ };
1425
+ let result = crate::browser::mouse_click(config, x, y, button)
1426
+ .await
1427
+ .map_err(OrchestrationError::Agent)?;
1428
+ Ok(result)
1429
+ }
1430
+ "browser_key_press" => {
1431
+ let key = if !input.key.is_empty() {
1432
+ &input.key
1433
+ } else {
1434
+ return Err(OrchestrationError::Agent(
1435
+ "browser_key_press requires 'key'".into(),
1436
+ ));
1437
+ };
1438
+ let result = crate::browser::key_press(config, key)
1439
+ .await
1440
+ .map_err(OrchestrationError::Agent)?;
1441
+ Ok(result)
1442
+ }
1443
+ "browser_screenshot" => {
1444
+ let data = crate::browser::screenshot(config)
1445
+ .await
1446
+ .map_err(OrchestrationError::Agent)?;
1447
+ Ok(format!("data:image/png;base64,{data}"))
1448
+ }
1336
1449
  "memory_recall" => {
1337
1450
  let query = required(&input.query, "query")?;
1338
1451
  let query_lower = query.to_ascii_lowercase();
@@ -1486,7 +1599,7 @@ where
1486
1599
  path: file_name.to_owned(),
1487
1600
  content: input.file_content.clone(),
1488
1601
  })
1489
- .map_err(|e| OrchestrationError::Agent(e))?;
1602
+ .map_err(OrchestrationError::Agent)?;
1490
1603
 
1491
1604
  match approved {
1492
1605
  ApprovalOutcome::Approved => {
@@ -1509,7 +1622,7 @@ where
1509
1622
  name: name.to_owned(),
1510
1623
  instruction: instruction.to_owned(),
1511
1624
  })
1512
- .map_err(|e| OrchestrationError::Agent(e))?;
1625
+ .map_err(OrchestrationError::Agent)?;
1513
1626
 
1514
1627
  match approved {
1515
1628
  ApprovalOutcome::Approved => Ok(execute_native_plugin(config, name, instruction)
@@ -1527,7 +1640,7 @@ where
1527
1640
  tool: tool.to_owned(),
1528
1641
  arguments: input.arguments.clone(),
1529
1642
  })
1530
- .map_err(|e| OrchestrationError::Agent(e))?;
1643
+ .map_err(OrchestrationError::Agent)?;
1531
1644
 
1532
1645
  match approved {
1533
1646
  ApprovalOutcome::Approved => Ok(serde_json::to_string_pretty(
@@ -1541,6 +1654,25 @@ where
1541
1654
  ApprovalOutcome::Intercepted(obs) => Ok(obs),
1542
1655
  }
1543
1656
  }
1657
+ "mcp_list_tools" => {
1658
+ let server = required(&input.server, "server")?;
1659
+ let approved = approve_cb(&AgentApproval::McpTool {
1660
+ server: server.to_owned(),
1661
+ tool: "list_tools".to_owned(),
1662
+ arguments: serde_json::json!({}),
1663
+ })
1664
+ .map_err(OrchestrationError::Agent)?;
1665
+
1666
+ match approved {
1667
+ ApprovalOutcome::Approved => Ok(serde_json::to_string_pretty(
1668
+ &crate::mcp::list_server_tools(config, server)
1669
+ .map_err(|e| OrchestrationError::Agent(e.to_string()))?,
1670
+ )
1671
+ .map_err(|e| OrchestrationError::Agent(e.to_string()))?),
1672
+ ApprovalOutcome::Denied => Ok(format!("User denied MCP list tools: {}", server)),
1673
+ ApprovalOutcome::Intercepted(obs) => Ok(obs),
1674
+ }
1675
+ }
1544
1676
  "run_shell" => {
1545
1677
  let command = required(&input.command, "command")?;
1546
1678
  let mode = classify_shell_command(command).mode.as_str().to_owned();
@@ -1548,7 +1680,7 @@ where
1548
1680
  command: command.to_owned(),
1549
1681
  mode,
1550
1682
  })
1551
- .map_err(|e| OrchestrationError::Agent(e))?;
1683
+ .map_err(OrchestrationError::Agent)?;
1552
1684
 
1553
1685
  match approved {
1554
1686
  ApprovalOutcome::Approved => run_shell(root, config, command),
@@ -1593,7 +1725,7 @@ where
1593
1725
  hunks: patch.hunks.clone(),
1594
1726
  diff,
1595
1727
  })
1596
- .map_err(|e| OrchestrationError::Agent(e))?;
1728
+ .map_err(OrchestrationError::Agent)?;
1597
1729
 
1598
1730
  match approved {
1599
1731
  ApprovalOutcome::Approved => {
@@ -1629,7 +1761,7 @@ where
1629
1761
  content: input.file_content.clone(),
1630
1762
  diff,
1631
1763
  })
1632
- .map_err(|e| OrchestrationError::Agent(e))?;
1764
+ .map_err(OrchestrationError::Agent)?;
1633
1765
 
1634
1766
  match approved {
1635
1767
  ApprovalOutcome::Approved => {
@@ -1823,13 +1955,13 @@ fn view_image(path: &Path, config: &MintConfig) -> Result<String, OrchestrationE
1823
1955
  }
1824
1956
  let bytes = std::fs::read(&path)
1825
1957
  .map_err(|e| OrchestrationError::Agent(format!("cannot read image: {e}")))?;
1826
- Ok(serde_json::to_string_pretty(&serde_json::json!({
1958
+ serde_json::to_string_pretty(&serde_json::json!({
1827
1959
  "path": path,
1828
1960
  "bytes": bytes.len(),
1829
1961
  "mime": mime,
1830
1962
  "dataUri": format!("data:{mime};base64,{}", BASE64_STANDARD.encode(bytes)),
1831
1963
  }))
1832
- .map_err(|e| OrchestrationError::Agent(e.to_string()))?)
1964
+ .map_err(|e| OrchestrationError::Agent(e.to_string()))
1833
1965
  }
1834
1966
 
1835
1967
  fn run_shell(
@@ -1845,14 +1977,13 @@ fn run_shell(
1845
1977
 
1846
1978
  let mut hint = "";
1847
1979
  let cmd_lower = command.to_lowercase();
1848
- if output.success {
1849
- if cmd_lower.contains("open")
1980
+ if output.success
1981
+ && (cmd_lower.contains("open")
1850
1982
  || cmd_lower.contains("launch")
1851
1983
  || cmd_lower.contains("chrome")
1852
- || cmd_lower.contains("firefox")
1853
- {
1854
- hint = "\nNote: Opening URLs, files, folders, or launching applications are background processes. Even if there are warnings or stdout/stderr outputs, since the command exited successfully with status 0, the operation has succeeded and you should now use the 'finish' action to inform the user.";
1855
- }
1984
+ || cmd_lower.contains("firefox"))
1985
+ {
1986
+ hint = "\nNote: Opening URLs, files, folders, or launching applications are background processes. Even if there are warnings or stdout/stderr outputs, since the command exited successfully with status 0, the operation has succeeded and you should now use the 'finish' action to inform the user.";
1856
1987
  }
1857
1988
 
1858
1989
  Ok(format!(
@@ -1897,7 +2028,9 @@ fn action_fingerprint(decision: &AgentDecision) -> String {
1897
2028
  }
1898
2029
 
1899
2030
  fn initial_observation(task: &str, root: &Path, skills: &str) -> String {
1900
- let now = chrono::Local::now().format("%Y-%m-%d %H:%M:%S %Z").to_string();
2031
+ let now = chrono::Local::now()
2032
+ .format("%Y-%m-%d %H:%M:%S %Z")
2033
+ .to_string();
1901
2034
  let mut observation = format!(
1902
2035
  "Current Time: {now}\nTask: {task}\nWorkspace: {}\nLearned skills:\n{}\n",
1903
2036
  root.display(),
@@ -2247,6 +2380,7 @@ Example response:
2247
2380
  chat_id: None,
2248
2381
  image_data_uri: None,
2249
2382
  audio_data_uri: None,
2383
+ video_data_uri: None,
2250
2384
  document_attachment: None,
2251
2385
  workspace_path: None,
2252
2386
  agent_id: None,
@@ -2271,19 +2405,19 @@ Example response:
2271
2405
  text_reply.to_string()
2272
2406
  };
2273
2407
 
2274
- if let Ok(value) = serde_json::from_str::<serde_json::Value>(&clean_json) {
2275
- if let Some(obj) = value.as_object() {
2276
- if let Some(new_name) = obj.get("name").and_then(|v| v.as_str()) {
2277
- let trimmed_name = new_name.trim();
2278
- if !trimmed_name.is_empty() && trimmed_name != current_name {
2279
- memory.set_profile("name", trimmed_name)?;
2280
- }
2408
+ if let Ok(value) = serde_json::from_str::<serde_json::Value>(&clean_json)
2409
+ && let Some(obj) = value.as_object()
2410
+ {
2411
+ if let Some(new_name) = obj.get("name").and_then(|v| v.as_str()) {
2412
+ let trimmed_name = new_name.trim();
2413
+ if !trimmed_name.is_empty() && trimmed_name != current_name {
2414
+ memory.set_profile("name", trimmed_name)?;
2281
2415
  }
2282
- if let Some(new_pref) = obj.get("preferences").and_then(|v| v.as_str()) {
2283
- let trimmed_pref = new_pref.trim();
2284
- if !trimmed_pref.is_empty() && trimmed_pref != current_pref {
2285
- memory.set_profile("preferences", trimmed_pref)?;
2286
- }
2416
+ }
2417
+ if let Some(new_pref) = obj.get("preferences").and_then(|v| v.as_str()) {
2418
+ let trimmed_pref = new_pref.trim();
2419
+ if !trimmed_pref.is_empty() && trimmed_pref != current_pref {
2420
+ memory.set_profile("preferences", trimmed_pref)?;
2287
2421
  }
2288
2422
  }
2289
2423
  }
@@ -2294,6 +2428,7 @@ Example response:
2294
2428
  #[cfg(test)]
2295
2429
  mod tests {
2296
2430
  use super::*;
2431
+ use crate::config::AgentConfig;
2297
2432
 
2298
2433
  #[test]
2299
2434
  fn preserves_request_without_history() {
@@ -2306,13 +2441,17 @@ mod tests {
2306
2441
  chat_id: None,
2307
2442
  image_data_uri: None,
2308
2443
  audio_data_uri: None,
2444
+ video_data_uri: None,
2309
2445
  document_attachment: None,
2310
2446
  workspace_path: None,
2311
2447
  agent_id: None,
2312
2448
  };
2313
- assert_eq!(
2314
- enrich_request(&store, &request).unwrap().system_instruction,
2315
- "system"
2449
+ let config = MintConfig::default();
2450
+ assert!(
2451
+ enrich_request(&config, &store, &request)
2452
+ .unwrap()
2453
+ .system_instruction
2454
+ .starts_with("system")
2316
2455
  );
2317
2456
  }
2318
2457
 
@@ -2418,6 +2557,7 @@ mod tests {
2418
2557
  #[test]
2419
2558
  fn test_resolve_agent_config() {
2420
2559
  let mut config = MintConfig::default();
2560
+ config.enable_agent_collaboration = true;
2421
2561
  config.agents = vec![
2422
2562
  AgentConfig {
2423
2563
  id: "planner".into(),
@@ -2447,12 +2587,13 @@ mod tests {
2447
2587
  assert_eq!(name, Some("Planner".to_string()));
2448
2588
  assert_eq!(model, Some("gpt-4o".to_string()));
2449
2589
 
2450
- let (cfg, instr, name, model) = resolve_agent_config(&config, None, &[]);
2590
+ let (cfg, instr, name, _model) = resolve_agent_config(&config, None, &[]);
2451
2591
  assert_eq!(cfg.ai_provider, "openai");
2452
2592
  assert_eq!(instr, "Planner instruction");
2453
2593
  assert_eq!(name, Some("Planner".to_string()));
2454
2594
 
2455
- let trajectory = vec!["Step 1:\n- Action: create_plan\n- Observation: all planned".to_string()];
2595
+ let trajectory =
2596
+ vec!["Step 1:\n- Action: create_plan\n- Observation: all planned".to_string()];
2456
2597
  let (cfg, instr, name, model) = resolve_agent_config(&config, None, &trajectory);
2457
2598
  assert_eq!(cfg.ai_provider, "gemini");
2458
2599
  assert_eq!(instr, "Coder instruction");