@pheem49/mint 1.8.2 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +55 -3
- package/Cargo.toml +2 -1
- package/Release_Note.md +297 -107
- package/bin/mint +0 -0
- package/crates/mint-cli/Cargo.toml +1 -0
- package/crates/mint-cli/src/actions.rs +233 -0
- package/crates/mint-cli/src/agent.rs +382 -127
- package/crates/mint-cli/src/gmail.rs +22 -2
- package/crates/mint-cli/src/image.rs +8 -8
- package/crates/mint-cli/src/interactive.rs +2888 -0
- package/crates/mint-cli/src/main.rs +391 -2506
- package/crates/mint-cli/src/markdown.rs +294 -0
- package/crates/mint-cli/src/onboard.rs +152 -109
- package/crates/mint-cli/src/setup.rs +143 -143
- package/crates/mint-cli/src/updater.rs +48 -6
- package/crates/mint-core/Cargo.toml +1 -0
- package/crates/mint-core/src/agent_loop.rs +1 -0
- package/crates/mint-core/src/api_server.rs +431 -65
- package/crates/mint-core/src/browser.rs +630 -102
- package/crates/mint-core/src/channels.rs +105 -6
- package/crates/mint-core/src/chat.rs +174 -11
- package/crates/mint-core/src/code_tools.rs +47 -31
- package/crates/mint-core/src/config.rs +302 -17
- package/crates/mint-core/src/image_gen.rs +299 -38
- package/crates/mint-core/src/lib.rs +10 -7
- package/crates/mint-core/src/mcp.rs +119 -6
- package/crates/mint-core/src/memory.rs +6 -1
- package/crates/mint-core/src/orchestration.rs +237 -96
- package/crates/mint-core/src/pictures.rs +67 -5
- package/crates/mint-core/src/semantic.rs +1 -1
- package/crates/mint-core/src/shell.rs +1 -1
- package/crates/mint-core/src/skills.rs +128 -25
- package/crates/mint-core/src/video_gen.rs +235 -0
- package/package.json +8 -2
- package/shared-modules/linux-audio/fluidsynth2-static.json +9 -3
- package/shared-modules/linux-audio/fluidsynth2.json +8 -2
- package/shared-modules/linux-audio/lilv.json +26 -19
- package/shared-modules/linux-audio/lv2.json +0 -1
- package/shared-modules/luajit/luajit.json +2 -2
- package/skills-lock.json +41 -0
- package/src/renderer/index-web.html +1 -1
- package/src/renderer/index.html +1 -1
- package/src/renderer/shared/components/AgentActivityTable.tsx +25 -6
- package/src/renderer/shared/constants/slashCommands.ts +27 -0
- package/src/renderer/shared/platform.ts +3 -0
- package/src/renderer/shared/types.ts +4 -0
- package/src/renderer/shared/utils/approval.tsx +86 -7
- package/src/renderer/shared/utils/fileIcons.ts +135 -0
- package/src/renderer/shared/utils/markdown.tsx +217 -88
- package/src/renderer/shared/utils/slashCommandProcessor.ts +239 -0
- package/src/renderer/src/components/ChatPanel.tsx +486 -69
- package/src/renderer/src/components/DashboardSidebar.tsx +11 -2
- package/src/renderer/src/components/ImageStudioPanel.tsx +87 -4
- package/src/renderer/src/components/MintDashboard.tsx +195 -7
- package/src/renderer/src/components/PicturesLibrary.tsx +156 -15
- package/src/renderer/src/components/Settings/AgentsTab.tsx +21 -2
- package/src/renderer/src/components/Settings/AutomationTab.tsx +316 -18
- package/src/renderer/src/components/Settings/GeneralTab.tsx +795 -413
- package/src/renderer/src/components/Settings/PluginsTab.tsx +304 -70
- package/src/renderer/src/components/Settings/ThemeTab.tsx +6 -1
- package/src/renderer/src/components/SettingsWindow.tsx +106 -3
- package/src/renderer/src/components/VeoStudioPanel.tsx +549 -0
- package/src/renderer/src/components/WorkspacePanel.tsx +7 -125
- package/src/renderer/src/css/image-studio.css +106 -0
- package/src/renderer/src/css/settings.css +158 -0
- package/src/renderer/src/css/styles.css +368 -1
- package/src/renderer/src/css/veo-studio.css +626 -0
- package/src/renderer/src/index.css +1 -0
- package/src/renderer/src/tauri.ts +144 -17
- package/src/renderer/src-web/components/ChatPanel.tsx +385 -93
- package/src/renderer/src-web/components/DashboardSidebar.tsx +16 -1
- package/src/renderer/src-web/components/ImageStudioPanel.tsx +87 -4
- package/src/renderer/src-web/components/MintDashboard.tsx +156 -5
- package/src/renderer/src-web/components/PicturesLibrary.tsx +155 -15
- package/src/renderer/src-web/components/Settings/AgentsTab.tsx +21 -2
- package/src/renderer/src-web/components/Settings/AutomationTab.tsx +316 -18
- package/src/renderer/src-web/components/Settings/GeneralTab.tsx +795 -411
- package/src/renderer/src-web/components/Settings/PluginsTab.tsx +304 -70
- package/src/renderer/src-web/components/Settings/ThemeTab.tsx +6 -1
- package/src/renderer/src-web/components/SettingsWindow.tsx +110 -3
- package/src/renderer/src-web/components/VeoStudioPanel.tsx +549 -0
- package/src/renderer/src-web/css/image-studio.css +106 -0
- package/src/renderer/src-web/css/settings.css +158 -0
- package/src/renderer/src-web/css/styles.css +369 -1
- package/src/renderer/src-web/css/veo-studio.css +650 -0
- package/src/renderer/src-web/index.css +1 -0
- package/src/renderer/src-web/tauri.ts +135 -16
- package/src-tauri/src/desktop.rs +18 -16
- package/src-tauri/src/discord_rpc.rs +1 -1
- package/src-tauri/src/lib.rs +155 -25
- package/src-tauri/src/webhooks.rs +45 -31
- package/src-tauri/src/workflows.rs +1 -0
- package/src-tauri/tauri.conf.json +1 -1
- package/temp_gridgeist/AGENTS.md +8 -0
- package/temp_gridgeist/CHANGELOG.md +130 -0
- package/temp_gridgeist/LICENSE +21 -0
- package/temp_gridgeist/README.md +323 -0
- package/temp_gridgeist/README.th.md +325 -0
- package/temp_gridgeist/skills/gridgeist/SKILL.md +88 -0
- package/temp_gridgeist/skills/gridgeist/agents/openai.yaml +9 -0
- package/temp_gridgeist/skills/gridgeist/assets/gridgeist-small.svg +10 -0
- package/temp_gridgeist/skills/gridgeist/assets/gridgeist.png +0 -0
- package/temp_gridgeist/skills/gridgeist/assets/gridgeist.svg +18 -0
- package/temp_gridgeist/skills/gridgeist/references/design-language.md +88 -0
- package/temp_gridgeist/skills/gridgeist/references/review-checklist.md +128 -0
- package/temp_gridgeist/skills/gridgeist/references/system-contract.md +109 -0
|
@@ -71,15 +71,14 @@ pub async fn resolve_github_links(message: &str, config: &MintConfig) -> String
|
|
|
71
71
|
.collect();
|
|
72
72
|
|
|
73
73
|
let repo_key = format!("{owner}/{repo_clean}");
|
|
74
|
-
if resolved_repos.insert(repo_key.clone())
|
|
75
|
-
|
|
74
|
+
if resolved_repos.insert(repo_key.clone())
|
|
75
|
+
&& let Ok(summary) =
|
|
76
76
|
crate::code_tools::fetch_github_repo_summary(owner, &repo_clean).await
|
|
77
|
-
|
|
78
|
-
|
|
77
|
+
{
|
|
78
|
+
resolved_msg.push_str(&format!(
|
|
79
79
|
"\n\n--- Auto-Resolved GitHub Metadata for {} ---\n{}\n--------------------------------------------",
|
|
80
80
|
repo_key, summary
|
|
81
81
|
));
|
|
82
|
-
}
|
|
83
82
|
}
|
|
84
83
|
}
|
|
85
84
|
}
|
|
@@ -210,18 +209,18 @@ fn enrich_request(
|
|
|
210
209
|
let mut enriched = request.clone();
|
|
211
210
|
|
|
212
211
|
let mut profile_instructions = String::new();
|
|
213
|
-
if let Ok(Some(name)) = memory.get_profile("name")
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
}
|
|
212
|
+
if let Ok(Some(name)) = memory.get_profile("name")
|
|
213
|
+
&& !name.trim().is_empty()
|
|
214
|
+
{
|
|
215
|
+
profile_instructions.push_str(&format!("User Name: {}\n", name.trim()));
|
|
217
216
|
}
|
|
218
|
-
if let Ok(Some(preferences)) = memory.get_profile("preferences")
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
)
|
|
224
|
-
|
|
217
|
+
if let Ok(Some(preferences)) = memory.get_profile("preferences")
|
|
218
|
+
&& !preferences.trim().is_empty()
|
|
219
|
+
{
|
|
220
|
+
profile_instructions.push_str(&format!(
|
|
221
|
+
"User Preferences & Profile:\n{}\n",
|
|
222
|
+
preferences.trim()
|
|
223
|
+
));
|
|
225
224
|
}
|
|
226
225
|
|
|
227
226
|
if !profile_instructions.is_empty() {
|
|
@@ -239,8 +238,8 @@ fn enrich_request(
|
|
|
239
238
|
"\n\n[Active Environment Context]\n\
|
|
240
239
|
You are running on: {}\n\
|
|
241
240
|
Using AI Model: {}\n",
|
|
242
|
-
|
|
243
|
-
|
|
241
|
+
config.ai_provider,
|
|
242
|
+
config.active_model()
|
|
244
243
|
);
|
|
245
244
|
enriched.system_instruction.push_str(&active_model_info);
|
|
246
245
|
|
|
@@ -303,6 +302,7 @@ pub fn build_system_prompt(config: &MintConfig) -> String {
|
|
|
303
302
|
"note_write",
|
|
304
303
|
"run_plugin",
|
|
305
304
|
"mcp_tool",
|
|
305
|
+
"mcp_list_tools",
|
|
306
306
|
"run_shell",
|
|
307
307
|
"verify",
|
|
308
308
|
"apply_patch",
|
|
@@ -314,6 +314,10 @@ pub fn build_system_prompt(config: &MintConfig) -> String {
|
|
|
314
314
|
allowed_actions.push("browser_click");
|
|
315
315
|
allowed_actions.push("browser_type");
|
|
316
316
|
allowed_actions.push("browser_read");
|
|
317
|
+
allowed_actions.push("browser_mouse_move");
|
|
318
|
+
allowed_actions.push("browser_mouse_click");
|
|
319
|
+
allowed_actions.push("browser_key_press");
|
|
320
|
+
allowed_actions.push("browser_screenshot");
|
|
317
321
|
}
|
|
318
322
|
|
|
319
323
|
allowed_actions.retain(|action| !config.disabled_tools.contains(&action.to_string()));
|
|
@@ -355,15 +359,27 @@ pub fn build_system_prompt(config: &MintConfig) -> String {
|
|
|
355
359
|
input_formats.push("- browser_open: {\"url\":\"https://example.com\"}");
|
|
356
360
|
}
|
|
357
361
|
if allowed_actions.contains(&"browser_click") {
|
|
358
|
-
input_formats.push("- browser_click: {\"selector\":\"button.submit-btn\"}");
|
|
362
|
+
input_formats.push("- browser_click: {\"selector\":\"button.submit-btn\"} (CSS selector, or text=Login, contains=Submit, xpath=//button)");
|
|
359
363
|
}
|
|
360
364
|
if allowed_actions.contains(&"browser_type") {
|
|
361
365
|
input_formats
|
|
362
|
-
.push("- browser_type: {\"selector\":\"input.search-bar\", \"text\":\"search query\"}");
|
|
366
|
+
.push("- browser_type: {\"selector\":\"input.search-bar\", \"text\":\"search query\"} (CSS selector, or text=Placeholder, contains=Label)");
|
|
363
367
|
}
|
|
364
368
|
if allowed_actions.contains(&"browser_read") {
|
|
365
369
|
input_formats.push("- browser_read: {}");
|
|
366
370
|
}
|
|
371
|
+
if allowed_actions.contains(&"browser_mouse_move") {
|
|
372
|
+
input_formats.push("- browser_mouse_move: {\"x\":640,\"y\":360}");
|
|
373
|
+
}
|
|
374
|
+
if allowed_actions.contains(&"browser_mouse_click") {
|
|
375
|
+
input_formats.push("- browser_mouse_click: {\"x\":640,\"y\":360,\"button\":\"left\"}");
|
|
376
|
+
}
|
|
377
|
+
if allowed_actions.contains(&"browser_key_press") {
|
|
378
|
+
input_formats.push("- browser_key_press: {\"key\":\"Enter\"} (keys: Enter,Tab,Escape,Backspace,Delete,ArrowUp,ArrowDown,ArrowLeft,ArrowRight,F1-F12,Space)");
|
|
379
|
+
}
|
|
380
|
+
if allowed_actions.contains(&"browser_screenshot") {
|
|
381
|
+
input_formats.push("- browser_screenshot: {}");
|
|
382
|
+
}
|
|
367
383
|
if allowed_actions.contains(&"memory_recall") {
|
|
368
384
|
input_formats.push("- memory_recall: {\"query\":\"what did user say about X\"}");
|
|
369
385
|
}
|
|
@@ -411,8 +427,19 @@ pub fn build_system_prompt(config: &MintConfig) -> String {
|
|
|
411
427
|
if allowed_actions.contains(&"run_plugin") {
|
|
412
428
|
input_formats.push("- run_plugin: {\"name\":\"gmail|google_calendar|notion|docker|spotify|obsidian|system_metrics\",\"instruction\":\"instruction string\"}");
|
|
413
429
|
}
|
|
430
|
+
let mcp_str;
|
|
414
431
|
if allowed_actions.contains(&"mcp_tool") {
|
|
415
|
-
|
|
432
|
+
let servers = crate::mcp::list_mcp_servers()
|
|
433
|
+
.map(|m| m.keys().cloned().collect::<Vec<String>>().join(", "))
|
|
434
|
+
.unwrap_or_default();
|
|
435
|
+
mcp_str = format!(
|
|
436
|
+
"- mcp_tool: {{\"server\":\"<name>\",\"tool\":\"tool-name\",\"arguments\":{{}}}} (Available configured servers: {})",
|
|
437
|
+
servers
|
|
438
|
+
);
|
|
439
|
+
input_formats.push(&mcp_str);
|
|
440
|
+
}
|
|
441
|
+
if allowed_actions.contains(&"mcp_list_tools") {
|
|
442
|
+
input_formats.push("- mcp_list_tools: {\"server\":\"configured-server\"}");
|
|
416
443
|
}
|
|
417
444
|
if allowed_actions.contains(&"run_shell") {
|
|
418
445
|
input_formats.push(
|
|
@@ -448,7 +475,7 @@ pub fn build_system_prompt(config: &MintConfig) -> String {
|
|
|
448
475
|
);
|
|
449
476
|
}
|
|
450
477
|
if allowed_actions.contains(&"apply_patch") && allowed_actions.contains(&"write_file") {
|
|
451
|
-
rules.push("3. Use apply_patch for all edits to existing files
|
|
478
|
+
rules.push("3. Use apply_patch for all edits to existing files (write_file is only for creating new files). For edits across multiple sections of a file, supply separate small hunks in the 'hunks' array (one hunk per edit location). Keep oldText in each hunk minimal (only 1-3 exact lines needed to pinpoint the edit). NEVER bundle multiple separate edits into a single giant hunk that spans the entire file.");
|
|
452
479
|
}
|
|
453
480
|
if allowed_actions.contains(&"run_shell")
|
|
454
481
|
|| allowed_actions.contains(&"write_file")
|
|
@@ -469,16 +496,28 @@ pub fn build_system_prompt(config: &MintConfig) -> String {
|
|
|
469
496
|
rules.push("7a. Use browser_open to navigate the virtual browser to a URL.");
|
|
470
497
|
}
|
|
471
498
|
if allowed_actions.contains(&"browser_click") {
|
|
472
|
-
rules.push("7b. Use browser_click to click elements
|
|
499
|
+
rules.push("7b. Use browser_click to click elements. Selector can be: CSS selector (button.class, #id, [attr=val]), text=ExactText to find by visible text, contains=PartialText for partial match, or xpath=//expr for XPath. Prefer text= or contains= when the element has visible text but no unique CSS class.");
|
|
473
500
|
}
|
|
474
501
|
if allowed_actions.contains(&"browser_type") {
|
|
475
|
-
rules.push("7c. Use browser_type to enter text
|
|
502
|
+
rules.push("7c. Use browser_type to enter text into form fields or search boxes. Selector supports the same formats as browser_click.");
|
|
476
503
|
}
|
|
477
504
|
if allowed_actions.contains(&"browser_read") {
|
|
478
505
|
rules.push(
|
|
479
506
|
"7d. Use browser_read to read the text content of the active page to analyze it.",
|
|
480
507
|
);
|
|
481
508
|
}
|
|
509
|
+
if allowed_actions.contains(&"browser_mouse_move") {
|
|
510
|
+
rules.push("7e. Use browser_mouse_move to move the real mouse cursor to absolute (x,y) coordinates.");
|
|
511
|
+
}
|
|
512
|
+
if allowed_actions.contains(&"browser_mouse_click") {
|
|
513
|
+
rules.push("7f. Use browser_mouse_click to perform a native mouse click at (x,y) coordinates. Use browser_screenshot first to find the target position.");
|
|
514
|
+
}
|
|
515
|
+
if allowed_actions.contains(&"browser_key_press") {
|
|
516
|
+
rules.push("7g. Use browser_key_press to press a keyboard key (e.g. Enter, Tab, Escape, ArrowDown). Use after browser_type or browser_mouse_click to interact with focused elements.");
|
|
517
|
+
}
|
|
518
|
+
if allowed_actions.contains(&"browser_screenshot") {
|
|
519
|
+
rules.push("7h. Use browser_screenshot to capture the current page as a PNG image (base64). Use it to inspect the visual state of the page before deciding where to click.");
|
|
520
|
+
}
|
|
482
521
|
if allowed_actions.contains(&"memory_recall") {
|
|
483
522
|
rules.push("8. Use memory_recall to search past interactions before asking the user to repeat context.");
|
|
484
523
|
}
|
|
@@ -659,6 +698,14 @@ struct AgentInput {
|
|
|
659
698
|
selector: String,
|
|
660
699
|
#[serde(default)]
|
|
661
700
|
text: String,
|
|
701
|
+
#[serde(default)]
|
|
702
|
+
x: Option<f64>,
|
|
703
|
+
#[serde(default)]
|
|
704
|
+
y: Option<f64>,
|
|
705
|
+
#[serde(default)]
|
|
706
|
+
button: String,
|
|
707
|
+
#[serde(default)]
|
|
708
|
+
key: String,
|
|
662
709
|
}
|
|
663
710
|
|
|
664
711
|
#[derive(Debug, Deserialize, Serialize)]
|
|
@@ -685,7 +732,8 @@ fn resolve_agent_config(
|
|
|
685
732
|
return (config.clone(), "".to_string(), None, None);
|
|
686
733
|
}
|
|
687
734
|
|
|
688
|
-
let enabled_agents: Vec<&crate::config::AgentConfig> =
|
|
735
|
+
let enabled_agents: Vec<&crate::config::AgentConfig> =
|
|
736
|
+
config.agents.iter().filter(|a| a.enabled).collect();
|
|
689
737
|
if enabled_agents.is_empty() {
|
|
690
738
|
return (config.clone(), "".to_string(), None, None);
|
|
691
739
|
}
|
|
@@ -694,7 +742,9 @@ fn resolve_agent_config(
|
|
|
694
742
|
enabled_agents.iter().find(|a| a.id == id).copied()
|
|
695
743
|
} else {
|
|
696
744
|
// Multi-Agent Pipeline Collaboration (Planner -> Coder -> Reviewer)
|
|
697
|
-
let plan_created = trajectory
|
|
745
|
+
let plan_created = trajectory
|
|
746
|
+
.iter()
|
|
747
|
+
.any(|step| step.contains("- Action: create_plan"));
|
|
698
748
|
if !plan_created {
|
|
699
749
|
if let Some(planner) = enabled_agents.iter().find(|a| a.id == "planner") {
|
|
700
750
|
Some(*planner)
|
|
@@ -708,7 +758,9 @@ fn resolve_agent_config(
|
|
|
708
758
|
});
|
|
709
759
|
if edits_made {
|
|
710
760
|
if let Some(last_step) = trajectory.last() {
|
|
711
|
-
if last_step.contains("- Action: write_file")
|
|
761
|
+
if last_step.contains("- Action: write_file")
|
|
762
|
+
|| last_step.contains("- Action: apply_patch")
|
|
763
|
+
{
|
|
712
764
|
if let Some(reviewer) = enabled_agents.iter().find(|a| a.id == "reviewer") {
|
|
713
765
|
Some(*reviewer)
|
|
714
766
|
} else {
|
|
@@ -741,21 +793,26 @@ fn resolve_agent_config(
|
|
|
741
793
|
cfg_clone.local_model_name = agent.model.clone();
|
|
742
794
|
cfg_clone.ollama_model = agent.model.clone();
|
|
743
795
|
|
|
744
|
-
if let Some(key) = &agent.api_key
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
}
|
|
796
|
+
if let Some(key) = &agent.api_key
|
|
797
|
+
&& !key.trim().is_empty()
|
|
798
|
+
{
|
|
799
|
+
match agent.provider.as_str() {
|
|
800
|
+
"gemini" => cfg_clone.api_key = key.clone(),
|
|
801
|
+
"openai" => cfg_clone.openai_api_key = key.clone(),
|
|
802
|
+
"anthropic" => cfg_clone.anthropic_api_key = key.clone(),
|
|
803
|
+
"openrouter" => cfg_clone.openrouter_api_key = key.clone(),
|
|
804
|
+
"deepseek" => cfg_clone.deepseek_api_key = key.clone(),
|
|
805
|
+
"huggingface" => cfg_clone.hf_api_key = key.clone(),
|
|
806
|
+
_ => {}
|
|
755
807
|
}
|
|
756
808
|
}
|
|
757
809
|
|
|
758
|
-
(
|
|
810
|
+
(
|
|
811
|
+
cfg_clone,
|
|
812
|
+
agent.system_instruction.clone(),
|
|
813
|
+
Some(agent.name.clone()),
|
|
814
|
+
Some(agent.model.clone()),
|
|
815
|
+
)
|
|
759
816
|
}
|
|
760
817
|
|
|
761
818
|
pub async fn orchestrate_agent_loop<Approve, Progress, Chunk>(
|
|
@@ -763,6 +820,7 @@ pub async fn orchestrate_agent_loop<Approve, Progress, Chunk>(
|
|
|
763
820
|
task: &str,
|
|
764
821
|
root: &Path,
|
|
765
822
|
image_data_uri: Option<String>,
|
|
823
|
+
video_data_uri: Option<String>,
|
|
766
824
|
chat_id: Option<&str>,
|
|
767
825
|
agent_id: Option<&str>,
|
|
768
826
|
fast_mode: bool,
|
|
@@ -784,30 +842,32 @@ where
|
|
|
784
842
|
))
|
|
785
843
|
})?;
|
|
786
844
|
let resolved_task = resolve_github_links(task, config).await;
|
|
787
|
-
let skills = crate::skills::learned_skills_context(Some(&root)).unwrap_or_default();
|
|
788
|
-
let mut observation = initial_observation(&resolved_task, &root, &skills);
|
|
789
|
-
let mut pending_image = image_data_uri;
|
|
790
|
-
|
|
791
|
-
let mut system_prompt = build_system_prompt(config);
|
|
792
845
|
let chat_id = chat_id
|
|
793
846
|
.map(str::trim)
|
|
794
847
|
.filter(|chat_id| !chat_id.is_empty())
|
|
795
848
|
.unwrap_or(DEFAULT_CONVERSATION_ID);
|
|
849
|
+
let skills =
|
|
850
|
+
crate::skills::learned_skills_context(Some(&root), Some(chat_id)).unwrap_or_default();
|
|
851
|
+
let mut observation = initial_observation(&resolved_task, &root, &skills);
|
|
852
|
+
let mut pending_image = image_data_uri;
|
|
853
|
+
let mut pending_video = video_data_uri;
|
|
854
|
+
|
|
855
|
+
let mut system_prompt = build_system_prompt(config);
|
|
796
856
|
|
|
797
857
|
if let Ok(memory) = MemoryStore::open_default() {
|
|
798
858
|
let mut profile_instructions = String::new();
|
|
799
|
-
if let Ok(Some(name)) = memory.get_profile("name")
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
}
|
|
859
|
+
if let Ok(Some(name)) = memory.get_profile("name")
|
|
860
|
+
&& !name.trim().is_empty()
|
|
861
|
+
{
|
|
862
|
+
profile_instructions.push_str(&format!("User Name: {}\n", name.trim()));
|
|
803
863
|
}
|
|
804
|
-
if let Ok(Some(preferences)) = memory.get_profile("preferences")
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
)
|
|
810
|
-
|
|
864
|
+
if let Ok(Some(preferences)) = memory.get_profile("preferences")
|
|
865
|
+
&& !preferences.trim().is_empty()
|
|
866
|
+
{
|
|
867
|
+
profile_instructions.push_str(&format!(
|
|
868
|
+
"User Preferences & Profile:\n{}\n",
|
|
869
|
+
preferences.trim()
|
|
870
|
+
));
|
|
811
871
|
}
|
|
812
872
|
|
|
813
873
|
if !profile_instructions.is_empty() {
|
|
@@ -844,7 +904,8 @@ where
|
|
|
844
904
|
let mut trajectory: Vec<String> = Vec::new();
|
|
845
905
|
|
|
846
906
|
for step in 1..=MAX_STEPS {
|
|
847
|
-
let (active_config, agent_instruction, active_agent_name, active_model_name) =
|
|
907
|
+
let (active_config, agent_instruction, active_agent_name, active_model_name) =
|
|
908
|
+
resolve_agent_config(config, agent_id, &trajectory);
|
|
848
909
|
|
|
849
910
|
progress(AgentProgress::Thinking {
|
|
850
911
|
elapsed_secs: started_at.elapsed().as_secs(),
|
|
@@ -858,12 +919,14 @@ where
|
|
|
858
919
|
"\n\n[Active Environment Context]\n\
|
|
859
920
|
You are running on: {}\n\
|
|
860
921
|
Using AI Model: {}\n",
|
|
861
|
-
|
|
862
|
-
model_name
|
|
922
|
+
active_config.ai_provider, model_name
|
|
863
923
|
));
|
|
864
924
|
}
|
|
865
925
|
if !agent_instruction.is_empty() {
|
|
866
|
-
active_system_prompt.push_str(&format!(
|
|
926
|
+
active_system_prompt.push_str(&format!(
|
|
927
|
+
"\n\nYour Current Role & System Instructions:\n{}",
|
|
928
|
+
agent_instruction
|
|
929
|
+
));
|
|
867
930
|
}
|
|
868
931
|
|
|
869
932
|
let (response, fallback) = send_chat_with_fallback(
|
|
@@ -874,6 +937,7 @@ where
|
|
|
874
937
|
chat_id: Some(chat_id.to_owned()),
|
|
875
938
|
image_data_uri: pending_image.take(),
|
|
876
939
|
audio_data_uri: None,
|
|
940
|
+
video_data_uri: pending_video.take(),
|
|
877
941
|
document_attachment: None,
|
|
878
942
|
workspace_path: None,
|
|
879
943
|
agent_id: None,
|
|
@@ -903,6 +967,7 @@ where
|
|
|
903
967
|
chat_id: Some(chat_id.to_owned()),
|
|
904
968
|
image_data_uri: None,
|
|
905
969
|
audio_data_uri: None,
|
|
970
|
+
video_data_uri: None,
|
|
906
971
|
document_attachment: None,
|
|
907
972
|
workspace_path: None,
|
|
908
973
|
agent_id: None,
|
|
@@ -1261,14 +1326,14 @@ where
|
|
|
1261
1326
|
if crate::is_browser_running(config).await {
|
|
1262
1327
|
let result = crate::browser::navigate(config, url)
|
|
1263
1328
|
.await
|
|
1264
|
-
.map_err(
|
|
1329
|
+
.map_err(OrchestrationError::Agent)?;
|
|
1265
1330
|
Ok(result)
|
|
1266
1331
|
} else {
|
|
1267
1332
|
let opened = if cfg!(target_os = "macos") {
|
|
1268
1333
|
std::process::Command::new("open").arg(url).spawn().is_ok()
|
|
1269
1334
|
} else if cfg!(target_os = "windows") {
|
|
1270
1335
|
std::process::Command::new("cmd")
|
|
1271
|
-
.args(
|
|
1336
|
+
.args(["/C", "start", url])
|
|
1272
1337
|
.spawn()
|
|
1273
1338
|
.is_ok()
|
|
1274
1339
|
} else {
|
|
@@ -1300,7 +1365,7 @@ where
|
|
|
1300
1365
|
};
|
|
1301
1366
|
let result = crate::browser::click(config, selector)
|
|
1302
1367
|
.await
|
|
1303
|
-
.map_err(
|
|
1368
|
+
.map_err(OrchestrationError::Agent)?;
|
|
1304
1369
|
Ok(result)
|
|
1305
1370
|
}
|
|
1306
1371
|
"browser_type" => {
|
|
@@ -1324,15 +1389,63 @@ where
|
|
|
1324
1389
|
};
|
|
1325
1390
|
let result = crate::browser::type_text(config, selector, text)
|
|
1326
1391
|
.await
|
|
1327
|
-
.map_err(
|
|
1392
|
+
.map_err(OrchestrationError::Agent)?;
|
|
1328
1393
|
Ok(result)
|
|
1329
1394
|
}
|
|
1330
1395
|
"browser_read" => {
|
|
1331
1396
|
let result = crate::browser::read_page_text(config)
|
|
1332
1397
|
.await
|
|
1333
|
-
.map_err(
|
|
1398
|
+
.map_err(OrchestrationError::Agent)?;
|
|
1334
1399
|
Ok(result)
|
|
1335
1400
|
}
|
|
1401
|
+
"browser_mouse_move" => {
|
|
1402
|
+
let x = input.x.ok_or_else(|| {
|
|
1403
|
+
OrchestrationError::Agent("browser_mouse_move requires 'x'".into())
|
|
1404
|
+
})?;
|
|
1405
|
+
let y = input.y.ok_or_else(|| {
|
|
1406
|
+
OrchestrationError::Agent("browser_mouse_move requires 'y'".into())
|
|
1407
|
+
})?;
|
|
1408
|
+
let result = crate::browser::mouse_move(config, x, y)
|
|
1409
|
+
.await
|
|
1410
|
+
.map_err(OrchestrationError::Agent)?;
|
|
1411
|
+
Ok(result)
|
|
1412
|
+
}
|
|
1413
|
+
"browser_mouse_click" => {
|
|
1414
|
+
let x = input.x.ok_or_else(|| {
|
|
1415
|
+
OrchestrationError::Agent("browser_mouse_click requires 'x'".into())
|
|
1416
|
+
})?;
|
|
1417
|
+
let y = input.y.ok_or_else(|| {
|
|
1418
|
+
OrchestrationError::Agent("browser_mouse_click requires 'y'".into())
|
|
1419
|
+
})?;
|
|
1420
|
+
let button = if input.button.is_empty() {
|
|
1421
|
+
"left"
|
|
1422
|
+
} else {
|
|
1423
|
+
&input.button
|
|
1424
|
+
};
|
|
1425
|
+
let result = crate::browser::mouse_click(config, x, y, button)
|
|
1426
|
+
.await
|
|
1427
|
+
.map_err(OrchestrationError::Agent)?;
|
|
1428
|
+
Ok(result)
|
|
1429
|
+
}
|
|
1430
|
+
"browser_key_press" => {
|
|
1431
|
+
let key = if !input.key.is_empty() {
|
|
1432
|
+
&input.key
|
|
1433
|
+
} else {
|
|
1434
|
+
return Err(OrchestrationError::Agent(
|
|
1435
|
+
"browser_key_press requires 'key'".into(),
|
|
1436
|
+
));
|
|
1437
|
+
};
|
|
1438
|
+
let result = crate::browser::key_press(config, key)
|
|
1439
|
+
.await
|
|
1440
|
+
.map_err(OrchestrationError::Agent)?;
|
|
1441
|
+
Ok(result)
|
|
1442
|
+
}
|
|
1443
|
+
"browser_screenshot" => {
|
|
1444
|
+
let data = crate::browser::screenshot(config)
|
|
1445
|
+
.await
|
|
1446
|
+
.map_err(OrchestrationError::Agent)?;
|
|
1447
|
+
Ok(format!("data:image/png;base64,{data}"))
|
|
1448
|
+
}
|
|
1336
1449
|
"memory_recall" => {
|
|
1337
1450
|
let query = required(&input.query, "query")?;
|
|
1338
1451
|
let query_lower = query.to_ascii_lowercase();
|
|
@@ -1486,7 +1599,7 @@ where
|
|
|
1486
1599
|
path: file_name.to_owned(),
|
|
1487
1600
|
content: input.file_content.clone(),
|
|
1488
1601
|
})
|
|
1489
|
-
.map_err(
|
|
1602
|
+
.map_err(OrchestrationError::Agent)?;
|
|
1490
1603
|
|
|
1491
1604
|
match approved {
|
|
1492
1605
|
ApprovalOutcome::Approved => {
|
|
@@ -1509,7 +1622,7 @@ where
|
|
|
1509
1622
|
name: name.to_owned(),
|
|
1510
1623
|
instruction: instruction.to_owned(),
|
|
1511
1624
|
})
|
|
1512
|
-
.map_err(
|
|
1625
|
+
.map_err(OrchestrationError::Agent)?;
|
|
1513
1626
|
|
|
1514
1627
|
match approved {
|
|
1515
1628
|
ApprovalOutcome::Approved => Ok(execute_native_plugin(config, name, instruction)
|
|
@@ -1527,7 +1640,7 @@ where
|
|
|
1527
1640
|
tool: tool.to_owned(),
|
|
1528
1641
|
arguments: input.arguments.clone(),
|
|
1529
1642
|
})
|
|
1530
|
-
.map_err(
|
|
1643
|
+
.map_err(OrchestrationError::Agent)?;
|
|
1531
1644
|
|
|
1532
1645
|
match approved {
|
|
1533
1646
|
ApprovalOutcome::Approved => Ok(serde_json::to_string_pretty(
|
|
@@ -1541,6 +1654,25 @@ where
|
|
|
1541
1654
|
ApprovalOutcome::Intercepted(obs) => Ok(obs),
|
|
1542
1655
|
}
|
|
1543
1656
|
}
|
|
1657
|
+
"mcp_list_tools" => {
|
|
1658
|
+
let server = required(&input.server, "server")?;
|
|
1659
|
+
let approved = approve_cb(&AgentApproval::McpTool {
|
|
1660
|
+
server: server.to_owned(),
|
|
1661
|
+
tool: "list_tools".to_owned(),
|
|
1662
|
+
arguments: serde_json::json!({}),
|
|
1663
|
+
})
|
|
1664
|
+
.map_err(OrchestrationError::Agent)?;
|
|
1665
|
+
|
|
1666
|
+
match approved {
|
|
1667
|
+
ApprovalOutcome::Approved => Ok(serde_json::to_string_pretty(
|
|
1668
|
+
&crate::mcp::list_server_tools(config, server)
|
|
1669
|
+
.map_err(|e| OrchestrationError::Agent(e.to_string()))?,
|
|
1670
|
+
)
|
|
1671
|
+
.map_err(|e| OrchestrationError::Agent(e.to_string()))?),
|
|
1672
|
+
ApprovalOutcome::Denied => Ok(format!("User denied MCP list tools: {}", server)),
|
|
1673
|
+
ApprovalOutcome::Intercepted(obs) => Ok(obs),
|
|
1674
|
+
}
|
|
1675
|
+
}
|
|
1544
1676
|
"run_shell" => {
|
|
1545
1677
|
let command = required(&input.command, "command")?;
|
|
1546
1678
|
let mode = classify_shell_command(command).mode.as_str().to_owned();
|
|
@@ -1548,7 +1680,7 @@ where
|
|
|
1548
1680
|
command: command.to_owned(),
|
|
1549
1681
|
mode,
|
|
1550
1682
|
})
|
|
1551
|
-
.map_err(
|
|
1683
|
+
.map_err(OrchestrationError::Agent)?;
|
|
1552
1684
|
|
|
1553
1685
|
match approved {
|
|
1554
1686
|
ApprovalOutcome::Approved => run_shell(root, config, command),
|
|
@@ -1593,7 +1725,7 @@ where
|
|
|
1593
1725
|
hunks: patch.hunks.clone(),
|
|
1594
1726
|
diff,
|
|
1595
1727
|
})
|
|
1596
|
-
.map_err(
|
|
1728
|
+
.map_err(OrchestrationError::Agent)?;
|
|
1597
1729
|
|
|
1598
1730
|
match approved {
|
|
1599
1731
|
ApprovalOutcome::Approved => {
|
|
@@ -1629,7 +1761,7 @@ where
|
|
|
1629
1761
|
content: input.file_content.clone(),
|
|
1630
1762
|
diff,
|
|
1631
1763
|
})
|
|
1632
|
-
.map_err(
|
|
1764
|
+
.map_err(OrchestrationError::Agent)?;
|
|
1633
1765
|
|
|
1634
1766
|
match approved {
|
|
1635
1767
|
ApprovalOutcome::Approved => {
|
|
@@ -1823,13 +1955,13 @@ fn view_image(path: &Path, config: &MintConfig) -> Result<String, OrchestrationE
|
|
|
1823
1955
|
}
|
|
1824
1956
|
let bytes = std::fs::read(&path)
|
|
1825
1957
|
.map_err(|e| OrchestrationError::Agent(format!("cannot read image: {e}")))?;
|
|
1826
|
-
|
|
1958
|
+
serde_json::to_string_pretty(&serde_json::json!({
|
|
1827
1959
|
"path": path,
|
|
1828
1960
|
"bytes": bytes.len(),
|
|
1829
1961
|
"mime": mime,
|
|
1830
1962
|
"dataUri": format!("data:{mime};base64,{}", BASE64_STANDARD.encode(bytes)),
|
|
1831
1963
|
}))
|
|
1832
|
-
.map_err(|e| OrchestrationError::Agent(e.to_string()))
|
|
1964
|
+
.map_err(|e| OrchestrationError::Agent(e.to_string()))
|
|
1833
1965
|
}
|
|
1834
1966
|
|
|
1835
1967
|
fn run_shell(
|
|
@@ -1845,14 +1977,13 @@ fn run_shell(
|
|
|
1845
1977
|
|
|
1846
1978
|
let mut hint = "";
|
|
1847
1979
|
let cmd_lower = command.to_lowercase();
|
|
1848
|
-
if output.success
|
|
1849
|
-
|
|
1980
|
+
if output.success
|
|
1981
|
+
&& (cmd_lower.contains("open")
|
|
1850
1982
|
|| cmd_lower.contains("launch")
|
|
1851
1983
|
|| cmd_lower.contains("chrome")
|
|
1852
|
-
|| cmd_lower.contains("firefox")
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
}
|
|
1984
|
+
|| cmd_lower.contains("firefox"))
|
|
1985
|
+
{
|
|
1986
|
+
hint = "\nNote: Opening URLs, files, folders, or launching applications are background processes. Even if there are warnings or stdout/stderr outputs, since the command exited successfully with status 0, the operation has succeeded and you should now use the 'finish' action to inform the user.";
|
|
1856
1987
|
}
|
|
1857
1988
|
|
|
1858
1989
|
Ok(format!(
|
|
@@ -1897,7 +2028,9 @@ fn action_fingerprint(decision: &AgentDecision) -> String {
|
|
|
1897
2028
|
}
|
|
1898
2029
|
|
|
1899
2030
|
fn initial_observation(task: &str, root: &Path, skills: &str) -> String {
|
|
1900
|
-
let now = chrono::Local::now()
|
|
2031
|
+
let now = chrono::Local::now()
|
|
2032
|
+
.format("%Y-%m-%d %H:%M:%S %Z")
|
|
2033
|
+
.to_string();
|
|
1901
2034
|
let mut observation = format!(
|
|
1902
2035
|
"Current Time: {now}\nTask: {task}\nWorkspace: {}\nLearned skills:\n{}\n",
|
|
1903
2036
|
root.display(),
|
|
@@ -2247,6 +2380,7 @@ Example response:
|
|
|
2247
2380
|
chat_id: None,
|
|
2248
2381
|
image_data_uri: None,
|
|
2249
2382
|
audio_data_uri: None,
|
|
2383
|
+
video_data_uri: None,
|
|
2250
2384
|
document_attachment: None,
|
|
2251
2385
|
workspace_path: None,
|
|
2252
2386
|
agent_id: None,
|
|
@@ -2271,19 +2405,19 @@ Example response:
|
|
|
2271
2405
|
text_reply.to_string()
|
|
2272
2406
|
};
|
|
2273
2407
|
|
|
2274
|
-
if let Ok(value) = serde_json::from_str::<serde_json::Value>(&clean_json)
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
|
|
2279
|
-
|
|
2280
|
-
|
|
2408
|
+
if let Ok(value) = serde_json::from_str::<serde_json::Value>(&clean_json)
|
|
2409
|
+
&& let Some(obj) = value.as_object()
|
|
2410
|
+
{
|
|
2411
|
+
if let Some(new_name) = obj.get("name").and_then(|v| v.as_str()) {
|
|
2412
|
+
let trimmed_name = new_name.trim();
|
|
2413
|
+
if !trimmed_name.is_empty() && trimmed_name != current_name {
|
|
2414
|
+
memory.set_profile("name", trimmed_name)?;
|
|
2281
2415
|
}
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2416
|
+
}
|
|
2417
|
+
if let Some(new_pref) = obj.get("preferences").and_then(|v| v.as_str()) {
|
|
2418
|
+
let trimmed_pref = new_pref.trim();
|
|
2419
|
+
if !trimmed_pref.is_empty() && trimmed_pref != current_pref {
|
|
2420
|
+
memory.set_profile("preferences", trimmed_pref)?;
|
|
2287
2421
|
}
|
|
2288
2422
|
}
|
|
2289
2423
|
}
|
|
@@ -2294,6 +2428,7 @@ Example response:
|
|
|
2294
2428
|
#[cfg(test)]
|
|
2295
2429
|
mod tests {
|
|
2296
2430
|
use super::*;
|
|
2431
|
+
use crate::config::AgentConfig;
|
|
2297
2432
|
|
|
2298
2433
|
#[test]
|
|
2299
2434
|
fn preserves_request_without_history() {
|
|
@@ -2306,13 +2441,17 @@ mod tests {
|
|
|
2306
2441
|
chat_id: None,
|
|
2307
2442
|
image_data_uri: None,
|
|
2308
2443
|
audio_data_uri: None,
|
|
2444
|
+
video_data_uri: None,
|
|
2309
2445
|
document_attachment: None,
|
|
2310
2446
|
workspace_path: None,
|
|
2311
2447
|
agent_id: None,
|
|
2312
2448
|
};
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2449
|
+
let config = MintConfig::default();
|
|
2450
|
+
assert!(
|
|
2451
|
+
enrich_request(&config, &store, &request)
|
|
2452
|
+
.unwrap()
|
|
2453
|
+
.system_instruction
|
|
2454
|
+
.starts_with("system")
|
|
2316
2455
|
);
|
|
2317
2456
|
}
|
|
2318
2457
|
|
|
@@ -2418,6 +2557,7 @@ mod tests {
|
|
|
2418
2557
|
#[test]
|
|
2419
2558
|
fn test_resolve_agent_config() {
|
|
2420
2559
|
let mut config = MintConfig::default();
|
|
2560
|
+
config.enable_agent_collaboration = true;
|
|
2421
2561
|
config.agents = vec![
|
|
2422
2562
|
AgentConfig {
|
|
2423
2563
|
id: "planner".into(),
|
|
@@ -2447,12 +2587,13 @@ mod tests {
|
|
|
2447
2587
|
assert_eq!(name, Some("Planner".to_string()));
|
|
2448
2588
|
assert_eq!(model, Some("gpt-4o".to_string()));
|
|
2449
2589
|
|
|
2450
|
-
let (cfg, instr, name,
|
|
2590
|
+
let (cfg, instr, name, _model) = resolve_agent_config(&config, None, &[]);
|
|
2451
2591
|
assert_eq!(cfg.ai_provider, "openai");
|
|
2452
2592
|
assert_eq!(instr, "Planner instruction");
|
|
2453
2593
|
assert_eq!(name, Some("Planner".to_string()));
|
|
2454
2594
|
|
|
2455
|
-
let trajectory =
|
|
2595
|
+
let trajectory =
|
|
2596
|
+
vec!["Step 1:\n- Action: create_plan\n- Observation: all planned".to_string()];
|
|
2456
2597
|
let (cfg, instr, name, model) = resolve_agent_config(&config, None, &trajectory);
|
|
2457
2598
|
assert_eq!(cfg.ai_provider, "gemini");
|
|
2458
2599
|
assert_eq!(instr, "Coder instruction");
|