@pheem49/mint 1.15.0 → 1.17.0

package/src-tauri/src/background/headless.rs

@@ -0,0 +1,230 @@
+use serde::Deserialize;
+use std::{path::Path, time::Duration};
+
+use mint_core::{
+    CodeEdit, KnowledgeStore, MintConfig, Task, TaskStore, load_config, parse_agent_json,
+    propose_code_edits, run_agent_loop,
+};
+use serde_json::{Value, json};
+use tauri::{AppHandle, Emitter, Manager};
+
+use mint_core::browser::{click, navigate, read_page_text, type_text};
+
+const MAX_STEPS: usize = 20;
+const SYSTEM_PROMPT: &str = r#"You are Mint's native background task agent. Return only JSON:
+{"thought":"short progress note","action":"done|propose_folder|propose_write_file|open_url|browser_read|browser_click|browser_type|knowledge_search|propose_bash","target":"path, URL, selector, query, command, or final result","data":"optional file content"}
+Use only one action per response. Background tasks never mutate the filesystem or execute shell commands. Use propose_folder, propose_write_file, and propose_bash so Mint can record a proposal for explicit user approval."#;
+
+#[derive(Debug, Deserialize)]
+struct AgentAction {
+    #[serde(default)]
+    thought: String,
+    action: String,
+    #[serde(default)]
+    target: String,
+    #[serde(default)]
+    data: String,
+}
+
+pub fn start_headless_queue(app: AppHandle) {
+    tauri::async_runtime::spawn(async move {
+        if let Ok(store) = TaskStore::open_default() {
+            let _ = store.resume_running();
+        }
+        loop {
+            tokio::time::sleep(Duration::from_secs(15)).await;
+            let Ok(config) = load_config() else {
+                continue;
+            };
+            if config
+                .extra
+                .get("enableHeadlessTaskQueue")
+                .and_then(Value::as_bool)
+                != Some(true)
+            {
+                continue;
+            }
+            let _ = run_next_task(&app).await;
+        }
+    });
+}
+
+pub async fn run_next_task(app: &AppHandle) -> Result<Option<Task>, String> {
+    let store = TaskStore::open_default().map_err(|error| error.to_string())?;
+    let Some(task) = store.pending().map_err(|error| error.to_string())? else {
+        return Ok(None);
+    };
+    store
+        .update_status(&task.id, "running", None)
+        .map_err(|error| error.to_string())?;
+    checkpoint(&store, &task.id, "started", &task.description)?;
+    emit(
+        app,
+        &format!("Started queued task: {}", task.description),
+        "info",
+    );
+
+    match execute_task(&store, &task).await {
+        Ok(result) => {
+            store
+                .add_artifact(
+                    &task.id,
+                    json!({ "type": "final_result", "content": result }),
+                )
+                .map_err(|error| error.to_string())?;
+            let completed = store
+                .update_status(&task.id, "completed", Some(Value::String(result.clone())))
+                .map_err(|error| error.to_string())?;
+            emit(app, &format!("Queued task completed: {result}"), "info");
+            Ok(completed)
+        }
+        Err(error) => {
+            let failed = store
+                .fail_with_retry(&task.id, &error)
+                .map_err(|store_error| store_error.to_string())?;
+            emit(app, &format!("Queued task failed: {error}"), "warning");
+            Ok(failed)
+        }
+    }
+}
+
+async fn execute_task(store: &TaskStore, task: &Task) -> Result<String, String> {
+    let config = load_config().map_err(|error| error.to_string())?;
+    let observer_store = store.clone();
+    let observer_task = task.clone();
+    let executor_store = store.clone();
+    let executor_task = task.clone();
+    let executor_config = config.clone();
+    run_agent_loop(
+        &config,
+        SYSTEM_PROMPT,
+        format!("Task: {}\nChoose the first action.", task.description),
+        MAX_STEPS,
+        |raw| parse_agent_json(raw).map_err(|error| error.to_string()),
+        |action: &AgentAction| (action.action == "done").then(|| action.target.clone()),
+        move |step, action| {
+            let store = executor_store.clone();
+            let task = executor_task.clone();
+            let config = executor_config.clone();
+            Box::pin(async move {
+                let observation = execute_action(&config, &store, &task, step, &action).await?;
+                checkpoint(&store, &task.id, "observation", &observation)?;
+                Ok(observation)
+            })
+        },
+        move |step, action| {
+            observer_store
+                .add_checkpoint(
+                    &observer_task.id,
+                    json!({
+                        "phase": "native_agent_step",
+                        "step": step,
+                        "thought": action.thought,
+                        "action": action.action,
+                        "target": action.target,
+                    }),
+                )
+                .map(|_| ())
+                .map_err(|error| error.to_string())
+        },
+    )
+    .await
+    .map_err(|error| error.to_string())
+}
+
+async fn execute_action(
+    config: &MintConfig,
+    store: &TaskStore,
+    task: &Task,
+    step: usize,
+    action: &AgentAction,
+) -> Result<String, String> {
+    match action.action.as_str() {
+        "propose_folder" => {
+            store
+                .add_artifact(
+                    &task.id,
+                    json!({ "type": "folder_proposal", "path": action.target, "description": format!("Proposed by native task step {step}; explicit approval required") }),
+                )
+                .map_err(|error| error.to_string())?;
+            Ok(format!(
+                "folder proposal recorded but not applied: {}",
+                action.target
+            ))
+        }
+        "propose_write_file" => {
+            let root = std::env::current_dir().map_err(|error| error.to_string())?;
+            let proposal = propose_code_edits(
+                &root,
+                &[CodeEdit {
+                    path: Path::new(&action.target).to_path_buf(),
+                    content: action.data.clone(),
+                }],
+                config,
+            )
+            .map_err(|error| error.to_string())?;
+            store
+                .add_artifact(
+                    &task.id,
+                    json!({ "type": "file_edit_proposal", "proposal": proposal, "description": format!("Proposed by native task step {step}; explicit approval required") }),
+                )
+                .map_err(|error| error.to_string())?;
+            Ok(format!(
+                "file edit proposal recorded but not applied: {}",
+                action.target
+            ))
+        }
+        "open_url" => navigate(config, &action.target).await,
+        "browser_read" => read_page_text(config).await,
+        "browser_click" => click(config, &action.target).await,
+        "browser_type" => type_text(config, &action.target, &action.data).await,
+        "knowledge_search" => {
+            let hits = KnowledgeStore::open_default()
+                .map_err(|error| error.to_string())?
+                .search(&action.target, 5)
+                .map_err(|error| error.to_string())?;
+            serde_json::to_string(&hits).map_err(|error| error.to_string())
+        }
+        "propose_bash" => Ok(format!(
+            "command proposal recorded but not executed: {}",
+            action.target
+        )),
+        other => Err(format!("unsupported native background action '{other}'")),
+    }
+}
+
+fn checkpoint(store: &TaskStore, id: &str, phase: &str, message: &str) -> Result<(), String> {
+    store
+        .add_checkpoint(id, json!({ "phase": phase, "message": message }))
+        .map(|_| ())
+        .map_err(|error| error.to_string())
+}
+
+fn emit(app: &AppHandle, message: &str, kind: &str) {
+    if let Some(main) = app.get_webview_window("main") {
+        let _ = main.emit(
+            "proactive-notification",
+            json!({ "message": message, "type": kind }),
+        );
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parses_json_wrapped_in_provider_text() {
+        let action: AgentAction =
+            parse_agent_json("```json\n{\"action\":\"done\",\"target\":\"ok\"}\n```").unwrap();
+        assert_eq!(action.action, "done");
+        assert_eq!(action.target, "ok");
+    }
+}
+
+#[tauri::command]
+pub(crate) async fn run_next_queued_task(
+    app: AppHandle,
+) -> Result<Option<mint_core::Task>, String> {
+    run_next_task(&app).await
+}

package/src-tauri/src/background/mod.rs

@@ -0,0 +1,4 @@
+//! Desktop-owned background loops and their notification adapters.
+pub(crate) mod events;
+pub(crate) mod headless;
+pub(crate) mod proactive;

package/src-tauri/src/background/proactive.rs

@@ -0,0 +1,262 @@
+use crate::native::capture::capture_screen_bytes;
+use std::{
+    collections::BTreeMap,
+    fs,
+    path::PathBuf,
+    sync::{
+        LazyLock, Mutex,
+        atomic::{AtomicBool, Ordering},
+    },
+    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
+};
+
+use base64::{Engine as _, engine::general_purpose::STANDARD};
+use mint_core::{MintConfig, config_path, load_config};
+use serde::{Deserialize, Serialize};
+use serde_json::{Value, json};
+use tauri::{AppHandle, Emitter, Manager};
+
+const DEFAULT_INTERVAL_SECONDS: u64 = 60;
+const DEFAULT_COOLDOWN_SECONDS: u64 = 120;
+const MAX_CONTEXT_HISTORY: usize = 20;
+const PROMPT: &str = r#"You are Mint's desktop suggestion engine. Analyze the screenshot and return only JSON:
+{"context":"short English description","message":"short Thai message or null","suggestions":[{"label":"1-3 words","action":{"type":"open_url|open_app|search|none","target":"..."}}]}
+Return an empty suggestions array when there is no clear useful opportunity. Provide at most 4 suggestions. Never suggest shell commands."#;
+
+static ENABLED: AtomicBool = AtomicBool::new(false);
+static LAST_SUGGESTION: LazyLock<Mutex<Option<(Instant, String)>>> =
+    LazyLock::new(|| Mutex::new(None));
+
+#[derive(Debug, Default, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct BehaviorMemory {
+    #[serde(default)]
+    app_frequency: BTreeMap<String, u64>,
+    #[serde(default)]
+    context_history: Vec<BehaviorEntry>,
+    #[serde(default)]
+    last_updated: Option<String>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct BehaviorEntry {
+    context: String,
+    time: String,
+}
+
+pub fn set_enabled(enabled: bool) {
+    ENABLED.store(enabled, Ordering::Relaxed);
+}
+
+pub fn record_behavior(context: &str) -> Result<(), String> {
+    let context = context.trim();
+    if context.is_empty() {
+        return Ok(());
+    }
+    let mut memory = load_behavior()?;
+    memory.context_history.insert(
+        0,
+        BehaviorEntry {
+            context: context.into(),
+            time: timestamp(),
+        },
+    );
+    memory.context_history.truncate(MAX_CONTEXT_HISTORY);
+    for app in [
+        "YouTube", "Chrome", "Firefox", "VS Code", "Spotify", "Terminal", "Google", "Discord",
+        "Slack", "Gmail", "GitHub", "Figma", "Notion",
+    ] {
+        if context
+            .to_ascii_lowercase()
+            .contains(&app.to_ascii_lowercase())
+        {
+            *memory.app_frequency.entry(app.into()).or_default() += 1;
+        }
+    }
+    memory.last_updated = Some(timestamp());
+    save_behavior(&memory)
+}
+
+pub fn start_loop(app: AppHandle) {
+    tauri::async_runtime::spawn(async move {
+        loop {
+            if !ENABLED.load(Ordering::Relaxed) {
+                tokio::time::sleep(Duration::from_secs(2)).await;
+                continue;
+            }
+            let config = match load_config() {
+                Ok(config) => config,
+                Err(_) => {
+                    tokio::time::sleep(Duration::from_secs(DEFAULT_INTERVAL_SECONDS)).await;
+                    continue;
+                }
+            };
+            if let Ok(Some(suggestion)) = analyze(&config).await {
+                if let Some(context) = suggestion["context"].as_str() {
+                    let _ = record_behavior(context);
+                }
+                if let Some(main) = app.get_webview_window("main") {
+                    let _ = main.emit("proactive-suggestion", suggestion);
+                }
+            }
+            tokio::time::sleep(Duration::from_secs(config_u64(
+                &config,
+                "proactiveInterval",
+                DEFAULT_INTERVAL_SECONDS,
+            )))
+            .await;
+        }
+    });
+}
+
+async fn analyze(config: &MintConfig) -> Result<Option<Value>, String> {
+    let key = if config.api_key.trim().is_empty() {
+        std::env::var("GEMINI_API_KEY").unwrap_or_default()
+    } else {
+        config.api_key.clone()
+    };
+    if key.trim().is_empty() || cooling_down(config) {
+        return Ok(None);
+    }
+    let image = capture_screen_bytes()?;
+    let response: Value = mint_core::HTTP_CLIENT
+        .clone()
+        .post(format!(
+            "https://generativelanguage.googleapis.com/v1beta/models/{}:generateContent?key={key}",
+            config.gemini_model
+        ))
+        .json(&json!({
+            "systemInstruction": { "parts": [{ "text": PROMPT }] },
+            "contents": [{ "role": "user", "parts": [
+                { "text": format!("Behavior context: {}", behavior_summary()?) },
+                { "inlineData": { "mimeType": "image/png", "data": STANDARD.encode(image) } }
+            ]}],
+            "generationConfig": { "responseMimeType": "application/json" }
+        }))
+        .send()
+        .await
+        .map_err(|error| error.to_string())?
+        .error_for_status()
+        .map_err(|error| error.to_string())?
+        .json()
+        .await
+        .map_err(|error| error.to_string())?;
+    let Some(raw) = response["candidates"][0]["content"]["parts"][0]["text"].as_str() else {
+        return Ok(None);
+    };
+    let mut suggestion: Value = serde_json::from_str(raw).map_err(|error| error.to_string())?;
+    let has_message = suggestion["message"].as_str().is_some();
+    let Some(items) = suggestion["suggestions"].as_array_mut() else {
+        return Ok(None);
+    };
+    items.retain(|item| {
+        item["action"]["type"]
+            .as_str()
+            .is_some_and(|kind| matches!(kind, "open_url" | "open_app" | "search" | "none"))
+    });
+    items.truncate(4);
+    if !has_message || items.is_empty() {
+        return Ok(None);
+    }
+    let context = suggestion["context"]
+        .as_str()
+        .unwrap_or_default()
+        .to_owned();
+    let mut last = LAST_SUGGESTION.lock().map_err(|error| error.to_string())?;
+    if last
+        .as_ref()
+        .is_some_and(|(_, previous)| previous == &context)
+    {
+        return Ok(None);
+    }
+    *last = Some((Instant::now(), context));
+    Ok(Some(suggestion))
+}
+
+fn cooling_down(config: &MintConfig) -> bool {
+    LAST_SUGGESTION
+        .lock()
+        .ok()
+        .and_then(|value| value.as_ref().map(|(time, _)| time.elapsed()))
+        .is_some_and(|elapsed| {
+            elapsed
+                < Duration::from_secs(config_u64(
+                    config,
+                    "proactiveCooldown",
+                    DEFAULT_COOLDOWN_SECONDS,
+                ))
+        })
+}
+
+fn behavior_summary() -> Result<String, String> {
+    let memory = load_behavior()?;
+    let recent = memory
+        .context_history
+        .iter()
+        .take(3)
+        .map(|entry| entry.context.as_str())
+        .collect::<Vec<_>>()
+        .join(" | ");
+    let mut apps = memory.app_frequency.into_iter().collect::<Vec<_>>();
+    apps.sort_by(|left, right| right.1.cmp(&left.1));
+    let apps = apps
+        .into_iter()
+        .take(5)
+        .map(|(app, count)| format!("{app} ({count}x)"))
+        .collect::<Vec<_>>()
+        .join(", ");
+    Ok(format!(
+        "Frequent apps: {apps}. Recent activities: {recent}."
+    ))
+}
+
+fn behavior_path() -> Result<PathBuf, String> {
+    Ok(config_path()
+        .map_err(|error| error.to_string())?
+        .with_file_name("behavior_memory.json"))
+}
+
+fn load_behavior() -> Result<BehaviorMemory, String> {
+    let path = behavior_path()?;
+    if !path.exists() {
+        return Ok(BehaviorMemory::default());
+    }
+    serde_json::from_str(&fs::read_to_string(&path).map_err(|error| error.to_string())?)
+        .map_err(|error| error.to_string())
+}
+
+fn save_behavior(memory: &BehaviorMemory) -> Result<(), String> {
+    let path = behavior_path()?;
+    let directory = path
+        .parent()
+        .ok_or_else(|| "behavior memory directory is unavailable".to_string())?;
+    fs::create_dir_all(directory).map_err(|error| error.to_string())?;
+    let raw = serde_json::to_string_pretty(memory).map_err(|error| error.to_string())?;
+    fs::write(path, format!("{raw}\n")).map_err(|error| error.to_string())
+}
+
+fn config_u64(config: &MintConfig, key: &str, default: u64) -> u64 {
+    config
+        .extra
+        .get(key)
+        .and_then(Value::as_u64)
+        .unwrap_or(default)
+        .max(5)
+}
+
+fn timestamp() -> String {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_secs()
+        .to_string()
+}
+
+#[tauri::command]
+pub(crate) fn toggle_proactive(enabled: bool) {
+    set_enabled(enabled);
+}
+#[tauri::command]
+pub(crate) fn save_behavior_context(context: String) -> Result<(), String> {
+    record_behavior(&context)
+}

package/src-tauri/src/commands/approvals.rs

@@ -0,0 +1,20 @@
+//! Native event delivery for the shared approval lifecycle.
+use mint_core::{AgentApproval, ApprovalOutcome};
+use tauri::{AppHandle, Emitter, Manager};
+
+pub type ApprovalsState = mint_core::ApprovalRegistry;
+
+pub(crate) fn callback(
+    app: AppHandle,
+    chat_id: String,
+) -> impl FnMut(&AgentApproval) -> Result<ApprovalOutcome, String> + Send {
+    move |approval| {
+        app.state::<ApprovalsState>().request_blocking(|token| {
+            app.emit(
+                "tool-approval-requested",
+                serde_json::json!({ "token": token, "approval": approval, "chatId": chat_id }),
+            )
+            .map_err(|error| error.to_string())
+        })
+    }
+}

package/src-tauri/src/commands/auth.rs

@@ -0,0 +1,127 @@
+use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
+use mint_core::{
+    AuthUser, MemoryStore, config_path, get_user, load_config, login_user, register_user,
+    save_avatar_file, update_profile,
+};
+use serde_json::Value;
+use std::fs;
+use std::path::PathBuf;
+
+#[tauri::command]
+pub(crate) fn get_profile_value(key: String) -> Result<Option<String>, String> {
+    if let Ok(memory) = MemoryStore::open_default() {
+        if let Ok(Some(val)) = memory.get_profile(&key) {
+            if !val.trim().is_empty() {
+                return Ok(Some(val));
+            }
+        }
+    }
+    if let Ok(cfg) = load_config() {
+        if key == "veoModel" {
+            let m = mint_core::media::video_models::active_video_model_for_provider(&cfg, "veo");
+            if !m.is_empty() {
+                return Ok(Some(m.to_string()));
+            }
+        } else if key == "videoGenProvider" {
+            let p = cfg
+                .extra
+                .get("videoGenProvider")
+                .and_then(|v| v.as_str())
+                .unwrap_or("veo");
+            return Ok(Some(p.to_string()));
+        }
+    }
+    Ok(None)
+}
+
+#[tauri::command]
+pub(crate) fn set_profile_value(key: String, value: String) -> Result<(), String> {
+    MemoryStore::open_default()
+        .and_then(|memory| memory.set_profile(&key, &value))
+        .map_err(|error| error.to_string())
+}
+
+/// Path to the file that remembers which shared-store user is currently
+/// logged into the desktop app (`~/.config/mint/session.json`). Desktop is a
+/// single trusted process per launch, so a bearer token isn't needed here —
+/// unlike the web-mode API server (see mint_core::auth session tokens).
+fn desktop_session_path() -> Result<PathBuf, String> {
+    Ok(config_path()
+        .map_err(|error| error.to_string())?
+        .with_file_name("session.json"))
+}
+
+fn write_desktop_session(user_id: &str) -> Result<(), String> {
+    let path = desktop_session_path()?;
+    let contents = serde_json::json!({ "userId": user_id }).to_string();
+    fs::write(path, contents).map_err(|error| error.to_string())
+}
+
+fn read_desktop_session_user_id() -> Option<String> {
+    let path = desktop_session_path().ok()?;
+    let contents = fs::read_to_string(path).ok()?;
+    let value: Value = serde_json::from_str(&contents).ok()?;
+    value.get("userId")?.as_str().map(str::to_string)
+}
+
+#[tauri::command]
+pub(crate) fn auth_register(
+    name: Option<String>,
+    email: String,
+    password: String,
+) -> Result<AuthUser, String> {
+    let user = register_user(name, &email, &password).map_err(|error| error.to_string())?;
+    write_desktop_session(&user.id)?;
+    Ok(user)
+}
+
+#[tauri::command]
+pub(crate) fn auth_login(email: String, password: String) -> Result<AuthUser, String> {
+    let user = login_user(&email, &password).map_err(|error| error.to_string())?;
+    write_desktop_session(&user.id)?;
+    Ok(user)
+}
+
+#[tauri::command]
+pub(crate) fn auth_logout() -> Result<(), String> {
+    let path = desktop_session_path()?;
+    if path.exists() {
+        fs::remove_file(path).map_err(|error| error.to_string())?;
+    }
+    Ok(())
+}
+
+#[tauri::command]
+pub(crate) fn auth_current_user() -> Result<Option<AuthUser>, String> {
+    let Some(user_id) = read_desktop_session_user_id() else {
+        return Ok(None);
+    };
+    get_user(&user_id).map_err(|error| error.to_string())
+}
+
+#[tauri::command]
+pub(crate) fn auth_update_profile(
+    name: Option<String>,
+    image: Option<String>,
+) -> Result<AuthUser, String> {
+    let Some(user_id) = read_desktop_session_user_id() else {
+        return Err("Not logged in".to_string());
+    };
+    update_profile(&user_id, name, image).map_err(|error| error.to_string())
+}
+
+#[tauri::command]
+pub(crate) fn auth_upload_avatar(
+    file_name: String,
+    data_base64: String,
+) -> Result<AuthUser, String> {
+    let Some(user_id) = read_desktop_session_user_id() else {
+        return Err("Not logged in".to_string());
+    };
+    let bytes = BASE64
+        .decode(data_base64.as_bytes())
+        .map_err(|_| "Invalid image data".to_string())?;
+    let extension = file_name.rsplit('.').next().unwrap_or("png").to_lowercase();
+    let url = save_avatar_file(&bytes, &extension).map_err(|error| error.to_string())?;
+    update_profile(&user_id, None, Some(url)).map_err(|error| error.to_string())
+}