@pheem49/mint 1.15.0 → 1.17.0

package/crates/mint-core/src/agent/chat.rs

@@ -343,6 +343,43 @@ const MAX_DOCUMENT_CONTEXT_CHARS: usize = 400_000;
 static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
 
 impl ChatRequest {
+    pub(crate) fn with_stored_document_context(
+        &self,
+        _config: &crate::MintConfig,
+        path: &std::path::Path,
+    ) -> Result<ChatRequest, String> {
+        let Some(document) = &self.document_attachment else {
+            return Ok(self.clone());
+        };
+        let extracted =
+            crate::knowledge::extract_managed_document_text(path).map_err(|e| e.to_string())?;
+        self.with_document_text(document, extracted)
+    }
+    fn with_document_text(
+        &self,
+        document: &DocumentAttachment,
+        extracted: String,
+    ) -> Result<ChatRequest, String> {
+        let mut context: String = extracted.chars().take(MAX_DOCUMENT_CONTEXT_CHARS).collect();
+        if extracted.chars().count() > MAX_DOCUMENT_CONTEXT_CHARS {
+            context.push_str("\n\n[Document text truncated because it is long.]");
+        }
+
+        let filename = document.filename.trim();
+        let filename = if filename.is_empty() {
+            "attached document"
+        } else {
+            filename
+        };
+
+        let mut enriched = self.clone();
+        enriched.document_attachment = None;
+        enriched.message = format!(
+            "{}\n\nAttached document: {filename}\nUse the extracted document text below when answering. If the user asks for a summary, summarize this document.\n\n--- Extracted document text ---\n{context}\n--- End extracted document text ---",
+            self.message
+        );
+        Ok(enriched)
+    }
     pub fn with_document_context(&self, config: &crate::MintConfig) -> Result<ChatRequest, String> {
         let Some(document) = &self.document_attachment else {
             return Ok(self.clone());
@@ -404,25 +441,7 @@ impl ChatRequest {
         let _ = std::fs::remove_file(&path);
         let extracted = extracted?;
 
-        let mut context: String = extracted.chars().take(MAX_DOCUMENT_CONTEXT_CHARS).collect();
-        if extracted.chars().count() > MAX_DOCUMENT_CONTEXT_CHARS {
-            context.push_str("\n\n[Document text truncated because it is long.]");
-        }
-
-        let filename = document.filename.trim();
-        let filename = if filename.is_empty() {
-            "attached document"
-        } else {
-            filename
-        };
-
-        let mut enriched = self.clone();
-        enriched.document_attachment = None;
-        enriched.message = format!(
-            "{}\n\nAttached document: {filename}\nUse the extracted document text below when answering. If the user asks for a summary, summarize this document.\n\n--- Extracted document text ---\n{context}\n--- End extracted document text ---",
-            self.message
-        );
-        Ok(enriched)
+        self.with_document_text(document, extracted)
     }
 }
 
@@ -970,6 +989,112 @@ async fn send_openai_style_request(
     Ok(response.json().await?)
 }
 
+// Preserve malformed arguments so the executor can reject them. Replacing them
+// with {} would silently turn a requested directory into the workspace root.
+fn decode_tool_arguments(arguments: &Value) -> Value {
+    match arguments.as_str() {
+        Some(raw) => sanitize_and_parse_tool_arguments(raw).unwrap_or_else(|| arguments.clone()),
+        None => arguments.clone(),
+    }
+}
+
+/// Sanitizes LLM tool call argument string: strips markdown fences, DeepSeek DSML tokens,
+/// thinking tags, extracts inner JSON objects, and handles trailing commas before failing.
+fn sanitize_and_parse_tool_arguments(raw: &str) -> Option<Value> {
+    let trimmed = raw.trim();
+    if trimmed.is_empty() {
+        return None;
+    }
+    if let Ok(val) = serde_json::from_str::<Value>(trimmed) {
+        return Some(val);
+    }
+
+    // 1. Strip markdown code fences if wrapped in ```json ... ```
+    let without_fences = if let Some(stripped) = trimmed.strip_prefix("```json") {
+        stripped.strip_suffix("```").unwrap_or(stripped).trim()
+    } else if let Some(stripped) = trimmed.strip_prefix("```") {
+        stripped.strip_suffix("```").unwrap_or(stripped).trim()
+    } else {
+        trimmed
+    };
+    if let Ok(val) = serde_json::from_str::<Value>(without_fences) {
+        return Some(val);
+    }
+
+    // 2. Strip known special tokens (DeepSeek DSML markers, thinking tags, end of sentence markers)
+    let cleaned = without_fences
+        .replace("</｜｜DSML｜｜", "")
+        .replace("<｜｜DSML｜｜", "")
+        .replace("<｜end of sentence｜>", "")
+        .replace("</think>", "");
+
+    if let Ok(val) = serde_json::from_str::<Value>(&cleaned) {
+        return Some(val);
+    }
+
+    // 3. Extract JSON object boundary between first '{' and last '}'
+    if let (Some(start), Some(end)) = (cleaned.find('{'), cleaned.rfind('}')) {
+        if start < end {
+            let slice = &cleaned[start..=end];
+            if let Ok(val) = serde_json::from_str::<Value>(slice) {
+                return Some(val);
+            }
+            // 4. Strip trailing commas before closing braces/brackets
+            let without_trailing_commas = strip_json_trailing_commas(slice);
+            if let Ok(val) = serde_json::from_str::<Value>(&without_trailing_commas) {
+                return Some(val);
+            }
+        }
+    }
+
+    None
+}
+
+/// Simple non-destructive removal of trailing commas before closing braces/brackets
+fn strip_json_trailing_commas(input: &str) -> String {
+    let mut out = String::with_capacity(input.len());
+    let mut chars = input.chars().peekable();
+    let mut in_str = false;
+    let mut escape = false;
+
+    while let Some(ch) = chars.next() {
+        if in_str {
+            if escape {
+                escape = false;
+            } else if ch == '\\' {
+                escape = true;
+            } else if ch == '"' {
+                in_str = false;
+            }
+            out.push(ch);
+        } else {
+            if ch == '"' {
+                in_str = true;
+                out.push(ch);
+            } else if ch == ',' {
+                // Check if following non-whitespace character is '}' or ']'
+                let mut peek_chars = chars.clone();
+                let mut is_trailing = false;
+                while let Some(next_ch) = peek_chars.next() {
+                    if next_ch.is_whitespace() {
+                        continue;
+                    }
+                    if next_ch == '}' || next_ch == ']' {
+                        is_trailing = true;
+                    }
+                    break;
+                }
+                if !is_trailing {
+                    out.push(ch);
+                }
+            } else {
+                out.push(ch);
+            }
+        }
+    }
+    out
+}
+
 /// Parses an OpenAI Chat Completions-shaped response (`choices[0].message`),
 /// extracting both plain text and any native `tool_calls`. Shared by OpenAI,
 /// custom OpenAI-compatible providers, and (for its response shape only) reused
@@ -999,10 +1124,7 @@ fn parse_openai_style_reply(model: String, response: &Value) -> Result<ProviderR
                 .iter()
                 .filter_map(|call| {
                     let name = call["function"]["name"].as_str()?.to_string();
-                    let input = call["function"]["arguments"]
-                        .as_str()
-                        .and_then(|raw| serde_json::from_str(raw).ok())
-                        .unwrap_or_else(|| json!({}));
+                    let input = decode_tool_arguments(&call["function"]["arguments"]);
                     Some(ToolCall {
                         id: call["id"].as_str().unwrap_or_default().to_string(),
                         name,
@@ -1064,11 +1186,10 @@ async fn call_ollama(
     let native = request.messages.is_some()
         && config.tool_calling_mode() == crate::config::ToolCallingMode::Native;
     let mut body = if native {
-        let messages = request.messages.as_deref().unwrap_or(&[]);
         let mut payload = json!({
             "model": model,
             "stream": false,
-            "messages": ollama_native_messages(messages),
+            "messages": ollama_request_messages(request),
         });
         if let Some(tools) = &request.tools {
             payload["tools"] = json!(
@@ -1127,6 +1248,19 @@ async fn call_ollama(
 /// Unlike OpenAI, Ollama expects tool-call `arguments` as a JSON object (not a
 /// stringified JSON blob) and doesn't require a `tool_call_id` on tool-result
 /// messages.
+fn ollama_request_messages(request: &ChatRequest) -> Vec<Value> {
+    let history = request.messages.as_deref().unwrap_or(&[]);
+    let mut result = ollama_native_messages(history);
+    if !request.system_instruction.is_empty() && !history.iter().any(|m| m.role == ChatRole::System)
+    {
+        result.insert(
+            0,
+            json!({"role":"system","content":request.system_instruction}),
+        );
+    }
+    result
+}
+
 fn ollama_native_messages(messages: &[ChatMessage]) -> Vec<Value> {
     let mut result = Vec::new();
     for message in messages {
@@ -1228,11 +1362,7 @@ fn parse_ollama_reply(model: String, response: &Value) -> Result<ProviderReply,
                 .filter_map(|(index, call)| {
                     let name = call["function"]["name"].as_str()?.to_string();
                     let arguments = &call["function"]["arguments"];
-                    let input = if let Some(raw) = arguments.as_str() {
-                        serde_json::from_str(raw).unwrap_or_else(|_| json!({}))
-                    } else {
-                        arguments.clone()
-                    };
+                    let input = decode_tool_arguments(arguments);
                     let id = call["id"]
                         .as_str()
                         .map(str::to_owned)
@@ -1498,6 +1628,19 @@ fn anthropic_content_block(block: &ContentBlock) -> Result<Value, ChatError> {
 /// `tool_use` content blocks (Anthropic may return a `tool_use` block before,
 /// after, or interleaved with text — unlike the old code, this does not assume
 /// the first block is text).
+// Anthropic reports uncached, cache-read and cache-created input separately.
+fn anthropic_input_tokens(usage: &Value) -> Option<u32> {
+    [
+        "input_tokens",
+        "cache_read_input_tokens",
+        "cache_creation_input_tokens",
+    ]
+    .into_iter()
+    .filter_map(|key| usage[key].as_u64())
+    .reduce(u64::saturating_add)
+    .map(|n| n.min(u32::MAX as u64) as u32)
+}
+
 fn parse_anthropic_reply(model: String, response: &Value) -> Result<ProviderReply, ChatError> {
     let blocks = response["content"].as_array().cloned().unwrap_or_default();
     let mut text = String::new();
@@ -1544,7 +1687,7 @@ fn parse_anthropic_reply(model: String, response: &Value) -> Result<ProviderRepl
                 thought: Some(th.clone()),
                 tool_calls: None,
                 stop_reason: response["stop_reason"].as_str().map(str::to_owned),
-                input_tokens: response["usage"]["input_tokens"].as_u64().map(|n| n as u32),
+                input_tokens: anthropic_input_tokens(&response["usage"]),
                 output_tokens: response["usage"]["output_tokens"]
                     .as_u64()
                     .map(|n| n as u32),
@@ -1558,7 +1701,7 @@ fn parse_anthropic_reply(model: String, response: &Value) -> Result<ProviderRepl
         thought,
         tool_calls: (!tool_calls.is_empty()).then_some(tool_calls),
         stop_reason: response["stop_reason"].as_str().map(str::to_owned),
-        input_tokens: response["usage"]["input_tokens"].as_u64().map(|n| n as u32),
+        input_tokens: anthropic_input_tokens(&response["usage"]),
         output_tokens: response["usage"]["output_tokens"]
             .as_u64()
             .map(|n| n as u32),
@@ -1692,7 +1835,7 @@ fn ollama_stream_payload(
         let mut value = json!({
             "model": model,
             "stream": true,
-            "messages": ollama_native_messages(request.messages.as_deref().unwrap_or(&[])),
+            "messages": ollama_request_messages(request),
         });
         if let Some(tools) = &request.tools {
             value["tools"] = json!(
@@ -2137,9 +2280,7 @@ impl StreamReplyAccumulator {
     {
         match value["type"].as_str() {
             Some("message_start") => {
-                self.input_tokens = value["message"]["usage"]["input_tokens"]
-                    .as_u64()
-                    .map(|n| n as u32);
+                self.input_tokens = anthropic_input_tokens(&value["message"]["usage"]);
             }
             Some("content_block_start") => {
                 let block_index = value["index"].as_u64().unwrap_or(0) as usize;
@@ -2214,9 +2355,9 @@ impl StreamReplyAccumulator {
             .into_iter()
             .filter(|tool| !tool.name.is_empty())
             .map(|tool| {
-                let input = tool.input.unwrap_or_else(|| {
-                    serde_json::from_str(&tool.arguments).unwrap_or_else(|_| json!({}))
-                });
+                let input = tool
+                    .input
+                    .unwrap_or_else(|| decode_tool_arguments(&Value::String(tool.arguments)));
                 ToolCall {
                     id: tool.id,
                     name: tool.name,
@@ -2747,6 +2888,10 @@ fn openai_native_messages(messages: &[ChatMessage]) -> Result<Vec<Value>, ChatEr
                 };
                 if !tool_calls.is_empty() {
                     entry["tool_calls"] = json!(tool_calls);
+                    // DeepSeek and other reasoning-enabled OpenAI-compatible APIs enforce that
+                    // any assistant message containing `tool_calls` must include `reasoning_content`
+                    // when reasoning/thinking mode is active. Providing `""` prevents HTTP 400 rejection.
+                    entry["reasoning_content"] = json!("");
                 }
                 result.push(entry);
             }
@@ -2995,6 +3140,38 @@ mod tests {
         assert!(parse_stream_value(StreamFormat::Anthropic, "data: [DONE]").is_none());
     }
 
+    #[test]
+    fn malformed_tool_arguments_survive_provider_parsing_for_rejection() {
+        let malformed = r#"{"path":"~/Downloads/YOUTUBE_DL""#;
+        let function = json!({"name": "list_files", "arguments": malformed});
+        let openai = parse_openai_style_reply(
+            "test".into(),
+            &json!({
+                "choices": [{"message": {"tool_calls": [{"id": "call_1", "function": function}]}}]
+            }),
+        )
+        .unwrap();
+        let ollama = parse_ollama_reply(
+            "test".into(),
+            &json!({
+                "message": {"tool_calls": [{"function": function}]}
+            }),
+        )
+        .unwrap();
+        let mut accumulator = StreamReplyAccumulator::default();
+        accumulator.consume_openai(&json!({
+            "choices": [{"delta": {"tool_calls": [{"index": 0, "id": "call_1", "function": function}]}}]
+        }), &mut |_| {});
+        let streamed = accumulator.finish("test".into()).unwrap();
+        for reply in [openai, ollama, streamed] {
+            let calls = reply.tool_calls.unwrap();
+            assert_eq!(calls.len(), 1);
+            assert_eq!(calls[0].input, Value::String(malformed.into()));
+        }
+        assert_eq!(decode_tool_arguments(&json!("{}")), json!({}));
+        assert_eq!(decode_tool_arguments(&Value::Null), Value::Null);
+    }
+
     #[test]
     fn accumulates_openai_reasoning_tools_and_usage() {
         let mut accumulator = StreamReplyAccumulator::default();
@@ -3168,6 +3345,32 @@ mod tests {
         assert_eq!(payload["system"][0]["cache_control"]["type"], "ephemeral");
     }
 
+    #[test]
+    fn review_anthropic_tool_results_preserve_failure_and_success_flags() {
+        for (text, is_error) in [
+            ("User denied MCP tool call", true),
+            ("MCP call was not approved; user feedback", true),
+            ("Error: literal diagnostic label", false),
+        ] {
+            let request = native_request(
+                vec![ChatMessage {
+                    role: ChatRole::Tool,
+                    content: vec![ContentBlock::ToolResult {
+                        tool_use_id: "mcp-call".into(),
+                        content: text.into(),
+                        is_error,
+                    }],
+                }],
+                None,
+            );
+            let payload =
+                anthropic_chat_payload(&MintConfig::default(), "claude-x", &request, false)
+                    .unwrap();
+            assert_eq!(payload["messages"][0]["content"][0]["is_error"], is_error);
+            assert_eq!(payload["messages"][0]["content"][0]["content"], text);
+        }
+    }
+
     #[test]
     fn anthropic_payload_omits_the_cache_breakpoint_for_an_empty_system_prompt() {
         let mut request = native_request(vec![ChatMessage::text(ChatRole::User, "hi")], None);
@@ -3319,6 +3522,124 @@ mod tests {
         );
     }
 
+    #[test]
+    fn openai_native_messages_attaches_reasoning_content_for_tool_calls() {
+        let messages = vec![ChatMessage {
+            role: ChatRole::Assistant,
+            content: vec![ContentBlock::ToolUse {
+                id: "call_abc".into(),
+                name: "read_file".into(),
+                input: json!({"path": "src/main.rs"}),
+                thought_signature: None,
+            }],
+        }];
+        let built = openai_native_messages(&messages).unwrap();
+        assert_eq!(built.len(), 1);
+        assert_eq!(built[0]["role"], "assistant");
+        assert_eq!(built[0]["reasoning_content"], json!(""));
+        assert_eq!(built[0]["tool_calls"][0]["function"]["name"], "read_file");
+    }
+
+    #[test]
+    fn openai_serializes_compacted_history_with_role_alternation() {
+        let messages = vec![
+            ChatMessage::text(ChatRole::User, "Fix the bug"),
+            ChatMessage::text(
+                ChatRole::Assistant,
+                "Summary of earlier steps:\n- read file.rs\n- found syntax error",
+            ),
+            ChatMessage::text(
+                ChatRole::User,
+                "Please continue the task from where you left off based on the summary above.",
+            ),
+            ChatMessage {
+                role: ChatRole::Assistant,
+                content: vec![ContentBlock::ToolUse {
+                    id: "call_1".into(),
+                    name: "apply_patch".into(),
+                    input: json!({"path": "file.rs", "patch": "..."}),
+                    thought_signature: None,
+                }],
+            },
+            ChatMessage {
+                role: ChatRole::Tool,
+                content: vec![ContentBlock::ToolResult {
+                    tool_use_id: "call_1".into(),
+                    content: "Patch applied".into(),
+                    is_error: false,
+                }],
+            },
+        ];
+        let built = openai_native_messages(&messages).unwrap();
+        assert_eq!(built.len(), 5);
+        assert_eq!(built[0]["role"], "user");
+        assert_eq!(built[1]["role"], "assistant");
+        assert_eq!(built[2]["role"], "user");
+        assert_eq!(built[3]["role"], "assistant");
+        assert_eq!(built[4]["role"], "tool");
+    }
+
+    #[test]
+    fn anthropic_serializes_compacted_history_with_role_alternation() {
+        let messages = vec![
+            ChatMessage::text(ChatRole::User, "Fix the bug"),
+            ChatMessage::text(ChatRole::Assistant, "Summary of earlier steps"),
+            ChatMessage::text(ChatRole::User, "Please continue"),
+            ChatMessage {
+                role: ChatRole::Assistant,
+                content: vec![ContentBlock::ToolUse {
+                    id: "toolu_1".into(),
+                    name: "read_file".into(),
+                    input: json!({"path": "file.rs"}),
+                    thought_signature: None,
+                }],
+            },
+            ChatMessage {
+                role: ChatRole::Tool,
+                content: vec![ContentBlock::ToolResult {
+                    tool_use_id: "toolu_1".into(),
+                    content: "Code content".into(),
+                    is_error: false,
+                }],
+            },
+        ];
+        let built = anthropic_messages(&messages).unwrap();
+        assert_eq!(built.len(), 5);
+        assert_eq!(built[0]["role"], "user");
+        assert_eq!(built[1]["role"], "assistant");
+        assert_eq!(built[2]["role"], "user");
+        assert_eq!(built[3]["role"], "assistant");
+        assert_eq!(built[4]["role"], "user");
+    }
+
+    #[test]
+    fn gemini_serializes_compacted_history_with_role_alternation() {
+        let messages = vec![
+            ChatMessage::text(ChatRole::User, "Fix the bug"),
+            ChatMessage::text(ChatRole::Assistant, "Summary of earlier steps"),
+            ChatMessage::text(ChatRole::User, "Please continue"),
+            ChatMessage {
+                role: ChatRole::Assistant,
+                content: vec![ContentBlock::ToolUse {
+                    id: "call_1".into(),
+                    name: "read_file".into(),
+                    input: json!({"path": "file.rs"}),
+                    thought_signature: None,
+                }],
+            },
+            ChatMessage {
+                role: ChatRole::Tool,
+                content: vec![ContentBlock::ToolResult {
+                    tool_use_id: "call_1".into(),
+                    content: "Code content".into(),
+                    is_error: false,
+                }],
+            },
+        ];
+        let built = gemini_native_contents(&messages).unwrap();
+        assert!(built.len() >= 4);
+    }
+
     #[test]
     fn openai_allows_audio_attachments() {
         let request = ChatRequest {
@@ -3758,6 +4079,32 @@ mod tests {
             (Some(80), Some(40))
         );
     }
+    #[test]
+    fn anthropic_cached_prompt_tokens_count_toward_context_in_json_and_streams() {
+        let usage = json!({"input_tokens":100,"cache_read_input_tokens":98_000,"cache_creation_input_tokens":1_000,"output_tokens":400,"cache_creation":{"ephemeral_5m_input_tokens":1_000}});
+        let reply = parse_anthropic_reply(
+            "claude-test".into(),
+            &json!({"content":[{"type":"text","text":"Ready"}],"usage":usage}),
+        )
+        .unwrap();
+        assert_eq!(reply.input_tokens, Some(99_100));
+        let mut stream = StreamReplyAccumulator::default();
+        stream.consume_anthropic(
+            &json!({"type":"message_start","message":{"usage":usage}}),
+            &mut |_| {},
+        );
+        stream.consume_anthropic(
+            &json!({"type":"content_block_delta","delta":{"type":"text_delta","text":"Ready"}}),
+            &mut |_| {},
+        );
+        stream.consume_anthropic(
+            &json!({"type":"message_delta","usage":{"output_tokens":400}}),
+            &mut |_| {},
+        );
+        let reply = stream.finish("claude-test".into()).unwrap();
+        assert_eq!(reply.input_tokens, Some(99_100));
+        assert_eq!(reply.output_tokens, Some(400));
+    }
 
     #[test]
     fn missing_usage_data_yields_no_token_counts() {
@@ -4024,4 +4371,24 @@ mod tests {
             Some("Ollama internal thinking trace...")
         );
     }
+
+    #[test]
+    fn decode_tool_arguments_sanitizes_fences_tokens_and_trailing_commas() {
+        // 1. Markdown fences
+        let fenced =
+            Value::String("```json\n{\"command\": \"curl -s https://example.com\"}\n```".into());
+        let decoded = decode_tool_arguments(&fenced);
+        assert_eq!(decoded["command"], "curl -s https://example.com");
+
+        // 2. DeepSeek DSML token
+        let dsml = Value::String("{\"command\": \"ls -la\"}</｜｜DSML｜｜".into());
+        let decoded_dsml = decode_tool_arguments(&dsml);
+        assert_eq!(decoded_dsml["command"], "ls -la");
+
+        // 3. Trailing commas
+        let trailing = Value::String("{\"command\": \"echo ok\", \"flag\": true, }".into());
+        let decoded_trailing = decode_tool_arguments(&trailing);
+        assert_eq!(decoded_trailing["command"], "echo ok");
+        assert_eq!(decoded_trailing["flag"], true);
+    }
 }

package/crates/mint-core/src/agent/code_tools.rs

@@ -543,6 +543,15 @@ pub fn apply_code_edits(
     }
     let mut applied = Vec::new();
     for (edit, preview) in edits.iter().zip(prepared) {
+        let pending = crate::system::workspace_history::begin_edit(&root, &preview.path).map_err(
+            |message| CodeInspectionError::Write {
+                path: preview.path.clone(),
+                source: std::io::Error::other(message),
+            },
+        )?;
+        if sha256(&read_optional_content(&preview.path)?) != preview.before_sha256 {
+            return Err(CodeInspectionError::StaleProposal(preview.path.clone()));
+        }
         if let Some(parent) = preview.path.parent() {
             fs::create_dir_all(parent).map_err(|source| CodeInspectionError::Write {
                 path: parent.to_path_buf(),
@@ -550,6 +559,12 @@ pub fn apply_code_edits(
             })?;
         }
         write_atomic(&preview.path, &edit.content)?;
+        pending
+            .commit()
+            .map_err(|message| CodeInspectionError::Write {
+                path: preview.path.clone(),
+                source: std::io::Error::other(message),
+            })?;
         applied.push(AppliedCodeEdit {
             path: preview.path,
             created: !preview.existed,