ft-scout 8.0.1 → 8.0.3

package/bin/src/engine/agentEngine.js

@@ -9,7 +9,7 @@ import { saveAgentSession, getRecentSessionsSummary } from '../utils/session.js'
 import { safeNote, renderMarkdown } from '../utils/markdown.js';
 import { openai, callOpenAIWithRetry, isQuotaExceededError, sanitizeMessage, sanitizeMessages } from './llm.js';
 import { checkFileSyntax, verifyAndSelfHealFiles, executeSmartCommand, extractErrorDiagnostics } from './verifier.js';
-import { openApp, executeInApp } from './appControl.js';
+import { openApp, executeInApp, cleanupScreenshots } from './appControl.js';
 import { speakText, listenSpeechToText } from './voiceEngine.js';
 const execAsync = promisify(exec);
 export function robustSnippetReplace(origContent, target, replacement) {
@@ -284,13 +284,13 @@ export const AGENT_TOOLS = [
     },
     {
         name: 'app_action',
-        description: 'Execute interactive desktop/browser scratchpad automation actions, screen takeover, mouse cursor positioning & click takeover, keyboard keystrokes, hotkeys, screen seeing/screenshot capture, visual screen analysis, focus locks, or command sequences without shell commands (e.g. action: "takeover", "capture_screen", "see_screen", "analyze_screen", "get_screen_info", "click_app", "move_mouse", "type_text", "send_keys", "key_combo", "lock_app", "fetch_page", "navigate", "search", "send_dm", "exec_command", "open_file").',
+        description: 'Execute interactive desktop/browser scratchpad automation actions, screen seeing & analysis, UI element clicking, screen takeover, mouse cursor positioning & click takeover, mouse scrolling, window management, keyboard keystrokes, hotkeys, focus locks, or command sequences without shell commands (e.g. action: "see_screen", "analyze_screen", "capture_screen", "click_element", "click_app", "scroll", "list_windows", "focus_window", "takeover", "move_mouse", "type_text", "send_keys", "key_combo", "lock_app", "fetch_page", "navigate", "search", "send_dm", "exec_command", "open_file"). All temporary screenshot files are automatically deleted upon task completion.',
         parameters: {
             type: 'object',
             properties: {
                 app: { type: 'string', description: 'Target application name, window title, screen, or category ("browser", "terminal", "editor", "notepad", "chrome", "desktop")' },
-                action: { type: 'string', description: 'Action type ("takeover", "capture_screen", "see_screen", "analyze_screen", "get_screen_info", "type_text", "click_app", "move_mouse", "send_keys", "key_combo", "lock_app", "fetch_page", "search", "send_dm", "exec_command", "open_file")' },
-                payload: { description: 'Action details/payload object or string (e.g. text string, { text: "...", enter: true }, { x: 100, y: 200, button: "left" }, { keyCombo: "ctrl+v" }, { duration: 3000 }, URL, search query, command, or file path)' },
+                action: { type: 'string', description: 'Action type ("see_screen", "analyze_screen", "capture_screen", "click_element", "click_app", "scroll", "list_windows", "focus_window", "takeover", "send_mail", "type_text", "move_mouse", "send_keys", "key_combo", "lock_app", "fetch_page", "search", "send_dm", "exec_command", "open_file")' },
+                payload: { description: 'Action details/payload object or string (e.g. element name/description to click { element: "Search Google" }, coordinates { x: 100, y: 200 }, scroll { direction: "down", amount: 4 }, text string, { text: "...", enter: true }, { keyCombo: "ctrl+v" }, { duration: 3000 }, URL, search query, command, or file path)' },
             },
             required: ['app', 'action'],
         },
@@ -399,12 +399,26 @@ export class AgentExecutionLoop {
     readFilesHistory = new Set();
     consecutiveReads = 0;
     consecutiveCommands = 0;
+    consecutiveScreenChecks = 0;
+    toolCallHistory = [];
     scratchpadState = { plan: [], completedSteps: [], notes: '' };
     constructor(options) {
         this.cwd = process.cwd();
         this.autoApprove = Boolean(options?.autoApprove);
         this.maxSteps = options?.maxSteps || 30;
         this.projectName = options?.projectName || path.basename(this.cwd);
+        // Ensure temporary screenshot cleanup on early exit or interrupt
+        const onExitOrInterrupt = () => {
+            try {
+                cleanupScreenshots();
+            }
+            catch { }
+        };
+        process.once('exit', onExitOrInterrupt);
+        process.once('SIGINT', () => {
+            onExitOrInterrupt();
+            process.exit(0);
+        });
     }
     getModifiedFiles() {
         return Array.from(this.modifiedFiles);
@@ -430,6 +444,8 @@ export class AgentExecutionLoop {
         this.currentStep = 0;
         this.consecutiveReads = 0;
         this.consecutiveCommands = 0;
+        this.consecutiveScreenChecks = 0;
+        this.toolCallHistory = [];
         this.scratchpadState = { plan: [], completedSteps: [], notes: '' };
     }
     updateScratchpad(plan, completedSteps, notes) {
@@ -472,7 +488,104 @@ export class AgentExecutionLoop {
         }
         return summary;
     }
-    handleToolConsecutiveTracking(fnName, step) {
+    autoSyncScratchpadFromText(text) {
+        try {
+            const planLines = text.match(/[-*]\s*\[([ xX✓/])\]\s*(.*)/g);
+            if (planLines && planLines.length > 0) {
+                const plan = [];
+                const completed = [];
+                planLines.forEach((l, idx) => {
+                    const isDone = l.includes('[x]') || l.includes('[X]') || l.includes('[✓]');
+                    const cleanText = l.replace(/^[-*]\s*\[[ xX✓/]\]\s*/, '').trim();
+                    plan.push(cleanText);
+                    if (isDone)
+                        completed.push(idx);
+                });
+                this.scratchpadState.plan = plan;
+                this.scratchpadState.completedSteps = completed;
+            }
+            const obsMatch = text.match(/Observations?:\s*([^\n]+(?:\n[^\n]+)*)/i);
+            const thoughtMatch = text.match(/Thought:\s*([^\n]+(?:\n[^\n]+)*)/i);
+            const notes = [
+                thoughtMatch ? `Thought: ${thoughtMatch[1]?.trim()}` : '',
+                obsMatch ? `Observations: ${obsMatch[1]?.trim()}` : '',
+            ].filter(Boolean).join('\n\n');
+            if (notes) {
+                this.scratchpadState.notes = notes;
+            }
+            const ftDir = path.join(this.cwd, '.ft');
+            if (!fs.existsSync(ftDir))
+                fs.mkdirSync(ftDir, { recursive: true });
+            const scratchpadPath = path.join(ftDir, 'scratchpad.md');
+            let mdContent = `# Scout Agent Scratchpad & Working Memory\n\n`;
+            if (this.scratchpadState.plan.length > 0) {
+                mdContent += `## Plan Checklist\n`;
+                this.scratchpadState.plan.forEach((item, idx) => {
+                    const isDone = this.scratchpadState.completedSteps.includes(idx);
+                    mdContent += `- [${isDone ? 'x' : ' '}] Step ${idx + 1}: ${item}\n`;
+                });
+                mdContent += `\n`;
+            }
+            if (this.scratchpadState.notes) {
+                mdContent += `## Working Memory & Observations\n${this.scratchpadState.notes}\n`;
+            }
+            fs.writeFileSync(scratchpadPath, mdContent, 'utf-8');
+        }
+        catch { }
+    }
+    handleToolConsecutiveTracking(fnName, args, step) {
+        const argsKey = JSON.stringify(args || {});
+        this.toolCallHistory.push({ fnName, argsKey });
+        if (this.toolCallHistory.length > 10) {
+            this.toolCallHistory.shift();
+        }
+        // 1. Direct duplicate call check: Same tool + exact same arguments executed consecutively
+        const len = this.toolCallHistory.length;
+        if (len >= 2) {
+            const prev = this.toolCallHistory[len - 2];
+            const curr = this.toolCallHistory[len - 1];
+            if (prev && curr && prev.fnName === curr.fnName && prev.argsKey === curr.argsKey && !['task_completed', 'ask_user'].includes(fnName)) {
+                this.historyMessages.push({
+                    role: 'user',
+                    content: `CRITICAL ACTION LOOP DETECTED (Step ${step}): You called "${fnName}" with the EXACT same arguments as the previous step! Repeating this action will not change the outcome. DO NOT call this tool with these arguments again. Change your strategy immediately: search with different terms, navigate to the correct page, take a concrete input action, or call "task_completed" if done.`,
+                });
+            }
+        }
+        // 2. Ping-Pong / Oscillation loop check: A -> B -> A -> B
+        if (len >= 4) {
+            const a1 = this.toolCallHistory[len - 4];
+            const b1 = this.toolCallHistory[len - 3];
+            const a2 = this.toolCallHistory[len - 2];
+            const b2 = this.toolCallHistory[len - 1];
+            if (a1 && b1 && a2 && b2 &&
+                a1.fnName === a2.fnName && a1.argsKey === a2.argsKey &&
+                b1.fnName === b2.fnName && b1.argsKey === b2.argsKey &&
+                (a1.fnName !== b1.fnName || a1.argsKey !== b1.argsKey)) {
+                this.historyMessages.push({
+                    role: 'user',
+                    content: `CRITICAL OSCILLATION LOOP DETECTED (Step ${step}): You are stuck alternating between "${a1.fnName}" and "${b1.fnName}" without progressing! STOP repeating these actions immediately. If the page or app is not responding as expected, adopt an alternative path, use web search, launch the desktop app directly, or conclude with task_completed.`,
+                });
+            }
+        }
+        // 3. Screen inspection stagnation tracking
+        if (fnName === 'app_action') {
+            const act = String(args?.action || '').toLowerCase();
+            if (['see_screen', 'analyze_screen', 'capture_screen'].includes(act)) {
+                this.consecutiveScreenChecks++;
+                if (this.consecutiveScreenChecks >= 2) {
+                    this.historyMessages.push({
+                        role: 'user',
+                        content: `URGENT ACTION MANDATE (Step ${step}): You have inspected the screen ${this.consecutiveScreenChecks} times consecutively without taking any physical action! Do not keep calling see_screen. Take a concrete action now: click an element ("click_element"), enter text ("type_text"), open/focus a window, or complete the task.`,
+                    });
+                }
+            }
+            else {
+                this.consecutiveScreenChecks = 0;
+            }
+        }
+        else {
+            this.consecutiveScreenChecks = 0;
+        }
         if (['read_file', 'list_dir', 'glob_search', 'grep_search', 'tree_view', 'file_info'].includes(fnName)) {
             this.consecutiveReads++;
             this.consecutiveCommands = 0;
@@ -541,17 +654,45 @@ Core Directives & Behavioral Guidelines:
 19. EXTERNAL APP TAKEOVER & CONTROL PROTOCOL: When requested by user prompt to open, take over, or work inside external applications (browser, terminal, VS Code, Notepad, social apps like Instagram, WhatsApp, Twitter/X, Telegram, or custom apps):
  a. Launch App: Use \`open_app\` to launch or focus the target application with optional URL, file path, or initial script.
  b. Social DM & Messaging Automation: For Instagram, WhatsApp, Twitter/X, or Telegram messaging requests (e.g. "open instagram and send message to @user"), immediately invoke \`app_action\` with action "send_dm" or "open_dm" (or \`open_app\`) specifying the target username/phone and message text so the agent automatically opens the direct messaging link in the browser!
- c. Work Inside App: Use \`app_action\` or \`run_command\` to execute actions inside the app context (e.g. fetching browser page content, running commands inside terminal, searching web, opening files in editor).
-20. RECKLESS SCRATCHPAD DESKTOP CONTROL, SCREEN TAKEOVER & VISUAL ANALYSIS (ZERO SHELL COMMANDS):
- a. Mouse & Keyboard Screen Takeover: During scratchpad execution, you act as a full desktop automation agent that takes over the mouse cursor, screen focus, and keyboard typing on the specified screen/app using \`app_action\` (\`action: "takeover"\`, \`action: "click_app"\`, \`action: "move_mouse"\`, \`action: "type_text"\`, \`action: "send_keys"\`, \`action: "key_combo"\`).
- b. Native Screen Seeing & Visual Analysis (No Shell Commands): You can view, capture, and visually analyze the specified screen/window directly using \`app_action\` (\`action: "capture_screen"\`, \`action: "see_screen"\`, \`action: "analyze_screen"\`, \`action: "get_screen_info"\`). DO NOT run shell commands (\`run_command\`) to capture or inspect screens—\`app_action\` handles native screen capture, visual analysis, mouse takeover, and key input internally!
- c. CLI Credential Input Prompt: If an application requires login credentials, passwords, 2FA codes, or secret tokens to proceed, call \`ask_user\` tool with a clear prompt. This presents a secure, interactive input bar directly in the user's running terminal CLI. Once the user enters the secret, take the received input, inject it into the target application window via \`app_action\` (\`action: "type_text"\`).
-21. MANDATORY TAKEOVER TOOL INVOCATION MANDATE: Whenever the user goal requests to open, launch, take over, click, type, or interact with an external app or desktop screen (e.g. 'take over browser and open website', 'open notepad and type', 'take over desktop'), YOU MUST CALL \`open_app\` AND \`app_action\` TOOLS (with \`action: "takeover"\`, \`action: "click_app"\`, \`action: "type_text"\`, \`action: "send_keys"\`, \`action: "capture_screen"\`) IN YOUR VERY FIRST STEP! You MUST NOT return a plain text answer or claim you did it in text without calling the tool! The tool calls ARE MANDATORY for executing the physical takeover.
+ c. Email & Gmail Automation: When requested to send or compose an email (e.g. "send mail to frontterrain@gmail.com saying...", "takeover mail.google.com and send mail"):
+    - Immediately invoke \`app_action\` with \`action: "send_mail"\` (or \`action: "send_dm"\`) specifying the target recipient email and body text. The agent automatically constructs the direct Gmail compose URL (\`https://mail.google.com/mail/?view=cm&fs=1&to=<recipient>&su=<subject>&body=<body>\`) which pre-populates the compose window and dispatches the email via Ctrl+Enter!
+    - Alternatively, navigate directly to \`https://mail.google.com/mail/?view=cm&fs=1&to=<recipient>&su=...&body=...\` and trigger hotkey \`app_action(action: "key_combo", payload: { keyCombo: "ctrl+enter" })\`. NEVER click blind coordinates like (20, 20) in a web browser!
+ d. Work Inside App: Use \`app_action\` or \`run_command\` to execute actions inside the app context (e.g. fetching browser page content, running commands inside terminal, searching web, opening files in editor).
+ e. Universal Gaming & External Application Protocol:
+     - Desktop Game/App Launch: To open or play ANY installed game or app on the PC (Steam games, Epic Games, Minecraft, Roblox, Discord, Spotify, etc.), call open_app("<game_name>"). The universal OS launcher automatically resolves installed Windows/macOS applications and store packages via system registry and launches them directly!
+     - Web Game Play: If the game is web-based (e.g. Chess.com, Slither.io, 2048, Poki) or not installed locally, NEVER guess speculative URL subpaths! Use app_action(app: "browser", action: "search", payload: { query: "<game_name> play online official" }) or navigate to the official domain homepage to find the verified play URL.
+     - Autonomous Gameplay Progression Loop:
+       1. Window Focus: Use app_action(action: "focus_window", payload: { app: "<game_name>" }) to bring the window front-and-center.
+       2. Screen Observation: Use app_action(action: "see_screen") to inspect the UI, loading screen, or active state.
+       3. Menu Traversal: Click menu buttons ("Play", "Start Game", "New Game", "Continue") using app_action(action: "click_element", payload: { element: "Play" }) .
+       4. Interactive Controls: Send gameplay controls using app_action(action: "send_keys" / "type_text" / "key_combo") with standard gaming keys (WASD, Arrow keys, Space, Enter, Escape, mouse clicks/drags).
+       5. Conclude task with task_completed when the requested gameplay actions or objectives are achieved.
+20. DESKTOP APP TAKEOVER & AUTOMATION PROTOCOL:
+ a. STEP 1 SCREEN TAKEOVER: Whenever the user goal asks to take over the screen, control the desktop, or automate an external app (e.g. 'take over screen', 'open maps and click', 'take over desktop', 'open notepad'):
+    - Your VERY FIRST tool call in Step 1 MUST be \`app_action\` with \`action: "takeover"\` (e.g. \`app: "desktop"\` or the target app).
+    - Calling \`app_action\` with \`action: "takeover"\` immediately activates the full-screen sky-blue aura HUD, displays the warning banner "⚡ Scout is on the screen.", blocks external input interruptions, and takes over the mouse cursor!
+    - Then immediately proceed to launch/navigate with \`open_app\` or \`app_action\` (\`action: "click_element"\` / \`action: "click_app"\` / \`action: "type_text"\` / \`action: "send_keys"\`).
+ b. SCREEN CAPTURE & VISION DIRECTIVE: You CAN take screenshots and visually inspect or analyze the screen using \`app_action\` with \`action: "see_screen"\`, \`"analyze_screen"\`, or \`"capture_screen"\` whenever needed to verify UI state, check screen contents, or inspect open windows. All temporary screenshot files are automatically and securely deleted upon task completion for privacy and storage cleanliness.
+ c. VISUAL ELEMENT GROUNDING (click_element): Instead of guessing blind coordinates (x, y), click buttons, inputs, or menus by descriptive label using \`app_action(action: "click_element", payload: { element: "Search" })\`. Vision AI and native OS UI automation will locate the element and click it accurately.
+ d. WINDOW & SCROLL CONTROLS: Use \`app_action(action: "scroll", payload: { direction: "down", amount: 4 })\` to scroll pages. Use \`app_action(action: "list_windows")\` to see all open windows, and \`app_action(action: "focus_window", payload: { app: "chrome" })\` to bring a window front-and-center.
+ e. CLI Credential Input Prompt: If an application requires login credentials, passwords, 2FA codes, or secret tokens to proceed, call \`ask_user\` tool with a clear prompt. This presents a secure, interactive input bar directly in the user's running terminal CLI. Once the user enters the secret, take the received input, inject it into the target application window via \`app_action\` (\`action: "type_text"\`).
+21. MANDATORY TAKEOVER TOOL INVOCATION MANDATE: Whenever the user goal requests to open, launch, take over, click, type, or interact with an external app or desktop screen (e.g. 'take over browser and open website', 'open notepad and type', 'take over desktop', 'take over screen'):
+ YOU MUST CALL \`app_action(action: "takeover")\` IN STEP 1. Then call \`open_app\` and \`app_action\` (\`click_element\`, \`click_app\`, \`type_text\`, \`send_keys\`). DO NOT call \`read_file\` or \`write_file\` for workspace code files when asked to take over external desktop apps! The takeover tool call is MANDATORY for executing the physical takeover.
 22. BROWSER DIRECT URL NAVIGATION MANDATE: When asked to open or navigate to a specific website or web app (e.g. Apple Maps, GitHub, YouTube, etc.), NEVER call Google Search or issue repeated \`app_action: search\` calls with text queries! IMMEDIATELY pass the exact URL (e.g. "https://maps.apple.com") to \`open_app(app: "browser", target: "https://maps.apple.com")\` or \`app_action(app: "browser", action: "navigate", payload: { url: "https://maps.apple.com" })\`. Direct URL navigation must always target the exact site URL directly without putting queries into Google Search!
-23. WORKING MEMORY & SCRATCHPAD PROTOCOL: For multi-step complex goals, maintain an active working memory scratchpad using the \`agent_scratchpad\` tool.
- a. Plan Checklist: Call \`agent_scratchpad(action: "update", plan: ["step 1", "step 2", ...])\` at the start of complex multi-step tasks.
- b. Reasoning Block: In every step, write a clear 1-2 sentence \`<scratchpad>\` reasoning block in your response before invoking tool calls to explain current state, working notes, and step selection.
- c. Real-Time Tracking: Update completed steps using \`agent_scratchpad(action: "update", completedSteps: [...])\` as you finish tasks so state is saved in \`.ft/scratchpad.md\`.
+23. WORKING MEMORY & STRUCTURED SCRATCHPAD PROTOCOL:
+ Like Antigravity and leading autonomous agents, maintain disciplined working memory. In EVERY step, start your response with a structured <scratchpad> reasoning block before returning tool calls:
+ \`\`\`markdown
+ <scratchpad>
+ Thought: [1-2 sentences on what you are doing on this turn and why]
+ Plan:
+   [x] 1. [Completed step]
+   [/] 2. [In-progress step]
+   [ ] 3. [Next upcoming step]
+ Observations: [What you learned from the last tool result or screen capture]
+ Next Action: [The exact tool you are invoking now]
+ </scratchpad>
+ \`\`\`
+ This keeps your reasoning crystal-clear, ensures plan progression, and syncs automatically with .ft/scratchpad.md.
 
 ${this.getScratchpadPromptContext()}
 
@@ -695,192 +836,229 @@ When returning tool calls, use standard OpenAI function calling format or JSON t
         let step = 0;
         let finalSummary = '';
         const stepDurations = [];
-        while (step < this.maxSteps) {
-            step++;
-            this.currentStep = step;
-            const stepStartTime = Date.now();
-            const avgStepMs = stepDurations.length > 0
-                ? stepDurations.reduce((a, b) => a + b, 0) / stepDurations.length
-                : 12000;
-            const remainingSteps = (this.maxSteps - step + 1);
-            const estSecsLeft = Math.max(5, Math.round((remainingSteps * avgStepMs) / 1000));
-            const estLeftStr = estSecsLeft >= 60
-                ? `${Math.floor(estSecsLeft / 60)}m ${estSecsLeft % 60}s`
-                : `${estSecsLeft}s`;
-            const stepSpinner = spinner();
-            stepSpinner.start(chalk.cyan(`Scout is Working (Step ${step}/${this.maxSteps} • Est. completion: ~${estLeftStr} left)`));
-            try {
-                let response;
+        try {
+            while (step < this.maxSteps) {
+                step++;
+                this.currentStep = step;
+                const stepStartTime = Date.now();
+                const avgStepMs = stepDurations.length > 0
+                    ? stepDurations.reduce((a, b) => a + b, 0) / stepDurations.length
+                    : 12000;
+                const remainingSteps = (this.maxSteps - step + 1);
+                const estSecsLeft = Math.max(5, Math.round((remainingSteps * avgStepMs) / 1000));
+                const estLeftStr = estSecsLeft >= 60
+                    ? `${Math.floor(estSecsLeft / 60)}m ${estSecsLeft % 60}s`
+                    : `${estSecsLeft}s`;
+                const stepSpinner = spinner();
+                stepSpinner.start(chalk.cyan(`Scout is Working (Step ${step}/${this.maxSteps} • Est. completion: ~${estLeftStr} left)`));
                 try {
-                    response = await callOpenAIWithRetry(async (model) => {
-                        return await openai.chat.completions.create({
-                            model,
-                            messages: sanitizeMessages(this.historyMessages),
-                            tools: AGENT_TOOLS.map((t) => ({ type: 'function', function: t })),
-                            tool_choice: 'auto',
-                            temperature: 0.1,
-                        });
-                    });
-                }
-                catch (err) {
-                    const errStr = String(err?.message || err?.error || err || '').toLowerCase();
-                    if (errStr.includes('tool') || errStr.includes('400') || errStr.includes('not supported') || errStr.includes('reasoning')) {
+                    let response;
+                    try {
                         response = await callOpenAIWithRetry(async (model) => {
                             return await openai.chat.completions.create({
                                 model,
                                 messages: sanitizeMessages(this.historyMessages),
+                                tools: AGENT_TOOLS.map((t) => ({ type: 'function', function: t })),
+                                tool_choice: 'auto',
                                 temperature: 0.1,
+                                presence_penalty: 0.1,
+                                frequency_penalty: 0.1,
                             });
                         });
                     }
-                    else {
-                        throw err;
-                    }
-                }
-                const choice = response.choices[0];
-                if (!choice) {
-                    stepSpinner.stop(chalk.yellow('No response from AI model. Retrying step...'));
-                    continue;
-                }
-                const msg = sanitizeMessage(choice.message);
-                this.historyMessages.push(msg);
-                // Render assistant's thought/reasoning text if provided on this turn
-                const textContent = msg.content || '';
-                if (textContent.trim()) {
-                    safeNote(renderMarkdown(textContent), `Scout Thought (Step ${step})`);
-                }
-                // Check native tool calls
-                if (msg.tool_calls && msg.tool_calls.length > 0) {
-                    stepSpinner.stop(chalk.green(`Step ${step}: Scout issued ${msg.tool_calls.length} tool call(s).`));
-                    for (const tc of msg.tool_calls) {
-                        if (tc.type === 'function' && tc.function) {
-                            const fnName = tc.function.name;
-                            let args = {};
-                            try {
-                                args = JSON.parse(tc.function.arguments || '{}');
-                            }
-                            catch { }
-                            const toolResult = await this.dispatchToolCall(fnName, args);
-                            this.historyMessages.push({
+                    catch (err) {
+                        const errStr = String(err?.message || err?.error || err || '').toLowerCase();
+                        if (errStr.includes('tool') || errStr.includes('400') || errStr.includes('not supported') || errStr.includes('reasoning')) {
+                            // Model doesn't support native function calling — inject tool-call formatting hint
+                            // so extractJsonToolCall can parse the response as a structured tool invocation
+                            const toolHintMsg = {
                                 role: 'user',
-                                tool_call_id: tc.id,
-                                content: `Tool Execution Result (${fnName}):\n${toolResult.result}`,
+                                content: `IMPORTANT: This model does not support native function/tool calling. You MUST format your tool invocations as a JSON code block in your response like this:
+\`\`\`json
+{ "tool": "tool_name", "args": { ... } }
+\`\`\`
+Available tools: read_file, write_file, edit_file, run_command, list_dir, grep_search, glob_search, tree_view, file_info, multi_edit_file, fetch_url, git_diff, open_app, app_action, agent_scratchpad, ask_user, speak_text, task_completed.
+For app takeover/control use: { "tool": "app_action", "args": { "app": "browser", "action": "takeover" } }
+For opening apps use: { "tool": "open_app", "args": { "app": "chrome", "target": "https://..." } }
+For clicking use: { "tool": "app_action", "args": { "app": "desktop", "action": "click_app", "payload": { "x": 500, "y": 300 } } }
+For typing text use: { "tool": "app_action", "args": { "app": "browser", "action": "type_text", "payload": { "text": "...", "enter": true } } }
+You MUST output exactly ONE JSON code block per tool call. Do NOT describe what you would do in plain text—output the JSON tool call directly!`,
+                            };
+                            const fallbackMessages = [...this.historyMessages, toolHintMsg];
+                            response = await callOpenAIWithRetry(async (model) => {
+                                return await openai.chat.completions.create({
+                                    model,
+                                    messages: sanitizeMessages(fallbackMessages),
+                                    temperature: 0.1,
+                                    presence_penalty: 0.1,
+                                    frequency_penalty: 0.1,
+                                });
                             });
-                            this.handleToolConsecutiveTracking(fnName, step);
-                            if (fnName === 'task_completed') {
-                                finalSummary = args.summary || toolResult.result;
-                                if (this.modifiedFiles.size > 0) {
-                                    const filesList = getDirectoryFiles(this.cwd);
-                                    const healRes = await verifyAndSelfHealFiles(Array.from(this.modifiedFiles), this.cwd, this.projectName, filesList, { maxRetries: 3 });
-                                    if (healRes.verifiedFiles.length > 0) {
-                                        safeNote(chalk.green(` Self-Healing Verification Confirmed: ${healRes.verifiedFiles.length} file(s) syntax & build clean!`), ' Code Verification Clean');
-                                    }
-                                    if (healRes.remainingErrors.length > 0) {
-                                        safeNote(chalk.yellow(`️ Remaining verification issues:\n${healRes.remainingErrors.join('\n')}`), '️ Verification Warning');
-                                    }
+                        }
+                        else {
+                            throw err;
+                        }
+                    }
+                    const choice = response.choices[0];
+                    if (!choice) {
+                        stepSpinner.stop(chalk.yellow('No response from Scout. Retrying step...'));
+                        continue;
+                    }
+                    const msg = sanitizeMessage(choice.message);
+                    this.historyMessages.push(msg);
+                    // Render assistant's thought/scratchpad reasoning text if provided on this turn
+                    const textContent = msg.content || '';
+                    if (textContent.trim()) {
+                        const scratchMatch = textContent.match(/<scratchpad>([\s\S]*?)<\/scratchpad>/i);
+                        if (scratchMatch && scratchMatch[1]) {
+                            const scratchText = scratchMatch[1].trim();
+                            safeNote(renderMarkdown(scratchText), chalk.cyan.bold(`Scout Scratchpad & Working Memory (Step ${step})`));
+                            this.autoSyncScratchpadFromText(scratchText);
+                            const remainingText = textContent.replace(/<scratchpad>[\s\S]*?<\/scratchpad>/i, '').trim();
+                            if (remainingText) {
+                                safeNote(renderMarkdown(remainingText), `Scout Thought (Step ${step})`);
+                            }
+                        }
+                        else {
+                            safeNote(renderMarkdown(textContent), `Scout Thought (Step ${step})`);
+                        }
+                    }
+                    // Check native tool calls
+                    if (msg.tool_calls && msg.tool_calls.length > 0) {
+                        stepSpinner.stop(chalk.green(`Step ${step}: Scout issued ${msg.tool_calls.length} tool call(s).`));
+                        for (const tc of msg.tool_calls) {
+                            if (tc.type === 'function' && tc.function) {
+                                const fnName = tc.function.name;
+                                let args = {};
+                                try {
+                                    args = JSON.parse(tc.function.arguments || '{}');
                                 }
-                                saveAgentSession({
-                                    goal: userGoal,
-                                    summary: finalSummary,
-                                    modifiedFiles: Array.from(this.modifiedFiles),
+                                catch { }
+                                const toolResult = await this.dispatchToolCall(fnName, args);
+                                this.historyMessages.push({
+                                    role: 'user',
+                                    tool_call_id: tc.id,
+                                    content: `Tool Execution Result (${fnName}):\n${toolResult.result}`,
                                 });
-                                return { success: true, summary: finalSummary };
+                                this.handleToolConsecutiveTracking(fnName, args, step);
+                                if (fnName === 'task_completed') {
+                                    finalSummary = args.summary || toolResult.result;
+                                    if (this.modifiedFiles.size > 0) {
+                                        const filesList = getDirectoryFiles(this.cwd);
+                                        const healRes = await verifyAndSelfHealFiles(Array.from(this.modifiedFiles), this.cwd, this.projectName, filesList, { maxRetries: 3 });
+                                        if (healRes.verifiedFiles.length > 0) {
+                                            safeNote(chalk.green(` Self-Healing Verification Confirmed: ${healRes.verifiedFiles.length} file(s) syntax & build clean!`), ' Code Verification Clean');
+                                        }
+                                        if (healRes.remainingErrors.length > 0) {
+                                            safeNote(chalk.yellow(`️ Remaining verification issues:\n${healRes.remainingErrors.join('\n')}`), '️ Verification Warning');
+                                        }
+                                    }
+                                    saveAgentSession({
+                                        goal: userGoal,
+                                        summary: finalSummary,
+                                        modifiedFiles: Array.from(this.modifiedFiles),
+                                    });
+                                    return { success: true, summary: finalSummary };
+                                }
                             }
                         }
+                        continue;
                     }
-                    continue;
-                }
-                // Check text response or fallback JSON tool call
-                stepSpinner.stop(chalk.blue(`Step ${step} thinking complete.`));
-                const parsedJsonTool = this.extractJsonToolCall(textContent);
-                if (parsedJsonTool) {
-                    const toolResult = await this.dispatchToolCall(parsedJsonTool.tool, parsedJsonTool.args);
+                    // Check text response or fallback JSON tool call
+                    stepSpinner.stop(chalk.blue(`Step ${step} thinking complete.`));
+                    const parsedJsonTool = this.extractJsonToolCall(textContent);
+                    if (parsedJsonTool) {
+                        const toolResult = await this.dispatchToolCall(parsedJsonTool.tool, parsedJsonTool.args);
+                        this.historyMessages.push({
+                            role: 'user',
+                            content: `Tool Execution Result (${parsedJsonTool.tool}):\n${toolResult.result}`,
+                        });
+                        this.handleToolConsecutiveTracking(parsedJsonTool.tool, parsedJsonTool.args, step);
+                        if (parsedJsonTool.tool === 'task_completed') {
+                            finalSummary = parsedJsonTool.args?.summary || toolResult.result;
+                            saveAgentSession({
+                                goal: userGoal,
+                                summary: finalSummary,
+                                modifiedFiles: Array.from(this.modifiedFiles),
+                            });
+                            return { success: true, summary: finalSummary };
+                        }
+                        continue;
+                    }
+                    if (textContent.trim()) {
+                        let cleanedThought = textContent
+                            .replace(/[\u0600-\u06FF\u0750-\u077F\uAC00-\uD7AF\u3040-\u30FF\u4E00-\u9FFF\u0D80-\u0DFF]+/g, '')
+                            .trim();
+                        if (!cleanedThought || cleanedThought.length < 5) {
+                            cleanedThought = 'Analyzing codebase files and executing next tool operation...';
+                        }
+                        safeNote(renderMarkdown(cleanedThought), ` Scout Agent Thought (Step ${step})`);
+                        const lowerText = textContent.toLowerCase();
+                        const isExplicitCompletion = lowerText.includes('task is complete') ||
+                            lowerText.includes('task complete') ||
+                            lowerText.includes('goal completed') ||
+                            lowerText.includes('goal is completed') ||
+                            lowerText.includes('all tasks completed') ||
+                            lowerText.includes('i have completed') ||
+                            lowerText.includes('no further changes needed') ||
+                            lowerText.includes('the fix is complete') ||
+                            lowerText.includes('has been created') ||
+                            lowerText.includes('successfully created') ||
+                            lowerText.includes('created the file') ||
+                            lowerText.includes('file created') ||
+                            lowerText.includes('implementation complete') ||
+                            lowerText.includes('work is complete');
+                        // If explicit completion phrase found, OR files have already been modified and assistant returned a final summary without calling tools
+                        if (isExplicitCompletion || (this.modifiedFiles.size > 0 && !lowerText.includes('?') && textContent.length > 50)) {
+                            saveAgentSession({
+                                goal: userGoal,
+                                summary: textContent,
+                                modifiedFiles: Array.from(this.modifiedFiles),
+                            });
+                            return { success: true, summary: textContent };
+                        }
+                    }
+                    // If assistant responded with text without calling tools, prompt it to execute tools to complete the goal
                     this.historyMessages.push({
                         role: 'user',
-                        content: `Tool Execution Result (${parsedJsonTool.tool}):\n${toolResult.result}`,
+                        content: 'You provided a text response but have not called any tools (write_file, edit_file, run_command, task_completed). Please execute necessary tool calls to complete the user goal, or invoke task_completed if finished.',
                     });
-                    this.handleToolConsecutiveTracking(parsedJsonTool.tool, step);
-                    if (parsedJsonTool.tool === 'task_completed') {
-                        finalSummary = parsedJsonTool.args?.summary || toolResult.result;
-                        saveAgentSession({
-                            goal: userGoal,
-                            summary: finalSummary,
-                            modifiedFiles: Array.from(this.modifiedFiles),
-                        });
-                        return { success: true, summary: finalSummary };
-                    }
-                    continue;
+                    stepDurations.push(Date.now() - stepStartTime);
                 }
-                if (textContent.trim()) {
-                    let cleanedThought = textContent
-                        .replace(/[\u0600-\u06FF\u0750-\u077F\uAC00-\uD7AF\u3040-\u30FF\u4E00-\u9FFF\u0D80-\u0DFF]+/g, '')
-                        .trim();
-                    if (!cleanedThought || cleanedThought.length < 5) {
-                        cleanedThought = 'Analyzing codebase files and executing next tool operation...';
-                    }
-                    safeNote(renderMarkdown(cleanedThought), ` Scout Agent Thought (Step ${step})`);
-                    const lowerText = textContent.toLowerCase();
-                    const isExplicitCompletion = lowerText.includes('task is complete') ||
-                        lowerText.includes('task complete') ||
-                        lowerText.includes('goal completed') ||
-                        lowerText.includes('goal is completed') ||
-                        lowerText.includes('all tasks completed') ||
-                        lowerText.includes('i have completed') ||
-                        lowerText.includes('no further changes needed') ||
-                        lowerText.includes('the fix is complete') ||
-                        lowerText.includes('has been created') ||
-                        lowerText.includes('successfully created') ||
-                        lowerText.includes('created the file') ||
-                        lowerText.includes('file created') ||
-                        lowerText.includes('implementation complete') ||
-                        lowerText.includes('work is complete');
-                    // If explicit completion phrase found, OR files have already been modified and assistant returned a final summary without calling tools
-                    if (isExplicitCompletion || (this.modifiedFiles.size > 0 && !lowerText.includes('?') && textContent.length > 50)) {
-                        saveAgentSession({
-                            goal: userGoal,
-                            summary: textContent,
-                            modifiedFiles: Array.from(this.modifiedFiles),
-                        });
-                        return { success: true, summary: textContent };
+                catch (err) {
+                    stepDurations.push(Date.now() - stepStartTime);
+                    if (isQuotaExceededError(err)) {
+                        stepSpinner.stop(chalk.red(`Oops! it\'s not you, it\'s us`));
+                        safeNote(`${chalk.bold.red(`Something went wrong in (Step ${step}), please try again in a moment`)}\n\n` +
+                            `${chalk.yellow('The configured AI provider has reached its API usage limit or rate cap.')}\n` +
+                            `${chalk.dim('This is separate from your Scout credit balance shown by `scout quota`.')}\n\n` +
+                            `${chalk.bold.cyan(' Please come back and try again in a few hours (or check back later today).')}\n\n` +
+                            `${chalk.dim('Scout Agent session has ended gracefully to protect remaining workflow.')}`, '️ API Quota Limit Reached');
+                        return {
+                            success: false,
+                            summary: 'Something went wrong, please try again in a moment.',
+                        };
                     }
+                    stepSpinner.stop(chalk.red(`Step ${step} execution error: ${err?.message || String(err)}`));
+                    this.historyMessages.push({
+                        role: 'user',
+                        content: `Error in previous turn: ${err?.message || String(err)}. Please try alternative steps or call tools.`,
+                    });
                 }
-                // If assistant responded with text without calling tools, prompt it to execute tools to complete the goal
-                this.historyMessages.push({
-                    role: 'user',
-                    content: 'You provided a text response but have not called any tools (write_file, edit_file, run_command, task_completed). Please execute necessary tool calls to complete the user goal, or invoke task_completed if finished.',
-                });
-                stepDurations.push(Date.now() - stepStartTime);
-            }
-            catch (err) {
-                stepDurations.push(Date.now() - stepStartTime);
-                if (isQuotaExceededError(err)) {
-                    stepSpinner.stop(chalk.red(`Oops! it\'s not you, it\'s us`));
-                    safeNote(`${chalk.bold.red(`Something went wrong in (Step ${step}), please try again in a moment`)}\n\n` +
-                        `${chalk.yellow('The configured AI provider has reached its API usage limit or rate cap.')}\n` +
-                        `${chalk.dim('This is separate from your Scout credit balance shown by `scout quota`.')}\n\n` +
-                        `${chalk.bold.cyan(' Please come back and try again in a few hours (or check back later today).')}\n\n` +
-                        `${chalk.dim('Scout Agent session has ended gracefully to protect remaining workflow.')}`, '️ API Quota Limit Reached');
-                    return {
-                        success: false,
-                        summary: 'Something went wrong, please try again in a moment.',
-                    };
+                // If current max steps limit is reached while task is still in progress, auto-extend by +15 extra steps (up to 60 max threshold)
+                if (step >= this.maxSteps && this.maxSteps < 60) {
+                    this.maxSteps += 15;
+                    safeNote(chalk.bold.yellow(`⚡ Step limit reached while task is in progress. Automatically extending execution by +15 extra steps (New Max Limit: ${this.maxSteps})...`), ' Auto Extra Steps Extension');
                 }
-                stepSpinner.stop(chalk.red(`Step ${step} execution error: ${err?.message || String(err)}`));
-                this.historyMessages.push({
-                    role: 'user',
-                    content: `Error in previous turn: ${err?.message || String(err)}. Please try alternative steps or call tools.`,
-                });
-            }
-            // If current max steps limit is reached while task is still in progress, auto-extend by +15 extra steps (up to 60 max threshold)
-            if (step >= this.maxSteps && this.maxSteps < 60) {
-                this.maxSteps += 15;
-                safeNote(chalk.bold.yellow(`⚡ Step limit reached while task is in progress. Automatically extending execution by +15 extra steps (New Max Limit: ${this.maxSteps})...`), ' Auto Extra Steps Extension');
             }
+            return {
+                success: false,
+                summary: `Reached max iteration steps limit (${this.maxSteps}). Modified files: ${Array.from(this.modifiedFiles).join(', ')}`,
+            };
+        }
+        finally {
+            cleanupScreenshots();
         }
-        return {
-            success: false,
-            summary: `Reached max iteration steps limit (${this.maxSteps}). Modified files: ${Array.from(this.modifiedFiles).join(', ')}`,
-        };
     }
     extractJsonToolCall(content) {
         if (!content)
@@ -900,7 +1078,10 @@ When returning tool calls, use standard OpenAI function calling format or JSON t
         const knownTools = [
             'read_file', 'write_file', 'edit_file', 'run_command', 'list_dir',
             'grep_search', 'glob_search', 'tree_view', 'file_info', 'multi_edit_file',
-            'fetch_url', 'git_diff', 'open_app', 'app_action', 'ask_user', 'task_completed'
+            'fetch_url', 'git_diff', 'open_app', 'app_action', 'agent_scratchpad',
+            'ask_user', 'speak_text', 'task_completed',
+            'desktop_action', 'type_text', 'click_app', 'send_keys', 'takeover',
+            'capture_screen', 'see_screen', 'analyze_screen', 'scratchpad',
         ];
         for (const toolName of knownTools) {
             const tagRegex = new RegExp(`<${toolName}>([\\s\\S]*?)<\\/${toolName}>`, 'i');
@@ -1374,7 +1555,7 @@ When returning tool calls, use standard OpenAI function calling format or JSON t
             case 'app_action': {
                 const appName = String(args.app || args.appName || 'desktop').trim();
                 const action = String(args.action || (name !== 'app_action' ? name : 'type_text')).trim();
-                const payload = args.payload !== undefined ? args.payload : (args.text || args.content || args.target || args.url || args.command || args);
+                const payload = args.payload !== undefined ? args.payload : (args.text || args.content || args.target || args.query || args.element || args.url || args.command || args);
                 const actionRes = await executeInApp(appName, action, payload);
                 return { result: actionRes.output };
             }
@@ -1407,6 +1588,7 @@ When returning tool calls, use standard OpenAI function calling format or JSON t
             case 'task_completed': {
                 const summaryStr = String(args.summary || 'Task completed successfully.');
                 speakText(summaryStr, { async: true });
+                cleanupScreenshots();
                 return { result: summaryStr };
             }
             default: