ft-scout 9.0.4 → 9.0.5

package/bin/src/engine/runtime/agentRuntime.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"agentRuntime.d.ts","sourceRoot":"","sources":["agentRuntime.ts"],"names":[],"mappings":"AAQA,OAAO,EAA0C,KAAK,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAEhG,OAAO,EAAE,cAAc,EAAsB,KAAK,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AAC7G,OAAO,EAAE,KAAK,mBAAmB,EAAsD,MAAM,mCAAmC,CAAC;AAEjI,OAAO,EAAE,KAAK,kBAAkB,EAA6B,MAAM,qBAAqB,CAAC;AACzF,OAAO,EAAE,cAAc,EAAyB,MAAM,sBAAsB,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAkB,MAAM,cAAc,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAGzD,MAAM,WAAW,mBAAmB;IAChC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iBAAiB,CAAC,EAAE,kBAAkB,CAAC;IACvC,kBAAkB,CAAC,EAAE,mBAAmB,CAAC;IACzC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,sBAAsB;IACnC,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,MAAM,EAAE,kBAAkB,CAAC,QAAQ,CAAC,CAAC;CACxC;AAED,qBAAa,YAAY;IACrB,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,WAAW,CAAU;IAC7B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,QAAQ,CAAC,CAAqB;IACtC,OAAO,CAAC,eAAe,CAA0C;IACjE,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,YAAY,CAAC,CAAqB;IAE1C,OAAO,CAAC,SAAS,CAAiB;IAClC,OAAO,CAAC,kBAAkB,CAAsB;IAChD,OAAO,CAAC,iBAAiB,CAAqB;IAC9C,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,OAAO,CAAc;IAC7B,OAAO,CAAC,YAAY,CAAsB;IAE1C,OAAO,CAAC,eAAe,CAAa;IACpC,OAAO,CAAC,aAAa,CAA0B;IAC/C,OAAO,CAAC,OAAO,CAAoB;IACnC,OAAO,CAAC,cAAc,CAAa;IACnC,OAAO,CAAC,WAAW,CAAa;IAChC,OAAO,CAAC,SAAS,CAAsB;IACvC,OAAO,CAAC,gBAAgB,CAA0B;IAClD,OAAO,CAAC,eAAe,CAAkD;IAEzE,YAAY,OAAO,CAAC,EAAE,mBAAmB,EAcxC;IAEM,YAAY,IAAI,cAAc,CAEpC;IAEM,qBAAqB,IAAI,mBAAmB,CAElD;IAEM,oBAAoB,IAAI,kBAAkB,CAEhD;IAEM,iBAAiB,IAAI,cAAc,CAEzC;IAEM,kBAAkB,IAAI,eAAe,CAE3C;IAEM,UAAU,IAAI,WAAW,CAE/B;IAEM,eAAe,IAAI,mBAAmB,CAE5C;IAEM,MAAM,CAAC,MAAM,GAAE,MAA8B,GAAG,IAAI,CAM1D;IAEM,eAAe,IAAI,OAAO,CAEhC;IAEM,gBAAgB,IAAI,MAAM,EAAE,CAElC;IAEM,UAAU,IAAI,UAAU,EAAE,CAEhC;IAEM,QAAQ;QAEP,UAAU;QACV,cAAc;QACd,kBAAkB;QAClB,SAAS;QACT,MAAM;QACN,MAAM;MAEb;IAEM,YAAY,IAAI,IAAI,CAY1B;IAED,OAAO,CAAC,iBAAiB;IAiDZ,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAqM1E;IAEY,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAiE9E;YAEa,oBAAoB;YA4LpB,mBAAmB;YA8DnB,gBAAgB;IAgM9B,OAAO,CAAC,mBAAmB;CAqB9B"}
+{"version":3,"file":"agentRuntime.d.ts","sourceRoot":"","sources":["agentRuntime.ts"],"names":[],"mappings":"AAQA,OAAO,EAA0C,KAAK,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAEhG,OAAO,EAAE,cAAc,EAAsB,KAAK,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AAC7G,OAAO,EAAE,KAAK,mBAAmB,EAAsD,MAAM,mCAAmC,CAAC;AAEjI,OAAO,EAAE,KAAK,kBAAkB,EAA6B,MAAM,qBAAqB,CAAC;AACzF,OAAO,EAAE,cAAc,EAAyB,MAAM,sBAAsB,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAkB,MAAM,cAAc,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAIzD,MAAM,WAAW,mBAAmB;IAChC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iBAAiB,CAAC,EAAE,kBAAkB,CAAC;IACvC,kBAAkB,CAAC,EAAE,mBAAmB,CAAC;IACzC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,sBAAsB;IACnC,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,MAAM,EAAE,kBAAkB,CAAC,QAAQ,CAAC,CAAC;CACxC;AAED,qBAAa,YAAY;IACrB,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,WAAW,CAAU;IAC7B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,QAAQ,CAAC,CAAqB;IACtC,OAAO,CAAC,eAAe,CAA0C;IACjE,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,YAAY,CAAC,CAAqB;IAE1C,OAAO,CAAC,SAAS,CAAiB;IAClC,OAAO,CAAC,kBAAkB,CAAsB;IAChD,OAAO,CAAC,iBAAiB,CAAqB;IAC9C,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,OAAO,CAAc;IAC7B,OAAO,CAAC,YAAY,CAAsB;IAE1C,OAAO,CAAC,eAAe,CAAa;IACpC,OAAO,CAAC,aAAa,CAA0B;IAC/C,OAAO,CAAC,OAAO,CAAoB;IACnC,OAAO,CAAC,cAAc,CAAa;IACnC,OAAO,CAAC,WAAW,CAAa;IAChC,OAAO,CAAC,SAAS,CAAsB;IACvC,OAAO,CAAC,gBAAgB,CAA0B;IAClD,OAAO,CAAC,eAAe,CAAkD;IACzE,OAAO,CAAC,eAAe,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,0BAA0B,CAAK;IACvD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAAQ;IAExD,YAAY,OAAO,CAAC,EAAE,mBAAmB,EAmBxC;IAEM,iBAAiB,IAAI,OAAO,CAKlC;IAEM,YAAY,IAAI,cAAc,CAEpC;IAEM,qBAAqB,IAAI,mBAAmB,CAElD;IAEM,oBAAoB,IAAI,kBAAkB,CAEhD;IAEM,iBAAiB,IAAI,cAAc,CAEzC;IAEM,kBAAkB,IAAI,eAAe,CAE3C;IAEM,UAAU,IAAI,WAAW,CAE/B;IAEM,eAAe,IAAI,mBAAmB,CAE5C;IAEM,MAAM,CAAC,MAAM,GAAE,MAA8B,GAAG,IAAI,CAM1D;IAEM,eAAe,IAAI,OAAO,CAEhC;IAEM,gBAAgB,IAAI,MAAM,EAAE,CAElC;IAEM,UAAU,IAAI,UAAU,EAAE,CAEhC;IAEM,QAAQ;QAEP,UAAU;QACV,cAAc;QACd,kBAAkB;QAClB,SAAS;QACT,MAAM;QACN,MAAM;MAEb;IAEM,YAAY,IAAI,IAAI,CAY1B;IAED,OAAO,CAAC,iBAAiB;IA2DZ,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,CAAC,CA4N1E;IAEY,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,gBAAgB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;QAAC,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,CAAC,CAE5J;IAEY,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,gBAAgB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;QAAC,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,CAAC,CAmF7J;YAEa,oBAAoB;IAgVlC;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAU9B;;;;OAIG;YACW,2BAA2B;IAgBzC;;;OAGG;IACH,OAAO,CAAC,mBAAmB;YAwBb,mBAAmB;YA8DnB,gBAAgB;IAsO9B,OAAO,CAAC,mBAAmB;CAqB9B"}

package/bin/src/engine/runtime/agentRuntime.js

@@ -17,6 +17,7 @@ import { ApprovalManager, RiskClassifier } from '../safety.js';
 import { TaskPlanner } from '../planner.js';
 import { UnifiedToolRegistry } from '../toolRegistry.js';
 import { robustSnippetReplace, buildDirectoryTree } from '../agentEngine.js';
+import { loadAuthConfig, isProOrHigher } from '../../utils/auth.js';
 export class AgentRuntime {
     cwd;
     autoApprove;
@@ -41,12 +42,20 @@ export class AgentRuntime {
     startTime = Date.now();
     readFilesHistory = new Set();
     toolCallHistory = [];
+    screenshotCount = 0;
+    userTier;
+    static MAX_SCREENSHOTS_IN_HISTORY = 5;
+    static SCREENSHOT_MAX_DIMENSION = 1280;
     constructor(options) {
         this.cwd = options?.workspaceDir || process.cwd();
         this.autoApprove = Boolean(options?.autoApprove);
-        this.maxSteps = options?.maxSteps || 30;
         this.mockMode = Boolean(options?.mockMode);
         this.language = options?.language;
+        this.userTier = (options?.tier || (options?.mockMode ? 'pro' : loadAuthConfig().currentTier) || 'free').toLowerCase();
+        // Free tier is bounded to basic direct actions (max 5 steps). Full autonomous multi-step loops require Pro/Enterprise.
+        const isPaid = this.mockMode || this.userTier === 'pro' || this.userTier === 'enterprise';
+        const requestedSteps = options?.maxSteps || 30;
+        this.maxSteps = isPaid ? requestedSteps : Math.min(requestedSteps, 5);
         this.workspace = new AgentWorkspace(options?.taskId, '', this.cwd);
         this.computerController = options?.computerController || (this.mockMode ? new MockComputerController() : new SystemComputerController(this.cwd));
         this.browserController = options?.browserController || new SemanticBrowserController(this.cwd);
@@ -55,6 +64,12 @@ export class AgentRuntime {
         this.planner = new TaskPlanner();
         this.toolRegistry = new UnifiedToolRegistry();
     }
+    isProOrEnterprise() {
+        if (this.mockMode || (this.computerController && this.computerController.constructor.name === 'MockComputerController')) {
+            return true;
+        }
+        return isProOrHigher(this.userTier);
+    }
     getWorkspace() {
         return this.workspace;
     }
@@ -118,6 +133,13 @@ export class AgentRuntime {
     buildSystemPrompt() {
         const filesList = getDirectoryFiles(this.cwd).slice(0, 100);
         const workspaceState = this.workspace.compactWorkspaceForPrompt();
+        const tierNotice = !this.isProOrEnterprise()
+            ? `\nTIER RESTRICTION (FREE DEVELOPER TIER):
+- You are operating under the Free Developer Tier.
+- Free developers have access to basic developer operations: code editing, creating new files and folders, reading files, and running terminal commands.
+- Scratchpad reasoning and autonomous computer/browser automation tools are locked to Pro & Enterprise plan users.
+- Do NOT attempt to use scratchpad, computer_* or browser_* tools. Complete coding and task requests using direct workspace tools (edit_file, write_file, read_file, run_command, list_dir, grep_search, etc.).`
+            : `\nTIER: PRO/ENTERPRISE (Full Autonomous Computer Control, Browser Automation & AI Scratchpad Unlocked).`;
         return `You are Scout Agent, an elite AI Principal Software Engineer & Autonomous Computer-Use Operator (developed by FrontTerrain).
 You do not merely explain or output chat answers—you autonomously operate the computer, applications, browser, and workspace tools to fulfill the user's objective end-to-end.
 
@@ -128,6 +150,8 @@ Working Directory: "${this.cwd}"
 Host OS Platform: "${process.platform}"
 Current Year: ${new Date().getFullYear()}
 
+${tierNotice}
+
 ${workspaceState}
 
 Available Workspace Files (sample):
@@ -220,12 +244,13 @@ Action Rules:
             stepSpinner.start(chalk.cyan(`Scout Operating (Step ${step}/${this.maxSteps})`));
             try {
                 let response;
+                const tierTools = this.toolRegistry.getToolsForTier(this.userTier);
                 try {
                     response = await callOpenAIWithRetry(async (model) => {
                         return await openai.chat.completions.create({
                             model,
                             messages: sanitizeMessages(this.historyMessages),
-                            tools: this.toolRegistry.getAllTools().map((t) => ({ type: 'function', function: t })),
+                            tools: tierTools.map((t) => ({ type: 'function', function: t })),
                             tool_choice: 'auto',
                             temperature: 0.1,
                         });
@@ -236,7 +261,7 @@ Action Rules:
                     if (errStr.includes('tool') || errStr.includes('400') || errStr.includes('not supported')) {
                         const fallbackMsg = {
                             role: 'user',
-                            content: `Format your tool call as a JSON code block:\n\`\`\`json\n{ "tool": "tool_name", "args": { ... } }\n\`\`\`\nAvailable tools: ${this.toolRegistry.getAllTools().map((t) => t.name).join(', ')}`,
+                            content: `Format your tool call as a JSON code block:\n\`\`\`json\n{ "tool": "tool_name", "args": { ... } }\n\`\`\`\nAvailable tools: ${tierTools.map((t) => t.name).join(', ')}`,
                         };
                         response = await callOpenAIWithRetry(async (model) => {
                             return await openai.chat.completions.create({
@@ -288,11 +313,33 @@ Action Rules:
                     if (this.isCancelled)
                         break;
                     const toolResult = await this.dispatchTool(tc.name, tc.args);
-                    this.historyMessages.push({
-                        role: 'user',
-                        tool_call_id: tc.id,
-                        content: `Tool Execution Result (${tc.name}):\n${toolResult.result}`,
-                    });
+                    // Build the tool result message with correct role
+                    if (tc.id) {
+                        // Native tool call: use 'tool' role with matching tool_call_id
+                        this.historyMessages.push({
+                            role: 'tool',
+                            tool_call_id: tc.id,
+                            content: typeof toolResult.result === 'string' ? toolResult.result : JSON.stringify(toolResult.result),
+                        });
+                    }
+                    else {
+                        // JSON-extracted tool call (no tc.id): use 'user' role
+                        this.historyMessages.push({
+                            role: 'user',
+                            content: `Tool Execution Result (${tc.name}):\n${toolResult.result}`,
+                        });
+                    }
+                    // If the tool returned screenshot data, append a vision message so the model can see the screen
+                    if (toolResult.screenshotBase64) {
+                        this.pruneOldScreenshots();
+                        this.historyMessages.push({
+                            role: 'user',
+                            content: [
+                                { type: 'text', text: `[Screenshot after ${tc.name}]: ${toolResult.screenshotSummary || 'Current screen state.'}` },
+                                { type: 'image_url', image_url: { url: `data:image/png;base64,${toolResult.screenshotBase64}`, detail: 'low' } },
+                            ],
+                        });
+                    }
                     if (tc.name === 'task_completed') {
                         finalSummary = tc.args.summary || toolResult.result;
                         if (this.modifiedFiles.size > 0) {
@@ -336,6 +383,9 @@ Action Rules:
             status: 'failed',
         };
     }
+    async executeTool(name, args) {
+        return await this.dispatchTool(name, args);
+    }
     async dispatchTool(name, args) {
         this.totalToolCalls++;
         const risk = this.approvalManager.checkAction(name, args);
@@ -380,17 +430,93 @@ Action Rules:
             };
         }
         // 1. Computer Control Tools
-        if (name.startsWith('computer_')) {
+        if (name.startsWith('computer_') || name === 'computer') {
+            if (!this.isProOrEnterprise()) {
+                const upgradeMsg = `[PLAN RESTRICTION]: Autonomous Computer Control tools (${name}) are exclusively available to Pro and Enterprise plan developers. Free tier developers can perform code editing, creating files/folders, and running terminal commands. Upgrade to Pro at https://tryscout.web.app to unlock autonomous computer & app control.`;
+                this.workspace.addObservation({
+                    tool: name,
+                    resultSummary: upgradeMsg,
+                    verified: false,
+                });
+                return { result: upgradeMsg };
+            }
             return await this.dispatchComputerTool(name, args);
         }
         // 2. Browser Automation Tools
         if (name.startsWith('browser_')) {
+            if (!this.isProOrEnterprise()) {
+                const upgradeMsg = `[PLAN RESTRICTION]: Autonomous Browser Control tools (${name}) are exclusively available to Pro and Enterprise plan developers. Free tier developers can perform code editing, creating files/folders, and running terminal commands. Upgrade to Pro at https://tryscout.web.app to unlock browser automation.`;
+                this.workspace.addObservation({
+                    tool: name,
+                    resultSummary: upgradeMsg,
+                    verified: false,
+                });
+                return { result: upgradeMsg };
+            }
             return await this.dispatchBrowserTool(name, args);
         }
         // 3. Core Direct Workspace Tools
         return await this.dispatchCoreTool(name, args);
     }
     async dispatchComputerTool(name, args) {
+        if (name === 'computer') {
+            const action = String(args.action || 'screenshot').toLowerCase().trim();
+            if (action === 'click' || action === 'left_click') {
+                name = 'computer_click';
+            }
+            else if (action === 'right_click') {
+                name = 'computer_right_click';
+            }
+            else if (action === 'double_click') {
+                name = 'computer_double_click';
+            }
+            else if (action === 'move' || action === 'mouse_move') {
+                name = 'computer_mouse_move';
+            }
+            else if (action === 'mouse_down') {
+                name = 'computer_mouse_down';
+            }
+            else if (action === 'mouse_up') {
+                name = 'computer_mouse_up';
+            }
+            else if (action === 'drag') {
+                name = 'computer_drag';
+                args.fromX = args.startX ?? args.fromX;
+                args.fromY = args.startY ?? args.fromY;
+                args.toX = args.endX ?? args.toX;
+                args.toY = args.endY ?? args.toY;
+            }
+            else if (action === 'scroll') {
+                name = 'computer_scroll';
+            }
+            else if (action === 'screenshot') {
+                name = 'computer_screenshot';
+            }
+            else if (action === 'type') {
+                name = 'computer_type';
+            }
+            else if (action === 'key_press' || action === 'key') {
+                name = 'computer_key_press';
+            }
+            else if (action === 'hotkey') {
+                name = 'computer_hotkey';
+            }
+            else if (action === 'open_app') {
+                name = 'computer_open_app';
+            }
+            else if (action === 'close_app') {
+                name = 'computer_close_app';
+            }
+            else if (action === 'switch_window') {
+                name = 'computer_switch_window';
+            }
+            else if (action === 'inspect_ui') {
+                name = 'computer_inspect_ui';
+            }
+            else if (action === 'diagnostic') {
+                name = 'computer_diagnostic';
+            }
+        }
         switch (name) {
             case 'computer_screenshot': {
                 const res = await this.computerController.screenshot();
@@ -406,18 +532,47 @@ Action Rules:
                 if (res.activeWindow) {
                     this.workspace.setActiveWindow(res.activeWindow);
                 }
-                const summary = `Screen captured: ${res.width}x${res.height}, Active window: "${res.activeWindow || 'Desktop'}"${res.path ? `, Saved: ${res.path}` : ''}`;
+                const summary = `Screen captured: ${res.width}x${res.height}, Active window: "${res.activeWindow || 'Desktop'}"`;
                 this.workspace.addObservation({ tool: name, resultSummary: summary, verified: true });
-                return { result: summary };
+                // Encode the screenshot as base64 for model vision
+                const base64Data = res.path ? this.readScreenshotAsBase64(res.path) : undefined;
+                this.screenshotCount++;
+                return {
+                    result: summary,
+                    screenshotBase64: base64Data,
+                    screenshotSummary: summary,
+                };
             }
             case 'computer_click': {
                 const x = Number(args.x);
                 const y = Number(args.y);
-                const res = await this.computerController.click(x, y, { button: args.button, clicks: 1 });
+                const btn = args.button || 'left';
+                const res = await this.computerController.click(x, y, { button: btn, clicks: 1 });
                 this.workspace.addObservation({ tool: name, resultSummary: res.output, verified: res.success });
                 const structuredRes = {
                     success: res.success,
                     action: 'click',
+                    coordinates: res.coordinates || { x, y },
+                    button: btn,
+                    output: res.output,
+                    ...(res.error ? { error: res.error } : {}),
+                    recoverable: res.recoverable ?? true,
+                    ...(res.activeWindow ? { activeWindow: res.activeWindow } : {}),
+                };
+                // Auto-capture screenshot after click so model sees the result
+                const postSnap = await this.capturePostActionScreenshot('click');
+                return { result: JSON.stringify(structuredRes, null, 2), ...postSnap };
+            }
+            case 'computer_right_click': {
+                const x = Number(args.x);
+                const y = Number(args.y);
+                const res = await this.computerController.rightClick(x, y);
+                this.workspace.addObservation({ tool: name, resultSummary: res.output, verified: res.success });
+                const structuredRes = {
+                    success: res.success,
+                    action: 'right_click',
+                    coordinates: res.coordinates || { x, y },
+                    button: 'right',
                     output: res.output,
                     ...(res.error ? { error: res.error } : {}),
                     recoverable: res.recoverable ?? true,
@@ -433,6 +588,42 @@ Action Rules:
                 const structuredRes = {
                     success: res.success,
                     action: 'double_click',
+                    coordinates: res.coordinates || { x, y },
+                    button: 'left',
+                    output: res.output,
+                    ...(res.error ? { error: res.error } : {}),
+                    recoverable: res.recoverable ?? true,
+                };
+                return { result: JSON.stringify(structuredRes, null, 2) };
+            }
+            case 'computer_mouse_down': {
+                const x = args.x !== undefined ? Number(args.x) : undefined;
+                const y = args.y !== undefined ? Number(args.y) : undefined;
+                const btn = args.button || 'left';
+                const res = await this.computerController.mouseDown(x, y, btn);
+                this.workspace.addObservation({ tool: name, resultSummary: res.output, verified: res.success });
+                const structuredRes = {
+                    success: res.success,
+                    action: 'mouse_down',
+                    coordinates: res.coordinates || (x !== undefined && y !== undefined ? { x, y } : undefined),
+                    button: btn,
+                    output: res.output,
+                    ...(res.error ? { error: res.error } : {}),
+                    recoverable: res.recoverable ?? true,
+                };
+                return { result: JSON.stringify(structuredRes, null, 2) };
+            }
+            case 'computer_mouse_up': {
+                const x = args.x !== undefined ? Number(args.x) : undefined;
+                const y = args.y !== undefined ? Number(args.y) : undefined;
+                const btn = args.button || 'left';
+                const res = await this.computerController.mouseUp(x, y, btn);
+                this.workspace.addObservation({ tool: name, resultSummary: res.output, verified: res.success });
+                const structuredRes = {
+                    success: res.success,
+                    action: 'mouse_up',
+                    coordinates: res.coordinates || (x !== undefined && y !== undefined ? { x, y } : undefined),
+                    button: btn,
                     output: res.output,
                     ...(res.error ? { error: res.error } : {}),
                     recoverable: res.recoverable ?? true,
@@ -452,7 +643,9 @@ Action Rules:
                     ...(res.activeWindow ? { activeWindow: res.activeWindow } : {}),
                     ...(res.expectedWindow ? { expectedWindow: res.expectedWindow } : {}),
                 };
-                return { result: JSON.stringify(structuredRes, null, 2) };
+                // Auto-capture screenshot after typing so model sees the result
+                const postSnap = await this.capturePostActionScreenshot('type');
+                return { result: JSON.stringify(structuredRes, null, 2), ...postSnap };
             }
             case 'computer_key_press': {
                 const key = String(args.key || '');
@@ -479,7 +672,9 @@ Action Rules:
                     recoverable: res.recoverable ?? true,
                     ...(res.activeWindow ? { activeWindow: res.activeWindow } : {}),
                 };
-                return { result: JSON.stringify(structuredRes, null, 2) };
+                // Auto-capture screenshot after hotkey so model sees the result
+                const postSnap = await this.capturePostActionScreenshot('hotkey');
+                return { result: JSON.stringify(structuredRes, null, 2), ...postSnap };
             }
             case 'computer_scroll': {
                 const deltaY = Number(args.deltaY || 0);
@@ -491,14 +686,37 @@ Action Rules:
             case 'computer_mouse_move': {
                 const x = Number(args.x);
                 const y = Number(args.y);
-                const res = await this.computerController.mouseMove(x, y);
+                const res = await this.computerController.move(x, y);
                 this.workspace.addObservation({ tool: name, resultSummary: res.output, verified: res.success });
-                return { result: res.output };
+                const structuredRes = {
+                    success: res.success,
+                    action: 'move',
+                    coordinates: res.coordinates || { x, y },
+                    output: res.output,
+                    ...(res.error ? { error: res.error } : {}),
+                    recoverable: res.recoverable ?? true,
+                    ...(res.activeWindow ? { activeWindow: res.activeWindow } : {}),
+                };
+                return { result: JSON.stringify(structuredRes, null, 2) };
             }
             case 'computer_drag': {
-                const res = await this.computerController.drag(Number(args.fromX), Number(args.fromY), Number(args.toX), Number(args.toY));
+                const fromX = Number(args.fromX ?? args.startX ?? 0);
+                const fromY = Number(args.fromY ?? args.startY ?? 0);
+                const toX = Number(args.toX ?? args.endX ?? 0);
+                const toY = Number(args.toY ?? args.endY ?? 0);
+                const res = await this.computerController.drag(fromX, fromY, toX, toY);
                 this.workspace.addObservation({ tool: name, resultSummary: res.output, verified: res.success });
-                return { result: res.output };
+                const structuredRes = {
+                    success: res.success,
+                    action: 'drag',
+                    from: { x: fromX, y: fromY },
+                    to: { x: toX, y: toY },
+                    coordinates: res.coordinates || { x: toX, y: toY },
+                    output: res.output,
+                    ...(res.error ? { error: res.error } : {}),
+                    recoverable: res.recoverable ?? true,
+                };
+                return { result: JSON.stringify(structuredRes, null, 2) };
             }
             case 'computer_wait': {
                 const ms = Math.min(10000, Math.max(100, Number(args.ms) || 1000));
@@ -519,7 +737,9 @@ Action Rules:
                     activeWindow: res.activeWindow || appName,
                     expectedWindow: res.expectedWindow || appName,
                 };
-                return { result: JSON.stringify(structuredRes, null, 2) };
+                // Auto-capture screenshot after opening app so model sees the window
+                const postSnap = await this.capturePostActionScreenshot('open_app');
+                return { result: JSON.stringify(structuredRes, null, 2), ...postSnap };
             }
             case 'computer_close_app': {
                 const appName = String(args.appName || args.app || '');
@@ -543,7 +763,9 @@ Action Rules:
                     activeWindow: res.activeWindow || targetWin,
                     expectedWindow: res.expectedWindow,
                 };
-                return { result: JSON.stringify(structuredRes, null, 2) };
+                // Auto-capture screenshot after switching so model sees the new window
+                const postSnap = await this.capturePostActionScreenshot('switch_window');
+                return { result: JSON.stringify(structuredRes, null, 2), ...postSnap };
             }
             case 'computer_inspect_ui': {
                 const ui = await this.computerController.inspectUI();
@@ -559,10 +781,80 @@ Action Rules:
                 };
                 return { result: JSON.stringify(structuredRes, null, 2) };
             }
+            case 'computer_diagnostic': {
+                const diag = await this.computerController.runDiagnostic();
+                return { result: JSON.stringify(diag, null, 2) };
+            }
             default:
                 return { result: `Unknown computer tool: "${name}"` };
         }
     }
+    /**
+     * Read a screenshot file from disk, resize it to save tokens, and return as base64.
+     * Returns undefined if the file cannot be read.
+     */
+    readScreenshotAsBase64(screenshotPath) {
+        try {
+            if (!fs.existsSync(screenshotPath))
+                return undefined;
+            const raw = fs.readFileSync(screenshotPath);
+            return raw.toString('base64');
+        }
+        catch {
+            return undefined;
+        }
+    }
+    /**
+     * Capture a post-action screenshot automatically.
+     * This gives the model vision of what happened after a screen-modifying action.
+     * Includes a brief delay to let the UI settle.
+     */
+    async capturePostActionScreenshot(actionName) {
+        try {
+            // Brief delay to let the UI settle after the action
+            await this.computerController.wait(350);
+            const snap = await this.computerController.screenshot();
+            if (!snap.path)
+                return {};
+            const base64Data = this.readScreenshotAsBase64(snap.path);
+            if (!base64Data)
+                return {};
+            this.screenshotCount++;
+            const summary = `Post-action screenshot (after ${actionName}): ${snap.width}x${snap.height}, Active window: "${snap.activeWindow || 'Desktop'}"`;
+            return { screenshotBase64: base64Data, screenshotSummary: summary };
+        }
+        catch {
+            return {};
+        }
+    }
+    /**
+     * Remove old screenshot vision messages from history to prevent context overflow.
+     * Keeps only the most recent MAX_SCREENSHOTS_IN_HISTORY screenshots.
+     */
+    pruneOldScreenshots() {
+        const maxKeep = AgentRuntime.MAX_SCREENSHOTS_IN_HISTORY;
+        // Find all vision messages (those with image_url content)
+        const visionIndices = [];
+        for (let i = 0; i < this.historyMessages.length; i++) {
+            const msg = this.historyMessages[i];
+            if (Array.isArray(msg.content)) {
+                const hasImage = msg.content.some((part) => part.type === 'image_url');
+                if (hasImage)
+                    visionIndices.push(i);
+            }
+        }
+        // If we have more than maxKeep, remove the oldest ones
+        if (visionIndices.length >= maxKeep) {
+            const toRemove = visionIndices.slice(0, visionIndices.length - maxKeep + 1);
+            // Remove in reverse order to preserve indices
+            for (let i = toRemove.length - 1; i >= 0; i--) {
+                const idx = toRemove[i];
+                if (idx !== undefined) {
+                    this.historyMessages.splice(idx, 1);
+                }
+            }
+        }
+    }
     async dispatchBrowserTool(name, args) {
         switch (name) {
             case 'browser_navigate': {
@@ -620,6 +912,45 @@ Action Rules:
     }
     async dispatchCoreTool(name, args) {
         switch (name) {
+            case 'scratchpad': {
+                if (!this.isProOrEnterprise()) {
+                    const upgradeMsg = `[PLAN RESTRICTION]: AI Scratchpad & Autonomous reasoning is an exclusive feature for Pro and Enterprise plan subscribers. Free tier developers can perform code editing, creating files/folders, and running commands. Upgrade to Pro at https://tryscout.web.app to unlock the AI Scratchpad.`;
+                    this.workspace.addObservation({
+                        tool: name,
+                        resultSummary: upgradeMsg,
+                        verified: false,
+                    });
+                    return { result: upgradeMsg };
+                }
+                const action = String(args.action || 'read').toLowerCase();
+                const scratchpadPath = path.resolve(this.cwd, '.ft', 'scratchpad.md');
+                fs.mkdirSync(path.dirname(scratchpadPath), { recursive: true });
+                if (action === 'write') {
+                    const content = String(args.content || '');
+                    fs.writeFileSync(scratchpadPath, content, 'utf-8');
+                    this.workspace.addObservation({ tool: name, resultSummary: `Scratchpad updated (${content.length} chars)`, verified: true });
+                    return { result: `Scratchpad updated successfully.` };
+                }
+                else if (action === 'append') {
+                    const content = String(args.content || '');
+                    fs.appendFileSync(scratchpadPath, `\n${content}`, 'utf-8');
+                    this.workspace.addObservation({ tool: name, resultSummary: `Appended to scratchpad`, verified: true });
+                    return { result: `Appended notes to scratchpad.` };
+                }
+                else if (action === 'clear') {
+                    fs.writeFileSync(scratchpadPath, '', 'utf-8');
+                    this.workspace.addObservation({ tool: name, resultSummary: `Cleared scratchpad`, verified: true });
+                    return { result: `Scratchpad cleared.` };
+                }
+                else {
+                    // read
+                    if (!fs.existsSync(scratchpadPath)) {
+                        return { result: `Scratchpad is currently empty.` };
+                    }
+                    const content = fs.readFileSync(scratchpadPath, 'utf-8');
+                    return { result: `Scratchpad content:\n${content || '(empty)'}` };
+                }
+            }
             case 'read_file': {
                 const targetPath = String(args.path || '').trim();
                 const fullPath = path.isAbsolute(targetPath) ? targetPath : path.resolve(this.cwd, targetPath);