ft-scout 9.0.2 → 9.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/.firebase/hosting.d2Vi.cache +7 -7
  2. package/README.md +76 -83
  3. package/bin/src/commands/agent.d.ts.map +1 -1
  4. package/bin/src/commands/agent.js +13 -223
  5. package/bin/src/commands/agent.js.map +1 -1
  6. package/bin/src/commands/plan.d.ts.map +1 -1
  7. package/bin/src/commands/plan.js +45 -18
  8. package/bin/src/commands/plan.js.map +1 -1
  9. package/bin/src/engine/actionVerifier.d.ts.map +1 -1
  10. package/bin/src/engine/actionVerifier.js +61 -9
  11. package/bin/src/engine/actionVerifier.js.map +1 -1
  12. package/bin/src/engine/agentEngine.d.ts.map +1 -1
  13. package/bin/src/engine/agentEngine.js +459 -547
  14. package/bin/src/engine/agentEngine.js.map +1 -1
  15. package/bin/src/engine/agentState.d.ts.map +1 -1
  16. package/bin/src/engine/agentState.js +0 -64
  17. package/bin/src/engine/agentState.js.map +1 -1
  18. package/bin/src/engine/browser/browserController.d.ts.map +1 -1
  19. package/bin/src/engine/browser/browserController.js +8 -0
  20. package/bin/src/engine/browser/browserController.js.map +1 -1
  21. package/bin/src/engine/computer/computerController.d.ts.map +1 -1
  22. package/bin/src/engine/computer/computerController.js +479 -269
  23. package/bin/src/engine/computer/computerController.js.map +1 -1
  24. package/bin/src/engine/computer/index.d.ts.map +1 -1
  25. package/bin/src/engine/computer/index.js +1 -0
  26. package/bin/src/engine/computer/index.js.map +1 -1
  27. package/bin/src/engine/computer/mockComputer.d.ts.map +1 -0
  28. package/bin/src/engine/computer/mockComputer.js +164 -0
  29. package/bin/src/engine/computer/mockComputer.js.map +1 -0
  30. package/bin/src/engine/runtime/agentRuntime.d.ts.map +1 -0
  31. package/bin/src/engine/runtime/agentRuntime.js +829 -0
  32. package/bin/src/engine/runtime/agentRuntime.js.map +1 -0
  33. package/bin/src/engine/runtime/index.d.ts.map +1 -0
  34. package/bin/src/engine/runtime/index.js +2 -0
  35. package/bin/src/engine/runtime/index.js.map +1 -0
  36. package/bin/src/engine/safety.d.ts.map +1 -1
  37. package/bin/src/engine/safety.js +49 -25
  38. package/bin/src/engine/safety.js.map +1 -1
  39. package/bin/src/engine/toolRegistry.d.ts.map +1 -1
  40. package/bin/src/engine/toolRegistry.js +162 -81
  41. package/bin/src/engine/toolRegistry.js.map +1 -1
  42. package/bin/src/engine/voiceEngine.d.ts.map +1 -1
  43. package/bin/src/engine/voiceEngine.js +0 -1
  44. package/bin/src/engine/voiceEngine.js.map +1 -1
  45. package/bin/src/engine/workspace/agentWorkspace.d.ts.map +1 -0
  46. package/bin/src/engine/workspace/agentWorkspace.js +484 -0
  47. package/bin/src/engine/workspace/agentWorkspace.js.map +1 -0
  48. package/bin/src/engine/workspace/index.d.ts.map +1 -0
  49. package/bin/src/engine/workspace/index.js +2 -0
  50. package/bin/src/engine/workspace/index.js.map +1 -0
  51. package/bin/src/index.js +4 -5
  52. package/bin/src/index.js.map +1 -1
  53. package/bin/src/server/server.d.ts.map +1 -1
  54. package/bin/src/server/server.js +25 -4
  55. package/bin/src/server/server.js.map +1 -1
  56. package/bin/src/utils/auth.d.ts.map +1 -1
  57. package/bin/src/utils/auth.js +67 -49
  58. package/bin/src/utils/auth.js.map +1 -1
  59. package/bin/src/utils/branding.js +1 -1
  60. package/bin/src/utils/payment.d.ts.map +1 -1
  61. package/bin/src/utils/payment.js +15 -1
  62. package/bin/src/utils/payment.js.map +1 -1
  63. package/bin/src/utils/pricing.js +1 -1
  64. package/bin/src/utils/pricing.js.map +1 -1
  65. package/bin/src/utils/razorpay.d.ts.map +1 -1
  66. package/bin/src/utils/razorpay.js +10 -2
  67. package/bin/src/utils/razorpay.js.map +1 -1
  68. package/downloads/invoice-2026-03.pdf +1 -0
  69. package/package.json +1 -1
  70. package/web/app.js +54 -135
  71. package/web/styles.css +2 -2
  72. package/bin/src/engine/appControl.d.ts.map +0 -1
  73. package/bin/src/engine/appControl.js +0 -2320
  74. package/bin/src/engine/appControl.js.map +0 -1
  75. package/bin/src/engine/liveScreenEngine.d.ts.map +0 -1
  76. package/bin/src/engine/liveScreenEngine.js +0 -421
  77. package/bin/src/engine/liveScreenEngine.js.map +0 -1
  78. package/bin/src/engine/scripts/winCapture.ps1 +0 -248
  79. package/downloads/INVOICE_2026_ACME.pdf +0 -1
@@ -9,16 +9,17 @@ import { saveAgentSession, getRecentSessionsSummary } from '../utils/session.js'
9
9
  import { safeNote, renderMarkdown } from '../utils/markdown.js';
10
10
  import { openai, callOpenAIWithRetry, isQuotaExceededError, sanitizeMessage, sanitizeMessages } from './llm.js';
11
11
  import { checkFileSyntax, verifyAndSelfHealFiles, executeSmartCommand, extractErrorDiagnostics } from './verifier.js';
12
- import { openApp, executeInApp, cleanupScreenshots } from './appControl.js';
13
12
  import { speakText, listenSpeechToText } from './voiceEngine.js';
14
- import { startLiveScreenShare, stopLiveScreenShare, isLiveScreenShareActive } from './liveScreenEngine.js';
15
13
  import { TaskStateManager } from './agentState.js';
16
- import { OSComputerController } from './computer/index.js';
17
14
  import { SemanticBrowserController } from './browser/index.js';
18
15
  import { ActionVerifier } from './actionVerifier.js';
19
16
  import { ApprovalManager, RiskClassifier } from './safety.js';
20
17
  import { TaskPlanner } from './planner.js';
21
18
  import { UnifiedToolRegistry } from './toolRegistry.js';
19
+ import { SystemComputerController, MockComputerController } from './computer/index.js';
20
+ import { AgentWorkspace, inferExecutionIntent, resolveDesktopPath } from './workspace/index.js';
21
+ import { AgentRuntime } from './runtime/index.js';
22
+ export { AgentRuntime, AgentWorkspace, inferExecutionIntent, resolveDesktopPath, SystemComputerController, MockComputerController };
22
23
  const execAsync = promisify(exec);
23
24
  export function robustSnippetReplace(origContent, target, replacement) {
24
25
  if (!origContent || !target) {
@@ -277,54 +278,6 @@ export const AGENT_TOOLS = [
277
278
  required: ['question'],
278
279
  },
279
280
  },
280
- {
281
- name: 'open_app',
282
- description: 'Open, launch, or focus an external application (browser, terminal, editor, VS Code, Notepad, or custom app) as requested by user prompt.',
283
- parameters: {
284
- type: 'object',
285
- properties: {
286
- app: { type: 'string', description: 'Application name or executable (e.g. "browser", "chrome", "edge", "terminal", "powershell", "code", "vscode", "notepad", or path)' },
287
- target: { type: 'string', description: 'Optional target URL (for browser), file/folder path (for editor), or initial command (for terminal)' },
288
- reason: { type: 'string', description: 'Explanation of why this app is being launched and taken over' },
289
- },
290
- required: ['app'],
291
- },
292
- },
293
- {
294
- name: 'app_action',
295
- description: 'Execute interactive desktop/browser scratchpad automation actions, live screen sharing with model, UI element clicking, screen takeover, mouse cursor positioning & click takeover, mouse scrolling, window management, keyboard keystrokes, hotkeys, focus locks, or command sequences without shell commands (e.g. action: "live_screen_share", "start_screen_share", "stop_screen_share", "live_screen_stream", "see_screen", "analyze_screen", "capture_screen", "click_element", "click_app", "scroll", "list_windows", "focus_window", "takeover", "move_mouse", "type_text", "send_keys", "key_combo", "lock_app", "fetch_page", "navigate", "search", "send_dm", "exec_command", "open_file"). All screen frames are streamed in-memory directly to the model with zero temporary files.',
296
- parameters: {
297
- type: 'object',
298
- properties: {
299
- app: { type: 'string', description: 'Target application name, window title, screen, or category ("browser", "terminal", "editor", "notepad", "chrome", "desktop")' },
300
- action: { type: 'string', description: 'Action type ("live_screen_share", "start_screen_share", "stop_screen_share", "live_screen_stream", "see_screen", "analyze_screen", "capture_screen", "click_element", "click_app", "scroll", "list_windows", "focus_window", "takeover", "send_mail", "type_text", "move_mouse", "send_keys", "key_combo", "lock_app", "fetch_page", "search", "send_dm", "exec_command", "open_file")' },
301
- payload: { description: 'Action details/payload object or string (e.g. element name/description to click { element: "Search Google" }, coordinates { x: 100, y: 200 }, scroll { direction: "down", amount: 4 }, text string, { text: "...", enter: true }, { keyCombo: "ctrl+v" }, { duration: 3000 }, URL, search query, command, or file path)' },
302
- },
303
- required: ['app', 'action'],
304
- },
305
- },
306
- {
307
- name: 'agent_scratchpad',
308
- description: 'Update, read, or clear the agent working memory scratchpad, step checklist, and working notes for multi-step goals.',
309
- parameters: {
310
- type: 'object',
311
- properties: {
312
- action: { type: 'string', description: 'Action type ("update", "read", "clear")' },
313
- plan: {
314
- type: 'array',
315
- description: 'List of step items in the goal execution plan',
316
- items: { type: 'string' },
317
- },
318
- completedSteps: {
319
- type: 'array',
320
- description: 'Indices of steps that are completed (0-based numbers)',
321
- items: { type: 'number' },
322
- },
323
- notes: { type: 'string', description: 'Working memory notes, findings, or scratchpad text' },
324
- },
325
- required: ['action'],
326
- },
327
- },
328
281
  {
329
282
  name: 'speak_text',
330
283
  description: 'Speak a text phrase out loud using Text-to-Speech (TTS) voice synthesis for voice feedback or audio summaries.',
@@ -400,7 +353,6 @@ export class AgentExecutionLoop {
400
353
  backups = [];
401
354
  autoApprove;
402
355
  maxSteps;
403
- projectName;
404
356
  startTime = Date.now();
405
357
  totalToolCalls = 0;
406
358
  currentStep = 0;
@@ -409,48 +361,32 @@ export class AgentExecutionLoop {
409
361
  consecutiveCommands = 0;
410
362
  consecutiveScreenChecks = 0;
411
363
  toolCallHistory = [];
412
- scratchpadState = { plan: [], completedSteps: [], notes: '' };
413
364
  taskStateManager;
414
- computerController;
415
365
  browserController;
366
+ computerController;
416
367
  actionVerifier;
417
368
  approvalManager;
418
369
  taskPlanner;
419
370
  toolRegistry;
420
371
  mockMode;
372
+ isCancelled = false;
373
+ cancelReason;
374
+ executionIntent;
421
375
  constructor(options) {
422
376
  this.cwd = process.cwd();
423
377
  this.autoApprove = Boolean(options?.autoApprove);
424
378
  this.maxSteps = options?.maxSteps || 30;
425
- this.projectName = options?.projectName || path.basename(this.cwd);
426
379
  this.mockMode = Boolean(options?.mockMode);
427
380
  this.taskStateManager = new TaskStateManager(options?.taskId, '', this.cwd);
428
- this.computerController = options?.computerController || new OSComputerController();
381
+ this.computerController = options?.computerController || (this.mockMode ? new MockComputerController() : new SystemComputerController(this.cwd));
429
382
  this.browserController = options?.browserController || new SemanticBrowserController(this.cwd);
430
383
  this.actionVerifier = new ActionVerifier(this.cwd);
431
384
  this.approvalManager = new ApprovalManager(this.autoApprove);
432
385
  this.taskPlanner = new TaskPlanner();
433
386
  this.toolRegistry = new UnifiedToolRegistry();
434
- if (options?.screenShare) {
435
- startLiveScreenShare().catch(() => { });
436
- }
437
- // Ensure temporary screenshot cleanup and screen share stop on early exit or interrupt
438
- const onExitOrInterrupt = () => {
439
- try {
440
- cleanupScreenshots();
441
- }
442
- catch { }
443
- try {
444
- if (isLiveScreenShareActive())
445
- stopLiveScreenShare();
446
- }
447
- catch { }
448
- };
449
- process.once('exit', onExitOrInterrupt);
450
- process.once('SIGINT', () => {
451
- onExitOrInterrupt();
452
- process.exit(0);
453
- });
387
+ }
388
+ getExecutionIntent() {
389
+ return this.executionIntent;
454
390
  }
455
391
  getTaskStateManager() {
456
392
  return this.taskStateManager;
@@ -458,12 +394,12 @@ export class AgentExecutionLoop {
458
394
  getStateManager() {
459
395
  return this.taskStateManager;
460
396
  }
461
- getComputerController() {
462
- return this.computerController;
463
- }
464
397
  getBrowserController() {
465
398
  return this.browserController;
466
399
  }
400
+ getComputerController() {
401
+ return this.computerController;
402
+ }
467
403
  getActionVerifier() {
468
404
  return this.actionVerifier;
469
405
  }
@@ -476,6 +412,15 @@ export class AgentExecutionLoop {
476
412
  getToolRegistry() {
477
413
  return this.toolRegistry;
478
414
  }
415
+ cancel(reason = 'User cancelled task') {
416
+ this.isCancelled = true;
417
+ this.cancelReason = reason;
418
+ this.taskStateManager.setStatus('failed');
419
+ this.taskStateManager.recordError(`Task execution cancelled: ${reason}`, 'runtime');
420
+ }
421
+ isTaskCancelled() {
422
+ return this.isCancelled;
423
+ }
479
424
  async resumeGoal(taskId) {
480
425
  const saved = TaskStateManager.loadTaskState(taskId, this.cwd);
481
426
  if (!saved) {
@@ -517,92 +462,7 @@ export class AgentExecutionLoop {
517
462
  this.consecutiveCommands = 0;
518
463
  this.consecutiveScreenChecks = 0;
519
464
  this.toolCallHistory = [];
520
- this.scratchpadState = { plan: [], completedSteps: [], notes: '' };
521
- }
522
- updateScratchpad(plan, completedSteps, notes) {
523
- if (plan !== undefined)
524
- this.scratchpadState.plan = plan;
525
- if (completedSteps !== undefined)
526
- this.scratchpadState.completedSteps = completedSteps;
527
- if (notes !== undefined)
528
- this.scratchpadState.notes = notes;
529
- try {
530
- const ftDir = path.join(this.cwd, '.ft');
531
- if (!fs.existsSync(ftDir)) {
532
- fs.mkdirSync(ftDir, { recursive: true });
533
- }
534
- const scratchpadPath = path.join(ftDir, 'scratchpad.md');
535
- let mdContent = `# Scout Agent Scratchpad\n\n`;
536
- if (this.scratchpadState.plan.length > 0) {
537
- mdContent += `## Plan Checklist\n`;
538
- this.scratchpadState.plan.forEach((item, idx) => {
539
- const isDone = this.scratchpadState.completedSteps.includes(idx);
540
- mdContent += `- [${isDone ? 'x' : ' '}] Step ${idx + 1}: ${item}\n`;
541
- });
542
- mdContent += `\n`;
543
- }
544
- if (this.scratchpadState.notes) {
545
- mdContent += `## Working Memory & Notes\n${this.scratchpadState.notes}\n`;
546
- }
547
- fs.writeFileSync(scratchpadPath, mdContent, 'utf-8');
548
- }
549
- catch {
550
- // Ignore write errors to prevent breaking execution
551
- }
552
- let summary = 'Agent Scratchpad updated successfully.\n';
553
- if (this.scratchpadState.plan.length > 0) {
554
- summary += `Plan (${this.scratchpadState.plan.length} items):\n` +
555
- this.scratchpadState.plan.map((item, idx) => ` ${this.scratchpadState.completedSteps.includes(idx) ? '[✓]' : '[ ]'} Step ${idx + 1}: ${item}`).join('\n') + '\n';
556
- }
557
- if (this.scratchpadState.notes) {
558
- summary += `Notes: ${this.scratchpadState.notes}`;
559
- }
560
- return summary;
561
- }
562
- autoSyncScratchpadFromText(text) {
563
- try {
564
- const planLines = text.match(/[-*]\s*\[([ xX✓/])\]\s*(.*)/g);
565
- if (planLines && planLines.length > 0) {
566
- const plan = [];
567
- const completed = [];
568
- planLines.forEach((l, idx) => {
569
- const isDone = l.includes('[x]') || l.includes('[X]') || l.includes('[✓]');
570
- const cleanText = l.replace(/^[-*]\s*\[[ xX✓/]\]\s*/, '').trim();
571
- plan.push(cleanText);
572
- if (isDone)
573
- completed.push(idx);
574
- });
575
- this.scratchpadState.plan = plan;
576
- this.scratchpadState.completedSteps = completed;
577
- }
578
- const obsMatch = text.match(/Observations?:\s*([^\n]+(?:\n[^\n]+)*)/i);
579
- const thoughtMatch = text.match(/Thought:\s*([^\n]+(?:\n[^\n]+)*)/i);
580
- const notes = [
581
- thoughtMatch ? `Thought: ${thoughtMatch[1]?.trim()}` : '',
582
- obsMatch ? `Observations: ${obsMatch[1]?.trim()}` : '',
583
- ].filter(Boolean).join('\n\n');
584
- if (notes) {
585
- this.scratchpadState.notes = notes;
586
- }
587
- const ftDir = path.join(this.cwd, '.ft');
588
- if (!fs.existsSync(ftDir))
589
- fs.mkdirSync(ftDir, { recursive: true });
590
- const scratchpadPath = path.join(ftDir, 'scratchpad.md');
591
- let mdContent = `# Scout Agent Scratchpad & Working Memory\n\n`;
592
- if (this.scratchpadState.plan.length > 0) {
593
- mdContent += `## Plan Checklist\n`;
594
- this.scratchpadState.plan.forEach((item, idx) => {
595
- const isDone = this.scratchpadState.completedSteps.includes(idx);
596
- mdContent += `- [${isDone ? 'x' : ' '}] Step ${idx + 1}: ${item}\n`;
597
- });
598
- mdContent += `\n`;
599
- }
600
- if (this.scratchpadState.notes) {
601
- mdContent += `## Working Memory & Observations\n${this.scratchpadState.notes}\n`;
602
- }
603
- fs.writeFileSync(scratchpadPath, mdContent, 'utf-8');
604
- }
605
- catch { }
465
+ this.executionIntent = undefined;
606
466
  }
607
467
  handleToolConsecutiveTracking(fnName, args, step) {
608
468
  const argsKey = JSON.stringify(args || {});
@@ -661,9 +521,12 @@ export class AgentExecutionLoop {
661
521
  this.consecutiveReads++;
662
522
  this.consecutiveCommands = 0;
663
523
  if (this.consecutiveReads >= 2) {
524
+ const actionHint = this.executionIntent?.requiredInteraction === 'computer'
525
+ ? `Take action in the requested application (${this.executionIntent.requiredApplications.join(', ') || 'Notepad'})! Use computer_type, computer_click, or computer_hotkey now.`
526
+ : `You MUST execute write_file or edit_file on this step to write or modify code and satisfy the user goal.`;
664
527
  this.historyMessages.push({
665
528
  role: 'user',
666
- content: `URGENT ACTION MANDATE (Step ${step}): You have issued ${this.consecutiveReads} exploration tool calls in a row without making any code changes! STOP READING FILES NOW. You MUST execute write_file or edit_file on this step to write or modify code and satisfy the user goal.`,
529
+ content: `URGENT ACTION MANDATE (Step ${step}): You have issued ${this.consecutiveReads} exploration tool calls in a row without making progress! ${actionHint}`,
667
530
  });
668
531
  }
669
532
  }
@@ -671,9 +534,12 @@ export class AgentExecutionLoop {
671
534
  this.consecutiveReads = 0;
672
535
  this.consecutiveCommands++;
673
536
  if (this.consecutiveCommands >= 2) {
537
+ const actionHint = this.executionIntent?.requiredInteraction === 'computer'
538
+ ? `Continue with computer interaction tools (computer_type, computer_hotkey) in the target application.`
539
+ : `Call 'write_file' or 'edit_file' NOW to write the required source code and files directly to disk!`;
674
540
  this.historyMessages.push({
675
541
  role: 'user',
676
- content: `URGENT ACTION MANDATE (Step ${step}): You have executed ${this.consecutiveCommands} shell commands in a row without creating or editing any files! STOP running shell commands. Call 'write_file' or 'edit_file' NOW to write the required source code and files directly to disk!`,
542
+ content: `URGENT ACTION MANDATE (Step ${step}): You have executed ${this.consecutiveCommands} shell commands in a row without progress! ${actionHint}`,
677
543
  });
678
544
  }
679
545
  }
@@ -684,115 +550,65 @@ export class AgentExecutionLoop {
684
550
  }
685
551
  systemPrompt() {
686
552
  const filesList = getDirectoryFiles(this.cwd).slice(0, 100);
687
- return `You are Scout Agent, an elite AI Principal Software Engineer & Autonomous Coding Assistant (built like Antigravity & Claude Code).
553
+ const intentInfo = this.executionIntent ? `
554
+ Task Execution Intent & Constraints:
555
+ • Required Applications: ${this.executionIntent.requiredApplications.length > 0 ? this.executionIntent.requiredApplications.join(', ') : 'None specified'}
556
+ • Required Interaction Type: ${this.executionIntent.requiredInteraction}
557
+ • Allow Alternative Tools (e.g. direct filesystem): ${this.executionIntent.allowAlternativeTools ? 'YES' : 'NO (STRICT)'}
558
+ • Target Destination: ${this.executionIntent.targetDestination || 'Default'}
559
+ • Target Artifact: ${this.executionIntent.targetArtifactName || 'None'}
560
+ ` : '';
561
+ return `You are Scout Agent, an elite AI Principal Software Engineer & Autonomous Computer-Use Operator (developed by FrontTerrain).
688
562
 
689
- Project Name: "${this.projectName}"
690
563
  Working Directory: "${this.cwd}"
691
564
  Host OS Platform: "${process.platform}"
692
565
  Current Date & Time: "${new Date().toISOString()}" (Current Year: ${new Date().getFullYear()})
693
-
566
+ ${intentInfo}
694
567
  Workspace Overview (first 100 files):
695
568
  ${filesList.join('\n')}
696
569
 
697
- Core Directives & Behavioral Guidelines:
698
- 1. REAL-TIME DATE AWARENESS: You are aware of real-time date and time (${new Date().getFullYear()}). Always use the current year (${new Date().getFullYear()}) and date for web searches, documentation, commit messages, and references. NEVER hardcode obsolete past years like 2024.
699
- 2. HIGH AGENCY & AUTONOMY: You act autonomously through step-by-step tool invocation to solve complex coding tasks, debug issues, build features, create projects from scratch, or answer architectural questions.
700
- 3. MULTILINGUAL & INTENT COMPREHENSION: Understand user intent in any language (English, Hinglish like "likho", "bnao", "code karo", "fix karo", "samjha do", Hindi, Spanish, etc.).
701
- 3. AUTONOMOUS TARGET FILE DISCOVERY: You have full autonomy to locate, inspect, and modify target workspace files. When given a goal without an explicit file path, use auto-discovered target files, \`grep_search\`, or \`glob_search\` to locate relevant code before editing.
702
- 4. ACTION-FIRST & DIRECT EXECUTION PRIORITY: When given a goal to fix, edit, modify, or create files, TAKE CODE MODIFICATION ACTIONS (\`write_file\`, \`edit_file\`, or \`run_command\`) IMMEDIATELY within the first 1-3 steps! DO NOT waste multiple steps reading unrelated asset files, sound generators, raw text files, init files, or requirements files unless specifically requested by the user.
703
- 5. PRE-LOADED CONTEXT RULE: If target files or auto-discovered files are already previewed in your context window history, DO NOT call \`read_file\` or \`list_dir\` on them again! Proceed directly to writing/editing code.
704
- 6. COMPLETE FILE CREATION & EDITING: Whenever asked to write, edit, create, implement, or fix code, YOU MUST CREATE / MODIFY ALL NECESSARY FILES required to completely fulfill the prompt (e.g. main source files, component files, styles, configs, tests, documentation). CALL \`write_file\` or \`edit_file\` for EVERY necessary file to save changes directly to disk in the workspace! Always provide a clear \`reason\` argument for why the edit is made.
705
- 7. RUN & VERIFY COMMANDS: Execute shell commands, tests, builds, and formatters using \`run_command\`. If a command or build fails, read the full error log, diagnose the root cause, and auto-fix the failing code immediately.
706
- 8. NO CODE TRUNCATION OR PLACEHOLDERS: Maintain 100% code integrity. NEVER replace existing code or HTML tags with placeholder comments (e.g. \`<!-- remaining content -->\` or \`// TODO: rest of code\`) or stripped skeleton structures. Always output complete, functional code.
707
- 9. REFORMATTING MINIFIED / LARGE FILES: When asked to format, beautify, or un-minify HTML, JS, CSS, or JSON files, DO NOT attempt to rewrite the file content manually using \`write_file\` or \`edit_file\` (which causes token truncation and content loss). ALWAYS use \`run_command\` to execute CLI formatters (e.g. \`npx prettier --write <file>\`, \`npx html-beautify -r <file>\`, \`npx js-beautify <file>\`, or run a quick Node script) to reformat files in-place on disk.
708
- 10. FROM-SCRATCH PROJECT CREATION & AUTOMATED TESTING PROTOCOL: When asked to build/create a project from scratch:
709
- a. Scaffold Architecture: Create configuration/manifest files (\`package.json\`, \`tsconfig.json\`, \`pyproject.toml\`, etc.) and folder structure using \`write_file\` or \`run_command\`.
710
- b. Write Source Implementation: Create all necessary source files with fully functional, complete, production-ready code (no skeleton placeholders or TODOs).
711
- c. Write Automated Tests: Create comprehensive unit/integration test suite files (e.g. \`test/*.test.ts\`, \`tests/test_*.py\`, etc.).
712
- d. Execute Tests & Build Verification: Run test suites and builds via \`run_command\` (e.g. \`npm test\`, \`npx vitest run\`, \`pytest\`, \`cargo test\`, \`go test ./...\`).
713
- e. Self-Heal & Pass: If tests or builds fail, inspect output logs, edit files using \`edit_file\` or \`write_file\` to resolve errors, re-run tests until 100% passing, and only call \`task_completed\` when all checks pass cleanly.
714
- 11. CRITICAL TERMINATION RULE: As soon as you have finished creating/modifying the necessary files, running verification checks, or answering the user request, YOU MUST CALL \`task_completed\` tool immediately with a clear summary of your work.
715
- 12. PRODUCTION-GRADE CODE QUALITY: Every line of code you write or edit MUST be clean, elegant, modular, production-ready, highly readable, self-documenting, formatted, and strictly typed.
716
- 13. HIGH AUTONOMY & NO DOUBLE PROMPTING: NEVER call \`ask_user\` to ask what code to write, what features to implement, or to re-confirm the prompt. Take immediate autonomous action using \`write_file\`, \`edit_file\`, \`run_command\`, \`read_file\`, or \`grep_search\` using best software engineering practices. ONLY call \`ask_user\` if a critical secret credential (API key/password) is strictly required.
717
- 14. EDITING & DEBUGGING MASTERY PROTOCOL:
718
- a. Full Context Reading: When editing or fixing bugs in existing files, inspect full code context. For files under 250 lines, reference the complete file content before modifying.
719
- b. Guaranteed Edit Success: Use \`edit_file\` for precise snippet updates. If \`edit_file\` returns a target snippet mismatch error, IMMEDIATELY call \`write_file\` with the complete corrected file code so the edit is 100% applied without getting stuck!
720
- c. Automated Error Diagnostics: After editing, check compiler/syntax outputs and test runner logs. If an error is detected, auto-heal the code immediately before concluding.
721
- 15. THOUGHT TRANSPARENCY PROTOCOL: In EVERY step before calling any tools or ending a turn, you MUST provide a clear 1-2 sentence explanation in your text response describing your current reasoning, what file/action you are taking, and why you are taking it.
722
- 16. OS-COMPATIBLE SHELL COMMANDS: Always ensure shell commands passed to \`run_command\` are compatible with the host OS ("${process.platform}"). On Windows (\`win32\`), do NOT use Linux-only builtins like \`touch\`, \`rm -rf\`, \`cat\`, or \`ls -la\` directly. Use \`write_file\` to create files, standard \`npm\`/\`npx\`/\`node\`/\`git\`/\`python\` commands, or PowerShell syntax.
723
- 17. STRICT USER INSTRUCTION & DIRECTIVE ALIGNMENT: You MUST strictly follow all exact user instructions, quantitative constraints, file paths, feature requirements, and architectural preferences specified in the user prompt. Execute the request step-by-step with zero deviation or hallucinated shortcuts, ensuring complete implementation within the allowed step limit.
724
- 18. DIRECT FILE CREATION VIA write_file ONLY (NO SHELL COMMANDS): Do NOT use \`run_command\` with \`touch\`, \`mkdir\`, \`echo > file\`, \`cat > file\`, or shell scripts to create files or folders! ALWAYS call \`write_file\` or \`edit_file\` directly to write source code to disk. The \`write_file\` tool automatically creates all necessary parent directories in one step.
725
- 19. EXTERNAL APP TAKEOVER & CONTROL PROTOCOL: When requested by user prompt to open, take over, or work inside external applications (browser, terminal, VS Code, Notepad, social apps like Instagram, WhatsApp, Twitter/X, Telegram, or custom apps):
726
- a. Launch App: Use \`open_app\` to launch or focus the target application with optional URL, file path, or initial script.
727
- b. Social DM & Messaging Automation: For Instagram, WhatsApp, Twitter/X, or Telegram messaging requests (e.g. "open instagram and send message to @user"), immediately invoke \`app_action\` with action "send_dm" or "open_dm" (or \`open_app\`) specifying the target username/phone and message text so the agent automatically opens the direct messaging link in the browser!
728
- c. Email & Gmail Automation: When requested to send or compose an email (e.g. "send mail to frontterrain@gmail.com saying...", "takeover mail.google.com and send mail"):
729
- - Immediately invoke \`app_action\` with \`action: "send_mail"\` (or \`action: "send_dm"\`) specifying the target recipient email and body text. The agent automatically constructs the direct Gmail compose URL (\`https://mail.google.com/mail/?view=cm&fs=1&to=<recipient>&su=<subject>&body=<body>\`) which pre-populates the compose window and dispatches the email via Ctrl+Enter!
730
- - Alternatively, navigate directly to \`https://mail.google.com/mail/?view=cm&fs=1&to=<recipient>&su=...&body=...\` and trigger hotkey \`app_action(action: "key_combo", payload: { keyCombo: "ctrl+enter" })\`. NEVER click blind coordinates like (20, 20) in a web browser!
731
- d. Work Inside App: Use \`app_action\` or \`run_command\` to execute actions inside the app context (e.g. fetching browser page content, running commands inside terminal, searching web, opening files in editor).
732
- e. Universal Gaming & External Application Protocol:
733
- - Desktop Game/App Launch: To open or play ANY installed game or app on the PC (Steam games, Epic Games, Minecraft, Roblox, Discord, Spotify, etc.), call open_app("<game_name>"). The universal OS launcher automatically resolves installed Windows/macOS applications and store packages via system registry and launches them directly!
734
- - Web Game Play: If the game is web-based (e.g. Chess.com, Slither.io, 2048, Poki) or not installed locally, NEVER guess speculative URL subpaths! Use app_action(app: "browser", action: "search", payload: { query: "<game_name> play online official" }) or navigate to the official domain homepage to find the verified play URL.
735
- - Autonomous Gameplay Progression Loop:
736
- 1. Window Focus: Use app_action(action: "focus_window", payload: { app: "<game_name>" }) to bring the window front-and-center.
737
- 2. Screen Observation: Use app_action(action: "see_screen") to inspect the UI, loading screen, or active state.
738
- 3. Menu Traversal: Click menu buttons ("Play", "Start Game", "New Game", "Continue") using app_action(action: "click_element", payload: { element: "Play" }) .
739
- 4. Interactive Controls: Send gameplay controls using app_action(action: "send_keys" / "type_text" / "key_combo") with standard gaming keys (WASD, Arrow keys, Space, Enter, Escape, mouse clicks/drags).
740
- 5. Conclude task with task_completed when the requested gameplay actions or objectives are achieved.
741
- 20. DESKTOP APP TAKEOVER & AUTOMATION PROTOCOL:
742
- a. STEP 1 SCREEN TAKEOVER: Whenever the user goal asks to take over the screen, control the desktop, or automate an external app (e.g. 'take over screen', 'open maps and click', 'take over desktop', 'open notepad'):
743
- - Your VERY FIRST tool call in Step 1 MUST be \`app_action\` with \`action: "takeover"\` (e.g. \`app: "desktop"\` or the target app).
744
- - Calling \`app_action\` with \`action: "takeover"\` immediately activates the full-screen sky-blue aura HUD, displays the warning banner "⚡ Scout is on the screen.", blocks external input interruptions, and takes over the mouse cursor!
745
- - Then immediately proceed to launch/navigate with \`open_app\` or \`app_action\` (\`action: "click_element"\` / \`action: "click_app"\` / \`action: "type_text"\` / \`action: "send_keys"\`).
746
- b. LIVE SCREEN SHARE & VISION STREAM DIRECTIVE: You have direct real-time live screen sharing with the model! Instead of taking, saving, and sending static screenshot files, stream the screen live in real-time or inspect the live screen share stream using \`app_action\` with \`action: "live_screen_share"\`, \`"start_screen_share"\`, \`"live_screen_stream"\`, \`"see_screen"\`, or \`"analyze_screen"\`. Screen frames are streamed in-memory directly to the model (zero temporary files written to disk).
747
- c. VISUAL ELEMENT GROUNDING (click_element): Instead of guessing blind coordinates (x, y), click buttons, inputs, or menus by descriptive label using \`app_action(action: "click_element", payload: { element: "Search" })\`. Vision AI and native OS UI automation will locate the element and click it accurately.
748
- d. WINDOW & SCROLL CONTROLS: Use \`app_action(action: "scroll", payload: { direction: "down", amount: 4 })\` to scroll pages. Use \`app_action(action: "list_windows")\` to see all open windows, and \`app_action(action: "focus_window", payload: { app: "chrome" })\` to bring a window front-and-center.
749
- e. CLI Credential Input Prompt: If an application requires login credentials, passwords, 2FA codes, or secret tokens to proceed, call \`ask_user\` tool with a clear prompt. This presents a secure, interactive input bar directly in the user's running terminal CLI. Once the user enters the secret, take the received input, inject it into the target application window via \`app_action\` (\`action: "type_text"\`).
750
- 21. MANDATORY TAKEOVER TOOL INVOCATION MANDATE: Whenever the user goal requests to open, launch, take over, click, type, or interact with an external app or desktop screen (e.g. 'take over browser and open website', 'open notepad and type', 'take over desktop', 'take over screen'):
751
- YOU MUST CALL \`app_action(action: "takeover")\` IN STEP 1. Then call \`open_app\` and \`app_action\` (\`click_element\`, \`click_app\`, \`type_text\`, \`send_keys\`). DO NOT call \`read_file\` or \`write_file\` for workspace code files when asked to take over external desktop apps! The takeover tool call is MANDATORY for executing the physical takeover.
752
- 22. BROWSER DIRECT URL NAVIGATION MANDATE: When asked to open or navigate to a specific website or web app (e.g. Apple Maps, GitHub, YouTube, etc.), NEVER call Google Search or issue repeated \`app_action: search\` calls with text queries! IMMEDIATELY pass the exact URL (e.g. "https://maps.apple.com") to \`open_app(app: "browser", target: "https://maps.apple.com")\` or \`app_action(app: "browser", action: "navigate", payload: { url: "https://maps.apple.com" })\`. Direct URL navigation must always target the exact site URL directly without putting queries into Google Search!
753
- 23. WORKING MEMORY & STRUCTURED SCRATCHPAD PROTOCOL:
754
- Like Antigravity and leading autonomous agents, maintain disciplined working memory. In EVERY step, start your response with a structured <scratchpad> reasoning block before returning tool calls:
755
- \`\`\`markdown
756
- <scratchpad>
757
- Thought: [1-2 sentences on what you are doing on this turn and why]
758
- Plan:
759
- [x] 1. [Completed step]
760
- [/] 2. [In-progress step]
761
- [ ] 3. [Next upcoming step]
762
- Observations: [What you learned from the last tool result or screen capture]
763
- Next Action: [The exact tool you are invoking now]
764
- </scratchpad>
765
- \`\`\`
766
- This keeps your reasoning crystal-clear, ensures plan progression, and syncs automatically with .ft/scratchpad.md.
570
+ CORE ARCHITECTURAL PRINCIPLES:
571
+ 1. USER INTENT > TOOL CONVENIENCE (CRITICAL MANDATE):
572
+ The easiest tool is NOT necessarily the correct tool. Scout optimizes for completing the user's ACTUAL REQUESTED WORKFLOW, not merely producing something that looks like the final artifact.
573
+ - Distinguish OUTCOME requirements (e.g. "Create a file containing X") from EXECUTION requirements (e.g. "Open Notepad, type X, save it as Y on Desktop").
574
+ - When the user explicitly specifies an application (e.g. Notepad, Excel, Chrome, VS Code) or computer interaction method, computer interaction tools (computer_open_app, computer_screenshot, computer_type, computer_hotkey, computer_switch_window, computer_click) MUST be used.
575
+ - NEVER silently substitute direct filesystem API calls (write_file) when computer/application interaction is required, unless the user explicitly permits alternative tools ("using any method you prefer").
576
+ - NEVER assume the repository root is the user's Desktop! If the user requests saving to Desktop, the file MUST be saved to the actual Desktop through the requested workflow.
767
577
 
768
- ${this.getScratchpadPromptContext()}
578
+ 2. COMPUTER ACTION FAILURE RECOVERY:
579
+ When a computer action fails or encounters a problem:
580
+ DO NOT immediately switch to an unrelated tool (like write_file).
581
+ Instead, follow the bounded computer recovery loop:
582
+ ACTION FAILED ➔ OBSERVE SCREEN (computer_screenshot / computer_inspect_ui) ➔ UNDERSTAND FAILURE (check if application is active/focused) ➔ RETRY / ALTERNATIVE COMPUTER ACTION (switch window, click text area, send hotkey/keystrokes) ➔ OBSERVE ➔ CONTINUE.
583
+ Only after repeated bounded computer recovery attempts fail should you report the failure to the user.
584
+
585
+ 3. ACTIVE APPLICATION TRACKING:
586
+ Before sending keyboard or mouse input (computer_type, computer_click, computer_hotkey), ensure that the expected application (e.g. Notepad) is the active foreground window. If not, use computer_switch_window or computer_open_app to focus it first.
587
+
588
+ 4. REAL-TIME DATE AWARENESS: You are aware of real-time date and time (${new Date().getFullYear()}). Always use the current year (${new Date().getFullYear()}) and date for web searches, documentation, commit messages, and references. NEVER hardcode obsolete past years like 2024.
589
+ 5. HIGH AGENCY & AUTONOMY: You act autonomously through step-by-step tool invocation to solve complex coding tasks, debug issues, build features, create projects from scratch, or operate desktop applications.
590
+ 6. MULTILINGUAL & INTENT COMPREHENSION: Understand user intent in any language (English, Hinglish like "likho", "bnao", "code karo", "fix karo", "samjha do", Hindi, Spanish, etc.).
591
+ 7. FOR CODE & REPO TASKS: When the task is a coding, project refactoring, or repository maintenance task, use direct code modification tools (write_file, edit_file, run_command, read_file, grep_search).
592
+ 8. RUN & VERIFY COMMANDS: Execute shell commands, tests, builds, and formatters using run_command.
593
+ 9. CRITICAL TERMINATION RULE: As soon as you have finished creating/modifying the necessary files, running verification checks, or completing the desktop application workflow, YOU MUST CALL task_completed tool immediately with a clear summary of your work.
594
+ 10. THOUGHT TRANSPARENCY PROTOCOL: In EVERY step before calling any tools or ending a turn, you MUST provide a clear 1-2 sentence explanation in your text response describing your current reasoning, what file/action you are taking, and why you are taking it.
769
595
 
770
596
  When returning tool calls, use standard OpenAI function calling format or JSON tool call payload format:
771
597
  \`\`\`json
772
598
  {
773
- "tool": "tool_name",
774
- "args": { ... }
599
+ "tool": "tool_name",
600
+ "args": { ... }
775
601
  }
776
602
  \`\`\`
777
603
  `.trim();
778
604
  }
779
- getScratchpadPromptContext() {
780
- if (this.scratchpadState.plan.length === 0 && !this.scratchpadState.notes) {
781
- return '';
782
- }
783
- let str = `ACTIVE SCRATCHPAD STATE (.ft/scratchpad.md):\n`;
784
- if (this.scratchpadState.plan.length > 0) {
785
- str += `Plan Checklist:\n` +
786
- this.scratchpadState.plan.map((item, idx) => ` ${this.scratchpadState.completedSteps.includes(idx) ? '[✓]' : '[ ]'} Step ${idx + 1}: ${item}`).join('\n') + '\n';
787
- }
788
- if (this.scratchpadState.notes) {
789
- str += `Working Notes: ${this.scratchpadState.notes}\n`;
790
- }
791
- return str;
792
- }
793
605
  async executeGoal(userGoal) {
794
606
  this.resetContext();
607
+ this.executionIntent = inferExecutionIntent(userGoal);
795
608
  this.taskStateManager.setObjective(userGoal);
609
+ if (this.computerController.setExpectedApplication && this.executionIntent.requiredApplications.length > 0) {
610
+ this.computerController.setExpectedApplication(this.executionIntent.requiredApplications[0] ?? null);
611
+ }
796
612
  const structuredPlan = this.taskPlanner.createPlan(userGoal);
797
613
  this.taskStateManager.setPlan(structuredPlan.steps);
798
614
  this.taskStateManager.setAvailableTools(this.toolRegistry.getAllTools().map((t) => t.name));
@@ -922,233 +738,215 @@ When returning tool calls, use standard OpenAI function calling format or JSON t
922
738
  let step = 0;
923
739
  let finalSummary = '';
924
740
  const stepDurations = [];
925
- try {
926
- while (step < this.maxSteps) {
927
- step++;
928
- this.currentStep = step;
929
- this.taskStateManager.startStep(step);
930
- const stepStartTime = Date.now();
931
- const avgStepMs = stepDurations.length > 0
932
- ? stepDurations.reduce((a, b) => a + b, 0) / stepDurations.length
933
- : 12000;
934
- const remainingSteps = (this.maxSteps - step + 1);
935
- const estSecsLeft = Math.max(5, Math.round((remainingSteps * avgStepMs) / 1000));
936
- const estLeftStr = estSecsLeft >= 60
937
- ? `${Math.floor(estSecsLeft / 60)}m ${estSecsLeft % 60}s`
938
- : `${estSecsLeft}s`;
939
- const stepSpinner = spinner();
940
- stepSpinner.start(chalk.cyan(`Scout is Working (Step ${step}/${this.maxSteps} Est. completion: ~${estLeftStr} left)`));
741
+ while (step < this.maxSteps) {
742
+ if (this.isCancelled) {
743
+ return {
744
+ success: false,
745
+ summary: `Task cancelled: ${this.cancelReason || 'User abort'}`,
746
+ };
747
+ }
748
+ step++;
749
+ this.currentStep = step;
750
+ this.taskStateManager.startStep(step);
751
+ const stepStartTime = Date.now();
752
+ const avgStepMs = stepDurations.length > 0
753
+ ? stepDurations.reduce((a, b) => a + b, 0) / stepDurations.length
754
+ : 12000;
755
+ const remainingSteps = (this.maxSteps - step + 1);
756
+ const estSecsLeft = Math.max(5, Math.round((remainingSteps * avgStepMs) / 1000));
757
+ const estLeftStr = estSecsLeft >= 60
758
+ ? `${Math.floor(estSecsLeft / 60)}m ${estSecsLeft % 60}s`
759
+ : `${estSecsLeft}s`;
760
+ const stepSpinner = spinner();
761
+ stepSpinner.start(chalk.cyan(`Scout is Working (Step ${step}/${this.maxSteps} • Est. completion: ~${estLeftStr} left)`));
762
+ try {
763
+ let response;
941
764
  try {
942
- let response;
943
- try {
765
+ response = await callOpenAIWithRetry(async (model) => {
766
+ return await openai.chat.completions.create({
767
+ model,
768
+ messages: sanitizeMessages(this.historyMessages),
769
+ tools: this.toolRegistry.getAllTools().map((t) => ({ type: 'function', function: t })),
770
+ tool_choice: 'auto',
771
+ temperature: 0.1,
772
+ presence_penalty: 0.1,
773
+ frequency_penalty: 0.1,
774
+ });
775
+ });
776
+ }
777
+ catch (err) {
778
+ const errStr = String(err?.message || err?.error || err || '').toLowerCase();
779
+ if (errStr.includes('tool') || errStr.includes('400') || errStr.includes('not supported') || errStr.includes('reasoning')) {
780
+ // Model doesn't support native function calling — inject tool-call formatting hint
781
+ // so extractJsonToolCall can parse the response as a structured tool invocation
782
+ const toolHintMsg = {
783
+ role: 'user',
784
+ content: `IMPORTANT: This model does not support native function/tool calling. You MUST format your tool invocations as a JSON code block in your response like this:
785
+ \`\`\`json
786
+ { "tool": "tool_name", "args": { ... } }
787
+ \`\`\`
788
+ Available tools: read_file, write_file, edit_file, run_command, list_dir, grep_search, glob_search, tree_view, file_info, multi_edit_file, fetch_url, git_diff, ask_user, speak_text, task_completed.
789
+ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what you would do in plain text—output the JSON tool call directly!`,
790
+ };
791
+ const fallbackMessages = [...this.historyMessages, toolHintMsg];
944
792
  response = await callOpenAIWithRetry(async (model) => {
945
793
  return await openai.chat.completions.create({
946
794
  model,
947
- messages: sanitizeMessages(this.historyMessages),
948
- tools: this.toolRegistry.getAllTools().map((t) => ({ type: 'function', function: t })),
949
- tool_choice: 'auto',
795
+ messages: sanitizeMessages(fallbackMessages),
950
796
  temperature: 0.1,
951
797
  presence_penalty: 0.1,
952
798
  frequency_penalty: 0.1,
953
799
  });
954
800
  });
955
801
  }
956
- catch (err) {
957
- const errStr = String(err?.message || err?.error || err || '').toLowerCase();
958
- if (errStr.includes('tool') || errStr.includes('400') || errStr.includes('not supported') || errStr.includes('reasoning')) {
959
- // Model doesn't support native function calling — inject tool-call formatting hint
960
- // so extractJsonToolCall can parse the response as a structured tool invocation
961
- const toolHintMsg = {
962
- role: 'user',
963
- content: `IMPORTANT: This model does not support native function/tool calling. You MUST format your tool invocations as a JSON code block in your response like this:
964
- \`\`\`json
965
- { "tool": "tool_name", "args": { ... } }
966
- \`\`\`
967
- Available tools: read_file, write_file, edit_file, run_command, list_dir, grep_search, glob_search, tree_view, file_info, multi_edit_file, fetch_url, git_diff, open_app, app_action, agent_scratchpad, ask_user, speak_text, task_completed.
968
- For app takeover/control use: { "tool": "app_action", "args": { "app": "browser", "action": "takeover" } }
969
- For opening apps use: { "tool": "open_app", "args": { "app": "chrome", "target": "https://..." } }
970
- For clicking use: { "tool": "app_action", "args": { "app": "desktop", "action": "click_app", "payload": { "x": 500, "y": 300 } } }
971
- For typing text use: { "tool": "app_action", "args": { "app": "browser", "action": "type_text", "payload": { "text": "...", "enter": true } } }
972
- You MUST output exactly ONE JSON code block per tool call. Do NOT describe what you would do in plain text—output the JSON tool call directly!`,
973
- };
974
- const fallbackMessages = [...this.historyMessages, toolHintMsg];
975
- response = await callOpenAIWithRetry(async (model) => {
976
- return await openai.chat.completions.create({
977
- model,
978
- messages: sanitizeMessages(fallbackMessages),
979
- temperature: 0.1,
980
- presence_penalty: 0.1,
981
- frequency_penalty: 0.1,
982
- });
983
- });
984
- }
985
- else {
986
- throw err;
987
- }
988
- }
989
- const choice = response.choices[0];
990
- if (!choice) {
991
- stepSpinner.stop(chalk.yellow('No response from Scout. Retrying step...'));
992
- continue;
802
+ else {
803
+ throw err;
993
804
  }
994
- const msg = sanitizeMessage(choice.message);
995
- this.historyMessages.push(msg);
996
- // Render assistant's thought/scratchpad reasoning text if provided on this turn
997
- const textContent = msg.content || '';
998
- if (textContent.trim()) {
999
- const scratchMatch = textContent.match(/<scratchpad>([\s\S]*?)<\/scratchpad>/i);
1000
- if (scratchMatch && scratchMatch[1]) {
1001
- const scratchText = scratchMatch[1].trim();
1002
- safeNote(renderMarkdown(scratchText), chalk.cyan.bold(`Scout Scratchpad & Working Memory (Step ${step})`));
1003
- this.autoSyncScratchpadFromText(scratchText);
1004
- const remainingText = textContent.replace(/<scratchpad>[\s\S]*?<\/scratchpad>/i, '').trim();
1005
- if (remainingText) {
1006
- safeNote(renderMarkdown(remainingText), `Scout Thought (Step ${step})`);
805
+ }
806
+ const choice = response.choices[0];
807
+ if (!choice) {
808
+ stepSpinner.stop(chalk.yellow('No response from Scout. Retrying step...'));
809
+ continue;
810
+ }
811
+ const msg = sanitizeMessage(choice.message);
812
+ this.historyMessages.push(msg);
813
+ // Render assistant's thought reasoning text if provided on this turn
814
+ const textContent = msg.content || '';
815
+ if (textContent.trim()) {
816
+ safeNote(renderMarkdown(textContent), `Scout Thought (Step ${step})`);
817
+ }
818
+ // Check native tool calls
819
+ if (msg.tool_calls && msg.tool_calls.length > 0) {
820
+ stepSpinner.stop(chalk.green(`Step ${step}: Scout issued ${msg.tool_calls.length} tool call(s).`));
821
+ for (const tc of msg.tool_calls) {
822
+ if (tc.type === 'function' && tc.function) {
823
+ const fnName = tc.function.name;
824
+ let args = {};
825
+ try {
826
+ args = JSON.parse(tc.function.arguments || '{}');
1007
827
  }
1008
- }
1009
- else {
1010
- safeNote(renderMarkdown(textContent), `Scout Thought (Step ${step})`);
1011
- }
1012
- }
1013
- // Check native tool calls
1014
- if (msg.tool_calls && msg.tool_calls.length > 0) {
1015
- stepSpinner.stop(chalk.green(`Step ${step}: Scout issued ${msg.tool_calls.length} tool call(s).`));
1016
- for (const tc of msg.tool_calls) {
1017
- if (tc.type === 'function' && tc.function) {
1018
- const fnName = tc.function.name;
1019
- let args = {};
1020
- try {
1021
- args = JSON.parse(tc.function.arguments || '{}');
1022
- }
1023
- catch { }
1024
- const toolResult = await this.dispatchToolCall(fnName, args);
1025
- this.historyMessages.push({
1026
- role: 'user',
1027
- tool_call_id: tc.id,
1028
- content: `Tool Execution Result (${fnName}):\n${toolResult.result}`,
1029
- });
1030
- this.handleToolConsecutiveTracking(fnName, args, step);
1031
- if (fnName === 'task_completed') {
1032
- finalSummary = args.summary || toolResult.result;
1033
- if (this.modifiedFiles.size > 0) {
1034
- const filesList = getDirectoryFiles(this.cwd);
1035
- const healRes = await verifyAndSelfHealFiles(Array.from(this.modifiedFiles), this.cwd, this.projectName, filesList, { maxRetries: 3 });
1036
- if (healRes.verifiedFiles.length > 0) {
1037
- safeNote(chalk.green(` Self-Healing Verification Confirmed: ${healRes.verifiedFiles.length} file(s) syntax & build clean!`), ' Code Verification Clean');
1038
- }
1039
- if (healRes.remainingErrors.length > 0) {
1040
- safeNote(chalk.yellow(`️ Remaining verification issues:\n${healRes.remainingErrors.join('\n')}`), '️ Verification Warning');
1041
- }
828
+ catch { }
829
+ const toolResult = await this.dispatchToolCall(fnName, args);
830
+ this.historyMessages.push({
831
+ role: 'user',
832
+ tool_call_id: tc.id,
833
+ content: `Tool Execution Result (${fnName}):\n${toolResult.result}`,
834
+ });
835
+ this.handleToolConsecutiveTracking(fnName, args, step);
836
+ if (fnName === 'task_completed') {
837
+ finalSummary = args.summary || toolResult.result;
838
+ if (this.modifiedFiles.size > 0) {
839
+ const filesList = getDirectoryFiles(this.cwd);
840
+ const healRes = await verifyAndSelfHealFiles(Array.from(this.modifiedFiles), this.cwd, path.basename(this.cwd), filesList, { maxRetries: 3 });
841
+ if (healRes.verifiedFiles.length > 0) {
842
+ safeNote(chalk.green(` Self-Healing Verification Confirmed: ${healRes.verifiedFiles.length} file(s) syntax & build clean!`), ' Code Verification Clean');
843
+ }
844
+ if (healRes.remainingErrors.length > 0) {
845
+ safeNote(chalk.yellow(`️ Remaining verification issues:\n${healRes.remainingErrors.join('\n')}`), '️ Verification Warning');
1042
846
  }
1043
- saveAgentSession({
1044
- goal: userGoal,
1045
- summary: finalSummary,
1046
- modifiedFiles: Array.from(this.modifiedFiles),
1047
- });
1048
- return { success: true, summary: finalSummary };
1049
847
  }
848
+ saveAgentSession({
849
+ goal: userGoal,
850
+ summary: finalSummary,
851
+ modifiedFiles: Array.from(this.modifiedFiles),
852
+ });
853
+ return { success: true, summary: finalSummary };
1050
854
  }
1051
855
  }
1052
- continue;
1053
- }
1054
- // Check text response or fallback JSON tool call
1055
- stepSpinner.stop(chalk.blue(`Step ${step} thinking complete.`));
1056
- const parsedJsonTool = this.extractJsonToolCall(textContent);
1057
- if (parsedJsonTool) {
1058
- const toolResult = await this.dispatchToolCall(parsedJsonTool.tool, parsedJsonTool.args);
1059
- this.historyMessages.push({
1060
- role: 'user',
1061
- content: `Tool Execution Result (${parsedJsonTool.tool}):\n${toolResult.result}`,
1062
- });
1063
- this.handleToolConsecutiveTracking(parsedJsonTool.tool, parsedJsonTool.args, step);
1064
- if (parsedJsonTool.tool === 'task_completed') {
1065
- finalSummary = parsedJsonTool.args?.summary || toolResult.result;
1066
- saveAgentSession({
1067
- goal: userGoal,
1068
- summary: finalSummary,
1069
- modifiedFiles: Array.from(this.modifiedFiles),
1070
- });
1071
- return { success: true, summary: finalSummary };
1072
- }
1073
- continue;
1074
856
  }
1075
- if (textContent.trim()) {
1076
- let cleanedThought = textContent
1077
- .replace(/[\u0600-\u06FF\u0750-\u077F\uAC00-\uD7AF\u3040-\u30FF\u4E00-\u9FFF\u0D80-\u0DFF]+/g, '')
1078
- .trim();
1079
- if (!cleanedThought || cleanedThought.length < 5) {
1080
- cleanedThought = 'Analyzing codebase files and executing next tool operation...';
1081
- }
1082
- safeNote(renderMarkdown(cleanedThought), ` Scout Agent Thought (Step ${step})`);
1083
- const lowerText = textContent.toLowerCase();
1084
- const isExplicitCompletion = lowerText.includes('task is complete') ||
1085
- lowerText.includes('task complete') ||
1086
- lowerText.includes('goal completed') ||
1087
- lowerText.includes('goal is completed') ||
1088
- lowerText.includes('all tasks completed') ||
1089
- lowerText.includes('i have completed') ||
1090
- lowerText.includes('no further changes needed') ||
1091
- lowerText.includes('the fix is complete') ||
1092
- lowerText.includes('has been created') ||
1093
- lowerText.includes('successfully created') ||
1094
- lowerText.includes('created the file') ||
1095
- lowerText.includes('file created') ||
1096
- lowerText.includes('implementation complete') ||
1097
- lowerText.includes('work is complete');
1098
- // If explicit completion phrase found, OR files have already been modified and assistant returned a final summary without calling tools
1099
- if (isExplicitCompletion || (this.modifiedFiles.size > 0 && !lowerText.includes('?') && textContent.length > 50)) {
1100
- saveAgentSession({
1101
- goal: userGoal,
1102
- summary: textContent,
1103
- modifiedFiles: Array.from(this.modifiedFiles),
1104
- });
1105
- return { success: true, summary: textContent };
1106
- }
1107
- }
1108
- // If assistant responded with text without calling tools, prompt it to execute tools to complete the goal
857
+ continue;
858
+ }
859
+ // Check text response or fallback JSON tool call
860
+ stepSpinner.stop(chalk.blue(`Step ${step} thinking complete.`));
861
+ const parsedJsonTool = this.extractJsonToolCall(textContent);
862
+ if (parsedJsonTool) {
863
+ const toolResult = await this.dispatchToolCall(parsedJsonTool.tool, parsedJsonTool.args);
1109
864
  this.historyMessages.push({
1110
865
  role: 'user',
1111
- content: 'You provided a text response but have not called any tools (write_file, edit_file, run_command, task_completed). Please execute necessary tool calls to complete the user goal, or invoke task_completed if finished.',
866
+ content: `Tool Execution Result (${parsedJsonTool.tool}):\n${toolResult.result}`,
1112
867
  });
1113
- stepDurations.push(Date.now() - stepStartTime);
868
+ this.handleToolConsecutiveTracking(parsedJsonTool.tool, parsedJsonTool.args, step);
869
+ if (parsedJsonTool.tool === 'task_completed') {
870
+ finalSummary = parsedJsonTool.args?.summary || toolResult.result;
871
+ saveAgentSession({
872
+ goal: userGoal,
873
+ summary: finalSummary,
874
+ modifiedFiles: Array.from(this.modifiedFiles),
875
+ });
876
+ return { success: true, summary: finalSummary };
877
+ }
878
+ continue;
1114
879
  }
1115
- catch (err) {
1116
- stepDurations.push(Date.now() - stepStartTime);
1117
- if (isQuotaExceededError(err)) {
1118
- stepSpinner.stop(chalk.red(`Oops! it\'s not you, it\'s us`));
1119
- safeNote(`${chalk.bold.red(`Something went wrong in (Step ${step}), please try again in a moment`)}\n\n` +
1120
- `${chalk.yellow('The configured AI provider has reached its API usage limit or rate cap.')}\n` +
1121
- `${chalk.dim('This is separate from your Scout credit balance shown by `scout quota`.')}\n\n` +
1122
- `${chalk.bold.cyan(' Please come back and try again in a few hours (or check back later today).')}\n\n` +
1123
- `${chalk.dim('Scout Agent session has ended gracefully to protect remaining workflow.')}`, '️ API Quota Limit Reached');
1124
- return {
1125
- success: false,
1126
- summary: 'Something went wrong, please try again in a moment.',
1127
- };
880
+ if (textContent.trim()) {
881
+ let cleanedThought = textContent
882
+ .replace(/[\u0600-\u06FF\u0750-\u077F\uAC00-\uD7AF\u3040-\u30FF\u4E00-\u9FFF\u0D80-\u0DFF]+/g, '')
883
+ .trim();
884
+ if (!cleanedThought || cleanedThought.length < 5) {
885
+ cleanedThought = 'Analyzing codebase files and executing next tool operation...';
886
+ }
887
+ safeNote(renderMarkdown(cleanedThought), ` Scout Agent Thought (Step ${step})`);
888
+ const lowerText = textContent.toLowerCase();
889
+ const isExplicitCompletion = lowerText.includes('task is complete') ||
890
+ lowerText.includes('task complete') ||
891
+ lowerText.includes('goal completed') ||
892
+ lowerText.includes('goal is completed') ||
893
+ lowerText.includes('all tasks completed') ||
894
+ lowerText.includes('i have completed') ||
895
+ lowerText.includes('no further changes needed') ||
896
+ lowerText.includes('the fix is complete') ||
897
+ lowerText.includes('has been created') ||
898
+ lowerText.includes('successfully created') ||
899
+ lowerText.includes('created the file') ||
900
+ lowerText.includes('file created') ||
901
+ lowerText.includes('implementation complete') ||
902
+ lowerText.includes('work is complete');
903
+ // If explicit completion phrase found, OR files have already been modified and assistant returned a final summary without calling tools
904
+ if (isExplicitCompletion || (this.modifiedFiles.size > 0 && !lowerText.includes('?') && textContent.length > 50)) {
905
+ saveAgentSession({
906
+ goal: userGoal,
907
+ summary: textContent,
908
+ modifiedFiles: Array.from(this.modifiedFiles),
909
+ });
910
+ return { success: true, summary: textContent };
1128
911
  }
1129
- stepSpinner.stop(chalk.red(`Step ${step} execution error: ${err?.message || String(err)}`));
1130
- this.historyMessages.push({
1131
- role: 'user',
1132
- content: `Error in previous turn: ${err?.message || String(err)}. Please try alternative steps or call tools.`,
1133
- });
1134
912
  }
1135
- // If current max steps limit is reached while task is still in progress, auto-extend by +15 extra steps (up to 60 max threshold)
1136
- if (step >= this.maxSteps && this.maxSteps < 60) {
1137
- this.maxSteps += 15;
1138
- safeNote(chalk.bold.yellow(`⚡ Step limit reached while task is in progress. Automatically extending execution by +15 extra steps (New Max Limit: ${this.maxSteps})...`), ' Auto Extra Steps Extension');
913
+ // If assistant responded with text without calling tools, prompt it to execute tools to complete the goal
914
+ this.historyMessages.push({
915
+ role: 'user',
916
+ content: 'You provided a text response but have not called any tools (write_file, edit_file, run_command, task_completed). Please execute necessary tool calls to complete the user goal, or invoke task_completed if finished.',
917
+ });
918
+ stepDurations.push(Date.now() - stepStartTime);
919
+ }
920
+ catch (err) {
921
+ stepDurations.push(Date.now() - stepStartTime);
922
+ if (isQuotaExceededError(err)) {
923
+ stepSpinner.stop(chalk.red(`Oops! it\'s not you, it\'s us`));
924
+ safeNote(`${chalk.bold.red(`Something went wrong in (Step ${step}), please try again in a moment`)}\n\n` +
925
+ `${chalk.yellow('The configured AI provider has reached its API usage limit or rate cap.')}\n` +
926
+ `${chalk.dim('This is separate from your Scout credit balance shown by `scout quota`.')}\n\n` +
927
+ `${chalk.bold.cyan(' Please come back and try again in a few hours (or check back later today).')}\n\n` +
928
+ `${chalk.dim('Scout Agent session has ended gracefully to protect remaining workflow.')}`, '️ API Quota Limit Reached');
929
+ return {
930
+ success: false,
931
+ summary: 'Something went wrong, please try again in a moment.',
932
+ };
1139
933
  }
934
+ stepSpinner.stop(chalk.red(`Step ${step} execution error: ${err?.message || String(err)}`));
935
+ this.historyMessages.push({
936
+ role: 'user',
937
+ content: `Error in previous turn: ${err?.message || String(err)}. Please try alternative steps or call tools.`,
938
+ });
1140
939
  }
1141
- return {
1142
- success: false,
1143
- summary: `Reached max iteration steps limit (${this.maxSteps}). Modified files: ${Array.from(this.modifiedFiles).join(', ')}`,
1144
- };
1145
- }
1146
- finally {
1147
- cleanupScreenshots();
1148
- if (isLiveScreenShareActive()) {
1149
- stopLiveScreenShare();
940
+ // If current max steps limit is reached while task is still in progress, auto-extend by +15 extra steps (up to 60 max threshold)
941
+ if (step >= this.maxSteps && this.maxSteps < 60) {
942
+ this.maxSteps += 15;
943
+ safeNote(chalk.bold.yellow(`⚡ Step limit reached while task is in progress. Automatically extending execution by +15 extra steps (New Max Limit: ${this.maxSteps})...`), ' Auto Extra Steps Extension');
1150
944
  }
1151
945
  }
946
+ return {
947
+ success: false,
948
+ summary: `Reached max iteration steps limit (${this.maxSteps}). Modified files: ${Array.from(this.modifiedFiles).join(', ')}`,
949
+ };
1152
950
  }
1153
951
  extractJsonToolCall(content) {
1154
952
  if (!content)
@@ -1168,10 +966,8 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
1168
966
  const knownTools = [
1169
967
  'read_file', 'write_file', 'edit_file', 'run_command', 'list_dir',
1170
968
  'grep_search', 'glob_search', 'tree_view', 'file_info', 'multi_edit_file',
1171
- 'fetch_url', 'git_diff', 'open_app', 'app_action', 'agent_scratchpad',
969
+ 'fetch_url', 'git_diff',
1172
970
  'ask_user', 'speak_text', 'task_completed',
1173
- 'desktop_action', 'type_text', 'click_app', 'send_keys', 'takeover',
1174
- 'capture_screen', 'see_screen', 'analyze_screen', 'scratchpad',
1175
971
  ];
1176
972
  for (const toolName of knownTools) {
1177
973
  const tagRegex = new RegExp(`<${toolName}>([\\s\\S]*?)<\\/${toolName}>`, 'i');
@@ -1227,10 +1023,6 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
1227
1023
  actionDesc = `Fetching URL: ${args.url || ''}`;
1228
1024
  else if (name === 'git_diff')
1229
1025
  actionDesc = `Retrieving workspace git diff`;
1230
- else if (name === 'open_app')
1231
- actionDesc = `Launching app: ${args.app || ''} (${args.target || 'default'})`;
1232
- else if (name === 'app_action')
1233
- actionDesc = `Executing app action: ${args.app || ''} -> ${args.action || ''}`;
1234
1026
  else if (name === 'browser_navigate')
1235
1027
  actionDesc = `Navigating browser: ${args.url || ''}`;
1236
1028
  else if (name === 'browser_inspect')
@@ -1244,19 +1036,25 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
1244
1036
  else if (name === 'browser_content')
1245
1037
  actionDesc = `Extracting web page content`;
1246
1038
  else if (name === 'computer_screenshot')
1247
- actionDesc = `Capturing computer screenshot`;
1039
+ actionDesc = `Observing computer screen`;
1248
1040
  else if (name === 'computer_click')
1249
- actionDesc = `Clicking coordinates (${args.x}, ${args.y})`;
1041
+ actionDesc = `Clicking screen coordinate: (${args.x}, ${args.y})`;
1042
+ else if (name === 'computer_double_click')
1043
+ actionDesc = `Double clicking coordinate: (${args.x}, ${args.y})`;
1250
1044
  else if (name === 'computer_type')
1251
- actionDesc = `Typing text keystrokes`;
1045
+ actionDesc = `Typing text: "${String(args.text || '').slice(0, 30)}"`;
1046
+ else if (name === 'computer_key_press')
1047
+ actionDesc = `Pressing key: ${args.key || ''}`;
1252
1048
  else if (name === 'computer_hotkey')
1253
- actionDesc = `Pressing hotkey: ${args.keys || ''}`;
1049
+ actionDesc = `Executing hotkey: ${(args.keys || []).join('+')}`;
1254
1050
  else if (name === 'computer_open_app')
1255
- actionDesc = `Opening app: ${args.name || ''}`;
1256
- else if (name === 'computer_switch_app')
1257
- actionDesc = `Switching to app: ${args.name || ''}`;
1258
- else if (name === 'computer_scroll')
1259
- actionDesc = `Scrolling ${args.direction || 'down'}`;
1051
+ actionDesc = `Launching application: ${args.appName || args.app || ''}`;
1052
+ else if (name === 'computer_close_app')
1053
+ actionDesc = `Closing application: ${args.appName || args.app || ''}`;
1054
+ else if (name === 'computer_switch_window')
1055
+ actionDesc = `Switching window: ${args.windowTitleOrApp || ''}`;
1056
+ else if (name === 'computer_inspect_ui')
1057
+ actionDesc = `Inspecting visible desktop UI`;
1260
1058
  else if (name === 'task_completed')
1261
1059
  actionDesc = `Task finished`;
1262
1060
  const badge = chalk.bgCyan.black.bold(` 🛠️ TOOL: ${name} `);
@@ -1280,6 +1078,27 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
1280
1078
  return { result: `[ACTION REJECTED BY USER]: ${decision.reason || 'User did not approve executing this action.'}` };
1281
1079
  }
1282
1080
  }
1081
+ // Validate against execution intent (USER INTENT > TOOL CONVENIENCE)
1082
+ if (this.executionIntent && !this.executionIntent.allowAlternativeTools) {
1083
+ if (this.executionIntent.requiredInteraction === 'computer') {
1084
+ if (['write_file', 'edit_file', 'multi_edit_file'].includes(name)) {
1085
+ const reqApps = this.executionIntent.requiredApplications.join(', ') || 'the requested desktop application';
1086
+ const violationMsg = `[EXECUTION INTENT VIOLATION BLOCKED]: User explicitly required computer/application interaction using ${reqApps}. You cannot silently substitute direct filesystem API calls (${name}). You must use computer interaction tools (computer_open_app, computer_switch_window, computer_click, computer_type, computer_hotkey) to perform this task in ${reqApps}.`;
1087
+ console.log(chalk.red.bold(`\n⛔ ${violationMsg}`));
1088
+ this.taskStateManager.addObservation(name, violationMsg, false);
1089
+ return {
1090
+ result: JSON.stringify({
1091
+ success: false,
1092
+ action: name,
1093
+ error: 'ExecutionIntentViolation',
1094
+ message: violationMsg,
1095
+ recoverable: true,
1096
+ suggestedAction: `Use computer interaction tools with ${reqApps}. Focus the window, type text into the app, and save through the application UI.`
1097
+ }, null, 2)
1098
+ };
1099
+ }
1100
+ }
1101
+ }
1283
1102
  switch (name) {
1284
1103
  case 'read_file': {
1285
1104
  const targetPath = String(args.path || '').trim();
@@ -1314,12 +1133,16 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
1314
1133
  return { result: `File: ${targetPath} (Lines ${start}-${end}/${lines.length}):\n${sliced}${minifiedNotice}` };
1315
1134
  }
1316
1135
  case 'write_file': {
1317
- const targetPath = String(args.path || '').trim();
1136
+ let targetPath = String(args.path || '').trim();
1318
1137
  let content = String(args.content || '');
1319
1138
  const reason = String(args.reason || args.why || args.description || 'Created/updated file content to fulfill user prompt.');
1320
1139
  if (content.includes('\\n') && !content.includes('\n')) {
1321
1140
  content = content.replace(/\\n/g, '\n').replace(/\\t/g, '\t');
1322
1141
  }
1142
+ if (this.executionIntent?.targetDestination === 'Desktop' || targetPath.toLowerCase().includes('desktop')) {
1143
+ const fileName = path.basename(targetPath);
1144
+ targetPath = resolveDesktopPath(fileName);
1145
+ }
1323
1146
  const fullPath = path.isAbsolute(targetPath) ? targetPath : path.resolve(this.cwd, targetPath);
1324
1147
  const fileExists = fs.existsSync(fullPath);
1325
1148
  const actionText = fileExists ? 'Editing file' : 'Creating file';
@@ -1659,49 +1482,6 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
1659
1482
  return { result: `Could not retrieve git diff: ${err?.message || String(err)}` };
1660
1483
  }
1661
1484
  }
1662
- case 'agent_scratchpad': {
1663
- const action = String(args.action || 'update').toLowerCase();
1664
- if (action === 'clear') {
1665
- this.scratchpadState = { plan: [], completedSteps: [], notes: '' };
1666
- try {
1667
- const scratchpadPath = path.join(this.cwd, '.ft', 'scratchpad.md');
1668
- if (fs.existsSync(scratchpadPath))
1669
- fs.unlinkSync(scratchpadPath);
1670
- }
1671
- catch { }
1672
- return { result: 'Agent Scratchpad cleared.' };
1673
- }
1674
- else if (action === 'read') {
1675
- const text = this.getScratchpadPromptContext() || 'Scratchpad is currently empty.';
1676
- return { result: text };
1677
- }
1678
- else {
1679
- const res = this.updateScratchpad(Array.isArray(args.plan) ? args.plan : undefined, Array.isArray(args.completedSteps) ? args.completedSteps : undefined, args.notes !== undefined ? String(args.notes) : undefined);
1680
- return { result: res };
1681
- }
1682
- }
1683
- case 'open_app': {
1684
- const appName = String(args.app || '').trim();
1685
- const target = args.target ? String(args.target) : undefined;
1686
- const launchRes = await openApp(appName, target, args);
1687
- return { result: launchRes.output };
1688
- }
1689
- case 'scratchpad':
1690
- case 'desktop_action':
1691
- case 'type_text':
1692
- case 'click_app':
1693
- case 'send_keys':
1694
- case 'takeover':
1695
- case 'capture_screen':
1696
- case 'see_screen':
1697
- case 'analyze_screen':
1698
- case 'app_action': {
1699
- const appName = String(args.app || args.appName || 'desktop').trim();
1700
- const action = String(args.action || (name !== 'app_action' ? name : 'type_text')).trim();
1701
- const payload = args.payload !== undefined ? args.payload : (args.text || args.content || args.target || args.query || args.element || args.url || args.command || args);
1702
- const actionRes = await executeInApp(appName, action, payload);
1703
- return { result: actionRes.output };
1704
- }
1705
1485
  case 'ask_user': {
1706
1486
  const questionText = String(args.question || args.prompt || 'Please provide input:').trim();
1707
1487
  const askSpin = spinner();
@@ -1781,56 +1561,188 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
1781
1561
  }
1782
1562
  case 'computer_screenshot': {
1783
1563
  const res = await this.computerController.screenshot();
1784
- this.taskStateManager.addObservation('computer_screenshot', res.output, res.success);
1785
- return { result: res.output };
1564
+ if (res.path) {
1565
+ this.taskStateManager.recordArtifact({
1566
+ filename: path.basename(res.path),
1567
+ path: res.path,
1568
+ type: 'image',
1569
+ description: `Screen observation (${res.width}x${res.height})`,
1570
+ application: res.activeWindow || 'Desktop',
1571
+ });
1572
+ }
1573
+ if (res.activeWindow) {
1574
+ this.taskStateManager.setActiveWindow(res.activeWindow);
1575
+ }
1576
+ const summary = `Screen captured (${res.width}x${res.height}), Active window: "${res.activeWindow || 'Desktop'}"${res.path ? `, Saved: ${res.path}` : ''}`;
1577
+ this.taskStateManager.addObservation('computer_screenshot', summary, true);
1578
+ return { result: summary };
1786
1579
  }
1787
1580
  case 'computer_click': {
1788
- const x = Number(args.x) || 0;
1789
- const y = Number(args.y) || 0;
1790
- const res = args.double
1791
- ? await this.computerController.double_click(x, y)
1792
- : await this.computerController.click(x, y, args.button || 'left');
1581
+ const x = Number(args.x);
1582
+ const y = Number(args.y);
1583
+ const res = await this.computerController.click(x, y, { button: args.button });
1793
1584
  this.taskStateManager.addObservation('computer_click', res.output, res.success, `(${x}, ${y})`);
1794
- return { result: res.output };
1585
+ const structuredRes = {
1586
+ success: res.success,
1587
+ action: 'click',
1588
+ output: res.output,
1589
+ ...(res.error ? { error: res.error } : {}),
1590
+ recoverable: res.recoverable ?? true,
1591
+ ...(res.activeWindow ? { activeWindow: res.activeWindow } : {}),
1592
+ };
1593
+ return { result: JSON.stringify(structuredRes, null, 2) };
1594
+ }
1595
+ case 'computer_double_click': {
1596
+ const x = Number(args.x);
1597
+ const y = Number(args.y);
1598
+ const res = await this.computerController.doubleClick(x, y);
1599
+ this.taskStateManager.addObservation('computer_double_click', res.output, res.success, `(${x}, ${y})`);
1600
+ const structuredRes = {
1601
+ success: res.success,
1602
+ action: 'double_click',
1603
+ output: res.output,
1604
+ ...(res.error ? { error: res.error } : {}),
1605
+ recoverable: res.recoverable ?? true,
1606
+ };
1607
+ return { result: JSON.stringify(structuredRes, null, 2) };
1795
1608
  }
1796
1609
  case 'computer_type': {
1797
- const text = String(args.text || '');
1798
- const res = await this.computerController.type(text);
1610
+ const textVal = String(args.text || '');
1611
+ const res = await this.computerController.type(textVal);
1799
1612
  this.taskStateManager.addObservation('computer_type', res.output, res.success);
1800
- return { result: res.output };
1613
+ const structuredRes = {
1614
+ success: res.success,
1615
+ action: 'type',
1616
+ output: res.output,
1617
+ ...(res.error ? { error: res.error } : {}),
1618
+ recoverable: res.recoverable ?? true,
1619
+ ...(res.activeWindow ? { activeWindow: res.activeWindow } : {}),
1620
+ ...(res.expectedWindow ? { expectedWindow: res.expectedWindow } : {}),
1621
+ };
1622
+ return { result: JSON.stringify(structuredRes, null, 2) };
1623
+ }
1624
+ case 'computer_key_press': {
1625
+ const key = String(args.key || '');
1626
+ const res = await this.computerController.keyPress(key);
1627
+ this.taskStateManager.addObservation('computer_key_press', res.output, res.success);
1628
+ const structuredRes = {
1629
+ success: res.success,
1630
+ action: 'key_press',
1631
+ output: res.output,
1632
+ ...(res.error ? { error: res.error } : {}),
1633
+ recoverable: res.recoverable ?? true,
1634
+ };
1635
+ return { result: JSON.stringify(structuredRes, null, 2) };
1801
1636
  }
1802
1637
  case 'computer_hotkey': {
1803
- const keys = String(args.keys || args.key || '');
1638
+ const keys = Array.isArray(args.keys) ? args.keys.map(String) : [String(args.keys)];
1804
1639
  const res = await this.computerController.hotkey(keys);
1805
- this.taskStateManager.addObservation('computer_hotkey', res.output, res.success, keys);
1640
+ this.taskStateManager.addObservation('computer_hotkey', res.output, res.success);
1641
+ const structuredRes = {
1642
+ success: res.success,
1643
+ action: 'hotkey',
1644
+ output: res.output,
1645
+ ...(res.error ? { error: res.error } : {}),
1646
+ recoverable: res.recoverable ?? true,
1647
+ ...(res.activeWindow ? { activeWindow: res.activeWindow } : {}),
1648
+ };
1649
+ return { result: JSON.stringify(structuredRes, null, 2) };
1650
+ }
1651
+ case 'computer_scroll': {
1652
+ const deltaY = Number(args.deltaY || 0);
1653
+ const deltaX = Number(args.deltaX || 0);
1654
+ const res = await this.computerController.scroll(deltaX, deltaY);
1655
+ this.taskStateManager.addObservation('computer_scroll', res.output, res.success);
1806
1656
  return { result: res.output };
1807
1657
  }
1808
- case 'computer_open_app': {
1809
- const name = String(args.name || args.app || '').trim();
1810
- const target = args.target ? String(args.target) : undefined;
1811
- this.taskStateManager.setActiveApplication(name);
1812
- const res = await this.computerController.open_application(name, target);
1813
- this.taskStateManager.addObservation('computer_open_app', res.output, res.success, name);
1658
+ case 'computer_mouse_move': {
1659
+ const x = Number(args.x);
1660
+ const y = Number(args.y);
1661
+ const res = await this.computerController.mouseMove(x, y);
1662
+ this.taskStateManager.addObservation('computer_mouse_move', res.output, res.success);
1814
1663
  return { result: res.output };
1815
1664
  }
1816
- case 'computer_switch_app': {
1817
- const name = String(args.name || args.app || '').trim();
1818
- this.taskStateManager.setActiveApplication(name);
1819
- const res = await this.computerController.switch_application(name);
1820
- this.taskStateManager.addObservation('computer_switch_app', res.output, res.success, name);
1665
+ case 'computer_drag': {
1666
+ const res = await this.computerController.drag(Number(args.fromX), Number(args.fromY), Number(args.toX), Number(args.toY));
1667
+ this.taskStateManager.addObservation('computer_drag', res.output, res.success);
1821
1668
  return { result: res.output };
1822
1669
  }
1823
- case 'computer_scroll': {
1824
- const dir = (args.direction || 'down');
1825
- const amount = Number(args.amount) || 4;
1826
- const res = await this.computerController.scroll(dir, amount);
1827
- this.taskStateManager.addObservation('computer_scroll', res.output, res.success);
1670
+ case 'computer_wait': {
1671
+ const ms = Math.min(10000, Math.max(100, Number(args.ms) || 1000));
1672
+ await this.computerController.wait(ms);
1673
+ return { result: `Waited for ${ms}ms` };
1674
+ }
1675
+ case 'computer_open_app': {
1676
+ const appName = String(args.appName || args.app || '');
1677
+ const res = await this.computerController.openApplication(appName, args.args);
1678
+ this.taskStateManager.setActiveApplication(appName);
1679
+ this.taskStateManager.addObservation('computer_open_app', res.output, res.success);
1680
+ const structuredRes = {
1681
+ success: res.success,
1682
+ action: 'open_app',
1683
+ output: res.output,
1684
+ ...(res.error ? { error: res.error } : {}),
1685
+ recoverable: res.recoverable ?? true,
1686
+ activeWindow: res.activeWindow || appName,
1687
+ expectedWindow: res.expectedWindow || appName,
1688
+ };
1689
+ return { result: JSON.stringify(structuredRes, null, 2) };
1690
+ }
1691
+ case 'computer_close_app': {
1692
+ const appName = String(args.appName || args.app || '');
1693
+ const res = await this.computerController.closeApplication(appName);
1694
+ this.taskStateManager.addObservation('computer_close_app', res.output, res.success);
1828
1695
  return { result: res.output };
1829
1696
  }
1697
+ case 'computer_switch_window': {
1698
+ const targetWin = String(args.windowTitleOrApp || args.target || '');
1699
+ const res = await this.computerController.switchWindow(targetWin);
1700
+ if (res.success) {
1701
+ this.taskStateManager.setActiveWindow(targetWin);
1702
+ }
1703
+ this.taskStateManager.addObservation('computer_switch_window', res.output, res.success);
1704
+ const structuredRes = {
1705
+ success: res.success,
1706
+ action: 'switch_window',
1707
+ output: res.output,
1708
+ ...(res.error ? { error: res.error } : {}),
1709
+ recoverable: res.recoverable ?? true,
1710
+ activeWindow: res.activeWindow || targetWin,
1711
+ expectedWindow: res.expectedWindow,
1712
+ };
1713
+ return { result: JSON.stringify(structuredRes, null, 2) };
1714
+ }
1715
+ case 'computer_inspect_ui': {
1716
+ const ui = await this.computerController.inspectUI();
1717
+ this.taskStateManager.setActiveWindow(ui.activeWindow);
1718
+ const outStr = `Active Window: "${ui.activeWindow}"\nVisible Windows: ${ui.windows.map((w) => `"${w}"`).join(', ')}`;
1719
+ this.taskStateManager.addObservation('computer_inspect_ui', outStr, true);
1720
+ const structuredRes = {
1721
+ success: true,
1722
+ action: 'inspect_ui',
1723
+ activeWindow: ui.activeWindow,
1724
+ windows: ui.windows,
1725
+ output: outStr,
1726
+ };
1727
+ return { result: JSON.stringify(structuredRes, null, 2) };
1728
+ }
1830
1729
  case 'task_completed': {
1831
1730
  const summaryStr = String(args.summary || 'Task completed successfully.');
1731
+ if (this.executionIntent?.targetDestination === 'Desktop' && this.executionIntent?.targetArtifactName) {
1732
+ const checkRes = await this.actionVerifier.verifyFileCreated(this.executionIntent.targetArtifactName, 1, 'Desktop');
1733
+ if (!checkRes.verified) {
1734
+ return {
1735
+ result: JSON.stringify({
1736
+ success: false,
1737
+ error: 'VerificationFailed',
1738
+ message: `Cannot conclude task as completed: ${checkRes.message}`,
1739
+ recoverable: true,
1740
+ suggestedAction: 'The file was not found on the Desktop (or was placed in a substitute path). Save the file to the user\'s Desktop via the application interface before completing.'
1741
+ }, null, 2)
1742
+ };
1743
+ }
1744
+ }
1832
1745
  speakText(summaryStr, { async: true });
1833
- cleanupScreenshots();
1834
1746
  this.taskStateManager.setStatus('completed');
1835
1747
  this.taskStateManager.completeStep(this.currentStep, summaryStr);
1836
1748
  return { result: summaryStr };