ft-scout 9.0.4 → 9.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/.firebase/hosting.d2Vi.cache +7 -7
  2. package/README.md +2 -2
  3. package/bin/src/commands/agent.d.ts.map +1 -1
  4. package/bin/src/commands/agent.js +21 -8
  5. package/bin/src/commands/agent.js.map +1 -1
  6. package/bin/src/engine/agentEngine.d.ts.map +1 -1
  7. package/bin/src/engine/agentEngine.js +253 -13
  8. package/bin/src/engine/agentEngine.js.map +1 -1
  9. package/bin/src/engine/computer/computerController.d.ts.map +1 -1
  10. package/bin/src/engine/computer/computerController.js +532 -296
  11. package/bin/src/engine/computer/computerController.js.map +1 -1
  12. package/bin/src/engine/computer/computerDiagnostic.d.ts.map +1 -0
  13. package/bin/src/engine/computer/computerDiagnostic.js +71 -0
  14. package/bin/src/engine/computer/computerDiagnostic.js.map +1 -0
  15. package/bin/src/engine/computer/index.d.ts.map +1 -1
  16. package/bin/src/engine/computer/index.js +1 -0
  17. package/bin/src/engine/computer/index.js.map +1 -1
  18. package/bin/src/engine/computer/mockComputer.d.ts.map +1 -1
  19. package/bin/src/engine/computer/mockComputer.js +104 -8
  20. package/bin/src/engine/computer/mockComputer.js.map +1 -1
  21. package/bin/src/engine/computer/scoutDriver.cs +873 -0
  22. package/bin/src/engine/computer/scout_driver.exe +0 -0
  23. package/bin/src/engine/runtime/agentRuntime.d.ts.map +1 -1
  24. package/bin/src/engine/runtime/agentRuntime.js +351 -20
  25. package/bin/src/engine/runtime/agentRuntime.js.map +1 -1
  26. package/bin/src/engine/safety.d.ts.map +1 -1
  27. package/bin/src/engine/safety.js +13 -3
  28. package/bin/src/engine/safety.js.map +1 -1
  29. package/bin/src/engine/toolRegistry.d.ts.map +1 -1
  30. package/bin/src/engine/toolRegistry.js +144 -0
  31. package/bin/src/engine/toolRegistry.js.map +1 -1
  32. package/bin/src/engine/workspace/agentWorkspace.d.ts.map +1 -1
  33. package/bin/src/engine/workspace/agentWorkspace.js +21 -0
  34. package/bin/src/engine/workspace/agentWorkspace.js.map +1 -1
  35. package/bin/src/index.js +4 -4
  36. package/bin/src/utils/auth.d.ts.map +1 -1
  37. package/bin/src/utils/auth.js +14 -1
  38. package/bin/src/utils/auth.js.map +1 -1
  39. package/bin/src/utils/branding.js +1 -1
  40. package/package.json +1 -1
  41. package/web/app.js +2 -2
  42. package/web/styles.css +2 -2
@@ -19,6 +19,7 @@ import { UnifiedToolRegistry } from './toolRegistry.js';
19
19
  import { SystemComputerController, MockComputerController } from './computer/index.js';
20
20
  import { AgentWorkspace, inferExecutionIntent, resolveDesktopPath } from './workspace/index.js';
21
21
  import { AgentRuntime } from './runtime/index.js';
22
+ import { loadAuthConfig, isProOrHigher } from '../utils/auth.js';
22
23
  export { AgentRuntime, AgentWorkspace, inferExecutionIntent, resolveDesktopPath, SystemComputerController, MockComputerController };
23
24
  const execAsync = promisify(exec);
24
25
  export function robustSnippetReplace(origContent, target, replacement) {
@@ -372,11 +373,16 @@ export class AgentExecutionLoop {
372
373
  isCancelled = false;
373
374
  cancelReason;
374
375
  executionIntent;
376
+ userTier;
375
377
  constructor(options) {
376
378
  this.cwd = process.cwd();
377
379
  this.autoApprove = Boolean(options?.autoApprove);
378
- this.maxSteps = options?.maxSteps || 30;
379
380
  this.mockMode = Boolean(options?.mockMode);
381
+ this.userTier = (options?.tier || loadAuthConfig().currentTier || 'free').toLowerCase();
382
+ // Free tier is bounded to basic direct actions (max 5 steps). Full autonomous multi-step loops require Pro/Enterprise.
383
+ const isPaid = this.mockMode || this.userTier === 'pro' || this.userTier === 'enterprise';
384
+ const requestedSteps = options?.maxSteps || 30;
385
+ this.maxSteps = isPaid ? requestedSteps : Math.min(requestedSteps, 5);
380
386
  this.taskStateManager = new TaskStateManager(options?.taskId, '', this.cwd);
381
387
  this.computerController = options?.computerController || (this.mockMode ? new MockComputerController() : new SystemComputerController(this.cwd));
382
388
  this.browserController = options?.browserController || new SemanticBrowserController(this.cwd);
@@ -385,6 +391,9 @@ export class AgentExecutionLoop {
385
391
  this.taskPlanner = new TaskPlanner();
386
392
  this.toolRegistry = new UnifiedToolRegistry();
387
393
  }
394
+ getUserTier() {
395
+ return this.userTier;
396
+ }
388
397
  getExecutionIntent() {
389
398
  return this.executionIntent;
390
399
  }
@@ -558,11 +567,27 @@ Task Execution Intent & Constraints:
558
567
  • Target Destination: ${this.executionIntent.targetDestination || 'Default'}
559
568
  • Target Artifact: ${this.executionIntent.targetArtifactName || 'None'}
560
569
  ` : '';
570
+ const isPaid = this.mockMode || this.userTier === 'pro' || this.userTier === 'enterprise';
571
+ const tierDirective = isPaid ? `
572
+ PLAN TIER & CAPABILITIES: ${this.userTier.toUpperCase()} (FULL ACCESS UNLOCKED)
573
+ • Autonomous Computer-Use, Browser Automation, and AI Scratchpad Reasoning are fully ENABLED.
574
+ • Use the 'scratchpad' tool to organize multi-step reasoning, break down complex goals, draft hypotheses, and track working memory across turns.
575
+ ` : `
576
+ PLAN TIER & CAPABILITIES: DEVELOPER FREE (BASIC DEVELOPER OPS ONLY)
577
+ • Scratchpadding and Autonomous Multi-Step Desktop/Browser Control are RESTRICTED to Pro & Enterprise plans.
578
+ • As a Free Tier developer assistant, you ONLY perform basic developer operations:
579
+ 1. Code editing: use edit_file or multi_edit_file to modify existing code.
580
+ 2. Creating new files & folders: use write_file to create new files and parent directories.
581
+ 3. Running commands: use run_command to execute terminal commands, builds, or tests.
582
+ 4. Reading & exploring: use read_file, list_dir, grep_search to inspect files.
583
+ • Do NOT attempt to use scratchpad, computer, or browser tools. Execute the requested file edit, file creation, or command directly and call task_completed immediately.
584
+ `;
561
585
  return `You are Scout Agent, an elite AI Principal Software Engineer & Autonomous Computer-Use Operator (developed by FrontTerrain).
562
586
 
563
587
  Working Directory: "${this.cwd}"
564
588
  Host OS Platform: "${process.platform}"
565
589
  Current Date & Time: "${new Date().toISOString()}" (Current Year: ${new Date().getFullYear()})
590
+ ${tierDirective}
566
591
  ${intentInfo}
567
592
  Workspace Overview (first 100 files):
568
593
  ${filesList.join('\n')}
@@ -611,7 +636,7 @@ When returning tool calls, use standard OpenAI function calling format or JSON t
611
636
  }
612
637
  const structuredPlan = this.taskPlanner.createPlan(userGoal);
613
638
  this.taskStateManager.setPlan(structuredPlan.steps);
614
- this.taskStateManager.setAvailableTools(this.toolRegistry.getAllTools().map((t) => t.name));
639
+ this.taskStateManager.setAvailableTools(this.toolRegistry.getToolsForTier(this.userTier).map((t) => t.name));
615
640
  if (this.mockMode) {
616
641
  for (let i = 1; i <= structuredPlan.steps.length; i++) {
617
642
  this.taskStateManager.startStep(i);
@@ -766,7 +791,7 @@ When returning tool calls, use standard OpenAI function calling format or JSON t
766
791
  return await openai.chat.completions.create({
767
792
  model,
768
793
  messages: sanitizeMessages(this.historyMessages),
769
- tools: this.toolRegistry.getAllTools().map((t) => ({ type: 'function', function: t })),
794
+ tools: this.toolRegistry.getToolsForTier(this.userTier).map((t) => ({ type: 'function', function: t })),
770
795
  tool_choice: 'auto',
771
796
  temperature: 0.1,
772
797
  presence_penalty: 0.1,
@@ -937,10 +962,17 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
937
962
  content: `Error in previous turn: ${err?.message || String(err)}. Please try alternative steps or call tools.`,
938
963
  });
939
964
  }
940
- // If current max steps limit is reached while task is still in progress, auto-extend by +15 extra steps (up to 60 max threshold)
941
- if (step >= this.maxSteps && this.maxSteps < 60) {
942
- this.maxSteps += 15;
943
- safeNote(chalk.bold.yellow(`⚡ Step limit reached while task is in progress. Automatically extending execution by +15 extra steps (New Max Limit: ${this.maxSteps})...`), ' Auto Extra Steps Extension');
965
+ // Step limit check: Auto-extend by +15 steps ONLY for Pro & Enterprise (up to 60 max threshold)
966
+ const isPaid = this.mockMode || this.userTier === 'pro' || this.userTier === 'enterprise';
967
+ if (step >= this.maxSteps) {
968
+ if (isPaid && this.maxSteps < 60) {
969
+ this.maxSteps += 15;
970
+ safeNote(chalk.bold.yellow(`⚡ Step limit reached while task is in progress. Automatically extending execution by +15 extra steps (New Max Limit: ${this.maxSteps})...`), ' Auto Extra Steps Extension');
971
+ }
972
+ else if (!isPaid) {
973
+ safeNote(chalk.bold.yellow(`⚡ Free Tier limit reached (${this.maxSteps} steps).\nFree developers can perform basic code editing, creating files/folders, and running commands.\nTo unlock unlimited autonomous multi-step loops and scratchpad reasoning, upgrade to Pro at https://tryscout.web.app/plans or run 'scout dashboard'.`), ' Free Tier Step Limit');
974
+ break;
975
+ }
944
976
  }
945
977
  }
946
978
  return {
@@ -967,7 +999,7 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
967
999
  'read_file', 'write_file', 'edit_file', 'run_command', 'list_dir',
968
1000
  'grep_search', 'glob_search', 'tree_view', 'file_info', 'multi_edit_file',
969
1001
  'fetch_url', 'git_diff',
970
- 'ask_user', 'speak_text', 'task_completed',
1002
+ 'ask_user', 'speak_text', 'task_completed', 'scratchpad',
971
1003
  ];
972
1004
  for (const toolName of knownTools) {
973
1005
  const tagRegex = new RegExp(`<${toolName}>([\\s\\S]*?)<\\/${toolName}>`, 'i');
@@ -1039,8 +1071,18 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
1039
1071
  actionDesc = `Observing computer screen`;
1040
1072
  else if (name === 'computer_click')
1041
1073
  actionDesc = `Clicking screen coordinate: (${args.x}, ${args.y})`;
1074
+ else if (name === 'computer_right_click')
1075
+ actionDesc = `Right clicking coordinate: (${args.x}, ${args.y})`;
1042
1076
  else if (name === 'computer_double_click')
1043
1077
  actionDesc = `Double clicking coordinate: (${args.x}, ${args.y})`;
1078
+ else if (name === 'computer_mouse_down')
1079
+ actionDesc = `Mouse down (${args.button || 'left'})`;
1080
+ else if (name === 'computer_mouse_up')
1081
+ actionDesc = `Mouse up (${args.button || 'left'})`;
1082
+ else if (name === 'computer_mouse_move')
1083
+ actionDesc = `Moving mouse to: (${args.x}, ${args.y})`;
1084
+ else if (name === 'computer_drag')
1085
+ actionDesc = `Dragging from (${args.fromX ?? args.startX}, ${args.fromY ?? args.startY}) to (${args.toX ?? args.endX}, ${args.toY ?? args.endY})`;
1044
1086
  else if (name === 'computer_type')
1045
1087
  actionDesc = `Typing text: "${String(args.text || '').slice(0, 30)}"`;
1046
1088
  else if (name === 'computer_key_press')
@@ -1055,11 +1097,68 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
1055
1097
  actionDesc = `Switching window: ${args.windowTitleOrApp || ''}`;
1056
1098
  else if (name === 'computer_inspect_ui')
1057
1099
  actionDesc = `Inspecting visible desktop UI`;
1100
+ else if (name === 'computer_diagnostic')
1101
+ actionDesc = `Running computer control diagnostics`;
1102
+ else if (name === 'computer')
1103
+ actionDesc = `Computer action: ${args.action || 'action'}`;
1104
+ else if (name === 'scratchpad')
1105
+ actionDesc = `Scratchpad (${args.action || 'read'})`;
1058
1106
  else if (name === 'task_completed')
1059
1107
  actionDesc = `Task finished`;
1060
1108
  const badge = chalk.bgCyan.black.bold(` 🛠️ TOOL: ${name} `);
1061
1109
  const descText = actionDesc ? chalk.yellow.bold(` → ${actionDesc}`) : '';
1062
1110
  console.log(`\n${badge}${descText} ${chalk.dim(JSON.stringify(args))}`);
1111
+ const isPaid = this.mockMode || this.userTier === 'pro' || this.userTier === 'enterprise';
1112
+ // 1. Scratchpad Feature Gating (Pro & Enterprise only)
1113
+ if (name === 'scratchpad') {
1114
+ if (!isPaid) {
1115
+ const blockedMsg = `[PLAN RESTRICTION: PRO / ENTERPRISE ONLY] The Scratchpad reasoning memory is exclusive to Pro and Enterprise tiers. Free tier developers can perform basic code editing (edit_file), creating files and folders (write_file), and running commands (run_command). To unlock the AI scratchpad, upgrade at https://tryscout.web.app/plans or run 'scout dashboard'.`;
1116
+ console.log(chalk.red.bold(`\n⛔ ${blockedMsg}`));
1117
+ this.taskStateManager.addObservation(name, blockedMsg, false);
1118
+ return { result: blockedMsg };
1119
+ }
1120
+ const action = String(args.action || 'read').toLowerCase().trim();
1121
+ const content = String(args.content || '');
1122
+ const ftDir = path.join(this.cwd, '.ft');
1123
+ const scratchPath = path.join(ftDir, 'scratchpad.md');
1124
+ fs.mkdirSync(ftDir, { recursive: true });
1125
+ let currentScratch = '';
1126
+ if (fs.existsSync(scratchPath)) {
1127
+ try {
1128
+ currentScratch = fs.readFileSync(scratchPath, 'utf-8');
1129
+ }
1130
+ catch { }
1131
+ }
1132
+ if (action === 'write') {
1133
+ fs.writeFileSync(scratchPath, content, 'utf-8');
1134
+ this.taskStateManager.addObservation(name, `Scratchpad updated (${content.length} chars)`, true);
1135
+ return { result: `Scratchpad updated successfully:\n${content}` };
1136
+ }
1137
+ else if (action === 'append') {
1138
+ const updated = currentScratch ? `${currentScratch}\n\n---\n<!-- ${new Date().toISOString()} -->\n${content}` : content;
1139
+ fs.writeFileSync(scratchPath, updated, 'utf-8');
1140
+ this.taskStateManager.addObservation(name, `Appended to scratchpad (${content.length} chars)`, true);
1141
+ return { result: `Appended to scratchpad. Current contents:\n${updated}` };
1142
+ }
1143
+ else if (action === 'clear') {
1144
+ fs.writeFileSync(scratchPath, '', 'utf-8');
1145
+ this.taskStateManager.addObservation(name, `Cleared scratchpad`, true);
1146
+ return { result: `Scratchpad cleared.` };
1147
+ }
1148
+ else {
1149
+ this.taskStateManager.addObservation(name, `Read scratchpad (${currentScratch.length} chars)`, true);
1150
+ return { result: currentScratch ? `Scratchpad Contents:\n${currentScratch}` : `Scratchpad is currently empty.` };
1151
+ }
1152
+ }
1153
+ // 2. Autonomous Computer & Browser Automation Gating (Pro & Enterprise only)
1154
+ if (name === 'computer' || name.startsWith('computer_') || name.startsWith('browser_')) {
1155
+ if (!isPaid) {
1156
+ const blockedMsg = `[PLAN RESTRICTION: PRO / ENTERPRISE ONLY] Autonomous Computer-Use & Browser Automation features are exclusive to Pro and Enterprise tiers. Free tier developers can perform basic code editing (edit_file), creating files and folders (write_file), and running commands (run_command) in the workspace. Upgrade to Pro at https://tryscout.web.app/plans or run 'scout dashboard'.`;
1157
+ console.log(chalk.red.bold(`\n⛔ ${blockedMsg}`));
1158
+ this.taskStateManager.addObservation(name, blockedMsg, false);
1159
+ return { result: blockedMsg };
1160
+ }
1161
+ }
1063
1162
  // Safety and human approval classification layer
1064
1163
  const risk = this.approvalManager.checkAction(name, args);
1065
1164
  if (risk.requiresApproval && !this.autoApprove) {
@@ -1099,6 +1198,64 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
1099
1198
  }
1100
1199
  }
1101
1200
  }
1201
+ if (name === 'computer') {
1202
+ const action = String(args.action || 'screenshot').toLowerCase().trim();
1203
+ if (action === 'click' || action === 'left_click') {
1204
+ name = 'computer_click';
1205
+ }
1206
+ else if (action === 'right_click') {
1207
+ name = 'computer_right_click';
1208
+ }
1209
+ else if (action === 'double_click') {
1210
+ name = 'computer_double_click';
1211
+ }
1212
+ else if (action === 'move' || action === 'mouse_move') {
1213
+ name = 'computer_mouse_move';
1214
+ }
1215
+ else if (action === 'mouse_down') {
1216
+ name = 'computer_mouse_down';
1217
+ }
1218
+ else if (action === 'mouse_up') {
1219
+ name = 'computer_mouse_up';
1220
+ }
1221
+ else if (action === 'drag') {
1222
+ name = 'computer_drag';
1223
+ args.fromX = args.startX ?? args.fromX;
1224
+ args.fromY = args.startY ?? args.fromY;
1225
+ args.toX = args.endX ?? args.toX;
1226
+ args.toY = args.endY ?? args.toY;
1227
+ }
1228
+ else if (action === 'scroll') {
1229
+ name = 'computer_scroll';
1230
+ }
1231
+ else if (action === 'screenshot') {
1232
+ name = 'computer_screenshot';
1233
+ }
1234
+ else if (action === 'type') {
1235
+ name = 'computer_type';
1236
+ }
1237
+ else if (action === 'key_press' || action === 'key') {
1238
+ name = 'computer_key_press';
1239
+ }
1240
+ else if (action === 'hotkey') {
1241
+ name = 'computer_hotkey';
1242
+ }
1243
+ else if (action === 'open_app') {
1244
+ name = 'computer_open_app';
1245
+ }
1246
+ else if (action === 'close_app') {
1247
+ name = 'computer_close_app';
1248
+ }
1249
+ else if (action === 'switch_window') {
1250
+ name = 'computer_switch_window';
1251
+ }
1252
+ else if (action === 'inspect_ui') {
1253
+ name = 'computer_inspect_ui';
1254
+ }
1255
+ else if (action === 'diagnostic') {
1256
+ name = 'computer_diagnostic';
1257
+ }
1258
+ }
1102
1259
  switch (name) {
1103
1260
  case 'read_file': {
1104
1261
  const targetPath = String(args.path || '').trim();
@@ -1580,11 +1737,31 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
1580
1737
  case 'computer_click': {
1581
1738
  const x = Number(args.x);
1582
1739
  const y = Number(args.y);
1583
- const res = await this.computerController.click(x, y, { button: args.button });
1740
+ const btn = args.button || 'left';
1741
+ const res = await this.computerController.click(x, y, { button: btn, clicks: 1 });
1584
1742
  this.taskStateManager.addObservation('computer_click', res.output, res.success, `(${x}, ${y})`);
1585
1743
  const structuredRes = {
1586
1744
  success: res.success,
1587
1745
  action: 'click',
1746
+ coordinates: res.coordinates || { x, y },
1747
+ button: btn,
1748
+ output: res.output,
1749
+ ...(res.error ? { error: res.error } : {}),
1750
+ recoverable: res.recoverable ?? true,
1751
+ ...(res.activeWindow ? { activeWindow: res.activeWindow } : {}),
1752
+ };
1753
+ return { result: JSON.stringify(structuredRes, null, 2) };
1754
+ }
1755
+ case 'computer_right_click': {
1756
+ const x = Number(args.x);
1757
+ const y = Number(args.y);
1758
+ const res = await this.computerController.rightClick(x, y);
1759
+ this.taskStateManager.addObservation('computer_right_click', res.output, res.success, `(${x}, ${y})`);
1760
+ const structuredRes = {
1761
+ success: res.success,
1762
+ action: 'right_click',
1763
+ coordinates: res.coordinates || { x, y },
1764
+ button: 'right',
1588
1765
  output: res.output,
1589
1766
  ...(res.error ? { error: res.error } : {}),
1590
1767
  recoverable: res.recoverable ?? true,
@@ -1600,6 +1777,42 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
1600
1777
  const structuredRes = {
1601
1778
  success: res.success,
1602
1779
  action: 'double_click',
1780
+ coordinates: res.coordinates || { x, y },
1781
+ button: 'left',
1782
+ output: res.output,
1783
+ ...(res.error ? { error: res.error } : {}),
1784
+ recoverable: res.recoverable ?? true,
1785
+ };
1786
+ return { result: JSON.stringify(structuredRes, null, 2) };
1787
+ }
1788
+ case 'computer_mouse_down': {
1789
+ const x = args.x !== undefined ? Number(args.x) : undefined;
1790
+ const y = args.y !== undefined ? Number(args.y) : undefined;
1791
+ const btn = args.button || 'left';
1792
+ const res = await this.computerController.mouseDown(x, y, btn);
1793
+ this.taskStateManager.addObservation('computer_mouse_down', res.output, res.success);
1794
+ const structuredRes = {
1795
+ success: res.success,
1796
+ action: 'mouse_down',
1797
+ coordinates: res.coordinates || (x !== undefined && y !== undefined ? { x, y } : undefined),
1798
+ button: btn,
1799
+ output: res.output,
1800
+ ...(res.error ? { error: res.error } : {}),
1801
+ recoverable: res.recoverable ?? true,
1802
+ };
1803
+ return { result: JSON.stringify(structuredRes, null, 2) };
1804
+ }
1805
+ case 'computer_mouse_up': {
1806
+ const x = args.x !== undefined ? Number(args.x) : undefined;
1807
+ const y = args.y !== undefined ? Number(args.y) : undefined;
1808
+ const btn = args.button || 'left';
1809
+ const res = await this.computerController.mouseUp(x, y, btn);
1810
+ this.taskStateManager.addObservation('computer_mouse_up', res.output, res.success);
1811
+ const structuredRes = {
1812
+ success: res.success,
1813
+ action: 'mouse_up',
1814
+ coordinates: res.coordinates || (x !== undefined && y !== undefined ? { x, y } : undefined),
1815
+ button: btn,
1603
1816
  output: res.output,
1604
1817
  ...(res.error ? { error: res.error } : {}),
1605
1818
  recoverable: res.recoverable ?? true,
@@ -1658,14 +1871,41 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
1658
1871
  case 'computer_mouse_move': {
1659
1872
  const x = Number(args.x);
1660
1873
  const y = Number(args.y);
1661
- const res = await this.computerController.mouseMove(x, y);
1874
+ const res = await this.computerController.move(x, y);
1662
1875
  this.taskStateManager.addObservation('computer_mouse_move', res.output, res.success);
1663
- return { result: res.output };
1876
+ const structuredRes = {
1877
+ success: res.success,
1878
+ action: 'move',
1879
+ coordinates: res.coordinates || { x, y },
1880
+ output: res.output,
1881
+ ...(res.error ? { error: res.error } : {}),
1882
+ recoverable: res.recoverable ?? true,
1883
+ ...(res.activeWindow ? { activeWindow: res.activeWindow } : {}),
1884
+ };
1885
+ return { result: JSON.stringify(structuredRes, null, 2) };
1664
1886
  }
1665
1887
  case 'computer_drag': {
1666
- const res = await this.computerController.drag(Number(args.fromX), Number(args.fromY), Number(args.toX), Number(args.toY));
1888
+ const fromX = Number(args.fromX ?? args.startX ?? 0);
1889
+ const fromY = Number(args.fromY ?? args.startY ?? 0);
1890
+ const toX = Number(args.toX ?? args.endX ?? 0);
1891
+ const toY = Number(args.toY ?? args.endY ?? 0);
1892
+ const res = await this.computerController.drag(fromX, fromY, toX, toY);
1667
1893
  this.taskStateManager.addObservation('computer_drag', res.output, res.success);
1668
- return { result: res.output };
1894
+ const structuredRes = {
1895
+ success: res.success,
1896
+ action: 'drag',
1897
+ from: { x: fromX, y: fromY },
1898
+ to: { x: toX, y: toY },
1899
+ coordinates: res.coordinates || { x: toX, y: toY },
1900
+ output: res.output,
1901
+ ...(res.error ? { error: res.error } : {}),
1902
+ recoverable: res.recoverable ?? true,
1903
+ };
1904
+ return { result: JSON.stringify(structuredRes, null, 2) };
1905
+ }
1906
+ case 'computer_diagnostic': {
1907
+ const diag = await this.computerController.runDiagnostic();
1908
+ return { result: JSON.stringify(diag, null, 2) };
1669
1909
  }
1670
1910
  case 'computer_wait': {
1671
1911
  const ms = Math.min(10000, Math.max(100, Number(args.ms) || 1000));