ft-scout 9.0.4 → 9.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/.firebase/hosting.d2Vi.cache +8 -7
  2. package/AUTONOMOUS_FEATURES.md +192 -0
  3. package/README.md +67 -35
  4. package/bin/src/commands/agent.d.ts.map +1 -1
  5. package/bin/src/commands/agent.js +21 -8
  6. package/bin/src/commands/agent.js.map +1 -1
  7. package/bin/src/commands/login.d.ts.map +1 -1
  8. package/bin/src/commands/login.js +3 -1
  9. package/bin/src/commands/login.js.map +1 -1
  10. package/bin/src/engine/actionVerifier.d.ts.map +1 -1
  11. package/bin/src/engine/actionVerifier.js +9 -1
  12. package/bin/src/engine/actionVerifier.js.map +1 -1
  13. package/bin/src/engine/agentEngine.d.ts.map +1 -1
  14. package/bin/src/engine/agentEngine.js +319 -13
  15. package/bin/src/engine/agentEngine.js.map +1 -1
  16. package/bin/src/engine/browser/browserController.d.ts.map +1 -1
  17. package/bin/src/engine/browser/browserController.js +53 -13
  18. package/bin/src/engine/browser/browserController.js.map +1 -1
  19. package/bin/src/engine/computer/computerController.d.ts.map +1 -1
  20. package/bin/src/engine/computer/computerController.js +564 -296
  21. package/bin/src/engine/computer/computerController.js.map +1 -1
  22. package/bin/src/engine/computer/computerDiagnostic.d.ts.map +1 -0
  23. package/bin/src/engine/computer/computerDiagnostic.js +71 -0
  24. package/bin/src/engine/computer/computerDiagnostic.js.map +1 -0
  25. package/bin/src/engine/computer/index.d.ts.map +1 -1
  26. package/bin/src/engine/computer/index.js +1 -0
  27. package/bin/src/engine/computer/index.js.map +1 -1
  28. package/bin/src/engine/computer/mockComputer.d.ts.map +1 -1
  29. package/bin/src/engine/computer/mockComputer.js +104 -8
  30. package/bin/src/engine/computer/mockComputer.js.map +1 -1
  31. package/bin/src/engine/computer/scoutDriver.cs +1161 -0
  32. package/bin/src/engine/computer/scout_driver.exe +0 -0
  33. package/bin/src/engine/runtime/agentRuntime.d.ts.map +1 -1
  34. package/bin/src/engine/runtime/agentRuntime.js +375 -20
  35. package/bin/src/engine/runtime/agentRuntime.js.map +1 -1
  36. package/bin/src/engine/safety.d.ts.map +1 -1
  37. package/bin/src/engine/safety.js +17 -6
  38. package/bin/src/engine/safety.js.map +1 -1
  39. package/bin/src/engine/toolRegistry.d.ts.map +1 -1
  40. package/bin/src/engine/toolRegistry.js +209 -0
  41. package/bin/src/engine/toolRegistry.js.map +1 -1
  42. package/bin/src/engine/workspace/agentWorkspace.d.ts.map +1 -1
  43. package/bin/src/engine/workspace/agentWorkspace.js +21 -0
  44. package/bin/src/engine/workspace/agentWorkspace.js.map +1 -1
  45. package/bin/src/index.js +5 -4
  46. package/bin/src/index.js.map +1 -1
  47. package/bin/src/utils/auth.d.ts.map +1 -1
  48. package/bin/src/utils/auth.js +14 -1
  49. package/bin/src/utils/auth.js.map +1 -1
  50. package/bin/src/utils/branding.js +1 -1
  51. package/bin/src/utils/holoseneAuth.d.ts.map +1 -1
  52. package/bin/src/utils/holoseneAuth.js +169 -41
  53. package/bin/src/utils/holoseneAuth.js.map +1 -1
  54. package/firebase.json +8 -0
  55. package/package.json +1 -1
  56. package/web/app.js +3 -3
  57. package/web/styles.css +2 -2
@@ -19,6 +19,7 @@ import { UnifiedToolRegistry } from './toolRegistry.js';
19
19
  import { SystemComputerController, MockComputerController } from './computer/index.js';
20
20
  import { AgentWorkspace, inferExecutionIntent, resolveDesktopPath } from './workspace/index.js';
21
21
  import { AgentRuntime } from './runtime/index.js';
22
+ import { loadAuthConfig, isProOrHigher } from '../utils/auth.js';
22
23
  export { AgentRuntime, AgentWorkspace, inferExecutionIntent, resolveDesktopPath, SystemComputerController, MockComputerController };
23
24
  const execAsync = promisify(exec);
24
25
  export function robustSnippetReplace(origContent, target, replacement) {
@@ -372,11 +373,16 @@ export class AgentExecutionLoop {
372
373
  isCancelled = false;
373
374
  cancelReason;
374
375
  executionIntent;
376
+ userTier;
375
377
  constructor(options) {
376
378
  this.cwd = process.cwd();
377
379
  this.autoApprove = Boolean(options?.autoApprove);
378
- this.maxSteps = options?.maxSteps || 30;
379
380
  this.mockMode = Boolean(options?.mockMode);
381
+ this.userTier = (options?.tier || loadAuthConfig().currentTier || 'free').toLowerCase();
382
+ // Free tier is bounded to basic direct actions (max 5 steps). Full autonomous multi-step loops require Pro/Enterprise.
383
+ const isPaid = this.mockMode || this.userTier === 'pro' || this.userTier === 'enterprise';
384
+ const requestedSteps = options?.maxSteps || 30;
385
+ this.maxSteps = isPaid ? requestedSteps : Math.min(requestedSteps, 5);
380
386
  this.taskStateManager = new TaskStateManager(options?.taskId, '', this.cwd);
381
387
  this.computerController = options?.computerController || (this.mockMode ? new MockComputerController() : new SystemComputerController(this.cwd));
382
388
  this.browserController = options?.browserController || new SemanticBrowserController(this.cwd);
@@ -385,6 +391,9 @@ export class AgentExecutionLoop {
385
391
  this.taskPlanner = new TaskPlanner();
386
392
  this.toolRegistry = new UnifiedToolRegistry();
387
393
  }
394
+ getUserTier() {
395
+ return this.userTier;
396
+ }
388
397
  getExecutionIntent() {
389
398
  return this.executionIntent;
390
399
  }
@@ -558,11 +567,27 @@ Task Execution Intent & Constraints:
558
567
  • Target Destination: ${this.executionIntent.targetDestination || 'Default'}
559
568
  • Target Artifact: ${this.executionIntent.targetArtifactName || 'None'}
560
569
  ` : '';
570
+ const isPaid = this.mockMode || this.userTier === 'pro' || this.userTier === 'enterprise';
571
+ const tierDirective = isPaid ? `
572
+ PLAN TIER & CAPABILITIES: ${this.userTier.toUpperCase()} (FULL ACCESS UNLOCKED)
573
+ • Autonomous Computer-Use, Browser Automation, and AI Scratchpad Reasoning are fully ENABLED.
574
+ • Use the 'scratchpad' tool to organize multi-step reasoning, break down complex goals, draft hypotheses, and track working memory across turns.
575
+ ` : `
576
+ PLAN TIER & CAPABILITIES: DEVELOPER FREE (BASIC DEVELOPER OPS ONLY)
577
+ • Scratchpadding and Autonomous Multi-Step Desktop/Browser Control are RESTRICTED to Pro & Enterprise plans.
578
+ • As a Free Tier developer assistant, you ONLY perform basic developer operations:
579
+ 1. Code editing: use edit_file or multi_edit_file to modify existing code.
580
+ 2. Creating new files & folders: use write_file to create new files and parent directories.
581
+ 3. Running commands: use run_command to execute terminal commands, builds, or tests.
582
+ 4. Reading & exploring: use read_file, list_dir, grep_search to inspect files.
583
+ • Do NOT attempt to use scratchpad, computer, or browser tools. Execute the requested file edit, file creation, or command directly and call task_completed immediately.
584
+ `;
561
585
  return `You are Scout Agent, an elite AI Principal Software Engineer & Autonomous Computer-Use Operator (developed by FrontTerrain).
562
586
 
563
587
  Working Directory: "${this.cwd}"
564
588
  Host OS Platform: "${process.platform}"
565
589
  Current Date & Time: "${new Date().toISOString()}" (Current Year: ${new Date().getFullYear()})
590
+ ${tierDirective}
566
591
  ${intentInfo}
567
592
  Workspace Overview (first 100 files):
568
593
  ${filesList.join('\n')}
@@ -611,7 +636,7 @@ When returning tool calls, use standard OpenAI function calling format or JSON t
611
636
  }
612
637
  const structuredPlan = this.taskPlanner.createPlan(userGoal);
613
638
  this.taskStateManager.setPlan(structuredPlan.steps);
614
- this.taskStateManager.setAvailableTools(this.toolRegistry.getAllTools().map((t) => t.name));
639
+ this.taskStateManager.setAvailableTools(this.toolRegistry.getToolsForTier(this.userTier).map((t) => t.name));
615
640
  if (this.mockMode) {
616
641
  for (let i = 1; i <= structuredPlan.steps.length; i++) {
617
642
  this.taskStateManager.startStep(i);
@@ -766,7 +791,7 @@ When returning tool calls, use standard OpenAI function calling format or JSON t
766
791
  return await openai.chat.completions.create({
767
792
  model,
768
793
  messages: sanitizeMessages(this.historyMessages),
769
- tools: this.toolRegistry.getAllTools().map((t) => ({ type: 'function', function: t })),
794
+ tools: this.toolRegistry.getToolsForTier(this.userTier).map((t) => ({ type: 'function', function: t })),
770
795
  tool_choice: 'auto',
771
796
  temperature: 0.1,
772
797
  presence_penalty: 0.1,
@@ -937,10 +962,17 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
937
962
  content: `Error in previous turn: ${err?.message || String(err)}. Please try alternative steps or call tools.`,
938
963
  });
939
964
  }
940
- // If current max steps limit is reached while task is still in progress, auto-extend by +15 extra steps (up to 60 max threshold)
941
- if (step >= this.maxSteps && this.maxSteps < 60) {
942
- this.maxSteps += 15;
943
- safeNote(chalk.bold.yellow(`⚡ Step limit reached while task is in progress. Automatically extending execution by +15 extra steps (New Max Limit: ${this.maxSteps})...`), ' Auto Extra Steps Extension');
965
+ // Step limit check: Auto-extend by +15 steps ONLY for Pro & Enterprise (up to 60 max threshold)
966
+ const isPaid = this.mockMode || this.userTier === 'pro' || this.userTier === 'enterprise';
967
+ if (step >= this.maxSteps) {
968
+ if (isPaid && this.maxSteps < 60) {
969
+ this.maxSteps += 15;
970
+ safeNote(chalk.bold.yellow(`⚡ Step limit reached while task is in progress. Automatically extending execution by +15 extra steps (New Max Limit: ${this.maxSteps})...`), ' Auto Extra Steps Extension');
971
+ }
972
+ else if (!isPaid) {
973
+ safeNote(chalk.bold.yellow(`⚡ Free Tier limit reached (${this.maxSteps} steps).\nFree developers can perform basic code editing, creating files/folders, and running commands.\nTo unlock unlimited autonomous multi-step loops and scratchpad reasoning, upgrade to Pro at https://scout.frontterrain.com/plans or run 'scout dashboard'.`), ' Free Tier Step Limit');
974
+ break;
975
+ }
944
976
  }
945
977
  }
946
978
  return {
@@ -967,7 +999,7 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
967
999
  'read_file', 'write_file', 'edit_file', 'run_command', 'list_dir',
968
1000
  'grep_search', 'glob_search', 'tree_view', 'file_info', 'multi_edit_file',
969
1001
  'fetch_url', 'git_diff',
970
- 'ask_user', 'speak_text', 'task_completed',
1002
+ 'ask_user', 'speak_text', 'task_completed', 'scratchpad',
971
1003
  ];
972
1004
  for (const toolName of knownTools) {
973
1005
  const tagRegex = new RegExp(`<${toolName}>([\\s\\S]*?)<\\/${toolName}>`, 'i');
@@ -1039,8 +1071,18 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
1039
1071
  actionDesc = `Observing computer screen`;
1040
1072
  else if (name === 'computer_click')
1041
1073
  actionDesc = `Clicking screen coordinate: (${args.x}, ${args.y})`;
1074
+ else if (name === 'computer_right_click')
1075
+ actionDesc = `Right clicking coordinate: (${args.x}, ${args.y})`;
1042
1076
  else if (name === 'computer_double_click')
1043
1077
  actionDesc = `Double clicking coordinate: (${args.x}, ${args.y})`;
1078
+ else if (name === 'computer_mouse_down')
1079
+ actionDesc = `Mouse down (${args.button || 'left'})`;
1080
+ else if (name === 'computer_mouse_up')
1081
+ actionDesc = `Mouse up (${args.button || 'left'})`;
1082
+ else if (name === 'computer_mouse_move')
1083
+ actionDesc = `Moving mouse to: (${args.x}, ${args.y})`;
1084
+ else if (name === 'computer_drag')
1085
+ actionDesc = `Dragging from (${args.fromX ?? args.startX}, ${args.fromY ?? args.startY}) to (${args.toX ?? args.endX}, ${args.toY ?? args.endY})`;
1044
1086
  else if (name === 'computer_type')
1045
1087
  actionDesc = `Typing text: "${String(args.text || '').slice(0, 30)}"`;
1046
1088
  else if (name === 'computer_key_press')
@@ -1055,11 +1097,68 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
1055
1097
  actionDesc = `Switching window: ${args.windowTitleOrApp || ''}`;
1056
1098
  else if (name === 'computer_inspect_ui')
1057
1099
  actionDesc = `Inspecting visible desktop UI`;
1100
+ else if (name === 'computer_diagnostic')
1101
+ actionDesc = `Running computer control diagnostics`;
1102
+ else if (name === 'computer')
1103
+ actionDesc = `Computer action: ${args.action || 'action'}`;
1104
+ else if (name === 'scratchpad')
1105
+ actionDesc = `Scratchpad (${args.action || 'read'})`;
1058
1106
  else if (name === 'task_completed')
1059
1107
  actionDesc = `Task finished`;
1060
1108
  const badge = chalk.bgCyan.black.bold(` 🛠️ TOOL: ${name} `);
1061
1109
  const descText = actionDesc ? chalk.yellow.bold(` → ${actionDesc}`) : '';
1062
1110
  console.log(`\n${badge}${descText} ${chalk.dim(JSON.stringify(args))}`);
1111
+ const isPaid = this.mockMode || this.userTier === 'pro' || this.userTier === 'enterprise';
1112
+ // 1. Scratchpad Feature Gating (Pro & Enterprise only)
1113
+ if (name === 'scratchpad') {
1114
+ if (!isPaid) {
1115
+ const blockedMsg = `[PLAN RESTRICTION: PRO / ENTERPRISE ONLY] The Scratchpad reasoning memory is exclusive to Pro and Enterprise tiers. Free tier developers can perform basic code editing (edit_file), creating files and folders (write_file), and running commands (run_command). To unlock the AI scratchpad, upgrade at https://scout.frontterrain.com/plans or run 'scout dashboard'.`;
1116
+ console.log(chalk.red.bold(`\n⛔ ${blockedMsg}`));
1117
+ this.taskStateManager.addObservation(name, blockedMsg, false);
1118
+ return { result: blockedMsg };
1119
+ }
1120
+ const action = String(args.action || 'read').toLowerCase().trim();
1121
+ const content = String(args.content || '');
1122
+ const ftDir = path.join(this.cwd, '.ft');
1123
+ const scratchPath = path.join(ftDir, 'scratchpad.md');
1124
+ fs.mkdirSync(ftDir, { recursive: true });
1125
+ let currentScratch = '';
1126
+ if (fs.existsSync(scratchPath)) {
1127
+ try {
1128
+ currentScratch = fs.readFileSync(scratchPath, 'utf-8');
1129
+ }
1130
+ catch { }
1131
+ }
1132
+ if (action === 'write') {
1133
+ fs.writeFileSync(scratchPath, content, 'utf-8');
1134
+ this.taskStateManager.addObservation(name, `Scratchpad updated (${content.length} chars)`, true);
1135
+ return { result: `Scratchpad updated successfully:\n${content}` };
1136
+ }
1137
+ else if (action === 'append') {
1138
+ const updated = currentScratch ? `${currentScratch}\n\n---\n<!-- ${new Date().toISOString()} -->\n${content}` : content;
1139
+ fs.writeFileSync(scratchPath, updated, 'utf-8');
1140
+ this.taskStateManager.addObservation(name, `Appended to scratchpad (${content.length} chars)`, true);
1141
+ return { result: `Appended to scratchpad. Current contents:\n${updated}` };
1142
+ }
1143
+ else if (action === 'clear') {
1144
+ fs.writeFileSync(scratchPath, '', 'utf-8');
1145
+ this.taskStateManager.addObservation(name, `Cleared scratchpad`, true);
1146
+ return { result: `Scratchpad cleared.` };
1147
+ }
1148
+ else {
1149
+ this.taskStateManager.addObservation(name, `Read scratchpad (${currentScratch.length} chars)`, true);
1150
+ return { result: currentScratch ? `Scratchpad Contents:\n${currentScratch}` : `Scratchpad is currently empty.` };
1151
+ }
1152
+ }
1153
+ // 2. Autonomous Computer & Browser Automation Gating (Pro & Enterprise only)
1154
+ if (name === 'computer' || name.startsWith('computer_') || name.startsWith('browser_')) {
1155
+ if (!isPaid) {
1156
+ const blockedMsg = `[PLAN RESTRICTION: PRO / ENTERPRISE ONLY] Autonomous Computer-Use & Browser Automation features are exclusive to Pro and Enterprise tiers. Free tier developers can perform basic code editing (edit_file), creating files and folders (write_file), and running commands (run_command) in the workspace. Upgrade to Pro at https://scout.frontterrain.com/plans or run 'scout dashboard'.`;
1157
+ console.log(chalk.red.bold(`\n⛔ ${blockedMsg}`));
1158
+ this.taskStateManager.addObservation(name, blockedMsg, false);
1159
+ return { result: blockedMsg };
1160
+ }
1161
+ }
1063
1162
  // Safety and human approval classification layer
1064
1163
  const risk = this.approvalManager.checkAction(name, args);
1065
1164
  if (risk.requiresApproval && !this.autoApprove) {
@@ -1099,6 +1198,82 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
1099
1198
  }
1100
1199
  }
1101
1200
  }
1201
+ if (name === 'computer') {
1202
+ const action = String(args.action || 'screenshot').toLowerCase().trim();
1203
+ if (action === 'click' || action === 'left_click') {
1204
+ name = 'computer_click';
1205
+ }
1206
+ else if (action === 'right_click') {
1207
+ name = 'computer_right_click';
1208
+ }
1209
+ else if (action === 'double_click') {
1210
+ name = 'computer_double_click';
1211
+ }
1212
+ else if (action === 'move' || action === 'mouse_move') {
1213
+ name = 'computer_mouse_move';
1214
+ }
1215
+ else if (action === 'mouse_down') {
1216
+ name = 'computer_mouse_down';
1217
+ }
1218
+ else if (action === 'mouse_up') {
1219
+ name = 'computer_mouse_up';
1220
+ }
1221
+ else if (action === 'drag') {
1222
+ name = 'computer_drag';
1223
+ args.fromX = args.startX ?? args.fromX;
1224
+ args.fromY = args.startY ?? args.fromY;
1225
+ args.toX = args.endX ?? args.toX;
1226
+ args.toY = args.endY ?? args.toY;
1227
+ }
1228
+ else if (action === 'scroll') {
1229
+ name = 'computer_scroll';
1230
+ }
1231
+ else if (action === 'screenshot') {
1232
+ name = 'computer_screenshot';
1233
+ }
1234
+ else if (action === 'type') {
1235
+ name = 'computer_type';
1236
+ }
1237
+ else if (action === 'key_press' || action === 'key') {
1238
+ name = 'computer_key_press';
1239
+ }
1240
+ else if (action === 'hotkey') {
1241
+ name = 'computer_hotkey';
1242
+ }
1243
+ else if (action === 'open_app') {
1244
+ name = 'computer_open_app';
1245
+ }
1246
+ else if (action === 'close_app') {
1247
+ name = 'computer_close_app';
1248
+ }
1249
+ else if (action === 'switch_window') {
1250
+ name = 'computer_switch_window';
1251
+ }
1252
+ else if (action === 'inspect_ui') {
1253
+ name = 'computer_inspect_ui';
1254
+ }
1255
+ else if (action === 'diagnostic') {
1256
+ name = 'computer_diagnostic';
1257
+ }
1258
+ }
1259
+ // Tier Gating Enforcement
1260
+ if (!isPaid) {
1261
+ if (name.startsWith('computer_') || name === 'computer') {
1262
+ const upgradeMsg = `[PLAN RESTRICTION]: Autonomous Computer Control tools (${name}) are exclusively available to Pro and Enterprise plan developers. Free tier developers can perform code editing, creating files/folders, and running terminal commands. Upgrade to Pro at https://scout.frontterrain.com to unlock autonomous computer & app control.`;
1263
+ this.taskStateManager.addObservation(name, upgradeMsg, false);
1264
+ return { result: upgradeMsg };
1265
+ }
1266
+ if (name.startsWith('browser_')) {
1267
+ const upgradeMsg = `[PLAN RESTRICTION]: Autonomous Browser Control tools (${name}) are exclusively available to Pro and Enterprise plan developers. Free tier developers can perform code editing, creating files/folders, and running terminal commands. Upgrade to Pro at https://scout.frontterrain.com to unlock browser automation.`;
1268
+ this.taskStateManager.addObservation(name, upgradeMsg, false);
1269
+ return { result: upgradeMsg };
1270
+ }
1271
+ if (name === 'scratchpad') {
1272
+ const upgradeMsg = `[PLAN RESTRICTION]: AI Scratchpad & Autonomous reasoning is an exclusive feature for Pro and Enterprise plan subscribers. Free tier developers can perform code editing, creating files/folders, and running commands. Upgrade to Pro at https://scout.frontterrain.com to unlock the AI Scratchpad.`;
1273
+ this.taskStateManager.addObservation(name, upgradeMsg, false);
1274
+ return { result: upgradeMsg };
1275
+ }
1276
+ }
1102
1277
  switch (name) {
1103
1278
  case 'read_file': {
1104
1279
  const targetPath = String(args.path || '').trim();
@@ -1559,6 +1734,54 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
1559
1734
  this.taskStateManager.addObservation('browser_content', `Extracted content (${res.output.length} chars)`, res.success);
1560
1735
  return { result: res.output };
1561
1736
  }
1737
+ case 'browser_screenshot': {
1738
+ const res = await this.browserController.screenshot();
1739
+ this.taskStateManager.addObservation('browser_screenshot', res.output, res.success, res.url);
1740
+ return { result: res.output };
1741
+ }
1742
+ case 'browser_select': {
1743
+ const selector = String(args.selector || args.target || '');
1744
+ const val = String(args.value || args.val || '');
1745
+ const res = await this.browserController.select(selector, val);
1746
+ this.taskStateManager.addObservation('browser_select', res.output, res.success, selector);
1747
+ return { result: res.output };
1748
+ }
1749
+ case 'browser_scroll': {
1750
+ const direction = (args.direction || 'down');
1751
+ const amount = Number(args.amount) || 4;
1752
+ const res = await this.browserController.scroll(direction, amount);
1753
+ this.taskStateManager.addObservation('browser_scroll', res.output, res.success);
1754
+ return { result: res.output };
1755
+ }
1756
+ case 'scratchpad': {
1757
+ const action = String(args.action || 'read').toLowerCase().trim();
1758
+ const scratchpadPath = path.resolve(this.cwd, '.ft', 'scratchpad.md');
1759
+ fs.mkdirSync(path.dirname(scratchpadPath), { recursive: true });
1760
+ if (action === 'read') {
1761
+ const content = fs.existsSync(scratchpadPath) ? fs.readFileSync(scratchpadPath, 'utf-8') : '(Scratchpad is empty)';
1762
+ this.taskStateManager.addObservation('scratchpad', 'Read scratchpad content', true);
1763
+ return { result: `Scratchpad Content:\n${content || '(empty)'}` };
1764
+ }
1765
+ else if (action === 'write') {
1766
+ const newContent = String(args.content || '');
1767
+ fs.writeFileSync(scratchpadPath, newContent, 'utf-8');
1768
+ this.taskStateManager.addObservation('scratchpad', `Wrote scratchpad (${newContent.length} chars)`, true);
1769
+ return { result: `Scratchpad updated (${newContent.length} chars).` };
1770
+ }
1771
+ else if (action === 'append') {
1772
+ const appendContent = String(args.content || '');
1773
+ fs.appendFileSync(scratchpadPath, `\n${appendContent}`, 'utf-8');
1774
+ this.taskStateManager.addObservation('scratchpad', `Appended to scratchpad (${appendContent.length} chars)`, true);
1775
+ return { result: `Appended to scratchpad (${appendContent.length} chars).` };
1776
+ }
1777
+ else if (action === 'clear') {
1778
+ if (fs.existsSync(scratchpadPath))
1779
+ fs.unlinkSync(scratchpadPath);
1780
+ this.taskStateManager.addObservation('scratchpad', 'Cleared scratchpad', true);
1781
+ return { result: 'Scratchpad cleared.' };
1782
+ }
1783
+ return { result: `Unknown scratchpad action: "${action}". Supported: read, write, append, clear.` };
1784
+ }
1562
1785
  case 'computer_screenshot': {
1563
1786
  const res = await this.computerController.screenshot();
1564
1787
  if (res.path) {
@@ -1580,11 +1803,31 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
1580
1803
  case 'computer_click': {
1581
1804
  const x = Number(args.x);
1582
1805
  const y = Number(args.y);
1583
- const res = await this.computerController.click(x, y, { button: args.button });
1806
+ const btn = args.button || 'left';
1807
+ const res = await this.computerController.click(x, y, { button: btn, clicks: 1 });
1584
1808
  this.taskStateManager.addObservation('computer_click', res.output, res.success, `(${x}, ${y})`);
1585
1809
  const structuredRes = {
1586
1810
  success: res.success,
1587
1811
  action: 'click',
1812
+ coordinates: res.coordinates || { x, y },
1813
+ button: btn,
1814
+ output: res.output,
1815
+ ...(res.error ? { error: res.error } : {}),
1816
+ recoverable: res.recoverable ?? true,
1817
+ ...(res.activeWindow ? { activeWindow: res.activeWindow } : {}),
1818
+ };
1819
+ return { result: JSON.stringify(structuredRes, null, 2) };
1820
+ }
1821
+ case 'computer_right_click': {
1822
+ const x = Number(args.x);
1823
+ const y = Number(args.y);
1824
+ const res = await this.computerController.rightClick(x, y);
1825
+ this.taskStateManager.addObservation('computer_right_click', res.output, res.success, `(${x}, ${y})`);
1826
+ const structuredRes = {
1827
+ success: res.success,
1828
+ action: 'right_click',
1829
+ coordinates: res.coordinates || { x, y },
1830
+ button: 'right',
1588
1831
  output: res.output,
1589
1832
  ...(res.error ? { error: res.error } : {}),
1590
1833
  recoverable: res.recoverable ?? true,
@@ -1600,6 +1843,42 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
1600
1843
  const structuredRes = {
1601
1844
  success: res.success,
1602
1845
  action: 'double_click',
1846
+ coordinates: res.coordinates || { x, y },
1847
+ button: 'left',
1848
+ output: res.output,
1849
+ ...(res.error ? { error: res.error } : {}),
1850
+ recoverable: res.recoverable ?? true,
1851
+ };
1852
+ return { result: JSON.stringify(structuredRes, null, 2) };
1853
+ }
1854
+ case 'computer_mouse_down': {
1855
+ const x = args.x !== undefined ? Number(args.x) : undefined;
1856
+ const y = args.y !== undefined ? Number(args.y) : undefined;
1857
+ const btn = args.button || 'left';
1858
+ const res = await this.computerController.mouseDown(x, y, btn);
1859
+ this.taskStateManager.addObservation('computer_mouse_down', res.output, res.success);
1860
+ const structuredRes = {
1861
+ success: res.success,
1862
+ action: 'mouse_down',
1863
+ coordinates: res.coordinates || (x !== undefined && y !== undefined ? { x, y } : undefined),
1864
+ button: btn,
1865
+ output: res.output,
1866
+ ...(res.error ? { error: res.error } : {}),
1867
+ recoverable: res.recoverable ?? true,
1868
+ };
1869
+ return { result: JSON.stringify(structuredRes, null, 2) };
1870
+ }
1871
+ case 'computer_mouse_up': {
1872
+ const x = args.x !== undefined ? Number(args.x) : undefined;
1873
+ const y = args.y !== undefined ? Number(args.y) : undefined;
1874
+ const btn = args.button || 'left';
1875
+ const res = await this.computerController.mouseUp(x, y, btn);
1876
+ this.taskStateManager.addObservation('computer_mouse_up', res.output, res.success);
1877
+ const structuredRes = {
1878
+ success: res.success,
1879
+ action: 'mouse_up',
1880
+ coordinates: res.coordinates || (x !== undefined && y !== undefined ? { x, y } : undefined),
1881
+ button: btn,
1603
1882
  output: res.output,
1604
1883
  ...(res.error ? { error: res.error } : {}),
1605
1884
  recoverable: res.recoverable ?? true,
@@ -1658,14 +1937,41 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
1658
1937
  case 'computer_mouse_move': {
1659
1938
  const x = Number(args.x);
1660
1939
  const y = Number(args.y);
1661
- const res = await this.computerController.mouseMove(x, y);
1940
+ const res = await this.computerController.move(x, y);
1662
1941
  this.taskStateManager.addObservation('computer_mouse_move', res.output, res.success);
1663
- return { result: res.output };
1942
+ const structuredRes = {
1943
+ success: res.success,
1944
+ action: 'move',
1945
+ coordinates: res.coordinates || { x, y },
1946
+ output: res.output,
1947
+ ...(res.error ? { error: res.error } : {}),
1948
+ recoverable: res.recoverable ?? true,
1949
+ ...(res.activeWindow ? { activeWindow: res.activeWindow } : {}),
1950
+ };
1951
+ return { result: JSON.stringify(structuredRes, null, 2) };
1664
1952
  }
1665
1953
  case 'computer_drag': {
1666
- const res = await this.computerController.drag(Number(args.fromX), Number(args.fromY), Number(args.toX), Number(args.toY));
1954
+ const fromX = Number(args.fromX ?? args.startX ?? 0);
1955
+ const fromY = Number(args.fromY ?? args.startY ?? 0);
1956
+ const toX = Number(args.toX ?? args.endX ?? 0);
1957
+ const toY = Number(args.toY ?? args.endY ?? 0);
1958
+ const res = await this.computerController.drag(fromX, fromY, toX, toY);
1667
1959
  this.taskStateManager.addObservation('computer_drag', res.output, res.success);
1668
- return { result: res.output };
1960
+ const structuredRes = {
1961
+ success: res.success,
1962
+ action: 'drag',
1963
+ from: { x: fromX, y: fromY },
1964
+ to: { x: toX, y: toY },
1965
+ coordinates: res.coordinates || { x: toX, y: toY },
1966
+ output: res.output,
1967
+ ...(res.error ? { error: res.error } : {}),
1968
+ recoverable: res.recoverable ?? true,
1969
+ };
1970
+ return { result: JSON.stringify(structuredRes, null, 2) };
1971
+ }
1972
+ case 'computer_diagnostic': {
1973
+ const diag = await this.computerController.runDiagnostic();
1974
+ return { result: JSON.stringify(diag, null, 2) };
1669
1975
  }
1670
1976
  case 'computer_wait': {
1671
1977
  const ms = Math.min(10000, Math.max(100, Number(args.ms) || 1000));