ft-scout 9.0.4 → 9.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.firebase/hosting.d2Vi.cache +7 -7
- package/README.md +2 -2
- package/bin/src/commands/agent.d.ts.map +1 -1
- package/bin/src/commands/agent.js +21 -8
- package/bin/src/commands/agent.js.map +1 -1
- package/bin/src/engine/agentEngine.d.ts.map +1 -1
- package/bin/src/engine/agentEngine.js +253 -13
- package/bin/src/engine/agentEngine.js.map +1 -1
- package/bin/src/engine/computer/computerController.d.ts.map +1 -1
- package/bin/src/engine/computer/computerController.js +532 -296
- package/bin/src/engine/computer/computerController.js.map +1 -1
- package/bin/src/engine/computer/computerDiagnostic.d.ts.map +1 -0
- package/bin/src/engine/computer/computerDiagnostic.js +71 -0
- package/bin/src/engine/computer/computerDiagnostic.js.map +1 -0
- package/bin/src/engine/computer/index.d.ts.map +1 -1
- package/bin/src/engine/computer/index.js +1 -0
- package/bin/src/engine/computer/index.js.map +1 -1
- package/bin/src/engine/computer/mockComputer.d.ts.map +1 -1
- package/bin/src/engine/computer/mockComputer.js +104 -8
- package/bin/src/engine/computer/mockComputer.js.map +1 -1
- package/bin/src/engine/computer/scoutDriver.cs +873 -0
- package/bin/src/engine/computer/scout_driver.exe +0 -0
- package/bin/src/engine/runtime/agentRuntime.d.ts.map +1 -1
- package/bin/src/engine/runtime/agentRuntime.js +351 -20
- package/bin/src/engine/runtime/agentRuntime.js.map +1 -1
- package/bin/src/engine/safety.d.ts.map +1 -1
- package/bin/src/engine/safety.js +13 -3
- package/bin/src/engine/safety.js.map +1 -1
- package/bin/src/engine/toolRegistry.d.ts.map +1 -1
- package/bin/src/engine/toolRegistry.js +144 -0
- package/bin/src/engine/toolRegistry.js.map +1 -1
- package/bin/src/engine/workspace/agentWorkspace.d.ts.map +1 -1
- package/bin/src/engine/workspace/agentWorkspace.js +21 -0
- package/bin/src/engine/workspace/agentWorkspace.js.map +1 -1
- package/bin/src/index.js +4 -4
- package/bin/src/utils/auth.d.ts.map +1 -1
- package/bin/src/utils/auth.js +14 -1
- package/bin/src/utils/auth.js.map +1 -1
- package/bin/src/utils/branding.js +1 -1
- package/package.json +1 -1
- package/web/app.js +2 -2
- package/web/styles.css +2 -2
|
@@ -19,6 +19,7 @@ import { UnifiedToolRegistry } from './toolRegistry.js';
|
|
|
19
19
|
import { SystemComputerController, MockComputerController } from './computer/index.js';
|
|
20
20
|
import { AgentWorkspace, inferExecutionIntent, resolveDesktopPath } from './workspace/index.js';
|
|
21
21
|
import { AgentRuntime } from './runtime/index.js';
|
|
22
|
+
import { loadAuthConfig, isProOrHigher } from '../utils/auth.js';
|
|
22
23
|
export { AgentRuntime, AgentWorkspace, inferExecutionIntent, resolveDesktopPath, SystemComputerController, MockComputerController };
|
|
23
24
|
const execAsync = promisify(exec);
|
|
24
25
|
export function robustSnippetReplace(origContent, target, replacement) {
|
|
@@ -372,11 +373,16 @@ export class AgentExecutionLoop {
|
|
|
372
373
|
isCancelled = false;
|
|
373
374
|
cancelReason;
|
|
374
375
|
executionIntent;
|
|
376
|
+
userTier;
|
|
375
377
|
constructor(options) {
|
|
376
378
|
this.cwd = process.cwd();
|
|
377
379
|
this.autoApprove = Boolean(options?.autoApprove);
|
|
378
|
-
this.maxSteps = options?.maxSteps || 30;
|
|
379
380
|
this.mockMode = Boolean(options?.mockMode);
|
|
381
|
+
this.userTier = (options?.tier || loadAuthConfig().currentTier || 'free').toLowerCase();
|
|
382
|
+
// Free tier is bounded to basic direct actions (max 5 steps). Full autonomous multi-step loops require Pro/Enterprise.
|
|
383
|
+
const isPaid = this.mockMode || this.userTier === 'pro' || this.userTier === 'enterprise';
|
|
384
|
+
const requestedSteps = options?.maxSteps || 30;
|
|
385
|
+
this.maxSteps = isPaid ? requestedSteps : Math.min(requestedSteps, 5);
|
|
380
386
|
this.taskStateManager = new TaskStateManager(options?.taskId, '', this.cwd);
|
|
381
387
|
this.computerController = options?.computerController || (this.mockMode ? new MockComputerController() : new SystemComputerController(this.cwd));
|
|
382
388
|
this.browserController = options?.browserController || new SemanticBrowserController(this.cwd);
|
|
@@ -385,6 +391,9 @@ export class AgentExecutionLoop {
|
|
|
385
391
|
this.taskPlanner = new TaskPlanner();
|
|
386
392
|
this.toolRegistry = new UnifiedToolRegistry();
|
|
387
393
|
}
|
|
394
|
+
getUserTier() {
|
|
395
|
+
return this.userTier;
|
|
396
|
+
}
|
|
388
397
|
getExecutionIntent() {
|
|
389
398
|
return this.executionIntent;
|
|
390
399
|
}
|
|
@@ -558,11 +567,27 @@ Task Execution Intent & Constraints:
|
|
|
558
567
|
• Target Destination: ${this.executionIntent.targetDestination || 'Default'}
|
|
559
568
|
• Target Artifact: ${this.executionIntent.targetArtifactName || 'None'}
|
|
560
569
|
` : '';
|
|
570
|
+
const isPaid = this.mockMode || this.userTier === 'pro' || this.userTier === 'enterprise';
|
|
571
|
+
const tierDirective = isPaid ? `
|
|
572
|
+
PLAN TIER & CAPABILITIES: ${this.userTier.toUpperCase()} (FULL ACCESS UNLOCKED)
|
|
573
|
+
• Autonomous Computer-Use, Browser Automation, and AI Scratchpad Reasoning are fully ENABLED.
|
|
574
|
+
• Use the 'scratchpad' tool to organize multi-step reasoning, break down complex goals, draft hypotheses, and track working memory across turns.
|
|
575
|
+
` : `
|
|
576
|
+
PLAN TIER & CAPABILITIES: DEVELOPER FREE (BASIC DEVELOPER OPS ONLY)
|
|
577
|
+
• Scratchpadding and Autonomous Multi-Step Desktop/Browser Control are RESTRICTED to Pro & Enterprise plans.
|
|
578
|
+
• As a Free Tier developer assistant, you ONLY perform basic developer operations:
|
|
579
|
+
1. Code editing: use edit_file or multi_edit_file to modify existing code.
|
|
580
|
+
2. Creating new files & folders: use write_file to create new files and parent directories.
|
|
581
|
+
3. Running commands: use run_command to execute terminal commands, builds, or tests.
|
|
582
|
+
4. Reading & exploring: use read_file, list_dir, grep_search to inspect files.
|
|
583
|
+
• Do NOT attempt to use scratchpad, computer, or browser tools. Execute the requested file edit, file creation, or command directly and call task_completed immediately.
|
|
584
|
+
`;
|
|
561
585
|
return `You are Scout Agent, an elite AI Principal Software Engineer & Autonomous Computer-Use Operator (developed by FrontTerrain).
|
|
562
586
|
|
|
563
587
|
Working Directory: "${this.cwd}"
|
|
564
588
|
Host OS Platform: "${process.platform}"
|
|
565
589
|
Current Date & Time: "${new Date().toISOString()}" (Current Year: ${new Date().getFullYear()})
|
|
590
|
+
${tierDirective}
|
|
566
591
|
${intentInfo}
|
|
567
592
|
Workspace Overview (first 100 files):
|
|
568
593
|
${filesList.join('\n')}
|
|
@@ -611,7 +636,7 @@ When returning tool calls, use standard OpenAI function calling format or JSON t
|
|
|
611
636
|
}
|
|
612
637
|
const structuredPlan = this.taskPlanner.createPlan(userGoal);
|
|
613
638
|
this.taskStateManager.setPlan(structuredPlan.steps);
|
|
614
|
-
this.taskStateManager.setAvailableTools(this.toolRegistry.
|
|
639
|
+
this.taskStateManager.setAvailableTools(this.toolRegistry.getToolsForTier(this.userTier).map((t) => t.name));
|
|
615
640
|
if (this.mockMode) {
|
|
616
641
|
for (let i = 1; i <= structuredPlan.steps.length; i++) {
|
|
617
642
|
this.taskStateManager.startStep(i);
|
|
@@ -766,7 +791,7 @@ When returning tool calls, use standard OpenAI function calling format or JSON t
|
|
|
766
791
|
return await openai.chat.completions.create({
|
|
767
792
|
model,
|
|
768
793
|
messages: sanitizeMessages(this.historyMessages),
|
|
769
|
-
tools: this.toolRegistry.
|
|
794
|
+
tools: this.toolRegistry.getToolsForTier(this.userTier).map((t) => ({ type: 'function', function: t })),
|
|
770
795
|
tool_choice: 'auto',
|
|
771
796
|
temperature: 0.1,
|
|
772
797
|
presence_penalty: 0.1,
|
|
@@ -937,10 +962,17 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
|
|
|
937
962
|
content: `Error in previous turn: ${err?.message || String(err)}. Please try alternative steps or call tools.`,
|
|
938
963
|
});
|
|
939
964
|
}
|
|
940
|
-
//
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
965
|
+
// Step limit check: Auto-extend by +15 steps ONLY for Pro & Enterprise (up to 60 max threshold)
|
|
966
|
+
const isPaid = this.mockMode || this.userTier === 'pro' || this.userTier === 'enterprise';
|
|
967
|
+
if (step >= this.maxSteps) {
|
|
968
|
+
if (isPaid && this.maxSteps < 60) {
|
|
969
|
+
this.maxSteps += 15;
|
|
970
|
+
safeNote(chalk.bold.yellow(`⚡ Step limit reached while task is in progress. Automatically extending execution by +15 extra steps (New Max Limit: ${this.maxSteps})...`), ' Auto Extra Steps Extension');
|
|
971
|
+
}
|
|
972
|
+
else if (!isPaid) {
|
|
973
|
+
safeNote(chalk.bold.yellow(`⚡ Free Tier limit reached (${this.maxSteps} steps).\nFree developers can perform basic code editing, creating files/folders, and running commands.\nTo unlock unlimited autonomous multi-step loops and scratchpad reasoning, upgrade to Pro at https://tryscout.web.app/plans or run 'scout dashboard'.`), ' Free Tier Step Limit');
|
|
974
|
+
break;
|
|
975
|
+
}
|
|
944
976
|
}
|
|
945
977
|
}
|
|
946
978
|
return {
|
|
@@ -967,7 +999,7 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
|
|
|
967
999
|
'read_file', 'write_file', 'edit_file', 'run_command', 'list_dir',
|
|
968
1000
|
'grep_search', 'glob_search', 'tree_view', 'file_info', 'multi_edit_file',
|
|
969
1001
|
'fetch_url', 'git_diff',
|
|
970
|
-
'ask_user', 'speak_text', 'task_completed',
|
|
1002
|
+
'ask_user', 'speak_text', 'task_completed', 'scratchpad',
|
|
971
1003
|
];
|
|
972
1004
|
for (const toolName of knownTools) {
|
|
973
1005
|
const tagRegex = new RegExp(`<${toolName}>([\\s\\S]*?)<\\/${toolName}>`, 'i');
|
|
@@ -1039,8 +1071,18 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
|
|
|
1039
1071
|
actionDesc = `Observing computer screen`;
|
|
1040
1072
|
else if (name === 'computer_click')
|
|
1041
1073
|
actionDesc = `Clicking screen coordinate: (${args.x}, ${args.y})`;
|
|
1074
|
+
else if (name === 'computer_right_click')
|
|
1075
|
+
actionDesc = `Right clicking coordinate: (${args.x}, ${args.y})`;
|
|
1042
1076
|
else if (name === 'computer_double_click')
|
|
1043
1077
|
actionDesc = `Double clicking coordinate: (${args.x}, ${args.y})`;
|
|
1078
|
+
else if (name === 'computer_mouse_down')
|
|
1079
|
+
actionDesc = `Mouse down (${args.button || 'left'})`;
|
|
1080
|
+
else if (name === 'computer_mouse_up')
|
|
1081
|
+
actionDesc = `Mouse up (${args.button || 'left'})`;
|
|
1082
|
+
else if (name === 'computer_mouse_move')
|
|
1083
|
+
actionDesc = `Moving mouse to: (${args.x}, ${args.y})`;
|
|
1084
|
+
else if (name === 'computer_drag')
|
|
1085
|
+
actionDesc = `Dragging from (${args.fromX ?? args.startX}, ${args.fromY ?? args.startY}) to (${args.toX ?? args.endX}, ${args.toY ?? args.endY})`;
|
|
1044
1086
|
else if (name === 'computer_type')
|
|
1045
1087
|
actionDesc = `Typing text: "${String(args.text || '').slice(0, 30)}"`;
|
|
1046
1088
|
else if (name === 'computer_key_press')
|
|
@@ -1055,11 +1097,68 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
|
|
|
1055
1097
|
actionDesc = `Switching window: ${args.windowTitleOrApp || ''}`;
|
|
1056
1098
|
else if (name === 'computer_inspect_ui')
|
|
1057
1099
|
actionDesc = `Inspecting visible desktop UI`;
|
|
1100
|
+
else if (name === 'computer_diagnostic')
|
|
1101
|
+
actionDesc = `Running computer control diagnostics`;
|
|
1102
|
+
else if (name === 'computer')
|
|
1103
|
+
actionDesc = `Computer action: ${args.action || 'action'}`;
|
|
1104
|
+
else if (name === 'scratchpad')
|
|
1105
|
+
actionDesc = `Scratchpad (${args.action || 'read'})`;
|
|
1058
1106
|
else if (name === 'task_completed')
|
|
1059
1107
|
actionDesc = `Task finished`;
|
|
1060
1108
|
const badge = chalk.bgCyan.black.bold(` 🛠️ TOOL: ${name} `);
|
|
1061
1109
|
const descText = actionDesc ? chalk.yellow.bold(` → ${actionDesc}`) : '';
|
|
1062
1110
|
console.log(`\n${badge}${descText} ${chalk.dim(JSON.stringify(args))}`);
|
|
1111
|
+
const isPaid = this.mockMode || this.userTier === 'pro' || this.userTier === 'enterprise';
|
|
1112
|
+
// 1. Scratchpad Feature Gating (Pro & Enterprise only)
|
|
1113
|
+
if (name === 'scratchpad') {
|
|
1114
|
+
if (!isPaid) {
|
|
1115
|
+
const blockedMsg = `[PLAN RESTRICTION: PRO / ENTERPRISE ONLY] The Scratchpad reasoning memory is exclusive to Pro and Enterprise tiers. Free tier developers can perform basic code editing (edit_file), creating files and folders (write_file), and running commands (run_command). To unlock the AI scratchpad, upgrade at https://tryscout.web.app/plans or run 'scout dashboard'.`;
|
|
1116
|
+
console.log(chalk.red.bold(`\n⛔ ${blockedMsg}`));
|
|
1117
|
+
this.taskStateManager.addObservation(name, blockedMsg, false);
|
|
1118
|
+
return { result: blockedMsg };
|
|
1119
|
+
}
|
|
1120
|
+
const action = String(args.action || 'read').toLowerCase().trim();
|
|
1121
|
+
const content = String(args.content || '');
|
|
1122
|
+
const ftDir = path.join(this.cwd, '.ft');
|
|
1123
|
+
const scratchPath = path.join(ftDir, 'scratchpad.md');
|
|
1124
|
+
fs.mkdirSync(ftDir, { recursive: true });
|
|
1125
|
+
let currentScratch = '';
|
|
1126
|
+
if (fs.existsSync(scratchPath)) {
|
|
1127
|
+
try {
|
|
1128
|
+
currentScratch = fs.readFileSync(scratchPath, 'utf-8');
|
|
1129
|
+
}
|
|
1130
|
+
catch { }
|
|
1131
|
+
}
|
|
1132
|
+
if (action === 'write') {
|
|
1133
|
+
fs.writeFileSync(scratchPath, content, 'utf-8');
|
|
1134
|
+
this.taskStateManager.addObservation(name, `Scratchpad updated (${content.length} chars)`, true);
|
|
1135
|
+
return { result: `Scratchpad updated successfully:\n${content}` };
|
|
1136
|
+
}
|
|
1137
|
+
else if (action === 'append') {
|
|
1138
|
+
const updated = currentScratch ? `${currentScratch}\n\n---\n<!-- ${new Date().toISOString()} -->\n${content}` : content;
|
|
1139
|
+
fs.writeFileSync(scratchPath, updated, 'utf-8');
|
|
1140
|
+
this.taskStateManager.addObservation(name, `Appended to scratchpad (${content.length} chars)`, true);
|
|
1141
|
+
return { result: `Appended to scratchpad. Current contents:\n${updated}` };
|
|
1142
|
+
}
|
|
1143
|
+
else if (action === 'clear') {
|
|
1144
|
+
fs.writeFileSync(scratchPath, '', 'utf-8');
|
|
1145
|
+
this.taskStateManager.addObservation(name, `Cleared scratchpad`, true);
|
|
1146
|
+
return { result: `Scratchpad cleared.` };
|
|
1147
|
+
}
|
|
1148
|
+
else {
|
|
1149
|
+
this.taskStateManager.addObservation(name, `Read scratchpad (${currentScratch.length} chars)`, true);
|
|
1150
|
+
return { result: currentScratch ? `Scratchpad Contents:\n${currentScratch}` : `Scratchpad is currently empty.` };
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
// 2. Autonomous Computer & Browser Automation Gating (Pro & Enterprise only)
|
|
1154
|
+
if (name === 'computer' || name.startsWith('computer_') || name.startsWith('browser_')) {
|
|
1155
|
+
if (!isPaid) {
|
|
1156
|
+
const blockedMsg = `[PLAN RESTRICTION: PRO / ENTERPRISE ONLY] Autonomous Computer-Use & Browser Automation features are exclusive to Pro and Enterprise tiers. Free tier developers can perform basic code editing (edit_file), creating files and folders (write_file), and running commands (run_command) in the workspace. Upgrade to Pro at https://tryscout.web.app/plans or run 'scout dashboard'.`;
|
|
1157
|
+
console.log(chalk.red.bold(`\n⛔ ${blockedMsg}`));
|
|
1158
|
+
this.taskStateManager.addObservation(name, blockedMsg, false);
|
|
1159
|
+
return { result: blockedMsg };
|
|
1160
|
+
}
|
|
1161
|
+
}
|
|
1063
1162
|
// Safety and human approval classification layer
|
|
1064
1163
|
const risk = this.approvalManager.checkAction(name, args);
|
|
1065
1164
|
if (risk.requiresApproval && !this.autoApprove) {
|
|
@@ -1099,6 +1198,64 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
|
|
|
1099
1198
|
}
|
|
1100
1199
|
}
|
|
1101
1200
|
}
|
|
1201
|
+
if (name === 'computer') {
|
|
1202
|
+
const action = String(args.action || 'screenshot').toLowerCase().trim();
|
|
1203
|
+
if (action === 'click' || action === 'left_click') {
|
|
1204
|
+
name = 'computer_click';
|
|
1205
|
+
}
|
|
1206
|
+
else if (action === 'right_click') {
|
|
1207
|
+
name = 'computer_right_click';
|
|
1208
|
+
}
|
|
1209
|
+
else if (action === 'double_click') {
|
|
1210
|
+
name = 'computer_double_click';
|
|
1211
|
+
}
|
|
1212
|
+
else if (action === 'move' || action === 'mouse_move') {
|
|
1213
|
+
name = 'computer_mouse_move';
|
|
1214
|
+
}
|
|
1215
|
+
else if (action === 'mouse_down') {
|
|
1216
|
+
name = 'computer_mouse_down';
|
|
1217
|
+
}
|
|
1218
|
+
else if (action === 'mouse_up') {
|
|
1219
|
+
name = 'computer_mouse_up';
|
|
1220
|
+
}
|
|
1221
|
+
else if (action === 'drag') {
|
|
1222
|
+
name = 'computer_drag';
|
|
1223
|
+
args.fromX = args.startX ?? args.fromX;
|
|
1224
|
+
args.fromY = args.startY ?? args.fromY;
|
|
1225
|
+
args.toX = args.endX ?? args.toX;
|
|
1226
|
+
args.toY = args.endY ?? args.toY;
|
|
1227
|
+
}
|
|
1228
|
+
else if (action === 'scroll') {
|
|
1229
|
+
name = 'computer_scroll';
|
|
1230
|
+
}
|
|
1231
|
+
else if (action === 'screenshot') {
|
|
1232
|
+
name = 'computer_screenshot';
|
|
1233
|
+
}
|
|
1234
|
+
else if (action === 'type') {
|
|
1235
|
+
name = 'computer_type';
|
|
1236
|
+
}
|
|
1237
|
+
else if (action === 'key_press' || action === 'key') {
|
|
1238
|
+
name = 'computer_key_press';
|
|
1239
|
+
}
|
|
1240
|
+
else if (action === 'hotkey') {
|
|
1241
|
+
name = 'computer_hotkey';
|
|
1242
|
+
}
|
|
1243
|
+
else if (action === 'open_app') {
|
|
1244
|
+
name = 'computer_open_app';
|
|
1245
|
+
}
|
|
1246
|
+
else if (action === 'close_app') {
|
|
1247
|
+
name = 'computer_close_app';
|
|
1248
|
+
}
|
|
1249
|
+
else if (action === 'switch_window') {
|
|
1250
|
+
name = 'computer_switch_window';
|
|
1251
|
+
}
|
|
1252
|
+
else if (action === 'inspect_ui') {
|
|
1253
|
+
name = 'computer_inspect_ui';
|
|
1254
|
+
}
|
|
1255
|
+
else if (action === 'diagnostic') {
|
|
1256
|
+
name = 'computer_diagnostic';
|
|
1257
|
+
}
|
|
1258
|
+
}
|
|
1102
1259
|
switch (name) {
|
|
1103
1260
|
case 'read_file': {
|
|
1104
1261
|
const targetPath = String(args.path || '').trim();
|
|
@@ -1580,11 +1737,31 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
|
|
|
1580
1737
|
case 'computer_click': {
|
|
1581
1738
|
const x = Number(args.x);
|
|
1582
1739
|
const y = Number(args.y);
|
|
1583
|
-
const
|
|
1740
|
+
const btn = args.button || 'left';
|
|
1741
|
+
const res = await this.computerController.click(x, y, { button: btn, clicks: 1 });
|
|
1584
1742
|
this.taskStateManager.addObservation('computer_click', res.output, res.success, `(${x}, ${y})`);
|
|
1585
1743
|
const structuredRes = {
|
|
1586
1744
|
success: res.success,
|
|
1587
1745
|
action: 'click',
|
|
1746
|
+
coordinates: res.coordinates || { x, y },
|
|
1747
|
+
button: btn,
|
|
1748
|
+
output: res.output,
|
|
1749
|
+
...(res.error ? { error: res.error } : {}),
|
|
1750
|
+
recoverable: res.recoverable ?? true,
|
|
1751
|
+
...(res.activeWindow ? { activeWindow: res.activeWindow } : {}),
|
|
1752
|
+
};
|
|
1753
|
+
return { result: JSON.stringify(structuredRes, null, 2) };
|
|
1754
|
+
}
|
|
1755
|
+
case 'computer_right_click': {
|
|
1756
|
+
const x = Number(args.x);
|
|
1757
|
+
const y = Number(args.y);
|
|
1758
|
+
const res = await this.computerController.rightClick(x, y);
|
|
1759
|
+
this.taskStateManager.addObservation('computer_right_click', res.output, res.success, `(${x}, ${y})`);
|
|
1760
|
+
const structuredRes = {
|
|
1761
|
+
success: res.success,
|
|
1762
|
+
action: 'right_click',
|
|
1763
|
+
coordinates: res.coordinates || { x, y },
|
|
1764
|
+
button: 'right',
|
|
1588
1765
|
output: res.output,
|
|
1589
1766
|
...(res.error ? { error: res.error } : {}),
|
|
1590
1767
|
recoverable: res.recoverable ?? true,
|
|
@@ -1600,6 +1777,42 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
|
|
|
1600
1777
|
const structuredRes = {
|
|
1601
1778
|
success: res.success,
|
|
1602
1779
|
action: 'double_click',
|
|
1780
|
+
coordinates: res.coordinates || { x, y },
|
|
1781
|
+
button: 'left',
|
|
1782
|
+
output: res.output,
|
|
1783
|
+
...(res.error ? { error: res.error } : {}),
|
|
1784
|
+
recoverable: res.recoverable ?? true,
|
|
1785
|
+
};
|
|
1786
|
+
return { result: JSON.stringify(structuredRes, null, 2) };
|
|
1787
|
+
}
|
|
1788
|
+
case 'computer_mouse_down': {
|
|
1789
|
+
const x = args.x !== undefined ? Number(args.x) : undefined;
|
|
1790
|
+
const y = args.y !== undefined ? Number(args.y) : undefined;
|
|
1791
|
+
const btn = args.button || 'left';
|
|
1792
|
+
const res = await this.computerController.mouseDown(x, y, btn);
|
|
1793
|
+
this.taskStateManager.addObservation('computer_mouse_down', res.output, res.success);
|
|
1794
|
+
const structuredRes = {
|
|
1795
|
+
success: res.success,
|
|
1796
|
+
action: 'mouse_down',
|
|
1797
|
+
coordinates: res.coordinates || (x !== undefined && y !== undefined ? { x, y } : undefined),
|
|
1798
|
+
button: btn,
|
|
1799
|
+
output: res.output,
|
|
1800
|
+
...(res.error ? { error: res.error } : {}),
|
|
1801
|
+
recoverable: res.recoverable ?? true,
|
|
1802
|
+
};
|
|
1803
|
+
return { result: JSON.stringify(structuredRes, null, 2) };
|
|
1804
|
+
}
|
|
1805
|
+
case 'computer_mouse_up': {
|
|
1806
|
+
const x = args.x !== undefined ? Number(args.x) : undefined;
|
|
1807
|
+
const y = args.y !== undefined ? Number(args.y) : undefined;
|
|
1808
|
+
const btn = args.button || 'left';
|
|
1809
|
+
const res = await this.computerController.mouseUp(x, y, btn);
|
|
1810
|
+
this.taskStateManager.addObservation('computer_mouse_up', res.output, res.success);
|
|
1811
|
+
const structuredRes = {
|
|
1812
|
+
success: res.success,
|
|
1813
|
+
action: 'mouse_up',
|
|
1814
|
+
coordinates: res.coordinates || (x !== undefined && y !== undefined ? { x, y } : undefined),
|
|
1815
|
+
button: btn,
|
|
1603
1816
|
output: res.output,
|
|
1604
1817
|
...(res.error ? { error: res.error } : {}),
|
|
1605
1818
|
recoverable: res.recoverable ?? true,
|
|
@@ -1658,14 +1871,41 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
|
|
|
1658
1871
|
case 'computer_mouse_move': {
|
|
1659
1872
|
const x = Number(args.x);
|
|
1660
1873
|
const y = Number(args.y);
|
|
1661
|
-
const res = await this.computerController.
|
|
1874
|
+
const res = await this.computerController.move(x, y);
|
|
1662
1875
|
this.taskStateManager.addObservation('computer_mouse_move', res.output, res.success);
|
|
1663
|
-
|
|
1876
|
+
const structuredRes = {
|
|
1877
|
+
success: res.success,
|
|
1878
|
+
action: 'move',
|
|
1879
|
+
coordinates: res.coordinates || { x, y },
|
|
1880
|
+
output: res.output,
|
|
1881
|
+
...(res.error ? { error: res.error } : {}),
|
|
1882
|
+
recoverable: res.recoverable ?? true,
|
|
1883
|
+
...(res.activeWindow ? { activeWindow: res.activeWindow } : {}),
|
|
1884
|
+
};
|
|
1885
|
+
return { result: JSON.stringify(structuredRes, null, 2) };
|
|
1664
1886
|
}
|
|
1665
1887
|
case 'computer_drag': {
|
|
1666
|
-
const
|
|
1888
|
+
const fromX = Number(args.fromX ?? args.startX ?? 0);
|
|
1889
|
+
const fromY = Number(args.fromY ?? args.startY ?? 0);
|
|
1890
|
+
const toX = Number(args.toX ?? args.endX ?? 0);
|
|
1891
|
+
const toY = Number(args.toY ?? args.endY ?? 0);
|
|
1892
|
+
const res = await this.computerController.drag(fromX, fromY, toX, toY);
|
|
1667
1893
|
this.taskStateManager.addObservation('computer_drag', res.output, res.success);
|
|
1668
|
-
|
|
1894
|
+
const structuredRes = {
|
|
1895
|
+
success: res.success,
|
|
1896
|
+
action: 'drag',
|
|
1897
|
+
from: { x: fromX, y: fromY },
|
|
1898
|
+
to: { x: toX, y: toY },
|
|
1899
|
+
coordinates: res.coordinates || { x: toX, y: toY },
|
|
1900
|
+
output: res.output,
|
|
1901
|
+
...(res.error ? { error: res.error } : {}),
|
|
1902
|
+
recoverable: res.recoverable ?? true,
|
|
1903
|
+
};
|
|
1904
|
+
return { result: JSON.stringify(structuredRes, null, 2) };
|
|
1905
|
+
}
|
|
1906
|
+
case 'computer_diagnostic': {
|
|
1907
|
+
const diag = await this.computerController.runDiagnostic();
|
|
1908
|
+
return { result: JSON.stringify(diag, null, 2) };
|
|
1669
1909
|
}
|
|
1670
1910
|
case 'computer_wait': {
|
|
1671
1911
|
const ms = Math.min(10000, Math.max(100, Number(args.ms) || 1000));
|