ft-scout 9.0.2 → 9.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.firebase/hosting.d2Vi.cache +7 -7
- package/README.md +76 -83
- package/bin/src/commands/agent.d.ts.map +1 -1
- package/bin/src/commands/agent.js +13 -223
- package/bin/src/commands/agent.js.map +1 -1
- package/bin/src/commands/plan.d.ts.map +1 -1
- package/bin/src/commands/plan.js +45 -18
- package/bin/src/commands/plan.js.map +1 -1
- package/bin/src/engine/actionVerifier.d.ts.map +1 -1
- package/bin/src/engine/actionVerifier.js +61 -9
- package/bin/src/engine/actionVerifier.js.map +1 -1
- package/bin/src/engine/agentEngine.d.ts.map +1 -1
- package/bin/src/engine/agentEngine.js +459 -547
- package/bin/src/engine/agentEngine.js.map +1 -1
- package/bin/src/engine/agentState.d.ts.map +1 -1
- package/bin/src/engine/agentState.js +0 -64
- package/bin/src/engine/agentState.js.map +1 -1
- package/bin/src/engine/browser/browserController.d.ts.map +1 -1
- package/bin/src/engine/browser/browserController.js +8 -0
- package/bin/src/engine/browser/browserController.js.map +1 -1
- package/bin/src/engine/computer/computerController.d.ts.map +1 -1
- package/bin/src/engine/computer/computerController.js +479 -269
- package/bin/src/engine/computer/computerController.js.map +1 -1
- package/bin/src/engine/computer/index.d.ts.map +1 -1
- package/bin/src/engine/computer/index.js +1 -0
- package/bin/src/engine/computer/index.js.map +1 -1
- package/bin/src/engine/computer/mockComputer.d.ts.map +1 -0
- package/bin/src/engine/computer/mockComputer.js +164 -0
- package/bin/src/engine/computer/mockComputer.js.map +1 -0
- package/bin/src/engine/runtime/agentRuntime.d.ts.map +1 -0
- package/bin/src/engine/runtime/agentRuntime.js +829 -0
- package/bin/src/engine/runtime/agentRuntime.js.map +1 -0
- package/bin/src/engine/runtime/index.d.ts.map +1 -0
- package/bin/src/engine/runtime/index.js +2 -0
- package/bin/src/engine/runtime/index.js.map +1 -0
- package/bin/src/engine/safety.d.ts.map +1 -1
- package/bin/src/engine/safety.js +49 -25
- package/bin/src/engine/safety.js.map +1 -1
- package/bin/src/engine/toolRegistry.d.ts.map +1 -1
- package/bin/src/engine/toolRegistry.js +162 -81
- package/bin/src/engine/toolRegistry.js.map +1 -1
- package/bin/src/engine/voiceEngine.d.ts.map +1 -1
- package/bin/src/engine/voiceEngine.js +0 -1
- package/bin/src/engine/voiceEngine.js.map +1 -1
- package/bin/src/engine/workspace/agentWorkspace.d.ts.map +1 -0
- package/bin/src/engine/workspace/agentWorkspace.js +484 -0
- package/bin/src/engine/workspace/agentWorkspace.js.map +1 -0
- package/bin/src/engine/workspace/index.d.ts.map +1 -0
- package/bin/src/engine/workspace/index.js +2 -0
- package/bin/src/engine/workspace/index.js.map +1 -0
- package/bin/src/index.js +4 -5
- package/bin/src/index.js.map +1 -1
- package/bin/src/server/server.d.ts.map +1 -1
- package/bin/src/server/server.js +25 -4
- package/bin/src/server/server.js.map +1 -1
- package/bin/src/utils/auth.d.ts.map +1 -1
- package/bin/src/utils/auth.js +67 -49
- package/bin/src/utils/auth.js.map +1 -1
- package/bin/src/utils/branding.js +1 -1
- package/bin/src/utils/payment.d.ts.map +1 -1
- package/bin/src/utils/payment.js +15 -1
- package/bin/src/utils/payment.js.map +1 -1
- package/bin/src/utils/pricing.js +1 -1
- package/bin/src/utils/pricing.js.map +1 -1
- package/bin/src/utils/razorpay.d.ts.map +1 -1
- package/bin/src/utils/razorpay.js +10 -2
- package/bin/src/utils/razorpay.js.map +1 -1
- package/downloads/invoice-2026-03.pdf +1 -0
- package/package.json +1 -1
- package/web/app.js +54 -135
- package/web/styles.css +2 -2
- package/bin/src/engine/appControl.d.ts.map +0 -1
- package/bin/src/engine/appControl.js +0 -2320
- package/bin/src/engine/appControl.js.map +0 -1
- package/bin/src/engine/liveScreenEngine.d.ts.map +0 -1
- package/bin/src/engine/liveScreenEngine.js +0 -421
- package/bin/src/engine/liveScreenEngine.js.map +0 -1
- package/bin/src/engine/scripts/winCapture.ps1 +0 -248
- package/downloads/INVOICE_2026_ACME.pdf +0 -1
|
@@ -9,16 +9,17 @@ import { saveAgentSession, getRecentSessionsSummary } from '../utils/session.js'
|
|
|
9
9
|
import { safeNote, renderMarkdown } from '../utils/markdown.js';
|
|
10
10
|
import { openai, callOpenAIWithRetry, isQuotaExceededError, sanitizeMessage, sanitizeMessages } from './llm.js';
|
|
11
11
|
import { checkFileSyntax, verifyAndSelfHealFiles, executeSmartCommand, extractErrorDiagnostics } from './verifier.js';
|
|
12
|
-
import { openApp, executeInApp, cleanupScreenshots } from './appControl.js';
|
|
13
12
|
import { speakText, listenSpeechToText } from './voiceEngine.js';
|
|
14
|
-
import { startLiveScreenShare, stopLiveScreenShare, isLiveScreenShareActive } from './liveScreenEngine.js';
|
|
15
13
|
import { TaskStateManager } from './agentState.js';
|
|
16
|
-
import { OSComputerController } from './computer/index.js';
|
|
17
14
|
import { SemanticBrowserController } from './browser/index.js';
|
|
18
15
|
import { ActionVerifier } from './actionVerifier.js';
|
|
19
16
|
import { ApprovalManager, RiskClassifier } from './safety.js';
|
|
20
17
|
import { TaskPlanner } from './planner.js';
|
|
21
18
|
import { UnifiedToolRegistry } from './toolRegistry.js';
|
|
19
|
+
import { SystemComputerController, MockComputerController } from './computer/index.js';
|
|
20
|
+
import { AgentWorkspace, inferExecutionIntent, resolveDesktopPath } from './workspace/index.js';
|
|
21
|
+
import { AgentRuntime } from './runtime/index.js';
|
|
22
|
+
export { AgentRuntime, AgentWorkspace, inferExecutionIntent, resolveDesktopPath, SystemComputerController, MockComputerController };
|
|
22
23
|
const execAsync = promisify(exec);
|
|
23
24
|
export function robustSnippetReplace(origContent, target, replacement) {
|
|
24
25
|
if (!origContent || !target) {
|
|
@@ -277,54 +278,6 @@ export const AGENT_TOOLS = [
|
|
|
277
278
|
required: ['question'],
|
|
278
279
|
},
|
|
279
280
|
},
|
|
280
|
-
{
|
|
281
|
-
name: 'open_app',
|
|
282
|
-
description: 'Open, launch, or focus an external application (browser, terminal, editor, VS Code, Notepad, or custom app) as requested by user prompt.',
|
|
283
|
-
parameters: {
|
|
284
|
-
type: 'object',
|
|
285
|
-
properties: {
|
|
286
|
-
app: { type: 'string', description: 'Application name or executable (e.g. "browser", "chrome", "edge", "terminal", "powershell", "code", "vscode", "notepad", or path)' },
|
|
287
|
-
target: { type: 'string', description: 'Optional target URL (for browser), file/folder path (for editor), or initial command (for terminal)' },
|
|
288
|
-
reason: { type: 'string', description: 'Explanation of why this app is being launched and taken over' },
|
|
289
|
-
},
|
|
290
|
-
required: ['app'],
|
|
291
|
-
},
|
|
292
|
-
},
|
|
293
|
-
{
|
|
294
|
-
name: 'app_action',
|
|
295
|
-
description: 'Execute interactive desktop/browser scratchpad automation actions, live screen sharing with model, UI element clicking, screen takeover, mouse cursor positioning & click takeover, mouse scrolling, window management, keyboard keystrokes, hotkeys, focus locks, or command sequences without shell commands (e.g. action: "live_screen_share", "start_screen_share", "stop_screen_share", "live_screen_stream", "see_screen", "analyze_screen", "capture_screen", "click_element", "click_app", "scroll", "list_windows", "focus_window", "takeover", "move_mouse", "type_text", "send_keys", "key_combo", "lock_app", "fetch_page", "navigate", "search", "send_dm", "exec_command", "open_file"). All screen frames are streamed in-memory directly to the model with zero temporary files.',
|
|
296
|
-
parameters: {
|
|
297
|
-
type: 'object',
|
|
298
|
-
properties: {
|
|
299
|
-
app: { type: 'string', description: 'Target application name, window title, screen, or category ("browser", "terminal", "editor", "notepad", "chrome", "desktop")' },
|
|
300
|
-
action: { type: 'string', description: 'Action type ("live_screen_share", "start_screen_share", "stop_screen_share", "live_screen_stream", "see_screen", "analyze_screen", "capture_screen", "click_element", "click_app", "scroll", "list_windows", "focus_window", "takeover", "send_mail", "type_text", "move_mouse", "send_keys", "key_combo", "lock_app", "fetch_page", "search", "send_dm", "exec_command", "open_file")' },
|
|
301
|
-
payload: { description: 'Action details/payload object or string (e.g. element name/description to click { element: "Search Google" }, coordinates { x: 100, y: 200 }, scroll { direction: "down", amount: 4 }, text string, { text: "...", enter: true }, { keyCombo: "ctrl+v" }, { duration: 3000 }, URL, search query, command, or file path)' },
|
|
302
|
-
},
|
|
303
|
-
required: ['app', 'action'],
|
|
304
|
-
},
|
|
305
|
-
},
|
|
306
|
-
{
|
|
307
|
-
name: 'agent_scratchpad',
|
|
308
|
-
description: 'Update, read, or clear the agent working memory scratchpad, step checklist, and working notes for multi-step goals.',
|
|
309
|
-
parameters: {
|
|
310
|
-
type: 'object',
|
|
311
|
-
properties: {
|
|
312
|
-
action: { type: 'string', description: 'Action type ("update", "read", "clear")' },
|
|
313
|
-
plan: {
|
|
314
|
-
type: 'array',
|
|
315
|
-
description: 'List of step items in the goal execution plan',
|
|
316
|
-
items: { type: 'string' },
|
|
317
|
-
},
|
|
318
|
-
completedSteps: {
|
|
319
|
-
type: 'array',
|
|
320
|
-
description: 'Indices of steps that are completed (0-based numbers)',
|
|
321
|
-
items: { type: 'number' },
|
|
322
|
-
},
|
|
323
|
-
notes: { type: 'string', description: 'Working memory notes, findings, or scratchpad text' },
|
|
324
|
-
},
|
|
325
|
-
required: ['action'],
|
|
326
|
-
},
|
|
327
|
-
},
|
|
328
281
|
{
|
|
329
282
|
name: 'speak_text',
|
|
330
283
|
description: 'Speak a text phrase out loud using Text-to-Speech (TTS) voice synthesis for voice feedback or audio summaries.',
|
|
@@ -400,7 +353,6 @@ export class AgentExecutionLoop {
|
|
|
400
353
|
backups = [];
|
|
401
354
|
autoApprove;
|
|
402
355
|
maxSteps;
|
|
403
|
-
projectName;
|
|
404
356
|
startTime = Date.now();
|
|
405
357
|
totalToolCalls = 0;
|
|
406
358
|
currentStep = 0;
|
|
@@ -409,48 +361,32 @@ export class AgentExecutionLoop {
|
|
|
409
361
|
consecutiveCommands = 0;
|
|
410
362
|
consecutiveScreenChecks = 0;
|
|
411
363
|
toolCallHistory = [];
|
|
412
|
-
scratchpadState = { plan: [], completedSteps: [], notes: '' };
|
|
413
364
|
taskStateManager;
|
|
414
|
-
computerController;
|
|
415
365
|
browserController;
|
|
366
|
+
computerController;
|
|
416
367
|
actionVerifier;
|
|
417
368
|
approvalManager;
|
|
418
369
|
taskPlanner;
|
|
419
370
|
toolRegistry;
|
|
420
371
|
mockMode;
|
|
372
|
+
isCancelled = false;
|
|
373
|
+
cancelReason;
|
|
374
|
+
executionIntent;
|
|
421
375
|
constructor(options) {
|
|
422
376
|
this.cwd = process.cwd();
|
|
423
377
|
this.autoApprove = Boolean(options?.autoApprove);
|
|
424
378
|
this.maxSteps = options?.maxSteps || 30;
|
|
425
|
-
this.projectName = options?.projectName || path.basename(this.cwd);
|
|
426
379
|
this.mockMode = Boolean(options?.mockMode);
|
|
427
380
|
this.taskStateManager = new TaskStateManager(options?.taskId, '', this.cwd);
|
|
428
|
-
this.computerController = options?.computerController || new
|
|
381
|
+
this.computerController = options?.computerController || (this.mockMode ? new MockComputerController() : new SystemComputerController(this.cwd));
|
|
429
382
|
this.browserController = options?.browserController || new SemanticBrowserController(this.cwd);
|
|
430
383
|
this.actionVerifier = new ActionVerifier(this.cwd);
|
|
431
384
|
this.approvalManager = new ApprovalManager(this.autoApprove);
|
|
432
385
|
this.taskPlanner = new TaskPlanner();
|
|
433
386
|
this.toolRegistry = new UnifiedToolRegistry();
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
// Ensure temporary screenshot cleanup and screen share stop on early exit or interrupt
|
|
438
|
-
const onExitOrInterrupt = () => {
|
|
439
|
-
try {
|
|
440
|
-
cleanupScreenshots();
|
|
441
|
-
}
|
|
442
|
-
catch { }
|
|
443
|
-
try {
|
|
444
|
-
if (isLiveScreenShareActive())
|
|
445
|
-
stopLiveScreenShare();
|
|
446
|
-
}
|
|
447
|
-
catch { }
|
|
448
|
-
};
|
|
449
|
-
process.once('exit', onExitOrInterrupt);
|
|
450
|
-
process.once('SIGINT', () => {
|
|
451
|
-
onExitOrInterrupt();
|
|
452
|
-
process.exit(0);
|
|
453
|
-
});
|
|
387
|
+
}
|
|
388
|
+
getExecutionIntent() {
|
|
389
|
+
return this.executionIntent;
|
|
454
390
|
}
|
|
455
391
|
getTaskStateManager() {
|
|
456
392
|
return this.taskStateManager;
|
|
@@ -458,12 +394,12 @@ export class AgentExecutionLoop {
|
|
|
458
394
|
getStateManager() {
|
|
459
395
|
return this.taskStateManager;
|
|
460
396
|
}
|
|
461
|
-
getComputerController() {
|
|
462
|
-
return this.computerController;
|
|
463
|
-
}
|
|
464
397
|
getBrowserController() {
|
|
465
398
|
return this.browserController;
|
|
466
399
|
}
|
|
400
|
+
getComputerController() {
|
|
401
|
+
return this.computerController;
|
|
402
|
+
}
|
|
467
403
|
getActionVerifier() {
|
|
468
404
|
return this.actionVerifier;
|
|
469
405
|
}
|
|
@@ -476,6 +412,15 @@ export class AgentExecutionLoop {
|
|
|
476
412
|
getToolRegistry() {
|
|
477
413
|
return this.toolRegistry;
|
|
478
414
|
}
|
|
415
|
+
cancel(reason = 'User cancelled task') {
|
|
416
|
+
this.isCancelled = true;
|
|
417
|
+
this.cancelReason = reason;
|
|
418
|
+
this.taskStateManager.setStatus('failed');
|
|
419
|
+
this.taskStateManager.recordError(`Task execution cancelled: ${reason}`, 'runtime');
|
|
420
|
+
}
|
|
421
|
+
isTaskCancelled() {
|
|
422
|
+
return this.isCancelled;
|
|
423
|
+
}
|
|
479
424
|
async resumeGoal(taskId) {
|
|
480
425
|
const saved = TaskStateManager.loadTaskState(taskId, this.cwd);
|
|
481
426
|
if (!saved) {
|
|
@@ -517,92 +462,7 @@ export class AgentExecutionLoop {
|
|
|
517
462
|
this.consecutiveCommands = 0;
|
|
518
463
|
this.consecutiveScreenChecks = 0;
|
|
519
464
|
this.toolCallHistory = [];
|
|
520
|
-
this.
|
|
521
|
-
}
|
|
522
|
-
updateScratchpad(plan, completedSteps, notes) {
|
|
523
|
-
if (plan !== undefined)
|
|
524
|
-
this.scratchpadState.plan = plan;
|
|
525
|
-
if (completedSteps !== undefined)
|
|
526
|
-
this.scratchpadState.completedSteps = completedSteps;
|
|
527
|
-
if (notes !== undefined)
|
|
528
|
-
this.scratchpadState.notes = notes;
|
|
529
|
-
try {
|
|
530
|
-
const ftDir = path.join(this.cwd, '.ft');
|
|
531
|
-
if (!fs.existsSync(ftDir)) {
|
|
532
|
-
fs.mkdirSync(ftDir, { recursive: true });
|
|
533
|
-
}
|
|
534
|
-
const scratchpadPath = path.join(ftDir, 'scratchpad.md');
|
|
535
|
-
let mdContent = `# Scout Agent Scratchpad\n\n`;
|
|
536
|
-
if (this.scratchpadState.plan.length > 0) {
|
|
537
|
-
mdContent += `## Plan Checklist\n`;
|
|
538
|
-
this.scratchpadState.plan.forEach((item, idx) => {
|
|
539
|
-
const isDone = this.scratchpadState.completedSteps.includes(idx);
|
|
540
|
-
mdContent += `- [${isDone ? 'x' : ' '}] Step ${idx + 1}: ${item}\n`;
|
|
541
|
-
});
|
|
542
|
-
mdContent += `\n`;
|
|
543
|
-
}
|
|
544
|
-
if (this.scratchpadState.notes) {
|
|
545
|
-
mdContent += `## Working Memory & Notes\n${this.scratchpadState.notes}\n`;
|
|
546
|
-
}
|
|
547
|
-
fs.writeFileSync(scratchpadPath, mdContent, 'utf-8');
|
|
548
|
-
}
|
|
549
|
-
catch {
|
|
550
|
-
// Ignore write errors to prevent breaking execution
|
|
551
|
-
}
|
|
552
|
-
let summary = 'Agent Scratchpad updated successfully.\n';
|
|
553
|
-
if (this.scratchpadState.plan.length > 0) {
|
|
554
|
-
summary += `Plan (${this.scratchpadState.plan.length} items):\n` +
|
|
555
|
-
this.scratchpadState.plan.map((item, idx) => ` ${this.scratchpadState.completedSteps.includes(idx) ? '[✓]' : '[ ]'} Step ${idx + 1}: ${item}`).join('\n') + '\n';
|
|
556
|
-
}
|
|
557
|
-
if (this.scratchpadState.notes) {
|
|
558
|
-
summary += `Notes: ${this.scratchpadState.notes}`;
|
|
559
|
-
}
|
|
560
|
-
return summary;
|
|
561
|
-
}
|
|
562
|
-
autoSyncScratchpadFromText(text) {
|
|
563
|
-
try {
|
|
564
|
-
const planLines = text.match(/[-*]\s*\[([ xX✓/])\]\s*(.*)/g);
|
|
565
|
-
if (planLines && planLines.length > 0) {
|
|
566
|
-
const plan = [];
|
|
567
|
-
const completed = [];
|
|
568
|
-
planLines.forEach((l, idx) => {
|
|
569
|
-
const isDone = l.includes('[x]') || l.includes('[X]') || l.includes('[✓]');
|
|
570
|
-
const cleanText = l.replace(/^[-*]\s*\[[ xX✓/]\]\s*/, '').trim();
|
|
571
|
-
plan.push(cleanText);
|
|
572
|
-
if (isDone)
|
|
573
|
-
completed.push(idx);
|
|
574
|
-
});
|
|
575
|
-
this.scratchpadState.plan = plan;
|
|
576
|
-
this.scratchpadState.completedSteps = completed;
|
|
577
|
-
}
|
|
578
|
-
const obsMatch = text.match(/Observations?:\s*([^\n]+(?:\n[^\n]+)*)/i);
|
|
579
|
-
const thoughtMatch = text.match(/Thought:\s*([^\n]+(?:\n[^\n]+)*)/i);
|
|
580
|
-
const notes = [
|
|
581
|
-
thoughtMatch ? `Thought: ${thoughtMatch[1]?.trim()}` : '',
|
|
582
|
-
obsMatch ? `Observations: ${obsMatch[1]?.trim()}` : '',
|
|
583
|
-
].filter(Boolean).join('\n\n');
|
|
584
|
-
if (notes) {
|
|
585
|
-
this.scratchpadState.notes = notes;
|
|
586
|
-
}
|
|
587
|
-
const ftDir = path.join(this.cwd, '.ft');
|
|
588
|
-
if (!fs.existsSync(ftDir))
|
|
589
|
-
fs.mkdirSync(ftDir, { recursive: true });
|
|
590
|
-
const scratchpadPath = path.join(ftDir, 'scratchpad.md');
|
|
591
|
-
let mdContent = `# Scout Agent Scratchpad & Working Memory\n\n`;
|
|
592
|
-
if (this.scratchpadState.plan.length > 0) {
|
|
593
|
-
mdContent += `## Plan Checklist\n`;
|
|
594
|
-
this.scratchpadState.plan.forEach((item, idx) => {
|
|
595
|
-
const isDone = this.scratchpadState.completedSteps.includes(idx);
|
|
596
|
-
mdContent += `- [${isDone ? 'x' : ' '}] Step ${idx + 1}: ${item}\n`;
|
|
597
|
-
});
|
|
598
|
-
mdContent += `\n`;
|
|
599
|
-
}
|
|
600
|
-
if (this.scratchpadState.notes) {
|
|
601
|
-
mdContent += `## Working Memory & Observations\n${this.scratchpadState.notes}\n`;
|
|
602
|
-
}
|
|
603
|
-
fs.writeFileSync(scratchpadPath, mdContent, 'utf-8');
|
|
604
|
-
}
|
|
605
|
-
catch { }
|
|
465
|
+
this.executionIntent = undefined;
|
|
606
466
|
}
|
|
607
467
|
handleToolConsecutiveTracking(fnName, args, step) {
|
|
608
468
|
const argsKey = JSON.stringify(args || {});
|
|
@@ -661,9 +521,12 @@ export class AgentExecutionLoop {
|
|
|
661
521
|
this.consecutiveReads++;
|
|
662
522
|
this.consecutiveCommands = 0;
|
|
663
523
|
if (this.consecutiveReads >= 2) {
|
|
524
|
+
const actionHint = this.executionIntent?.requiredInteraction === 'computer'
|
|
525
|
+
? `Take action in the requested application (${this.executionIntent.requiredApplications.join(', ') || 'Notepad'})! Use computer_type, computer_click, or computer_hotkey now.`
|
|
526
|
+
: `You MUST execute write_file or edit_file on this step to write or modify code and satisfy the user goal.`;
|
|
664
527
|
this.historyMessages.push({
|
|
665
528
|
role: 'user',
|
|
666
|
-
content: `URGENT ACTION MANDATE (Step ${step}): You have issued ${this.consecutiveReads} exploration tool calls in a row without making
|
|
529
|
+
content: `URGENT ACTION MANDATE (Step ${step}): You have issued ${this.consecutiveReads} exploration tool calls in a row without making progress! ${actionHint}`,
|
|
667
530
|
});
|
|
668
531
|
}
|
|
669
532
|
}
|
|
@@ -671,9 +534,12 @@ export class AgentExecutionLoop {
|
|
|
671
534
|
this.consecutiveReads = 0;
|
|
672
535
|
this.consecutiveCommands++;
|
|
673
536
|
if (this.consecutiveCommands >= 2) {
|
|
537
|
+
const actionHint = this.executionIntent?.requiredInteraction === 'computer'
|
|
538
|
+
? `Continue with computer interaction tools (computer_type, computer_hotkey) in the target application.`
|
|
539
|
+
: `Call 'write_file' or 'edit_file' NOW to write the required source code and files directly to disk!`;
|
|
674
540
|
this.historyMessages.push({
|
|
675
541
|
role: 'user',
|
|
676
|
-
content: `URGENT ACTION MANDATE (Step ${step}): You have executed ${this.consecutiveCommands} shell commands in a row without
|
|
542
|
+
content: `URGENT ACTION MANDATE (Step ${step}): You have executed ${this.consecutiveCommands} shell commands in a row without progress! ${actionHint}`,
|
|
677
543
|
});
|
|
678
544
|
}
|
|
679
545
|
}
|
|
@@ -684,115 +550,65 @@ export class AgentExecutionLoop {
|
|
|
684
550
|
}
|
|
685
551
|
systemPrompt() {
|
|
686
552
|
const filesList = getDirectoryFiles(this.cwd).slice(0, 100);
|
|
687
|
-
|
|
553
|
+
const intentInfo = this.executionIntent ? `
|
|
554
|
+
Task Execution Intent & Constraints:
|
|
555
|
+
• Required Applications: ${this.executionIntent.requiredApplications.length > 0 ? this.executionIntent.requiredApplications.join(', ') : 'None specified'}
|
|
556
|
+
• Required Interaction Type: ${this.executionIntent.requiredInteraction}
|
|
557
|
+
• Allow Alternative Tools (e.g. direct filesystem): ${this.executionIntent.allowAlternativeTools ? 'YES' : 'NO (STRICT)'}
|
|
558
|
+
• Target Destination: ${this.executionIntent.targetDestination || 'Default'}
|
|
559
|
+
• Target Artifact: ${this.executionIntent.targetArtifactName || 'None'}
|
|
560
|
+
` : '';
|
|
561
|
+
return `You are Scout Agent, an elite AI Principal Software Engineer & Autonomous Computer-Use Operator (developed by FrontTerrain).
|
|
688
562
|
|
|
689
|
-
Project Name: "${this.projectName}"
|
|
690
563
|
Working Directory: "${this.cwd}"
|
|
691
564
|
Host OS Platform: "${process.platform}"
|
|
692
565
|
Current Date & Time: "${new Date().toISOString()}" (Current Year: ${new Date().getFullYear()})
|
|
693
|
-
|
|
566
|
+
${intentInfo}
|
|
694
567
|
Workspace Overview (first 100 files):
|
|
695
568
|
${filesList.join('\n')}
|
|
696
569
|
|
|
697
|
-
|
|
698
|
-
1.
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
6. COMPLETE FILE CREATION & EDITING: Whenever asked to write, edit, create, implement, or fix code, YOU MUST CREATE / MODIFY ALL NECESSARY FILES required to completely fulfill the prompt (e.g. main source files, component files, styles, configs, tests, documentation). CALL \`write_file\` or \`edit_file\` for EVERY necessary file to save changes directly to disk in the workspace! Always provide a clear \`reason\` argument for why the edit is made.
|
|
705
|
-
7. RUN & VERIFY COMMANDS: Execute shell commands, tests, builds, and formatters using \`run_command\`. If a command or build fails, read the full error log, diagnose the root cause, and auto-fix the failing code immediately.
|
|
706
|
-
8. NO CODE TRUNCATION OR PLACEHOLDERS: Maintain 100% code integrity. NEVER replace existing code or HTML tags with placeholder comments (e.g. \`<!-- remaining content -->\` or \`// TODO: rest of code\`) or stripped skeleton structures. Always output complete, functional code.
|
|
707
|
-
9. REFORMATTING MINIFIED / LARGE FILES: When asked to format, beautify, or un-minify HTML, JS, CSS, or JSON files, DO NOT attempt to rewrite the file content manually using \`write_file\` or \`edit_file\` (which causes token truncation and content loss). ALWAYS use \`run_command\` to execute CLI formatters (e.g. \`npx prettier --write <file>\`, \`npx html-beautify -r <file>\`, \`npx js-beautify <file>\`, or run a quick Node script) to reformat files in-place on disk.
|
|
708
|
-
10. FROM-SCRATCH PROJECT CREATION & AUTOMATED TESTING PROTOCOL: When asked to build/create a project from scratch:
|
|
709
|
-
a. Scaffold Architecture: Create configuration/manifest files (\`package.json\`, \`tsconfig.json\`, \`pyproject.toml\`, etc.) and folder structure using \`write_file\` or \`run_command\`.
|
|
710
|
-
b. Write Source Implementation: Create all necessary source files with fully functional, complete, production-ready code (no skeleton placeholders or TODOs).
|
|
711
|
-
c. Write Automated Tests: Create comprehensive unit/integration test suite files (e.g. \`test/*.test.ts\`, \`tests/test_*.py\`, etc.).
|
|
712
|
-
d. Execute Tests & Build Verification: Run test suites and builds via \`run_command\` (e.g. \`npm test\`, \`npx vitest run\`, \`pytest\`, \`cargo test\`, \`go test ./...\`).
|
|
713
|
-
e. Self-Heal & Pass: If tests or builds fail, inspect output logs, edit files using \`edit_file\` or \`write_file\` to resolve errors, re-run tests until 100% passing, and only call \`task_completed\` when all checks pass cleanly.
|
|
714
|
-
11. CRITICAL TERMINATION RULE: As soon as you have finished creating/modifying the necessary files, running verification checks, or answering the user request, YOU MUST CALL \`task_completed\` tool immediately with a clear summary of your work.
|
|
715
|
-
12. PRODUCTION-GRADE CODE QUALITY: Every line of code you write or edit MUST be clean, elegant, modular, production-ready, highly readable, self-documenting, formatted, and strictly typed.
|
|
716
|
-
13. HIGH AUTONOMY & NO DOUBLE PROMPTING: NEVER call \`ask_user\` to ask what code to write, what features to implement, or to re-confirm the prompt. Take immediate autonomous action using \`write_file\`, \`edit_file\`, \`run_command\`, \`read_file\`, or \`grep_search\` using best software engineering practices. ONLY call \`ask_user\` if a critical secret credential (API key/password) is strictly required.
|
|
717
|
-
14. EDITING & DEBUGGING MASTERY PROTOCOL:
|
|
718
|
-
a. Full Context Reading: When editing or fixing bugs in existing files, inspect full code context. For files under 250 lines, reference the complete file content before modifying.
|
|
719
|
-
b. Guaranteed Edit Success: Use \`edit_file\` for precise snippet updates. If \`edit_file\` returns a target snippet mismatch error, IMMEDIATELY call \`write_file\` with the complete corrected file code so the edit is 100% applied without getting stuck!
|
|
720
|
-
c. Automated Error Diagnostics: After editing, check compiler/syntax outputs and test runner logs. If an error is detected, auto-heal the code immediately before concluding.
|
|
721
|
-
15. THOUGHT TRANSPARENCY PROTOCOL: In EVERY step before calling any tools or ending a turn, you MUST provide a clear 1-2 sentence explanation in your text response describing your current reasoning, what file/action you are taking, and why you are taking it.
|
|
722
|
-
16. OS-COMPATIBLE SHELL COMMANDS: Always ensure shell commands passed to \`run_command\` are compatible with the host OS ("${process.platform}"). On Windows (\`win32\`), do NOT use Linux-only builtins like \`touch\`, \`rm -rf\`, \`cat\`, or \`ls -la\` directly. Use \`write_file\` to create files, standard \`npm\`/\`npx\`/\`node\`/\`git\`/\`python\` commands, or PowerShell syntax.
|
|
723
|
-
17. STRICT USER INSTRUCTION & DIRECTIVE ALIGNMENT: You MUST strictly follow all exact user instructions, quantitative constraints, file paths, feature requirements, and architectural preferences specified in the user prompt. Execute the request step-by-step with zero deviation or hallucinated shortcuts, ensuring complete implementation within the allowed step limit.
|
|
724
|
-
18. DIRECT FILE CREATION VIA write_file ONLY (NO SHELL COMMANDS): Do NOT use \`run_command\` with \`touch\`, \`mkdir\`, \`echo > file\`, \`cat > file\`, or shell scripts to create files or folders! ALWAYS call \`write_file\` or \`edit_file\` directly to write source code to disk. The \`write_file\` tool automatically creates all necessary parent directories in one step.
|
|
725
|
-
19. EXTERNAL APP TAKEOVER & CONTROL PROTOCOL: When requested by user prompt to open, take over, or work inside external applications (browser, terminal, VS Code, Notepad, social apps like Instagram, WhatsApp, Twitter/X, Telegram, or custom apps):
|
|
726
|
-
a. Launch App: Use \`open_app\` to launch or focus the target application with optional URL, file path, or initial script.
|
|
727
|
-
b. Social DM & Messaging Automation: For Instagram, WhatsApp, Twitter/X, or Telegram messaging requests (e.g. "open instagram and send message to @user"), immediately invoke \`app_action\` with action "send_dm" or "open_dm" (or \`open_app\`) specifying the target username/phone and message text so the agent automatically opens the direct messaging link in the browser!
|
|
728
|
-
c. Email & Gmail Automation: When requested to send or compose an email (e.g. "send mail to frontterrain@gmail.com saying...", "takeover mail.google.com and send mail"):
|
|
729
|
-
- Immediately invoke \`app_action\` with \`action: "send_mail"\` (or \`action: "send_dm"\`) specifying the target recipient email and body text. The agent automatically constructs the direct Gmail compose URL (\`https://mail.google.com/mail/?view=cm&fs=1&to=<recipient>&su=<subject>&body=<body>\`) which pre-populates the compose window and dispatches the email via Ctrl+Enter!
|
|
730
|
-
- Alternatively, navigate directly to \`https://mail.google.com/mail/?view=cm&fs=1&to=<recipient>&su=...&body=...\` and trigger hotkey \`app_action(action: "key_combo", payload: { keyCombo: "ctrl+enter" })\`. NEVER click blind coordinates like (20, 20) in a web browser!
|
|
731
|
-
d. Work Inside App: Use \`app_action\` or \`run_command\` to execute actions inside the app context (e.g. fetching browser page content, running commands inside terminal, searching web, opening files in editor).
|
|
732
|
-
e. Universal Gaming & External Application Protocol:
|
|
733
|
-
- Desktop Game/App Launch: To open or play ANY installed game or app on the PC (Steam games, Epic Games, Minecraft, Roblox, Discord, Spotify, etc.), call open_app("<game_name>"). The universal OS launcher automatically resolves installed Windows/macOS applications and store packages via system registry and launches them directly!
|
|
734
|
-
- Web Game Play: If the game is web-based (e.g. Chess.com, Slither.io, 2048, Poki) or not installed locally, NEVER guess speculative URL subpaths! Use app_action(app: "browser", action: "search", payload: { query: "<game_name> play online official" }) or navigate to the official domain homepage to find the verified play URL.
|
|
735
|
-
- Autonomous Gameplay Progression Loop:
|
|
736
|
-
1. Window Focus: Use app_action(action: "focus_window", payload: { app: "<game_name>" }) to bring the window front-and-center.
|
|
737
|
-
2. Screen Observation: Use app_action(action: "see_screen") to inspect the UI, loading screen, or active state.
|
|
738
|
-
3. Menu Traversal: Click menu buttons ("Play", "Start Game", "New Game", "Continue") using app_action(action: "click_element", payload: { element: "Play" }) .
|
|
739
|
-
4. Interactive Controls: Send gameplay controls using app_action(action: "send_keys" / "type_text" / "key_combo") with standard gaming keys (WASD, Arrow keys, Space, Enter, Escape, mouse clicks/drags).
|
|
740
|
-
5. Conclude task with task_completed when the requested gameplay actions or objectives are achieved.
|
|
741
|
-
20. DESKTOP APP TAKEOVER & AUTOMATION PROTOCOL:
|
|
742
|
-
a. STEP 1 SCREEN TAKEOVER: Whenever the user goal asks to take over the screen, control the desktop, or automate an external app (e.g. 'take over screen', 'open maps and click', 'take over desktop', 'open notepad'):
|
|
743
|
-
- Your VERY FIRST tool call in Step 1 MUST be \`app_action\` with \`action: "takeover"\` (e.g. \`app: "desktop"\` or the target app).
|
|
744
|
-
- Calling \`app_action\` with \`action: "takeover"\` immediately activates the full-screen sky-blue aura HUD, displays the warning banner "⚡ Scout is on the screen.", blocks external input interruptions, and takes over the mouse cursor!
|
|
745
|
-
- Then immediately proceed to launch/navigate with \`open_app\` or \`app_action\` (\`action: "click_element"\` / \`action: "click_app"\` / \`action: "type_text"\` / \`action: "send_keys"\`).
|
|
746
|
-
b. LIVE SCREEN SHARE & VISION STREAM DIRECTIVE: You have direct real-time live screen sharing with the model! Instead of taking, saving, and sending static screenshot files, stream the screen live in real-time or inspect the live screen share stream using \`app_action\` with \`action: "live_screen_share"\`, \`"start_screen_share"\`, \`"live_screen_stream"\`, \`"see_screen"\`, or \`"analyze_screen"\`. Screen frames are streamed in-memory directly to the model (zero temporary files written to disk).
|
|
747
|
-
c. VISUAL ELEMENT GROUNDING (click_element): Instead of guessing blind coordinates (x, y), click buttons, inputs, or menus by descriptive label using \`app_action(action: "click_element", payload: { element: "Search" })\`. Vision AI and native OS UI automation will locate the element and click it accurately.
|
|
748
|
-
d. WINDOW & SCROLL CONTROLS: Use \`app_action(action: "scroll", payload: { direction: "down", amount: 4 })\` to scroll pages. Use \`app_action(action: "list_windows")\` to see all open windows, and \`app_action(action: "focus_window", payload: { app: "chrome" })\` to bring a window front-and-center.
|
|
749
|
-
e. CLI Credential Input Prompt: If an application requires login credentials, passwords, 2FA codes, or secret tokens to proceed, call \`ask_user\` tool with a clear prompt. This presents a secure, interactive input bar directly in the user's running terminal CLI. Once the user enters the secret, take the received input, inject it into the target application window via \`app_action\` (\`action: "type_text"\`).
|
|
750
|
-
21. MANDATORY TAKEOVER TOOL INVOCATION MANDATE: Whenever the user goal requests to open, launch, take over, click, type, or interact with an external app or desktop screen (e.g. 'take over browser and open website', 'open notepad and type', 'take over desktop', 'take over screen'):
|
|
751
|
-
YOU MUST CALL \`app_action(action: "takeover")\` IN STEP 1. Then call \`open_app\` and \`app_action\` (\`click_element\`, \`click_app\`, \`type_text\`, \`send_keys\`). DO NOT call \`read_file\` or \`write_file\` for workspace code files when asked to take over external desktop apps! The takeover tool call is MANDATORY for executing the physical takeover.
|
|
752
|
-
22. BROWSER DIRECT URL NAVIGATION MANDATE: When asked to open or navigate to a specific website or web app (e.g. Apple Maps, GitHub, YouTube, etc.), NEVER call Google Search or issue repeated \`app_action: search\` calls with text queries! IMMEDIATELY pass the exact URL (e.g. "https://maps.apple.com") to \`open_app(app: "browser", target: "https://maps.apple.com")\` or \`app_action(app: "browser", action: "navigate", payload: { url: "https://maps.apple.com" })\`. Direct URL navigation must always target the exact site URL directly without putting queries into Google Search!
|
|
753
|
-
23. WORKING MEMORY & STRUCTURED SCRATCHPAD PROTOCOL:
|
|
754
|
-
Like Antigravity and leading autonomous agents, maintain disciplined working memory. In EVERY step, start your response with a structured <scratchpad> reasoning block before returning tool calls:
|
|
755
|
-
\`\`\`markdown
|
|
756
|
-
<scratchpad>
|
|
757
|
-
Thought: [1-2 sentences on what you are doing on this turn and why]
|
|
758
|
-
Plan:
|
|
759
|
-
[x] 1. [Completed step]
|
|
760
|
-
[/] 2. [In-progress step]
|
|
761
|
-
[ ] 3. [Next upcoming step]
|
|
762
|
-
Observations: [What you learned from the last tool result or screen capture]
|
|
763
|
-
Next Action: [The exact tool you are invoking now]
|
|
764
|
-
</scratchpad>
|
|
765
|
-
\`\`\`
|
|
766
|
-
This keeps your reasoning crystal-clear, ensures plan progression, and syncs automatically with .ft/scratchpad.md.
|
|
570
|
+
CORE ARCHITECTURAL PRINCIPLES:
|
|
571
|
+
1. USER INTENT > TOOL CONVENIENCE (CRITICAL MANDATE):
|
|
572
|
+
The easiest tool is NOT necessarily the correct tool. Scout optimizes for completing the user's ACTUAL REQUESTED WORKFLOW, not merely producing something that looks like the final artifact.
|
|
573
|
+
- Distinguish OUTCOME requirements (e.g. "Create a file containing X") from EXECUTION requirements (e.g. "Open Notepad, type X, save it as Y on Desktop").
|
|
574
|
+
- When the user explicitly specifies an application (e.g. Notepad, Excel, Chrome, VS Code) or computer interaction method, computer interaction tools (computer_open_app, computer_screenshot, computer_type, computer_hotkey, computer_switch_window, computer_click) MUST be used.
|
|
575
|
+
- NEVER silently substitute direct filesystem API calls (write_file) when computer/application interaction is required, unless the user explicitly permits alternative tools ("using any method you prefer").
|
|
576
|
+
- NEVER assume the repository root is the user's Desktop! If the user requests saving to Desktop, the file MUST be saved to the actual Desktop through the requested workflow.
|
|
767
577
|
|
|
768
|
-
|
|
578
|
+
2. COMPUTER ACTION FAILURE RECOVERY:
|
|
579
|
+
When a computer action fails or encounters a problem:
|
|
580
|
+
DO NOT immediately switch to an unrelated tool (like write_file).
|
|
581
|
+
Instead, follow the bounded computer recovery loop:
|
|
582
|
+
ACTION FAILED ➔ OBSERVE SCREEN (computer_screenshot / computer_inspect_ui) ➔ UNDERSTAND FAILURE (check if application is active/focused) ➔ RETRY / ALTERNATIVE COMPUTER ACTION (switch window, click text area, send hotkey/keystrokes) ➔ OBSERVE ➔ CONTINUE.
|
|
583
|
+
Only after repeated bounded computer recovery attempts fail should you report the failure to the user.
|
|
584
|
+
|
|
585
|
+
3. ACTIVE APPLICATION TRACKING:
|
|
586
|
+
Before sending keyboard or mouse input (computer_type, computer_click, computer_hotkey), ensure that the expected application (e.g. Notepad) is the active foreground window. If not, use computer_switch_window or computer_open_app to focus it first.
|
|
587
|
+
|
|
588
|
+
4. REAL-TIME DATE AWARENESS: You are aware of real-time date and time (${new Date().getFullYear()}). Always use the current year (${new Date().getFullYear()}) and date for web searches, documentation, commit messages, and references. NEVER hardcode obsolete past years like 2024.
|
|
589
|
+
5. HIGH AGENCY & AUTONOMY: You act autonomously through step-by-step tool invocation to solve complex coding tasks, debug issues, build features, create projects from scratch, or operate desktop applications.
|
|
590
|
+
6. MULTILINGUAL & INTENT COMPREHENSION: Understand user intent in any language (English, Hinglish like "likho", "bnao", "code karo", "fix karo", "samjha do", Hindi, Spanish, etc.).
|
|
591
|
+
7. FOR CODE & REPO TASKS: When the task is a coding, project refactoring, or repository maintenance task, use direct code modification tools (write_file, edit_file, run_command, read_file, grep_search).
|
|
592
|
+
8. RUN & VERIFY COMMANDS: Execute shell commands, tests, builds, and formatters using run_command.
|
|
593
|
+
9. CRITICAL TERMINATION RULE: As soon as you have finished creating/modifying the necessary files, running verification checks, or completing the desktop application workflow, YOU MUST CALL task_completed tool immediately with a clear summary of your work.
|
|
594
|
+
10. THOUGHT TRANSPARENCY PROTOCOL: In EVERY step before calling any tools or ending a turn, you MUST provide a clear 1-2 sentence explanation in your text response describing your current reasoning, what file/action you are taking, and why you are taking it.
|
|
769
595
|
|
|
770
596
|
When returning tool calls, use standard OpenAI function calling format or JSON tool call payload format:
|
|
771
597
|
\`\`\`json
|
|
772
598
|
{
|
|
773
|
-
|
|
774
|
-
|
|
599
|
+
"tool": "tool_name",
|
|
600
|
+
"args": { ... }
|
|
775
601
|
}
|
|
776
602
|
\`\`\`
|
|
777
603
|
`.trim();
|
|
778
604
|
}
|
|
779
|
-
getScratchpadPromptContext() {
|
|
780
|
-
if (this.scratchpadState.plan.length === 0 && !this.scratchpadState.notes) {
|
|
781
|
-
return '';
|
|
782
|
-
}
|
|
783
|
-
let str = `ACTIVE SCRATCHPAD STATE (.ft/scratchpad.md):\n`;
|
|
784
|
-
if (this.scratchpadState.plan.length > 0) {
|
|
785
|
-
str += `Plan Checklist:\n` +
|
|
786
|
-
this.scratchpadState.plan.map((item, idx) => ` ${this.scratchpadState.completedSteps.includes(idx) ? '[✓]' : '[ ]'} Step ${idx + 1}: ${item}`).join('\n') + '\n';
|
|
787
|
-
}
|
|
788
|
-
if (this.scratchpadState.notes) {
|
|
789
|
-
str += `Working Notes: ${this.scratchpadState.notes}\n`;
|
|
790
|
-
}
|
|
791
|
-
return str;
|
|
792
|
-
}
|
|
793
605
|
async executeGoal(userGoal) {
|
|
794
606
|
this.resetContext();
|
|
607
|
+
this.executionIntent = inferExecutionIntent(userGoal);
|
|
795
608
|
this.taskStateManager.setObjective(userGoal);
|
|
609
|
+
if (this.computerController.setExpectedApplication && this.executionIntent.requiredApplications.length > 0) {
|
|
610
|
+
this.computerController.setExpectedApplication(this.executionIntent.requiredApplications[0] ?? null);
|
|
611
|
+
}
|
|
796
612
|
const structuredPlan = this.taskPlanner.createPlan(userGoal);
|
|
797
613
|
this.taskStateManager.setPlan(structuredPlan.steps);
|
|
798
614
|
this.taskStateManager.setAvailableTools(this.toolRegistry.getAllTools().map((t) => t.name));
|
|
@@ -922,233 +738,215 @@ When returning tool calls, use standard OpenAI function calling format or JSON t
|
|
|
922
738
|
let step = 0;
|
|
923
739
|
let finalSummary = '';
|
|
924
740
|
const stepDurations = [];
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
741
|
+
while (step < this.maxSteps) {
|
|
742
|
+
if (this.isCancelled) {
|
|
743
|
+
return {
|
|
744
|
+
success: false,
|
|
745
|
+
summary: `Task cancelled: ${this.cancelReason || 'User abort'}`,
|
|
746
|
+
};
|
|
747
|
+
}
|
|
748
|
+
step++;
|
|
749
|
+
this.currentStep = step;
|
|
750
|
+
this.taskStateManager.startStep(step);
|
|
751
|
+
const stepStartTime = Date.now();
|
|
752
|
+
const avgStepMs = stepDurations.length > 0
|
|
753
|
+
? stepDurations.reduce((a, b) => a + b, 0) / stepDurations.length
|
|
754
|
+
: 12000;
|
|
755
|
+
const remainingSteps = (this.maxSteps - step + 1);
|
|
756
|
+
const estSecsLeft = Math.max(5, Math.round((remainingSteps * avgStepMs) / 1000));
|
|
757
|
+
const estLeftStr = estSecsLeft >= 60
|
|
758
|
+
? `${Math.floor(estSecsLeft / 60)}m ${estSecsLeft % 60}s`
|
|
759
|
+
: `${estSecsLeft}s`;
|
|
760
|
+
const stepSpinner = spinner();
|
|
761
|
+
stepSpinner.start(chalk.cyan(`Scout is Working (Step ${step}/${this.maxSteps} • Est. completion: ~${estLeftStr} left)`));
|
|
762
|
+
try {
|
|
763
|
+
let response;
|
|
941
764
|
try {
|
|
942
|
-
|
|
943
|
-
|
|
765
|
+
response = await callOpenAIWithRetry(async (model) => {
|
|
766
|
+
return await openai.chat.completions.create({
|
|
767
|
+
model,
|
|
768
|
+
messages: sanitizeMessages(this.historyMessages),
|
|
769
|
+
tools: this.toolRegistry.getAllTools().map((t) => ({ type: 'function', function: t })),
|
|
770
|
+
tool_choice: 'auto',
|
|
771
|
+
temperature: 0.1,
|
|
772
|
+
presence_penalty: 0.1,
|
|
773
|
+
frequency_penalty: 0.1,
|
|
774
|
+
});
|
|
775
|
+
});
|
|
776
|
+
}
|
|
777
|
+
catch (err) {
|
|
778
|
+
const errStr = String(err?.message || err?.error || err || '').toLowerCase();
|
|
779
|
+
if (errStr.includes('tool') || errStr.includes('400') || errStr.includes('not supported') || errStr.includes('reasoning')) {
|
|
780
|
+
// Model doesn't support native function calling — inject tool-call formatting hint
|
|
781
|
+
// so extractJsonToolCall can parse the response as a structured tool invocation
|
|
782
|
+
const toolHintMsg = {
|
|
783
|
+
role: 'user',
|
|
784
|
+
content: `IMPORTANT: This model does not support native function/tool calling. You MUST format your tool invocations as a JSON code block in your response like this:
|
|
785
|
+
\`\`\`json
|
|
786
|
+
{ "tool": "tool_name", "args": { ... } }
|
|
787
|
+
\`\`\`
|
|
788
|
+
Available tools: read_file, write_file, edit_file, run_command, list_dir, grep_search, glob_search, tree_view, file_info, multi_edit_file, fetch_url, git_diff, ask_user, speak_text, task_completed.
|
|
789
|
+
You MUST output exactly ONE JSON code block per tool call. Do NOT describe what you would do in plain text—output the JSON tool call directly!`,
|
|
790
|
+
};
|
|
791
|
+
const fallbackMessages = [...this.historyMessages, toolHintMsg];
|
|
944
792
|
response = await callOpenAIWithRetry(async (model) => {
|
|
945
793
|
return await openai.chat.completions.create({
|
|
946
794
|
model,
|
|
947
|
-
messages: sanitizeMessages(
|
|
948
|
-
tools: this.toolRegistry.getAllTools().map((t) => ({ type: 'function', function: t })),
|
|
949
|
-
tool_choice: 'auto',
|
|
795
|
+
messages: sanitizeMessages(fallbackMessages),
|
|
950
796
|
temperature: 0.1,
|
|
951
797
|
presence_penalty: 0.1,
|
|
952
798
|
frequency_penalty: 0.1,
|
|
953
799
|
});
|
|
954
800
|
});
|
|
955
801
|
}
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
if (errStr.includes('tool') || errStr.includes('400') || errStr.includes('not supported') || errStr.includes('reasoning')) {
|
|
959
|
-
// Model doesn't support native function calling — inject tool-call formatting hint
|
|
960
|
-
// so extractJsonToolCall can parse the response as a structured tool invocation
|
|
961
|
-
const toolHintMsg = {
|
|
962
|
-
role: 'user',
|
|
963
|
-
content: `IMPORTANT: This model does not support native function/tool calling. You MUST format your tool invocations as a JSON code block in your response like this:
|
|
964
|
-
\`\`\`json
|
|
965
|
-
{ "tool": "tool_name", "args": { ... } }
|
|
966
|
-
\`\`\`
|
|
967
|
-
Available tools: read_file, write_file, edit_file, run_command, list_dir, grep_search, glob_search, tree_view, file_info, multi_edit_file, fetch_url, git_diff, open_app, app_action, agent_scratchpad, ask_user, speak_text, task_completed.
|
|
968
|
-
For app takeover/control use: { "tool": "app_action", "args": { "app": "browser", "action": "takeover" } }
|
|
969
|
-
For opening apps use: { "tool": "open_app", "args": { "app": "chrome", "target": "https://..." } }
|
|
970
|
-
For clicking use: { "tool": "app_action", "args": { "app": "desktop", "action": "click_app", "payload": { "x": 500, "y": 300 } } }
|
|
971
|
-
For typing text use: { "tool": "app_action", "args": { "app": "browser", "action": "type_text", "payload": { "text": "...", "enter": true } } }
|
|
972
|
-
You MUST output exactly ONE JSON code block per tool call. Do NOT describe what you would do in plain text—output the JSON tool call directly!`,
|
|
973
|
-
};
|
|
974
|
-
const fallbackMessages = [...this.historyMessages, toolHintMsg];
|
|
975
|
-
response = await callOpenAIWithRetry(async (model) => {
|
|
976
|
-
return await openai.chat.completions.create({
|
|
977
|
-
model,
|
|
978
|
-
messages: sanitizeMessages(fallbackMessages),
|
|
979
|
-
temperature: 0.1,
|
|
980
|
-
presence_penalty: 0.1,
|
|
981
|
-
frequency_penalty: 0.1,
|
|
982
|
-
});
|
|
983
|
-
});
|
|
984
|
-
}
|
|
985
|
-
else {
|
|
986
|
-
throw err;
|
|
987
|
-
}
|
|
988
|
-
}
|
|
989
|
-
const choice = response.choices[0];
|
|
990
|
-
if (!choice) {
|
|
991
|
-
stepSpinner.stop(chalk.yellow('No response from Scout. Retrying step...'));
|
|
992
|
-
continue;
|
|
802
|
+
else {
|
|
803
|
+
throw err;
|
|
993
804
|
}
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
805
|
+
}
|
|
806
|
+
const choice = response.choices[0];
|
|
807
|
+
if (!choice) {
|
|
808
|
+
stepSpinner.stop(chalk.yellow('No response from Scout. Retrying step...'));
|
|
809
|
+
continue;
|
|
810
|
+
}
|
|
811
|
+
const msg = sanitizeMessage(choice.message);
|
|
812
|
+
this.historyMessages.push(msg);
|
|
813
|
+
// Render assistant's thought reasoning text if provided on this turn
|
|
814
|
+
const textContent = msg.content || '';
|
|
815
|
+
if (textContent.trim()) {
|
|
816
|
+
safeNote(renderMarkdown(textContent), `Scout Thought (Step ${step})`);
|
|
817
|
+
}
|
|
818
|
+
// Check native tool calls
|
|
819
|
+
if (msg.tool_calls && msg.tool_calls.length > 0) {
|
|
820
|
+
stepSpinner.stop(chalk.green(`Step ${step}: Scout issued ${msg.tool_calls.length} tool call(s).`));
|
|
821
|
+
for (const tc of msg.tool_calls) {
|
|
822
|
+
if (tc.type === 'function' && tc.function) {
|
|
823
|
+
const fnName = tc.function.name;
|
|
824
|
+
let args = {};
|
|
825
|
+
try {
|
|
826
|
+
args = JSON.parse(tc.function.arguments || '{}');
|
|
1007
827
|
}
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
role: 'user',
|
|
1027
|
-
tool_call_id: tc.id,
|
|
1028
|
-
content: `Tool Execution Result (${fnName}):\n${toolResult.result}`,
|
|
1029
|
-
});
|
|
1030
|
-
this.handleToolConsecutiveTracking(fnName, args, step);
|
|
1031
|
-
if (fnName === 'task_completed') {
|
|
1032
|
-
finalSummary = args.summary || toolResult.result;
|
|
1033
|
-
if (this.modifiedFiles.size > 0) {
|
|
1034
|
-
const filesList = getDirectoryFiles(this.cwd);
|
|
1035
|
-
const healRes = await verifyAndSelfHealFiles(Array.from(this.modifiedFiles), this.cwd, this.projectName, filesList, { maxRetries: 3 });
|
|
1036
|
-
if (healRes.verifiedFiles.length > 0) {
|
|
1037
|
-
safeNote(chalk.green(` Self-Healing Verification Confirmed: ${healRes.verifiedFiles.length} file(s) syntax & build clean!`), ' Code Verification Clean');
|
|
1038
|
-
}
|
|
1039
|
-
if (healRes.remainingErrors.length > 0) {
|
|
1040
|
-
safeNote(chalk.yellow(`️ Remaining verification issues:\n${healRes.remainingErrors.join('\n')}`), '️ Verification Warning');
|
|
1041
|
-
}
|
|
828
|
+
catch { }
|
|
829
|
+
const toolResult = await this.dispatchToolCall(fnName, args);
|
|
830
|
+
this.historyMessages.push({
|
|
831
|
+
role: 'user',
|
|
832
|
+
tool_call_id: tc.id,
|
|
833
|
+
content: `Tool Execution Result (${fnName}):\n${toolResult.result}`,
|
|
834
|
+
});
|
|
835
|
+
this.handleToolConsecutiveTracking(fnName, args, step);
|
|
836
|
+
if (fnName === 'task_completed') {
|
|
837
|
+
finalSummary = args.summary || toolResult.result;
|
|
838
|
+
if (this.modifiedFiles.size > 0) {
|
|
839
|
+
const filesList = getDirectoryFiles(this.cwd);
|
|
840
|
+
const healRes = await verifyAndSelfHealFiles(Array.from(this.modifiedFiles), this.cwd, path.basename(this.cwd), filesList, { maxRetries: 3 });
|
|
841
|
+
if (healRes.verifiedFiles.length > 0) {
|
|
842
|
+
safeNote(chalk.green(` Self-Healing Verification Confirmed: ${healRes.verifiedFiles.length} file(s) syntax & build clean!`), ' Code Verification Clean');
|
|
843
|
+
}
|
|
844
|
+
if (healRes.remainingErrors.length > 0) {
|
|
845
|
+
safeNote(chalk.yellow(`️ Remaining verification issues:\n${healRes.remainingErrors.join('\n')}`), '️ Verification Warning');
|
|
1042
846
|
}
|
|
1043
|
-
saveAgentSession({
|
|
1044
|
-
goal: userGoal,
|
|
1045
|
-
summary: finalSummary,
|
|
1046
|
-
modifiedFiles: Array.from(this.modifiedFiles),
|
|
1047
|
-
});
|
|
1048
|
-
return { success: true, summary: finalSummary };
|
|
1049
847
|
}
|
|
848
|
+
saveAgentSession({
|
|
849
|
+
goal: userGoal,
|
|
850
|
+
summary: finalSummary,
|
|
851
|
+
modifiedFiles: Array.from(this.modifiedFiles),
|
|
852
|
+
});
|
|
853
|
+
return { success: true, summary: finalSummary };
|
|
1050
854
|
}
|
|
1051
855
|
}
|
|
1052
|
-
continue;
|
|
1053
|
-
}
|
|
1054
|
-
// Check text response or fallback JSON tool call
|
|
1055
|
-
stepSpinner.stop(chalk.blue(`Step ${step} thinking complete.`));
|
|
1056
|
-
const parsedJsonTool = this.extractJsonToolCall(textContent);
|
|
1057
|
-
if (parsedJsonTool) {
|
|
1058
|
-
const toolResult = await this.dispatchToolCall(parsedJsonTool.tool, parsedJsonTool.args);
|
|
1059
|
-
this.historyMessages.push({
|
|
1060
|
-
role: 'user',
|
|
1061
|
-
content: `Tool Execution Result (${parsedJsonTool.tool}):\n${toolResult.result}`,
|
|
1062
|
-
});
|
|
1063
|
-
this.handleToolConsecutiveTracking(parsedJsonTool.tool, parsedJsonTool.args, step);
|
|
1064
|
-
if (parsedJsonTool.tool === 'task_completed') {
|
|
1065
|
-
finalSummary = parsedJsonTool.args?.summary || toolResult.result;
|
|
1066
|
-
saveAgentSession({
|
|
1067
|
-
goal: userGoal,
|
|
1068
|
-
summary: finalSummary,
|
|
1069
|
-
modifiedFiles: Array.from(this.modifiedFiles),
|
|
1070
|
-
});
|
|
1071
|
-
return { success: true, summary: finalSummary };
|
|
1072
|
-
}
|
|
1073
|
-
continue;
|
|
1074
856
|
}
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
safeNote(renderMarkdown(cleanedThought), ` Scout Agent Thought (Step ${step})`);
|
|
1083
|
-
const lowerText = textContent.toLowerCase();
|
|
1084
|
-
const isExplicitCompletion = lowerText.includes('task is complete') ||
|
|
1085
|
-
lowerText.includes('task complete') ||
|
|
1086
|
-
lowerText.includes('goal completed') ||
|
|
1087
|
-
lowerText.includes('goal is completed') ||
|
|
1088
|
-
lowerText.includes('all tasks completed') ||
|
|
1089
|
-
lowerText.includes('i have completed') ||
|
|
1090
|
-
lowerText.includes('no further changes needed') ||
|
|
1091
|
-
lowerText.includes('the fix is complete') ||
|
|
1092
|
-
lowerText.includes('has been created') ||
|
|
1093
|
-
lowerText.includes('successfully created') ||
|
|
1094
|
-
lowerText.includes('created the file') ||
|
|
1095
|
-
lowerText.includes('file created') ||
|
|
1096
|
-
lowerText.includes('implementation complete') ||
|
|
1097
|
-
lowerText.includes('work is complete');
|
|
1098
|
-
// If explicit completion phrase found, OR files have already been modified and assistant returned a final summary without calling tools
|
|
1099
|
-
if (isExplicitCompletion || (this.modifiedFiles.size > 0 && !lowerText.includes('?') && textContent.length > 50)) {
|
|
1100
|
-
saveAgentSession({
|
|
1101
|
-
goal: userGoal,
|
|
1102
|
-
summary: textContent,
|
|
1103
|
-
modifiedFiles: Array.from(this.modifiedFiles),
|
|
1104
|
-
});
|
|
1105
|
-
return { success: true, summary: textContent };
|
|
1106
|
-
}
|
|
1107
|
-
}
|
|
1108
|
-
// If assistant responded with text without calling tools, prompt it to execute tools to complete the goal
|
|
857
|
+
continue;
|
|
858
|
+
}
|
|
859
|
+
// Check text response or fallback JSON tool call
|
|
860
|
+
stepSpinner.stop(chalk.blue(`Step ${step} thinking complete.`));
|
|
861
|
+
const parsedJsonTool = this.extractJsonToolCall(textContent);
|
|
862
|
+
if (parsedJsonTool) {
|
|
863
|
+
const toolResult = await this.dispatchToolCall(parsedJsonTool.tool, parsedJsonTool.args);
|
|
1109
864
|
this.historyMessages.push({
|
|
1110
865
|
role: 'user',
|
|
1111
|
-
content:
|
|
866
|
+
content: `Tool Execution Result (${parsedJsonTool.tool}):\n${toolResult.result}`,
|
|
1112
867
|
});
|
|
1113
|
-
|
|
868
|
+
this.handleToolConsecutiveTracking(parsedJsonTool.tool, parsedJsonTool.args, step);
|
|
869
|
+
if (parsedJsonTool.tool === 'task_completed') {
|
|
870
|
+
finalSummary = parsedJsonTool.args?.summary || toolResult.result;
|
|
871
|
+
saveAgentSession({
|
|
872
|
+
goal: userGoal,
|
|
873
|
+
summary: finalSummary,
|
|
874
|
+
modifiedFiles: Array.from(this.modifiedFiles),
|
|
875
|
+
});
|
|
876
|
+
return { success: true, summary: finalSummary };
|
|
877
|
+
}
|
|
878
|
+
continue;
|
|
1114
879
|
}
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
880
|
+
if (textContent.trim()) {
|
|
881
|
+
let cleanedThought = textContent
|
|
882
|
+
.replace(/[\u0600-\u06FF\u0750-\u077F\uAC00-\uD7AF\u3040-\u30FF\u4E00-\u9FFF\u0D80-\u0DFF]+/g, '')
|
|
883
|
+
.trim();
|
|
884
|
+
if (!cleanedThought || cleanedThought.length < 5) {
|
|
885
|
+
cleanedThought = 'Analyzing codebase files and executing next tool operation...';
|
|
886
|
+
}
|
|
887
|
+
safeNote(renderMarkdown(cleanedThought), ` Scout Agent Thought (Step ${step})`);
|
|
888
|
+
const lowerText = textContent.toLowerCase();
|
|
889
|
+
const isExplicitCompletion = lowerText.includes('task is complete') ||
|
|
890
|
+
lowerText.includes('task complete') ||
|
|
891
|
+
lowerText.includes('goal completed') ||
|
|
892
|
+
lowerText.includes('goal is completed') ||
|
|
893
|
+
lowerText.includes('all tasks completed') ||
|
|
894
|
+
lowerText.includes('i have completed') ||
|
|
895
|
+
lowerText.includes('no further changes needed') ||
|
|
896
|
+
lowerText.includes('the fix is complete') ||
|
|
897
|
+
lowerText.includes('has been created') ||
|
|
898
|
+
lowerText.includes('successfully created') ||
|
|
899
|
+
lowerText.includes('created the file') ||
|
|
900
|
+
lowerText.includes('file created') ||
|
|
901
|
+
lowerText.includes('implementation complete') ||
|
|
902
|
+
lowerText.includes('work is complete');
|
|
903
|
+
// If explicit completion phrase found, OR files have already been modified and assistant returned a final summary without calling tools
|
|
904
|
+
if (isExplicitCompletion || (this.modifiedFiles.size > 0 && !lowerText.includes('?') && textContent.length > 50)) {
|
|
905
|
+
saveAgentSession({
|
|
906
|
+
goal: userGoal,
|
|
907
|
+
summary: textContent,
|
|
908
|
+
modifiedFiles: Array.from(this.modifiedFiles),
|
|
909
|
+
});
|
|
910
|
+
return { success: true, summary: textContent };
|
|
1128
911
|
}
|
|
1129
|
-
stepSpinner.stop(chalk.red(`Step ${step} execution error: ${err?.message || String(err)}`));
|
|
1130
|
-
this.historyMessages.push({
|
|
1131
|
-
role: 'user',
|
|
1132
|
-
content: `Error in previous turn: ${err?.message || String(err)}. Please try alternative steps or call tools.`,
|
|
1133
|
-
});
|
|
1134
912
|
}
|
|
1135
|
-
// If
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
913
|
+
// If assistant responded with text without calling tools, prompt it to execute tools to complete the goal
|
|
914
|
+
this.historyMessages.push({
|
|
915
|
+
role: 'user',
|
|
916
|
+
content: 'You provided a text response but have not called any tools (write_file, edit_file, run_command, task_completed). Please execute necessary tool calls to complete the user goal, or invoke task_completed if finished.',
|
|
917
|
+
});
|
|
918
|
+
stepDurations.push(Date.now() - stepStartTime);
|
|
919
|
+
}
|
|
920
|
+
catch (err) {
|
|
921
|
+
stepDurations.push(Date.now() - stepStartTime);
|
|
922
|
+
if (isQuotaExceededError(err)) {
|
|
923
|
+
stepSpinner.stop(chalk.red(`Oops! it\'s not you, it\'s us`));
|
|
924
|
+
safeNote(`${chalk.bold.red(`Something went wrong in (Step ${step}), please try again in a moment`)}\n\n` +
|
|
925
|
+
`${chalk.yellow('The configured AI provider has reached its API usage limit or rate cap.')}\n` +
|
|
926
|
+
`${chalk.dim('This is separate from your Scout credit balance shown by `scout quota`.')}\n\n` +
|
|
927
|
+
`${chalk.bold.cyan(' Please come back and try again in a few hours (or check back later today).')}\n\n` +
|
|
928
|
+
`${chalk.dim('Scout Agent session has ended gracefully to protect remaining workflow.')}`, '️ API Quota Limit Reached');
|
|
929
|
+
return {
|
|
930
|
+
success: false,
|
|
931
|
+
summary: 'Something went wrong, please try again in a moment.',
|
|
932
|
+
};
|
|
1139
933
|
}
|
|
934
|
+
stepSpinner.stop(chalk.red(`Step ${step} execution error: ${err?.message || String(err)}`));
|
|
935
|
+
this.historyMessages.push({
|
|
936
|
+
role: 'user',
|
|
937
|
+
content: `Error in previous turn: ${err?.message || String(err)}. Please try alternative steps or call tools.`,
|
|
938
|
+
});
|
|
1140
939
|
}
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
}
|
|
1146
|
-
finally {
|
|
1147
|
-
cleanupScreenshots();
|
|
1148
|
-
if (isLiveScreenShareActive()) {
|
|
1149
|
-
stopLiveScreenShare();
|
|
940
|
+
// If current max steps limit is reached while task is still in progress, auto-extend by +15 extra steps (up to 60 max threshold)
|
|
941
|
+
if (step >= this.maxSteps && this.maxSteps < 60) {
|
|
942
|
+
this.maxSteps += 15;
|
|
943
|
+
safeNote(chalk.bold.yellow(`⚡ Step limit reached while task is in progress. Automatically extending execution by +15 extra steps (New Max Limit: ${this.maxSteps})...`), ' Auto Extra Steps Extension');
|
|
1150
944
|
}
|
|
1151
945
|
}
|
|
946
|
+
return {
|
|
947
|
+
success: false,
|
|
948
|
+
summary: `Reached max iteration steps limit (${this.maxSteps}). Modified files: ${Array.from(this.modifiedFiles).join(', ')}`,
|
|
949
|
+
};
|
|
1152
950
|
}
|
|
1153
951
|
extractJsonToolCall(content) {
|
|
1154
952
|
if (!content)
|
|
@@ -1168,10 +966,8 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
|
|
|
1168
966
|
const knownTools = [
|
|
1169
967
|
'read_file', 'write_file', 'edit_file', 'run_command', 'list_dir',
|
|
1170
968
|
'grep_search', 'glob_search', 'tree_view', 'file_info', 'multi_edit_file',
|
|
1171
|
-
'fetch_url', 'git_diff',
|
|
969
|
+
'fetch_url', 'git_diff',
|
|
1172
970
|
'ask_user', 'speak_text', 'task_completed',
|
|
1173
|
-
'desktop_action', 'type_text', 'click_app', 'send_keys', 'takeover',
|
|
1174
|
-
'capture_screen', 'see_screen', 'analyze_screen', 'scratchpad',
|
|
1175
971
|
];
|
|
1176
972
|
for (const toolName of knownTools) {
|
|
1177
973
|
const tagRegex = new RegExp(`<${toolName}>([\\s\\S]*?)<\\/${toolName}>`, 'i');
|
|
@@ -1227,10 +1023,6 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
|
|
|
1227
1023
|
actionDesc = `Fetching URL: ${args.url || ''}`;
|
|
1228
1024
|
else if (name === 'git_diff')
|
|
1229
1025
|
actionDesc = `Retrieving workspace git diff`;
|
|
1230
|
-
else if (name === 'open_app')
|
|
1231
|
-
actionDesc = `Launching app: ${args.app || ''} (${args.target || 'default'})`;
|
|
1232
|
-
else if (name === 'app_action')
|
|
1233
|
-
actionDesc = `Executing app action: ${args.app || ''} -> ${args.action || ''}`;
|
|
1234
1026
|
else if (name === 'browser_navigate')
|
|
1235
1027
|
actionDesc = `Navigating browser: ${args.url || ''}`;
|
|
1236
1028
|
else if (name === 'browser_inspect')
|
|
@@ -1244,19 +1036,25 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
|
|
|
1244
1036
|
else if (name === 'browser_content')
|
|
1245
1037
|
actionDesc = `Extracting web page content`;
|
|
1246
1038
|
else if (name === 'computer_screenshot')
|
|
1247
|
-
actionDesc = `
|
|
1039
|
+
actionDesc = `Observing computer screen`;
|
|
1248
1040
|
else if (name === 'computer_click')
|
|
1249
|
-
actionDesc = `Clicking
|
|
1041
|
+
actionDesc = `Clicking screen coordinate: (${args.x}, ${args.y})`;
|
|
1042
|
+
else if (name === 'computer_double_click')
|
|
1043
|
+
actionDesc = `Double clicking coordinate: (${args.x}, ${args.y})`;
|
|
1250
1044
|
else if (name === 'computer_type')
|
|
1251
|
-
actionDesc = `Typing text
|
|
1045
|
+
actionDesc = `Typing text: "${String(args.text || '').slice(0, 30)}"`;
|
|
1046
|
+
else if (name === 'computer_key_press')
|
|
1047
|
+
actionDesc = `Pressing key: ${args.key || ''}`;
|
|
1252
1048
|
else if (name === 'computer_hotkey')
|
|
1253
|
-
actionDesc = `
|
|
1049
|
+
actionDesc = `Executing hotkey: ${(args.keys || []).join('+')}`;
|
|
1254
1050
|
else if (name === 'computer_open_app')
|
|
1255
|
-
actionDesc = `
|
|
1256
|
-
else if (name === '
|
|
1257
|
-
actionDesc = `
|
|
1258
|
-
else if (name === '
|
|
1259
|
-
actionDesc = `
|
|
1051
|
+
actionDesc = `Launching application: ${args.appName || args.app || ''}`;
|
|
1052
|
+
else if (name === 'computer_close_app')
|
|
1053
|
+
actionDesc = `Closing application: ${args.appName || args.app || ''}`;
|
|
1054
|
+
else if (name === 'computer_switch_window')
|
|
1055
|
+
actionDesc = `Switching window: ${args.windowTitleOrApp || ''}`;
|
|
1056
|
+
else if (name === 'computer_inspect_ui')
|
|
1057
|
+
actionDesc = `Inspecting visible desktop UI`;
|
|
1260
1058
|
else if (name === 'task_completed')
|
|
1261
1059
|
actionDesc = `Task finished`;
|
|
1262
1060
|
const badge = chalk.bgCyan.black.bold(` 🛠️ TOOL: ${name} `);
|
|
@@ -1280,6 +1078,27 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
|
|
|
1280
1078
|
return { result: `[ACTION REJECTED BY USER]: ${decision.reason || 'User did not approve executing this action.'}` };
|
|
1281
1079
|
}
|
|
1282
1080
|
}
|
|
1081
|
+
// Validate against execution intent (USER INTENT > TOOL CONVENIENCE)
|
|
1082
|
+
if (this.executionIntent && !this.executionIntent.allowAlternativeTools) {
|
|
1083
|
+
if (this.executionIntent.requiredInteraction === 'computer') {
|
|
1084
|
+
if (['write_file', 'edit_file', 'multi_edit_file'].includes(name)) {
|
|
1085
|
+
const reqApps = this.executionIntent.requiredApplications.join(', ') || 'the requested desktop application';
|
|
1086
|
+
const violationMsg = `[EXECUTION INTENT VIOLATION BLOCKED]: User explicitly required computer/application interaction using ${reqApps}. You cannot silently substitute direct filesystem API calls (${name}). You must use computer interaction tools (computer_open_app, computer_switch_window, computer_click, computer_type, computer_hotkey) to perform this task in ${reqApps}.`;
|
|
1087
|
+
console.log(chalk.red.bold(`\n⛔ ${violationMsg}`));
|
|
1088
|
+
this.taskStateManager.addObservation(name, violationMsg, false);
|
|
1089
|
+
return {
|
|
1090
|
+
result: JSON.stringify({
|
|
1091
|
+
success: false,
|
|
1092
|
+
action: name,
|
|
1093
|
+
error: 'ExecutionIntentViolation',
|
|
1094
|
+
message: violationMsg,
|
|
1095
|
+
recoverable: true,
|
|
1096
|
+
suggestedAction: `Use computer interaction tools with ${reqApps}. Focus the window, type text into the app, and save through the application UI.`
|
|
1097
|
+
}, null, 2)
|
|
1098
|
+
};
|
|
1099
|
+
}
|
|
1100
|
+
}
|
|
1101
|
+
}
|
|
1283
1102
|
switch (name) {
|
|
1284
1103
|
case 'read_file': {
|
|
1285
1104
|
const targetPath = String(args.path || '').trim();
|
|
@@ -1314,12 +1133,16 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
|
|
|
1314
1133
|
return { result: `File: ${targetPath} (Lines ${start}-${end}/${lines.length}):\n${sliced}${minifiedNotice}` };
|
|
1315
1134
|
}
|
|
1316
1135
|
case 'write_file': {
|
|
1317
|
-
|
|
1136
|
+
let targetPath = String(args.path || '').trim();
|
|
1318
1137
|
let content = String(args.content || '');
|
|
1319
1138
|
const reason = String(args.reason || args.why || args.description || 'Created/updated file content to fulfill user prompt.');
|
|
1320
1139
|
if (content.includes('\\n') && !content.includes('\n')) {
|
|
1321
1140
|
content = content.replace(/\\n/g, '\n').replace(/\\t/g, '\t');
|
|
1322
1141
|
}
|
|
1142
|
+
if (this.executionIntent?.targetDestination === 'Desktop' || targetPath.toLowerCase().includes('desktop')) {
|
|
1143
|
+
const fileName = path.basename(targetPath);
|
|
1144
|
+
targetPath = resolveDesktopPath(fileName);
|
|
1145
|
+
}
|
|
1323
1146
|
const fullPath = path.isAbsolute(targetPath) ? targetPath : path.resolve(this.cwd, targetPath);
|
|
1324
1147
|
const fileExists = fs.existsSync(fullPath);
|
|
1325
1148
|
const actionText = fileExists ? 'Editing file' : 'Creating file';
|
|
@@ -1659,49 +1482,6 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
|
|
|
1659
1482
|
return { result: `Could not retrieve git diff: ${err?.message || String(err)}` };
|
|
1660
1483
|
}
|
|
1661
1484
|
}
|
|
1662
|
-
case 'agent_scratchpad': {
|
|
1663
|
-
const action = String(args.action || 'update').toLowerCase();
|
|
1664
|
-
if (action === 'clear') {
|
|
1665
|
-
this.scratchpadState = { plan: [], completedSteps: [], notes: '' };
|
|
1666
|
-
try {
|
|
1667
|
-
const scratchpadPath = path.join(this.cwd, '.ft', 'scratchpad.md');
|
|
1668
|
-
if (fs.existsSync(scratchpadPath))
|
|
1669
|
-
fs.unlinkSync(scratchpadPath);
|
|
1670
|
-
}
|
|
1671
|
-
catch { }
|
|
1672
|
-
return { result: 'Agent Scratchpad cleared.' };
|
|
1673
|
-
}
|
|
1674
|
-
else if (action === 'read') {
|
|
1675
|
-
const text = this.getScratchpadPromptContext() || 'Scratchpad is currently empty.';
|
|
1676
|
-
return { result: text };
|
|
1677
|
-
}
|
|
1678
|
-
else {
|
|
1679
|
-
const res = this.updateScratchpad(Array.isArray(args.plan) ? args.plan : undefined, Array.isArray(args.completedSteps) ? args.completedSteps : undefined, args.notes !== undefined ? String(args.notes) : undefined);
|
|
1680
|
-
return { result: res };
|
|
1681
|
-
}
|
|
1682
|
-
}
|
|
1683
|
-
case 'open_app': {
|
|
1684
|
-
const appName = String(args.app || '').trim();
|
|
1685
|
-
const target = args.target ? String(args.target) : undefined;
|
|
1686
|
-
const launchRes = await openApp(appName, target, args);
|
|
1687
|
-
return { result: launchRes.output };
|
|
1688
|
-
}
|
|
1689
|
-
case 'scratchpad':
|
|
1690
|
-
case 'desktop_action':
|
|
1691
|
-
case 'type_text':
|
|
1692
|
-
case 'click_app':
|
|
1693
|
-
case 'send_keys':
|
|
1694
|
-
case 'takeover':
|
|
1695
|
-
case 'capture_screen':
|
|
1696
|
-
case 'see_screen':
|
|
1697
|
-
case 'analyze_screen':
|
|
1698
|
-
case 'app_action': {
|
|
1699
|
-
const appName = String(args.app || args.appName || 'desktop').trim();
|
|
1700
|
-
const action = String(args.action || (name !== 'app_action' ? name : 'type_text')).trim();
|
|
1701
|
-
const payload = args.payload !== undefined ? args.payload : (args.text || args.content || args.target || args.query || args.element || args.url || args.command || args);
|
|
1702
|
-
const actionRes = await executeInApp(appName, action, payload);
|
|
1703
|
-
return { result: actionRes.output };
|
|
1704
|
-
}
|
|
1705
1485
|
case 'ask_user': {
|
|
1706
1486
|
const questionText = String(args.question || args.prompt || 'Please provide input:').trim();
|
|
1707
1487
|
const askSpin = spinner();
|
|
@@ -1781,56 +1561,188 @@ You MUST output exactly ONE JSON code block per tool call. Do NOT describe what
|
|
|
1781
1561
|
}
|
|
1782
1562
|
case 'computer_screenshot': {
|
|
1783
1563
|
const res = await this.computerController.screenshot();
|
|
1784
|
-
|
|
1785
|
-
|
|
1564
|
+
if (res.path) {
|
|
1565
|
+
this.taskStateManager.recordArtifact({
|
|
1566
|
+
filename: path.basename(res.path),
|
|
1567
|
+
path: res.path,
|
|
1568
|
+
type: 'image',
|
|
1569
|
+
description: `Screen observation (${res.width}x${res.height})`,
|
|
1570
|
+
application: res.activeWindow || 'Desktop',
|
|
1571
|
+
});
|
|
1572
|
+
}
|
|
1573
|
+
if (res.activeWindow) {
|
|
1574
|
+
this.taskStateManager.setActiveWindow(res.activeWindow);
|
|
1575
|
+
}
|
|
1576
|
+
const summary = `Screen captured (${res.width}x${res.height}), Active window: "${res.activeWindow || 'Desktop'}"${res.path ? `, Saved: ${res.path}` : ''}`;
|
|
1577
|
+
this.taskStateManager.addObservation('computer_screenshot', summary, true);
|
|
1578
|
+
return { result: summary };
|
|
1786
1579
|
}
|
|
1787
1580
|
case 'computer_click': {
|
|
1788
|
-
const x = Number(args.x)
|
|
1789
|
-
const y = Number(args.y)
|
|
1790
|
-
const res = args.
|
|
1791
|
-
? await this.computerController.double_click(x, y)
|
|
1792
|
-
: await this.computerController.click(x, y, args.button || 'left');
|
|
1581
|
+
const x = Number(args.x);
|
|
1582
|
+
const y = Number(args.y);
|
|
1583
|
+
const res = await this.computerController.click(x, y, { button: args.button });
|
|
1793
1584
|
this.taskStateManager.addObservation('computer_click', res.output, res.success, `(${x}, ${y})`);
|
|
1794
|
-
|
|
1585
|
+
const structuredRes = {
|
|
1586
|
+
success: res.success,
|
|
1587
|
+
action: 'click',
|
|
1588
|
+
output: res.output,
|
|
1589
|
+
...(res.error ? { error: res.error } : {}),
|
|
1590
|
+
recoverable: res.recoverable ?? true,
|
|
1591
|
+
...(res.activeWindow ? { activeWindow: res.activeWindow } : {}),
|
|
1592
|
+
};
|
|
1593
|
+
return { result: JSON.stringify(structuredRes, null, 2) };
|
|
1594
|
+
}
|
|
1595
|
+
case 'computer_double_click': {
|
|
1596
|
+
const x = Number(args.x);
|
|
1597
|
+
const y = Number(args.y);
|
|
1598
|
+
const res = await this.computerController.doubleClick(x, y);
|
|
1599
|
+
this.taskStateManager.addObservation('computer_double_click', res.output, res.success, `(${x}, ${y})`);
|
|
1600
|
+
const structuredRes = {
|
|
1601
|
+
success: res.success,
|
|
1602
|
+
action: 'double_click',
|
|
1603
|
+
output: res.output,
|
|
1604
|
+
...(res.error ? { error: res.error } : {}),
|
|
1605
|
+
recoverable: res.recoverable ?? true,
|
|
1606
|
+
};
|
|
1607
|
+
return { result: JSON.stringify(structuredRes, null, 2) };
|
|
1795
1608
|
}
|
|
1796
1609
|
case 'computer_type': {
|
|
1797
|
-
const
|
|
1798
|
-
const res = await this.computerController.type(
|
|
1610
|
+
const textVal = String(args.text || '');
|
|
1611
|
+
const res = await this.computerController.type(textVal);
|
|
1799
1612
|
this.taskStateManager.addObservation('computer_type', res.output, res.success);
|
|
1800
|
-
|
|
1613
|
+
const structuredRes = {
|
|
1614
|
+
success: res.success,
|
|
1615
|
+
action: 'type',
|
|
1616
|
+
output: res.output,
|
|
1617
|
+
...(res.error ? { error: res.error } : {}),
|
|
1618
|
+
recoverable: res.recoverable ?? true,
|
|
1619
|
+
...(res.activeWindow ? { activeWindow: res.activeWindow } : {}),
|
|
1620
|
+
...(res.expectedWindow ? { expectedWindow: res.expectedWindow } : {}),
|
|
1621
|
+
};
|
|
1622
|
+
return { result: JSON.stringify(structuredRes, null, 2) };
|
|
1623
|
+
}
|
|
1624
|
+
case 'computer_key_press': {
|
|
1625
|
+
const key = String(args.key || '');
|
|
1626
|
+
const res = await this.computerController.keyPress(key);
|
|
1627
|
+
this.taskStateManager.addObservation('computer_key_press', res.output, res.success);
|
|
1628
|
+
const structuredRes = {
|
|
1629
|
+
success: res.success,
|
|
1630
|
+
action: 'key_press',
|
|
1631
|
+
output: res.output,
|
|
1632
|
+
...(res.error ? { error: res.error } : {}),
|
|
1633
|
+
recoverable: res.recoverable ?? true,
|
|
1634
|
+
};
|
|
1635
|
+
return { result: JSON.stringify(structuredRes, null, 2) };
|
|
1801
1636
|
}
|
|
1802
1637
|
case 'computer_hotkey': {
|
|
1803
|
-
const keys =
|
|
1638
|
+
const keys = Array.isArray(args.keys) ? args.keys.map(String) : [String(args.keys)];
|
|
1804
1639
|
const res = await this.computerController.hotkey(keys);
|
|
1805
|
-
this.taskStateManager.addObservation('computer_hotkey', res.output, res.success
|
|
1640
|
+
this.taskStateManager.addObservation('computer_hotkey', res.output, res.success);
|
|
1641
|
+
const structuredRes = {
|
|
1642
|
+
success: res.success,
|
|
1643
|
+
action: 'hotkey',
|
|
1644
|
+
output: res.output,
|
|
1645
|
+
...(res.error ? { error: res.error } : {}),
|
|
1646
|
+
recoverable: res.recoverable ?? true,
|
|
1647
|
+
...(res.activeWindow ? { activeWindow: res.activeWindow } : {}),
|
|
1648
|
+
};
|
|
1649
|
+
return { result: JSON.stringify(structuredRes, null, 2) };
|
|
1650
|
+
}
|
|
1651
|
+
case 'computer_scroll': {
|
|
1652
|
+
const deltaY = Number(args.deltaY || 0);
|
|
1653
|
+
const deltaX = Number(args.deltaX || 0);
|
|
1654
|
+
const res = await this.computerController.scroll(deltaX, deltaY);
|
|
1655
|
+
this.taskStateManager.addObservation('computer_scroll', res.output, res.success);
|
|
1806
1656
|
return { result: res.output };
|
|
1807
1657
|
}
|
|
1808
|
-
case '
|
|
1809
|
-
const
|
|
1810
|
-
const
|
|
1811
|
-
this.
|
|
1812
|
-
|
|
1813
|
-
this.taskStateManager.addObservation('computer_open_app', res.output, res.success, name);
|
|
1658
|
+
case 'computer_mouse_move': {
|
|
1659
|
+
const x = Number(args.x);
|
|
1660
|
+
const y = Number(args.y);
|
|
1661
|
+
const res = await this.computerController.mouseMove(x, y);
|
|
1662
|
+
this.taskStateManager.addObservation('computer_mouse_move', res.output, res.success);
|
|
1814
1663
|
return { result: res.output };
|
|
1815
1664
|
}
|
|
1816
|
-
case '
|
|
1817
|
-
const
|
|
1818
|
-
this.taskStateManager.
|
|
1819
|
-
const res = await this.computerController.switch_application(name);
|
|
1820
|
-
this.taskStateManager.addObservation('computer_switch_app', res.output, res.success, name);
|
|
1665
|
+
case 'computer_drag': {
|
|
1666
|
+
const res = await this.computerController.drag(Number(args.fromX), Number(args.fromY), Number(args.toX), Number(args.toY));
|
|
1667
|
+
this.taskStateManager.addObservation('computer_drag', res.output, res.success);
|
|
1821
1668
|
return { result: res.output };
|
|
1822
1669
|
}
|
|
1823
|
-
case '
|
|
1824
|
-
const
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1670
|
+
case 'computer_wait': {
|
|
1671
|
+
const ms = Math.min(10000, Math.max(100, Number(args.ms) || 1000));
|
|
1672
|
+
await this.computerController.wait(ms);
|
|
1673
|
+
return { result: `Waited for ${ms}ms` };
|
|
1674
|
+
}
|
|
1675
|
+
case 'computer_open_app': {
|
|
1676
|
+
const appName = String(args.appName || args.app || '');
|
|
1677
|
+
const res = await this.computerController.openApplication(appName, args.args);
|
|
1678
|
+
this.taskStateManager.setActiveApplication(appName);
|
|
1679
|
+
this.taskStateManager.addObservation('computer_open_app', res.output, res.success);
|
|
1680
|
+
const structuredRes = {
|
|
1681
|
+
success: res.success,
|
|
1682
|
+
action: 'open_app',
|
|
1683
|
+
output: res.output,
|
|
1684
|
+
...(res.error ? { error: res.error } : {}),
|
|
1685
|
+
recoverable: res.recoverable ?? true,
|
|
1686
|
+
activeWindow: res.activeWindow || appName,
|
|
1687
|
+
expectedWindow: res.expectedWindow || appName,
|
|
1688
|
+
};
|
|
1689
|
+
return { result: JSON.stringify(structuredRes, null, 2) };
|
|
1690
|
+
}
|
|
1691
|
+
case 'computer_close_app': {
|
|
1692
|
+
const appName = String(args.appName || args.app || '');
|
|
1693
|
+
const res = await this.computerController.closeApplication(appName);
|
|
1694
|
+
this.taskStateManager.addObservation('computer_close_app', res.output, res.success);
|
|
1828
1695
|
return { result: res.output };
|
|
1829
1696
|
}
|
|
1697
|
+
case 'computer_switch_window': {
|
|
1698
|
+
const targetWin = String(args.windowTitleOrApp || args.target || '');
|
|
1699
|
+
const res = await this.computerController.switchWindow(targetWin);
|
|
1700
|
+
if (res.success) {
|
|
1701
|
+
this.taskStateManager.setActiveWindow(targetWin);
|
|
1702
|
+
}
|
|
1703
|
+
this.taskStateManager.addObservation('computer_switch_window', res.output, res.success);
|
|
1704
|
+
const structuredRes = {
|
|
1705
|
+
success: res.success,
|
|
1706
|
+
action: 'switch_window',
|
|
1707
|
+
output: res.output,
|
|
1708
|
+
...(res.error ? { error: res.error } : {}),
|
|
1709
|
+
recoverable: res.recoverable ?? true,
|
|
1710
|
+
activeWindow: res.activeWindow || targetWin,
|
|
1711
|
+
expectedWindow: res.expectedWindow,
|
|
1712
|
+
};
|
|
1713
|
+
return { result: JSON.stringify(structuredRes, null, 2) };
|
|
1714
|
+
}
|
|
1715
|
+
case 'computer_inspect_ui': {
|
|
1716
|
+
const ui = await this.computerController.inspectUI();
|
|
1717
|
+
this.taskStateManager.setActiveWindow(ui.activeWindow);
|
|
1718
|
+
const outStr = `Active Window: "${ui.activeWindow}"\nVisible Windows: ${ui.windows.map((w) => `"${w}"`).join(', ')}`;
|
|
1719
|
+
this.taskStateManager.addObservation('computer_inspect_ui', outStr, true);
|
|
1720
|
+
const structuredRes = {
|
|
1721
|
+
success: true,
|
|
1722
|
+
action: 'inspect_ui',
|
|
1723
|
+
activeWindow: ui.activeWindow,
|
|
1724
|
+
windows: ui.windows,
|
|
1725
|
+
output: outStr,
|
|
1726
|
+
};
|
|
1727
|
+
return { result: JSON.stringify(structuredRes, null, 2) };
|
|
1728
|
+
}
|
|
1830
1729
|
case 'task_completed': {
|
|
1831
1730
|
const summaryStr = String(args.summary || 'Task completed successfully.');
|
|
1731
|
+
if (this.executionIntent?.targetDestination === 'Desktop' && this.executionIntent?.targetArtifactName) {
|
|
1732
|
+
const checkRes = await this.actionVerifier.verifyFileCreated(this.executionIntent.targetArtifactName, 1, 'Desktop');
|
|
1733
|
+
if (!checkRes.verified) {
|
|
1734
|
+
return {
|
|
1735
|
+
result: JSON.stringify({
|
|
1736
|
+
success: false,
|
|
1737
|
+
error: 'VerificationFailed',
|
|
1738
|
+
message: `Cannot conclude task as completed: ${checkRes.message}`,
|
|
1739
|
+
recoverable: true,
|
|
1740
|
+
suggestedAction: 'The file was not found on the Desktop (or was placed in a substitute path). Save the file to the user\'s Desktop via the application interface before completing.'
|
|
1741
|
+
}, null, 2)
|
|
1742
|
+
};
|
|
1743
|
+
}
|
|
1744
|
+
}
|
|
1832
1745
|
speakText(summaryStr, { async: true });
|
|
1833
|
-
cleanupScreenshots();
|
|
1834
1746
|
this.taskStateManager.setStatus('completed');
|
|
1835
1747
|
this.taskStateManager.completeStep(this.currentStep, summaryStr);
|
|
1836
1748
|
return { result: summaryStr };
|