ft-scout 9.4.6 → 9.4.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/.firebase/hosting.d2Vi.cache +8 -8
  2. package/.scoutrules +13 -0
  3. package/AUTONOMOUS_FEATURES.md +15 -10
  4. package/README.md +588 -482
  5. package/bin/src/commands/agent.d.ts.map +1 -1
  6. package/bin/src/commands/agent.js +73 -25
  7. package/bin/src/commands/agent.js.map +1 -1
  8. package/bin/src/commands/authCommand.d.ts.map +1 -1
  9. package/bin/src/commands/authCommand.js +4 -10
  10. package/bin/src/commands/authCommand.js.map +1 -1
  11. package/bin/src/commands/benchmark.d.ts.map +1 -1
  12. package/bin/src/commands/benchmark.js +0 -5
  13. package/bin/src/commands/benchmark.js.map +1 -1
  14. package/bin/src/commands/checkpoint.d.ts.map +1 -0
  15. package/bin/src/commands/checkpoint.js +49 -0
  16. package/bin/src/commands/checkpoint.js.map +1 -0
  17. package/bin/src/commands/init.d.ts.map +1 -1
  18. package/bin/src/commands/init.js +14 -17
  19. package/bin/src/commands/init.js.map +1 -1
  20. package/bin/src/commands/login.d.ts.map +1 -1
  21. package/bin/src/commands/login.js +2 -17
  22. package/bin/src/commands/login.js.map +1 -1
  23. package/bin/src/commands/plan.d.ts.map +1 -1
  24. package/bin/src/commands/plan.js +36 -6
  25. package/bin/src/commands/plan.js.map +1 -1
  26. package/bin/src/commands/pr.d.ts.map +1 -0
  27. package/bin/src/commands/pr.js +191 -0
  28. package/bin/src/commands/pr.js.map +1 -0
  29. package/bin/src/commands/watch.d.ts.map +1 -0
  30. package/bin/src/commands/watch.js +149 -0
  31. package/bin/src/commands/watch.js.map +1 -0
  32. package/bin/src/engine/agentEngine.d.ts.map +1 -1
  33. package/bin/src/engine/agentEngine.js +492 -375
  34. package/bin/src/engine/agentEngine.js.map +1 -1
  35. package/bin/src/engine/checkpoints/checkpointManager.d.ts.map +1 -0
  36. package/bin/src/engine/checkpoints/checkpointManager.js +171 -0
  37. package/bin/src/engine/checkpoints/checkpointManager.js.map +1 -0
  38. package/bin/src/engine/computer/computerController.d.ts.map +1 -1
  39. package/bin/src/engine/computer/computerController.js +49 -0
  40. package/bin/src/engine/computer/computerController.js.map +1 -1
  41. package/bin/src/engine/computer/operatorHud.d.ts.map +1 -0
  42. package/bin/src/engine/computer/operatorHud.js +126 -0
  43. package/bin/src/engine/computer/operatorHud.js.map +1 -0
  44. package/bin/src/engine/llm.d.ts.map +1 -1
  45. package/bin/src/engine/llm.js +21 -51
  46. package/bin/src/engine/llm.js.map +1 -1
  47. package/bin/src/engine/runtime/agentRuntime.d.ts.map +1 -1
  48. package/bin/src/engine/runtime/agentRuntime.js +308 -206
  49. package/bin/src/engine/runtime/agentRuntime.js.map +1 -1
  50. package/bin/src/engine/subagents/subagentManager.d.ts.map +1 -0
  51. package/bin/src/engine/subagents/subagentManager.js +96 -0
  52. package/bin/src/engine/subagents/subagentManager.js.map +1 -0
  53. package/bin/src/engine/toolRegistry.js +1 -1
  54. package/bin/src/engine/toolRegistry.js.map +1 -1
  55. package/bin/src/engine/voiceEngine.d.ts.map +1 -1
  56. package/bin/src/engine/voiceEngine.js +27 -238
  57. package/bin/src/engine/voiceEngine.js.map +1 -1
  58. package/bin/src/engine/workspace/agentWorkspace.d.ts.map +1 -1
  59. package/bin/src/engine/workspace/agentWorkspace.js +22 -9
  60. package/bin/src/engine/workspace/agentWorkspace.js.map +1 -1
  61. package/bin/src/index.js +76 -9
  62. package/bin/src/index.js.map +1 -1
  63. package/bin/src/mcp/client.js +1 -1
  64. package/bin/src/server/server.d.ts.map +1 -1
  65. package/bin/src/server/server.js +37 -3
  66. package/bin/src/server/server.js.map +1 -1
  67. package/bin/src/utils/admin.d.ts.map +1 -1
  68. package/bin/src/utils/admin.js +63 -0
  69. package/bin/src/utils/admin.js.map +1 -1
  70. package/bin/src/utils/auth.d.ts.map +1 -1
  71. package/bin/src/utils/auth.js +10 -5
  72. package/bin/src/utils/auth.js.map +1 -1
  73. package/bin/src/utils/branding.js +1 -1
  74. package/bin/src/utils/claudeUI.d.ts.map +1 -0
  75. package/bin/src/utils/claudeUI.js +404 -0
  76. package/bin/src/utils/claudeUI.js.map +1 -0
  77. package/bin/src/utils/interruptHandler.d.ts.map +1 -0
  78. package/bin/src/utils/interruptHandler.js +121 -0
  79. package/bin/src/utils/interruptHandler.js.map +1 -0
  80. package/bin/src/utils/payment.d.ts.map +1 -1
  81. package/bin/src/utils/payment.js +1 -1
  82. package/bin/src/utils/payment.js.map +1 -1
  83. package/bin/src/utils/pricing.d.ts.map +1 -1
  84. package/bin/src/utils/pricing.js +106 -45
  85. package/bin/src/utils/pricing.js.map +1 -1
  86. package/bin/src/utils/qrBridge.d.ts.map +1 -1
  87. package/bin/src/utils/qrBridge.js +17 -17
  88. package/bin/src/utils/qrBridge.js.map +1 -1
  89. package/bin/src/utils/rules.d.ts.map +1 -0
  90. package/bin/src/utils/rules.js +105 -0
  91. package/bin/src/utils/rules.js.map +1 -0
  92. package/bin/src/utils/updateCheck.d.ts.map +1 -1
  93. package/bin/src/utils/updateCheck.js +12 -1
  94. package/bin/src/utils/updateCheck.js.map +1 -1
  95. package/firebase.json +1 -1
  96. package/package.json +1 -1
  97. package/web/app.js +110 -52
  98. package/web/styles.css +3 -3
@@ -20,6 +20,9 @@ import { robustSnippetReplace, buildDirectoryTree } from '../agentEngine.js';
20
20
  import { loadAuthConfig, isProOrHigher } from '../../utils/auth.js';
21
21
  import { getMCPManager } from '../../mcp/index.js';
22
22
  import { compress, LEVEL_FAST } from 'codrop';
23
+ import { renderStepHeader, renderThoughtBox, renderToolCall, renderToolResult, renderTaskCompletedCard, } from '../../utils/claudeUI.js';
24
+ import { loadWorkspaceRules } from '../../utils/rules.js';
25
+ import { InterruptManager } from '../../utils/interruptHandler.js';
23
26
  export class AgentRuntime {
24
27
  cwd;
25
28
  autoApprove;
@@ -54,10 +57,10 @@ export class AgentRuntime {
54
57
  this.mockMode = Boolean(options?.mockMode);
55
58
  this.language = options?.language;
56
59
  this.userTier = (options?.tier || (options?.mockMode ? 'pro' : loadAuthConfig().currentTier) || 'free').toLowerCase();
57
- // Free tier is bounded to basic direct actions (max 5 steps). Full autonomous multi-step loops require Pro/Enterprise.
58
- const isPaid = this.mockMode || this.userTier === 'pro' || this.userTier === 'enterprise';
59
- const requestedSteps = options?.maxSteps || 30;
60
- this.maxSteps = isPaid ? requestedSteps : Math.min(requestedSteps, 5);
60
+ // Free tier is bounded to basic direct actions (max 5 steps). Full autonomous multi-step loops require Pro/Enterprise (unlimited steps).
61
+ const isPaid = this.mockMode || isProOrHigher(this.userTier);
62
+ const requestedSteps = options?.maxSteps;
63
+ this.maxSteps = isPaid ? (requestedSteps || Infinity) : Math.min(requestedSteps || 5, 5);
61
64
  this.workspace = new AgentWorkspace(options?.taskId, '', this.cwd);
62
65
  this.computerController = options?.computerController || (this.mockMode ? new MockComputerController() : new SystemComputerController(this.cwd));
63
66
  this.browserController = options?.browserController || new SemanticBrowserController(this.cwd);
@@ -136,6 +139,8 @@ export class AgentRuntime {
136
139
  buildSystemPrompt() {
137
140
  const filesList = getDirectoryFiles(this.cwd).slice(0, 100);
138
141
  const workspaceState = this.workspace.compactWorkspaceForPrompt();
142
+ const workspaceRules = loadWorkspaceRules(this.cwd);
143
+ const rulesSection = workspaceRules.found ? `\n${workspaceRules.formattedPrompt}\n` : '';
139
144
  const tierNotice = !this.isProOrEnterprise()
140
145
  ? `\nTIER RESTRICTION (FREE DEVELOPER TIER):
141
146
  - You are operating under the Free Developer Tier.
@@ -156,7 +161,7 @@ Current Year: ${new Date().getFullYear()}
156
161
  ${tierNotice}
157
162
 
158
163
  ${workspaceState}
159
-
164
+ ${rulesSection}
160
165
  Available Workspace Files (sample):
161
166
  ${filesList.join('\n')}
162
167
 
@@ -227,246 +232,333 @@ Action Rules:
227
232
  this.historyMessages.push({ role: 'user', content: userGoal });
228
233
  let step = 0;
229
234
  let finalSummary = '';
230
- while (step < this.maxSteps) {
231
- if (this.isCancelled) {
232
- return {
233
- success: false,
234
- summary: `Task cancelled: ${this.cancelReason || 'User abort'}`,
235
- taskId,
236
- stepsTaken: step,
237
- modifiedFiles: Array.from(this.modifiedFiles),
238
- status: 'cancelled',
239
- };
240
- }
241
- step++;
242
- this.currentStep = step;
243
- this.workspace.startStep(step);
244
- // Context Compaction check: If history is long, compact middle turns using workspace state
245
- if (this.historyMessages.length > 10) {
246
- this.historyMessages = this.workspace.compactHistoryMessages(this.historyMessages, 6);
247
- }
248
- const stepSpinner = spinner();
249
- stepSpinner.start(chalk.cyan(`Scout Operating (Step ${step}/${this.maxSteps})`));
250
- try {
251
- let response;
252
- const mcpRouting = getMCPManager().routeTools(userGoal);
253
- const mcpToolDefs = mcpRouting.tools.map((mt) => {
254
- let desc = mt.description;
255
- if (mt.originalName === 'execute_blender_code') {
256
- desc = 'Execute Python (bpy) code inside Blender. Use this to procedurally create, model, texture, color, and render 3D meshes, anime/game characters, clothing, hair, materials, cel-shading, lights, cameras, and scenes directly.';
257
- }
258
- else if (mt.originalName === 'generate_3d') {
259
- desc = 'Cloud AI text-to-3d generator (requires external API key). If keys are not set, use execute_blender_code to model directly.';
235
+ const interruptManager = new InterruptManager({
236
+ getStepInfo: () => ({
237
+ step: this.currentStep || 1,
238
+ maxSteps: this.maxSteps,
239
+ goal: userGoal,
240
+ modifiedFilesCount: this.modifiedFiles.size,
241
+ }),
242
+ onPause: () => {
243
+ this.workspace.setStatus('paused');
244
+ },
245
+ onResume: () => {
246
+ this.workspace.setStatus('running');
247
+ },
248
+ onFeedback: (feedback) => {
249
+ this.historyMessages.push({
250
+ role: 'user',
251
+ content: `[USER STEERING FEEDBACK (INTERRUPTED AT STEP ${this.currentStep})]:\n"${feedback}"\n\nPlease adapt your execution plan accordingly and follow these user steering instructions immediately.`,
252
+ });
253
+ },
254
+ onRevert: () => {
255
+ if (this.backups.length === 0)
256
+ return false;
257
+ for (const b of this.backups) {
258
+ try {
259
+ fs.writeFileSync(b.filePath, b.originalContent, 'utf-8');
260
260
  }
261
+ catch { }
262
+ }
263
+ this.modifiedFiles.clear();
264
+ return true;
265
+ },
266
+ onAbort: (reason) => {
267
+ this.isCancelled = true;
268
+ this.cancelReason = reason;
269
+ },
270
+ });
271
+ interruptManager.attach();
272
+ try {
273
+ while (step < this.maxSteps) {
274
+ if (this.isCancelled) {
261
275
  return {
262
- name: mt.originalName,
263
- description: `[MCP: ${mt.serverName}] ${desc.length > 250 ? desc.slice(0, 247) + '...' : desc}`,
264
- parameters: {
265
- type: 'object',
266
- properties: mt.inputSchema.properties || {},
267
- required: mt.inputSchema.required || [],
268
- },
269
- source: 'mcp',
270
- mcpServerId: mt.serverId,
271
- mcpServerName: mt.serverName,
276
+ success: false,
277
+ summary: `Task cancelled: ${this.cancelReason || 'User abort'}`,
278
+ taskId,
279
+ stepsTaken: step,
280
+ modifiedFiles: Array.from(this.modifiedFiles),
281
+ status: 'cancelled',
272
282
  };
273
- });
274
- const goalLower = (userGoal || '').toLowerCase();
275
- const needsBrowser = goalLower.includes('browser') || goalLower.includes('website') || goalLower.includes('webpage') || goalLower.includes('http') || goalLower.includes('url') || goalLower.includes('scrape') || goalLower.includes('chrome');
276
- const needsComputer = goalLower.includes('computer') || goalLower.includes('desktop') || goalLower.includes('mouse') || goalLower.includes('coordinate') || goalLower.includes('drag') || goalLower.includes('double click');
277
- let baseTools = this.toolRegistry.getToolsForTier(this.userTier);
278
- if (mcpToolDefs.length > 0 && !needsBrowser && !needsComputer) {
279
- const ESSENTIAL_BASE_TOOLS = new Set(['read_file', 'write_file', 'edit_file', 'run_command', 'list_dir', 'grep_search', 'task_completed', 'ask_user', 'speak_text', 'scratchpad', 'computer_open_app']);
280
- baseTools = baseTools.filter((t) => ESSENTIAL_BASE_TOOLS.has(t.name));
281
283
  }
282
- else if (!needsBrowser) {
283
- baseTools = baseTools.filter((t) => !t.name.startsWith('browser_'));
284
+ step++;
285
+ this.currentStep = step;
286
+ this.workspace.startStep(step);
287
+ // Context Compaction check: If history is long, compact middle turns using workspace state
288
+ if (this.historyMessages.length > 10) {
289
+ this.historyMessages = this.workspace.compactHistoryMessages(this.historyMessages, 6);
284
290
  }
285
- const tierTools = [...baseTools, ...mcpToolDefs];
291
+ const isPaid = this.isProOrEnterprise();
292
+ renderStepHeader({
293
+ step,
294
+ maxSteps: this.maxSteps,
295
+ startTimeMs: this.workspace.getData().createdAt ? new Date(this.workspace.getData().createdAt).getTime() : Date.now(),
296
+ tokensConsumed: 0,
297
+ userTier: this.userTier,
298
+ });
299
+ const stepSpinner = spinner();
300
+ const stepText = isPaid && !Number.isFinite(this.maxSteps)
301
+ ? `Scout Operating (Step ${step})`
302
+ : `Scout Operating (Step ${step}/${this.maxSteps})`;
303
+ stepSpinner.start(chalk.cyan(stepText));
286
304
  try {
287
- response = await callOpenAIWithRetry(async (model) => {
288
- return await openai.chat.completions.create({
289
- model,
290
- messages: sanitizeMessages(this.historyMessages),
291
- tools: tierTools.map((t) => ({ type: 'function', function: t })),
292
- tool_choice: 'auto',
293
- temperature: 0.1,
294
- });
295
- });
296
- }
297
- catch (err) {
298
- const errStr = String(err?.message || err || '').toLowerCase();
299
- if (errStr.includes('tool') || errStr.includes('400') || errStr.includes('not supported')) {
300
- const fallbackMsg = {
301
- role: 'user',
302
- content: `Format your tool call as a JSON code block:\n\`\`\`json\n{ "tool": "tool_name", "args": { ... } }\n\`\`\`\nAvailable tools: ${tierTools.map((t) => t.name).join(', ')}`,
305
+ let response;
306
+ const mcpRouting = getMCPManager().routeTools(userGoal);
307
+ const mcpToolDefs = mcpRouting.tools.map((mt) => {
308
+ let desc = mt.description;
309
+ if (mt.originalName === 'execute_blender_code') {
310
+ desc = 'Execute Python (bpy) code inside Blender. Use this to procedurally create, model, texture, color, and render 3D meshes, anime/game characters, clothing, hair, materials, cel-shading, lights, cameras, and scenes directly.';
311
+ }
312
+ else if (mt.originalName === 'generate_3d') {
313
+ desc = 'Cloud AI text-to-3d generator (requires external API key). If keys are not set, use execute_blender_code to model directly.';
314
+ }
315
+ return {
316
+ name: mt.originalName,
317
+ description: `[MCP: ${mt.serverName}] ${desc.length > 250 ? desc.slice(0, 247) + '...' : desc}`,
318
+ parameters: {
319
+ type: 'object',
320
+ properties: mt.inputSchema.properties || {},
321
+ required: mt.inputSchema.required || [],
322
+ },
323
+ source: 'mcp',
324
+ mcpServerId: mt.serverId,
325
+ mcpServerName: mt.serverName,
303
326
  };
327
+ });
328
+ const goalLower = (userGoal || '').toLowerCase();
329
+ const needsBrowser = goalLower.includes('browser') || goalLower.includes('website') || goalLower.includes('webpage') || goalLower.includes('http') || goalLower.includes('url') || goalLower.includes('scrape') || goalLower.includes('chrome');
330
+ const needsComputer = goalLower.includes('computer') || goalLower.includes('desktop') || goalLower.includes('mouse') || goalLower.includes('coordinate') || goalLower.includes('drag') || goalLower.includes('double click');
331
+ let baseTools = this.toolRegistry.getToolsForTier(this.userTier);
332
+ if (mcpToolDefs.length > 0 && !needsBrowser && !needsComputer) {
333
+ const ESSENTIAL_BASE_TOOLS = new Set(['read_file', 'write_file', 'edit_file', 'run_command', 'list_dir', 'grep_search', 'task_completed', 'ask_user', 'speak_text', 'scratchpad', 'computer_open_app']);
334
+ baseTools = baseTools.filter((t) => ESSENTIAL_BASE_TOOLS.has(t.name));
335
+ }
336
+ else if (!needsBrowser) {
337
+ baseTools = baseTools.filter((t) => !t.name.startsWith('browser_'));
338
+ }
339
+ const tierTools = [...baseTools, ...mcpToolDefs];
340
+ try {
304
341
  response = await callOpenAIWithRetry(async (model) => {
305
342
  return await openai.chat.completions.create({
306
343
  model,
307
- messages: sanitizeMessages([...this.historyMessages, fallbackMsg]),
344
+ messages: sanitizeMessages(this.historyMessages),
345
+ tools: tierTools.map((t) => ({ type: 'function', function: t })),
346
+ tool_choice: 'auto',
308
347
  temperature: 0.1,
309
348
  });
310
349
  });
311
350
  }
312
- else {
313
- throw err;
314
- }
315
- }
316
- const choice = response.choices[0];
317
- if (!choice) {
318
- stepSpinner.stop(chalk.yellow('No response from model. Retrying step...'));
319
- continue;
320
- }
321
- const msg = sanitizeMessage(choice.message);
322
- this.historyMessages.push(msg);
323
- if (msg.content && msg.content.trim()) {
324
- safeNote(renderMarkdown(msg.content), `Scout Step ${step}`);
325
- }
326
- let toolCallsToExecute = [];
327
- if (msg.tool_calls && msg.tool_calls.length > 0) {
328
- for (const tc of msg.tool_calls) {
329
- if (tc.type === 'function' && tc.function) {
330
- try {
331
- toolCallsToExecute.push({
332
- name: tc.function.name,
333
- args: JSON.parse(tc.function.arguments || '{}'),
334
- id: tc.id,
351
+ catch (err) {
352
+ const errStr = String(err?.message || err || '').toLowerCase();
353
+ if (errStr.includes('tool') || errStr.includes('400') || errStr.includes('not supported')) {
354
+ const fallbackMsg = {
355
+ role: 'user',
356
+ content: `Format your tool call as a JSON code block:\n\`\`\`json\n{ "tool": "tool_name", "args": { ... } }\n\`\`\`\nAvailable tools: ${tierTools.map((t) => t.name).join(', ')}`,
357
+ };
358
+ response = await callOpenAIWithRetry(async (model) => {
359
+ return await openai.chat.completions.create({
360
+ model,
361
+ messages: sanitizeMessages([...this.historyMessages, fallbackMsg]),
362
+ temperature: 0.1,
335
363
  });
336
- }
337
- catch {
338
- toolCallsToExecute.push({ name: tc.function.name, args: {}, id: tc.id });
339
- }
364
+ });
365
+ }
366
+ else {
367
+ throw err;
340
368
  }
341
369
  }
342
- }
343
- else if (msg.content) {
344
- const parsed = this.extractJsonToolCall(msg.content);
345
- if (parsed) {
346
- toolCallsToExecute.push({ name: parsed.tool, args: parsed.args });
370
+ const choice = response.choices[0];
371
+ if (!choice) {
372
+ stepSpinner.stop(chalk.yellow('No response from model. Retrying step...'));
373
+ continue;
347
374
  }
348
- }
349
- stepSpinner.stop(chalk.green(`Step ${step}: Executing ${toolCallsToExecute.length} action(s).`));
350
- for (const tc of toolCallsToExecute) {
351
- if (this.isCancelled)
352
- break;
353
- const toolResult = await this.dispatchTool(tc.name, tc.args);
354
- // Build the tool result message with correct role
355
- const rawContent = typeof toolResult.result === 'string' ? toolResult.result : JSON.stringify(toolResult.result);
356
- let cappedContent = rawContent;
357
- if (rawContent.length > 800) {
358
- try {
359
- const origBytes = Buffer.byteLength(rawContent, 'utf-8');
360
- const cdp = compress(Buffer.from(rawContent, 'utf-8'), LEVEL_FAST);
361
- const ratio = (origBytes / Math.max(1, cdp.length)).toFixed(1);
362
- cappedContent = rawContent.slice(0, 800) + `\n... [Codrop: ${origBytes}B compressed to ${cdp.length}B (${ratio}x ratio) for LLM context]`;
363
- }
364
- catch {
365
- cappedContent = rawContent.slice(0, 800) + '\n... [output truncated for token efficiency]';
366
- }
375
+ const msg = sanitizeMessage(choice.message);
376
+ this.historyMessages.push(msg);
377
+ if (msg.content && msg.content.trim()) {
378
+ renderThoughtBox(msg.content, step);
367
379
  }
368
- if (tc.id) {
369
- // Native tool call: use 'tool' role with matching tool_call_id
370
- this.historyMessages.push({
371
- role: 'tool',
372
- tool_call_id: tc.id,
373
- content: cappedContent,
374
- });
380
+ let toolCallsToExecute = [];
381
+ if (msg.tool_calls && msg.tool_calls.length > 0) {
382
+ for (const tc of msg.tool_calls) {
383
+ if (tc.type === 'function' && tc.function) {
384
+ try {
385
+ toolCallsToExecute.push({
386
+ name: tc.function.name,
387
+ args: JSON.parse(tc.function.arguments || '{}'),
388
+ id: tc.id,
389
+ });
390
+ }
391
+ catch {
392
+ toolCallsToExecute.push({ name: tc.function.name, args: {}, id: tc.id });
393
+ }
394
+ }
395
+ }
375
396
  }
376
- else {
377
- // JSON-extracted tool call (no tc.id): use 'user' role
378
- this.historyMessages.push({
379
- role: 'user',
380
- content: `Tool Execution Result (${tc.name}):\n${cappedContent}`,
381
- });
397
+ else if (msg.content) {
398
+ const parsed = this.extractJsonToolCall(msg.content);
399
+ if (parsed) {
400
+ toolCallsToExecute.push({ name: parsed.tool, args: parsed.args });
401
+ }
382
402
  }
383
- // If the tool returned screenshot data, append a vision message so the model can see the screen
384
- if (toolResult.screenshotBase64) {
385
- this.pruneOldScreenshots();
386
- this.historyMessages.push({
387
- role: 'user',
388
- content: [
389
- { type: 'text', text: `[Screenshot after ${tc.name}]: ${toolResult.screenshotSummary || 'Current screen state.'}` },
390
- { type: 'image_url', image_url: { url: `data:image/png;base64,${toolResult.screenshotBase64}`, detail: 'low' } },
391
- ],
403
+ stepSpinner.stop(chalk.green(`Step ${step}: Executing ${toolCallsToExecute.length} action(s).`));
404
+ for (const tc of toolCallsToExecute) {
405
+ if (this.isCancelled)
406
+ break;
407
+ renderToolCall({ name: tc.name, args: tc.args, step });
408
+ const toolStartTime = Date.now();
409
+ const toolResult = await this.dispatchTool(tc.name, tc.args);
410
+ const durationMs = Date.now() - toolStartTime;
411
+ renderToolResult({
412
+ name: tc.name,
413
+ args: tc.args,
414
+ result: typeof toolResult.result === 'string' ? toolResult.result : JSON.stringify(toolResult.result),
415
+ durationMs,
416
+ success: !String(toolResult.result).includes('[PLAN RESTRICTION') && !String(toolResult.result).includes('[ACTION BLOCKED'),
392
417
  });
393
- }
394
- // Automatic immediate completion for simple app actions (open/close/switch window)
395
- const simpleAppCheck = checkSimpleAppAction(userGoal);
396
- if (simpleAppCheck.isSimple && (tc.name === 'computer_open_app' || tc.name === 'computer_close_app' || tc.name === 'computer_switch_window')) {
397
- let callSucceeded = false;
398
- try {
399
- const parsed = typeof toolResult.result === 'string' ? JSON.parse(toolResult.result) : toolResult.result;
400
- callSucceeded = parsed?.success !== false;
418
+ // Build the tool result message with correct role
419
+ const rawContent = typeof toolResult.result === 'string' ? toolResult.result : JSON.stringify(toolResult.result);
420
+ let cappedContent = rawContent;
421
+ if (rawContent.length > 800) {
422
+ try {
423
+ const origBytes = Buffer.byteLength(rawContent, 'utf-8');
424
+ const cdp = compress(Buffer.from(rawContent, 'utf-8'), LEVEL_FAST);
425
+ const ratio = (origBytes / Math.max(1, cdp.length)).toFixed(1);
426
+ cappedContent = rawContent.slice(0, 800) + `\n... [Codrop: ${origBytes}B compressed to ${cdp.length}B (${ratio}x ratio) for LLM context]`;
427
+ }
428
+ catch {
429
+ cappedContent = rawContent.slice(0, 800) + '\n... [output truncated for token efficiency]';
430
+ }
431
+ }
432
+ if (tc.id) {
433
+ // Native tool call: use 'tool' role with matching tool_call_id
434
+ this.historyMessages.push({
435
+ role: 'tool',
436
+ tool_call_id: tc.id,
437
+ content: cappedContent,
438
+ });
401
439
  }
402
- catch {
403
- const str = String(toolResult.result || '').toLowerCase();
404
- callSucceeded = !str.includes('failed') && !str.includes('error');
440
+ else {
441
+ // JSON-extracted tool call (no tc.id): use 'user' role
442
+ this.historyMessages.push({
443
+ role: 'user',
444
+ content: `Tool Execution Result (${tc.name}):\n${cappedContent}`,
445
+ });
446
+ }
447
+ // If the tool returned screenshot data, append a vision message so the model can see the screen
448
+ if (toolResult.screenshotBase64) {
449
+ this.pruneOldScreenshots();
450
+ this.historyMessages.push({
451
+ role: 'user',
452
+ content: [
453
+ { type: 'text', text: `[Screenshot after ${tc.name}]: ${toolResult.screenshotSummary || 'Current screen state.'}` },
454
+ { type: 'image_url', image_url: { url: `data:image/png;base64,${toolResult.screenshotBase64}`, detail: 'low' } },
455
+ ],
456
+ });
405
457
  }
406
- if (callSucceeded) {
407
- const appName = simpleAppCheck.appName || 'application';
408
- const actionVerb = simpleAppCheck.action === 'close' ? 'closed' : (simpleAppCheck.action === 'switch' ? 'focused' : 'launched');
409
- finalSummary = `I've ${actionVerb} ${appName} for you. It's ready on your screen!`;
410
- safeNote(chalk.green(` ${finalSummary}`), 'Task Completed');
458
+ // Automatic immediate completion for simple app actions (open/close/switch window)
459
+ const simpleAppCheck = checkSimpleAppAction(userGoal);
460
+ if (simpleAppCheck.isSimple && (tc.name === 'computer_open_app' || tc.name === 'computer_close_app' || tc.name === 'computer_switch_window')) {
461
+ let callSucceeded = false;
462
+ try {
463
+ const parsed = typeof toolResult.result === 'string' ? JSON.parse(toolResult.result) : toolResult.result;
464
+ callSucceeded = parsed?.success !== false;
465
+ }
466
+ catch {
467
+ const str = String(toolResult.result || '').toLowerCase();
468
+ callSucceeded = !str.includes('failed') && !str.includes('error');
469
+ }
470
+ if (callSucceeded) {
471
+ const appName = simpleAppCheck.appName || 'application';
472
+ const actionVerb = simpleAppCheck.action === 'close' ? 'closed' : (simpleAppCheck.action === 'switch' ? 'focused' : 'launched');
473
+ finalSummary = `I've ${actionVerb} ${appName} for you. It's ready on your screen!`;
474
+ renderTaskCompletedCard({
475
+ objective: userGoal,
476
+ summary: finalSummary,
477
+ modifiedFiles: [],
478
+ steps: step,
479
+ maxSteps: this.maxSteps,
480
+ totalTokens: 0,
481
+ totalToolCalls: step,
482
+ elapsedSecs: Math.round((Date.now() - (this.workspace.getData().createdAt ? new Date(this.workspace.getData().createdAt).getTime() : Date.now())) / 1000),
483
+ userTier: this.userTier,
484
+ });
485
+ this.workspace.setStatus('completed');
486
+ this.workspace.completeStep(step, finalSummary);
487
+ saveAgentSession({
488
+ goal: userGoal,
489
+ summary: finalSummary,
490
+ modifiedFiles: [],
491
+ });
492
+ return {
493
+ success: true,
494
+ summary: finalSummary,
495
+ taskId,
496
+ stepsTaken: step,
497
+ modifiedFiles: [],
498
+ status: 'completed',
499
+ };
500
+ }
501
+ }
502
+ if (tc.name === 'task_completed') {
503
+ finalSummary = tc.args.summary || toolResult.result;
504
+ if (this.modifiedFiles.size > 0) {
505
+ const filesList = getDirectoryFiles(this.cwd);
506
+ await verifyAndSelfHealFiles(Array.from(this.modifiedFiles), this.cwd, path.basename(this.cwd), filesList, { maxRetries: 3 });
507
+ }
508
+ renderTaskCompletedCard({
509
+ objective: userGoal,
510
+ summary: finalSummary,
511
+ modifiedFiles: Array.from(this.modifiedFiles),
512
+ steps: step,
513
+ maxSteps: this.maxSteps,
514
+ totalTokens: 0,
515
+ totalToolCalls: step,
516
+ elapsedSecs: Math.round((Date.now() - (this.workspace.getData().createdAt ? new Date(this.workspace.getData().createdAt).getTime() : Date.now())) / 1000),
517
+ userTier: this.userTier,
518
+ });
411
519
  this.workspace.setStatus('completed');
412
520
  this.workspace.completeStep(step, finalSummary);
413
521
  saveAgentSession({
414
522
  goal: userGoal,
415
523
  summary: finalSummary,
416
- modifiedFiles: [],
524
+ modifiedFiles: Array.from(this.modifiedFiles),
417
525
  });
418
526
  return {
419
527
  success: true,
420
528
  summary: finalSummary,
421
529
  taskId,
422
530
  stepsTaken: step,
423
- modifiedFiles: [],
531
+ modifiedFiles: Array.from(this.modifiedFiles),
424
532
  status: 'completed',
425
533
  };
426
534
  }
427
535
  }
428
- if (tc.name === 'task_completed') {
429
- finalSummary = tc.args.summary || toolResult.result;
430
- if (this.modifiedFiles.size > 0) {
431
- const filesList = getDirectoryFiles(this.cwd);
432
- await verifyAndSelfHealFiles(Array.from(this.modifiedFiles), this.cwd, path.basename(this.cwd), filesList, { maxRetries: 3 });
433
- }
434
- this.workspace.setStatus('completed');
435
- this.workspace.completeStep(step, finalSummary);
436
- saveAgentSession({
437
- goal: userGoal,
438
- summary: finalSummary,
439
- modifiedFiles: Array.from(this.modifiedFiles),
440
- });
441
- return {
442
- success: true,
443
- summary: finalSummary,
444
- taskId,
445
- stepsTaken: step,
446
- modifiedFiles: Array.from(this.modifiedFiles),
447
- status: 'completed',
448
- };
449
- }
536
+ }
537
+ catch (err) {
538
+ stepSpinner.stop(chalk.red(`Step ${step} error: ${err?.message || String(err)}`));
539
+ this.workspace.recordError(err?.message || String(err), 'runtime', undefined, false);
540
+ this.historyMessages.push({
541
+ role: 'user',
542
+ content: `System Error encountered on step ${step}: ${err?.message || String(err)}. Diagnose and adapt action.`,
543
+ });
450
544
  }
451
545
  }
452
- catch (err) {
453
- stepSpinner.stop(chalk.red(`Step ${step} error: ${err?.message || String(err)}`));
454
- this.workspace.recordError(err?.message || String(err), 'runtime', undefined, false);
455
- this.historyMessages.push({
456
- role: 'user',
457
- content: `System Error encountered on step ${step}: ${err?.message || String(err)}. Diagnose and adapt action.`,
458
- });
459
- }
546
+ this.workspace.setStatus('failed');
547
+ const isPaid = this.isProOrEnterprise();
548
+ return {
549
+ success: false,
550
+ summary: isPaid
551
+ ? `Task ended after ${step} iteration steps. Modified files: ${Array.from(this.modifiedFiles).join(', ')}`
552
+ : `Reached max iteration steps limit (${this.maxSteps}). Modified files: ${Array.from(this.modifiedFiles).join(', ')}`,
553
+ taskId,
554
+ stepsTaken: step,
555
+ modifiedFiles: Array.from(this.modifiedFiles),
556
+ status: 'failed',
557
+ };
558
+ }
559
+ finally {
560
+ interruptManager.detach();
460
561
  }
461
- this.workspace.setStatus('failed');
462
- return {
463
- success: false,
464
- summary: `Reached max iteration steps limit (${this.maxSteps}). Modified files: ${Array.from(this.modifiedFiles).join(', ')}`,
465
- taskId,
466
- stepsTaken: step,
467
- modifiedFiles: Array.from(this.modifiedFiles),
468
- status: 'failed',
469
- };
470
562
  }
471
563
  async executeTool(name, args) {
472
564
  return await this.dispatchTool(name, args);
@@ -601,6 +693,9 @@ Action Rules:
601
693
  else if (action === 'diagnostic') {
602
694
  name = 'computer_diagnostic';
603
695
  }
696
+ else if (action === 'wait') {
697
+ name = 'computer_wait';
698
+ }
604
699
  }
605
700
  switch (name) {
606
701
  case 'computer_screenshot': {
@@ -872,6 +967,15 @@ Action Rules:
872
967
  const diag = await this.computerController.runDiagnostic();
873
968
  return { result: JSON.stringify(diag, null, 2) };
874
969
  }
970
+ case 'computer': {
971
+ return {
972
+ result: JSON.stringify({
973
+ success: false,
974
+ error: 'InvalidComputerAction',
975
+ message: `Unknown computer action: "${args?.action}". Supported actions: click, right_click, double_click, move, mouse_down, mouse_up, drag, scroll, screenshot, type, key_press, hotkey, open_app, close_app, switch_window, inspect_ui, diagnostic, wait.`
976
+ }, null, 2)
977
+ };
978
+ }
875
979
  default:
876
980
  return { result: `Unknown computer tool: "${name}"` };
877
981
  }
@@ -1227,8 +1331,7 @@ Action Rules:
1227
1331
  }
1228
1332
  case 'speak_text': {
1229
1333
  const txt = String(args.text || '');
1230
- speakText(txt, { async: true });
1231
- return { result: `Spoke: "${txt}"` };
1334
+ return { result: `[TTS DISABLED]: Text-to-speech audio synthesis has been disabled. Text displayed: "${txt}"` };
1232
1335
  }
1233
1336
  case 'task_completed': {
1234
1337
  const summary = String(args.summary || 'Completed');
@@ -1247,7 +1350,6 @@ Action Rules:
1247
1350
  };
1248
1351
  }
1249
1352
  }
1250
- speakText(summary, { async: true });
1251
1353
  return { result: summary };
1252
1354
  }
1253
1355
  default: {