@timetotest/cli 0.3.0 → 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/README.md +12 -2
  2. package/dist/bin/ttt.js +0 -6
  3. package/dist/bin/ttt.js.map +1 -1
  4. package/dist/package.json +2 -1
  5. package/dist/src/commands/chat/ChatApp.js +102 -61
  6. package/dist/src/commands/chat/ChatApp.js.map +1 -1
  7. package/dist/src/commands/chat/OnboardingApp.js +49 -0
  8. package/dist/src/commands/chat/OnboardingApp.js.map +1 -0
  9. package/dist/src/commands/chat/components/Banner.js +1 -1
  10. package/dist/src/commands/chat/components/Banner.js.map +1 -1
  11. package/dist/src/commands/chat/components/ChatInput.js +16 -3
  12. package/dist/src/commands/chat/components/ChatInput.js.map +1 -1
  13. package/dist/src/commands/chat/components/TodoPanel.js +71 -0
  14. package/dist/src/commands/chat/components/TodoPanel.js.map +1 -0
  15. package/dist/src/commands/chat-ink.js +116 -292
  16. package/dist/src/commands/chat-ink.js.map +1 -1
  17. package/dist/src/lib/__tests__/code-mode-integration.test.js +45 -356
  18. package/dist/src/lib/__tests__/code-mode-integration.test.js.map +1 -1
  19. package/dist/src/lib/__tests__/tool-executor-mode-gating.test.js +46 -0
  20. package/dist/src/lib/__tests__/tool-executor-mode-gating.test.js.map +1 -0
  21. package/dist/src/lib/__tests__/ui-browser-integration.test.js +35 -0
  22. package/dist/src/lib/__tests__/ui-browser-integration.test.js.map +1 -0
  23. package/dist/src/lib/agent-orchestrator.js +16 -717
  24. package/dist/src/lib/agent-orchestrator.js.map +1 -1
  25. package/dist/src/lib/backend-loop-client.js +364 -0
  26. package/dist/src/lib/backend-loop-client.js.map +1 -0
  27. package/dist/src/lib/cli-tool-manifest.js +71 -0
  28. package/dist/src/lib/cli-tool-manifest.js.map +1 -0
  29. package/dist/src/lib/config.js +60 -9
  30. package/dist/src/lib/config.js.map +1 -1
  31. package/dist/src/lib/conversation/turns.js +58 -0
  32. package/dist/src/lib/conversation/turns.js.map +1 -0
  33. package/dist/src/lib/events.js +20 -4
  34. package/dist/src/lib/events.js.map +1 -1
  35. package/dist/src/lib/http.js +7 -2
  36. package/dist/src/lib/http.js.map +1 -1
  37. package/dist/src/lib/prompts/templates.js +18 -0
  38. package/dist/src/lib/prompts/templates.js.map +1 -1
  39. package/dist/src/lib/session-manager.js +74 -33
  40. package/dist/src/lib/session-manager.js.map +1 -1
  41. package/dist/src/lib/socket.js +15 -3
  42. package/dist/src/lib/socket.js.map +1 -1
  43. package/dist/src/lib/todo.js +7 -0
  44. package/dist/src/lib/todo.js.map +1 -0
  45. package/dist/src/lib/tool-executor.js +196 -51
  46. package/dist/src/lib/tool-executor.js.map +1 -1
  47. package/dist/src/lib/tui/events.js +10 -9
  48. package/dist/src/lib/tui/events.js.map +1 -1
  49. package/dist/src/lib/utils/json.js +15 -0
  50. package/dist/src/lib/utils/json.js.map +1 -0
  51. package/package.json +2 -1
  52. package/dist/src/commands/report.js +0 -25
  53. package/dist/src/commands/report.js.map +0 -1
  54. package/dist/src/commands/restart.js +0 -17
  55. package/dist/src/commands/restart.js.map +0 -1
  56. package/dist/src/commands/share.js +0 -18
  57. package/dist/src/commands/share.js.map +0 -1
  58. package/dist/src/lib/context-compactor.js +0 -310
  59. package/dist/src/lib/context-compactor.js.map +0 -1
  60. package/dist/src/lib/tool-registry.js +0 -971
  61. package/dist/src/lib/tool-registry.js.map +0 -1
  62. package/dist/src/lib/tool-result-pruner.js +0 -384
  63. package/dist/src/lib/tool-result-pruner.js.map +0 -1
@@ -1,728 +1,27 @@
1
1
  /**
2
- * Agent Orchestrator - Main chat loop and backend integration
3
- * Manages conversation, tool calling, and execution
2
+ * AgentOrchestrator (CLI)
3
+ *
4
+ * The canonical agent loop now runs in the backend. This class is a thin client
5
+ * that:
6
+ * - starts/resumes backend sessions
7
+ * - registers CLI local tools (filesystem, browser MCP, command exec) via Socket.IO
8
+ * - executes backend-requested tool calls locally and returns results
9
+ *
10
+ * This intentionally replaces the previous TS-based ReAct loop to reduce
11
+ * duplicated orchestration code.
4
12
  */
5
- import { EventEmitter } from "events";
6
- import { SessionManager } from "./session-manager.js";
7
- import { ToolExecutor } from "./tool-executor.js";
8
- import { getToolSchemasByDomain } from "./tool-registry.js";
9
- import { isTestingMode } from "./testing-mode.js";
10
- import { configManager } from "./config.js";
11
- import { toolResultPruner } from "./tool-result-pruner.js";
12
- import { ContextCompactor } from "./context-compactor.js";
13
- import { createAgentHttpClient, } from "./http.js";
14
- import { build_cli_system_prompt } from "./prompts/builder.js";
15
- export class AgentOrchestrator extends EventEmitter {
16
- sessionManager;
17
- toolExecutor;
18
- config;
19
- conversationId;
20
- httpClient;
21
- isCancelled = false;
22
- currentModel;
23
- recentToolCalls = [];
24
- static MAX_REPEATED_CALLS = 3;
25
- contextCompactor;
26
- // Reasoning chaining state (matches backend implementation)
27
- lastResponseId = null;
28
- pendingFunctionOutputs = null;
29
- geminiThoughtSignatures = []; // Gemini-specific reasoning chaining
13
+ import { BackendLoopClient } from "./backend-loop-client.js";
14
+ export class AgentOrchestrator extends BackendLoopClient {
30
15
  constructor(config, sessionId) {
31
- super(); // Initialize EventEmitter
32
- this.config = config;
33
- this.sessionManager = new SessionManager(sessionId);
34
- this.conversationId = this.sessionManager.getSessionId();
35
- this.httpClient = createAgentHttpClient({
36
- baseUrl: config.apiUrl,
16
+ const backendCfg = {
17
+ apiUrl: config.apiUrl,
37
18
  token: config.token,
38
- });
39
- this.contextCompactor = new ContextCompactor();
40
- // Initialize execution context
41
- const executionContext = {
42
- sessionManager: this.sessionManager,
43
- testingMode: config.testingMode,
44
- apiContext: {},
45
- browserContext: {},
46
- };
47
- this.toolExecutor = new ToolExecutor(executionContext);
48
- // Update session config
49
- this.sessionManager.updateConfig({
50
- mode: config.mode,
51
19
  testingMode: config.testingMode,
52
20
  baseUrl: config.baseUrl,
53
21
  apiBaseUrl: config.apiBaseUrl,
54
- });
55
- }
56
- async chat(userMessage) {
57
- this.isCancelled = false; // Reset cancellation flag
58
- this.recentToolCalls = []; // Reset tool call tracking for new conversation turn
59
- // Note: We do NOT reset lastResponseId or pendingFunctionOutputs here
60
- // They persist across the conversation to enable reasoning chaining
61
- // Add user message to history
62
- this.sessionManager.addMessage({
63
- role: "user",
64
- content: userMessage,
65
- });
66
- const messages = this.sessionManager.getMessages();
67
- const toolSchemas = this.getAvailableToolSchemas();
68
- try {
69
- if (this.config.mode === "local") {
70
- return await this.handleLocalMode(messages, toolSchemas);
71
- }
72
- else {
73
- throw new Error("Remote mode is no longer supported in this version.");
74
- }
75
- }
76
- catch (error) {
77
- return `Error: ${error.message}`;
78
- }
79
- }
80
- cancel() {
81
- this.isCancelled = true;
82
- this.emit("agent_cancelled", {
83
- message: "Agent execution cancelled",
84
- data: { cancelled: true },
85
- });
86
- }
87
- setTestingMode(mode) {
88
- this.config.testingMode = mode;
89
- this.sessionManager.updateConfig({
90
- testingMode: mode,
91
- baseUrl: this.config.baseUrl,
92
- apiBaseUrl: this.config.apiBaseUrl,
93
- });
94
- this.toolExecutor.setTestingMode(mode, {
95
- baseUrl: this.config.baseUrl,
96
- apiBaseUrl: this.config.apiBaseUrl,
97
- });
98
- }
99
- /**
100
- * Set the permission prompt function for user interaction
101
- */
102
- setPermissionPromptFn(fn) {
103
- this.toolExecutor.setPermissionPromptFn(fn);
104
- }
105
- async refreshHttpClient() {
106
- // Refresh HTTP client with current token from config file
107
- const { getAuthToken } = await import("./config.js");
108
- const currentToken = getAuthToken();
109
- if (currentToken) {
110
- this.config.token = currentToken; // Update instance config
111
- }
112
- this.httpClient = createAgentHttpClient({
113
- baseUrl: this.config.apiUrl,
114
- token: this.config.token,
115
- });
116
- }
117
- async handleLocalMode(messages, toolSchemas) {
118
- // Auto-compact context if approaching token limits
119
- this.autoCompactIfNeeded();
120
- // Build system prompt with context
121
- const systemPrompt = build_cli_system_prompt("unified_agent_react", {
122
- user_goal: "Help the user with testing and automation tasks",
123
- base_url: this.config.baseUrl,
124
- api_base_url: this.config.apiBaseUrl,
125
- test_type: this.config.testingMode,
126
- mode: "local",
127
- });
128
- // Format messages and prepend system prompt
129
- const formattedMessages = this.formatMessages(messages);
130
- const messagesWithSystem = [
131
- {
132
- role: "system",
133
- content: systemPrompt,
134
- },
135
- ...formattedMessages,
136
- ];
137
- // Add latest screenshot to context if available
138
- const messagesWithScreenshot = this.addLatestScreenshotToContext(messagesWithSystem);
139
- const payload = {
140
- messages: messagesWithScreenshot,
141
- tools: toolSchemas,
142
- tool_choice: "auto",
143
- context: {
144
- test_id: this.conversationId,
145
- mode: "local",
146
- ...(this.currentModel && { ai_model: this.currentModel }),
147
- },
148
- mode: "local",
149
- // Reasoning chaining (matches backend implementation):
150
- // - function_outputs: Used by all providers to send tool results
151
- // - previous_response_id: OpenAI-specific extended reasoning chaining
152
- // (backend filters this for non-OpenAI providers)
153
- ...(this.pendingFunctionOutputs && {
154
- function_outputs: this.pendingFunctionOutputs,
155
- }),
156
- ...(this.lastResponseId && { previous_response_id: this.lastResponseId }),
157
- };
158
- // Check for cancellation before HTTP request
159
- if (this.isCancelled) {
160
- return "Agent execution cancelled";
161
- }
162
- let result;
163
- try {
164
- result = await this.httpClient.postAgentReason(payload);
165
- }
166
- catch (error) {
167
- // Handle 401 errors by refreshing the HTTP client and retrying once
168
- if (error?.response?.status === 401) {
169
- await this.refreshHttpClient();
170
- result = await this.httpClient.postAgentReason(payload);
171
- }
172
- else {
173
- throw error;
174
- }
175
- }
176
- // Check for cancellation after HTTP request
177
- if (this.isCancelled) {
178
- return "Agent execution cancelled";
179
- }
180
- // Capture response ID for reasoning chaining (OpenAI-specific)
181
- this.lastResponseId = result.id || null;
182
- // Capture Gemini thought signatures for reasoning chaining
183
- if (result.gemini_thought_signatures) {
184
- this.geminiThoughtSignatures = result.gemini_thought_signatures;
185
- }
186
- // Clear pending function outputs after they've been sent
187
- if (this.pendingFunctionOutputs) {
188
- this.pendingFunctionOutputs = null;
189
- }
190
- const toolCall = this.extractToolCall(result);
191
- if (toolCall) {
192
- // IMPORTANT: Generate the tool call ID once and reuse everywhere to prevent mismatch
193
- // If toolCall.id is undefined, we generate a fallback ID that MUST be used consistently
194
- // in both the assistant message (with tool_calls) and the tool result message
195
- const resolvedCallId = toolCall.id || `call_${Date.now()}`;
196
- // Extract and emit assistant reasoning text first
197
- const assistantReasoning = this.extractAssistantText(result);
198
- if (assistantReasoning) {
199
- this.emit("assistant_reasoning", {
200
- message: assistantReasoning,
201
- data: {
202
- reasoning: assistantReasoning,
203
- },
204
- });
205
- // Add assistant message with tool_calls to match backend format
206
- const assistantMessage = {
207
- role: "assistant",
208
- content: assistantReasoning,
209
- toolCalls: [
210
- {
211
- id: resolvedCallId,
212
- name: toolCall.name,
213
- arguments: toolCall.arguments,
214
- },
215
- ],
216
- };
217
- // Add Gemini thought signatures if present (for reasoning chaining)
218
- if (this.geminiThoughtSignatures.length > 0) {
219
- assistantMessage.gemini_thought_signatures = [
220
- ...this.geminiThoughtSignatures,
221
- ];
222
- }
223
- this.sessionManager.addMessage(assistantMessage);
224
- }
225
- else {
226
- // No reasoning text, but still need to add assistant message with tool call
227
- const assistantMessage = {
228
- role: "assistant",
229
- content: "",
230
- toolCalls: [
231
- {
232
- id: resolvedCallId,
233
- name: toolCall.name,
234
- arguments: toolCall.arguments,
235
- },
236
- ],
237
- };
238
- // Add Gemini thought signatures if present (for reasoning chaining)
239
- if (this.geminiThoughtSignatures.length > 0) {
240
- assistantMessage.gemini_thought_signatures = [
241
- ...this.geminiThoughtSignatures,
242
- ];
243
- }
244
- this.sessionManager.addMessage(assistantMessage);
245
- }
246
- // Parse arguments for both event emission and tool execution
247
- const parsedArguments = typeof toolCall.arguments === "string"
248
- ? JSON.parse(toolCall.arguments)
249
- : toolCall.arguments;
250
- // Check for loop detection - same tool called repeatedly with similar args
251
- const loopWarning = this.checkForToolLoop(toolCall.name, parsedArguments);
252
- if (loopWarning) {
253
- // Inject a warning message to help the agent break out of the loop
254
- this.sessionManager.addMessage({
255
- role: "user",
256
- content: loopWarning,
257
- });
258
- // Continue to next iteration with the warning in context
259
- return this.handleLocalMode(this.sessionManager.getMessages(), this.getAvailableToolSchemas());
260
- }
261
- // Emit tool start event
262
- this.emit("tool_start", {
263
- tool: toolCall.name,
264
- message: `Running ${toolCall.name}...`,
265
- data: {
266
- tool: toolCall.name,
267
- arguments: parsedArguments,
268
- },
269
- });
270
- // Check for cancellation before tool execution
271
- if (this.isCancelled) {
272
- return "Agent execution cancelled";
273
- }
274
- const toolResult = await this.toolExecutor.execute({
275
- name: toolCall.name,
276
- arguments: parsedArguments,
277
- });
278
- const pendingScreenshot = this.toolExecutor.consumePendingScreenshot();
279
- // Check for cancellation after tool execution
280
- if (this.isCancelled) {
281
- return "Agent execution cancelled";
282
- }
283
- // Emit tool result event
284
- this.emit("tool_result", {
285
- tool: toolCall.name,
286
- message: `Completed ${toolCall.name}`,
287
- data: {
288
- tool: toolCall.name,
289
- result: toolResult,
290
- success: toolResult?.success !== false,
291
- },
292
- });
293
- if (toolCall.name === "set_testing_mode" &&
294
- toolResult?.success &&
295
- isTestingMode(toolResult.mode)) {
296
- this.config.testingMode = toolResult.mode;
297
- // Save mode preference for next session
298
- configManager.setLastMode(toolResult.mode).catch(() => {
299
- // Ignore save errors - non-critical
300
- });
301
- }
302
- // Handle finish tool - stop the agent loop
303
- if (toolCall.name === "finish" && toolResult?.finished) {
304
- this.sessionManager.addMessage({
305
- role: "tool",
306
- content: JSON.stringify(toolResult),
307
- toolCall: {
308
- id: resolvedCallId,
309
- name: toolCall.name,
310
- arguments: toolCall.arguments,
311
- },
312
- });
313
- return toolResult.summary || toolResult.message || "Task completed";
314
- }
315
- // Add tool result message with proper tool_call_id to match backend format
316
- // Note: resolvedCallId was generated at the top of this block to ensure consistency
317
- this.sessionManager.addMessage({
318
- role: "tool",
319
- content: JSON.stringify(toolResult),
320
- toolCall: {
321
- id: resolvedCallId,
322
- name: toolCall.name,
323
- arguments: toolCall.arguments,
324
- },
325
- });
326
- // Add to pending function outputs for reasoning chaining (matches backend)
327
- const functionOutput = {
328
- call_id: resolvedCallId,
329
- output: JSON.stringify(toolResult),
330
- };
331
- if (!this.pendingFunctionOutputs) {
332
- this.pendingFunctionOutputs = [];
333
- }
334
- this.pendingFunctionOutputs.push(functionOutput);
335
- if (pendingScreenshot) {
336
- const screenshotMessageId = this.handleScreenshotInHistory(pendingScreenshot.data, pendingScreenshot.action);
337
- if (toolCall.name === "browser_take_screenshot" &&
338
- screenshotMessageId) {
339
- this.sessionManager.addMessage({
340
- role: "tool",
341
- content: JSON.stringify({
342
- success: true,
343
- screenshot_message_id: screenshotMessageId,
344
- }),
345
- toolCall: {
346
- id: `${toolCall.id || Date.now()}-link`,
347
- name: toolCall.name,
348
- arguments: { linked_message_id: screenshotMessageId },
349
- },
350
- });
351
- }
352
- }
353
- // Prune tool results to optimize conversation history
354
- this.pruneToolResults(toolCall.name);
355
- // Check for cancellation before continuing
356
- if (this.isCancelled) {
357
- return "Agent execution cancelled";
358
- }
359
- return this.handleLocalMode(this.sessionManager.getMessages(), this.getAvailableToolSchemas());
360
- }
361
- const assistantMessage = this.extractAssistantText(result);
362
- this.sessionManager.addMessage({
363
- role: "assistant",
364
- content: assistantMessage,
365
- });
366
- return assistantMessage;
367
- }
368
- getAvailableToolSchemas() {
369
- const testingMode = this.config.testingMode;
370
- const schemas = getToolSchemasByDomain(testingMode);
371
- return Object.values(schemas).map((schema) => ({
372
- type: "function",
373
- function: schema,
374
- }));
375
- }
376
- /**
377
- * Check if the agent is stuck in a loop calling the same tool repeatedly
378
- * Returns a warning message if loop detected, null otherwise
379
- */
380
- checkForToolLoop(toolName, args) {
381
- const argsKey = JSON.stringify(args);
382
- const callSignature = { name: toolName, args: argsKey };
383
- // Track this call
384
- this.recentToolCalls.push(callSignature);
385
- // Keep only recent calls (last 10)
386
- if (this.recentToolCalls.length > 10) {
387
- this.recentToolCalls.shift();
388
- }
389
- // Count how many times this exact tool has been called (regardless of args for now)
390
- // This catches cases where the agent keeps calling the same tool with slightly different args
391
- const sameToolCalls = this.recentToolCalls.filter((call) => call.name === toolName);
392
- if (sameToolCalls.length >= AgentOrchestrator.MAX_REPEATED_CALLS) {
393
- // Check if args are similar
394
- const similarCalls = sameToolCalls.filter((call) => this.areSimilarArgs(call.args, argsKey));
395
- if (similarCalls.length >= AgentOrchestrator.MAX_REPEATED_CALLS) {
396
- // Clear the tracking to allow fresh attempts after the warning
397
- this.recentToolCalls = [];
398
- return `[System Notice] You've called ${toolName} ${similarCalls.length} times with similar arguments. This appears to be a loop.
399
-
400
- STOP and do one of the following:
401
- 1. If you already have the information you need from previous tool results, USE IT to answer the user's question
402
- 2. If the tool isn't returning what you expected, try a DIFFERENT tool or approach
403
- 3. If you're stuck, explain what you're trying to accomplish and ask for guidance
404
-
405
- Do NOT call ${toolName} again with the same or similar arguments.`;
406
- }
407
- }
408
- return null;
409
- }
410
- /**
411
- * Check if two argument strings are similar (exact match or minor variations)
412
- */
413
- areSimilarArgs(args1, args2) {
414
- // Exact match
415
- if (args1 === args2)
416
- return true;
417
- // Parse and compare key fields for common tools
418
- try {
419
- const parsed1 = JSON.parse(args1);
420
- const parsed2 = JSON.parse(args2);
421
- // For directory/file tools, compare the path field
422
- if ("path" in parsed1 && "path" in parsed2) {
423
- // Normalize paths for comparison
424
- const path1 = (parsed1.path || ".").replace(/^\.\//, "");
425
- const path2 = (parsed2.path || ".").replace(/^\.\//, "");
426
- return path1 === path2;
427
- }
428
- // For grep_search, compare query and path
429
- if ("query" in parsed1 && "query" in parsed2) {
430
- return parsed1.query === parsed2.query && parsed1.path === parsed2.path;
431
- }
432
- // For search_files, compare pattern
433
- if ("pattern" in parsed1 && "pattern" in parsed2) {
434
- return parsed1.pattern === parsed2.pattern;
435
- }
436
- }
437
- catch {
438
- // If parsing fails, fall back to string comparison
439
- }
440
- return false;
441
- }
442
- formatMessages(messages) {
443
- return messages
444
- .filter((msg) => msg.metadata?.type !== "screenshot")
445
- .map((msg) => {
446
- const base = {
447
- role: msg.role,
448
- content: typeof msg.content === "string"
449
- ? msg.content
450
- : JSON.stringify(msg.content),
451
- };
452
- // Handle assistant messages with tool calls
453
- if (msg.role === "assistant" &&
454
- msg.toolCalls &&
455
- msg.toolCalls.length > 0) {
456
- base.tool_calls = msg.toolCalls.map((tc) => ({
457
- id: tc.id,
458
- type: "function",
459
- function: {
460
- name: tc.name,
461
- arguments: typeof tc.arguments === "string"
462
- ? tc.arguments
463
- : JSON.stringify(tc.arguments),
464
- },
465
- }));
466
- // Include Gemini thought signatures for reasoning chaining
467
- if (msg.gemini_thought_signatures &&
468
- msg.gemini_thought_signatures.length > 0) {
469
- base.gemini_thought_signatures = msg.gemini_thought_signatures;
470
- }
471
- }
472
- // Handle tool result messages
473
- if (msg.role === "tool" && msg.toolCall) {
474
- base.name = msg.toolCall.name;
475
- base.tool_call_id = msg.toolCall.id;
476
- }
477
- return base;
478
- });
479
- }
480
- getMode() {
481
- return "local";
482
- }
483
- getTestingMode() {
484
- return this.config.testingMode;
485
- }
486
- getSessionId() {
487
- return this.sessionManager.getSessionId();
488
- }
489
- getSessionPath() {
490
- return this.sessionManager.getSessionPath();
491
- }
492
- getConversationHistory() {
493
- return this.sessionManager.getMessages();
494
- }
495
- setModel(modelId) {
496
- this.currentModel = modelId;
497
- }
498
- extractToolCall(result) {
499
- // Check for single function call first
500
- if (result.function_call &&
501
- result.function_call.name &&
502
- result.function_call.arguments) {
503
- return {
504
- id: result.function_call.id,
505
- name: result.function_call.name,
506
- arguments: result.function_call.arguments,
507
- };
508
- }
509
- // Check for multiple function calls
510
- if (result.multiple &&
511
- Array.isArray(result.multiple) &&
512
- result.multiple.length > 0) {
513
- const call = result.multiple[0];
514
- if (call && call.name && call.arguments) {
515
- return {
516
- id: call.id,
517
- name: call.name,
518
- arguments: call.arguments,
519
- };
520
- }
521
- }
522
- // Legacy support for old format
523
- if (result.name && result.arguments) {
524
- return {
525
- id: result.context?.tool_call_id,
526
- name: result.name,
527
- arguments: result.arguments,
528
- };
529
- }
530
- return null;
531
- }
532
- extractAssistantText(result) {
533
- if (typeof result.content === "string") {
534
- return result.content;
535
- }
536
- if (Array.isArray(result.content)) {
537
- const text = result.content
538
- .map((chunk) => {
539
- if (!chunk)
540
- return "";
541
- if (typeof chunk === "string")
542
- return chunk;
543
- if (typeof chunk?.text === "string")
544
- return chunk.text;
545
- return "";
546
- })
547
- .filter(Boolean)
548
- .join("\n")
549
- .trim();
550
- if (text) {
551
- return text;
552
- }
553
- }
554
- // Check reasoning field as fallback
555
- if (result.reasoning) {
556
- return result.reasoning;
557
- }
558
- return result.message || "";
559
- }
560
- handleScreenshotInHistory(screenshotData, actionName) {
561
- try {
562
- // Create a screenshot message for the conversation history
563
- const screenshotMessage = {
564
- id: `screenshot-${Date.now()}`,
565
- role: "assistant",
566
- content: "",
567
- timestamp: Date.now(),
568
- metadata: {
569
- type: "screenshot",
570
- action_name: actionName,
571
- screenshot_data: screenshotData,
572
- page_url: screenshotData.page_url,
573
- page_title: screenshotData.page_title,
574
- image_url: screenshotData.image_url, // CDN URL for AI usage
575
- cloudinary_uploaded: screenshotData.cloudinary_uploaded,
576
- },
577
- };
578
- // Add screenshot message to conversation history
579
- this.sessionManager.addMessage(screenshotMessage);
580
- // Remove any previous screenshot messages to keep only the latest
581
- this.removePreviousScreenshotMessages(screenshotMessage.id);
582
- return screenshotMessage.id;
583
- }
584
- catch (error) {
585
- // Don't fail the main flow if screenshot handling fails
586
- return null;
587
- }
588
- }
589
- removePreviousScreenshotMessages(currentScreenshotId) {
590
- try {
591
- const messages = this.sessionManager.getMessages();
592
- const filteredMessages = messages.filter((msg) => {
593
- // Keep the current screenshot message
594
- if (msg.id === currentScreenshotId) {
595
- return true;
596
- }
597
- // Remove other screenshot messages
598
- if (msg.metadata?.type === "screenshot") {
599
- return false;
600
- }
601
- return true;
602
- });
603
- // Clear and re-add filtered messages
604
- this.sessionManager.clearMessages();
605
- filteredMessages.forEach((msg) => this.sessionManager.addMessage(msg));
606
- }
607
- catch (error) {
608
- // Ignore cleanup errors
609
- }
610
- }
611
- addLatestScreenshotToContext(messages) {
612
- // Only add screenshots for UI testing mode
613
- if (this.config.testingMode !== "ui") {
614
- return messages;
615
- }
616
- try {
617
- const conversationMessages = this.sessionManager.getMessages();
618
- // Find the latest screenshot message
619
- const latestScreenshotMessage = conversationMessages
620
- .filter((msg) => msg.metadata?.type === "screenshot")
621
- .sort((a, b) => (b.timestamp || 0) - (a.timestamp || 0))[0];
622
- if (!latestScreenshotMessage?.metadata?.image_url) {
623
- return messages;
624
- }
625
- const imageUrl = latestScreenshotMessage.metadata.image_url;
626
- const pageUrl = latestScreenshotMessage.metadata.page_url ??
627
- latestScreenshotMessage.metadata.screenshot_data?.url ??
628
- latestScreenshotMessage.metadata.screenshot_data?.page_url ??
629
- "unknown";
630
- const pageTitle = latestScreenshotMessage.metadata.page_title ??
631
- latestScreenshotMessage.metadata.screenshot_data?.page_title ??
632
- "unknown";
633
- const actionName = latestScreenshotMessage.metadata.action_name ??
634
- latestScreenshotMessage.metadata.screenshot_data?.action_name ??
635
- "snapshot_refresh";
636
- // Add screenshot as image message (similar to backend)
637
- const screenshotMessage = {
638
- role: "user",
639
- content: [
640
- {
641
- type: "image_url",
642
- image_url: {
643
- url: imageUrl,
644
- detail: "low",
645
- },
646
- },
647
- {
648
- type: "text",
649
- text: `Current page screenshot after ${actionName?.replace(/_/g, " ")}: ${pageTitle} (${pageUrl})`,
650
- },
651
- ],
652
- };
653
- return [...messages, screenshotMessage];
654
- }
655
- catch (error) {
656
- // Don't fail if screenshot context addition fails
657
- return messages;
658
- }
659
- }
660
- pruneToolResults(toolName) {
661
- try {
662
- const messages = this.sessionManager.getMessages();
663
- // Prune tool results
664
- const prunedMessages = toolResultPruner.pruneToolResults(messages, toolName);
665
- // Cap conversation history if needed
666
- const cappedMessages = toolResultPruner.capConversationHistory(prunedMessages);
667
- // Update session with pruned messages
668
- if (cappedMessages.length !== messages.length) {
669
- this.sessionManager.clearMessages();
670
- cappedMessages.forEach((msg) => this.sessionManager.addMessage(msg));
671
- }
672
- }
673
- catch (error) {
674
- // Don't fail on pruning errors
675
- }
676
- }
677
- /**
678
- * Get context compaction statistics
679
- */
680
- getContextStats() {
681
- const messages = this.sessionManager.getMessages();
682
- return this.contextCompactor.getStats(messages);
683
- }
684
- /**
685
- * Manually compact context if needed
686
- */
687
- async compactContext() {
688
- const messages = this.sessionManager.getMessages();
689
- const result = await this.contextCompactor.compact(messages, this.httpClient);
690
- if (result.removedCount > 0) {
691
- this.sessionManager.clearMessages();
692
- result.compactedMessages.forEach((msg) => this.sessionManager.addMessage(msg));
693
- }
694
- return {
695
- success: true,
696
- stats: {
697
- removedCount: result.removedCount,
698
- tokensBeforeCompaction: result.tokensBeforeCompaction,
699
- tokensAfterCompaction: result.tokensAfterCompaction,
700
- strategy: result.strategy,
701
- summaryGenerated: result.summaryGenerated,
702
- },
22
+ socketUrlOverride: config.socketUrlOverride,
703
23
  };
704
- }
705
- /**
706
- * Auto-compact context before sending to backend if needed
707
- */
708
- async autoCompactIfNeeded() {
709
- const messages = this.sessionManager.getMessages();
710
- if (this.contextCompactor.needsCompaction(messages)) {
711
- const result = await this.contextCompactor.compact(messages, this.httpClient);
712
- this.sessionManager.clearMessages();
713
- result.compactedMessages.forEach((msg) => this.sessionManager.addMessage(msg));
714
- // Emit event to notify UI
715
- this.emit("context_compacted", {
716
- message: "Context automatically compacted to manage token limits",
717
- data: {
718
- removedCount: result.removedCount,
719
- tokensBeforeCompaction: result.tokensBeforeCompaction,
720
- tokensAfterCompaction: result.tokensAfterCompaction,
721
- strategy: result.strategy,
722
- summaryGenerated: result.summaryGenerated,
723
- },
724
- });
725
- }
24
+ super(backendCfg, sessionId);
726
25
  }
727
26
  }
728
27
  //# sourceMappingURL=agent-orchestrator.js.map