explorbot 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -5
- package/bin/explorbot-cli.ts +29 -7
- package/boat/api-tester/src/config.ts +45 -3
- package/boat/doc-collector/src/ai/documentarian.ts +8 -2
- package/boat/doc-collector/src/ai/tools.ts +29 -16
- package/boat/doc-collector/src/cli.ts +2 -6
- package/boat/doc-collector/src/config.ts +2 -0
- package/boat/doc-collector/src/docbot.ts +42 -14
- package/boat/doc-collector/src/interaction-screenshots.ts +160 -0
- package/boat/doc-collector/src/screenshots.ts +24 -16
- package/dist/bin/explorbot-cli.js +26 -6
- package/dist/boat/api-tester/src/config.js +43 -4
- package/dist/boat/doc-collector/src/ai/documentarian.js +7 -2
- package/dist/boat/doc-collector/src/ai/tools.js +21 -15
- package/dist/boat/doc-collector/src/cli.js +2 -5
- package/dist/boat/doc-collector/src/config.js +1 -0
- package/dist/boat/doc-collector/src/docbot.js +39 -13
- package/dist/boat/doc-collector/src/interaction-screenshots.js +156 -0
- package/dist/boat/doc-collector/src/screenshots.js +25 -16
- package/dist/models.json +30 -0
- package/dist/package.json +8 -2
- package/dist/src/action.d.ts +11 -3
- package/dist/src/action.js +22 -5
- package/dist/src/ai/agent.d.ts +17 -0
- package/dist/src/ai/captain/idle-mode.js +1 -1
- package/dist/src/ai/captain/test-mode.js +1 -1
- package/dist/src/ai/captain/web-mode.js +18 -22
- package/dist/src/ai/captain.js +13 -26
- package/dist/src/ai/driller.d.ts +2 -13
- package/dist/src/ai/driller.js +23 -38
- package/dist/src/ai/historian/codeceptjs.js +1 -1
- package/dist/src/ai/historian/playwright.d.ts +3 -2
- package/dist/src/ai/historian/playwright.js +1 -1
- package/dist/src/ai/historian/screencast.d.ts +2 -1
- package/dist/src/ai/historian/screencast.js +2 -2
- package/dist/src/ai/historian.d.ts +5 -4
- package/dist/src/ai/navigator.d.ts +6 -2
- package/dist/src/ai/navigator.js +26 -22
- package/dist/src/ai/pilot.d.ts +13 -6
- package/dist/src/ai/pilot.js +49 -44
- package/dist/src/ai/planner.d.ts +3 -2
- package/dist/src/ai/planner.js +19 -17
- package/dist/src/ai/provider.js +0 -1
- package/dist/src/ai/quartermaster.d.ts +3 -2
- package/dist/src/ai/quartermaster.js +4 -4
- package/dist/src/ai/rerunner.d.ts +2 -11
- package/dist/src/ai/rerunner.js +9 -22
- package/dist/src/ai/researcher/coordinates.js +4 -3
- package/dist/src/ai/researcher/deep-analysis.d.ts +2 -0
- package/dist/src/ai/researcher/deep-analysis.js +6 -6
- package/dist/src/ai/researcher/locators.d.ts +9 -2
- package/dist/src/ai/researcher/locators.js +65 -8
- package/dist/src/ai/researcher/sections.d.ts +2 -0
- package/dist/src/ai/researcher/sections.js +2 -2
- package/dist/src/ai/researcher.js +29 -54
- package/dist/src/ai/task-agent.d.ts +18 -5
- package/dist/src/ai/task-agent.js +35 -6
- package/dist/src/ai/tester.d.ts +6 -12
- package/dist/src/ai/tester.js +37 -51
- package/dist/src/ai/tools.d.ts +11 -10
- package/dist/src/ai/tools.js +24 -36
- package/dist/src/command-handler.js +1 -1
- package/dist/src/commands/context-aria-command.js +1 -1
- package/dist/src/commands/context-command.js +7 -5
- package/dist/src/commands/context-data-command.js +2 -2
- package/dist/src/commands/context-experience-command.js +2 -2
- package/dist/src/commands/context-html-command.js +2 -2
- package/dist/src/commands/context-knowledge-command.js +1 -1
- package/dist/src/commands/drill-command.js +1 -1
- package/dist/src/commands/explore-command.js +4 -3
- package/dist/src/commands/freesail-command.js +2 -2
- package/dist/src/commands/learn-command.js +4 -4
- package/dist/src/commands/path-command.js +1 -1
- package/dist/src/commands/research-command.js +1 -1
- package/dist/src/commands/test-command.js +1 -1
- package/dist/src/components/App.js +1 -1
- package/dist/src/config.d.ts +17 -0
- package/dist/src/config.js +144 -10
- package/dist/src/experience-tracker.js +1 -1
- package/dist/src/explorbot.d.ts +17 -8
- package/dist/src/explorbot.js +74 -36
- package/dist/src/explorer.d.ts +67 -91
- package/dist/src/explorer.js +327 -567
- package/dist/src/state-manager.d.ts +6 -1
- package/dist/src/state-manager.js +1 -0
- package/dist/src/utils/hooks-runner.js +2 -4
- package/dist/src/utils/test-files.js +1 -1
- package/dist/src/utils/web-annotate.d.ts +5 -0
- package/dist/src/utils/web-annotate.js +58 -0
- package/dist/src/utils/web-eidx.d.ts +2 -0
- package/dist/src/utils/web-eidx.js +20 -0
- package/dist/src/utils/web-element.d.ts +1 -0
- package/dist/src/utils/web-element.js +29 -0
- package/models.json +30 -0
- package/package.json +8 -2
- package/src/action.ts +25 -5
- package/src/ai/agent.ts +20 -0
- package/src/ai/captain/idle-mode.ts +1 -1
- package/src/ai/captain/test-mode.ts +1 -1
- package/src/ai/captain/web-mode.ts +18 -23
- package/src/ai/captain.ts +12 -25
- package/src/ai/driller.ts +50 -67
- package/src/ai/historian/codeceptjs.ts +1 -1
- package/src/ai/historian/playwright.ts +4 -3
- package/src/ai/historian/screencast.ts +4 -3
- package/src/ai/historian.ts +3 -2
- package/src/ai/navigator.ts +30 -25
- package/src/ai/pilot.ts +55 -47
- package/src/ai/planner.ts +22 -20
- package/src/ai/provider.ts +0 -1
- package/src/ai/quartermaster.ts +5 -4
- package/src/ai/rerunner.ts +10 -29
- package/src/ai/researcher/coordinates.ts +4 -3
- package/src/ai/researcher/deep-analysis.ts +8 -6
- package/src/ai/researcher/locators.ts +66 -9
- package/src/ai/researcher/sections.ts +4 -2
- package/src/ai/researcher.ts +32 -61
- package/src/ai/task-agent.ts +45 -11
- package/src/ai/tester.ts +40 -57
- package/src/ai/tools.ts +32 -53
- package/src/command-handler.ts +1 -1
- package/src/commands/context-aria-command.ts +1 -1
- package/src/commands/context-command.ts +7 -5
- package/src/commands/context-data-command.ts +2 -2
- package/src/commands/context-experience-command.ts +2 -2
- package/src/commands/context-html-command.ts +2 -2
- package/src/commands/context-knowledge-command.ts +1 -1
- package/src/commands/drill-command.ts +1 -1
- package/src/commands/explore-command.ts +4 -3
- package/src/commands/freesail-command.ts +2 -2
- package/src/commands/learn-command.ts +4 -4
- package/src/commands/path-command.ts +1 -1
- package/src/commands/research-command.ts +1 -1
- package/src/commands/test-command.ts +1 -1
- package/src/components/App.tsx +1 -1
- package/src/config.ts +173 -11
- package/src/experience-tracker.ts +1 -1
- package/src/explorbot.ts +78 -39
- package/src/explorer.ts +388 -626
- package/src/state-manager.ts +7 -1
- package/src/utils/hooks-runner.ts +2 -4
- package/src/utils/test-files.ts +1 -1
- package/src/utils/web-annotate.ts +64 -0
- package/src/utils/web-eidx.ts +21 -0
- package/src/utils/web-element.ts +30 -0
package/src/ai/tester.ts
CHANGED
|
@@ -5,18 +5,17 @@ import dedent from 'dedent';
|
|
|
5
5
|
import { z } from 'zod';
|
|
6
6
|
import { ActionResult } from '../action-result.ts';
|
|
7
7
|
import { clearActivity, setActivity } from '../activity.ts';
|
|
8
|
-
import type {
|
|
9
|
-
import type
|
|
8
|
+
import type { RequestStore } from '../api/request-store.ts';
|
|
9
|
+
import type { TestRun } from '../explorer.ts';
|
|
10
10
|
import { Observability } from '../observability.ts';
|
|
11
11
|
import type { StateTransition } from '../state-manager.ts';
|
|
12
12
|
import { Stats } from '../stats.ts';
|
|
13
13
|
import { type Test, TestResult, type TestResultType } from '../test-plan.ts';
|
|
14
14
|
import { detectFocusArea, extractFocusedElement } from '../utils/aria.ts';
|
|
15
15
|
import { ErrorPageError, isErrorPage } from '../utils/error-page.ts';
|
|
16
|
-
import { HooksRunner } from '../utils/hooks-runner.ts';
|
|
17
16
|
import { createDebug, tag } from '../utils/logger.ts';
|
|
18
17
|
import { loop } from '../utils/loop.ts';
|
|
19
|
-
import type { Agent } from './agent.ts';
|
|
18
|
+
import type { Agent, AgentDeps } from './agent.ts';
|
|
20
19
|
import type { Captain } from './captain.ts';
|
|
21
20
|
import type { Conversation } from './conversation.ts';
|
|
22
21
|
import { Navigator } from './navigator.ts';
|
|
@@ -44,8 +43,8 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
44
43
|
protected readonly ACTION_TOOLS = ['click', 'hover', 'pressKey', 'form'];
|
|
45
44
|
protected readonly SPECIAL_CONTEXT_ACTION_TOOLS = ['exitIframe'];
|
|
46
45
|
emoji = '🧪';
|
|
47
|
-
private
|
|
48
|
-
private
|
|
46
|
+
private requestStore: RequestStore;
|
|
47
|
+
private testRun: TestRun | null = null;
|
|
49
48
|
private currentConversation: Conversation | null = null;
|
|
50
49
|
private pilot: Pilot | null = null;
|
|
51
50
|
private captain: Captain | null = null;
|
|
@@ -60,38 +59,23 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
60
59
|
private previousStateHash: string | null = null;
|
|
61
60
|
private pageStateHash: string | null = null;
|
|
62
61
|
private pageActionResult: ActionResult | null = null;
|
|
63
|
-
private hooksRunner: HooksRunner;
|
|
64
62
|
private seenUiMapUrls = new Set<string>();
|
|
65
63
|
private lastAnalyzedStateHash: string | null = null;
|
|
66
64
|
private stalledIterations = 0;
|
|
67
65
|
private readonly MAX_STALLED_ITERATIONS = 3;
|
|
68
66
|
|
|
69
|
-
constructor(
|
|
70
|
-
super();
|
|
71
|
-
this.
|
|
72
|
-
this.provider = provider;
|
|
67
|
+
constructor(deps: AgentDeps, researcher: Researcher, navigator: Navigator, agentTools?: any) {
|
|
68
|
+
super(deps);
|
|
69
|
+
this.requestStore = deps.requestStore;
|
|
73
70
|
this.researcher = researcher;
|
|
74
71
|
this.navigator = navigator;
|
|
75
72
|
this.agentTools = agentTools;
|
|
76
|
-
this.hooksRunner = new HooksRunner(explorer, explorer.getConfig());
|
|
77
73
|
}
|
|
78
74
|
|
|
79
75
|
protected getNavigator(): Navigator {
|
|
80
76
|
return this.navigator;
|
|
81
77
|
}
|
|
82
78
|
|
|
83
|
-
protected getExperienceTracker(): ExperienceTracker {
|
|
84
|
-
return this.explorer.getStateManager().getExperienceTracker();
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
protected getKnowledgeTracker() {
|
|
88
|
-
return this.explorer.getKnowledgeTracker();
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
protected getProvider(): Provider {
|
|
92
|
-
return this.provider;
|
|
93
|
-
}
|
|
94
|
-
|
|
95
79
|
setPilot(pilot: Pilot): void {
|
|
96
80
|
this.pilot = pilot;
|
|
97
81
|
}
|
|
@@ -101,11 +85,11 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
101
85
|
}
|
|
102
86
|
|
|
103
87
|
private getCurrentState(): ActionResult {
|
|
104
|
-
return ActionResult.fromState(this.
|
|
88
|
+
return ActionResult.fromState(this.stateManager.getCurrentState()!);
|
|
105
89
|
}
|
|
106
90
|
|
|
107
91
|
private get progressCheckInterval(): number {
|
|
108
|
-
return (this.
|
|
92
|
+
return (this.config.ai?.agents?.tester as any)?.progressCheckInterval ?? 3;
|
|
109
93
|
}
|
|
110
94
|
|
|
111
95
|
getConversation(): Conversation | null {
|
|
@@ -114,7 +98,7 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
114
98
|
|
|
115
99
|
async test(task: Test): Promise<{ success: boolean }> {
|
|
116
100
|
Stats.tests++;
|
|
117
|
-
const state = this.
|
|
101
|
+
const state = this.stateManager.getCurrentState();
|
|
118
102
|
if (!state) throw new Error('No state found');
|
|
119
103
|
|
|
120
104
|
setActivity(`🧪 Testing: ${task.scenario}`, 'action');
|
|
@@ -126,20 +110,20 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
126
110
|
this.seenUiMapUrls.clear();
|
|
127
111
|
this.lastAnalyzedStateHash = null;
|
|
128
112
|
this.stalledIterations = 0;
|
|
129
|
-
this.
|
|
113
|
+
this.stateManager.clearHistory();
|
|
130
114
|
this.resetFailureCount();
|
|
131
115
|
this.pilot?.reset();
|
|
132
116
|
|
|
133
|
-
const requestStore = this.
|
|
134
|
-
requestStore
|
|
135
|
-
const offFailedRequest = requestStore
|
|
117
|
+
const requestStore = this.requestStore;
|
|
118
|
+
requestStore.clear();
|
|
119
|
+
const offFailedRequest = requestStore.onFailedRequest((r) => {
|
|
136
120
|
task.addNote(`Network error: ${r.method} ${r.path} → ${r.status}`, TestResult.FAILED);
|
|
137
121
|
});
|
|
138
122
|
|
|
139
123
|
const initialState = ActionResult.fromState(state);
|
|
140
124
|
if (isErrorPage(initialState)) {
|
|
141
125
|
task.start();
|
|
142
|
-
await this.explorer.
|
|
126
|
+
this.testRun = await this.explorer.beginTest(task);
|
|
143
127
|
offFailedRequest?.();
|
|
144
128
|
return await this.abortStartedTestOnErrorPage(task, initialState);
|
|
145
129
|
}
|
|
@@ -196,7 +180,8 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
196
180
|
|
|
197
181
|
debugLog('Starting test execution with tools');
|
|
198
182
|
|
|
199
|
-
|
|
183
|
+
this.testRun = await this.explorer.beginTest(task);
|
|
184
|
+
if (!this.testRun.started) {
|
|
200
185
|
offFailedRequest?.();
|
|
201
186
|
await this.cleanupStartedTest(task);
|
|
202
187
|
return { success: task.isSuccessful };
|
|
@@ -214,7 +199,7 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
214
199
|
}
|
|
215
200
|
}
|
|
216
201
|
|
|
217
|
-
const startState = this.
|
|
202
|
+
const startState = this.stateManager.getCurrentState();
|
|
218
203
|
if (startState) {
|
|
219
204
|
task.addUrlNote(startState);
|
|
220
205
|
const startActionResult = ActionResult.fromState(startState);
|
|
@@ -226,14 +211,14 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
226
211
|
const currentUrl = startState?.url || task.startUrl || '';
|
|
227
212
|
await this.hooksRunner.runBeforeHook('tester', currentUrl);
|
|
228
213
|
|
|
229
|
-
const offStateChange = this.
|
|
214
|
+
const offStateChange = this.stateManager.onStateChange((event: StateTransition) => {
|
|
230
215
|
if (task.hasFinished) return;
|
|
231
216
|
if (event.toState?.url === event.fromState?.url) return;
|
|
232
217
|
if (event.toState) task.addUrlNote(event.toState, event.fromState || undefined);
|
|
233
218
|
task.states.push(event.toState);
|
|
234
219
|
});
|
|
235
220
|
|
|
236
|
-
const codeceptjsTools = createCodeceptJSTools(this.
|
|
221
|
+
const codeceptjsTools = createCodeceptJSTools(this.toolDeps, task);
|
|
237
222
|
let assertionPerformed = false;
|
|
238
223
|
let extensions = 0;
|
|
239
224
|
let shouldContinue = true;
|
|
@@ -244,7 +229,7 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
244
229
|
await loop(
|
|
245
230
|
async ({ stop, pause, iteration, userInput }) => {
|
|
246
231
|
debugLog('iteration', iteration);
|
|
247
|
-
if (!(await this.explorer.
|
|
232
|
+
if (!(await this.explorer.recover()).ok) {
|
|
248
233
|
task.addNote('Browser page is unavailable');
|
|
249
234
|
task.finish(TestResult.FAILED);
|
|
250
235
|
stop();
|
|
@@ -258,12 +243,12 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
258
243
|
...this.agentTools,
|
|
259
244
|
};
|
|
260
245
|
if (currentState.isInsideIframe) {
|
|
261
|
-
Object.assign(tools, createIframeTools(this.
|
|
246
|
+
Object.assign(tools, createIframeTools(this.toolDeps));
|
|
262
247
|
}
|
|
263
248
|
|
|
264
249
|
debugLog(`Test ${task.scenario} iteration ${iteration}`);
|
|
265
250
|
|
|
266
|
-
if (this.
|
|
251
|
+
if (this.stateManager.isInDeadLoop()) {
|
|
267
252
|
task.addNote('Dead loop detected. Stopped');
|
|
268
253
|
stop();
|
|
269
254
|
return;
|
|
@@ -412,7 +397,7 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
412
397
|
|
|
413
398
|
if (task.hasFinished) break;
|
|
414
399
|
|
|
415
|
-
if (!(await this.explorer.
|
|
400
|
+
if (!(await this.explorer.recover()).ok) break;
|
|
416
401
|
|
|
417
402
|
const finalState = this.getCurrentState();
|
|
418
403
|
const wantsContinue = await this.pilot!.finalReview(task, finalState, conversation, this.navigator);
|
|
@@ -431,7 +416,7 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
431
416
|
shouldContinue = true;
|
|
432
417
|
}
|
|
433
418
|
|
|
434
|
-
const finalUrl = this.
|
|
419
|
+
const finalUrl = this.stateManager.getCurrentState()?.url || currentUrl;
|
|
435
420
|
await this.hooksRunner.runAfterHook('tester', finalUrl);
|
|
436
421
|
|
|
437
422
|
await this.getHistorian().saveSession(task, initialState, conversation);
|
|
@@ -443,7 +428,7 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
443
428
|
offStateChange();
|
|
444
429
|
offFailedRequest?.();
|
|
445
430
|
await this.finishTest(task);
|
|
446
|
-
await this.
|
|
431
|
+
await this.testRun?.stop(this.buildStopTestMeta(task));
|
|
447
432
|
|
|
448
433
|
return {
|
|
449
434
|
success: task.isSuccessful,
|
|
@@ -559,7 +544,7 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
559
544
|
}
|
|
560
545
|
|
|
561
546
|
if (currentState.isInsideIframe) {
|
|
562
|
-
const iframeInfo = currentState.iframeURL ||
|
|
547
|
+
const iframeInfo = currentState.iframeURL || 'iframe context active';
|
|
563
548
|
context += dedent`
|
|
564
549
|
<iframe_context>
|
|
565
550
|
INSIDE IFRAME: ${iframeInfo}
|
|
@@ -568,10 +553,10 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
568
553
|
`;
|
|
569
554
|
}
|
|
570
555
|
|
|
571
|
-
|
|
572
|
-
|
|
556
|
+
const otherTabs = this.stateManager.otherTabs;
|
|
557
|
+
if (otherTabs.length > 0) {
|
|
573
558
|
context += multipleTabsRule(otherTabs);
|
|
574
|
-
this.
|
|
559
|
+
this.stateManager.otherTabs = [];
|
|
575
560
|
}
|
|
576
561
|
|
|
577
562
|
if (isNewUrl) {
|
|
@@ -675,7 +660,7 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
675
660
|
task.addNote(error.message, TestResult.FAILED, actionResult.screenshotFile, actionResult.fullUrl || actionResult.url);
|
|
676
661
|
task.finish(TestResult.FAILED);
|
|
677
662
|
this.finishTest(task);
|
|
678
|
-
await this.
|
|
663
|
+
await this.testRun?.stop(this.buildStopTestMeta(task));
|
|
679
664
|
clearActivity(true);
|
|
680
665
|
return { success: false };
|
|
681
666
|
}
|
|
@@ -790,7 +775,7 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
790
775
|
|
|
791
776
|
${dataProtectionRules}
|
|
792
777
|
|
|
793
|
-
${this.provider.getSystemPromptForAgent('tester', this.
|
|
778
|
+
${this.provider.getSystemPromptForAgent('tester', this.stateManager.getCurrentState()?.url) || ''}
|
|
794
779
|
`;
|
|
795
780
|
}
|
|
796
781
|
|
|
@@ -837,7 +822,7 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
837
822
|
}
|
|
838
823
|
|
|
839
824
|
private buildAvailableFiles(): string {
|
|
840
|
-
const userFiles = this.
|
|
825
|
+
const userFiles = this.config.files || {};
|
|
841
826
|
const codeceptDir = (global as any).codecept_dir || process.cwd();
|
|
842
827
|
const lines: string[] = [];
|
|
843
828
|
|
|
@@ -895,11 +880,9 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
895
880
|
reason: z.string().optional().describe('Explanation why reset is the only option'),
|
|
896
881
|
}),
|
|
897
882
|
execute: async ({ reason }) => {
|
|
898
|
-
|
|
899
|
-
await this.explorer.switchToMainFrame();
|
|
900
|
-
}
|
|
883
|
+
await this.explorer.action().exitIframe();
|
|
901
884
|
|
|
902
|
-
const currentState = this.
|
|
885
|
+
const currentState = this.stateManager.getCurrentState();
|
|
903
886
|
const currentUrl = currentState?.fullUrl || currentState?.url;
|
|
904
887
|
if (currentUrl === resetUrl!) {
|
|
905
888
|
return {
|
|
@@ -927,7 +910,7 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
927
910
|
const explanation = reason ? `${reason} (RESET)` : 'Resetting to initial page';
|
|
928
911
|
const targetUrl = resetUrl!;
|
|
929
912
|
task.addNote(explanation);
|
|
930
|
-
const resetAction = this.explorer.
|
|
913
|
+
const resetAction = this.explorer.action();
|
|
931
914
|
const success = await resetAction.attempt(`I.amOnPage(${JSON.stringify(targetUrl)})`, explanation);
|
|
932
915
|
|
|
933
916
|
if (success) {
|
|
@@ -1076,7 +1059,7 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
1076
1059
|
mappedStatus = TestResult.FAILED;
|
|
1077
1060
|
}
|
|
1078
1061
|
|
|
1079
|
-
const screenshotFile = this.
|
|
1062
|
+
const screenshotFile = this.stateManager.getCurrentState()?.screenshotFile;
|
|
1080
1063
|
|
|
1081
1064
|
for (const noteText of input.notes) {
|
|
1082
1065
|
task.addNote(noteText, mappedStatus, screenshotFile);
|
|
@@ -1117,7 +1100,7 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
1117
1100
|
const message = error instanceof Error ? error.message : String(error);
|
|
1118
1101
|
if (!task.hasFinished) task.addNote(`Execution error: ${message}`);
|
|
1119
1102
|
|
|
1120
|
-
const result = await this.explorer.
|
|
1103
|
+
const result = await this.explorer.recover(error);
|
|
1121
1104
|
tag('info').log(`Browser supervisor: ${result.action} - ${result.message}`);
|
|
1122
1105
|
task.addNote(result.message);
|
|
1123
1106
|
|
|
@@ -1152,7 +1135,7 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
1152
1135
|
|
|
1153
1136
|
private async cleanupStartedTest(task: Test): Promise<void> {
|
|
1154
1137
|
await this.finishTest(task);
|
|
1155
|
-
await this.
|
|
1138
|
+
await this.testRun?.stop({
|
|
1156
1139
|
startUrl: task.startUrl,
|
|
1157
1140
|
style: task.style,
|
|
1158
1141
|
sessionName: task.sessionName,
|
package/src/ai/tools.ts
CHANGED
|
@@ -3,13 +3,13 @@ import dedent from 'dedent';
|
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
import { ActionResult, type PageDiff, type ToolResultMetadata } from '../action-result.ts';
|
|
5
5
|
import type { ExperienceTracker } from '../experience-tracker.ts';
|
|
6
|
-
import type Explorer from '../explorer.ts';
|
|
7
6
|
import { type Task, TestResult } from '../test-plan.js';
|
|
8
7
|
import { LARGE_ARIA_CHANGE_THRESHOLD, extractFocusedElement } from '../utils/aria.ts';
|
|
9
8
|
import { isFatalBrowserError } from '../utils/browser-errors.ts';
|
|
10
9
|
import { createDebug, tag } from '../utils/logger.js';
|
|
11
10
|
import { pause } from '../utils/loop.js';
|
|
12
11
|
import { WebElement } from '../utils/web-element.ts';
|
|
12
|
+
import type { ToolDeps } from './agent.ts';
|
|
13
13
|
import { Navigator } from './navigator.ts';
|
|
14
14
|
import type { AIProvider } from './provider.ts';
|
|
15
15
|
import { Researcher } from './researcher.ts';
|
|
@@ -18,11 +18,16 @@ import { isInteractive } from './task-agent.ts';
|
|
|
18
18
|
|
|
19
19
|
const debugLog = createDebug('explorbot:tools');
|
|
20
20
|
|
|
21
|
-
|
|
21
|
+
interface AgentToolDeps extends ToolDeps {
|
|
22
|
+
researcher: Researcher;
|
|
23
|
+
navigator: Navigator;
|
|
24
|
+
supervisor?: boolean;
|
|
25
|
+
withExperience?: boolean;
|
|
26
|
+
}
|
|
22
27
|
|
|
23
|
-
export
|
|
24
|
-
const stateManager = explorer.getStateManager();
|
|
28
|
+
export const ASSERTION_TOOLS = ['verify'] as const;
|
|
25
29
|
|
|
30
|
+
export function createCodeceptJSTools({ explorer, stateManager, ai }: ToolDeps, task: Task) {
|
|
26
31
|
return {
|
|
27
32
|
click: tool({
|
|
28
33
|
description: dedent`
|
|
@@ -81,7 +86,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
|
|
|
81
86
|
});
|
|
82
87
|
|
|
83
88
|
const previousState = ActionResult.fromState(stateManager.getCurrentState()!);
|
|
84
|
-
const action = explorer.
|
|
89
|
+
const action = explorer.action();
|
|
85
90
|
const attempts: Array<{ command: string; success: boolean; error?: string }> = [];
|
|
86
91
|
|
|
87
92
|
for (let i = 0; i < commands.length; i++) {
|
|
@@ -101,7 +106,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
|
|
|
101
106
|
|
|
102
107
|
let disambiguated = null;
|
|
103
108
|
if (attempts.some((a) => a.error?.toLowerCase().includes(MULTIPLE_ELEMENTS_PATTERN))) {
|
|
104
|
-
disambiguated = await disambiguateElements(action.lastError, explanation,
|
|
109
|
+
disambiguated = await disambiguateElements(action.lastError, explanation, ai);
|
|
105
110
|
}
|
|
106
111
|
|
|
107
112
|
if (disambiguated) {
|
|
@@ -193,7 +198,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
|
|
|
193
198
|
});
|
|
194
199
|
|
|
195
200
|
const previousState = ActionResult.fromState(stateManager.getCurrentState()!);
|
|
196
|
-
const action = explorer.
|
|
201
|
+
const action = explorer.action();
|
|
197
202
|
const attempts: Array<{ command: string; success: boolean; error?: string }> = [];
|
|
198
203
|
|
|
199
204
|
for (const command of commands) {
|
|
@@ -263,7 +268,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
|
|
|
263
268
|
|
|
264
269
|
if (!isSingleChar && !isStandardKey) {
|
|
265
270
|
const previousState = ActionResult.fromState(stateManager.getCurrentState()!);
|
|
266
|
-
const action = explorer.
|
|
271
|
+
const action = explorer.action();
|
|
267
272
|
const typeCommand = `I.type(${JSON.stringify(key)})`;
|
|
268
273
|
await action.attempt(typeCommand, explanation);
|
|
269
274
|
const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, key);
|
|
@@ -306,7 +311,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
|
|
|
306
311
|
}
|
|
307
312
|
|
|
308
313
|
const previousState = ActionResult.fromState(stateManager.getCurrentState()!);
|
|
309
|
-
const action = explorer.
|
|
314
|
+
const action = explorer.action();
|
|
310
315
|
|
|
311
316
|
let pressKeyCommand: string;
|
|
312
317
|
if (modifier) {
|
|
@@ -406,14 +411,9 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
|
|
|
406
411
|
|
|
407
412
|
const previousState = ActionResult.fromState(stateManager.getCurrentState()!);
|
|
408
413
|
const formLocator = codeLines[0] || 'form';
|
|
409
|
-
const action = explorer.
|
|
410
|
-
const wasInIframe = await explorer.isInsideIframe();
|
|
414
|
+
const action = explorer.action();
|
|
411
415
|
await action.attempt(codeBlock, explanation);
|
|
412
416
|
|
|
413
|
-
if (action.lastError && !wasInIframe && (await explorer.isInsideIframe())) {
|
|
414
|
-
await explorer.switchToMainFrame();
|
|
415
|
-
}
|
|
416
|
-
|
|
417
417
|
const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, formLocator);
|
|
418
418
|
|
|
419
419
|
if (action.lastError) {
|
|
@@ -422,7 +422,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
|
|
|
422
422
|
|
|
423
423
|
let formSuggestion = 'Look into error message and identify which commands passed and which failed. Continue execution using step-by-step approach using click() and form() tools.';
|
|
424
424
|
if (message.toLowerCase().includes(MULTIPLE_ELEMENTS_PATTERN)) {
|
|
425
|
-
const disambiguated = await disambiguateElements(action.lastError, explanation,
|
|
425
|
+
const disambiguated = await disambiguateElements(action.lastError, explanation, ai);
|
|
426
426
|
if (disambiguated) {
|
|
427
427
|
formSuggestion = `Multiple elements matched. Add step.opts({ elementIndex: ${disambiguated.position} }) to the failing command. Fallback locator: ${disambiguated.xpath}`;
|
|
428
428
|
}
|
|
@@ -471,9 +471,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
|
|
|
471
471
|
};
|
|
472
472
|
}
|
|
473
473
|
|
|
474
|
-
export function createIframeTools(explorer:
|
|
475
|
-
const stateManager = explorer.getStateManager();
|
|
476
|
-
|
|
474
|
+
export function createIframeTools({ explorer, stateManager }: ToolDeps) {
|
|
477
475
|
return {
|
|
478
476
|
exitIframe: tool({
|
|
479
477
|
description: dedent`
|
|
@@ -495,9 +493,9 @@ export function createIframeTools(explorer: Explorer) {
|
|
|
495
493
|
});
|
|
496
494
|
}
|
|
497
495
|
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
const nextState =
|
|
496
|
+
const action = explorer.action();
|
|
497
|
+
await action.execute('I.switchTo()');
|
|
498
|
+
const nextState = action.getActionResult()!;
|
|
501
499
|
const toolResult = await nextState.toToolResult(previousState, 'I.switchTo()');
|
|
502
500
|
|
|
503
501
|
return successToolResult('exitIframe', {
|
|
@@ -540,19 +538,7 @@ export function createLearnExperienceTool({ getExperienceTracker, getState }: {
|
|
|
540
538
|
});
|
|
541
539
|
}
|
|
542
540
|
|
|
543
|
-
export function createAgentTools({
|
|
544
|
-
explorer,
|
|
545
|
-
researcher,
|
|
546
|
-
navigator,
|
|
547
|
-
supervisor,
|
|
548
|
-
withExperience,
|
|
549
|
-
}: {
|
|
550
|
-
explorer: Explorer;
|
|
551
|
-
researcher: Researcher;
|
|
552
|
-
navigator: Navigator;
|
|
553
|
-
supervisor?: boolean;
|
|
554
|
-
withExperience?: boolean;
|
|
555
|
-
}): any {
|
|
541
|
+
export function createAgentTools({ explorer, stateManager, ai, researcher, navigator, supervisor, withExperience }: AgentToolDeps): any {
|
|
556
542
|
let visionDisabled = false;
|
|
557
543
|
|
|
558
544
|
const tools: Record<string, any> = {
|
|
@@ -577,7 +563,7 @@ export function createAgentTools({
|
|
|
577
563
|
}
|
|
578
564
|
|
|
579
565
|
try {
|
|
580
|
-
const actionResult = await explorer.
|
|
566
|
+
const actionResult = await explorer.capture({ screenshot: true });
|
|
581
567
|
|
|
582
568
|
if (!actionResult.screenshot) {
|
|
583
569
|
return failedToolResult('see', 'Failed to capture screenshot for analysis');
|
|
@@ -625,7 +611,6 @@ export function createAgentTools({
|
|
|
625
611
|
}),
|
|
626
612
|
execute: async ({ reason }) => {
|
|
627
613
|
try {
|
|
628
|
-
const stateManager = explorer.getStateManager();
|
|
629
614
|
const currentState = stateManager.getCurrentState();
|
|
630
615
|
|
|
631
616
|
if (!currentState) {
|
|
@@ -666,7 +651,7 @@ export function createAgentTools({
|
|
|
666
651
|
}),
|
|
667
652
|
execute: async ({ assertion }) => {
|
|
668
653
|
try {
|
|
669
|
-
const currentState =
|
|
654
|
+
const currentState = stateManager.getCurrentState();
|
|
670
655
|
const verifications = currentState?.verifications;
|
|
671
656
|
|
|
672
657
|
if (verifications?.[assertion] !== undefined) {
|
|
@@ -677,7 +662,7 @@ export function createAgentTools({
|
|
|
677
662
|
});
|
|
678
663
|
}
|
|
679
664
|
|
|
680
|
-
const actionResult = await explorer.
|
|
665
|
+
const actionResult = await explorer.capture();
|
|
681
666
|
const result = await navigator.verifyState(assertion, actionResult);
|
|
682
667
|
|
|
683
668
|
if (result.verified) {
|
|
@@ -727,7 +712,6 @@ export function createAgentTools({
|
|
|
727
712
|
}),
|
|
728
713
|
execute: async ({ reason }) => {
|
|
729
714
|
try {
|
|
730
|
-
const stateManager = explorer.getStateManager();
|
|
731
715
|
const currentState = stateManager.getCurrentState();
|
|
732
716
|
|
|
733
717
|
if (!currentState) {
|
|
@@ -771,7 +755,6 @@ export function createAgentTools({
|
|
|
771
755
|
}),
|
|
772
756
|
execute: async ({ instruction }) => {
|
|
773
757
|
try {
|
|
774
|
-
const stateManager = explorer.getStateManager();
|
|
775
758
|
const currentState = stateManager.getCurrentState();
|
|
776
759
|
|
|
777
760
|
if (!currentState) {
|
|
@@ -826,7 +809,6 @@ export function createAgentTools({
|
|
|
826
809
|
}
|
|
827
810
|
|
|
828
811
|
try {
|
|
829
|
-
const stateManager = explorer.getStateManager();
|
|
830
812
|
const currentState = stateManager.getCurrentState();
|
|
831
813
|
|
|
832
814
|
if (!currentState) {
|
|
@@ -834,8 +816,8 @@ export function createAgentTools({
|
|
|
834
816
|
}
|
|
835
817
|
|
|
836
818
|
const previousState = ActionResult.fromState(currentState);
|
|
837
|
-
const action = explorer.
|
|
838
|
-
const actionResult = await explorer.
|
|
819
|
+
const action = explorer.action();
|
|
820
|
+
const actionResult = await explorer.capture({ screenshot: true });
|
|
839
821
|
|
|
840
822
|
if (!actionResult.screenshot) {
|
|
841
823
|
return failedToolResult('visualClick', 'Failed to capture screenshot for visual analysis');
|
|
@@ -897,7 +879,6 @@ export function createAgentTools({
|
|
|
897
879
|
reason: z.string().describe('Why you need to go back'),
|
|
898
880
|
}),
|
|
899
881
|
execute: async ({ reason }) => {
|
|
900
|
-
const stateManager = explorer.getStateManager();
|
|
901
882
|
const currentState = stateManager.getCurrentState();
|
|
902
883
|
const currentUrl = currentState?.fullUrl || currentState?.url;
|
|
903
884
|
const history = stateManager.getStateHistory();
|
|
@@ -920,7 +901,7 @@ export function createAgentTools({
|
|
|
920
901
|
let previousState: ActionResult | null = null;
|
|
921
902
|
if (currentState) previousState = ActionResult.fromState(currentState);
|
|
922
903
|
|
|
923
|
-
const action = explorer.
|
|
904
|
+
const action = explorer.action();
|
|
924
905
|
const success = await action.attempt(`I.amOnPage(${JSON.stringify(targetUrl)})`, `${reason} (BACK to ${targetUrl})`);
|
|
925
906
|
|
|
926
907
|
if (success) {
|
|
@@ -942,7 +923,7 @@ export function createAgentTools({
|
|
|
942
923
|
description: 'List all previously visited page states (deduped by URL). Use to find pages to navigate back to.',
|
|
943
924
|
inputSchema: z.object({}),
|
|
944
925
|
execute: async () => {
|
|
945
|
-
const history =
|
|
926
|
+
const history = stateManager.getStateHistory();
|
|
946
927
|
const seen = new Set<string>();
|
|
947
928
|
const states = history
|
|
948
929
|
.map((t) => t.toState)
|
|
@@ -975,7 +956,6 @@ export function createAgentTools({
|
|
|
975
956
|
reason: z.string().describe('What element you are looking for and why'),
|
|
976
957
|
}),
|
|
977
958
|
execute: async ({ xpath, reason }) => {
|
|
978
|
-
const stateManager = explorer.getStateManager();
|
|
979
959
|
const currentState = stateManager.getCurrentState();
|
|
980
960
|
|
|
981
961
|
if (!currentState) {
|
|
@@ -1001,9 +981,9 @@ export function createAgentTools({
|
|
|
1001
981
|
});
|
|
1002
982
|
}
|
|
1003
983
|
|
|
1004
|
-
const action = explorer.
|
|
984
|
+
const action = explorer.action();
|
|
1005
985
|
const visible = await action.attempt(`I.seeElement(${JSON.stringify(xpath)})`, 'xpathCheck visibility');
|
|
1006
|
-
const liveElement = await WebElement.fromPlaywrightLocator(
|
|
986
|
+
const liveElement = await explorer.withPage((page) => WebElement.fromPlaywrightLocator(page.locator(`xpath=${xpath}`)));
|
|
1007
987
|
|
|
1008
988
|
const matchesSummary = result.elements.map((el, i) => `${i + 1}. <${el.tag} ${el.keyAttrs}> text="${el.text}" html: ${el.outerHTML}`).join('\n');
|
|
1009
989
|
|
|
@@ -1033,9 +1013,8 @@ export function createAgentTools({
|
|
|
1033
1013
|
|
|
1034
1014
|
if (withExperience !== false) {
|
|
1035
1015
|
tools.learnExperience = createLearnExperienceTool({
|
|
1036
|
-
getExperienceTracker: () =>
|
|
1016
|
+
getExperienceTracker: () => stateManager.getExperienceTracker(),
|
|
1037
1017
|
getState: () => {
|
|
1038
|
-
const stateManager = explorer.getStateManager();
|
|
1039
1018
|
const currentState = stateManager.getCurrentState();
|
|
1040
1019
|
return currentState ? ActionResult.fromState(currentState) : null;
|
|
1041
1020
|
},
|
|
@@ -1270,7 +1249,7 @@ async function disambiguateElements(error: Error | null | undefined, explanation
|
|
|
1270
1249
|
},
|
|
1271
1250
|
],
|
|
1272
1251
|
schema,
|
|
1273
|
-
provider.
|
|
1252
|
+
provider.getModelForAgent(),
|
|
1274
1253
|
{ agentName: 'disambiguator', timeout: 15000 }
|
|
1275
1254
|
);
|
|
1276
1255
|
|
package/src/command-handler.ts
CHANGED
|
@@ -178,7 +178,7 @@ export class CommandHandler implements InputManager {
|
|
|
178
178
|
}
|
|
179
179
|
|
|
180
180
|
private async executeBrowserCommand(input: string): Promise<void> {
|
|
181
|
-
const action = this.explorBot.getExplorer().
|
|
181
|
+
const action = this.explorBot.getExplorer().action();
|
|
182
182
|
await action.execute(input);
|
|
183
183
|
}
|
|
184
184
|
|
|
@@ -6,7 +6,7 @@ export class ContextAriaCommand extends BaseCommand {
|
|
|
6
6
|
description = 'Print full ARIA snapshot for current page';
|
|
7
7
|
|
|
8
8
|
async execute(_args: string): Promise<void> {
|
|
9
|
-
const state = this.explorBot.
|
|
9
|
+
const state = this.explorBot.stateManager().getCurrentState();
|
|
10
10
|
|
|
11
11
|
if (!state) {
|
|
12
12
|
throw new Error('No active page to snapshot');
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { ActionResult } from '../action-result.js';
|
|
2
2
|
import { Researcher } from '../ai/researcher.js';
|
|
3
|
+
import { visuallyAnnotateContainers } from '../ai/researcher/coordinates.js';
|
|
3
4
|
import { outputPath } from '../config.js';
|
|
4
5
|
import { type ContextData, type ContextMode, formatContextSummary } from '../utils/context-formatter.js';
|
|
5
6
|
import { tag } from '../utils/logger.js';
|
|
6
7
|
import { extractValidContainers } from '../utils/research-parser.js';
|
|
8
|
+
import { annotatePageElements } from '../utils/web-annotate.js';
|
|
7
9
|
import { BaseCommand, type Suggestion } from './base-command.js';
|
|
8
10
|
|
|
9
11
|
export class ContextCommand extends BaseCommand {
|
|
@@ -25,7 +27,7 @@ export class ContextCommand extends BaseCommand {
|
|
|
25
27
|
|
|
26
28
|
async execute(args: string): Promise<void> {
|
|
27
29
|
const explorer = this.explorBot.getExplorer();
|
|
28
|
-
const state =
|
|
30
|
+
const state = this.explorBot.stateManager().getCurrentState();
|
|
29
31
|
|
|
30
32
|
if (!state) {
|
|
31
33
|
throw new Error('No active page to show context for');
|
|
@@ -34,16 +36,16 @@ export class ContextCommand extends BaseCommand {
|
|
|
34
36
|
const { opts } = this.parseArgs(args);
|
|
35
37
|
const isVisual = !!(opts.visual || opts.screenshot);
|
|
36
38
|
|
|
37
|
-
await explorer.
|
|
39
|
+
await explorer.withPage(annotatePageElements);
|
|
38
40
|
|
|
39
41
|
if (isVisual) {
|
|
40
42
|
const cachedResearch = Researcher.getCachedResearch(state);
|
|
41
43
|
const containers = cachedResearch ? extractValidContainers(cachedResearch) : [];
|
|
42
|
-
await explorer.
|
|
44
|
+
await explorer.withPage((page) => visuallyAnnotateContainers(page, containers));
|
|
43
45
|
}
|
|
44
46
|
|
|
45
|
-
const actionResult = await explorer.
|
|
46
|
-
const experienceTracker =
|
|
47
|
+
const actionResult = await explorer.capture({ screenshot: isVisual });
|
|
48
|
+
const experienceTracker = this.explorBot.experienceTracker();
|
|
47
49
|
const knowledgeTracker = this.explorBot.knowledgeTracker();
|
|
48
50
|
|
|
49
51
|
let mode: ContextMode = 'compact';
|
|
@@ -8,7 +8,7 @@ export class ContextDataCommand extends BaseCommand {
|
|
|
8
8
|
|
|
9
9
|
async execute(_args: string): Promise<void> {
|
|
10
10
|
const explorer = this.explorBot.getExplorer();
|
|
11
|
-
const state =
|
|
11
|
+
const state = this.explorBot.stateManager().getCurrentState();
|
|
12
12
|
|
|
13
13
|
if (!state) {
|
|
14
14
|
throw new Error('No active page to extract data from');
|
|
@@ -18,7 +18,7 @@ export class ContextDataCommand extends BaseCommand {
|
|
|
18
18
|
|
|
19
19
|
if (!actionResult.html || actionResult.html.trim().length < 100) {
|
|
20
20
|
tag('info').log('Capturing fresh page content...');
|
|
21
|
-
const freshResult = await explorer.
|
|
21
|
+
const freshResult = await explorer.capture();
|
|
22
22
|
const table = await this.explorBot.agentResearcher().extractData(freshResult);
|
|
23
23
|
tag('multiline').log(table);
|
|
24
24
|
return;
|