explorbot 0.1.27 → 0.1.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +83 -245
- package/bin/explorbot-cli.ts +1 -0
- package/dist/bin/explorbot-cli.js +1 -0
- package/dist/package.json +8 -6
- package/dist/rules/navigator/verification-actions.md +2 -0
- package/dist/src/action-result.js +2 -0
- package/dist/src/action.js +88 -7
- package/dist/src/ai/captain/file-tools.js +100 -0
- package/dist/src/ai/captain/idle-mode.js +70 -6
- package/dist/src/ai/captain/web-mode.js +36 -6
- package/dist/src/ai/captain.js +87 -19
- package/dist/src/ai/fisherman.js +14 -3
- package/dist/src/ai/historian/screencast.js +11 -2
- package/dist/src/ai/navigator.js +5 -2
- package/dist/src/ai/pilot.js +52 -9
- package/dist/src/ai/planner.js +16 -5
- package/dist/src/ai/provider.js +53 -18
- package/dist/src/ai/researcher/coordinates.js +2 -3
- package/dist/src/ai/researcher/deep-analysis.js +3 -4
- package/dist/src/ai/researcher/locators.js +1 -2
- package/dist/src/ai/researcher.js +24 -19
- package/dist/src/ai/rules.js +44 -0
- package/dist/src/ai/task-agent.js +1 -0
- package/dist/src/ai/tester.js +161 -46
- package/dist/src/ai/tools.js +84 -8
- package/dist/src/commands/explore-command.js +6 -1
- package/dist/src/components/LogPane.js +4 -3
- package/dist/src/explorbot.js +7 -2
- package/dist/src/explorer.js +270 -35
- package/dist/src/stats.js +16 -0
- package/dist/src/utils/aria.js +66 -6
- package/dist/src/utils/browser-errors.js +23 -0
- package/dist/src/utils/error-page.js +17 -2
- package/dist/src/utils/logger.js +2 -2
- package/package.json +8 -6
- package/rules/navigator/verification-actions.md +2 -0
- package/src/action-result.ts +2 -0
- package/src/action.ts +83 -7
- package/src/ai/captain/file-tools.ts +126 -0
- package/src/ai/captain/idle-mode.ts +72 -6
- package/src/ai/captain/mixin.ts +1 -1
- package/src/ai/captain/web-mode.ts +40 -5
- package/src/ai/captain.ts +94 -20
- package/src/ai/fisherman.ts +14 -3
- package/src/ai/historian/screencast.ts +11 -2
- package/src/ai/navigator.ts +6 -2
- package/src/ai/pilot.ts +53 -9
- package/src/ai/planner.ts +16 -5
- package/src/ai/provider.ts +51 -19
- package/src/ai/researcher/coordinates.ts +2 -3
- package/src/ai/researcher/deep-analysis.ts +3 -4
- package/src/ai/researcher/locators.ts +1 -2
- package/src/ai/researcher.ts +25 -19
- package/src/ai/rules.ts +46 -0
- package/src/ai/task-agent.ts +1 -1
- package/src/ai/tester.ts +175 -48
- package/src/ai/tools.ts +97 -8
- package/src/commands/explore-command.ts +6 -1
- package/src/components/LogPane.tsx +4 -3
- package/src/config.ts +1 -0
- package/src/explorbot.ts +6 -2
- package/src/explorer.ts +295 -38
- package/src/state-manager.ts +2 -0
- package/src/stats.ts +18 -0
- package/src/utils/aria.ts +63 -6
- package/src/utils/browser-errors.ts +25 -0
- package/src/utils/error-page.ts +16 -3
- package/src/utils/logger.ts +3 -3
package/src/ai/tester.ts
CHANGED
|
@@ -4,7 +4,7 @@ import { tool } from 'ai';
|
|
|
4
4
|
import dedent from 'dedent';
|
|
5
5
|
import { z } from 'zod';
|
|
6
6
|
import { ActionResult } from '../action-result.ts';
|
|
7
|
-
import { setActivity } from '../activity.ts';
|
|
7
|
+
import { clearActivity, setActivity } from '../activity.ts';
|
|
8
8
|
import { ConfigParser } from '../config.ts';
|
|
9
9
|
import type { ExperienceTracker } from '../experience-tracker.ts';
|
|
10
10
|
import type Explorer from '../explorer.ts';
|
|
@@ -13,7 +13,7 @@ import type { StateTransition, WebPageState } from '../state-manager.ts';
|
|
|
13
13
|
import { Stats } from '../stats.ts';
|
|
14
14
|
import { type Note, type Test, TestResult, type TestResultType } from '../test-plan.ts';
|
|
15
15
|
import { detectFocusArea, extractFocusedElement } from '../utils/aria.ts';
|
|
16
|
-
import { ErrorPageError } from '../utils/error-page.ts';
|
|
16
|
+
import { ErrorPageError, isErrorPage } from '../utils/error-page.ts';
|
|
17
17
|
import { HooksRunner } from '../utils/hooks-runner.ts';
|
|
18
18
|
import { codeToMarkdown } from '../utils/html.ts';
|
|
19
19
|
import { createDebug, tag } from '../utils/logger.ts';
|
|
@@ -25,7 +25,7 @@ import { Navigator } from './navigator.ts';
|
|
|
25
25
|
import type { Pilot } from './pilot.ts';
|
|
26
26
|
import { Provider } from './provider.ts';
|
|
27
27
|
import { Researcher } from './researcher.ts';
|
|
28
|
-
import { actionRule, focusedElementRule, formRequirementsRule, locatorRule, multipleTabsRule,
|
|
28
|
+
import { actionRule, capabilityGroundingRule, dataProtectionRules, focusedElementRule, formRequirementsRule, locatorRule, multipleTabsRule, sectionContextRule } from './rules.ts';
|
|
29
29
|
import { TaskAgent } from './task-agent.ts';
|
|
30
30
|
import { createCodeceptJSTools, createSpecialContextTools } from './tools.ts';
|
|
31
31
|
|
|
@@ -43,7 +43,7 @@ const SAMPLE_FILES: Record<string, string> = {
|
|
|
43
43
|
};
|
|
44
44
|
|
|
45
45
|
export class Tester extends TaskAgent implements Agent {
|
|
46
|
-
protected readonly ACTION_TOOLS = ['click', 'pressKey', 'form'];
|
|
46
|
+
protected readonly ACTION_TOOLS = ['click', 'hover', 'pressKey', 'form'];
|
|
47
47
|
protected readonly SPECIAL_CONTEXT_ACTION_TOOLS = ['exitIframe'];
|
|
48
48
|
emoji = '🧪';
|
|
49
49
|
private explorer: Explorer;
|
|
@@ -66,6 +66,8 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
66
66
|
private hooksRunner: HooksRunner;
|
|
67
67
|
private seenUiMapUrls = new Set<string>();
|
|
68
68
|
private lastAnalyzedStateHash: string | null = null;
|
|
69
|
+
private stalledIterations = 0;
|
|
70
|
+
private readonly MAX_STALLED_ITERATIONS = 3;
|
|
69
71
|
|
|
70
72
|
constructor(explorer: Explorer, provider: Provider, researcher: Researcher, navigator: Navigator, agentTools?: any) {
|
|
71
73
|
super();
|
|
@@ -126,6 +128,7 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
126
128
|
this.pageActionResult = null;
|
|
127
129
|
this.seenUiMapUrls.clear();
|
|
128
130
|
this.lastAnalyzedStateHash = null;
|
|
131
|
+
this.stalledIterations = 0;
|
|
129
132
|
this.explorer.getStateManager().clearHistory();
|
|
130
133
|
this.resetFailureCount();
|
|
131
134
|
this.pilot?.reset();
|
|
@@ -136,18 +139,13 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
136
139
|
task.addNote(`Network error: ${r.method} ${r.path} → ${r.status}`, TestResult.FAILED);
|
|
137
140
|
});
|
|
138
141
|
|
|
139
|
-
const page = this.explorer.playwrightHelper?.page;
|
|
140
|
-
const onPageError = (err: Error) => {
|
|
141
|
-
task.addNote(`Console error: ${err.message}`, TestResult.FAILED);
|
|
142
|
-
};
|
|
143
|
-
const onConsoleMessage = (msg: any) => {
|
|
144
|
-
if (msg.type() !== 'error') return;
|
|
145
|
-
task.addNote(`Console error: ${msg.text()}`, TestResult.FAILED);
|
|
146
|
-
};
|
|
147
|
-
page?.on('pageerror', onPageError);
|
|
148
|
-
page?.on('console', onConsoleMessage);
|
|
149
|
-
|
|
150
142
|
const initialState = ActionResult.fromState(state);
|
|
143
|
+
if (isErrorPage(initialState)) {
|
|
144
|
+
task.start();
|
|
145
|
+
await this.explorer.startTest(task);
|
|
146
|
+
offFailedRequest?.();
|
|
147
|
+
return await this.abortStartedTestOnErrorPage(task, initialState);
|
|
148
|
+
}
|
|
151
149
|
|
|
152
150
|
const conversation = this.provider.startConversation(this.getSystemMessage(), 'tester');
|
|
153
151
|
conversation.markLastMessageCacheable();
|
|
@@ -176,20 +174,18 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
176
174
|
expected: task.expected,
|
|
177
175
|
},
|
|
178
176
|
},
|
|
179
|
-
async () => this.runTestSession(task, initialState, conversation, { offFailedRequest
|
|
177
|
+
async () => this.runTestSession(task, initialState, conversation, { offFailedRequest })
|
|
180
178
|
);
|
|
181
179
|
}
|
|
182
180
|
|
|
183
|
-
private async runTestSession(task: Test, initialState: ActionResult, conversation: Conversation, handlers:
|
|
184
|
-
const { offFailedRequest
|
|
181
|
+
private async runTestSession(task: Test, initialState: ActionResult, conversation: Conversation, handlers: TestSessionHandlers): Promise<{ success: boolean }> {
|
|
182
|
+
const { offFailedRequest } = handlers;
|
|
185
183
|
|
|
186
184
|
if (this.pilot) {
|
|
187
185
|
try {
|
|
188
186
|
const plan = await this.pilot.planTest(task, initialState);
|
|
189
187
|
if (task.hasFinished) {
|
|
190
188
|
offFailedRequest?.();
|
|
191
|
-
page?.off('pageerror', onPageError);
|
|
192
|
-
page?.off('console', onConsoleMessage);
|
|
193
189
|
return { success: task.isSuccessful };
|
|
194
190
|
}
|
|
195
191
|
if (plan) {
|
|
@@ -201,22 +197,39 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
201
197
|
task.addNote(`Planning failed: ${message}`, TestResult.FAILED);
|
|
202
198
|
task.finish(TestResult.FAILED);
|
|
203
199
|
offFailedRequest?.();
|
|
204
|
-
page?.off('pageerror', onPageError);
|
|
205
|
-
page?.off('console', onConsoleMessage);
|
|
206
200
|
return { success: false };
|
|
207
201
|
}
|
|
208
202
|
}
|
|
209
203
|
|
|
210
204
|
debugLog('Starting test execution with tools');
|
|
211
205
|
|
|
212
|
-
|
|
213
|
-
|
|
206
|
+
if (!(await this.explorer.startTest(task))) {
|
|
207
|
+
offFailedRequest?.();
|
|
208
|
+
await this.cleanupStartedTest(task);
|
|
209
|
+
return { success: task.isSuccessful };
|
|
210
|
+
}
|
|
214
211
|
|
|
215
212
|
debugLog(`Navigating to ${task.startUrl}`);
|
|
216
|
-
|
|
213
|
+
try {
|
|
214
|
+
await this.explorer.visit(task.startUrl!);
|
|
215
|
+
} catch (error) {
|
|
216
|
+
const result = await this.handleLoopError(task, error);
|
|
217
|
+
if (result === 'stop') {
|
|
218
|
+
offFailedRequest?.();
|
|
219
|
+
await this.cleanupStartedTest(task);
|
|
220
|
+
return { success: task.isSuccessful };
|
|
221
|
+
}
|
|
222
|
+
}
|
|
217
223
|
|
|
218
224
|
const startState = this.explorer.getStateManager().getCurrentState();
|
|
219
|
-
if (startState)
|
|
225
|
+
if (startState) {
|
|
226
|
+
task.addUrlNote(startState);
|
|
227
|
+
const startActionResult = ActionResult.fromState(startState);
|
|
228
|
+
if (isErrorPage(startActionResult)) {
|
|
229
|
+
offFailedRequest?.();
|
|
230
|
+
return await this.abortStartedTestOnErrorPage(task, startActionResult);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
220
233
|
const currentUrl = startState?.url || task.startUrl || '';
|
|
221
234
|
await this.hooksRunner.runBeforeHook('tester', currentUrl);
|
|
222
235
|
|
|
@@ -238,6 +251,12 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
238
251
|
await loop(
|
|
239
252
|
async ({ stop, pause, iteration, userInput }) => {
|
|
240
253
|
debugLog('iteration', iteration);
|
|
254
|
+
if (!(await this.explorer.ensurePageAvailable())) {
|
|
255
|
+
task.addNote('Browser page is unavailable');
|
|
256
|
+
task.finish(TestResult.FAILED);
|
|
257
|
+
stop();
|
|
258
|
+
return;
|
|
259
|
+
}
|
|
241
260
|
const currentState = this.getCurrentState();
|
|
242
261
|
|
|
243
262
|
const tools = {
|
|
@@ -332,6 +351,11 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
332
351
|
});
|
|
333
352
|
}
|
|
334
353
|
|
|
354
|
+
if (this.shouldStopForStalledExecution(task, currentState, result?.toolExecutions || [])) {
|
|
355
|
+
stop();
|
|
356
|
+
return;
|
|
357
|
+
}
|
|
358
|
+
|
|
335
359
|
if (assertionPerformed) {
|
|
336
360
|
const message = result?.toolExecutions?.find((execution: any) => execution.toolName === 'verify')?.output?.message || '';
|
|
337
361
|
task.addNote(message, wasSuccessful ? TestResult.PASSED : TestResult.FAILED);
|
|
@@ -385,22 +409,16 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
385
409
|
}
|
|
386
410
|
: undefined,
|
|
387
411
|
catch: async ({ error, stop }) => {
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
if (!task.hasFinished) {
|
|
391
|
-
task.addNote(`Execution error: ${message}`);
|
|
392
|
-
}
|
|
393
|
-
if (error instanceof Error && error.name === 'AbortError') {
|
|
394
|
-
stop();
|
|
395
|
-
return;
|
|
396
|
-
}
|
|
397
|
-
conversation.addUserText(`Previous AI call failed: ${message}. Take a different approach on the next step.`);
|
|
412
|
+
const result = await this.handleLoopError(task, error);
|
|
413
|
+
if (result === 'stop') stop();
|
|
398
414
|
},
|
|
399
415
|
}
|
|
400
416
|
);
|
|
401
417
|
|
|
402
418
|
if (task.hasFinished) break;
|
|
403
419
|
|
|
420
|
+
if (!(await this.explorer.ensurePageAvailable())) break;
|
|
421
|
+
|
|
404
422
|
const finalState = this.getCurrentState();
|
|
405
423
|
const wantsContinue = await this.pilot!.finalReview(task, finalState, conversation, this.navigator);
|
|
406
424
|
|
|
@@ -429,14 +447,8 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
429
447
|
|
|
430
448
|
offStateChange();
|
|
431
449
|
offFailedRequest?.();
|
|
432
|
-
page?.off('pageerror', onPageError);
|
|
433
|
-
page?.off('console', onConsoleMessage);
|
|
434
450
|
await this.finishTest(task);
|
|
435
|
-
await this.explorer.stopTest(task,
|
|
436
|
-
startUrl: task.startUrl,
|
|
437
|
-
style: task.style,
|
|
438
|
-
sessionName: task.sessionName,
|
|
439
|
-
});
|
|
451
|
+
await this.explorer.stopTest(task, this.buildStopTestMeta(task));
|
|
440
452
|
|
|
441
453
|
return {
|
|
442
454
|
success: task.isSuccessful,
|
|
@@ -452,6 +464,31 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
452
464
|
return true;
|
|
453
465
|
}
|
|
454
466
|
|
|
467
|
+
private shouldStopForStalledExecution(task: Test, previousState: ActionResult, toolExecutions: any[]): boolean {
|
|
468
|
+
if (task.hasFinished) return false;
|
|
469
|
+
|
|
470
|
+
const currentState = this.getCurrentState();
|
|
471
|
+
const stateChanged = previousState.url !== currentState.url || previousState.hash !== currentState.hash;
|
|
472
|
+
const actionTools = [...this.ACTION_TOOLS, ...this.SPECIAL_CONTEXT_ACTION_TOOLS];
|
|
473
|
+
const hasSuccessfulAction = toolExecutions.some((execution) => execution.wasSuccessful && actionTools.includes(execution.toolName));
|
|
474
|
+
const hasSuccessfulAssertion = toolExecutions.some((execution) => execution.wasSuccessful && this.ASSERTION_TOOLS.includes(execution.toolName));
|
|
475
|
+
|
|
476
|
+
if (stateChanged || hasSuccessfulAction || hasSuccessfulAssertion) {
|
|
477
|
+
this.stalledIterations = 0;
|
|
478
|
+
return false;
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
const hasNoBrowserProgress = toolExecutions.length === 0 || toolExecutions.every((execution) => !actionTools.includes(execution.toolName) || !execution.wasSuccessful);
|
|
482
|
+
if (!hasNoBrowserProgress) return false;
|
|
483
|
+
|
|
484
|
+
this.stalledIterations++;
|
|
485
|
+
if (this.stalledIterations < this.MAX_STALLED_ITERATIONS) return false;
|
|
486
|
+
|
|
487
|
+
task.addNote('No browser progress after repeated attempts on unchanged page', TestResult.FAILED);
|
|
488
|
+
task.finish(TestResult.FAILED);
|
|
489
|
+
return true;
|
|
490
|
+
}
|
|
491
|
+
|
|
455
492
|
private async prepareInstructionsForNextStep(task: Test): Promise<string> {
|
|
456
493
|
let outcomeStatus = dedent`
|
|
457
494
|
<task>
|
|
@@ -460,6 +497,8 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
460
497
|
|
|
461
498
|
<rules>
|
|
462
499
|
Use tools ${this.ACTION_TOOLS.join(', ')} to interact with the page.
|
|
500
|
+
Use tool names exactly as listed in this prompt. Do not invent combined tool names, aliases, or names with channel markers such as "commentary".
|
|
501
|
+
Match each tool input schema exactly. Do not invent parameter names or pass extra fields.
|
|
463
502
|
Do not do unsuccesful clicks again.
|
|
464
503
|
Do not run same tool calls with same parameters again.
|
|
465
504
|
</rules>
|
|
@@ -689,6 +728,26 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
689
728
|
}
|
|
690
729
|
}
|
|
691
730
|
|
|
731
|
+
private async abortStartedTestOnErrorPage(task: Test, actionResult: ActionResult): Promise<{ success: boolean }> {
|
|
732
|
+
const error = new ErrorPageError(actionResult.url || task.startUrl || '', actionResult.title, actionResult.httpStatus);
|
|
733
|
+
tag('warning').log(error.message);
|
|
734
|
+
task.addNote(error.message, TestResult.FAILED, actionResult.screenshotFile, actionResult.fullUrl || actionResult.url);
|
|
735
|
+
task.finish(TestResult.FAILED);
|
|
736
|
+
this.finishTest(task);
|
|
737
|
+
await this.explorer.stopTest(task, this.buildStopTestMeta(task));
|
|
738
|
+
clearActivity(true);
|
|
739
|
+
return { success: false };
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
private buildStopTestMeta(task: Test): Record<string, string> {
|
|
743
|
+
const meta: Record<string, string> = {
|
|
744
|
+
startUrl: task.startUrl,
|
|
745
|
+
};
|
|
746
|
+
if (task.style) meta.style = task.style;
|
|
747
|
+
if (task.sessionName) meta.sessionName = task.sessionName;
|
|
748
|
+
return meta;
|
|
749
|
+
}
|
|
750
|
+
|
|
692
751
|
getSystemMessage(): string {
|
|
693
752
|
return dedent`
|
|
694
753
|
<role>
|
|
@@ -722,6 +781,7 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
722
781
|
<rules>
|
|
723
782
|
- Refer to UI Map from <page_ui_map> to understand the page structure and its main elements
|
|
724
783
|
- Use only elements that exist in the provided ARIA tree or HTML, <page_aria> and <page_html>
|
|
784
|
+
- Use tool input schemas exactly as documented. Do not invent parameter names or add fields not listed by the tool schema.
|
|
725
785
|
- Use click() for buttons, links, and clickable elements ONLY - do NOT include I.fillField() or I.type() commands in click() tool
|
|
726
786
|
- click() commands array is for FALLBACK LOCATORS of the SAME element, NOT for clicking different elements in sequence. If you need to click two different elements, make two separate click() calls.
|
|
727
787
|
- Use form() for text input (I.fillField, I.type), dropdown selection (I.selectOption), file uploads (I.attachFile), and multi-step form interactions
|
|
@@ -734,6 +794,7 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
734
794
|
- NEVER call record(status: "success") if your last verify() or see() call FAILED. A failed check means the outcome is NOT confirmed — use record(status: "fail") instead, or retry with a different approach.
|
|
735
795
|
- Use finish() to complete the test, not record(). record() is for intermediate notes.
|
|
736
796
|
- Call finish(verify) when all goals are achieved — provide an assertion to verify
|
|
797
|
+
- NEVER call finish() with a negative assertion that says the goal did NOT happen. If the goal cannot be achieved after real attempts, record the blocker and call stop().
|
|
737
798
|
- ONLY call stop() if the scenario itself is completely irrelevant to this page and no expectations can be achieved
|
|
738
799
|
- Use reset() ONLY as a last resort when the current page cannot host the scenario. Never reset after a successful flow just because an assertion or milestone did not match — verify differently or record() the finding instead. Reset is destructive and does not undo server-side side effects.
|
|
739
800
|
- Be precise with locators (CSS or XPath)
|
|
@@ -747,6 +808,13 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
747
808
|
- When you interact with form with inputs, ensure that you click corresponding button to save its data
|
|
748
809
|
- Follow <locator_priority> rules when selecting locators for all tools
|
|
749
810
|
- Before retrying your actions check maybe they already achived expected results. Use see() tool for that
|
|
811
|
+
- If the current URL is already a create/edit/new form and the scenario is about creating/editing that entity, fill and submit that form. Do not click the list-page "New" button again from inside the form.
|
|
812
|
+
- If the scenario is about search/filter/sort/tabs/list inspection and the current URL is a create/edit/new form, go back or reset to the stable list page before interacting with list controls.
|
|
813
|
+
- When selecting related entities from a list, do not choose rows/options/cards marked as "0 items", "0 tests", or otherwise empty if the scenario requires selecting real content.
|
|
814
|
+
- In selection pickers, counters such as "Selected 0", "Matched tests 0", or disabled Save/Apply mean the selection did not register. Choose a non-empty item or change filters before submitting.
|
|
815
|
+
- A passed form/click command only means the command executed. If a required field remains empty, submit stays disabled, or the expected text is not visible, treat the action as not completed and correct the missing field/state.
|
|
816
|
+
- For filter/tab scenarios, success requires BOTH: the requested filter/tab is visibly active/selected AND the list content matches that filter. Do not finish from only one of these signals.
|
|
817
|
+
- Empty-state text such as "No matched items" only proves a filter when the requested filter/tab is active and the empty state belongs to the filtered list.
|
|
750
818
|
- When filling complex form with lot of actions performed, use see() to look which fields were filled and which are not
|
|
751
819
|
- When verify() fails, use see() to visually confirm the result — visual confirmation is equally valid evidence
|
|
752
820
|
- For visual state verification (active tabs, selected items, counts, colors), prefer see() over DOM-based verify()
|
|
@@ -775,6 +843,10 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
775
843
|
|
|
776
844
|
${formRequirementsRule}
|
|
777
845
|
|
|
846
|
+
${capabilityGroundingRule}
|
|
847
|
+
|
|
848
|
+
${dataProtectionRules}
|
|
849
|
+
|
|
778
850
|
${this.provider.getSystemPromptForAgent('tester', this.explorer.getStateManager().getCurrentState()?.url) || ''}
|
|
779
851
|
`;
|
|
780
852
|
}
|
|
@@ -798,10 +870,13 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
798
870
|
Try to achieve as many goals as possible.
|
|
799
871
|
If goal is not achievable, log that and skip to next one.
|
|
800
872
|
Do not hallucinate that goal was achieved when it was not.
|
|
873
|
+
If the scenario action could not be completed, do not finish with a verification of the failure state.
|
|
801
874
|
When creating or editing items via form() or type() you should include ${task.sessionName} in the value (if it is not restricted by the application logic)
|
|
802
875
|
Initial page URL: ${actionResult.url}
|
|
803
876
|
|
|
804
|
-
${
|
|
877
|
+
${capabilityGroundingRule}
|
|
878
|
+
|
|
879
|
+
${dataProtectionRules}
|
|
805
880
|
|
|
806
881
|
${this.buildDeletionScope(task)}
|
|
807
882
|
|
|
@@ -939,12 +1014,13 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
939
1014
|
}),
|
|
940
1015
|
stop: tool({
|
|
941
1016
|
description: dedent`
|
|
942
|
-
Stop the current test because
|
|
943
|
-
Use this
|
|
944
|
-
|
|
1017
|
+
Stop the current test because it cannot be completed in the current session.
|
|
1018
|
+
Use this when the scenario is incompatible, required UI/data is absent, or repeated varied attempts
|
|
1019
|
+
show that automation cannot complete the workflow.
|
|
1020
|
+
Do NOT use this immediately after the first failed action — retry with a materially different approach first.
|
|
945
1021
|
`,
|
|
946
1022
|
inputSchema: z.object({
|
|
947
|
-
reason: z.string().describe('Explanation why the scenario
|
|
1023
|
+
reason: z.string().describe('Explanation why the scenario cannot be completed'),
|
|
948
1024
|
}),
|
|
949
1025
|
execute: async ({ reason }) => {
|
|
950
1026
|
task.addNote(`Stop requested: ${reason}`);
|
|
@@ -979,6 +1055,7 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
979
1055
|
Provide a specific assertion to verify the final state.
|
|
980
1056
|
The assertion MUST prove that YOUR ACTIONS changed the page state.
|
|
981
1057
|
Do NOT verify something that was already true before you started testing.
|
|
1058
|
+
Do NOT provide an assertion that verifies absence, failure, an empty state, or that the goal did not happen.
|
|
982
1059
|
|
|
983
1060
|
Examples of good assertions:
|
|
984
1061
|
- "New user 'john@example.com' is visible in the users list"
|
|
@@ -1093,4 +1170,54 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
1093
1170
|
}),
|
|
1094
1171
|
};
|
|
1095
1172
|
}
|
|
1173
|
+
|
|
1174
|
+
private async handleLoopError(task: Test, error: unknown): Promise<'continue' | 'stop'> {
|
|
1175
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1176
|
+
if (!task.hasFinished) task.addNote(`Execution error: ${message}`);
|
|
1177
|
+
|
|
1178
|
+
const result = await this.explorer.handleExecutionError(error);
|
|
1179
|
+
tag('info').log(`Browser supervisor: ${result.action} - ${result.message}`);
|
|
1180
|
+
task.addNote(result.message);
|
|
1181
|
+
|
|
1182
|
+
if (result.action === 'stop') {
|
|
1183
|
+
task.finish(TestResult.FAILED);
|
|
1184
|
+
return 'stop';
|
|
1185
|
+
}
|
|
1186
|
+
|
|
1187
|
+
if (result.recovered) {
|
|
1188
|
+
this.resetFailureCount();
|
|
1189
|
+
this.previousUrl = null;
|
|
1190
|
+
this.previousStateHash = null;
|
|
1191
|
+
this.stalledIterations = 0;
|
|
1192
|
+
} else if (this.shouldStopAfterStalledLoopError(task)) {
|
|
1193
|
+
return 'stop';
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1196
|
+
this.currentConversation?.addUserText(result.message);
|
|
1197
|
+
return 'continue';
|
|
1198
|
+
}
|
|
1199
|
+
|
|
1200
|
+
private shouldStopAfterStalledLoopError(task: Test): boolean {
|
|
1201
|
+
if (task.hasFinished) return false;
|
|
1202
|
+
|
|
1203
|
+
this.stalledIterations++;
|
|
1204
|
+
if (this.stalledIterations < this.MAX_STALLED_ITERATIONS) return false;
|
|
1205
|
+
|
|
1206
|
+
task.addNote('No browser progress after repeated execution errors', TestResult.FAILED);
|
|
1207
|
+
task.finish(TestResult.FAILED);
|
|
1208
|
+
return true;
|
|
1209
|
+
}
|
|
1210
|
+
|
|
1211
|
+
private async cleanupStartedTest(task: Test): Promise<void> {
|
|
1212
|
+
await this.finishTest(task);
|
|
1213
|
+
await this.explorer.stopTest(task, {
|
|
1214
|
+
startUrl: task.startUrl,
|
|
1215
|
+
style: task.style,
|
|
1216
|
+
sessionName: task.sessionName,
|
|
1217
|
+
});
|
|
1218
|
+
}
|
|
1219
|
+
}
|
|
1220
|
+
|
|
1221
|
+
interface TestSessionHandlers {
|
|
1222
|
+
offFailedRequest?: () => void;
|
|
1096
1223
|
}
|
package/src/ai/tools.ts
CHANGED
|
@@ -6,6 +6,7 @@ import type { ExperienceTracker } from '../experience-tracker.ts';
|
|
|
6
6
|
import type Explorer from '../explorer.ts';
|
|
7
7
|
import { type Task, TestResult } from '../test-plan.js';
|
|
8
8
|
import { extractFocusedElement } from '../utils/aria.ts';
|
|
9
|
+
import { isFatalBrowserError } from '../utils/browser-errors.ts';
|
|
9
10
|
import { createDebug, tag } from '../utils/logger.js';
|
|
10
11
|
import { pause } from '../utils/loop.js';
|
|
11
12
|
import { WebElement } from '../utils/web-element.ts';
|
|
@@ -17,7 +18,7 @@ import { isInteractive } from './task-agent.ts';
|
|
|
17
18
|
|
|
18
19
|
const debugLog = createDebug('explorbot:tools');
|
|
19
20
|
|
|
20
|
-
export const CODECEPT_TOOLS = ['click', 'pressKey', 'form'] as const;
|
|
21
|
+
export const CODECEPT_TOOLS = ['click', 'hover', 'pressKey', 'form'] as const;
|
|
21
22
|
export const ASSERTION_TOOLS = ['verify'] as const;
|
|
22
23
|
|
|
23
24
|
export function createCodeceptJSTools(explorer: Explorer, task: Task) {
|
|
@@ -159,6 +160,84 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
|
|
|
159
160
|
},
|
|
160
161
|
}),
|
|
161
162
|
|
|
163
|
+
hover: tool({
|
|
164
|
+
description: dedent`
|
|
165
|
+
Move the mouse cursor to an element to reveal hover-only controls.
|
|
166
|
+
|
|
167
|
+
Use this before clicking row actions, icon buttons, menus, or toolbars that appear only
|
|
168
|
+
when the user hovers a list item, table row, card, or tree node.
|
|
169
|
+
|
|
170
|
+
This tool ONLY accepts I.moveCursorTo(locator) commands. It does not click.
|
|
171
|
+
After hovering, use context(), see(), or click() the revealed control.
|
|
172
|
+
`,
|
|
173
|
+
inputSchema: z.object({
|
|
174
|
+
commands: z.array(z.string()).describe(dedent`
|
|
175
|
+
FALLBACK LOCATORS for ONE element to hover.
|
|
176
|
+
Order by reliability:
|
|
177
|
+
1. I.moveCursorTo(text, container)
|
|
178
|
+
2. I.moveCursorTo(ARIA, container)
|
|
179
|
+
3. I.moveCursorTo(CSS, container)
|
|
180
|
+
4. I.moveCursorTo(CSS) or I.moveCursorTo(XPath)
|
|
181
|
+
`),
|
|
182
|
+
explanation: z.string().describe('Why you are hovering this element'),
|
|
183
|
+
}),
|
|
184
|
+
execute: async ({ commands: rawCommands, explanation }) => {
|
|
185
|
+
const activeNote = task.startNote(explanation);
|
|
186
|
+
|
|
187
|
+
if (rawCommands.length === 0) {
|
|
188
|
+
activeNote.commit(TestResult.FAILED);
|
|
189
|
+
return failedToolResult('hover', 'No commands provided');
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
const invalidCommands = rawCommands.map((cmd) => cmd.trim()).filter((cmd) => cmd.startsWith('I.') && !cmd.startsWith('I.moveCursorTo'));
|
|
193
|
+
|
|
194
|
+
if (invalidCommands.length > 0) {
|
|
195
|
+
activeNote.commit(TestResult.FAILED);
|
|
196
|
+
return failedToolResult('hover', `Invalid commands: ${invalidCommands.join(', ')}. Hover tool only accepts I.moveCursorTo() commands.`, {
|
|
197
|
+
suggestion: 'Use click() to click elements, or form() for typing/selecting.',
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
const commands = rawCommands.map((cmd) => {
|
|
202
|
+
const trimmed = cmd.trim();
|
|
203
|
+
if (trimmed.startsWith('I.moveCursorTo')) return trimmed;
|
|
204
|
+
return `I.moveCursorTo(${JSON.stringify(trimmed)})`;
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
const previousState = ActionResult.fromState(stateManager.getCurrentState()!);
|
|
208
|
+
const action = explorer.createAction();
|
|
209
|
+
const attempts: Array<{ command: string; success: boolean; error?: string }> = [];
|
|
210
|
+
|
|
211
|
+
for (const command of commands) {
|
|
212
|
+
const success = await action.attempt(command, explanation, true);
|
|
213
|
+
attempts.push({
|
|
214
|
+
command,
|
|
215
|
+
success,
|
|
216
|
+
...(action.lastError && { error: action.lastError.toString() }),
|
|
217
|
+
});
|
|
218
|
+
|
|
219
|
+
if (!success) continue;
|
|
220
|
+
|
|
221
|
+
const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, command);
|
|
222
|
+
activeNote.commit(TestResult.PASSED);
|
|
223
|
+
return successToolResult('hover', { ...toolResult, attempts, code: command }, action);
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, commands[0]);
|
|
227
|
+
activeNote.commit(TestResult.FAILED);
|
|
228
|
+
return failedToolResult(
|
|
229
|
+
'hover',
|
|
230
|
+
'All hover commands failed',
|
|
231
|
+
{
|
|
232
|
+
...toolResult,
|
|
233
|
+
attempts,
|
|
234
|
+
suggestion: 'Use xpathCheck() to locate the row/card/tree node, or visualClick() if the hover target is only visually identifiable.',
|
|
235
|
+
},
|
|
236
|
+
action.lastError
|
|
237
|
+
);
|
|
238
|
+
},
|
|
239
|
+
}),
|
|
240
|
+
|
|
162
241
|
pressKey: tool({
|
|
163
242
|
description: dedent`
|
|
164
243
|
Press a keyboard key or key combination. Use this for special keys like Enter, Escape, Tab, Arrow keys, or key combinations with modifiers.
|
|
@@ -287,6 +366,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
|
|
|
287
366
|
suggestion: 'Verify the key name is correct. For typing text, use form() tool instead.',
|
|
288
367
|
});
|
|
289
368
|
} catch (error) {
|
|
369
|
+
throwIfFatalBrowserError(error);
|
|
290
370
|
activeNote.commit(TestResult.FAILED);
|
|
291
371
|
const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred';
|
|
292
372
|
return failedToolResult('pressKey', `PressKey tool failed: ${errorMessage}`);
|
|
@@ -405,6 +485,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
|
|
|
405
485
|
action
|
|
406
486
|
);
|
|
407
487
|
} catch (error) {
|
|
488
|
+
throwIfFatalBrowserError(error);
|
|
408
489
|
activeNote.commit(TestResult.FAILED);
|
|
409
490
|
const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred';
|
|
410
491
|
return failedToolResult('form', `Form tool failed: ${errorMessage}`);
|
|
@@ -444,8 +525,7 @@ export function createSpecialContextTools(explorer: Explorer, context: 'iframe')
|
|
|
444
525
|
|
|
445
526
|
await explorer.switchToMainFrame();
|
|
446
527
|
|
|
447
|
-
const
|
|
448
|
-
const nextState = await action.capturePageState();
|
|
528
|
+
const nextState = await explorer.capturePageState();
|
|
449
529
|
const toolResult = await nextState.toToolResult(previousState, 'I.switchTo()');
|
|
450
530
|
|
|
451
531
|
return successToolResult('exitIframe', {
|
|
@@ -454,6 +534,7 @@ export function createSpecialContextTools(explorer: Explorer, context: 'iframe')
|
|
|
454
534
|
code: 'I.switchTo()',
|
|
455
535
|
});
|
|
456
536
|
} catch (error) {
|
|
537
|
+
throwIfFatalBrowserError(error);
|
|
457
538
|
const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred';
|
|
458
539
|
return failedToolResult('exitIframe', `Failed to exit iframe: ${errorMessage}`);
|
|
459
540
|
}
|
|
@@ -485,6 +566,7 @@ export function createAgentTools({
|
|
|
485
566
|
Check the page contents based on current page state and screenshot.
|
|
486
567
|
This tool will trigger visual research to check the page contents on request.
|
|
487
568
|
Use it to verify the actions were performed correctly and the page is in the expected state.
|
|
569
|
+
Input schema has exactly one field: request. Do not pass text, reason, assertion, or other fields.
|
|
488
570
|
|
|
489
571
|
<example>
|
|
490
572
|
request: "Check current state of the Login form"
|
|
@@ -500,8 +582,7 @@ export function createAgentTools({
|
|
|
500
582
|
}
|
|
501
583
|
|
|
502
584
|
try {
|
|
503
|
-
const
|
|
504
|
-
const actionResult = await action.caputrePageWithScreenshot();
|
|
585
|
+
const actionResult = await explorer.capturePageWithScreenshot();
|
|
505
586
|
|
|
506
587
|
if (!actionResult.screenshot) {
|
|
507
588
|
return failedToolResult('see', 'Failed to capture screenshot for analysis');
|
|
@@ -519,6 +600,7 @@ export function createAgentTools({
|
|
|
519
600
|
suggestion: 'Visual confirmation is valid evidence for test results. Use record() to note the visual findings.',
|
|
520
601
|
});
|
|
521
602
|
} catch (error) {
|
|
603
|
+
throwIfFatalBrowserError(error);
|
|
522
604
|
const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred';
|
|
523
605
|
visionDisabled = true;
|
|
524
606
|
tag('warning').log('⚠️ Vision model is not available. Visual checks are disabled for this session.');
|
|
@@ -600,8 +682,7 @@ export function createAgentTools({
|
|
|
600
682
|
});
|
|
601
683
|
}
|
|
602
684
|
|
|
603
|
-
const
|
|
604
|
-
const actionResult = await action.capturePageState();
|
|
685
|
+
const actionResult = await explorer.capturePageState();
|
|
605
686
|
const result = await navigator.verifyState(assertion, actionResult);
|
|
606
687
|
|
|
607
688
|
if (result.verified) {
|
|
@@ -619,6 +700,7 @@ export function createAgentTools({
|
|
|
619
700
|
suggestion: 'The assertion could not be verified. Check if the condition is actually present on the page or try a different assertion.',
|
|
620
701
|
});
|
|
621
702
|
} catch (error) {
|
|
703
|
+
throwIfFatalBrowserError(error);
|
|
622
704
|
const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred';
|
|
623
705
|
return failedToolResult('verify', `Verify tool failed: ${errorMessage}`, {
|
|
624
706
|
error: errorMessage,
|
|
@@ -674,6 +756,7 @@ export function createAgentTools({
|
|
|
674
756
|
`,
|
|
675
757
|
});
|
|
676
758
|
} catch (error) {
|
|
759
|
+
throwIfFatalBrowserError(error);
|
|
677
760
|
const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred';
|
|
678
761
|
return failedToolResult('research', `Research tool failed: ${errorMessage}`, {
|
|
679
762
|
error: errorMessage,
|
|
@@ -718,6 +801,7 @@ export function createAgentTools({
|
|
|
718
801
|
suggestion: 'The action could not be completed. Try a different instruction or use more specific element descriptions.',
|
|
719
802
|
});
|
|
720
803
|
} catch (error) {
|
|
804
|
+
throwIfFatalBrowserError(error);
|
|
721
805
|
const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred';
|
|
722
806
|
return failedToolResult('interact', `Interact tool failed: ${errorMessage}`, {
|
|
723
807
|
error: errorMessage,
|
|
@@ -756,7 +840,7 @@ export function createAgentTools({
|
|
|
756
840
|
|
|
757
841
|
const previousState = ActionResult.fromState(currentState);
|
|
758
842
|
const action = explorer.createAction();
|
|
759
|
-
const actionResult = await
|
|
843
|
+
const actionResult = await explorer.capturePageWithScreenshot();
|
|
760
844
|
|
|
761
845
|
if (!actionResult.screenshot) {
|
|
762
846
|
return failedToolResult('visualClick', 'Failed to capture screenshot for visual analysis');
|
|
@@ -797,6 +881,7 @@ export function createAgentTools({
|
|
|
797
881
|
analysis: locationResult,
|
|
798
882
|
});
|
|
799
883
|
} catch (error) {
|
|
884
|
+
throwIfFatalBrowserError(error);
|
|
800
885
|
const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred';
|
|
801
886
|
visionDisabled = true;
|
|
802
887
|
tag('warning').log('⚠️ Vision model is not available. Visual clicks are disabled for this session.');
|
|
@@ -1018,6 +1103,10 @@ function cap(text: string | undefined | null, max: number): string {
|
|
|
1018
1103
|
return `${text.slice(0, max)}\n[...truncated; ${text.length - max} chars omitted...]`;
|
|
1019
1104
|
}
|
|
1020
1105
|
|
|
1106
|
+
function throwIfFatalBrowserError(error: unknown): void {
|
|
1107
|
+
if (isFatalBrowserError(error)) throw error;
|
|
1108
|
+
}
|
|
1109
|
+
|
|
1021
1110
|
function transformContainsCommand(command: string): string {
|
|
1022
1111
|
if (!command.includes(':contains(')) return command;
|
|
1023
1112
|
|
|
@@ -5,7 +5,7 @@ import { normalizeUrl } from '../state-manager.js';
|
|
|
5
5
|
import { Stats } from '../stats.js';
|
|
6
6
|
import { type Plan, type Test, TestResult } from '../test-plan.js';
|
|
7
7
|
import { getCliName } from '../utils/cli-name.ts';
|
|
8
|
-
import { ErrorPageError } from '../utils/error-page.ts';
|
|
8
|
+
import { ErrorPageError, getStateErrorPageError } from '../utils/error-page.ts';
|
|
9
9
|
import { tag } from '../utils/logger.js';
|
|
10
10
|
import { type NextStepSection, printNextSteps, relativeToCwd } from '../utils/next-steps.ts';
|
|
11
11
|
import { safeFilename } from '../utils/strings.ts';
|
|
@@ -56,6 +56,11 @@ export class ExploreCommand extends BaseCommand {
|
|
|
56
56
|
Stats.mode ??= 'explore';
|
|
57
57
|
Stats.focus ??= feature;
|
|
58
58
|
const mainUrl = this.getCurrentPageUrl();
|
|
59
|
+
const error = getStateErrorPageError(this.explorBot.getExplorer().getStateManager().getCurrentState());
|
|
60
|
+
if (error) {
|
|
61
|
+
tag('warning').log(error.message);
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
59
64
|
|
|
60
65
|
if (cfg.enabled) {
|
|
61
66
|
await this.runReuseMode(mainUrl, feature, cfg);
|