explorbot 0.1.27 → 0.1.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +83 -245
- package/bin/explorbot-cli.ts +1 -0
- package/dist/bin/explorbot-cli.js +1 -0
- package/dist/package.json +8 -6
- package/dist/rules/navigator/verification-actions.md +2 -0
- package/dist/src/action-result.js +2 -0
- package/dist/src/action.js +88 -7
- package/dist/src/ai/captain/file-tools.js +100 -0
- package/dist/src/ai/captain/idle-mode.js +70 -6
- package/dist/src/ai/captain/web-mode.js +36 -6
- package/dist/src/ai/captain.js +87 -19
- package/dist/src/ai/fisherman.js +14 -3
- package/dist/src/ai/historian/screencast.js +11 -2
- package/dist/src/ai/navigator.js +5 -2
- package/dist/src/ai/pilot.js +52 -9
- package/dist/src/ai/planner.js +16 -5
- package/dist/src/ai/provider.js +53 -18
- package/dist/src/ai/researcher/coordinates.js +2 -3
- package/dist/src/ai/researcher/deep-analysis.js +3 -4
- package/dist/src/ai/researcher/locators.js +1 -2
- package/dist/src/ai/researcher.js +24 -19
- package/dist/src/ai/rules.js +44 -0
- package/dist/src/ai/task-agent.js +1 -0
- package/dist/src/ai/tester.js +161 -46
- package/dist/src/ai/tools.js +84 -8
- package/dist/src/commands/explore-command.js +6 -1
- package/dist/src/components/LogPane.js +4 -3
- package/dist/src/explorbot.js +7 -2
- package/dist/src/explorer.js +270 -35
- package/dist/src/stats.js +16 -0
- package/dist/src/utils/aria.js +66 -6
- package/dist/src/utils/browser-errors.js +23 -0
- package/dist/src/utils/error-page.js +17 -2
- package/dist/src/utils/logger.js +2 -2
- package/package.json +8 -6
- package/rules/navigator/verification-actions.md +2 -0
- package/src/action-result.ts +2 -0
- package/src/action.ts +83 -7
- package/src/ai/captain/file-tools.ts +126 -0
- package/src/ai/captain/idle-mode.ts +72 -6
- package/src/ai/captain/mixin.ts +1 -1
- package/src/ai/captain/web-mode.ts +40 -5
- package/src/ai/captain.ts +94 -20
- package/src/ai/fisherman.ts +14 -3
- package/src/ai/historian/screencast.ts +11 -2
- package/src/ai/navigator.ts +6 -2
- package/src/ai/pilot.ts +53 -9
- package/src/ai/planner.ts +16 -5
- package/src/ai/provider.ts +51 -19
- package/src/ai/researcher/coordinates.ts +2 -3
- package/src/ai/researcher/deep-analysis.ts +3 -4
- package/src/ai/researcher/locators.ts +1 -2
- package/src/ai/researcher.ts +25 -19
- package/src/ai/rules.ts +46 -0
- package/src/ai/task-agent.ts +1 -1
- package/src/ai/tester.ts +175 -48
- package/src/ai/tools.ts +97 -8
- package/src/commands/explore-command.ts +6 -1
- package/src/components/LogPane.tsx +4 -3
- package/src/config.ts +1 -0
- package/src/explorbot.ts +6 -2
- package/src/explorer.ts +295 -38
- package/src/state-manager.ts +2 -0
- package/src/stats.ts +18 -0
- package/src/utils/aria.ts +63 -6
- package/src/utils/browser-errors.ts +25 -0
- package/src/utils/error-page.ts +16 -3
- package/src/utils/logger.ts +3 -3
|
@@ -47,6 +47,12 @@ export class Researcher extends ResearcherBase {
|
|
|
47
47
|
this.stateManager = explorer.getStateManager();
|
|
48
48
|
this.experienceTracker = this.stateManager.getExperienceTracker();
|
|
49
49
|
this.hooksRunner = new HooksRunner(explorer, explorer.getConfig());
|
|
50
|
+
const ai = explorer.getConfig().ai;
|
|
51
|
+
if (ai) {
|
|
52
|
+
ai.agents ??= {};
|
|
53
|
+
ai.agents.researcher ??= {};
|
|
54
|
+
ai.agents.researcher.reasoning ??= 'low';
|
|
55
|
+
}
|
|
50
56
|
}
|
|
51
57
|
getNavigator() {
|
|
52
58
|
throw new Error('not implemented');
|
|
@@ -97,11 +103,11 @@ export class Researcher extends ResearcherBase {
|
|
|
97
103
|
await this.hooksRunner.runBeforeHook('researcher', state.url);
|
|
98
104
|
const annotatedElements = await this.explorer.annotateElements();
|
|
99
105
|
debugLog(`Annotated ${annotatedElements.length} interactive elements with eidx`);
|
|
100
|
-
this.actionResult = await this.explorer.
|
|
106
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot && this.provider.hasVision() });
|
|
101
107
|
const condition = detectPageCondition(this.actionResult);
|
|
102
108
|
if (condition === 'error') {
|
|
103
109
|
tag('warning').log(`Detected error page at ${state.url}`);
|
|
104
|
-
throw new ErrorPageError(state.url, this.actionResult.title);
|
|
110
|
+
throw new ErrorPageError(state.url, this.actionResult.title, this.actionResult.httpStatus);
|
|
105
111
|
}
|
|
106
112
|
if (condition === 'loading') {
|
|
107
113
|
const settled = await this.waitUntilSettled(screenshot);
|
|
@@ -139,7 +145,7 @@ export class Researcher extends ResearcherBase {
|
|
|
139
145
|
catch (error) {
|
|
140
146
|
if (!(error instanceof ContextLengthError) || retriesLeft <= 0) {
|
|
141
147
|
if (error instanceof ContextLengthError) {
|
|
142
|
-
tag('warning').log('Output truncated. Try lowering reasoning effort or increasing
|
|
148
|
+
tag('warning').log('Output truncated. Try lowering reasoning effort or increasing maxOutputTokens in ai.config.');
|
|
143
149
|
}
|
|
144
150
|
throw error;
|
|
145
151
|
}
|
|
@@ -191,7 +197,7 @@ export class Researcher extends ResearcherBase {
|
|
|
191
197
|
// Must run BEFORE visuallyAnnotateContainers — annotation overlays inject z-index 99998+ which would pollute the scoring.
|
|
192
198
|
if (!interrupted() && this.hasScreenshotToAnalyze) {
|
|
193
199
|
const sections = parseResearchSections(result.text);
|
|
194
|
-
const focused = await detectFocusedSection(this.explorer.playwrightHelper.page, sections);
|
|
200
|
+
const focused = await this.explorer.runWithBrowserRecovery('detectFocusedSection', () => detectFocusedSection(this.explorer.playwrightHelper.page, sections));
|
|
195
201
|
if (focused)
|
|
196
202
|
markSectionAsFocused(result, focused);
|
|
197
203
|
}
|
|
@@ -204,7 +210,7 @@ export class Researcher extends ResearcherBase {
|
|
|
204
210
|
const freshBroken = freshContainerLocs.filter((l) => l.valid === false).map((l) => l.locator);
|
|
205
211
|
const containers = validContainers.filter((c) => !freshBroken.includes(c.css));
|
|
206
212
|
await this.visuallyAnnotateElements({ containers });
|
|
207
|
-
this.actionResult = await this.explorer.
|
|
213
|
+
this.actionResult = await this.explorer.capturePageWithScreenshot();
|
|
208
214
|
const visualResult = await this.analyzeScreenshotForVisualProps();
|
|
209
215
|
if (visualResult.elements.size > 0) {
|
|
210
216
|
await this.mergeVisualData(result, visualResult.elements);
|
|
@@ -277,7 +283,7 @@ export class Researcher extends ResearcherBase {
|
|
|
277
283
|
if (!this.actionResult) {
|
|
278
284
|
debugLog('No action result, navigating to URL');
|
|
279
285
|
await this.explorer.visit(url);
|
|
280
|
-
this.actionResult = await this.explorer.
|
|
286
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot });
|
|
281
287
|
return;
|
|
282
288
|
}
|
|
283
289
|
const isOnCurrentState = this.actionResult.getStateHash() === this.stateManager.getCurrentState()?.hash;
|
|
@@ -285,46 +291,47 @@ export class Researcher extends ResearcherBase {
|
|
|
285
291
|
const isEmpty = isBodyEmpty(stateHtml);
|
|
286
292
|
if (!isEmpty && isOnCurrentState) {
|
|
287
293
|
if ((!this.actionResult.screenshot && screenshot) || !this.actionResult.ariaSnapshot) {
|
|
288
|
-
this.actionResult = await this.explorer.
|
|
294
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot });
|
|
289
295
|
}
|
|
290
296
|
return;
|
|
291
297
|
}
|
|
292
298
|
if (isEmpty && isOnCurrentState) {
|
|
293
299
|
debugLog('HTML body empty on current URL, waiting for content');
|
|
294
300
|
tag('step').log('Page body is empty, waiting for content...');
|
|
301
|
+
await this.explorer.visit(url);
|
|
302
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false });
|
|
295
303
|
await this.waitUntilSettled(screenshot ?? false);
|
|
296
304
|
return;
|
|
297
305
|
}
|
|
298
306
|
debugLog('Not on current state, navigating to URL');
|
|
299
307
|
tag('step').log('Navigating to URL...');
|
|
300
308
|
await this.explorer.visit(url);
|
|
301
|
-
this.actionResult = await this.explorer.
|
|
309
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false });
|
|
302
310
|
}
|
|
303
311
|
async waitUntilSettled(screenshot) {
|
|
304
312
|
const errorPageTimeout = this.explorer.getConfig().ai?.agents?.researcher?.errorPageTimeout ?? 10;
|
|
305
313
|
if (errorPageTimeout <= 0)
|
|
306
314
|
return false;
|
|
307
|
-
const page = this.explorer.playwrightHelper.page;
|
|
308
315
|
const includeScreenshot = screenshot && this.provider.hasVision();
|
|
309
316
|
try {
|
|
310
|
-
await page?.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 });
|
|
317
|
+
await this.explorer.runWithBrowserRecovery('waitUntilSettled', () => this.explorer.playwrightHelper.page?.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 }));
|
|
311
318
|
}
|
|
312
319
|
catch { }
|
|
313
320
|
await this.explorer.annotateElements();
|
|
314
|
-
this.actionResult = await this.explorer.
|
|
321
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot });
|
|
315
322
|
let condition = detectPageCondition(this.actionResult);
|
|
316
323
|
if (condition === 'error') {
|
|
317
|
-
throw new ErrorPageError(this.actionResult.url, this.actionResult.title);
|
|
324
|
+
throw new ErrorPageError(this.actionResult.url, this.actionResult.title, this.actionResult.httpStatus);
|
|
318
325
|
}
|
|
319
326
|
if (condition === 'ok')
|
|
320
327
|
return true;
|
|
321
328
|
for (let i = 0; i < 3; i++) {
|
|
322
329
|
await new Promise((r) => setTimeout(r, 1000));
|
|
323
330
|
await this.explorer.annotateElements();
|
|
324
|
-
this.actionResult = await this.explorer.
|
|
331
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot });
|
|
325
332
|
condition = detectPageCondition(this.actionResult);
|
|
326
333
|
if (condition === 'error') {
|
|
327
|
-
throw new ErrorPageError(this.actionResult.url, this.actionResult.title);
|
|
334
|
+
throw new ErrorPageError(this.actionResult.url, this.actionResult.title, this.actionResult.httpStatus);
|
|
328
335
|
}
|
|
329
336
|
if (condition === 'ok')
|
|
330
337
|
return true;
|
|
@@ -681,15 +688,13 @@ export class Researcher extends ResearcherBase {
|
|
|
681
688
|
.join('\n\n');
|
|
682
689
|
}
|
|
683
690
|
async navigateTo(url) {
|
|
684
|
-
|
|
685
|
-
await action.execute(`I.amOnPage("${url}")`);
|
|
691
|
+
await this.explorer.visit(url);
|
|
686
692
|
}
|
|
687
693
|
async cancelInUi() {
|
|
688
694
|
const beforeAria = this.stateManager.getCurrentState()?.ariaSnapshot || null;
|
|
689
|
-
|
|
690
|
-
await action.execute('I.clickXY(0, 0)');
|
|
695
|
+
await this.explorer.executeAction('I.clickXY(0, 0)');
|
|
691
696
|
if (diffAriaSnapshots(beforeAria, this.stateManager.getCurrentState()?.ariaSnapshot || null))
|
|
692
697
|
return;
|
|
693
|
-
await
|
|
698
|
+
await this.explorer.executeAction(`I.pressKey('Escape')`);
|
|
694
699
|
}
|
|
695
700
|
}
|
package/dist/src/ai/rules.js
CHANGED
|
@@ -147,6 +147,50 @@ export const protectionRule = dedent `
|
|
|
147
147
|
Do not propose or perform delete/remove/archive actions on the entity that owns the current URL; propose such actions only on disposable children created within the scenario itself.
|
|
148
148
|
</important>
|
|
149
149
|
`;
|
|
150
|
+
export const dataProtectionRules = dedent `
|
|
151
|
+
<data_protection_rules>
|
|
152
|
+
${protectionRule}
|
|
153
|
+
|
|
154
|
+
If the user request, scenario, focus, or test instructions explicitly prohibit creating,
|
|
155
|
+
editing, updating, deleting, removing, or otherwise mutating data, do not perform those
|
|
156
|
+
actions through the UI, API preconditions, cleanup, fallback steps, or Fisherman.
|
|
157
|
+
|
|
158
|
+
Do not use Fisherman or API data preparation to bypass a no-mutation, read-only, search,
|
|
159
|
+
filter, tab, or list-inspection constraint. Use visible existing data when it is available.
|
|
160
|
+
If no suitable data exists, report the missing precondition instead of creating data.
|
|
161
|
+
|
|
162
|
+
Destructive actions are allowed only against disposable data created by the current scenario
|
|
163
|
+
or prepared for that scenario by Fisherman/API preconditions. Existing application data must
|
|
164
|
+
remain unchanged.
|
|
165
|
+
</data_protection_rules>
|
|
166
|
+
`;
|
|
167
|
+
export const capabilityGroundingRule = dedent `
|
|
168
|
+
<capability_grounding>
|
|
169
|
+
When a scenario depends on a named action, menu item, status, option, workflow, or feature,
|
|
170
|
+
that capability must be visible or explicitly confirmed in the current research/page context
|
|
171
|
+
for the same target entity type.
|
|
172
|
+
|
|
173
|
+
Do not transfer capabilities between similar entities, rows, lists, detail pages, or menus.
|
|
174
|
+
Do not replace a requested action with a synonym or related action unless the UI explicitly
|
|
175
|
+
shows that action for the target entity.
|
|
176
|
+
|
|
177
|
+
When an action is described as applying to an item, row, card, record, node, or entity,
|
|
178
|
+
the target must be grounded as that kind of data entity in the current context. Do not use
|
|
179
|
+
navigation links, filter tabs, counters, breadcrumbs, headings, toolbar controls, or other
|
|
180
|
+
page controls as the subject of row/entity actions.
|
|
181
|
+
|
|
182
|
+
When a scenario asks to open, view, inspect, or navigate to an entity detail view, success
|
|
183
|
+
requires evidence of that entity detail context. An active filter, selected tab, visible count,
|
|
184
|
+
or filtered list is not enough to prove an entity detail view opened.
|
|
185
|
+
|
|
186
|
+
Do not rewrite a scenario goal to match a similar outcome that happened accidentally. If the
|
|
187
|
+
requested entity detail/action/workflow was not achieved, report that mismatch instead of
|
|
188
|
+
passing the test for a related filter, tab, navigation, or status view.
|
|
189
|
+
|
|
190
|
+
If the required capability is not available for the target entity after reasonable discovery,
|
|
191
|
+
record the missing capability and stop instead of repeatedly trying unrelated locators.
|
|
192
|
+
</capability_grounding>
|
|
193
|
+
`;
|
|
150
194
|
export const focusedElementRule = dedent `
|
|
151
195
|
<focused_element_actions>
|
|
152
196
|
When a text input element is focused (textbox, combobox, contenteditable):
|
package/dist/src/ai/tester.js
CHANGED
|
@@ -4,17 +4,17 @@ import { tool } from 'ai';
|
|
|
4
4
|
import dedent from 'dedent';
|
|
5
5
|
import { z } from 'zod';
|
|
6
6
|
import { ActionResult } from "../action-result.js";
|
|
7
|
-
import { setActivity } from "../activity.js";
|
|
7
|
+
import { clearActivity, setActivity } from "../activity.js";
|
|
8
8
|
import { ConfigParser } from "../config.js";
|
|
9
9
|
import { Observability } from "../observability.js";
|
|
10
10
|
import { Stats } from "../stats.js";
|
|
11
11
|
import { TestResult } from "../test-plan.js";
|
|
12
12
|
import { detectFocusArea, extractFocusedElement } from "../utils/aria.js";
|
|
13
|
-
import { ErrorPageError } from "../utils/error-page.js";
|
|
13
|
+
import { ErrorPageError, isErrorPage } from "../utils/error-page.js";
|
|
14
14
|
import { HooksRunner } from "../utils/hooks-runner.js";
|
|
15
15
|
import { createDebug, tag } from "../utils/logger.js";
|
|
16
16
|
import { loop } from "../utils/loop.js";
|
|
17
|
-
import { actionRule, focusedElementRule, formRequirementsRule, locatorRule, multipleTabsRule,
|
|
17
|
+
import { actionRule, capabilityGroundingRule, dataProtectionRules, focusedElementRule, formRequirementsRule, locatorRule, multipleTabsRule, sectionContextRule } from "./rules.js";
|
|
18
18
|
import { TaskAgent } from "./task-agent.js";
|
|
19
19
|
import { createCodeceptJSTools, createSpecialContextTools } from "./tools.js";
|
|
20
20
|
const debugLog = createDebug('explorbot:tester');
|
|
@@ -29,7 +29,7 @@ const SAMPLE_FILES = {
|
|
|
29
29
|
'MP3 audio': 'sample.mp3',
|
|
30
30
|
};
|
|
31
31
|
export class Tester extends TaskAgent {
|
|
32
|
-
ACTION_TOOLS = ['click', 'pressKey', 'form'];
|
|
32
|
+
ACTION_TOOLS = ['click', 'hover', 'pressKey', 'form'];
|
|
33
33
|
SPECIAL_CONTEXT_ACTION_TOOLS = ['exitIframe'];
|
|
34
34
|
emoji = '🧪';
|
|
35
35
|
explorer;
|
|
@@ -51,6 +51,8 @@ export class Tester extends TaskAgent {
|
|
|
51
51
|
hooksRunner;
|
|
52
52
|
seenUiMapUrls = new Set();
|
|
53
53
|
lastAnalyzedStateHash = null;
|
|
54
|
+
stalledIterations = 0;
|
|
55
|
+
MAX_STALLED_ITERATIONS = 3;
|
|
54
56
|
constructor(explorer, provider, researcher, navigator, agentTools) {
|
|
55
57
|
super();
|
|
56
58
|
this.explorer = explorer;
|
|
@@ -99,6 +101,7 @@ export class Tester extends TaskAgent {
|
|
|
99
101
|
this.pageActionResult = null;
|
|
100
102
|
this.seenUiMapUrls.clear();
|
|
101
103
|
this.lastAnalyzedStateHash = null;
|
|
104
|
+
this.stalledIterations = 0;
|
|
102
105
|
this.explorer.getStateManager().clearHistory();
|
|
103
106
|
this.resetFailureCount();
|
|
104
107
|
this.pilot?.reset();
|
|
@@ -107,18 +110,13 @@ export class Tester extends TaskAgent {
|
|
|
107
110
|
const offFailedRequest = requestStore?.onFailedRequest((r) => {
|
|
108
111
|
task.addNote(`Network error: ${r.method} ${r.path} → ${r.status}`, TestResult.FAILED);
|
|
109
112
|
});
|
|
110
|
-
const page = this.explorer.playwrightHelper?.page;
|
|
111
|
-
const onPageError = (err) => {
|
|
112
|
-
task.addNote(`Console error: ${err.message}`, TestResult.FAILED);
|
|
113
|
-
};
|
|
114
|
-
const onConsoleMessage = (msg) => {
|
|
115
|
-
if (msg.type() !== 'error')
|
|
116
|
-
return;
|
|
117
|
-
task.addNote(`Console error: ${msg.text()}`, TestResult.FAILED);
|
|
118
|
-
};
|
|
119
|
-
page?.on('pageerror', onPageError);
|
|
120
|
-
page?.on('console', onConsoleMessage);
|
|
121
113
|
const initialState = ActionResult.fromState(state);
|
|
114
|
+
if (isErrorPage(initialState)) {
|
|
115
|
+
task.start();
|
|
116
|
+
await this.explorer.startTest(task);
|
|
117
|
+
offFailedRequest?.();
|
|
118
|
+
return await this.abortStartedTestOnErrorPage(task, initialState);
|
|
119
|
+
}
|
|
122
120
|
const conversation = this.provider.startConversation(this.getSystemMessage(), 'tester');
|
|
123
121
|
conversation.markLastMessageCacheable();
|
|
124
122
|
this.currentConversation = conversation;
|
|
@@ -140,17 +138,15 @@ export class Tester extends TaskAgent {
|
|
|
140
138
|
startUrl: task.startUrl,
|
|
141
139
|
expected: task.expected,
|
|
142
140
|
},
|
|
143
|
-
}, async () => this.runTestSession(task, initialState, conversation, { offFailedRequest
|
|
141
|
+
}, async () => this.runTestSession(task, initialState, conversation, { offFailedRequest }));
|
|
144
142
|
}
|
|
145
143
|
async runTestSession(task, initialState, conversation, handlers) {
|
|
146
|
-
const { offFailedRequest
|
|
144
|
+
const { offFailedRequest } = handlers;
|
|
147
145
|
if (this.pilot) {
|
|
148
146
|
try {
|
|
149
147
|
const plan = await this.pilot.planTest(task, initialState);
|
|
150
148
|
if (task.hasFinished) {
|
|
151
149
|
offFailedRequest?.();
|
|
152
|
-
page?.off('pageerror', onPageError);
|
|
153
|
-
page?.off('console', onConsoleMessage);
|
|
154
150
|
return { success: task.isSuccessful };
|
|
155
151
|
}
|
|
156
152
|
if (plan) {
|
|
@@ -163,19 +159,36 @@ export class Tester extends TaskAgent {
|
|
|
163
159
|
task.addNote(`Planning failed: ${message}`, TestResult.FAILED);
|
|
164
160
|
task.finish(TestResult.FAILED);
|
|
165
161
|
offFailedRequest?.();
|
|
166
|
-
page?.off('pageerror', onPageError);
|
|
167
|
-
page?.off('console', onConsoleMessage);
|
|
168
162
|
return { success: false };
|
|
169
163
|
}
|
|
170
164
|
}
|
|
171
165
|
debugLog('Starting test execution with tools');
|
|
172
|
-
|
|
173
|
-
|
|
166
|
+
if (!(await this.explorer.startTest(task))) {
|
|
167
|
+
offFailedRequest?.();
|
|
168
|
+
await this.cleanupStartedTest(task);
|
|
169
|
+
return { success: task.isSuccessful };
|
|
170
|
+
}
|
|
174
171
|
debugLog(`Navigating to ${task.startUrl}`);
|
|
175
|
-
|
|
172
|
+
try {
|
|
173
|
+
await this.explorer.visit(task.startUrl);
|
|
174
|
+
}
|
|
175
|
+
catch (error) {
|
|
176
|
+
const result = await this.handleLoopError(task, error);
|
|
177
|
+
if (result === 'stop') {
|
|
178
|
+
offFailedRequest?.();
|
|
179
|
+
await this.cleanupStartedTest(task);
|
|
180
|
+
return { success: task.isSuccessful };
|
|
181
|
+
}
|
|
182
|
+
}
|
|
176
183
|
const startState = this.explorer.getStateManager().getCurrentState();
|
|
177
|
-
if (startState)
|
|
184
|
+
if (startState) {
|
|
178
185
|
task.addUrlNote(startState);
|
|
186
|
+
const startActionResult = ActionResult.fromState(startState);
|
|
187
|
+
if (isErrorPage(startActionResult)) {
|
|
188
|
+
offFailedRequest?.();
|
|
189
|
+
return await this.abortStartedTestOnErrorPage(task, startActionResult);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
179
192
|
const currentUrl = startState?.url || task.startUrl || '';
|
|
180
193
|
await this.hooksRunner.runBeforeHook('tester', currentUrl);
|
|
181
194
|
const offStateChange = this.explorer.getStateManager().onStateChange((event) => {
|
|
@@ -195,6 +208,12 @@ export class Tester extends TaskAgent {
|
|
|
195
208
|
shouldContinue = false;
|
|
196
209
|
await loop(async ({ stop, pause, iteration, userInput }) => {
|
|
197
210
|
debugLog('iteration', iteration);
|
|
211
|
+
if (!(await this.explorer.ensurePageAvailable())) {
|
|
212
|
+
task.addNote('Browser page is unavailable');
|
|
213
|
+
task.finish(TestResult.FAILED);
|
|
214
|
+
stop();
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
198
217
|
const currentState = this.getCurrentState();
|
|
199
218
|
const tools = {
|
|
200
219
|
...codeceptjsTools,
|
|
@@ -277,6 +296,10 @@ export class Tester extends TaskAgent {
|
|
|
277
296
|
task.addNote(`Failed to ${execution.input.explanation} (${execution.toolName})`, TestResult.FAILED);
|
|
278
297
|
});
|
|
279
298
|
}
|
|
299
|
+
if (this.shouldStopForStalledExecution(task, currentState, result?.toolExecutions || [])) {
|
|
300
|
+
stop();
|
|
301
|
+
return;
|
|
302
|
+
}
|
|
280
303
|
if (assertionPerformed) {
|
|
281
304
|
const message = result?.toolExecutions?.find((execution) => execution.toolName === 'verify')?.output?.message || '';
|
|
282
305
|
task.addNote(message, wasSuccessful ? TestResult.PASSED : TestResult.FAILED);
|
|
@@ -327,20 +350,15 @@ export class Tester extends TaskAgent {
|
|
|
327
350
|
}
|
|
328
351
|
: undefined,
|
|
329
352
|
catch: async ({ error, stop }) => {
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
if (!task.hasFinished) {
|
|
333
|
-
task.addNote(`Execution error: ${message}`);
|
|
334
|
-
}
|
|
335
|
-
if (error instanceof Error && error.name === 'AbortError') {
|
|
353
|
+
const result = await this.handleLoopError(task, error);
|
|
354
|
+
if (result === 'stop')
|
|
336
355
|
stop();
|
|
337
|
-
return;
|
|
338
|
-
}
|
|
339
|
-
conversation.addUserText(`Previous AI call failed: ${message}. Take a different approach on the next step.`);
|
|
340
356
|
},
|
|
341
357
|
});
|
|
342
358
|
if (task.hasFinished)
|
|
343
359
|
break;
|
|
360
|
+
if (!(await this.explorer.ensurePageAvailable()))
|
|
361
|
+
break;
|
|
344
362
|
const finalState = this.getCurrentState();
|
|
345
363
|
const wantsContinue = await this.pilot.finalReview(task, finalState, conversation, this.navigator);
|
|
346
364
|
if (!wantsContinue || task.hasFinished)
|
|
@@ -366,14 +384,8 @@ export class Tester extends TaskAgent {
|
|
|
366
384
|
await this.getQuartermaster().analyzeSession(task, initialState, conversation);
|
|
367
385
|
offStateChange();
|
|
368
386
|
offFailedRequest?.();
|
|
369
|
-
page?.off('pageerror', onPageError);
|
|
370
|
-
page?.off('console', onConsoleMessage);
|
|
371
387
|
await this.finishTest(task);
|
|
372
|
-
await this.explorer.stopTest(task,
|
|
373
|
-
startUrl: task.startUrl,
|
|
374
|
-
style: task.style,
|
|
375
|
-
sessionName: task.sessionName,
|
|
376
|
-
});
|
|
388
|
+
await this.explorer.stopTest(task, this.buildStopTestMeta(task));
|
|
377
389
|
return {
|
|
378
390
|
success: task.isSuccessful,
|
|
379
391
|
...task,
|
|
@@ -390,6 +402,28 @@ export class Tester extends TaskAgent {
|
|
|
390
402
|
return false;
|
|
391
403
|
return true;
|
|
392
404
|
}
|
|
405
|
+
shouldStopForStalledExecution(task, previousState, toolExecutions) {
|
|
406
|
+
if (task.hasFinished)
|
|
407
|
+
return false;
|
|
408
|
+
const currentState = this.getCurrentState();
|
|
409
|
+
const stateChanged = previousState.url !== currentState.url || previousState.hash !== currentState.hash;
|
|
410
|
+
const actionTools = [...this.ACTION_TOOLS, ...this.SPECIAL_CONTEXT_ACTION_TOOLS];
|
|
411
|
+
const hasSuccessfulAction = toolExecutions.some((execution) => execution.wasSuccessful && actionTools.includes(execution.toolName));
|
|
412
|
+
const hasSuccessfulAssertion = toolExecutions.some((execution) => execution.wasSuccessful && this.ASSERTION_TOOLS.includes(execution.toolName));
|
|
413
|
+
if (stateChanged || hasSuccessfulAction || hasSuccessfulAssertion) {
|
|
414
|
+
this.stalledIterations = 0;
|
|
415
|
+
return false;
|
|
416
|
+
}
|
|
417
|
+
const hasNoBrowserProgress = toolExecutions.length === 0 || toolExecutions.every((execution) => !actionTools.includes(execution.toolName) || !execution.wasSuccessful);
|
|
418
|
+
if (!hasNoBrowserProgress)
|
|
419
|
+
return false;
|
|
420
|
+
this.stalledIterations++;
|
|
421
|
+
if (this.stalledIterations < this.MAX_STALLED_ITERATIONS)
|
|
422
|
+
return false;
|
|
423
|
+
task.addNote('No browser progress after repeated attempts on unchanged page', TestResult.FAILED);
|
|
424
|
+
task.finish(TestResult.FAILED);
|
|
425
|
+
return true;
|
|
426
|
+
}
|
|
393
427
|
async prepareInstructionsForNextStep(task) {
|
|
394
428
|
let outcomeStatus = dedent `
|
|
395
429
|
<task>
|
|
@@ -398,6 +432,8 @@ export class Tester extends TaskAgent {
|
|
|
398
432
|
|
|
399
433
|
<rules>
|
|
400
434
|
Use tools ${this.ACTION_TOOLS.join(', ')} to interact with the page.
|
|
435
|
+
Use tool names exactly as listed in this prompt. Do not invent combined tool names, aliases, or names with channel markers such as "commentary".
|
|
436
|
+
Match each tool input schema exactly. Do not invent parameter names or pass extra fields.
|
|
401
437
|
Do not do unsuccesful clicks again.
|
|
402
438
|
Do not run same tool calls with same parameters again.
|
|
403
439
|
</rules>
|
|
@@ -607,6 +643,26 @@ export class Tester extends TaskAgent {
|
|
|
607
643
|
tag('warning').log(`Test with no result: ${task.scenario}`);
|
|
608
644
|
}
|
|
609
645
|
}
|
|
646
|
+
async abortStartedTestOnErrorPage(task, actionResult) {
|
|
647
|
+
const error = new ErrorPageError(actionResult.url || task.startUrl || '', actionResult.title, actionResult.httpStatus);
|
|
648
|
+
tag('warning').log(error.message);
|
|
649
|
+
task.addNote(error.message, TestResult.FAILED, actionResult.screenshotFile, actionResult.fullUrl || actionResult.url);
|
|
650
|
+
task.finish(TestResult.FAILED);
|
|
651
|
+
this.finishTest(task);
|
|
652
|
+
await this.explorer.stopTest(task, this.buildStopTestMeta(task));
|
|
653
|
+
clearActivity(true);
|
|
654
|
+
return { success: false };
|
|
655
|
+
}
|
|
656
|
+
buildStopTestMeta(task) {
|
|
657
|
+
const meta = {
|
|
658
|
+
startUrl: task.startUrl,
|
|
659
|
+
};
|
|
660
|
+
if (task.style)
|
|
661
|
+
meta.style = task.style;
|
|
662
|
+
if (task.sessionName)
|
|
663
|
+
meta.sessionName = task.sessionName;
|
|
664
|
+
return meta;
|
|
665
|
+
}
|
|
610
666
|
getSystemMessage() {
|
|
611
667
|
return dedent `
|
|
612
668
|
<role>
|
|
@@ -640,6 +696,7 @@ export class Tester extends TaskAgent {
|
|
|
640
696
|
<rules>
|
|
641
697
|
- Refer to UI Map from <page_ui_map> to understand the page structure and its main elements
|
|
642
698
|
- Use only elements that exist in the provided ARIA tree or HTML, <page_aria> and <page_html>
|
|
699
|
+
- Use tool input schemas exactly as documented. Do not invent parameter names or add fields not listed by the tool schema.
|
|
643
700
|
- Use click() for buttons, links, and clickable elements ONLY - do NOT include I.fillField() or I.type() commands in click() tool
|
|
644
701
|
- click() commands array is for FALLBACK LOCATORS of the SAME element, NOT for clicking different elements in sequence. If you need to click two different elements, make two separate click() calls.
|
|
645
702
|
- Use form() for text input (I.fillField, I.type), dropdown selection (I.selectOption), file uploads (I.attachFile), and multi-step form interactions
|
|
@@ -652,6 +709,7 @@ export class Tester extends TaskAgent {
|
|
|
652
709
|
- NEVER call record(status: "success") if your last verify() or see() call FAILED. A failed check means the outcome is NOT confirmed — use record(status: "fail") instead, or retry with a different approach.
|
|
653
710
|
- Use finish() to complete the test, not record(). record() is for intermediate notes.
|
|
654
711
|
- Call finish(verify) when all goals are achieved — provide an assertion to verify
|
|
712
|
+
- NEVER call finish() with a negative assertion that says the goal did NOT happen. If the goal cannot be achieved after real attempts, record the blocker and call stop().
|
|
655
713
|
- ONLY call stop() if the scenario itself is completely irrelevant to this page and no expectations can be achieved
|
|
656
714
|
- Use reset() ONLY as a last resort when the current page cannot host the scenario. Never reset after a successful flow just because an assertion or milestone did not match — verify differently or record() the finding instead. Reset is destructive and does not undo server-side side effects.
|
|
657
715
|
- Be precise with locators (CSS or XPath)
|
|
@@ -665,6 +723,13 @@ export class Tester extends TaskAgent {
|
|
|
665
723
|
- When you interact with form with inputs, ensure that you click corresponding button to save its data
|
|
666
724
|
- Follow <locator_priority> rules when selecting locators for all tools
|
|
667
725
|
- Before retrying your actions check maybe they already achived expected results. Use see() tool for that
|
|
726
|
+
- If the current URL is already a create/edit/new form and the scenario is about creating/editing that entity, fill and submit that form. Do not click the list-page "New" button again from inside the form.
|
|
727
|
+
- If the scenario is about search/filter/sort/tabs/list inspection and the current URL is a create/edit/new form, go back or reset to the stable list page before interacting with list controls.
|
|
728
|
+
- When selecting related entities from a list, do not choose rows/options/cards marked as "0 items", "0 tests", or otherwise empty if the scenario requires selecting real content.
|
|
729
|
+
- In selection pickers, counters such as "Selected 0", "Matched tests 0", or disabled Save/Apply mean the selection did not register. Choose a non-empty item or change filters before submitting.
|
|
730
|
+
- A passed form/click command only means the command executed. If a required field remains empty, submit stays disabled, or the expected text is not visible, treat the action as not completed and correct the missing field/state.
|
|
731
|
+
- For filter/tab scenarios, success requires BOTH: the requested filter/tab is visibly active/selected AND the list content matches that filter. Do not finish from only one of these signals.
|
|
732
|
+
- Empty-state text such as "No matched items" only proves a filter when the requested filter/tab is active and the empty state belongs to the filtered list.
|
|
668
733
|
- When filling complex form with lot of actions performed, use see() to look which fields were filled and which are not
|
|
669
734
|
- When verify() fails, use see() to visually confirm the result — visual confirmation is equally valid evidence
|
|
670
735
|
- For visual state verification (active tabs, selected items, counts, colors), prefer see() over DOM-based verify()
|
|
@@ -693,6 +758,10 @@ export class Tester extends TaskAgent {
|
|
|
693
758
|
|
|
694
759
|
${formRequirementsRule}
|
|
695
760
|
|
|
761
|
+
${capabilityGroundingRule}
|
|
762
|
+
|
|
763
|
+
${dataProtectionRules}
|
|
764
|
+
|
|
696
765
|
${this.provider.getSystemPromptForAgent('tester', this.explorer.getStateManager().getCurrentState()?.url) || ''}
|
|
697
766
|
`;
|
|
698
767
|
}
|
|
@@ -714,10 +783,13 @@ export class Tester extends TaskAgent {
|
|
|
714
783
|
Try to achieve as many goals as possible.
|
|
715
784
|
If goal is not achievable, log that and skip to next one.
|
|
716
785
|
Do not hallucinate that goal was achieved when it was not.
|
|
786
|
+
If the scenario action could not be completed, do not finish with a verification of the failure state.
|
|
717
787
|
When creating or editing items via form() or type() you should include ${task.sessionName} in the value (if it is not restricted by the application logic)
|
|
718
788
|
Initial page URL: ${actionResult.url}
|
|
719
789
|
|
|
720
|
-
${
|
|
790
|
+
${capabilityGroundingRule}
|
|
791
|
+
|
|
792
|
+
${dataProtectionRules}
|
|
721
793
|
|
|
722
794
|
${this.buildDeletionScope(task)}
|
|
723
795
|
|
|
@@ -842,12 +914,13 @@ export class Tester extends TaskAgent {
|
|
|
842
914
|
}),
|
|
843
915
|
stop: tool({
|
|
844
916
|
description: dedent `
|
|
845
|
-
Stop the current test because
|
|
846
|
-
Use this
|
|
847
|
-
|
|
917
|
+
Stop the current test because it cannot be completed in the current session.
|
|
918
|
+
Use this when the scenario is incompatible, required UI/data is absent, or repeated varied attempts
|
|
919
|
+
show that automation cannot complete the workflow.
|
|
920
|
+
Do NOT use this immediately after the first failed action — retry with a materially different approach first.
|
|
848
921
|
`,
|
|
849
922
|
inputSchema: z.object({
|
|
850
|
-
reason: z.string().describe('Explanation why the scenario
|
|
923
|
+
reason: z.string().describe('Explanation why the scenario cannot be completed'),
|
|
851
924
|
}),
|
|
852
925
|
execute: async ({ reason }) => {
|
|
853
926
|
task.addNote(`Stop requested: ${reason}`);
|
|
@@ -881,6 +954,7 @@ export class Tester extends TaskAgent {
|
|
|
881
954
|
Provide a specific assertion to verify the final state.
|
|
882
955
|
The assertion MUST prove that YOUR ACTIONS changed the page state.
|
|
883
956
|
Do NOT verify something that was already true before you started testing.
|
|
957
|
+
Do NOT provide an assertion that verifies absence, failure, an empty state, or that the goal did not happen.
|
|
884
958
|
|
|
885
959
|
Examples of good assertions:
|
|
886
960
|
- "New user 'john@example.com' is visible in the users list"
|
|
@@ -991,4 +1065,45 @@ export class Tester extends TaskAgent {
|
|
|
991
1065
|
}),
|
|
992
1066
|
};
|
|
993
1067
|
}
|
|
1068
|
+
async handleLoopError(task, error) {
|
|
1069
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1070
|
+
if (!task.hasFinished)
|
|
1071
|
+
task.addNote(`Execution error: ${message}`);
|
|
1072
|
+
const result = await this.explorer.handleExecutionError(error);
|
|
1073
|
+
tag('info').log(`Browser supervisor: ${result.action} - ${result.message}`);
|
|
1074
|
+
task.addNote(result.message);
|
|
1075
|
+
if (result.action === 'stop') {
|
|
1076
|
+
task.finish(TestResult.FAILED);
|
|
1077
|
+
return 'stop';
|
|
1078
|
+
}
|
|
1079
|
+
if (result.recovered) {
|
|
1080
|
+
this.resetFailureCount();
|
|
1081
|
+
this.previousUrl = null;
|
|
1082
|
+
this.previousStateHash = null;
|
|
1083
|
+
this.stalledIterations = 0;
|
|
1084
|
+
}
|
|
1085
|
+
else if (this.shouldStopAfterStalledLoopError(task)) {
|
|
1086
|
+
return 'stop';
|
|
1087
|
+
}
|
|
1088
|
+
this.currentConversation?.addUserText(result.message);
|
|
1089
|
+
return 'continue';
|
|
1090
|
+
}
|
|
1091
|
+
shouldStopAfterStalledLoopError(task) {
|
|
1092
|
+
if (task.hasFinished)
|
|
1093
|
+
return false;
|
|
1094
|
+
this.stalledIterations++;
|
|
1095
|
+
if (this.stalledIterations < this.MAX_STALLED_ITERATIONS)
|
|
1096
|
+
return false;
|
|
1097
|
+
task.addNote('No browser progress after repeated execution errors', TestResult.FAILED);
|
|
1098
|
+
task.finish(TestResult.FAILED);
|
|
1099
|
+
return true;
|
|
1100
|
+
}
|
|
1101
|
+
async cleanupStartedTest(task) {
|
|
1102
|
+
await this.finishTest(task);
|
|
1103
|
+
await this.explorer.stopTest(task, {
|
|
1104
|
+
startUrl: task.startUrl,
|
|
1105
|
+
style: task.style,
|
|
1106
|
+
sessionName: task.sessionName,
|
|
1107
|
+
});
|
|
1108
|
+
}
|
|
994
1109
|
}
|