explorbot 0.1.26 → 0.1.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package.json +1 -1
- package/dist/src/action-result.js +2 -0
- package/dist/src/action.js +88 -7
- package/dist/src/ai/captain/file-tools.js +100 -0
- package/dist/src/ai/captain/idle-mode.js +70 -6
- package/dist/src/ai/captain/web-mode.js +36 -6
- package/dist/src/ai/captain.js +87 -19
- package/dist/src/ai/historian/screencast.js +11 -2
- package/dist/src/ai/navigator.js +5 -2
- package/dist/src/ai/pilot.js +33 -5
- package/dist/src/ai/researcher/cache.js +8 -0
- package/dist/src/ai/researcher/coordinates.js +2 -3
- package/dist/src/ai/researcher/deep-analysis.js +149 -65
- package/dist/src/ai/researcher/locators.js +1 -2
- package/dist/src/ai/researcher.js +17 -18
- package/dist/src/ai/task-agent.js +1 -0
- package/dist/src/ai/tester.js +91 -39
- package/dist/src/ai/tools.js +17 -7
- package/dist/src/commands/explore-command.js +6 -1
- package/dist/src/components/LogPane.js +4 -3
- package/dist/src/explorer.js +270 -35
- package/dist/src/utils/browser-errors.js +23 -0
- package/dist/src/utils/error-page.js +17 -2
- package/dist/src/utils/logger.js +2 -2
- package/package.json +1 -1
- package/src/action-result.ts +2 -0
- package/src/action.ts +83 -7
- package/src/ai/captain/file-tools.ts +126 -0
- package/src/ai/captain/idle-mode.ts +72 -6
- package/src/ai/captain/mixin.ts +1 -1
- package/src/ai/captain/web-mode.ts +40 -5
- package/src/ai/captain.ts +94 -20
- package/src/ai/historian/screencast.ts +11 -2
- package/src/ai/navigator.ts +6 -2
- package/src/ai/pilot.ts +34 -5
- package/src/ai/researcher/cache.ts +7 -0
- package/src/ai/researcher/coordinates.ts +2 -3
- package/src/ai/researcher/deep-analysis.ts +169 -72
- package/src/ai/researcher/locators.ts +1 -2
- package/src/ai/researcher.ts +17 -18
- package/src/ai/task-agent.ts +1 -1
- package/src/ai/tester.ts +101 -41
- package/src/ai/tools.ts +17 -7
- package/src/commands/explore-command.ts +6 -1
- package/src/components/LogPane.tsx +4 -3
- package/src/explorer.ts +295 -38
- package/src/state-manager.ts +2 -0
- package/src/utils/browser-errors.ts +25 -0
- package/src/utils/error-page.ts +16 -3
- package/src/utils/logger.ts +3 -3
|
@@ -2,9 +2,10 @@ import dedent from 'dedent';
|
|
|
2
2
|
import { ActionResult } from '../../action-result.js';
|
|
3
3
|
import { executionController } from "../../execution-controller.js";
|
|
4
4
|
import { detectFocusArea, diffAriaSnapshots } from "../../utils/aria.js";
|
|
5
|
+
import { extractCodeBlocks } from "../../utils/code-extractor.js";
|
|
5
6
|
import { tag } from '../../utils/logger.js';
|
|
6
7
|
import { mdq } from "../../utils/markdown-query.js";
|
|
7
|
-
import { getCachedResearch, saveResearch } from "./cache.js";
|
|
8
|
+
import { getCachedResearch, getPreviousResearch, saveResearch } from "./cache.js";
|
|
8
9
|
import { debugLog } from "./mixin.js";
|
|
9
10
|
import { parseResearchSections } from "./parser.js";
|
|
10
11
|
const DEFAULT_MAX_EXPANDABLE_CLICKS = 10;
|
|
@@ -13,13 +14,26 @@ export function WithDeepAnalysis(Base) {
|
|
|
13
14
|
async performDeepAnalysis(state, result) {
|
|
14
15
|
tag('info').log('Starting deep analysis of expandable elements');
|
|
15
16
|
await this.navigateTo(state.fullUrl || state.url);
|
|
16
|
-
let expandables = await this._discoverExpandables(result.text);
|
|
17
|
-
if (expandables.length === 0) {
|
|
18
|
-
tag('info').log('No expandable elements identified by AI');
|
|
19
|
-
return;
|
|
20
|
-
}
|
|
21
|
-
tag('substep').log(`Identified ${expandables.length} expandable elements`);
|
|
22
17
|
const maxClicks = this.explorer.getConfig().ai?.agents?.researcher?.maxExpandableClicks ?? DEFAULT_MAX_EXPANDABLE_CLICKS;
|
|
18
|
+
const expandedSections = [];
|
|
19
|
+
const navigationLinks = [];
|
|
20
|
+
let verifiedCodes = [];
|
|
21
|
+
let missing = [];
|
|
22
|
+
const previousSections = this._loadPreviousExtendedSections(state.hash || '');
|
|
23
|
+
if (previousSections.length > 0) {
|
|
24
|
+
tag('substep').log(`Replaying ${previousSections.length} previously discovered sections`);
|
|
25
|
+
const replay = await this._replayPreviousSections(state, previousSections, maxClicks);
|
|
26
|
+
expandedSections.push(...replay.verified);
|
|
27
|
+
verifiedCodes = replay.verifiedCodes;
|
|
28
|
+
missing = replay.missing;
|
|
29
|
+
tag('info').log(`Reused ${replay.verified.length}/${previousSections.length} previous sections, ${missing.length} to re-discover`);
|
|
30
|
+
if (missing.length === 0 && replay.verified.length >= maxClicks) {
|
|
31
|
+
tag('info').log('Page appears unchanged, reusing previous sections and skipping discovery');
|
|
32
|
+
this._appendExtendedResearch(result, expandedSections, navigationLinks);
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
let expandables = await this._discoverExpandables(result.text, missing, verifiedCodes);
|
|
23
37
|
if (expandables.length > maxClicks) {
|
|
24
38
|
expandables = await this._selectExpandables(expandables, state.fullUrl || state.url, maxClicks);
|
|
25
39
|
tag('substep').log(`Selected ${expandables.length} expandables to click (max: ${maxClicks})`);
|
|
@@ -29,9 +43,11 @@ export function WithDeepAnalysis(Base) {
|
|
|
29
43
|
commands: this._buildClickCommands(el),
|
|
30
44
|
description: el.name,
|
|
31
45
|
}))
|
|
32
|
-
.filter((el) => el.commands.length > 0)
|
|
46
|
+
.filter((el) => el.commands.length > 0)
|
|
47
|
+
.filter((el) => !el.commands.some((cmd) => verifiedCodes.includes(cmd)));
|
|
33
48
|
if (elements.length === 0) {
|
|
34
|
-
tag('info').log('No
|
|
49
|
+
tag('info').log('No new expandable elements to click');
|
|
50
|
+
this._appendExtendedResearch(result, expandedSections, navigationLinks);
|
|
35
51
|
return;
|
|
36
52
|
}
|
|
37
53
|
const expandableRows = elements.map((el) => `| ${el.description} | \`${el.commands[0]}\` |`).join('\n');
|
|
@@ -39,21 +55,9 @@ export function WithDeepAnalysis(Base) {
|
|
|
39
55
|
for (const el of elements)
|
|
40
56
|
debugLog(`Expandable: ${el.description} → ${el.commands[0]}`);
|
|
41
57
|
tag('substep').log(`Clicking ${elements.length} expandable elements`);
|
|
42
|
-
const expandedSections = [];
|
|
43
|
-
const navigationLinks = [];
|
|
44
58
|
await this._clickExpandableElements(elements, state, expandedSections, navigationLinks);
|
|
45
59
|
tag('info').log(`Deep analysis complete. Sections: ${expandedSections.length}, navigation links: ${navigationLinks.length}`);
|
|
46
|
-
|
|
47
|
-
if (dedupedSections.length !== expandedSections.length) {
|
|
48
|
-
tag('substep').log(`Deduplicated ${expandedSections.length} → ${dedupedSections.length} extended sections`);
|
|
49
|
-
}
|
|
50
|
-
if (dedupedSections.length > 0) {
|
|
51
|
-
result.text += `\n\n# Extended Research\n\n${dedupedSections.join('\n\n---\n\n')}`;
|
|
52
|
-
}
|
|
53
|
-
if (navigationLinks.length > 0) {
|
|
54
|
-
const links = navigationLinks.map((l) => `- \`${l.code}\` opens ${l.url}`).join('\n');
|
|
55
|
-
result.text += `\n\n## Navigation Links\n\n${links}`;
|
|
56
|
-
}
|
|
60
|
+
this._appendExtendedResearch(result, expandedSections, navigationLinks);
|
|
57
61
|
}
|
|
58
62
|
async researchOverlay(current, previous, pageStateHash) {
|
|
59
63
|
const focusArea = detectFocusArea(current.ariaSnapshot);
|
|
@@ -97,7 +101,66 @@ export function WithDeepAnalysis(Base) {
|
|
|
97
101
|
tag('substep').log(`Overlay research appended: ${focusArea.name}`);
|
|
98
102
|
return sectionMarkdown;
|
|
99
103
|
}
|
|
100
|
-
|
|
104
|
+
_loadPreviousExtendedSections(hash) {
|
|
105
|
+
if (!hash)
|
|
106
|
+
return [];
|
|
107
|
+
const previous = getPreviousResearch(hash);
|
|
108
|
+
if (!previous)
|
|
109
|
+
return [];
|
|
110
|
+
const sections = [];
|
|
111
|
+
for (const section of parseResearchSections(previous)) {
|
|
112
|
+
if (!section.isExtended)
|
|
113
|
+
continue;
|
|
114
|
+
const code = extractCodeBlocks(section.rawMarkdown)[0];
|
|
115
|
+
if (!code)
|
|
116
|
+
continue;
|
|
117
|
+
sections.push({ name: section.name, code });
|
|
118
|
+
}
|
|
119
|
+
return sections;
|
|
120
|
+
}
|
|
121
|
+
async _replayPreviousSections(state, prevSections, maxClicks) {
|
|
122
|
+
const originalAria = state.ariaSnapshot || '';
|
|
123
|
+
const verified = [];
|
|
124
|
+
const verifiedCodes = [];
|
|
125
|
+
const missing = [];
|
|
126
|
+
for (const section of prevSections.slice(0, maxClicks)) {
|
|
127
|
+
if (executionController.isInterrupted())
|
|
128
|
+
break;
|
|
129
|
+
let outcome;
|
|
130
|
+
try {
|
|
131
|
+
outcome = await this._executeAndAnalyze([section.code], section.name, state, originalAria, this._summarizeExpanded(verified));
|
|
132
|
+
}
|
|
133
|
+
catch (err) {
|
|
134
|
+
tag('warning').log(`Replay failed for "${section.name}": ${err instanceof Error ? err.message : err}`);
|
|
135
|
+
await this._restorePageState(state.url, originalAria).catch(() => { });
|
|
136
|
+
missing.push(section);
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
if (outcome.status === 'revealed') {
|
|
140
|
+
verified.push(outcome.sectionMarkdown);
|
|
141
|
+
verifiedCodes.push(section.code);
|
|
142
|
+
debugLog(`Replayed and verified section: ${section.name}`);
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
debugLog(`Could not replay previous section: ${section.name}`);
|
|
146
|
+
missing.push(section);
|
|
147
|
+
}
|
|
148
|
+
return { verified, verifiedCodes, missing };
|
|
149
|
+
}
|
|
150
|
+
_appendExtendedResearch(result, expandedSections, navigationLinks) {
|
|
151
|
+
const dedupedSections = this._deduplicateExpandedSections(expandedSections);
|
|
152
|
+
if (dedupedSections.length !== expandedSections.length) {
|
|
153
|
+
tag('substep').log(`Deduplicated ${expandedSections.length} → ${dedupedSections.length} extended sections`);
|
|
154
|
+
}
|
|
155
|
+
if (dedupedSections.length > 0) {
|
|
156
|
+
result.text += `\n\n# Extended Research\n\n${dedupedSections.join('\n\n---\n\n')}`;
|
|
157
|
+
}
|
|
158
|
+
if (navigationLinks.length > 0) {
|
|
159
|
+
const links = navigationLinks.map((l) => `- \`${l.code}\` opens ${l.url}`).join('\n');
|
|
160
|
+
result.text += `\n\n## Navigation Links\n\n${links}`;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
async _discoverExpandables(researchText, missing = [], verifiedCodes = []) {
|
|
101
164
|
const allElements = new Map();
|
|
102
165
|
for (const section of parseResearchSections(researchText)) {
|
|
103
166
|
for (const el of section.elements) {
|
|
@@ -108,6 +171,15 @@ export function WithDeepAnalysis(Base) {
|
|
|
108
171
|
if (allElements.size === 0)
|
|
109
172
|
return [];
|
|
110
173
|
const eidxList = [...allElements.keys()].join(', ');
|
|
174
|
+
let missingHint = '';
|
|
175
|
+
if (missing.length > 0) {
|
|
176
|
+
const list = missing.map((s) => `- "${s.name}" (previously revealed via ${s.code})`).join('\n');
|
|
177
|
+
missingHint = dedent `
|
|
178
|
+
|
|
179
|
+
These sections were present on a previous visit but their trigger could not be replayed now — the element may have moved or been renamed. Prioritize finding the element that now reveals each:
|
|
180
|
+
${list}
|
|
181
|
+
`;
|
|
182
|
+
}
|
|
111
183
|
const textPrompt = dedent `
|
|
112
184
|
From this UI research, identify elements that could reveal hidden UI when clicked
|
|
113
185
|
(dropdown menus, popups, expandable panels, accordion sections, overflow menus, tab switches).
|
|
@@ -115,6 +187,7 @@ export function WithDeepAnalysis(Base) {
|
|
|
115
187
|
Available eidx refs: ${eidxList}
|
|
116
188
|
|
|
117
189
|
${researchText}
|
|
190
|
+
${missingHint}
|
|
118
191
|
|
|
119
192
|
Rules:
|
|
120
193
|
- Only pick elements that HIDE content until clicked (menus, dropdowns, accordions, tabs)
|
|
@@ -136,6 +209,7 @@ export function WithDeepAnalysis(Base) {
|
|
|
136
209
|
|
|
137
210
|
Look for: overflow/ellipsis menus, chevron dropdowns, hamburger menus,
|
|
138
211
|
gear/settings buttons, accordion toggles, tab switches, filter buttons.
|
|
212
|
+
${missingHint}
|
|
139
213
|
|
|
140
214
|
Rules:
|
|
141
215
|
- For repeated icons (same icon on every list row), pick only the FIRST one
|
|
@@ -238,10 +312,9 @@ export function WithDeepAnalysis(Base) {
|
|
|
238
312
|
const isCoordinateClick = el.commands[0].startsWith('I.clickXY(');
|
|
239
313
|
if (!isCoordinateClick) {
|
|
240
314
|
const hoverCmd = el.commands[0].replace('I.click(', 'I.moveCursorTo(');
|
|
241
|
-
|
|
242
|
-
await hoverAction.attempt(hoverCmd, undefined, false);
|
|
315
|
+
await this.explorer.attemptAction(hoverCmd, undefined, false);
|
|
243
316
|
await new Promise((r) => setTimeout(r, 500));
|
|
244
|
-
await this.explorer.
|
|
317
|
+
await this.explorer.capturePageState();
|
|
245
318
|
const hoverAR = ActionResult.fromState(this.stateManager.getCurrentState());
|
|
246
319
|
const hoverDiff = await hoverAR.diff(previousState);
|
|
247
320
|
await hoverDiff.calculate();
|
|
@@ -258,48 +331,15 @@ export function WithDeepAnalysis(Base) {
|
|
|
258
331
|
await this._restorePageState(state.url, originalAria);
|
|
259
332
|
}
|
|
260
333
|
}
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
if (await action.attempt(cmd, undefined, false)) {
|
|
265
|
-
clickCode = cmd;
|
|
266
|
-
break;
|
|
267
|
-
}
|
|
268
|
-
}
|
|
269
|
-
if (!clickCode) {
|
|
270
|
-
debugLog(`Click failed: ${el.description.slice(0, 80)}`);
|
|
334
|
+
const outcome = await this._executeAndAnalyze(el.commands, el.description, state, originalAria, this._summarizeExpanded(expandedSections));
|
|
335
|
+
if (outcome.status === 'navigated') {
|
|
336
|
+
navigationLinks.push({ code: outcome.code, url: outcome.url });
|
|
271
337
|
continue;
|
|
272
338
|
}
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
try {
|
|
276
|
-
await this.explorer.createAction().capturePageState();
|
|
277
|
-
const currAR = ActionResult.fromState(this.stateManager.getCurrentState());
|
|
278
|
-
diff = await currAR.diff(previousState);
|
|
279
|
-
await diff.calculate();
|
|
280
|
-
}
|
|
281
|
-
catch (err) {
|
|
282
|
-
tag('warning').log(`State capture failed after click: ${err instanceof Error ? err.message : err}`);
|
|
283
|
-
await this._restorePageState(state.url, originalAria);
|
|
284
|
-
continue;
|
|
285
|
-
}
|
|
286
|
-
if (diff.urlHasChanged()) {
|
|
287
|
-
debugLog(`Click navigated to ${this.stateManager.getCurrentState()?.url}`);
|
|
288
|
-
navigationLinks.push({ code: clickCode, url: this.stateManager.getCurrentState()?.url || '' });
|
|
289
|
-
await this.navigateTo(state.url);
|
|
290
|
-
continue;
|
|
291
|
-
}
|
|
292
|
-
const clickHtmlSize = diff.htmlParts.reduce((sum, p) => sum + p.subtree.length, 0);
|
|
293
|
-
if (!diff.ariaChanged && clickHtmlSize <= 150) {
|
|
294
|
-
debugLog(`No changes from: ${el.description.slice(0, 80)}`);
|
|
295
|
-
continue;
|
|
296
|
-
}
|
|
297
|
-
const sectionMarkdown = await this._analyzeExpandedAction(clickCode, el.description, diff, this._summarizeExpanded(expandedSections));
|
|
298
|
-
if (sectionMarkdown) {
|
|
299
|
-
expandedSections.push(sectionMarkdown);
|
|
339
|
+
if (outcome.status === 'revealed') {
|
|
340
|
+
expandedSections.push(outcome.sectionMarkdown);
|
|
300
341
|
debugLog(`Captured section from: ${el.description.slice(0, 80)}`);
|
|
301
342
|
}
|
|
302
|
-
await this._restorePageState(state.url, originalAria);
|
|
303
343
|
}
|
|
304
344
|
catch (err) {
|
|
305
345
|
tag('warning').log(`Expandable click failed for "${el.description.slice(0, 80)}": ${err instanceof Error ? err.message : err}`);
|
|
@@ -310,10 +350,54 @@ export function WithDeepAnalysis(Base) {
|
|
|
310
350
|
}
|
|
311
351
|
}
|
|
312
352
|
}
|
|
353
|
+
async _executeAndAnalyze(commands, description, state, originalAria, alreadyExpanded) {
|
|
354
|
+
const previousState = ActionResult.fromState(this.stateManager.getCurrentState());
|
|
355
|
+
let clickCode = null;
|
|
356
|
+
const action = this.explorer.createAction();
|
|
357
|
+
for (const cmd of commands) {
|
|
358
|
+
if (await action.attempt(cmd, undefined, false)) {
|
|
359
|
+
clickCode = cmd;
|
|
360
|
+
break;
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
if (!clickCode) {
|
|
364
|
+
debugLog(`Click failed: ${description.slice(0, 80)}`);
|
|
365
|
+
return { status: 'failed' };
|
|
366
|
+
}
|
|
367
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
368
|
+
let diff;
|
|
369
|
+
try {
|
|
370
|
+
await this.explorer.createAction().capturePageState();
|
|
371
|
+
const currAR = ActionResult.fromState(this.stateManager.getCurrentState());
|
|
372
|
+
diff = await currAR.diff(previousState);
|
|
373
|
+
await diff.calculate();
|
|
374
|
+
}
|
|
375
|
+
catch (err) {
|
|
376
|
+
tag('warning').log(`State capture failed after click: ${err instanceof Error ? err.message : err}`);
|
|
377
|
+
await this._restorePageState(state.url, originalAria);
|
|
378
|
+
return { status: 'failed' };
|
|
379
|
+
}
|
|
380
|
+
if (diff.urlHasChanged()) {
|
|
381
|
+
const url = this.stateManager.getCurrentState()?.url || '';
|
|
382
|
+
debugLog(`Click navigated to ${url}`);
|
|
383
|
+
await this.navigateTo(state.url);
|
|
384
|
+
return { status: 'navigated', code: clickCode, url };
|
|
385
|
+
}
|
|
386
|
+
const clickHtmlSize = diff.htmlParts.reduce((sum, p) => sum + p.subtree.length, 0);
|
|
387
|
+
if (!diff.ariaChanged && clickHtmlSize <= 150) {
|
|
388
|
+
debugLog(`No changes from: ${description.slice(0, 80)}`);
|
|
389
|
+
return { status: 'none', code: clickCode };
|
|
390
|
+
}
|
|
391
|
+
const sectionMarkdown = await this._analyzeExpandedAction(clickCode, description, diff, alreadyExpanded);
|
|
392
|
+
await this._restorePageState(state.url, originalAria);
|
|
393
|
+
if (!sectionMarkdown)
|
|
394
|
+
return { status: 'none', code: clickCode };
|
|
395
|
+
return { status: 'revealed', code: clickCode, sectionMarkdown };
|
|
396
|
+
}
|
|
313
397
|
async _restorePageState(url, originalAria) {
|
|
314
398
|
try {
|
|
315
399
|
await this.cancelInUi();
|
|
316
|
-
await this.explorer.
|
|
400
|
+
await this.explorer.capturePageState();
|
|
317
401
|
const currentAria = this.stateManager.getCurrentState()?.ariaSnapshot || '';
|
|
318
402
|
if (!diffAriaSnapshots(originalAria, currentAria))
|
|
319
403
|
return;
|
|
@@ -173,8 +173,7 @@ export function WithLocators(Base) {
|
|
|
173
173
|
}
|
|
174
174
|
}
|
|
175
175
|
if (needsXpath.length > 0) {
|
|
176
|
-
const
|
|
177
|
-
const webElements = await WebElement.fromEidxList(page, needsXpath);
|
|
176
|
+
const webElements = await this.explorer.runWithBrowserRecovery('backfillBrokenLocators', () => WebElement.fromEidxList(this.explorer.playwrightHelper.page, needsXpath));
|
|
178
177
|
const changedSections = new Set();
|
|
179
178
|
for (const w of webElements) {
|
|
180
179
|
const entry = needsXpathEls.get(w.eidx);
|
|
@@ -97,11 +97,11 @@ export class Researcher extends ResearcherBase {
|
|
|
97
97
|
await this.hooksRunner.runBeforeHook('researcher', state.url);
|
|
98
98
|
const annotatedElements = await this.explorer.annotateElements();
|
|
99
99
|
debugLog(`Annotated ${annotatedElements.length} interactive elements with eidx`);
|
|
100
|
-
this.actionResult = await this.explorer.
|
|
100
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot && this.provider.hasVision() });
|
|
101
101
|
const condition = detectPageCondition(this.actionResult);
|
|
102
102
|
if (condition === 'error') {
|
|
103
103
|
tag('warning').log(`Detected error page at ${state.url}`);
|
|
104
|
-
throw new ErrorPageError(state.url, this.actionResult.title);
|
|
104
|
+
throw new ErrorPageError(state.url, this.actionResult.title, this.actionResult.httpStatus);
|
|
105
105
|
}
|
|
106
106
|
if (condition === 'loading') {
|
|
107
107
|
const settled = await this.waitUntilSettled(screenshot);
|
|
@@ -191,7 +191,7 @@ export class Researcher extends ResearcherBase {
|
|
|
191
191
|
// Must run BEFORE visuallyAnnotateContainers — annotation overlays inject z-index 99998+ which would pollute the scoring.
|
|
192
192
|
if (!interrupted() && this.hasScreenshotToAnalyze) {
|
|
193
193
|
const sections = parseResearchSections(result.text);
|
|
194
|
-
const focused = await detectFocusedSection(this.explorer.playwrightHelper.page, sections);
|
|
194
|
+
const focused = await this.explorer.runWithBrowserRecovery('detectFocusedSection', () => detectFocusedSection(this.explorer.playwrightHelper.page, sections));
|
|
195
195
|
if (focused)
|
|
196
196
|
markSectionAsFocused(result, focused);
|
|
197
197
|
}
|
|
@@ -204,7 +204,7 @@ export class Researcher extends ResearcherBase {
|
|
|
204
204
|
const freshBroken = freshContainerLocs.filter((l) => l.valid === false).map((l) => l.locator);
|
|
205
205
|
const containers = validContainers.filter((c) => !freshBroken.includes(c.css));
|
|
206
206
|
await this.visuallyAnnotateElements({ containers });
|
|
207
|
-
this.actionResult = await this.explorer.
|
|
207
|
+
this.actionResult = await this.explorer.capturePageWithScreenshot();
|
|
208
208
|
const visualResult = await this.analyzeScreenshotForVisualProps();
|
|
209
209
|
if (visualResult.elements.size > 0) {
|
|
210
210
|
await this.mergeVisualData(result, visualResult.elements);
|
|
@@ -277,7 +277,7 @@ export class Researcher extends ResearcherBase {
|
|
|
277
277
|
if (!this.actionResult) {
|
|
278
278
|
debugLog('No action result, navigating to URL');
|
|
279
279
|
await this.explorer.visit(url);
|
|
280
|
-
this.actionResult = await this.explorer.
|
|
280
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot });
|
|
281
281
|
return;
|
|
282
282
|
}
|
|
283
283
|
const isOnCurrentState = this.actionResult.getStateHash() === this.stateManager.getCurrentState()?.hash;
|
|
@@ -285,46 +285,47 @@ export class Researcher extends ResearcherBase {
|
|
|
285
285
|
const isEmpty = isBodyEmpty(stateHtml);
|
|
286
286
|
if (!isEmpty && isOnCurrentState) {
|
|
287
287
|
if ((!this.actionResult.screenshot && screenshot) || !this.actionResult.ariaSnapshot) {
|
|
288
|
-
this.actionResult = await this.explorer.
|
|
288
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot });
|
|
289
289
|
}
|
|
290
290
|
return;
|
|
291
291
|
}
|
|
292
292
|
if (isEmpty && isOnCurrentState) {
|
|
293
293
|
debugLog('HTML body empty on current URL, waiting for content');
|
|
294
294
|
tag('step').log('Page body is empty, waiting for content...');
|
|
295
|
+
await this.explorer.visit(url);
|
|
296
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false });
|
|
295
297
|
await this.waitUntilSettled(screenshot ?? false);
|
|
296
298
|
return;
|
|
297
299
|
}
|
|
298
300
|
debugLog('Not on current state, navigating to URL');
|
|
299
301
|
tag('step').log('Navigating to URL...');
|
|
300
302
|
await this.explorer.visit(url);
|
|
301
|
-
this.actionResult = await this.explorer.
|
|
303
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false });
|
|
302
304
|
}
|
|
303
305
|
async waitUntilSettled(screenshot) {
|
|
304
306
|
const errorPageTimeout = this.explorer.getConfig().ai?.agents?.researcher?.errorPageTimeout ?? 10;
|
|
305
307
|
if (errorPageTimeout <= 0)
|
|
306
308
|
return false;
|
|
307
|
-
const page = this.explorer.playwrightHelper.page;
|
|
308
309
|
const includeScreenshot = screenshot && this.provider.hasVision();
|
|
309
310
|
try {
|
|
310
|
-
await page?.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 });
|
|
311
|
+
await this.explorer.runWithBrowserRecovery('waitUntilSettled', () => this.explorer.playwrightHelper.page?.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 }));
|
|
311
312
|
}
|
|
312
313
|
catch { }
|
|
313
314
|
await this.explorer.annotateElements();
|
|
314
|
-
this.actionResult = await this.explorer.
|
|
315
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot });
|
|
315
316
|
let condition = detectPageCondition(this.actionResult);
|
|
316
317
|
if (condition === 'error') {
|
|
317
|
-
throw new ErrorPageError(this.actionResult.url, this.actionResult.title);
|
|
318
|
+
throw new ErrorPageError(this.actionResult.url, this.actionResult.title, this.actionResult.httpStatus);
|
|
318
319
|
}
|
|
319
320
|
if (condition === 'ok')
|
|
320
321
|
return true;
|
|
321
322
|
for (let i = 0; i < 3; i++) {
|
|
322
323
|
await new Promise((r) => setTimeout(r, 1000));
|
|
323
324
|
await this.explorer.annotateElements();
|
|
324
|
-
this.actionResult = await this.explorer.
|
|
325
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot });
|
|
325
326
|
condition = detectPageCondition(this.actionResult);
|
|
326
327
|
if (condition === 'error') {
|
|
327
|
-
throw new ErrorPageError(this.actionResult.url, this.actionResult.title);
|
|
328
|
+
throw new ErrorPageError(this.actionResult.url, this.actionResult.title, this.actionResult.httpStatus);
|
|
328
329
|
}
|
|
329
330
|
if (condition === 'ok')
|
|
330
331
|
return true;
|
|
@@ -681,15 +682,13 @@ export class Researcher extends ResearcherBase {
|
|
|
681
682
|
.join('\n\n');
|
|
682
683
|
}
|
|
683
684
|
async navigateTo(url) {
|
|
684
|
-
|
|
685
|
-
await action.execute(`I.amOnPage("${url}")`);
|
|
685
|
+
await this.explorer.visit(url);
|
|
686
686
|
}
|
|
687
687
|
async cancelInUi() {
|
|
688
688
|
const beforeAria = this.stateManager.getCurrentState()?.ariaSnapshot || null;
|
|
689
|
-
|
|
690
|
-
await action.execute('I.clickXY(0, 0)');
|
|
689
|
+
await this.explorer.executeAction('I.clickXY(0, 0)');
|
|
691
690
|
if (diffAriaSnapshots(beforeAria, this.stateManager.getCurrentState()?.ariaSnapshot || null))
|
|
692
691
|
return;
|
|
693
|
-
await
|
|
692
|
+
await this.explorer.executeAction(`I.pressKey('Escape')`);
|
|
694
693
|
}
|
|
695
694
|
}
|
package/dist/src/ai/tester.js
CHANGED
|
@@ -4,13 +4,13 @@ import { tool } from 'ai';
|
|
|
4
4
|
import dedent from 'dedent';
|
|
5
5
|
import { z } from 'zod';
|
|
6
6
|
import { ActionResult } from "../action-result.js";
|
|
7
|
-
import { setActivity } from "../activity.js";
|
|
7
|
+
import { clearActivity, setActivity } from "../activity.js";
|
|
8
8
|
import { ConfigParser } from "../config.js";
|
|
9
9
|
import { Observability } from "../observability.js";
|
|
10
10
|
import { Stats } from "../stats.js";
|
|
11
11
|
import { TestResult } from "../test-plan.js";
|
|
12
12
|
import { detectFocusArea, extractFocusedElement } from "../utils/aria.js";
|
|
13
|
-
import { ErrorPageError } from "../utils/error-page.js";
|
|
13
|
+
import { ErrorPageError, isErrorPage } from "../utils/error-page.js";
|
|
14
14
|
import { HooksRunner } from "../utils/hooks-runner.js";
|
|
15
15
|
import { createDebug, tag } from "../utils/logger.js";
|
|
16
16
|
import { loop } from "../utils/loop.js";
|
|
@@ -107,18 +107,13 @@ export class Tester extends TaskAgent {
|
|
|
107
107
|
const offFailedRequest = requestStore?.onFailedRequest((r) => {
|
|
108
108
|
task.addNote(`Network error: ${r.method} ${r.path} → ${r.status}`, TestResult.FAILED);
|
|
109
109
|
});
|
|
110
|
-
const page = this.explorer.playwrightHelper?.page;
|
|
111
|
-
const onPageError = (err) => {
|
|
112
|
-
task.addNote(`Console error: ${err.message}`, TestResult.FAILED);
|
|
113
|
-
};
|
|
114
|
-
const onConsoleMessage = (msg) => {
|
|
115
|
-
if (msg.type() !== 'error')
|
|
116
|
-
return;
|
|
117
|
-
task.addNote(`Console error: ${msg.text()}`, TestResult.FAILED);
|
|
118
|
-
};
|
|
119
|
-
page?.on('pageerror', onPageError);
|
|
120
|
-
page?.on('console', onConsoleMessage);
|
|
121
110
|
const initialState = ActionResult.fromState(state);
|
|
111
|
+
if (isErrorPage(initialState)) {
|
|
112
|
+
task.start();
|
|
113
|
+
await this.explorer.startTest(task);
|
|
114
|
+
offFailedRequest?.();
|
|
115
|
+
return await this.abortStartedTestOnErrorPage(task, initialState);
|
|
116
|
+
}
|
|
122
117
|
const conversation = this.provider.startConversation(this.getSystemMessage(), 'tester');
|
|
123
118
|
conversation.markLastMessageCacheable();
|
|
124
119
|
this.currentConversation = conversation;
|
|
@@ -140,17 +135,15 @@ export class Tester extends TaskAgent {
|
|
|
140
135
|
startUrl: task.startUrl,
|
|
141
136
|
expected: task.expected,
|
|
142
137
|
},
|
|
143
|
-
}, async () => this.runTestSession(task, initialState, conversation, { offFailedRequest
|
|
138
|
+
}, async () => this.runTestSession(task, initialState, conversation, { offFailedRequest }));
|
|
144
139
|
}
|
|
145
140
|
async runTestSession(task, initialState, conversation, handlers) {
|
|
146
|
-
const { offFailedRequest
|
|
141
|
+
const { offFailedRequest } = handlers;
|
|
147
142
|
if (this.pilot) {
|
|
148
143
|
try {
|
|
149
144
|
const plan = await this.pilot.planTest(task, initialState);
|
|
150
145
|
if (task.hasFinished) {
|
|
151
146
|
offFailedRequest?.();
|
|
152
|
-
page?.off('pageerror', onPageError);
|
|
153
|
-
page?.off('console', onConsoleMessage);
|
|
154
147
|
return { success: task.isSuccessful };
|
|
155
148
|
}
|
|
156
149
|
if (plan) {
|
|
@@ -163,19 +156,36 @@ export class Tester extends TaskAgent {
|
|
|
163
156
|
task.addNote(`Planning failed: ${message}`, TestResult.FAILED);
|
|
164
157
|
task.finish(TestResult.FAILED);
|
|
165
158
|
offFailedRequest?.();
|
|
166
|
-
page?.off('pageerror', onPageError);
|
|
167
|
-
page?.off('console', onConsoleMessage);
|
|
168
159
|
return { success: false };
|
|
169
160
|
}
|
|
170
161
|
}
|
|
171
162
|
debugLog('Starting test execution with tools');
|
|
172
|
-
|
|
173
|
-
|
|
163
|
+
if (!(await this.explorer.startTest(task))) {
|
|
164
|
+
offFailedRequest?.();
|
|
165
|
+
await this.cleanupStartedTest(task);
|
|
166
|
+
return { success: task.isSuccessful };
|
|
167
|
+
}
|
|
174
168
|
debugLog(`Navigating to ${task.startUrl}`);
|
|
175
|
-
|
|
169
|
+
try {
|
|
170
|
+
await this.explorer.visit(task.startUrl);
|
|
171
|
+
}
|
|
172
|
+
catch (error) {
|
|
173
|
+
const result = await this.handleLoopError(task, error);
|
|
174
|
+
if (result === 'stop') {
|
|
175
|
+
offFailedRequest?.();
|
|
176
|
+
await this.cleanupStartedTest(task);
|
|
177
|
+
return { success: task.isSuccessful };
|
|
178
|
+
}
|
|
179
|
+
}
|
|
176
180
|
const startState = this.explorer.getStateManager().getCurrentState();
|
|
177
|
-
if (startState)
|
|
181
|
+
if (startState) {
|
|
178
182
|
task.addUrlNote(startState);
|
|
183
|
+
const startActionResult = ActionResult.fromState(startState);
|
|
184
|
+
if (isErrorPage(startActionResult)) {
|
|
185
|
+
offFailedRequest?.();
|
|
186
|
+
return await this.abortStartedTestOnErrorPage(task, startActionResult);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
179
189
|
const currentUrl = startState?.url || task.startUrl || '';
|
|
180
190
|
await this.hooksRunner.runBeforeHook('tester', currentUrl);
|
|
181
191
|
const offStateChange = this.explorer.getStateManager().onStateChange((event) => {
|
|
@@ -195,6 +205,12 @@ export class Tester extends TaskAgent {
|
|
|
195
205
|
shouldContinue = false;
|
|
196
206
|
await loop(async ({ stop, pause, iteration, userInput }) => {
|
|
197
207
|
debugLog('iteration', iteration);
|
|
208
|
+
if (!(await this.explorer.ensurePageAvailable())) {
|
|
209
|
+
task.addNote('Browser page is unavailable');
|
|
210
|
+
task.finish(TestResult.FAILED);
|
|
211
|
+
stop();
|
|
212
|
+
return;
|
|
213
|
+
}
|
|
198
214
|
const currentState = this.getCurrentState();
|
|
199
215
|
const tools = {
|
|
200
216
|
...codeceptjsTools,
|
|
@@ -327,20 +343,15 @@ export class Tester extends TaskAgent {
|
|
|
327
343
|
}
|
|
328
344
|
: undefined,
|
|
329
345
|
catch: async ({ error, stop }) => {
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
if (!task.hasFinished) {
|
|
333
|
-
task.addNote(`Execution error: ${message}`);
|
|
334
|
-
}
|
|
335
|
-
if (error instanceof Error && error.name === 'AbortError') {
|
|
346
|
+
const result = await this.handleLoopError(task, error);
|
|
347
|
+
if (result === 'stop')
|
|
336
348
|
stop();
|
|
337
|
-
return;
|
|
338
|
-
}
|
|
339
|
-
conversation.addUserText(`Previous AI call failed: ${message}. Take a different approach on the next step.`);
|
|
340
349
|
},
|
|
341
350
|
});
|
|
342
351
|
if (task.hasFinished)
|
|
343
352
|
break;
|
|
353
|
+
if (!(await this.explorer.ensurePageAvailable()))
|
|
354
|
+
break;
|
|
344
355
|
const finalState = this.getCurrentState();
|
|
345
356
|
const wantsContinue = await this.pilot.finalReview(task, finalState, conversation, this.navigator);
|
|
346
357
|
if (!wantsContinue || task.hasFinished)
|
|
@@ -366,14 +377,8 @@ export class Tester extends TaskAgent {
|
|
|
366
377
|
await this.getQuartermaster().analyzeSession(task, initialState, conversation);
|
|
367
378
|
offStateChange();
|
|
368
379
|
offFailedRequest?.();
|
|
369
|
-
page?.off('pageerror', onPageError);
|
|
370
|
-
page?.off('console', onConsoleMessage);
|
|
371
380
|
await this.finishTest(task);
|
|
372
|
-
await this.explorer.stopTest(task,
|
|
373
|
-
startUrl: task.startUrl,
|
|
374
|
-
style: task.style,
|
|
375
|
-
sessionName: task.sessionName,
|
|
376
|
-
});
|
|
381
|
+
await this.explorer.stopTest(task, this.buildStopTestMeta(task));
|
|
377
382
|
return {
|
|
378
383
|
success: task.isSuccessful,
|
|
379
384
|
...task,
|
|
@@ -607,6 +612,26 @@ export class Tester extends TaskAgent {
|
|
|
607
612
|
tag('warning').log(`Test with no result: ${task.scenario}`);
|
|
608
613
|
}
|
|
609
614
|
}
|
|
615
|
+
async abortStartedTestOnErrorPage(task, actionResult) {
|
|
616
|
+
const error = new ErrorPageError(actionResult.url || task.startUrl || '', actionResult.title, actionResult.httpStatus);
|
|
617
|
+
tag('warning').log(error.message);
|
|
618
|
+
task.addNote(error.message, TestResult.FAILED, actionResult.screenshotFile, actionResult.fullUrl || actionResult.url);
|
|
619
|
+
task.finish(TestResult.FAILED);
|
|
620
|
+
this.finishTest(task);
|
|
621
|
+
await this.explorer.stopTest(task, this.buildStopTestMeta(task));
|
|
622
|
+
clearActivity(true);
|
|
623
|
+
return { success: false };
|
|
624
|
+
}
|
|
625
|
+
buildStopTestMeta(task) {
|
|
626
|
+
const meta = {
|
|
627
|
+
startUrl: task.startUrl,
|
|
628
|
+
};
|
|
629
|
+
if (task.style)
|
|
630
|
+
meta.style = task.style;
|
|
631
|
+
if (task.sessionName)
|
|
632
|
+
meta.sessionName = task.sessionName;
|
|
633
|
+
return meta;
|
|
634
|
+
}
|
|
610
635
|
getSystemMessage() {
|
|
611
636
|
return dedent `
|
|
612
637
|
<role>
|
|
@@ -991,4 +1016,31 @@ export class Tester extends TaskAgent {
|
|
|
991
1016
|
}),
|
|
992
1017
|
};
|
|
993
1018
|
}
|
|
1019
|
+
async handleLoopError(task, error) {
|
|
1020
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1021
|
+
if (!task.hasFinished)
|
|
1022
|
+
task.addNote(`Execution error: ${message}`);
|
|
1023
|
+
const result = await this.explorer.handleExecutionError(error);
|
|
1024
|
+
tag('info').log(`Browser supervisor: ${result.action} - ${result.message}`);
|
|
1025
|
+
task.addNote(result.message);
|
|
1026
|
+
if (result.action === 'stop') {
|
|
1027
|
+
task.finish(TestResult.FAILED);
|
|
1028
|
+
return 'stop';
|
|
1029
|
+
}
|
|
1030
|
+
if (result.recovered) {
|
|
1031
|
+
this.resetFailureCount();
|
|
1032
|
+
this.previousUrl = null;
|
|
1033
|
+
this.previousStateHash = null;
|
|
1034
|
+
}
|
|
1035
|
+
this.currentConversation?.addUserText(result.message);
|
|
1036
|
+
return 'continue';
|
|
1037
|
+
}
|
|
1038
|
+
async cleanupStartedTest(task) {
|
|
1039
|
+
await this.finishTest(task);
|
|
1040
|
+
await this.explorer.stopTest(task, {
|
|
1041
|
+
startUrl: task.startUrl,
|
|
1042
|
+
style: task.style,
|
|
1043
|
+
sessionName: task.sessionName,
|
|
1044
|
+
});
|
|
1045
|
+
}
|
|
994
1046
|
}
|