explorbot 0.1.26 → 0.1.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/dist/package.json +1 -1
  2. package/dist/src/action-result.js +2 -0
  3. package/dist/src/action.js +88 -7
  4. package/dist/src/ai/captain/file-tools.js +100 -0
  5. package/dist/src/ai/captain/idle-mode.js +70 -6
  6. package/dist/src/ai/captain/web-mode.js +36 -6
  7. package/dist/src/ai/captain.js +87 -19
  8. package/dist/src/ai/historian/screencast.js +11 -2
  9. package/dist/src/ai/navigator.js +5 -2
  10. package/dist/src/ai/pilot.js +33 -5
  11. package/dist/src/ai/researcher/cache.js +8 -0
  12. package/dist/src/ai/researcher/coordinates.js +2 -3
  13. package/dist/src/ai/researcher/deep-analysis.js +149 -65
  14. package/dist/src/ai/researcher/locators.js +1 -2
  15. package/dist/src/ai/researcher.js +17 -18
  16. package/dist/src/ai/task-agent.js +1 -0
  17. package/dist/src/ai/tester.js +91 -39
  18. package/dist/src/ai/tools.js +17 -7
  19. package/dist/src/commands/explore-command.js +6 -1
  20. package/dist/src/components/LogPane.js +4 -3
  21. package/dist/src/explorer.js +270 -35
  22. package/dist/src/utils/browser-errors.js +23 -0
  23. package/dist/src/utils/error-page.js +17 -2
  24. package/dist/src/utils/logger.js +2 -2
  25. package/package.json +1 -1
  26. package/src/action-result.ts +2 -0
  27. package/src/action.ts +83 -7
  28. package/src/ai/captain/file-tools.ts +126 -0
  29. package/src/ai/captain/idle-mode.ts +72 -6
  30. package/src/ai/captain/mixin.ts +1 -1
  31. package/src/ai/captain/web-mode.ts +40 -5
  32. package/src/ai/captain.ts +94 -20
  33. package/src/ai/historian/screencast.ts +11 -2
  34. package/src/ai/navigator.ts +6 -2
  35. package/src/ai/pilot.ts +34 -5
  36. package/src/ai/researcher/cache.ts +7 -0
  37. package/src/ai/researcher/coordinates.ts +2 -3
  38. package/src/ai/researcher/deep-analysis.ts +169 -72
  39. package/src/ai/researcher/locators.ts +1 -2
  40. package/src/ai/researcher.ts +17 -18
  41. package/src/ai/task-agent.ts +1 -1
  42. package/src/ai/tester.ts +101 -41
  43. package/src/ai/tools.ts +17 -7
  44. package/src/commands/explore-command.ts +6 -1
  45. package/src/components/LogPane.tsx +4 -3
  46. package/src/explorer.ts +295 -38
  47. package/src/state-manager.ts +2 -0
  48. package/src/utils/browser-errors.ts +25 -0
  49. package/src/utils/error-page.ts +16 -3
  50. package/src/utils/logger.ts +3 -3
@@ -2,9 +2,10 @@ import dedent from 'dedent';
2
2
  import { ActionResult } from '../../action-result.js';
3
3
  import { executionController } from "../../execution-controller.js";
4
4
  import { detectFocusArea, diffAriaSnapshots } from "../../utils/aria.js";
5
+ import { extractCodeBlocks } from "../../utils/code-extractor.js";
5
6
  import { tag } from '../../utils/logger.js';
6
7
  import { mdq } from "../../utils/markdown-query.js";
7
- import { getCachedResearch, saveResearch } from "./cache.js";
8
+ import { getCachedResearch, getPreviousResearch, saveResearch } from "./cache.js";
8
9
  import { debugLog } from "./mixin.js";
9
10
  import { parseResearchSections } from "./parser.js";
10
11
  const DEFAULT_MAX_EXPANDABLE_CLICKS = 10;
@@ -13,13 +14,26 @@ export function WithDeepAnalysis(Base) {
13
14
  async performDeepAnalysis(state, result) {
14
15
  tag('info').log('Starting deep analysis of expandable elements');
15
16
  await this.navigateTo(state.fullUrl || state.url);
16
- let expandables = await this._discoverExpandables(result.text);
17
- if (expandables.length === 0) {
18
- tag('info').log('No expandable elements identified by AI');
19
- return;
20
- }
21
- tag('substep').log(`Identified ${expandables.length} expandable elements`);
22
17
  const maxClicks = this.explorer.getConfig().ai?.agents?.researcher?.maxExpandableClicks ?? DEFAULT_MAX_EXPANDABLE_CLICKS;
18
+ const expandedSections = [];
19
+ const navigationLinks = [];
20
+ let verifiedCodes = [];
21
+ let missing = [];
22
+ const previousSections = this._loadPreviousExtendedSections(state.hash || '');
23
+ if (previousSections.length > 0) {
24
+ tag('substep').log(`Replaying ${previousSections.length} previously discovered sections`);
25
+ const replay = await this._replayPreviousSections(state, previousSections, maxClicks);
26
+ expandedSections.push(...replay.verified);
27
+ verifiedCodes = replay.verifiedCodes;
28
+ missing = replay.missing;
29
+ tag('info').log(`Reused ${replay.verified.length}/${previousSections.length} previous sections, ${missing.length} to re-discover`);
30
+ if (missing.length === 0 && replay.verified.length >= maxClicks) {
31
+ tag('info').log('Page appears unchanged, reusing previous sections and skipping discovery');
32
+ this._appendExtendedResearch(result, expandedSections, navigationLinks);
33
+ return;
34
+ }
35
+ }
36
+ let expandables = await this._discoverExpandables(result.text, missing, verifiedCodes);
23
37
  if (expandables.length > maxClicks) {
24
38
  expandables = await this._selectExpandables(expandables, state.fullUrl || state.url, maxClicks);
25
39
  tag('substep').log(`Selected ${expandables.length} expandables to click (max: ${maxClicks})`);
@@ -29,9 +43,11 @@ export function WithDeepAnalysis(Base) {
29
43
  commands: this._buildClickCommands(el),
30
44
  description: el.name,
31
45
  }))
32
- .filter((el) => el.commands.length > 0);
46
+ .filter((el) => el.commands.length > 0)
47
+ .filter((el) => !el.commands.some((cmd) => verifiedCodes.includes(cmd)));
33
48
  if (elements.length === 0) {
34
- tag('info').log('No expandables with valid locators');
49
+ tag('info').log('No new expandable elements to click');
50
+ this._appendExtendedResearch(result, expandedSections, navigationLinks);
35
51
  return;
36
52
  }
37
53
  const expandableRows = elements.map((el) => `| ${el.description} | \`${el.commands[0]}\` |`).join('\n');
@@ -39,21 +55,9 @@ export function WithDeepAnalysis(Base) {
39
55
  for (const el of elements)
40
56
  debugLog(`Expandable: ${el.description} → ${el.commands[0]}`);
41
57
  tag('substep').log(`Clicking ${elements.length} expandable elements`);
42
- const expandedSections = [];
43
- const navigationLinks = [];
44
58
  await this._clickExpandableElements(elements, state, expandedSections, navigationLinks);
45
59
  tag('info').log(`Deep analysis complete. Sections: ${expandedSections.length}, navigation links: ${navigationLinks.length}`);
46
- const dedupedSections = this._deduplicateExpandedSections(expandedSections);
47
- if (dedupedSections.length !== expandedSections.length) {
48
- tag('substep').log(`Deduplicated ${expandedSections.length} → ${dedupedSections.length} extended sections`);
49
- }
50
- if (dedupedSections.length > 0) {
51
- result.text += `\n\n# Extended Research\n\n${dedupedSections.join('\n\n---\n\n')}`;
52
- }
53
- if (navigationLinks.length > 0) {
54
- const links = navigationLinks.map((l) => `- \`${l.code}\` opens ${l.url}`).join('\n');
55
- result.text += `\n\n## Navigation Links\n\n${links}`;
56
- }
60
+ this._appendExtendedResearch(result, expandedSections, navigationLinks);
57
61
  }
58
62
  async researchOverlay(current, previous, pageStateHash) {
59
63
  const focusArea = detectFocusArea(current.ariaSnapshot);
@@ -97,7 +101,66 @@ export function WithDeepAnalysis(Base) {
97
101
  tag('substep').log(`Overlay research appended: ${focusArea.name}`);
98
102
  return sectionMarkdown;
99
103
  }
100
- async _discoverExpandables(researchText) {
104
+ _loadPreviousExtendedSections(hash) {
105
+ if (!hash)
106
+ return [];
107
+ const previous = getPreviousResearch(hash);
108
+ if (!previous)
109
+ return [];
110
+ const sections = [];
111
+ for (const section of parseResearchSections(previous)) {
112
+ if (!section.isExtended)
113
+ continue;
114
+ const code = extractCodeBlocks(section.rawMarkdown)[0];
115
+ if (!code)
116
+ continue;
117
+ sections.push({ name: section.name, code });
118
+ }
119
+ return sections;
120
+ }
121
+ async _replayPreviousSections(state, prevSections, maxClicks) {
122
+ const originalAria = state.ariaSnapshot || '';
123
+ const verified = [];
124
+ const verifiedCodes = [];
125
+ const missing = [];
126
+ for (const section of prevSections.slice(0, maxClicks)) {
127
+ if (executionController.isInterrupted())
128
+ break;
129
+ let outcome;
130
+ try {
131
+ outcome = await this._executeAndAnalyze([section.code], section.name, state, originalAria, this._summarizeExpanded(verified));
132
+ }
133
+ catch (err) {
134
+ tag('warning').log(`Replay failed for "${section.name}": ${err instanceof Error ? err.message : err}`);
135
+ await this._restorePageState(state.url, originalAria).catch(() => { });
136
+ missing.push(section);
137
+ continue;
138
+ }
139
+ if (outcome.status === 'revealed') {
140
+ verified.push(outcome.sectionMarkdown);
141
+ verifiedCodes.push(section.code);
142
+ debugLog(`Replayed and verified section: ${section.name}`);
143
+ continue;
144
+ }
145
+ debugLog(`Could not replay previous section: ${section.name}`);
146
+ missing.push(section);
147
+ }
148
+ return { verified, verifiedCodes, missing };
149
+ }
150
+ _appendExtendedResearch(result, expandedSections, navigationLinks) {
151
+ const dedupedSections = this._deduplicateExpandedSections(expandedSections);
152
+ if (dedupedSections.length !== expandedSections.length) {
153
+ tag('substep').log(`Deduplicated ${expandedSections.length} → ${dedupedSections.length} extended sections`);
154
+ }
155
+ if (dedupedSections.length > 0) {
156
+ result.text += `\n\n# Extended Research\n\n${dedupedSections.join('\n\n---\n\n')}`;
157
+ }
158
+ if (navigationLinks.length > 0) {
159
+ const links = navigationLinks.map((l) => `- \`${l.code}\` opens ${l.url}`).join('\n');
160
+ result.text += `\n\n## Navigation Links\n\n${links}`;
161
+ }
162
+ }
163
+ async _discoverExpandables(researchText, missing = [], verifiedCodes = []) {
101
164
  const allElements = new Map();
102
165
  for (const section of parseResearchSections(researchText)) {
103
166
  for (const el of section.elements) {
@@ -108,6 +171,15 @@ export function WithDeepAnalysis(Base) {
108
171
  if (allElements.size === 0)
109
172
  return [];
110
173
  const eidxList = [...allElements.keys()].join(', ');
174
+ let missingHint = '';
175
+ if (missing.length > 0) {
176
+ const list = missing.map((s) => `- "${s.name}" (previously revealed via ${s.code})`).join('\n');
177
+ missingHint = dedent `
178
+
179
+ These sections were present on a previous visit but their trigger could not be replayed now — the element may have moved or been renamed. Prioritize finding the element that now reveals each:
180
+ ${list}
181
+ `;
182
+ }
111
183
  const textPrompt = dedent `
112
184
  From this UI research, identify elements that could reveal hidden UI when clicked
113
185
  (dropdown menus, popups, expandable panels, accordion sections, overflow menus, tab switches).
@@ -115,6 +187,7 @@ export function WithDeepAnalysis(Base) {
115
187
  Available eidx refs: ${eidxList}
116
188
 
117
189
  ${researchText}
190
+ ${missingHint}
118
191
 
119
192
  Rules:
120
193
  - Only pick elements that HIDE content until clicked (menus, dropdowns, accordions, tabs)
@@ -136,6 +209,7 @@ export function WithDeepAnalysis(Base) {
136
209
 
137
210
  Look for: overflow/ellipsis menus, chevron dropdowns, hamburger menus,
138
211
  gear/settings buttons, accordion toggles, tab switches, filter buttons.
212
+ ${missingHint}
139
213
 
140
214
  Rules:
141
215
  - For repeated icons (same icon on every list row), pick only the FIRST one
@@ -238,10 +312,9 @@ export function WithDeepAnalysis(Base) {
238
312
  const isCoordinateClick = el.commands[0].startsWith('I.clickXY(');
239
313
  if (!isCoordinateClick) {
240
314
  const hoverCmd = el.commands[0].replace('I.click(', 'I.moveCursorTo(');
241
- const hoverAction = this.explorer.createAction();
242
- await hoverAction.attempt(hoverCmd, undefined, false);
315
+ await this.explorer.attemptAction(hoverCmd, undefined, false);
243
316
  await new Promise((r) => setTimeout(r, 500));
244
- await this.explorer.createAction().capturePageState();
317
+ await this.explorer.capturePageState();
245
318
  const hoverAR = ActionResult.fromState(this.stateManager.getCurrentState());
246
319
  const hoverDiff = await hoverAR.diff(previousState);
247
320
  await hoverDiff.calculate();
@@ -258,48 +331,15 @@ export function WithDeepAnalysis(Base) {
258
331
  await this._restorePageState(state.url, originalAria);
259
332
  }
260
333
  }
261
- let clickCode = null;
262
- const action = this.explorer.createAction();
263
- for (const cmd of el.commands) {
264
- if (await action.attempt(cmd, undefined, false)) {
265
- clickCode = cmd;
266
- break;
267
- }
268
- }
269
- if (!clickCode) {
270
- debugLog(`Click failed: ${el.description.slice(0, 80)}`);
334
+ const outcome = await this._executeAndAnalyze(el.commands, el.description, state, originalAria, this._summarizeExpanded(expandedSections));
335
+ if (outcome.status === 'navigated') {
336
+ navigationLinks.push({ code: outcome.code, url: outcome.url });
271
337
  continue;
272
338
  }
273
- await new Promise((r) => setTimeout(r, 500));
274
- let diff;
275
- try {
276
- await this.explorer.createAction().capturePageState();
277
- const currAR = ActionResult.fromState(this.stateManager.getCurrentState());
278
- diff = await currAR.diff(previousState);
279
- await diff.calculate();
280
- }
281
- catch (err) {
282
- tag('warning').log(`State capture failed after click: ${err instanceof Error ? err.message : err}`);
283
- await this._restorePageState(state.url, originalAria);
284
- continue;
285
- }
286
- if (diff.urlHasChanged()) {
287
- debugLog(`Click navigated to ${this.stateManager.getCurrentState()?.url}`);
288
- navigationLinks.push({ code: clickCode, url: this.stateManager.getCurrentState()?.url || '' });
289
- await this.navigateTo(state.url);
290
- continue;
291
- }
292
- const clickHtmlSize = diff.htmlParts.reduce((sum, p) => sum + p.subtree.length, 0);
293
- if (!diff.ariaChanged && clickHtmlSize <= 150) {
294
- debugLog(`No changes from: ${el.description.slice(0, 80)}`);
295
- continue;
296
- }
297
- const sectionMarkdown = await this._analyzeExpandedAction(clickCode, el.description, diff, this._summarizeExpanded(expandedSections));
298
- if (sectionMarkdown) {
299
- expandedSections.push(sectionMarkdown);
339
+ if (outcome.status === 'revealed') {
340
+ expandedSections.push(outcome.sectionMarkdown);
300
341
  debugLog(`Captured section from: ${el.description.slice(0, 80)}`);
301
342
  }
302
- await this._restorePageState(state.url, originalAria);
303
343
  }
304
344
  catch (err) {
305
345
  tag('warning').log(`Expandable click failed for "${el.description.slice(0, 80)}": ${err instanceof Error ? err.message : err}`);
@@ -310,10 +350,54 @@ export function WithDeepAnalysis(Base) {
310
350
  }
311
351
  }
312
352
  }
353
+ async _executeAndAnalyze(commands, description, state, originalAria, alreadyExpanded) {
354
+ const previousState = ActionResult.fromState(this.stateManager.getCurrentState());
355
+ let clickCode = null;
356
+ const action = this.explorer.createAction();
357
+ for (const cmd of commands) {
358
+ if (await action.attempt(cmd, undefined, false)) {
359
+ clickCode = cmd;
360
+ break;
361
+ }
362
+ }
363
+ if (!clickCode) {
364
+ debugLog(`Click failed: ${description.slice(0, 80)}`);
365
+ return { status: 'failed' };
366
+ }
367
+ await new Promise((r) => setTimeout(r, 500));
368
+ let diff;
369
+ try {
370
+ await this.explorer.createAction().capturePageState();
371
+ const currAR = ActionResult.fromState(this.stateManager.getCurrentState());
372
+ diff = await currAR.diff(previousState);
373
+ await diff.calculate();
374
+ }
375
+ catch (err) {
376
+ tag('warning').log(`State capture failed after click: ${err instanceof Error ? err.message : err}`);
377
+ await this._restorePageState(state.url, originalAria);
378
+ return { status: 'failed' };
379
+ }
380
+ if (diff.urlHasChanged()) {
381
+ const url = this.stateManager.getCurrentState()?.url || '';
382
+ debugLog(`Click navigated to ${url}`);
383
+ await this.navigateTo(state.url);
384
+ return { status: 'navigated', code: clickCode, url };
385
+ }
386
+ const clickHtmlSize = diff.htmlParts.reduce((sum, p) => sum + p.subtree.length, 0);
387
+ if (!diff.ariaChanged && clickHtmlSize <= 150) {
388
+ debugLog(`No changes from: ${description.slice(0, 80)}`);
389
+ return { status: 'none', code: clickCode };
390
+ }
391
+ const sectionMarkdown = await this._analyzeExpandedAction(clickCode, description, diff, alreadyExpanded);
392
+ await this._restorePageState(state.url, originalAria);
393
+ if (!sectionMarkdown)
394
+ return { status: 'none', code: clickCode };
395
+ return { status: 'revealed', code: clickCode, sectionMarkdown };
396
+ }
313
397
  async _restorePageState(url, originalAria) {
314
398
  try {
315
399
  await this.cancelInUi();
316
- await this.explorer.createAction().capturePageState();
400
+ await this.explorer.capturePageState();
317
401
  const currentAria = this.stateManager.getCurrentState()?.ariaSnapshot || '';
318
402
  if (!diffAriaSnapshots(originalAria, currentAria))
319
403
  return;
@@ -173,8 +173,7 @@ export function WithLocators(Base) {
173
173
  }
174
174
  }
175
175
  if (needsXpath.length > 0) {
176
- const page = this.explorer.playwrightHelper.page;
177
- const webElements = await WebElement.fromEidxList(page, needsXpath);
176
+ const webElements = await this.explorer.runWithBrowserRecovery('backfillBrokenLocators', () => WebElement.fromEidxList(this.explorer.playwrightHelper.page, needsXpath));
178
177
  const changedSections = new Set();
179
178
  for (const w of webElements) {
180
179
  const entry = needsXpathEls.get(w.eidx);
@@ -97,11 +97,11 @@ export class Researcher extends ResearcherBase {
97
97
  await this.hooksRunner.runBeforeHook('researcher', state.url);
98
98
  const annotatedElements = await this.explorer.annotateElements();
99
99
  debugLog(`Annotated ${annotatedElements.length} interactive elements with eidx`);
100
- this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot: screenshot && this.provider.hasVision() });
100
+ this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot && this.provider.hasVision() });
101
101
  const condition = detectPageCondition(this.actionResult);
102
102
  if (condition === 'error') {
103
103
  tag('warning').log(`Detected error page at ${state.url}`);
104
- throw new ErrorPageError(state.url, this.actionResult.title);
104
+ throw new ErrorPageError(state.url, this.actionResult.title, this.actionResult.httpStatus);
105
105
  }
106
106
  if (condition === 'loading') {
107
107
  const settled = await this.waitUntilSettled(screenshot);
@@ -191,7 +191,7 @@ export class Researcher extends ResearcherBase {
191
191
  // Must run BEFORE visuallyAnnotateContainers — annotation overlays inject z-index 99998+ which would pollute the scoring.
192
192
  if (!interrupted() && this.hasScreenshotToAnalyze) {
193
193
  const sections = parseResearchSections(result.text);
194
- const focused = await detectFocusedSection(this.explorer.playwrightHelper.page, sections);
194
+ const focused = await this.explorer.runWithBrowserRecovery('detectFocusedSection', () => detectFocusedSection(this.explorer.playwrightHelper.page, sections));
195
195
  if (focused)
196
196
  markSectionAsFocused(result, focused);
197
197
  }
@@ -204,7 +204,7 @@ export class Researcher extends ResearcherBase {
204
204
  const freshBroken = freshContainerLocs.filter((l) => l.valid === false).map((l) => l.locator);
205
205
  const containers = validContainers.filter((c) => !freshBroken.includes(c.css));
206
206
  await this.visuallyAnnotateElements({ containers });
207
- this.actionResult = await this.explorer.createAction().caputrePageWithScreenshot();
207
+ this.actionResult = await this.explorer.capturePageWithScreenshot();
208
208
  const visualResult = await this.analyzeScreenshotForVisualProps();
209
209
  if (visualResult.elements.size > 0) {
210
210
  await this.mergeVisualData(result, visualResult.elements);
@@ -277,7 +277,7 @@ export class Researcher extends ResearcherBase {
277
277
  if (!this.actionResult) {
278
278
  debugLog('No action result, navigating to URL');
279
279
  await this.explorer.visit(url);
280
- this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot: screenshot });
280
+ this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot });
281
281
  return;
282
282
  }
283
283
  const isOnCurrentState = this.actionResult.getStateHash() === this.stateManager.getCurrentState()?.hash;
@@ -285,46 +285,47 @@ export class Researcher extends ResearcherBase {
285
285
  const isEmpty = isBodyEmpty(stateHtml);
286
286
  if (!isEmpty && isOnCurrentState) {
287
287
  if ((!this.actionResult.screenshot && screenshot) || !this.actionResult.ariaSnapshot) {
288
- this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot: screenshot });
288
+ this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot });
289
289
  }
290
290
  return;
291
291
  }
292
292
  if (isEmpty && isOnCurrentState) {
293
293
  debugLog('HTML body empty on current URL, waiting for content');
294
294
  tag('step').log('Page body is empty, waiting for content...');
295
+ await this.explorer.visit(url);
296
+ this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false });
295
297
  await this.waitUntilSettled(screenshot ?? false);
296
298
  return;
297
299
  }
298
300
  debugLog('Not on current state, navigating to URL');
299
301
  tag('step').log('Navigating to URL...');
300
302
  await this.explorer.visit(url);
301
- this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot: screenshot ?? false });
303
+ this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false });
302
304
  }
303
305
  async waitUntilSettled(screenshot) {
304
306
  const errorPageTimeout = this.explorer.getConfig().ai?.agents?.researcher?.errorPageTimeout ?? 10;
305
307
  if (errorPageTimeout <= 0)
306
308
  return false;
307
- const page = this.explorer.playwrightHelper.page;
308
309
  const includeScreenshot = screenshot && this.provider.hasVision();
309
310
  try {
310
- await page?.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 });
311
+ await this.explorer.runWithBrowserRecovery('waitUntilSettled', () => this.explorer.playwrightHelper.page?.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 }));
311
312
  }
312
313
  catch { }
313
314
  await this.explorer.annotateElements();
314
- this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot });
315
+ this.actionResult = await this.explorer.capturePageState({ includeScreenshot });
315
316
  let condition = detectPageCondition(this.actionResult);
316
317
  if (condition === 'error') {
317
- throw new ErrorPageError(this.actionResult.url, this.actionResult.title);
318
+ throw new ErrorPageError(this.actionResult.url, this.actionResult.title, this.actionResult.httpStatus);
318
319
  }
319
320
  if (condition === 'ok')
320
321
  return true;
321
322
  for (let i = 0; i < 3; i++) {
322
323
  await new Promise((r) => setTimeout(r, 1000));
323
324
  await this.explorer.annotateElements();
324
- this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot });
325
+ this.actionResult = await this.explorer.capturePageState({ includeScreenshot });
325
326
  condition = detectPageCondition(this.actionResult);
326
327
  if (condition === 'error') {
327
- throw new ErrorPageError(this.actionResult.url, this.actionResult.title);
328
+ throw new ErrorPageError(this.actionResult.url, this.actionResult.title, this.actionResult.httpStatus);
328
329
  }
329
330
  if (condition === 'ok')
330
331
  return true;
@@ -681,15 +682,13 @@ export class Researcher extends ResearcherBase {
681
682
  .join('\n\n');
682
683
  }
683
684
  async navigateTo(url) {
684
- const action = this.explorer.createAction();
685
- await action.execute(`I.amOnPage("${url}")`);
685
+ await this.explorer.visit(url);
686
686
  }
687
687
  async cancelInUi() {
688
688
  const beforeAria = this.stateManager.getCurrentState()?.ariaSnapshot || null;
689
- const action = this.explorer.createAction();
690
- await action.execute('I.clickXY(0, 0)');
689
+ await this.explorer.executeAction('I.clickXY(0, 0)');
691
690
  if (diffAriaSnapshots(beforeAria, this.stateManager.getCurrentState()?.ariaSnapshot || null))
692
691
  return;
693
- await action.execute(`I.pressKey('Escape')`);
692
+ await this.explorer.executeAction(`I.pressKey('Escape')`);
694
693
  }
695
694
  }
@@ -14,6 +14,7 @@ export class TaskAgent {
14
14
  consecutiveFailures = 0;
15
15
  consecutiveEmptyResults = 0;
16
16
  recentToolCalls = [];
17
+ ACTION_TOOLS = [];
17
18
  _historian = null;
18
19
  _quartermaster = null;
19
20
  getKnowledge(actionResult) {
@@ -4,13 +4,13 @@ import { tool } from 'ai';
4
4
  import dedent from 'dedent';
5
5
  import { z } from 'zod';
6
6
  import { ActionResult } from "../action-result.js";
7
- import { setActivity } from "../activity.js";
7
+ import { clearActivity, setActivity } from "../activity.js";
8
8
  import { ConfigParser } from "../config.js";
9
9
  import { Observability } from "../observability.js";
10
10
  import { Stats } from "../stats.js";
11
11
  import { TestResult } from "../test-plan.js";
12
12
  import { detectFocusArea, extractFocusedElement } from "../utils/aria.js";
13
- import { ErrorPageError } from "../utils/error-page.js";
13
+ import { ErrorPageError, isErrorPage } from "../utils/error-page.js";
14
14
  import { HooksRunner } from "../utils/hooks-runner.js";
15
15
  import { createDebug, tag } from "../utils/logger.js";
16
16
  import { loop } from "../utils/loop.js";
@@ -107,18 +107,13 @@ export class Tester extends TaskAgent {
107
107
  const offFailedRequest = requestStore?.onFailedRequest((r) => {
108
108
  task.addNote(`Network error: ${r.method} ${r.path} → ${r.status}`, TestResult.FAILED);
109
109
  });
110
- const page = this.explorer.playwrightHelper?.page;
111
- const onPageError = (err) => {
112
- task.addNote(`Console error: ${err.message}`, TestResult.FAILED);
113
- };
114
- const onConsoleMessage = (msg) => {
115
- if (msg.type() !== 'error')
116
- return;
117
- task.addNote(`Console error: ${msg.text()}`, TestResult.FAILED);
118
- };
119
- page?.on('pageerror', onPageError);
120
- page?.on('console', onConsoleMessage);
121
110
  const initialState = ActionResult.fromState(state);
111
+ if (isErrorPage(initialState)) {
112
+ task.start();
113
+ await this.explorer.startTest(task);
114
+ offFailedRequest?.();
115
+ return await this.abortStartedTestOnErrorPage(task, initialState);
116
+ }
122
117
  const conversation = this.provider.startConversation(this.getSystemMessage(), 'tester');
123
118
  conversation.markLastMessageCacheable();
124
119
  this.currentConversation = conversation;
@@ -140,17 +135,15 @@ export class Tester extends TaskAgent {
140
135
  startUrl: task.startUrl,
141
136
  expected: task.expected,
142
137
  },
143
- }, async () => this.runTestSession(task, initialState, conversation, { offFailedRequest, page, onPageError, onConsoleMessage }));
138
+ }, async () => this.runTestSession(task, initialState, conversation, { offFailedRequest }));
144
139
  }
145
140
  async runTestSession(task, initialState, conversation, handlers) {
146
- const { offFailedRequest, page, onPageError, onConsoleMessage } = handlers;
141
+ const { offFailedRequest } = handlers;
147
142
  if (this.pilot) {
148
143
  try {
149
144
  const plan = await this.pilot.planTest(task, initialState);
150
145
  if (task.hasFinished) {
151
146
  offFailedRequest?.();
152
- page?.off('pageerror', onPageError);
153
- page?.off('console', onConsoleMessage);
154
147
  return { success: task.isSuccessful };
155
148
  }
156
149
  if (plan) {
@@ -163,19 +156,36 @@ export class Tester extends TaskAgent {
163
156
  task.addNote(`Planning failed: ${message}`, TestResult.FAILED);
164
157
  task.finish(TestResult.FAILED);
165
158
  offFailedRequest?.();
166
- page?.off('pageerror', onPageError);
167
- page?.off('console', onConsoleMessage);
168
159
  return { success: false };
169
160
  }
170
161
  }
171
162
  debugLog('Starting test execution with tools');
172
- task.start();
173
- await this.explorer.startTest(task);
163
+ if (!(await this.explorer.startTest(task))) {
164
+ offFailedRequest?.();
165
+ await this.cleanupStartedTest(task);
166
+ return { success: task.isSuccessful };
167
+ }
174
168
  debugLog(`Navigating to ${task.startUrl}`);
175
- await this.explorer.visit(task.startUrl);
169
+ try {
170
+ await this.explorer.visit(task.startUrl);
171
+ }
172
+ catch (error) {
173
+ const result = await this.handleLoopError(task, error);
174
+ if (result === 'stop') {
175
+ offFailedRequest?.();
176
+ await this.cleanupStartedTest(task);
177
+ return { success: task.isSuccessful };
178
+ }
179
+ }
176
180
  const startState = this.explorer.getStateManager().getCurrentState();
177
- if (startState)
181
+ if (startState) {
178
182
  task.addUrlNote(startState);
183
+ const startActionResult = ActionResult.fromState(startState);
184
+ if (isErrorPage(startActionResult)) {
185
+ offFailedRequest?.();
186
+ return await this.abortStartedTestOnErrorPage(task, startActionResult);
187
+ }
188
+ }
179
189
  const currentUrl = startState?.url || task.startUrl || '';
180
190
  await this.hooksRunner.runBeforeHook('tester', currentUrl);
181
191
  const offStateChange = this.explorer.getStateManager().onStateChange((event) => {
@@ -195,6 +205,12 @@ export class Tester extends TaskAgent {
195
205
  shouldContinue = false;
196
206
  await loop(async ({ stop, pause, iteration, userInput }) => {
197
207
  debugLog('iteration', iteration);
208
+ if (!(await this.explorer.ensurePageAvailable())) {
209
+ task.addNote('Browser page is unavailable');
210
+ task.finish(TestResult.FAILED);
211
+ stop();
212
+ return;
213
+ }
198
214
  const currentState = this.getCurrentState();
199
215
  const tools = {
200
216
  ...codeceptjsTools,
@@ -327,20 +343,15 @@ export class Tester extends TaskAgent {
327
343
  }
328
344
  : undefined,
329
345
  catch: async ({ error, stop }) => {
330
- tag('error').log(`Test execution error: ${error}`);
331
- const message = error instanceof Error ? error.message : String(error);
332
- if (!task.hasFinished) {
333
- task.addNote(`Execution error: ${message}`);
334
- }
335
- if (error instanceof Error && error.name === 'AbortError') {
346
+ const result = await this.handleLoopError(task, error);
347
+ if (result === 'stop')
336
348
  stop();
337
- return;
338
- }
339
- conversation.addUserText(`Previous AI call failed: ${message}. Take a different approach on the next step.`);
340
349
  },
341
350
  });
342
351
  if (task.hasFinished)
343
352
  break;
353
+ if (!(await this.explorer.ensurePageAvailable()))
354
+ break;
344
355
  const finalState = this.getCurrentState();
345
356
  const wantsContinue = await this.pilot.finalReview(task, finalState, conversation, this.navigator);
346
357
  if (!wantsContinue || task.hasFinished)
@@ -366,14 +377,8 @@ export class Tester extends TaskAgent {
366
377
  await this.getQuartermaster().analyzeSession(task, initialState, conversation);
367
378
  offStateChange();
368
379
  offFailedRequest?.();
369
- page?.off('pageerror', onPageError);
370
- page?.off('console', onConsoleMessage);
371
380
  await this.finishTest(task);
372
- await this.explorer.stopTest(task, {
373
- startUrl: task.startUrl,
374
- style: task.style,
375
- sessionName: task.sessionName,
376
- });
381
+ await this.explorer.stopTest(task, this.buildStopTestMeta(task));
377
382
  return {
378
383
  success: task.isSuccessful,
379
384
  ...task,
@@ -607,6 +612,26 @@ export class Tester extends TaskAgent {
607
612
  tag('warning').log(`Test with no result: ${task.scenario}`);
608
613
  }
609
614
  }
615
+ async abortStartedTestOnErrorPage(task, actionResult) {
616
+ const error = new ErrorPageError(actionResult.url || task.startUrl || '', actionResult.title, actionResult.httpStatus);
617
+ tag('warning').log(error.message);
618
+ task.addNote(error.message, TestResult.FAILED, actionResult.screenshotFile, actionResult.fullUrl || actionResult.url);
619
+ task.finish(TestResult.FAILED);
620
+ this.finishTest(task);
621
+ await this.explorer.stopTest(task, this.buildStopTestMeta(task));
622
+ clearActivity(true);
623
+ return { success: false };
624
+ }
625
+ buildStopTestMeta(task) {
626
+ const meta = {
627
+ startUrl: task.startUrl,
628
+ };
629
+ if (task.style)
630
+ meta.style = task.style;
631
+ if (task.sessionName)
632
+ meta.sessionName = task.sessionName;
633
+ return meta;
634
+ }
610
635
  getSystemMessage() {
611
636
  return dedent `
612
637
  <role>
@@ -991,4 +1016,31 @@ export class Tester extends TaskAgent {
991
1016
  }),
992
1017
  };
993
1018
  }
1019
+ async handleLoopError(task, error) {
1020
+ const message = error instanceof Error ? error.message : String(error);
1021
+ if (!task.hasFinished)
1022
+ task.addNote(`Execution error: ${message}`);
1023
+ const result = await this.explorer.handleExecutionError(error);
1024
+ tag('info').log(`Browser supervisor: ${result.action} - ${result.message}`);
1025
+ task.addNote(result.message);
1026
+ if (result.action === 'stop') {
1027
+ task.finish(TestResult.FAILED);
1028
+ return 'stop';
1029
+ }
1030
+ if (result.recovered) {
1031
+ this.resetFailureCount();
1032
+ this.previousUrl = null;
1033
+ this.previousStateHash = null;
1034
+ }
1035
+ this.currentConversation?.addUserText(result.message);
1036
+ return 'continue';
1037
+ }
1038
+ async cleanupStartedTest(task) {
1039
+ await this.finishTest(task);
1040
+ await this.explorer.stopTest(task, {
1041
+ startUrl: task.startUrl,
1042
+ style: task.style,
1043
+ sessionName: task.sessionName,
1044
+ });
1045
+ }
994
1046
  }