explorbot 0.2.5 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/boat/prima/README.md +96 -0
  2. package/boat/prima/package.json +14 -10
  3. package/boat/prima/src/cli.ts +5 -0
  4. package/boat/prima/src/prima.ts +17 -4
  5. package/dist/boat/prima/src/cli.js +7 -0
  6. package/dist/boat/prima/src/prima.js +18 -4
  7. package/dist/models.json +4 -4
  8. package/dist/package.json +6 -2
  9. package/dist/src/action-result.d.ts +13 -0
  10. package/dist/src/action-result.js +46 -15
  11. package/dist/src/action.d.ts +5 -2
  12. package/dist/src/action.js +48 -17
  13. package/dist/src/ai/captain/web-mode.js +1 -2
  14. package/dist/src/ai/captain.d.ts +20 -0
  15. package/dist/src/ai/captain.js +10 -1
  16. package/dist/src/ai/conversation.d.ts +1 -0
  17. package/dist/src/ai/conversation.js +3 -0
  18. package/dist/src/ai/driller.js +6 -2
  19. package/dist/src/ai/fisherman-tools.d.ts +40 -1
  20. package/dist/src/ai/fisherman-tools.js +39 -0
  21. package/dist/src/ai/fisherman.js +3 -2
  22. package/dist/src/ai/navigator.d.ts +2 -1
  23. package/dist/src/ai/navigator.js +5 -9
  24. package/dist/src/ai/pilot.js +51 -29
  25. package/dist/src/ai/planner/subpages.js +2 -16
  26. package/dist/src/ai/planner.js +1 -1
  27. package/dist/src/ai/provider.d.ts +3 -0
  28. package/dist/src/ai/provider.js +80 -17
  29. package/dist/src/ai/researcher/cache.d.ts +8 -3
  30. package/dist/src/ai/researcher/cache.js +13 -8
  31. package/dist/src/ai/researcher/deep-analysis.js +1 -1
  32. package/dist/src/ai/researcher/fingerprint-worker.js +21 -4
  33. package/dist/src/ai/researcher.js +4 -3
  34. package/dist/src/ai/rules.js +1 -5
  35. package/dist/src/ai/tester.d.ts +1 -1
  36. package/dist/src/ai/tester.js +34 -26
  37. package/dist/src/ai/tools.d.ts +8 -5
  38. package/dist/src/ai/tools.js +79 -56
  39. package/dist/src/commands/explore-command.js +22 -17
  40. package/dist/src/commands/init-command.js +13 -20
  41. package/dist/src/config.d.ts +2 -1
  42. package/dist/src/config.js +3 -1
  43. package/dist/src/experience-tracker.d.ts +2 -0
  44. package/dist/src/experience-tracker.js +12 -0
  45. package/dist/src/explorbot.js +1 -1
  46. package/dist/src/playwright-recorder.js +6 -12
  47. package/dist/src/test-plan.d.ts +8 -0
  48. package/dist/src/test-plan.js +11 -0
  49. package/dist/src/utils/html-diff.d.ts +5 -0
  50. package/dist/src/utils/html-diff.js +65 -6
  51. package/dist/src/utils/strings.d.ts +2 -0
  52. package/dist/src/utils/strings.js +32 -0
  53. package/dist/src/utils/url-matcher.d.ts +1 -0
  54. package/dist/src/utils/url-matcher.js +31 -2
  55. package/docs/basics/getting-started.md +33 -10
  56. package/docs/basics/providers.md +6 -4
  57. package/docs/contributing/npm-package.md +73 -4
  58. package/models.json +4 -4
  59. package/package.json +6 -2
  60. package/src/action-result.ts +61 -16
  61. package/src/action.ts +51 -17
  62. package/src/ai/captain/web-mode.ts +1 -2
  63. package/src/ai/captain.ts +9 -1
  64. package/src/ai/conversation.ts +3 -0
  65. package/src/ai/driller.ts +6 -2
  66. package/src/ai/fisherman-tools.ts +35 -0
  67. package/src/ai/fisherman.ts +3 -2
  68. package/src/ai/navigator.ts +6 -10
  69. package/src/ai/pilot.ts +54 -32
  70. package/src/ai/planner/subpages.ts +2 -13
  71. package/src/ai/planner.ts +1 -1
  72. package/src/ai/provider.ts +111 -41
  73. package/src/ai/researcher/cache.ts +17 -9
  74. package/src/ai/researcher/deep-analysis.ts +1 -1
  75. package/src/ai/researcher/fingerprint-worker.ts +23 -5
  76. package/src/ai/researcher.ts +4 -3
  77. package/src/ai/rules.ts +1 -5
  78. package/src/ai/tester.ts +32 -27
  79. package/src/ai/tools.ts +84 -60
  80. package/src/commands/explore-command.ts +17 -14
  81. package/src/commands/init-command.ts +14 -20
  82. package/src/config.ts +4 -2
  83. package/src/experience-tracker.ts +13 -0
  84. package/src/explorbot.ts +1 -1
  85. package/src/playwright-recorder.ts +6 -11
  86. package/src/test-plan.ts +18 -0
  87. package/src/utils/html-diff.ts +72 -7
  88. package/src/utils/strings.ts +36 -0
  89. package/src/utils/url-matcher.ts +27 -2
package/dist/src/ai/rules.js CHANGED
@@ -2,10 +2,6 @@ import dedent from 'dedent';
2
2
  export const recommendedCodeceptCommands = ['I.click', 'I.type', 'I.fillField', 'I.see', 'I.seeElement'];
3
3
  const locatorPriorityRule = dedent `
4
4
  <locator_priority>
5
- When the page context shows the element a ref, such as [ref=e14], there is no locator to select: click it with clickRef
6
- and that ref. A ref names one exact element, so it never matches the wrong one and never has to be narrowed. Everything
7
- below is for elements the context gives no ref for.
8
-
9
5
  Use the following priority when selecting locators:
10
6
 
11
7
  1. ARIA locators (first choice) - target browser's accessibility tree, most reliable
@@ -76,7 +72,7 @@ const locatorStrategyRule = dedent `
76
72
 
77
73
  NEVER include \`eidx\` attribute in any locator (ARIA, CSS, XPath). It is an internal annotation.
78
74
 
79
- If <aria> section is not present or element is not found there, fall back to CSS/XPath locators from <html> section.
75
+ If the element is not found in the ARIA snapshot, fall back to CSS/XPath locators from page HTML.
80
76
 
81
77
  Stick to semantic attributes like role, aria-*, id, class, name, data-id, etc.
82
78
  Avoid IDs that follow framework auto-generation patterns (these change on every page load):
package/dist/src/ai/tester.d.ts CHANGED
@@ -11,6 +11,7 @@ import { Researcher } from './researcher.js';
11
11
  import { TaskAgent } from './task-agent.js';
12
12
  export declare class Tester extends TaskAgent implements Agent {
13
13
  readonly ACTION_TOOLS: string[];
14
+ readonly DELEGATED_ACTION_TOOLS: string[];
14
15
  readonly SPECIAL_CONTEXT_ACTION_TOOLS: string[];
15
16
  emoji: string;
16
17
  requestStore: RequestStore;
@@ -49,7 +50,6 @@ export declare class Tester extends TaskAgent implements Agent {
49
50
  shouldStopForStalledExecution(task: Test, previousState: ActionResult, toolExecutions: any[]): boolean;
50
51
  prepareInstructionsForNextStep(task: Test): Promise<string>;
51
52
  reinjectContextIfNeeded(iteration: number, currentState: ActionResult): Promise<string>;
52
- interactiveAriaWithRefs(state: ActionResult): Promise<string>;
53
53
  finishTest(task: Test): void;
54
54
  abortStartedTestOnErrorPage(task: Test, actionResult: ActionResult): Promise<{
55
55
  success: boolean;
package/dist/src/ai/tester.js CHANGED
@@ -6,12 +6,14 @@ import { z } from 'zod';
6
6
  import { ActionResult } from "../action-result.js";
7
7
  import { clearActivity, setActivity } from "../activity.js";
8
8
  import { Observability } from "../observability.js";
9
+ import { normalizeUrl } from "../state-manager.js";
9
10
  import { Stats } from "../stats.js";
10
11
  import { TestResult } from "../test-plan.js";
11
- import { compactAriaSnapshot, detectFocusArea } from "../utils/aria.js";
12
+ import { detectFocusArea } from "../utils/aria.js";
12
13
  import { ErrorPageError, isErrorPage } from "../utils/error-page.js";
13
14
  import { createDebug, tag } from "../utils/logger.js";
14
15
  import { loop } from "../utils/loop.js";
16
+ import { compactErrorMessage } from "../utils/strings.js";
15
17
  import { actionRule, capabilityGroundingRule, dataProtectionRules, focusedElementRule, formRequirementsRule, locatorRule, multipleTabsRule, sectionContextRule } from "./rules.js";
16
18
  import { TaskAgent } from "./task-agent.js";
17
19
  import { createCodeceptJSTools, createIframeTools, withdrawVisionTools } from "./tools.js";
@@ -28,6 +30,7 @@ const SAMPLE_FILES = {
28
30
  };
29
31
  export class Tester extends TaskAgent {
30
32
  ACTION_TOOLS = ['click', 'hover', 'pressKey', 'form'];
33
+ DELEGATED_ACTION_TOOLS = ['interact'];
31
34
  SPECIAL_CONTEXT_ACTION_TOOLS = ['exitIframe'];
32
35
  emoji = '🧪';
33
36
  requestStore;
@@ -76,7 +79,7 @@ export class Tester extends TaskAgent {
76
79
  }
77
80
  async test(task, opts = {}) {
78
81
  Stats.tests++;
79
- const state = this.stateManager.getCurrentState();
82
+ let state = this.stateManager.getCurrentState();
80
83
  if (!state)
81
84
  throw new Error('No state found');
82
85
  setActivity(`🧪 Testing: ${task.scenario}`, 'action');
@@ -95,7 +98,23 @@ export class Tester extends TaskAgent {
95
98
  const offFailedRequest = requestStore.onFailedRequest((r) => {
96
99
  task.addObservation(`Network error: ${r.method} ${r.path} → ${r.status}`);
97
100
  });
98
- const initialState = ActionResult.fromState(state);
101
+ let initialState = ActionResult.fromState(state);
102
+ const currentUrl = state.fullUrl || state.url;
103
+ let startOnCurrentPage = opts.startOnCurrentPage;
104
+ if (isErrorPage(initialState) && !startOnCurrentPage && task.startUrl && normalizeUrl(currentUrl) !== normalizeUrl(task.startUrl)) {
105
+ debugLog(`Recovering from error page at ${currentUrl} by navigating to ${task.startUrl}`);
106
+ try {
107
+ await this.explorer.visit(task.startUrl);
108
+ state = this.stateManager.getCurrentState();
109
+ if (!state)
110
+ throw new Error('No state found after navigating to test start URL');
111
+ initialState = ActionResult.fromState(state);
112
+ startOnCurrentPage = true;
113
+ }
114
+ catch (error) {
115
+ debugLog(`Could not recover from error page: ${compactErrorMessage(error)}`);
116
+ }
117
+ }
99
118
  if (isErrorPage(initialState)) {
100
119
  task.start();
101
120
  this.testRun = await this.explorer.beginTest(task);
@@ -120,7 +139,7 @@ export class Tester extends TaskAgent {
120
139
  startUrl: task.startUrl,
121
140
  expected: task.expected,
122
141
  },
123
- }, async () => this.runTestSession(task, initialState, conversation, { offFailedRequest }, opts));
142
+ }, async () => this.runTestSession(task, initialState, conversation, { offFailedRequest }, { ...opts, startOnCurrentPage }));
124
143
  }
125
144
  async runTestSession(task, initialState, conversation, handlers, opts) {
126
145
  const { offFailedRequest } = handlers;
@@ -232,8 +251,6 @@ export class Tester extends TaskAgent {
232
251
  `);
233
252
  }
234
253
  conversation.cleanupTag('page_aria', '...cleaned aria snapshot...', 1);
235
- conversation.cleanupTag('page_html', '...cleaned HTML snapshot...', 1);
236
- conversation.cleanupTag('experience', '...cleaned experience...', 1);
237
254
  conversation.cleanupTag('applied_experience', '...cleaned past experience...', 1);
238
255
  conversation.cleanupTag('page_ui_map', '...cleaned UI map...', 1);
239
256
  conversation.cleanupTag('page_ui_map_overlay', '...cleaned UI overlay...', 1);
@@ -359,8 +376,6 @@ export class Tester extends TaskAgent {
359
376
  this.stalledIterations = 0;
360
377
  tag('info').log(`Pilot extending test (${extensions}/${this.MAX_EXTENSIONS})`);
361
378
  conversation.cleanupTag('page_aria', '...trimmed...', 1);
362
- conversation.cleanupTag('page_html', '...trimmed...', 0);
363
- conversation.cleanupTag('experience', '...trimmed...', 0);
364
379
  conversation.cleanupTag('page_ui_map', '...trimmed...', 0);
365
380
  conversation.cleanupTag('page_ui_map_overlay', '...trimmed...', 0);
366
381
  conversation.compactToolResults(1);
@@ -398,7 +413,7 @@ export class Tester extends TaskAgent {
398
413
  return false;
399
414
  const currentState = this.getCurrentState();
400
415
  const stateChanged = previousState.url !== currentState.url || previousState.hash !== currentState.hash;
401
- const actionTools = [...this.ACTION_TOOLS, ...this.SPECIAL_CONTEXT_ACTION_TOOLS];
416
+ const actionTools = [...this.ACTION_TOOLS, ...this.DELEGATED_ACTION_TOOLS, ...this.SPECIAL_CONTEXT_ACTION_TOOLS];
402
417
  const hasSuccessfulAction = toolExecutions.some((execution) => execution.wasSuccessful && actionTools.includes(execution.toolName));
403
418
  const hasSuccessfulAssertion = toolExecutions.some((execution) => execution.wasSuccessful && this.ASSERTION_TOOLS.includes(execution.toolName));
404
419
  if (stateChanged || hasSuccessfulAction || hasSuccessfulAssertion) {
@@ -422,7 +437,8 @@ export class Tester extends TaskAgent {
422
437
 
423
438
  <rules>
424
439
  Use tools ${this.ACTION_TOOLS.join(', ')} to interact with the page.
425
- Use tool names exactly as listed in this prompt. Do not invent combined tool names, aliases, or names with channel markers such as "commentary".
440
+ Fall back to interact() when those fail, when the step needs a sequence of actions, or when your context is not enough to locate the element.
441
+ Use tool names exactly as listed in this prompt. Do not invent combined tool names or aliases.
426
442
  Match each tool input schema exactly. Do not invent parameter names or pass extra fields.
427
443
  Do not do unsuccesful clicks again.
428
444
  Do not run same tool calls with same parameters again.
@@ -532,13 +548,13 @@ export class Tester extends TaskAgent {
532
548
  </page>
533
549
 
534
550
  <page_aria>
535
- ${await this.interactiveAriaWithRefs(currentState)}
551
+ ${currentState.getInteractiveARIA()}
536
552
  </page_aria>
537
553
  ${uiMapSection}
538
554
 
539
555
  Use <page_ui_map> to understand the page structure and its main elements.
540
- However, <page_ui_map> is not always up to date, use <page_aria> and <page_html> to understand the ACTUAL state of the page
541
- Do not interact with elements that are not listed in <page_aria> and <page_html>
556
+ However, <page_ui_map> is not always up to date, use <page_aria> to understand the ACTUAL state of the page
557
+ Do not interact with elements that are not listed in <page_aria> or in HTML returned by tools
542
558
  Refer to information on page sections in <page_ui_map> and use container CSS locators to interact with elements inside sections
543
559
  `;
544
560
  return context;
@@ -567,16 +583,10 @@ export class Tester extends TaskAgent {
567
583
  </page>
568
584
 
569
585
  <page_aria>
570
- ${await this.interactiveAriaWithRefs(currentState)}
586
+ ${currentState.getInteractiveARIA()}
571
587
  </page_aria>
572
588
  `;
573
589
  }
574
- async interactiveAriaWithRefs(state) {
575
- const withRefs = await Promise.resolve(this.explorer?.withPage?.((page) => page.locator('body').ariaSnapshot({ mode: 'ai' }))).catch(() => null);
576
- if (!withRefs)
577
- return state.getInteractiveARIA();
578
- return compactAriaSnapshot(withRefs, false);
579
- }
580
590
  finishTest(task) {
581
591
  if (!task.result) {
582
592
  if (task.hasAchievedAll())
@@ -646,7 +656,7 @@ export class Tester extends TaskAgent {
646
656
 
647
657
  <rules>
648
658
  - Refer to UI Map from <page_ui_map> to understand the page structure and its main elements
649
- - Use only elements that exist in the provided ARIA tree or HTML, <page_aria> and <page_html>
659
+ - Use only elements that exist in <page_aria> or in HTML returned by tools
650
660
  - Use tool input schemas exactly as documented. Do not invent parameter names or add fields not listed by the tool schema.
651
661
  - Use click() for buttons, links, and clickable elements ONLY - do NOT include I.fillField() or I.type() commands in click() tool
652
662
  - click() commands array is for FALLBACK LOCATORS of the SAME element, NOT for clicking different elements in sequence. If you need to click two different elements, make two separate click() calls.
@@ -669,7 +679,8 @@ export class Tester extends TaskAgent {
669
679
  - Check for error messages to understand if there are issues
670
680
  - Verify if data was correctly saved and changes are reflected on the page
671
681
  - By default, you receive accessibility tree data which shows interactive elements and page structure
672
- - Understand current context by following <page_html>, <page_aria>, and <page_ui_map>
682
+ - Full page HTML is never injected automatically. When ARIA is not enough, delegate to the tools that read it: verify() to assert, interact() to act
683
+ - Understand current context by following <page_aria> and <page_ui_map>
673
684
  - Before submitting form, check all inputs were filled in correctly using see() tool
674
685
  - When you interact with form with inputs, ensure that you click corresponding button to save its data
675
686
  - Follow <locator_priority> rules when selecting locators for all tools
@@ -720,7 +731,6 @@ export class Tester extends TaskAgent {
720
731
  }
721
732
  buildScenarioBlock(task, actionResult) {
722
733
  const knowledge = this.getKnowledge(actionResult);
723
- const experience = this.getExperience(actionResult);
724
734
  return dedent `
725
735
  <task>
726
736
  SCENARIO GOAL: ${task.scenario}
@@ -749,8 +759,6 @@ export class Tester extends TaskAgent {
749
759
  ${this.buildAvailableFiles()}
750
760
 
751
761
  ${knowledge}
752
-
753
- ${experience}
754
762
  `;
755
763
  }
756
764
  getDeletableSessionNames(task) {
@@ -860,7 +868,7 @@ export class Tester extends TaskAgent {
860
868
  explanation,
861
869
  };
862
870
  if (resetAction.lastError) {
863
- result.error = resetAction.lastError.toString();
871
+ result.error = compactErrorMessage(resetAction.lastError);
864
872
  }
865
873
  return result;
866
874
  },
package/dist/src/ai/tools.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import { ActionResult, type PageDiff } from '../action-result.js';
2
- import type { ExperienceTracker } from '../experience-tracker.js';
2
+ import { type ExperienceTracker } from '../experience-tracker.js';
3
3
  import { type Task } from '../test-plan.js';
4
4
  import type { ToolDeps } from './agent.js';
5
5
  import { Navigator } from './navigator.js';
@@ -16,10 +16,6 @@ export declare function createCodeceptJSTools({ explorer, stateManager, ai }: To
16
16
  commands: any;
17
17
  explanation: any;
18
18
  }, Record<string, any>, import("@ai-sdk/provider-utils").Context>>;
19
- clickRef: import("@ai-sdk/provider-utils").ExecutableTool<import("ai").Tool<{
20
- ref: any;
21
- element: any;
22
- }, Record<string, any>, import("@ai-sdk/provider-utils").Context>>;
23
19
  hover: import("@ai-sdk/provider-utils").ExecutableTool<import("ai").Tool<{
24
20
  commands: any;
25
21
  explanation: any;
@@ -34,6 +30,12 @@ export declare function createCodeceptJSTools({ explorer, stateManager, ai }: To
34
30
  explanation: any;
35
31
  }, Record<string, any>, import("@ai-sdk/provider-utils").Context>>;
36
32
  };
33
+ export declare function createRefTools({ explorer, stateManager }: ToolDeps, task: Task): {
34
+ clickRef: import("@ai-sdk/provider-utils").ExecutableTool<import("ai").Tool<{
35
+ ref: any;
36
+ element: any;
37
+ }, Record<string, any>, import("@ai-sdk/provider-utils").Context>>;
38
+ };
37
39
  export declare function createIframeTools({ explorer, stateManager }: ToolDeps): {
38
40
  exitIframe: import("@ai-sdk/provider-utils").ExecutableTool<import("ai").Tool<{
39
41
  reason?: string;
@@ -59,6 +61,7 @@ export declare function successToolResult(action: string, data?: Record<string,
59
61
  assertionSteps?: any[];
60
62
  }): Record<string, any>;
61
63
  export declare function isMajorPageChange(pageDiff: PageDiff): boolean;
64
+ export declare function hasFailedRequest(pageDiff: PageDiff): boolean;
62
65
  export declare function failedToolResult(action: string, message: string, data?: Record<string, any>, error?: Error | null): Promise<Record<string, any>>;
63
66
  export declare function withdrawVisionTools(tools: Record<string, any>): void;
64
67
  export declare function clickFailureSuggestion(attempts: Array<{
package/dist/src/ai/tools.js CHANGED
@@ -2,11 +2,13 @@ import { tool } from 'ai';
2
2
  import dedent from 'dedent';
3
3
  import { z } from 'zod';
4
4
  import { ActionResult } from "../action-result.js";
5
+ import { renderExperienceRecipes } from "../experience-tracker.js";
5
6
  import { Stats } from "../stats.js";
6
7
  import { TestResult } from '../test-plan.js';
7
8
  import { LARGE_ARIA_CHANGE_THRESHOLD } from "../utils/aria.js";
8
9
  import { isFatalBrowserError } from "../utils/browser-errors.js";
9
10
  import { createDebug, tag } from '../utils/logger.js';
11
+ import { compactErrorMessage } from "../utils/strings.js";
10
12
  import { pause } from '../utils/loop.js';
11
13
  import { WebElement } from "../utils/web-element.js";
12
14
  import { sectionContextRule } from "./rules.js";
@@ -19,10 +21,6 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }, task) {
19
21
  description: dedent `
20
22
  Click an element by trying multiple CodeceptJS commands in order until one succeeds.
21
23
 
22
- Use this only for elements the page context gives you no ref for. When the element shows a ref such as [ref=e14],
23
- call clickRef with that ref instead — composing a locator for an element that already has a ref is wasted work,
24
- and a locator can match several elements where a ref cannot.
25
-
26
24
  Follow <locator_priority> from system prompt for locator selection.
27
25
 
28
26
  I.click(locator) - click element matching locator
@@ -78,7 +76,7 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }, task) {
78
76
  const success = await action.attempt(command, explanation);
79
77
  const attempt = { command, success };
80
78
  if (action.lastError)
81
- attempt.error = action.lastError.toString();
79
+ attempt.error = errorText(action.lastError);
82
80
  attempts.push(attempt);
83
81
  if (success) {
84
82
  const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, command);
@@ -100,7 +98,7 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }, task) {
100
98
  retryCommands.push(`I.click('${disambiguated.xpath.replace(/'/g, "\\'")}')`);
101
99
  for (const retryCmd of retryCommands) {
102
100
  if (!(await action.attempt(retryCmd, explanation))) {
103
- attempts.push({ command: retryCmd, success: false, error: action.lastError?.toString() });
101
+ attempts.push({ command: retryCmd, success: false, error: errorText(action.lastError) });
104
102
  continue;
105
103
  }
106
104
  const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, retryCmd);
@@ -118,38 +116,6 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }, task) {
118
116
  }, action.lastError);
119
117
  },
120
118
  }),
121
- clickRef: tool({
122
- description: dedent `
123
- Click an element by the ref the page context gave it, e.g. [ref=e14].
124
-
125
- Prefer this over click() whenever the element you want carries a ref. A ref names one exact element, so it
126
- cannot match several by mistake and never needs disambiguating — it is the fastest way to click.
127
- Only pass a ref that appears in the page context you were given. Never invent or guess one.
128
- If it reports the ref is gone, the page has been rebuilt: get fresh context and use the new ref.
129
- `,
130
- inputSchema: z.object({
131
- ref: z.string().describe('The ref exactly as it appears in the page context, e.g. "e14"'),
132
- element: z.string().describe('Role and name of the element you are clicking, for the record'),
133
- }),
134
- execute: async ({ ref, element }) => {
135
- const activeNote = task.startNote(`Click ${element}`);
136
- const previousState = ActionResult.fromState(stateManager.getCurrentState());
137
- const action = explorer.action();
138
- const named = await describeRef(explorer, ref);
139
- const run = `I.usePlaywrightTo(${JSON.stringify(`click ${element}`)}, async ({ page }) => page.locator(${JSON.stringify(`aria-ref=${ref}`)}).click())`;
140
- if (!(await action.attempt(run, `Click ${element}`))) {
141
- activeNote.commit(TestResult.FAILED);
142
- return failedToolResult('clickRef', `Ref ${ref} could not be clicked: ${errorText(action.lastError)}`, {
143
- suggestion: 'The ref may belong to an older version of the page. Get fresh context and use the ref it gives, or fall back to click() with a locator.',
144
- });
145
- }
146
- // a ref belongs to this session only, so the run is reported as the locator a later test can replay
147
- const code = named ? `I.click(${JSON.stringify(named)})` : run;
148
- const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, code);
149
- await commitNote(activeNote, TestResult.PASSED, toolResult, action);
150
- return successToolResult('clickRef', { ...toolResult, code }, action);
151
- },
152
- }),
153
119
  hover: tool({
154
120
  description: dedent `
155
121
  Move the mouse cursor to an element to reveal hover-only controls.
@@ -197,7 +163,7 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }, task) {
197
163
  const success = await action.attempt(command, explanation);
198
164
  const attempt = { command, success };
199
165
  if (action.lastError)
200
- attempt.error = action.lastError.toString();
166
+ attempt.error = errorText(action.lastError);
201
167
  attempts.push(attempt);
202
168
  if (!success)
203
169
  continue;
@@ -264,7 +230,7 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }, task) {
264
230
  fallback: true,
265
231
  }, action);
266
232
  }
267
- const errorMsg = `pressKey fallback to type() failed: ${action.lastError?.toString()}`;
233
+ const errorMsg = `pressKey fallback to type() failed: ${errorText(action.lastError)}`;
268
234
  await commitNote(activeNote, TestResult.FAILED, toolResult, action);
269
235
  return failedToolResult('pressKey', errorMsg, {
270
236
  ...toolResult,
@@ -300,7 +266,7 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }, task) {
300
266
  code: pressKeyCommand,
301
267
  }, action);
302
268
  }
303
- const errorMsg = `pressKey() failed: ${action.lastError?.toString()}`;
269
+ const errorMsg = `pressKey() failed: ${errorText(action.lastError)}`;
304
270
  await commitNote(activeNote, TestResult.FAILED, toolResult, action);
305
271
  return failedToolResult('pressKey', errorMsg, {
306
272
  ...toolResult,
@@ -321,8 +287,6 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }, task) {
321
287
  Execute raw CodeceptJS code block with multiple commands.
322
288
  USE THIS TOOL for typing text into fields: I.fillField, I.type
323
289
 
324
- Do not put a click on a ref-bearing element in here — clickRef with its ref is cheaper and cannot mis-target.
325
-
326
290
  Follow <actions> from system prompt for available commands.
327
291
  Follow <locator_priority> from system prompt for locator selection.
328
292
 
@@ -377,7 +341,7 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }, task) {
377
341
  await action.attempt(codeBlock, explanation);
378
342
  const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, formLocator);
379
343
  if (action.lastError) {
380
- const message = action.lastError ? String(action.lastError) : 'Unknown error';
344
+ const message = errorText(action.lastError);
381
345
  await commitNote(activeNote, TestResult.FAILED, toolResult, action);
382
346
  let formSuggestion = 'Look into error message and identify which commands passed and which failed. Continue execution using step-by-step approach using click() and form() tools.';
383
347
  if (message.toLowerCase().includes(MULTIPLE_ELEMENTS_PATTERN)) {
@@ -419,6 +383,42 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }, task) {
419
383
  }),
420
384
  };
421
385
  }
386
+ export function createRefTools({ explorer, stateManager }, task) {
387
+ return {
388
+ clickRef: tool({
389
+ description: dedent `
390
+ Click an element by the ref the page context gave it, e.g. [ref=e14].
391
+
392
+ Prefer this over click() whenever the element you want carries a ref. A ref names one exact element, so it
393
+ cannot match several by mistake and never needs disambiguating — it is the fastest way to click.
394
+ Only pass a ref that appears in the page context you were given. Never invent or guess one.
395
+ If it reports the ref is gone, the page has been rebuilt: get fresh context and use the new ref.
396
+ `,
397
+ inputSchema: z.object({
398
+ ref: z.string().describe('The ref exactly as it appears in the page context, e.g. "e14"'),
399
+ element: z.string().describe('Role and name of the element you are clicking, for the record'),
400
+ }),
401
+ execute: async ({ ref, element }) => {
402
+ const activeNote = task.startNote(`Click ${element}`);
403
+ const previousState = ActionResult.fromState(stateManager.getCurrentState());
404
+ const action = explorer.action();
405
+ const named = await describeRef(explorer, ref);
406
+ const run = `I.usePlaywrightTo(${JSON.stringify(`click ${element}`)}, async ({ page }) => page.locator(${JSON.stringify(`aria-ref=${ref}`)}).click())`;
407
+ if (!(await action.attempt(run, `Click ${element}`))) {
408
+ activeNote.commit(TestResult.FAILED);
409
+ return failedToolResult('clickRef', `Ref ${ref} could not be clicked: ${errorText(action.lastError)}`, {
410
+ suggestion: 'The ref may belong to an older version of the page. Get fresh context and use the ref it gives, or fall back to click() with a locator.',
411
+ });
412
+ }
413
+ // a ref belongs to this session only, so the run is reported as the locator a later test can replay
414
+ const code = named ? `I.click(${JSON.stringify(named)})` : run;
415
+ const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, code);
416
+ await commitNote(activeNote, TestResult.PASSED, toolResult, action);
417
+ return successToolResult('clickRef', { ...toolResult, code }, action);
418
+ },
419
+ }),
420
+ };
421
+ }
422
422
  export function createIframeTools({ explorer, stateManager }) {
423
423
  return {
424
424
  exitIframe: tool({
@@ -535,7 +535,7 @@ export function createAgentTools({ explorer, stateManager, ai, researcher, navig
535
535
 
536
536
  DO NOT call this if:
537
537
  - You just performed an action (pageDiff already provided in response)
538
- - You already have recent <page_html>/<page_aria> in context
538
+ - You already have a recent <page_aria> snapshot in context
539
539
  - You're about to perform an action (you'll get pageDiff after)
540
540
 
541
541
  Call ONLY when:
@@ -675,12 +675,18 @@ export function createAgentTools({ explorer, stateManager, ai, researcher, navig
675
675
  }),
676
676
  interact: tool({
677
677
  description: dedent `
678
- Execute an action on the current page using AI-powered interaction.
679
- Use this to perform actions like clicking buttons, selecting options, filling forms, etc.
680
- The AI will generate and try multiple CodeceptJS code strategies to accomplish the instruction.
678
+ Delegate one step to the Navigator, which reads the full page HTML and tries multiple CodeceptJS strategies.
679
+ Slower than the direct action tools — use it as a fallback, not as the default.
680
+
681
+ Use when:
682
+ - direct action tools failed and you have no better locator to try
683
+ - the step needs a sequence of actions to complete
684
+ - the element is not in the context you have
685
+
686
+ Describe the outcome to reach, not the locator to use.
681
687
  `,
682
688
  inputSchema: z.object({
683
- instruction: z.string().describe('What action to perform on the page, e.g. "select new suite option", "click the Submit button"'),
689
+ instruction: z.string().describe('The step to perform on the page, described by its intent'),
684
690
  }),
685
691
  execute: async ({ instruction }) => {
686
692
  try {
@@ -690,7 +696,8 @@ export function createAgentTools({ explorer, stateManager, ai, researcher, navig
690
696
  }
691
697
  const previousState = ActionResult.fromState(currentState);
692
698
  const actionResult = ActionResult.fromState(currentState);
693
- const success = await navigator.resolveState(instruction, actionResult);
699
+ const experience = renderExperienceRecipes(explorer.activeTest?.getAppliedExperience(actionResult) ?? []);
700
+ const success = await navigator.resolveState(instruction, actionResult, { experience });
694
701
  const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, instruction);
695
702
  if (success) {
696
703
  return successToolResult('interact', {
@@ -824,7 +831,7 @@ export function createAgentTools({ explorer, stateManager, ai, researcher, navig
824
831
  }
825
832
  const failData = { suggestion: 'Try reset() to return to the starting page.' };
826
833
  if (action.lastError)
827
- failData.error = action.lastError.toString();
834
+ failData.error = errorText(action.lastError);
828
835
  return failedToolResult('back', `Failed to navigate back to ${targetUrl}`, failData);
829
836
  },
830
837
  }),
@@ -881,7 +888,7 @@ export function createAgentTools({ explorer, stateManager, ai, researcher, navig
881
888
  }
882
889
  if (result.totalFound === 0) {
883
890
  return failedToolResult('xpathCheck', `No elements matched XPath: ${xpath}`, {
884
- suggestion: 'Try a broader expression. Examples: //*[contains(@class, "btn")], //button, //*[contains(text(), "keyword")]',
891
+ suggestion: 'Do not guess another expression. Narrow down from what you know about the target: its role, its visible text, its nearest labelled ancestor. Add one constraint at a time.',
885
892
  });
886
893
  }
887
894
  const action = explorer.action();
@@ -963,7 +970,9 @@ export function createAgentTools({ explorer, stateManager, ai, researcher, navig
963
970
  withdrawVisionTools(tools);
964
971
  return tools;
965
972
  }
966
- const PAGE_DIFF_SUGGESTION = 'Analyze page diff. htmlParts shows what changed and WHERE — each part has a container selector. Use the container as context when clicking elements from the diff.';
973
+ const PAGE_DIFF_SUGGESTION = 'Analyze page diff. htmlParts shows what changed and WHERE — each part has a container selector. Use the container as context when clicking elements from the diff. messages holds text the app showed in response, requests the calls it made and consoleErrors what it logged.';
974
+ const FAILED_REQUEST_SUGGESTION = 'The server rejected a request made by this action (see requests). The UI accepted the interaction but the operation did not complete — read messages and consoleErrors for the reason and report it instead of repeating the action.';
975
+ const NAVIGATED_SUGGESTION = 'The action left the page. Elements are never compared across pages, so this diff carries the move itself and what the app announced in transit — an empty element diff does not mean nothing happened.';
967
976
  const ARIA_OUTPUT_CAP = 4000;
968
977
  const HTML_OUTPUT_CAP = 6000;
969
978
  const ANALYSIS_OUTPUT_CAP = 2000;
@@ -1001,7 +1010,7 @@ function transformContainsCommand(command) {
1001
1010
  }
1002
1011
  function errorText(error) {
1003
1012
  if (error instanceof Error)
1004
- return error.toString();
1013
+ return compactErrorMessage(error);
1005
1014
  return 'Unknown error occurred';
1006
1015
  }
1007
1016
  export async function commitNote(activeNote, result, toolResult, action) {
@@ -1037,7 +1046,13 @@ export function successToolResult(action, data, source) {
1037
1046
  const ariaChanges = data.pageDiff.ariaChanges || '';
1038
1047
  const urlChanged = data.pageDiff.urlChanged === true;
1039
1048
  const hasHtmlParts = Array.isArray(data.pageDiff.htmlParts) && data.pageDiff.htmlParts.length > 0;
1040
- if (isMajorPageChange(data.pageDiff)) {
1049
+ if (hasFailedRequest(data.pageDiff)) {
1050
+ suggestion = `${FAILED_REQUEST_SUGGESTION} ${suggestion}`;
1051
+ }
1052
+ else if (urlChanged) {
1053
+ suggestion = `${NAVIGATED_SUGGESTION} ${suggestion}`;
1054
+ }
1055
+ else if (isMajorPageChange(data.pageDiff)) {
1041
1056
  suggestion = `MAJOR PAGE CHANGE. Page entered a different mode. Check htmlParts and iframes in pageDiff before next action. ${suggestion}`;
1042
1057
  }
1043
1058
  else if (!urlChanged && !ariaChanges && !hasHtmlParts) {
@@ -1053,6 +1068,9 @@ export function successToolResult(action, data, source) {
1053
1068
  export function isMajorPageChange(pageDiff) {
1054
1069
  return pageDiff.urlChanged !== true && (pageDiff.ariaChangeCount ?? 0) >= LARGE_ARIA_CHANGE_THRESHOLD;
1055
1070
  }
1071
+ export function hasFailedRequest(pageDiff) {
1072
+ return (pageDiff.requests ?? []).some((request) => request.status >= 400);
1073
+ }
1056
1074
  function hasObservablePageChange(data) {
1057
1075
  if (!data?.pageDiff)
1058
1076
  return false;
@@ -1060,6 +1078,8 @@ function hasObservablePageChange(data) {
1060
1078
  return true;
1061
1079
  if (data.pageDiff.ariaChanges)
1062
1080
  return true;
1081
+ if (data.pageDiff.messages?.length)
1082
+ return true;
1063
1083
  return Array.isArray(data.pageDiff.htmlParts) && data.pageDiff.htmlParts.length > 0;
1064
1084
  }
1065
1085
  export async function failedToolResult(action, message, data, error) {
@@ -1111,6 +1131,9 @@ export function clickFailureSuggestion(attempts) {
1111
1131
  if (errors.some((e) => e.includes('is not visible'))) {
1112
1132
  return 'Element is in the DOM but not visible. Reveal it first — scroll to it, expand its section, or open the panel holding it.';
1113
1133
  }
1134
+ if (errors.some((e) => e.includes('SyntaxError'))) {
1135
+ return 'The command string never parsed as JavaScript — quotes or brackets do not match. No element was looked up, so this tells you nothing about the page. Re-emit the same intent as valid CodeceptJS.';
1136
+ }
1114
1137
  const notFound = errors.filter((e) => e.includes('was not found'));
1115
1138
  if (notFound.length && notFound.every((e) => e.includes('was not found inside element'))) {
1116
1139
  return 'Element was not found inside that container — the container is wrong or stale, and the element may exist elsewhere on the page. Retry the same locator WITHOUT a container, or verify the container with xpathCheck().';
@@ -1191,7 +1214,7 @@ function getNotFoundSuggestion(errorMessage) {
1191
1214
  Element was not found. The locator does not exist on this page.
1192
1215
  1. Use see() to visually analyze what elements are actually on the page
1193
1216
  2. Use context() to get fresh HTML and ARIA snapshot
1194
- 3. Use ONLY locators from <page_aria> or <page_html>
1217
+ 3. Use ONLY locators from <page_aria> or from HTML returned by context()
1195
1218
  4. Prefer ARIA locators: { "role": "button", "text": "visible text" }
1196
1219
  `;
1197
1220
  }
package/dist/src/commands/explore-command.js CHANGED
@@ -59,25 +59,30 @@ export class ExploreCommand extends BaseCommand {
59
59
  tag('warning').log(error.message);
60
60
  return;
61
61
  }
62
- if (cfg.enabled) {
63
- await this.runReuseMode(mainUrl, feature, cfg);
64
- }
65
- else {
66
- await this.runFreshMode(mainUrl, feature, cfg.styles);
67
- }
68
- const mainPlan = this.completedPlans[0];
69
- if (mainPlan)
70
- this.explorBot.setCurrentPlan(mainPlan);
71
- if (this.dryRun) {
62
+ try {
63
+ if (cfg.enabled) {
64
+ await this.runReuseMode(mainUrl, feature, cfg);
65
+ }
66
+ else {
67
+ await this.runFreshMode(mainUrl, feature, cfg.styles);
68
+ }
69
+ const mainPlan = this.completedPlans[0];
70
+ if (mainPlan)
71
+ this.explorBot.setCurrentPlan(mainPlan);
72
+ if (this.dryRun) {
73
+ this.printResults();
74
+ return;
75
+ }
76
+ if (mainUrl)
77
+ await this.explorBot.visit(mainUrl).catch((err) => tag('warning').log(`Could not return to ${mainUrl}: ${browserErrorMessage(err)}`));
78
+ const savedPath = this.explorBot.savePlans(this.completedPlans);
72
79
  this.printResults();
73
- return;
80
+ this.printNextSteps(savedPath);
81
+ }
82
+ finally {
83
+ if (!this.dryRun)
84
+ await this.explorBot.printSessionAnalysis();
74
85
  }
75
- if (mainUrl)
76
- await this.explorBot.visit(mainUrl).catch((err) => tag('warning').log(`Could not return to ${mainUrl}: ${browserErrorMessage(err)}`));
77
- const savedPath = this.explorBot.savePlans(this.completedPlans);
78
- this.printResults();
79
- await this.explorBot.printSessionAnalysis();
80
- this.printNextSteps(savedPath);
81
86
  }
82
87
  originLabel(test) {
83
88
  return this.oldTestRefs.has(test) ? 'OLD' : 'NEW';