explorbot 0.1.28 → 0.1.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/README.md +83 -245
  2. package/bin/explorbot-cli.ts +1 -0
  3. package/boat/doc-collector/src/ai/documentarian.ts +37 -13
  4. package/boat/doc-collector/src/ai/tools.ts +60 -20
  5. package/boat/doc-collector/src/cli.ts +3 -0
  6. package/boat/doc-collector/src/config.ts +7 -0
  7. package/boat/doc-collector/src/docbot.ts +23 -5
  8. package/boat/doc-collector/src/docs-renderer.ts +14 -1
  9. package/boat/doc-collector/src/screenshots.ts +126 -0
  10. package/dist/bin/explorbot-cli.js +1 -0
  11. package/dist/boat/doc-collector/src/ai/documentarian.js +15 -11
  12. package/dist/boat/doc-collector/src/ai/tools.js +53 -20
  13. package/dist/boat/doc-collector/src/cli.js +3 -0
  14. package/dist/boat/doc-collector/src/config.js +3 -0
  15. package/dist/boat/doc-collector/src/docbot.js +19 -4
  16. package/dist/boat/doc-collector/src/docs-renderer.js +12 -1
  17. package/dist/boat/doc-collector/src/screenshots.js +90 -0
  18. package/dist/package.json +8 -6
  19. package/dist/rules/navigator/verification-actions.md +2 -0
  20. package/dist/src/action.js +26 -23
  21. package/dist/src/ai/fisherman.js +14 -3
  22. package/dist/src/ai/historian/codeceptjs.js +3 -2
  23. package/dist/src/ai/historian/experience.js +48 -6
  24. package/dist/src/ai/historian/playwright.js +2 -1
  25. package/dist/src/ai/historian/utils.js +1 -19
  26. package/dist/src/ai/historian.js +1 -1
  27. package/dist/src/ai/pilot.js +19 -4
  28. package/dist/src/ai/planner.js +16 -5
  29. package/dist/src/ai/provider.js +53 -18
  30. package/dist/src/ai/quartermaster.js +2 -2
  31. package/dist/src/ai/researcher.js +7 -1
  32. package/dist/src/ai/rules.js +44 -0
  33. package/dist/src/ai/tester.js +73 -7
  34. package/dist/src/ai/tools.js +66 -1
  35. package/dist/src/experience-tracker.js +1 -1
  36. package/dist/src/explorbot.js +14 -3
  37. package/dist/src/explorer.js +30 -27
  38. package/dist/src/stats.js +16 -0
  39. package/dist/src/utils/aria.js +66 -6
  40. package/dist/src/utils/browser-errors.js +5 -0
  41. package/dist/src/utils/page-readiness.js +48 -0
  42. package/dist/src/utils/step-analyzer.js +68 -0
  43. package/package.json +8 -6
  44. package/rules/navigator/verification-actions.md +2 -0
  45. package/src/action.ts +24 -26
  46. package/src/ai/fisherman.ts +14 -3
  47. package/src/ai/historian/codeceptjs.ts +3 -2
  48. package/src/ai/historian/experience.ts +51 -6
  49. package/src/ai/historian/playwright.ts +2 -1
  50. package/src/ai/historian/utils.ts +1 -21
  51. package/src/ai/historian.ts +1 -1
  52. package/src/ai/pilot.ts +19 -4
  53. package/src/ai/planner.ts +16 -5
  54. package/src/ai/provider.ts +51 -19
  55. package/src/ai/quartermaster.ts +2 -2
  56. package/src/ai/researcher.ts +8 -1
  57. package/src/ai/rules.ts +46 -0
  58. package/src/ai/tester.ts +77 -7
  59. package/src/ai/tools.ts +79 -1
  60. package/src/config.ts +2 -0
  61. package/src/experience-tracker.ts +1 -1
  62. package/src/explorbot.ts +13 -3
  63. package/src/explorer.ts +28 -27
  64. package/src/stats.ts +18 -0
  65. package/src/utils/aria.ts +63 -6
  66. package/src/utils/browser-errors.ts +6 -0
  67. package/src/utils/page-readiness.ts +59 -0
  68. package/src/utils/step-analyzer.ts +73 -0
@@ -19,7 +19,7 @@ import { WithSubPages, getPlannedByStateHash, getRegisteredPlan, registerPlan }
19
19
  import { POSSIBLE_SECTIONS, Researcher } from "./researcher.js";
20
20
  import { findSimilarStateHash } from "./researcher/cache.js";
21
21
  import { hasFocusedSection } from "./researcher/focus.js";
22
- import { fileUploadRule, protectionRule } from "./rules.js";
22
+ import { capabilityGroundingRule, dataProtectionRules, fileUploadRule } from "./rules.js";
23
23
  const debugLog = createDebug('explorbot:planner');
24
24
  const TasksSchema = z.object({
25
25
  planName: z.string().describe('Short descriptive name for the test plan (e.g., "User Authentication Testing", "Product Catalog Navigation", "Form Validation Tests")'),
@@ -27,7 +27,7 @@ const TasksSchema = z.object({
27
27
  .array(z.object({
28
28
  scenario: z.string().describe('A single sentence describing what to test'),
29
29
  priority: z.enum(['critical', 'important', 'high', 'normal', 'low']).describe('Priority of the task based on business importance'),
30
- startUrl: z.string().nullable().describe('Start URL for the test if different from plan URL (only for tests on visited subpages)'),
30
+ startUrl: z.string().nullable().describe('Start URL for the test if different from plan URL. Use only stable feature/list/detail pages, not transient create/edit/modal URLs unless the scenario specifically starts inside that form.'),
31
31
  steps: z.array(z.string()).describe('List of steps to perform for this scenario. Each step should be a specific action (e.g., "Open the form", "Enter required data", "Submit the form"). Keep steps atomic and actionable.'),
32
32
  expectedOutcomes: z
33
33
  .array(z.string())
@@ -73,6 +73,9 @@ export class Planner extends PlannerBase {
73
73
  const featureDirective = feature
74
74
  ? `\n IMPORTANT: The user requested to focus specifically on: "${feature}"\n ALL scenarios MUST be directly related to this feature. Do not propose generic page tests unrelated to it.\n Use the user's exact wording to guide scenario names — do not substitute different entities (e.g., do not plan "suite" actions when user said "test").`
75
75
  : '';
76
+ const focusExistingDataDirective = feature
77
+ ? '\n If this focus asks for search, filter, tabs, sorting, or list behavior involving existing items, only use item names/values visible in the provided page research. If no concrete visible item names/values are present, do NOT propose scenarios that require an existing known item; propose no-match search, empty-state, clear-search, tab/filter empty-list, or other read-only list behavior instead.'
78
+ : '';
76
79
  return dedent `
77
80
  <role>
78
81
  You are ISTQB certified senior manual QA planning exploratory testing session of a web application.
@@ -96,7 +99,7 @@ export class Planner extends PlannerBase {
96
99
  Bad: "Open delete dropdown" + "Confirm deletion" — these are ONE test, not two.
97
100
  Bad: "Search for X" + "Verify search results" — searching and verifying is ONE test.
98
101
  Bad: "Leave field empty" + "Click submit" — that's one negative test, not two.
99
- If two scenarios cannot run independently (one requires the other to run first), merge them into one.${featureDirective}
102
+ If two scenarios cannot run independently (one requires the other to run first), merge them into one.${featureDirective}${focusExistingDataDirective}
100
103
  </task>
101
104
 
102
105
  ${customPrompt || ''}
@@ -301,6 +304,11 @@ export class Planner extends PlannerBase {
301
304
  If a scenario needs existing records, recipients, results, notifications, or other target data, propose it only when that data is visible or API preconditions can create it.
302
305
  If the page appears read-only, degraded, demo-limited, maintenance-like, or lacks write controls, prefer read-only scenarios such as opening panels, inspecting visible lists, filtering, searching, or verifying current state.
303
306
  Do not assume hidden data exists just because a control is present.
307
+ For scenarios that act on existing items or search/filter by existing values, use only item names or values visible in research, visited pages, or prior observed flows.
308
+ If the list is empty or no concrete item names are visible, do not invent "known" or "existing" items. Prefer empty-state, no-match search, clear-search, or read-only list behavior scenarios.
309
+ Search, filter, sorting, tab, and list scenarios must start from a stable page where those controls are visible; avoid transient create/edit/new URLs unless the scenario tests that form.
310
+ For option values and list items, use only visible or previously observed data; do not add create/update/delete setup unless the user explicitly requests that workflow.
311
+ Detail-view scenarios must target visible data entities from list rows, cards, tree nodes, or detail links; do not use filter tabs, counters, status tabs, breadcrumbs, or navigation controls as detail targets.
304
312
  DO NOT propose "verification-only" tests that merely open a UI element (modal, dropdown, panel) and check it exists.
305
313
  Every test must complete a meaningful action that changes application state or produces a business outcome.
306
314
  Opening a modal is NOT a test — performing an action INSIDE the modal IS a test.
@@ -311,7 +319,8 @@ export class Planner extends PlannerBase {
311
319
  Tests that only switch views, toggle filters, or paginate are LESS valuable — propose them only after data-changing tests are covered.
312
320
  If multiple ways to create or modify data exist (different types, different forms), propose a separate test for each.
313
321
  </priority_order>
314
- ${protectionRule}
322
+ ${capabilityGroundingRule}
323
+ ${dataProtectionRules}
315
324
  ${fileUploadRule}
316
325
  </rules>
317
326
 
@@ -466,7 +475,9 @@ export class Planner extends PlannerBase {
466
475
  .join('\n')}
467
476
 
468
477
  You MAY propose tests starting from these pages if they are relevant to the plan "${this.currentPlan.title}".
469
- Set startUrl for such tests. Ignore pages that belong to a different feature area.
478
+ Set startUrl for such tests only when the page is a stable feature/list/detail page.
479
+ Do not use create/edit/new/modal URLs as startUrl for scenarios that need the underlying page.
480
+ Ignore pages that belong to a different feature area.
470
481
  </context_from_previous_tests>
471
482
 
472
483
  Propose ONLY new scenarios that are NOT in the existing tests list.
@@ -1,6 +1,7 @@
1
+ import { OpenTelemetry } from '@ai-sdk/otel';
1
2
  import { LangfuseSpanProcessor } from '@langfuse/otel';
2
3
  import { NodeSDK } from '@opentelemetry/sdk-node';
3
- import { generateObject, generateText, stepCountIs } from 'ai';
4
+ import { generateObject, generateText, isStepCount, registerTelemetry } from 'ai';
4
5
  import { clearActivity, setActivity } from "../activity.js";
5
6
  import { executionController } from "../execution-controller.js";
6
7
  import { Observability } from "../observability.js";
@@ -16,10 +17,11 @@ class AiError extends Error {
16
17
  }
17
18
  export class ContextLengthError extends Error {
18
19
  }
20
+ let telemetryRegistered = false;
19
21
  function extractCachedTokens(usage) {
20
22
  if (!usage)
21
23
  return 0;
22
- const direct = usage.cachedInputTokens ?? usage.inputTokenDetails?.cacheReadTokens;
24
+ const direct = usage.inputTokenDetails?.cacheReadTokens ?? usage.cachedInputTokens;
23
25
  if (typeof direct === 'number')
24
26
  return direct;
25
27
  const raw = usage.raw;
@@ -78,7 +80,7 @@ export class Provider {
78
80
  await generateText({
79
81
  model: this.config.model,
80
82
  prompt: 'hi',
81
- maxTokens: 1,
83
+ maxOutputTokens: 1,
82
84
  });
83
85
  }
84
86
  catch (error) {
@@ -100,6 +102,18 @@ export class Provider {
100
102
  }
101
103
  return this.config.agenticModel || this.config.model;
102
104
  }
105
+ getConfiguredModels() {
106
+ const models = { model: this.getModelName(this.config.model) };
107
+ if (this.config.agenticModel)
108
+ models.agenticModel = this.getModelName(this.config.agenticModel);
109
+ if (this.config.visionModel)
110
+ models.visionModel = this.getModelName(this.config.visionModel);
111
+ for (const [agent, agentConfig] of Object.entries(this.config.agents || {})) {
112
+ if (agentConfig?.model)
113
+ models[agent] = this.getModelName(agentConfig.model);
114
+ }
115
+ return models;
116
+ }
103
117
  getSystemPromptForAgent(agentName, currentUrl) {
104
118
  const agentConfig = this.config.agents?.[agentName];
105
119
  const parts = [];
@@ -116,6 +130,12 @@ export class Provider {
116
130
  const agentConfig = this.config.agents?.[agentName];
117
131
  return agentConfig?.providerOptions;
118
132
  }
133
+ getReasoningForAgent(agentName) {
134
+ if (!agentName)
135
+ return undefined;
136
+ const agentConfig = this.config.agents?.[agentName];
137
+ return agentConfig?.reasoning;
138
+ }
119
139
  getRetryOptions(options = {}) {
120
140
  return {
121
141
  ...this.defaultRetryOptions,
@@ -133,6 +153,13 @@ export class Provider {
133
153
  providerOptions: { ...config.providerOptions, ...agentOptions },
134
154
  };
135
155
  }
156
+ finalizeConfig(config, options, telemetry) {
157
+ if (telemetry)
158
+ config.telemetry = telemetry;
159
+ const reasoning = this.getReasoningForAgent(options.agentName);
160
+ if (reasoning)
161
+ config.reasoning ??= reasoning;
162
+ }
136
163
  initLangfuse() {
137
164
  const langfuseConfig = this.config.langfuse;
138
165
  const publicKey = langfuseConfig?.publicKey || process.env.LANGFUSE_PUBLIC_KEY;
@@ -152,6 +179,10 @@ export class Provider {
152
179
  instrumentations: [],
153
180
  });
154
181
  void this.otelSdk.start();
182
+ if (!telemetryRegistered) {
183
+ registerTelemetry(new OpenTelemetry());
184
+ telemetryRegistered = true;
185
+ }
155
186
  this.telemetryEnabled = true;
156
187
  }
157
188
  getTelemetry(options) {
@@ -159,18 +190,18 @@ export class Provider {
159
190
  return undefined;
160
191
  }
161
192
  const runTelemetry = Observability.getTelemetry();
162
- if (!options.experimental_telemetry) {
163
- return runTelemetry || { isEnabled: true };
193
+ if (!options.telemetry) {
194
+ return runTelemetry;
164
195
  }
165
196
  if (!runTelemetry) {
166
- return options.experimental_telemetry;
197
+ return options.telemetry;
167
198
  }
168
199
  return {
169
200
  ...runTelemetry,
170
- ...options.experimental_telemetry,
201
+ ...options.telemetry,
171
202
  metadata: {
172
203
  ...runTelemetry.metadata,
173
- ...options.experimental_telemetry.metadata,
204
+ ...options.telemetry.metadata,
174
205
  },
175
206
  };
176
207
  }
@@ -210,13 +241,14 @@ export class Provider {
210
241
  promptLog(`Using model: ${modelName}`);
211
242
  const telemetry = this.getTelemetry(options);
212
243
  const config = this.mergeProviderOptions({
213
- maxTokens: 16384,
244
+ maxOutputTokens: 16384,
245
+ allowSystemInMessages: true,
214
246
  ...(this.config.config || {}),
215
247
  ...options,
216
- ...(telemetry ? { experimental_telemetry: telemetry } : {}),
217
248
  model,
218
249
  abortSignal: executionController.getAbortSignal(),
219
250
  }, options.agentName);
251
+ this.finalizeConfig(config, options, telemetry);
220
252
  promptLog(messages[messages.length - 1].content);
221
253
  try {
222
254
  const response = await withRetry(async () => {
@@ -224,7 +256,7 @@ export class Provider {
224
256
  if (!result.text) {
225
257
  debugLog(result);
226
258
  if (result.finishReason === 'length') {
227
- throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase maxTokens in config or use a model with higher output capacity.');
259
+ throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase maxOutputTokens in config or use a model with higher output capacity.');
228
260
  }
229
261
  throw new Error('No response text from AI');
230
262
  }
@@ -277,21 +309,22 @@ export class Provider {
277
309
  const telemetry = this.getTelemetry(options);
278
310
  const maxRoundtrips = options.maxToolRoundtrips ?? 5;
279
311
  const extraStop = options.stopWhen;
280
- const stopConditions = [stepCountIs(maxRoundtrips)];
312
+ const stopConditions = [isStepCount(maxRoundtrips)];
281
313
  if (extraStop)
282
314
  stopConditions.push(extraStop);
283
315
  const { stopWhen: _ignoredStopWhen, ...optionsWithoutStop } = options;
284
316
  const config = this.mergeProviderOptions({
285
317
  tools,
286
- maxTokens: 16384,
318
+ maxOutputTokens: 16384,
287
319
  toolChoice: 'auto',
320
+ allowSystemInMessages: true,
288
321
  ...(this.config.config || {}),
289
322
  ...optionsWithoutStop,
290
323
  stopWhen: stopConditions,
291
- ...(telemetry ? { experimental_telemetry: telemetry } : {}),
292
324
  model,
293
325
  abortSignal: executionController.getAbortSignal(),
294
326
  }, options.agentName);
327
+ this.finalizeConfig(config, options, telemetry);
295
328
  try {
296
329
  const response = await withRetry(async () => {
297
330
  const timeout = config.timeout || 30000;
@@ -306,7 +339,7 @@ export class Provider {
306
339
  ]));
307
340
  const hasToolCall = (result.toolCalls?.length || 0) > 0;
308
341
  if (!result.text && !hasToolCall && result.finishReason === 'length') {
309
- throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase maxTokens in config or use a model with higher output capacity.');
342
+ throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase maxOutputTokens in config or use a model with higher output capacity.');
310
343
  }
311
344
  return result;
312
345
  }
@@ -365,12 +398,13 @@ export class Provider {
365
398
  const telemetry = this.getTelemetry(options);
366
399
  const config = this.mergeProviderOptions({
367
400
  schema,
401
+ allowSystemInMessages: true,
368
402
  ...(this.config.config || {}),
369
403
  ...options,
370
- ...(telemetry ? { experimental_telemetry: telemetry } : {}),
371
404
  model: modelToUse,
372
405
  abortSignal: executionController.getAbortSignal(),
373
406
  }, options.agentName);
407
+ this.finalizeConfig(config, options, telemetry);
374
408
  try {
375
409
  promptLog(messages[messages.length - 1].content);
376
410
  const response = await withRetry(async () => {
@@ -552,12 +586,13 @@ export class Provider {
552
586
  ];
553
587
  const telemetry = this.getTelemetry({});
554
588
  const config = {
555
- maxTokens: 16384,
589
+ maxOutputTokens: 16384,
556
590
  ...(this.config.config || {}),
557
- ...(telemetry ? { experimental_telemetry: telemetry } : {}),
558
591
  model: this.config.visionModel,
559
592
  abortSignal: executionController.getAbortSignal(),
560
593
  };
594
+ if (telemetry)
595
+ config.telemetry = telemetry;
561
596
  try {
562
597
  promptLog(`Processing image with prompt: ${prompt}`);
563
598
  const response = await withRetry(async () => {
@@ -3,7 +3,7 @@ import { join } from 'node:path';
3
3
  import { z } from 'zod';
4
4
  import { ConfigParser } from "../config.js";
5
5
  import { createDebug, tag } from "../utils/logger.js";
6
- import { CODECEPT_TOOLS } from "./tools.js";
6
+ import { isCodeceptToolName } from "../utils/step-analyzer.js";
7
7
  const debugLog = createDebug('explorbot:quartermaster');
8
8
  export class Quartermaster {
9
9
  provider;
@@ -88,7 +88,7 @@ export class Quartermaster {
88
88
  const stateHash = initialState.getStateHash();
89
89
  const pageAnalysis = this.pageAnalyses.get(stateHash);
90
90
  const toolExecutions = conversation.getToolExecutions();
91
- const codeceptExecutions = toolExecutions.filter((e) => CODECEPT_TOOLS.includes(e.toolName));
91
+ const codeceptExecutions = toolExecutions.filter((e) => isCodeceptToolName(e.toolName));
92
92
  if (codeceptExecutions.length === 0 && !pageAnalysis?.axeViolations.length) {
93
93
  debugLog('No interactions or violations to analyze');
94
94
  return null;
@@ -47,6 +47,12 @@ export class Researcher extends ResearcherBase {
47
47
  this.stateManager = explorer.getStateManager();
48
48
  this.experienceTracker = this.stateManager.getExperienceTracker();
49
49
  this.hooksRunner = new HooksRunner(explorer, explorer.getConfig());
50
+ const ai = explorer.getConfig().ai;
51
+ if (ai) {
52
+ ai.agents ??= {};
53
+ ai.agents.researcher ??= {};
54
+ ai.agents.researcher.reasoning ??= 'low';
55
+ }
50
56
  }
51
57
  getNavigator() {
52
58
  throw new Error('not implemented');
@@ -139,7 +145,7 @@ export class Researcher extends ResearcherBase {
139
145
  catch (error) {
140
146
  if (!(error instanceof ContextLengthError) || retriesLeft <= 0) {
141
147
  if (error instanceof ContextLengthError) {
142
- tag('warning').log('Output truncated. Try lowering reasoning effort or increasing maxTokens in ai.config.');
148
+ tag('warning').log('Output truncated. Try lowering reasoning effort or increasing maxOutputTokens in ai.config.');
143
149
  }
144
150
  throw error;
145
151
  }
@@ -147,6 +147,50 @@ export const protectionRule = dedent `
147
147
  Do not propose or perform delete/remove/archive actions on the entity that owns the current URL; propose such actions only on disposable children created within the scenario itself.
148
148
  </important>
149
149
  `;
150
+ export const dataProtectionRules = dedent `
151
+ <data_protection_rules>
152
+ ${protectionRule}
153
+
154
+ If the user request, scenario, focus, or test instructions explicitly prohibit creating,
155
+ editing, updating, deleting, removing, or otherwise mutating data, do not perform those
156
+ actions through the UI, API preconditions, cleanup, fallback steps, or Fisherman.
157
+
158
+ Do not use Fisherman or API data preparation to bypass a no-mutation, read-only, search,
159
+ filter, tab, or list-inspection constraint. Use visible existing data when it is available.
160
+ If no suitable data exists, report the missing precondition instead of creating data.
161
+
162
+ Destructive actions are allowed only against disposable data created by the current scenario
163
+ or prepared for that scenario by Fisherman/API preconditions. Existing application data must
164
+ remain unchanged.
165
+ </data_protection_rules>
166
+ `;
167
+ export const capabilityGroundingRule = dedent `
168
+ <capability_grounding>
169
+ When a scenario depends on a named action, menu item, status, option, workflow, or feature,
170
+ that capability must be visible or explicitly confirmed in the current research/page context
171
+ for the same target entity type.
172
+
173
+ Do not transfer capabilities between similar entities, rows, lists, detail pages, or menus.
174
+ Do not replace a requested action with a synonym or related action unless the UI explicitly
175
+ shows that action for the target entity.
176
+
177
+ When an action is described as applying to an item, row, card, record, node, or entity,
178
+ the target must be grounded as that kind of data entity in the current context. Do not use
179
+ navigation links, filter tabs, counters, breadcrumbs, headings, toolbar controls, or other
180
+ page controls as the subject of row/entity actions.
181
+
182
+ When a scenario asks to open, view, inspect, or navigate to an entity detail view, success
183
+ requires evidence of that entity detail context. An active filter, selected tab, visible count,
184
+ or filtered list is not enough to prove an entity detail view opened.
185
+
186
+ Do not rewrite a scenario goal to match a similar outcome that happened accidentally. If the
187
+ requested entity detail/action/workflow was not achieved, report that mismatch instead of
188
+ passing the test for a related filter, tab, navigation, or status view.
189
+
190
+ If the required capability is not available for the target entity after reasonable discovery,
191
+ record the missing capability and stop instead of repeatedly trying unrelated locators.
192
+ </capability_grounding>
193
+ `;
150
194
  export const focusedElementRule = dedent `
151
195
  <focused_element_actions>
152
196
  When a text input element is focused (textbox, combobox, contenteditable):
@@ -14,7 +14,7 @@ import { ErrorPageError, isErrorPage } from "../utils/error-page.js";
14
14
  import { HooksRunner } from "../utils/hooks-runner.js";
15
15
  import { createDebug, tag } from "../utils/logger.js";
16
16
  import { loop } from "../utils/loop.js";
17
- import { actionRule, focusedElementRule, formRequirementsRule, locatorRule, multipleTabsRule, protectionRule, sectionContextRule } from "./rules.js";
17
+ import { actionRule, capabilityGroundingRule, dataProtectionRules, focusedElementRule, formRequirementsRule, locatorRule, multipleTabsRule, sectionContextRule } from "./rules.js";
18
18
  import { TaskAgent } from "./task-agent.js";
19
19
  import { createCodeceptJSTools, createSpecialContextTools } from "./tools.js";
20
20
  const debugLog = createDebug('explorbot:tester');
@@ -29,7 +29,7 @@ const SAMPLE_FILES = {
29
29
  'MP3 audio': 'sample.mp3',
30
30
  };
31
31
  export class Tester extends TaskAgent {
32
- ACTION_TOOLS = ['click', 'pressKey', 'form'];
32
+ ACTION_TOOLS = ['click', 'hover', 'pressKey', 'form'];
33
33
  SPECIAL_CONTEXT_ACTION_TOOLS = ['exitIframe'];
34
34
  emoji = '🧪';
35
35
  explorer;
@@ -51,6 +51,8 @@ export class Tester extends TaskAgent {
51
51
  hooksRunner;
52
52
  seenUiMapUrls = new Set();
53
53
  lastAnalyzedStateHash = null;
54
+ stalledIterations = 0;
55
+ MAX_STALLED_ITERATIONS = 3;
54
56
  constructor(explorer, provider, researcher, navigator, agentTools) {
55
57
  super();
56
58
  this.explorer = explorer;
@@ -99,6 +101,7 @@ export class Tester extends TaskAgent {
99
101
  this.pageActionResult = null;
100
102
  this.seenUiMapUrls.clear();
101
103
  this.lastAnalyzedStateHash = null;
104
+ this.stalledIterations = 0;
102
105
  this.explorer.getStateManager().clearHistory();
103
106
  this.resetFailureCount();
104
107
  this.pilot?.reset();
@@ -293,6 +296,10 @@ export class Tester extends TaskAgent {
293
296
  task.addNote(`Failed to ${execution.input.explanation} (${execution.toolName})`, TestResult.FAILED);
294
297
  });
295
298
  }
299
+ if (this.shouldStopForStalledExecution(task, currentState, result?.toolExecutions || [])) {
300
+ stop();
301
+ return;
302
+ }
296
303
  if (assertionPerformed) {
297
304
  const message = result?.toolExecutions?.find((execution) => execution.toolName === 'verify')?.output?.message || '';
298
305
  task.addNote(message, wasSuccessful ? TestResult.PASSED : TestResult.FAILED);
@@ -395,6 +402,28 @@ export class Tester extends TaskAgent {
395
402
  return false;
396
403
  return true;
397
404
  }
405
+ shouldStopForStalledExecution(task, previousState, toolExecutions) {
406
+ if (task.hasFinished)
407
+ return false;
408
+ const currentState = this.getCurrentState();
409
+ const stateChanged = previousState.url !== currentState.url || previousState.hash !== currentState.hash;
410
+ const actionTools = [...this.ACTION_TOOLS, ...this.SPECIAL_CONTEXT_ACTION_TOOLS];
411
+ const hasSuccessfulAction = toolExecutions.some((execution) => execution.wasSuccessful && actionTools.includes(execution.toolName));
412
+ const hasSuccessfulAssertion = toolExecutions.some((execution) => execution.wasSuccessful && this.ASSERTION_TOOLS.includes(execution.toolName));
413
+ if (stateChanged || hasSuccessfulAction || hasSuccessfulAssertion) {
414
+ this.stalledIterations = 0;
415
+ return false;
416
+ }
417
+ const hasNoBrowserProgress = toolExecutions.length === 0 || toolExecutions.every((execution) => !actionTools.includes(execution.toolName) || !execution.wasSuccessful);
418
+ if (!hasNoBrowserProgress)
419
+ return false;
420
+ this.stalledIterations++;
421
+ if (this.stalledIterations < this.MAX_STALLED_ITERATIONS)
422
+ return false;
423
+ task.addNote('No browser progress after repeated attempts on unchanged page', TestResult.FAILED);
424
+ task.finish(TestResult.FAILED);
425
+ return true;
426
+ }
398
427
  async prepareInstructionsForNextStep(task) {
399
428
  let outcomeStatus = dedent `
400
429
  <task>
@@ -403,6 +432,8 @@ export class Tester extends TaskAgent {
403
432
 
404
433
  <rules>
405
434
  Use tools ${this.ACTION_TOOLS.join(', ')} to interact with the page.
435
+ Use tool names exactly as listed in this prompt. Do not invent combined tool names, aliases, or names with channel markers such as "commentary".
436
+ Match each tool input schema exactly. Do not invent parameter names or pass extra fields.
406
437
  Do not do unsuccesful clicks again.
407
438
  Do not run same tool calls with same parameters again.
408
439
  </rules>
@@ -665,6 +696,7 @@ export class Tester extends TaskAgent {
665
696
  <rules>
666
697
  - Refer to UI Map from <page_ui_map> to understand the page structure and its main elements
667
698
  - Use only elements that exist in the provided ARIA tree or HTML, <page_aria> and <page_html>
699
+ - Use tool input schemas exactly as documented. Do not invent parameter names or add fields not listed by the tool schema.
668
700
  - Use click() for buttons, links, and clickable elements ONLY - do NOT include I.fillField() or I.type() commands in click() tool
669
701
  - click() commands array is for FALLBACK LOCATORS of the SAME element, NOT for clicking different elements in sequence. If you need to click two different elements, make two separate click() calls.
670
702
  - Use form() for text input (I.fillField, I.type), dropdown selection (I.selectOption), file uploads (I.attachFile), and multi-step form interactions
@@ -677,6 +709,7 @@ export class Tester extends TaskAgent {
677
709
  - NEVER call record(status: "success") if your last verify() or see() call FAILED. A failed check means the outcome is NOT confirmed — use record(status: "fail") instead, or retry with a different approach.
678
710
  - Use finish() to complete the test, not record(). record() is for intermediate notes.
679
711
  - Call finish(verify) when all goals are achieved — provide an assertion to verify
712
+ - NEVER call finish() with a negative assertion that says the goal did NOT happen. If the goal cannot be achieved after real attempts, record the blocker and call stop().
680
713
  - ONLY call stop() if the scenario itself is completely irrelevant to this page and no expectations can be achieved
681
714
  - Use reset() ONLY as a last resort when the current page cannot host the scenario. Never reset after a successful flow just because an assertion or milestone did not match — verify differently or record() the finding instead. Reset is destructive and does not undo server-side side effects.
682
715
  - Be precise with locators (CSS or XPath)
@@ -690,6 +723,13 @@ export class Tester extends TaskAgent {
690
723
  - When you interact with form with inputs, ensure that you click corresponding button to save its data
691
724
  - Follow <locator_priority> rules when selecting locators for all tools
692
725
  - Before retrying your actions check maybe they already achived expected results. Use see() tool for that
726
+ - If the current URL is already a create/edit/new form and the scenario is about creating/editing that entity, fill and submit that form. Do not click the list-page "New" button again from inside the form.
727
+ - If the scenario is about search/filter/sort/tabs/list inspection and the current URL is a create/edit/new form, go back or reset to the stable list page before interacting with list controls.
728
+ - When selecting related entities from a list, do not choose rows/options/cards marked as "0 items", "0 tests", or otherwise empty if the scenario requires selecting real content.
729
+ - In selection pickers, counters such as "Selected 0", "Matched tests 0", or disabled Save/Apply mean the selection did not register. Choose a non-empty item or change filters before submitting.
730
+ - A passed form/click command only means the command executed. If a required field remains empty, submit stays disabled, or the expected text is not visible, treat the action as not completed and correct the missing field/state.
731
+ - For filter/tab scenarios, success requires BOTH: the requested filter/tab is visibly active/selected AND the list content matches that filter. Do not finish from only one of these signals.
732
+ - Empty-state text such as "No matched items" only proves a filter when the requested filter/tab is active and the empty state belongs to the filtered list.
693
733
  - When filling complex form with lot of actions performed, use see() to look which fields were filled and which are not
694
734
  - When verify() fails, use see() to visually confirm the result — visual confirmation is equally valid evidence
695
735
  - For visual state verification (active tabs, selected items, counts, colors), prefer see() over DOM-based verify()
@@ -718,11 +758,16 @@ export class Tester extends TaskAgent {
718
758
 
719
759
  ${formRequirementsRule}
720
760
 
761
+ ${capabilityGroundingRule}
762
+
763
+ ${dataProtectionRules}
764
+
721
765
  ${this.provider.getSystemPromptForAgent('tester', this.explorer.getStateManager().getCurrentState()?.url) || ''}
722
766
  `;
723
767
  }
724
768
  buildScenarioBlock(task, actionResult) {
725
769
  const knowledge = this.getKnowledge(actionResult);
770
+ const experience = this.getExperience(actionResult);
726
771
  return dedent `
727
772
  <task>
728
773
  SCENARIO GOAL: ${task.scenario}
@@ -739,16 +784,21 @@ export class Tester extends TaskAgent {
739
784
  Try to achieve as many goals as possible.
740
785
  If goal is not achievable, log that and skip to next one.
741
786
  Do not hallucinate that goal was achieved when it was not.
787
+ If the scenario action could not be completed, do not finish with a verification of the failure state.
742
788
  When creating or editing items via form() or type() you should include ${task.sessionName} in the value (if it is not restricted by the application logic)
743
789
  Initial page URL: ${actionResult.url}
744
790
 
745
- ${protectionRule}
791
+ ${capabilityGroundingRule}
792
+
793
+ ${dataProtectionRules}
746
794
 
747
795
  ${this.buildDeletionScope(task)}
748
796
 
749
797
  ${this.buildAvailableFiles()}
750
798
 
751
799
  ${knowledge}
800
+
801
+ ${experience}
752
802
  `;
753
803
  }
754
804
  getDeletableSessionNames(task) {
@@ -867,12 +917,13 @@ export class Tester extends TaskAgent {
867
917
  }),
868
918
  stop: tool({
869
919
  description: dedent `
870
- Stop the current test because the scenario is fundamentally incompatible with the page.
871
- Use this ONLY when the scenario cannot be executed on the current page or application.
872
- Do NOT use this for failures — use reset() and retry instead.
920
+ Stop the current test because it cannot be completed in the current session.
921
+ Use this when the scenario is incompatible, required UI/data is absent, or repeated varied attempts
922
+ show that automation cannot complete the workflow.
923
+ Do NOT use this immediately after the first failed action — retry with a materially different approach first.
873
924
  `,
874
925
  inputSchema: z.object({
875
- reason: z.string().describe('Explanation why the scenario is incompatible'),
926
+ reason: z.string().describe('Explanation why the scenario cannot be completed'),
876
927
  }),
877
928
  execute: async ({ reason }) => {
878
929
  task.addNote(`Stop requested: ${reason}`);
@@ -906,6 +957,7 @@ export class Tester extends TaskAgent {
906
957
  Provide a specific assertion to verify the final state.
907
958
  The assertion MUST prove that YOUR ACTIONS changed the page state.
908
959
  Do NOT verify something that was already true before you started testing.
960
+ Do NOT provide an assertion that verifies absence, failure, an empty state, or that the goal did not happen.
909
961
 
910
962
  Examples of good assertions:
911
963
  - "New user 'john@example.com' is visible in the users list"
@@ -1031,10 +1083,24 @@ export class Tester extends TaskAgent {
1031
1083
  this.resetFailureCount();
1032
1084
  this.previousUrl = null;
1033
1085
  this.previousStateHash = null;
1086
+ this.stalledIterations = 0;
1087
+ }
1088
+ else if (this.shouldStopAfterStalledLoopError(task)) {
1089
+ return 'stop';
1034
1090
  }
1035
1091
  this.currentConversation?.addUserText(result.message);
1036
1092
  return 'continue';
1037
1093
  }
1094
+ shouldStopAfterStalledLoopError(task) {
1095
+ if (task.hasFinished)
1096
+ return false;
1097
+ this.stalledIterations++;
1098
+ if (this.stalledIterations < this.MAX_STALLED_ITERATIONS)
1099
+ return false;
1100
+ task.addNote('No browser progress after repeated execution errors', TestResult.FAILED);
1101
+ task.finish(TestResult.FAILED);
1102
+ return true;
1103
+ }
1038
1104
  async cleanupStartedTest(task) {
1039
1105
  await this.finishTest(task);
1040
1106
  await this.explorer.stopTest(task, {
@@ -11,7 +11,6 @@ import { WebElement } from "../utils/web-element.js";
11
11
  import { sectionContextRule } from "./rules.js";
12
12
  import { isInteractive } from "./task-agent.js";
13
13
  const debugLog = createDebug('explorbot:tools');
14
- export const CODECEPT_TOOLS = ['click', 'pressKey', 'form'];
15
14
  export const ASSERTION_TOOLS = ['verify'];
16
15
  export function createCodeceptJSTools(explorer, task) {
17
16
  const stateManager = explorer.getStateManager();
@@ -133,6 +132,71 @@ export function createCodeceptJSTools(explorer, task) {
133
132
  }, action.lastError);
134
133
  },
135
134
  }),
135
+ hover: tool({
136
+ description: dedent `
137
+ Move the mouse cursor to an element to reveal hover-only controls.
138
+
139
+ Use this before clicking row actions, icon buttons, menus, or toolbars that appear only
140
+ when the user hovers a list item, table row, card, or tree node.
141
+
142
+ This tool ONLY accepts I.moveCursorTo(locator) commands. It does not click.
143
+ After hovering, use context(), see(), or click() the revealed control.
144
+ `,
145
+ inputSchema: z.object({
146
+ commands: z.array(z.string()).describe(dedent `
147
+ FALLBACK LOCATORS for ONE element to hover.
148
+ Order by reliability:
149
+ 1. I.moveCursorTo(text, container)
150
+ 2. I.moveCursorTo(ARIA, container)
151
+ 3. I.moveCursorTo(CSS, container)
152
+ 4. I.moveCursorTo(CSS) or I.moveCursorTo(XPath)
153
+ `),
154
+ explanation: z.string().describe('Why you are hovering this element'),
155
+ }),
156
+ execute: async ({ commands: rawCommands, explanation }) => {
157
+ const activeNote = task.startNote(explanation);
158
+ if (rawCommands.length === 0) {
159
+ activeNote.commit(TestResult.FAILED);
160
+ return failedToolResult('hover', 'No commands provided');
161
+ }
162
+ const invalidCommands = rawCommands.map((cmd) => cmd.trim()).filter((cmd) => cmd.startsWith('I.') && !cmd.startsWith('I.moveCursorTo'));
163
+ if (invalidCommands.length > 0) {
164
+ activeNote.commit(TestResult.FAILED);
165
+ return failedToolResult('hover', `Invalid commands: ${invalidCommands.join(', ')}. Hover tool only accepts I.moveCursorTo() commands.`, {
166
+ suggestion: 'Use click() to click elements, or form() for typing/selecting.',
167
+ });
168
+ }
169
+ const commands = rawCommands.map((cmd) => {
170
+ const trimmed = cmd.trim();
171
+ if (trimmed.startsWith('I.moveCursorTo'))
172
+ return trimmed;
173
+ return `I.moveCursorTo(${JSON.stringify(trimmed)})`;
174
+ });
175
+ const previousState = ActionResult.fromState(stateManager.getCurrentState());
176
+ const action = explorer.createAction();
177
+ const attempts = [];
178
+ for (const command of commands) {
179
+ const success = await action.attempt(command, explanation, true);
180
+ attempts.push({
181
+ command,
182
+ success,
183
+ ...(action.lastError && { error: action.lastError.toString() }),
184
+ });
185
+ if (!success)
186
+ continue;
187
+ const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, command);
188
+ activeNote.commit(TestResult.PASSED);
189
+ return successToolResult('hover', { ...toolResult, attempts, code: command }, action);
190
+ }
191
+ const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, commands[0]);
192
+ activeNote.commit(TestResult.FAILED);
193
+ return failedToolResult('hover', 'All hover commands failed', {
194
+ ...toolResult,
195
+ attempts,
196
+ suggestion: 'Use xpathCheck() to locate the row/card/tree node, or visualClick() if the hover target is only visually identifiable.',
197
+ }, action.lastError);
198
+ },
199
+ }),
136
200
  pressKey: tool({
137
201
  description: dedent `
138
202
  Press a keyboard key or key combination. Use this for special keys like Enter, Escape, Tab, Arrow keys, or key combinations with modifiers.
@@ -404,6 +468,7 @@ export function createAgentTools({ explorer, researcher, navigator, experienceTr
404
468
  Check the page contents based on current page state and screenshot.
405
469
  This tool will trigger visual research to check the page contents on request.
406
470
  Use it to verify the actions were performed correctly and the page is in the expected state.
471
+ Input schema has exactly one field: request. Do not pass text, reason, assertion, or other fields.
407
472
 
408
473
  <example>
409
474
  request: "Check current state of the Login form"