explorbot 0.1.28 → 0.1.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -10,6 +10,7 @@ import { ErrorPageError } from "../utils/error-page.js";
10
10
  import { createDebug, tag } from "../utils/logger.js";
11
11
  const debugLog = createDebug('explorbot:pilot');
12
12
  import { truncateJson } from "../utils/strings.js";
13
+ import { capabilityGroundingRule, dataProtectionRules } from "./rules.js";
13
14
  import { isInteractive } from "./task-agent.js";
14
15
  const CHECK_TOOLS = ['verify', 'see', 'research', 'context'];
15
16
  const META_TOOLS = ['record', 'reset', 'stop', 'finish'];
@@ -71,7 +72,7 @@ export class Pilot {
71
72
  const notes = task.notesToString() || 'No notes recorded.';
72
73
  let visualAnalysis = '';
73
74
  let screenshotState = null;
74
- if (this.provider.hasVision()) {
75
+ if (type === 'finish' && this.provider.hasVision()) {
75
76
  try {
76
77
  screenshotState = await this.explorer.capturePageWithScreenshot();
77
78
  if (screenshotState.screenshot) {
@@ -139,7 +140,7 @@ export class Pilot {
139
140
  try {
140
141
  const response = await this.provider.generateObject(messages, schema, this.provider.getAgenticModel('pilot'), {
141
142
  agentName: 'pilot',
142
- experimental_telemetry: { functionId: 'pilot.reviewVerdict' },
143
+ telemetry: { functionId: 'pilot.reviewVerdict' },
143
144
  });
144
145
  const result = response?.object;
145
146
  if (!result) {
@@ -229,7 +230,7 @@ export class Pilot {
229
230
  try {
230
231
  const response = await this.provider.generateObject(messages, schema, this.provider.getAgenticModel('pilot'), {
231
232
  agentName: 'pilot',
232
- experimental_telemetry: { functionId: 'pilot.reviewReset' },
233
+ telemetry: { functionId: 'pilot.reviewReset' },
233
234
  });
234
235
  const result = response?.object;
235
236
  if (!result) {
@@ -333,6 +334,8 @@ export class Pilot {
333
334
  You are Pilot — final decision maker for test pass/fail. Tester requested ${type}. Review the
334
335
  evidence and commit to a verdict; "continue" only when evidence is genuinely insufficient.
335
336
 
337
+ ${capabilityGroundingRule}
338
+
336
339
  ${this.buildSharedEvidenceRules(task)}
337
340
 
338
341
  DECISION:
@@ -341,6 +344,8 @@ export class Pilot {
341
344
  Pick assertions DOM can express; for non-DOM regions (iframes, canvas, Monaco/CodeMirror), target a
342
345
  stable landmark (container, ARIA role) instead of literal inner text. Your "pass" stands even if the
343
346
  DOM assertion can't be made.
347
+ Do not pass when Tester achieved only a related navigation/filter/tab/status outcome instead of the
348
+ requested action, workflow, or entity detail goal.
344
349
  - "fail": scenario was attempted but the goal was not achieved.
345
350
  - "skipped": scenario is irrelevant to the app, OR systematic infrastructure failures (LLM errors,
346
351
  crashes) prevented testing. NOT for "test failed to interact" — that's "fail" or "continue".
@@ -374,6 +379,10 @@ export class Pilot {
374
379
 
375
380
  FIRST: Decide if precondition() is needed.
376
381
 
382
+ ${capabilityGroundingRule}
383
+
384
+ ${dataProtectionRules}
385
+
377
386
  Call precondition() WHEN:
378
387
  - The scenario edits/deletes/modifies an item, and you want a DISPOSABLE item to act on safely
379
388
  - The scenario needs specific data clearly NOT on the current page (e.g., items with specific statuses for filtering)
@@ -498,7 +507,7 @@ export class Pilot {
498
507
  maxToolRoundtrips: opts.maxToolRoundtrips ?? 0,
499
508
  agentName: 'pilot',
500
509
  stopWhen: opts.task ? () => opts.task.hasFinished : undefined,
501
- experimental_telemetry: { functionId },
510
+ telemetry: { functionId },
502
511
  });
503
512
  const text = result?.response?.text || '';
504
513
  const learned = (result?.toolExecutions || []).filter((e) => e.toolName === 'learnExperience' && e.output?.content).map((e) => e.output.content);
@@ -930,9 +939,13 @@ export class Pilot {
930
939
 
931
940
  Tester tools: click, pressKey, form, see, verify, context, research, xpathCheck, visualClick,
932
941
  back, getVisitedStates, reset, stop, finish, record.
942
+ Use tool names exactly as listed. Do not invent combined names, aliases, or names with channel markers such as "commentary".
943
+
944
+ ${capabilityGroundingRule}
933
945
 
934
946
  YOUR Pilot-only tool: precondition(description) — create FRESH disposable test data via API. Never
935
947
  request users. Use when:
948
+
936
949
  - Scenario edits/deletes/modifies an item → create a disposable target ("1 post").
937
950
  - Scenario needs auxiliary data (labels, categories, statuses for filtering).
938
951
  - Tester failed because required data is missing (empty dropdown, empty list).
@@ -942,6 +955,8 @@ export class Pilot {
942
955
  - Current page already shows the exact data needed.
943
956
  - Scenario tests navigation, search UI, or viewing.
944
957
 
958
+ ${dataProtectionRules}
959
+
945
960
  Describe WHAT to create, not what exists. RIGHT: precondition("1 test"). WRONG:
946
961
  precondition("1 test suite named Updated Suite with existing tests"). Keep descriptions short.
947
962
 
@@ -19,7 +19,7 @@ import { WithSubPages, getPlannedByStateHash, getRegisteredPlan, registerPlan }
19
19
  import { POSSIBLE_SECTIONS, Researcher } from "./researcher.js";
20
20
  import { findSimilarStateHash } from "./researcher/cache.js";
21
21
  import { hasFocusedSection } from "./researcher/focus.js";
22
- import { fileUploadRule, protectionRule } from "./rules.js";
22
+ import { capabilityGroundingRule, dataProtectionRules, fileUploadRule } from "./rules.js";
23
23
  const debugLog = createDebug('explorbot:planner');
24
24
  const TasksSchema = z.object({
25
25
  planName: z.string().describe('Short descriptive name for the test plan (e.g., "User Authentication Testing", "Product Catalog Navigation", "Form Validation Tests")'),
@@ -27,7 +27,7 @@ const TasksSchema = z.object({
27
27
  .array(z.object({
28
28
  scenario: z.string().describe('A single sentence describing what to test'),
29
29
  priority: z.enum(['critical', 'important', 'high', 'normal', 'low']).describe('Priority of the task based on business importance'),
30
- startUrl: z.string().nullable().describe('Start URL for the test if different from plan URL (only for tests on visited subpages)'),
30
+ startUrl: z.string().nullable().describe('Start URL for the test if different from plan URL. Use only stable feature/list/detail pages, not transient create/edit/modal URLs unless the scenario specifically starts inside that form.'),
31
31
  steps: z.array(z.string()).describe('List of steps to perform for this scenario. Each step should be a specific action (e.g., "Open the form", "Enter required data", "Submit the form"). Keep steps atomic and actionable.'),
32
32
  expectedOutcomes: z
33
33
  .array(z.string())
@@ -73,6 +73,9 @@ export class Planner extends PlannerBase {
73
73
  const featureDirective = feature
74
74
  ? `\n IMPORTANT: The user requested to focus specifically on: "${feature}"\n ALL scenarios MUST be directly related to this feature. Do not propose generic page tests unrelated to it.\n Use the user's exact wording to guide scenario names — do not substitute different entities (e.g., do not plan "suite" actions when user said "test").`
75
75
  : '';
76
+ const focusExistingDataDirective = feature
77
+ ? '\n If this focus asks for search, filter, tabs, sorting, or list behavior involving existing items, only use item names/values visible in the provided page research. If no concrete visible item names/values are present, do NOT propose scenarios that require an existing known item; propose no-match search, empty-state, clear-search, tab/filter empty-list, or other read-only list behavior instead.'
78
+ : '';
76
79
  return dedent `
77
80
  <role>
78
81
  You are ISTQB certified senior manual QA planning exploratory testing session of a web application.
@@ -96,7 +99,7 @@ export class Planner extends PlannerBase {
96
99
  Bad: "Open delete dropdown" + "Confirm deletion" — these are ONE test, not two.
97
100
  Bad: "Search for X" + "Verify search results" — searching and verifying is ONE test.
98
101
  Bad: "Leave field empty" + "Click submit" — that's one negative test, not two.
99
- If two scenarios cannot run independently (one requires the other to run first), merge them into one.${featureDirective}
102
+ If two scenarios cannot run independently (one requires the other to run first), merge them into one.${featureDirective}${focusExistingDataDirective}
100
103
  </task>
101
104
 
102
105
  ${customPrompt || ''}
@@ -301,6 +304,11 @@ export class Planner extends PlannerBase {
301
304
  If a scenario needs existing records, recipients, results, notifications, or other target data, propose it only when that data is visible or API preconditions can create it.
302
305
  If the page appears read-only, degraded, demo-limited, maintenance-like, or lacks write controls, prefer read-only scenarios such as opening panels, inspecting visible lists, filtering, searching, or verifying current state.
303
306
  Do not assume hidden data exists just because a control is present.
307
+ For scenarios that act on existing items or search/filter by existing values, use only item names or values visible in research, visited pages, or prior observed flows.
308
+ If the list is empty or no concrete item names are visible, do not invent "known" or "existing" items. Prefer empty-state, no-match search, clear-search, or read-only list behavior scenarios.
309
+ Search, filter, sorting, tab, and list scenarios must start from a stable page where those controls are visible; avoid transient create/edit/new URLs unless the scenario tests that form.
310
+ For option values and list items, use only visible or previously observed data; do not add create/update/delete setup unless the user explicitly requests that workflow.
311
+ Detail-view scenarios must target visible data entities from list rows, cards, tree nodes, or detail links; do not use filter tabs, counters, status tabs, breadcrumbs, or navigation controls as detail targets.
304
312
  DO NOT propose "verification-only" tests that merely open a UI element (modal, dropdown, panel) and check it exists.
305
313
  Every test must complete a meaningful action that changes application state or produces a business outcome.
306
314
  Opening a modal is NOT a test — performing an action INSIDE the modal IS a test.
@@ -311,7 +319,8 @@ export class Planner extends PlannerBase {
311
319
  Tests that only switch views, toggle filters, or paginate are LESS valuable — propose them only after data-changing tests are covered.
312
320
  If multiple ways to create or modify data exist (different types, different forms), propose a separate test for each.
313
321
  </priority_order>
314
- ${protectionRule}
322
+ ${capabilityGroundingRule}
323
+ ${dataProtectionRules}
315
324
  ${fileUploadRule}
316
325
  </rules>
317
326
 
@@ -466,7 +475,9 @@ export class Planner extends PlannerBase {
466
475
  .join('\n')}
467
476
 
468
477
  You MAY propose tests starting from these pages if they are relevant to the plan "${this.currentPlan.title}".
469
- Set startUrl for such tests. Ignore pages that belong to a different feature area.
478
+ Set startUrl for such tests only when the page is a stable feature/list/detail page.
479
+ Do not use create/edit/new/modal URLs as startUrl for scenarios that need the underlying page.
480
+ Ignore pages that belong to a different feature area.
470
481
  </context_from_previous_tests>
471
482
 
472
483
  Propose ONLY new scenarios that are NOT in the existing tests list.
@@ -1,6 +1,7 @@
1
+ import { OpenTelemetry } from '@ai-sdk/otel';
1
2
  import { LangfuseSpanProcessor } from '@langfuse/otel';
2
3
  import { NodeSDK } from '@opentelemetry/sdk-node';
3
- import { generateObject, generateText, stepCountIs } from 'ai';
4
+ import { generateObject, generateText, isStepCount, registerTelemetry } from 'ai';
4
5
  import { clearActivity, setActivity } from "../activity.js";
5
6
  import { executionController } from "../execution-controller.js";
6
7
  import { Observability } from "../observability.js";
@@ -16,10 +17,11 @@ class AiError extends Error {
16
17
  }
17
18
  export class ContextLengthError extends Error {
18
19
  }
20
+ let telemetryRegistered = false;
19
21
  function extractCachedTokens(usage) {
20
22
  if (!usage)
21
23
  return 0;
22
- const direct = usage.cachedInputTokens ?? usage.inputTokenDetails?.cacheReadTokens;
24
+ const direct = usage.inputTokenDetails?.cacheReadTokens ?? usage.cachedInputTokens;
23
25
  if (typeof direct === 'number')
24
26
  return direct;
25
27
  const raw = usage.raw;
@@ -78,7 +80,7 @@ export class Provider {
78
80
  await generateText({
79
81
  model: this.config.model,
80
82
  prompt: 'hi',
81
- maxTokens: 1,
83
+ maxOutputTokens: 1,
82
84
  });
83
85
  }
84
86
  catch (error) {
@@ -100,6 +102,18 @@ export class Provider {
100
102
  }
101
103
  return this.config.agenticModel || this.config.model;
102
104
  }
105
+ getConfiguredModels() {
106
+ const models = { model: this.getModelName(this.config.model) };
107
+ if (this.config.agenticModel)
108
+ models.agenticModel = this.getModelName(this.config.agenticModel);
109
+ if (this.config.visionModel)
110
+ models.visionModel = this.getModelName(this.config.visionModel);
111
+ for (const [agent, agentConfig] of Object.entries(this.config.agents || {})) {
112
+ if (agentConfig?.model)
113
+ models[agent] = this.getModelName(agentConfig.model);
114
+ }
115
+ return models;
116
+ }
103
117
  getSystemPromptForAgent(agentName, currentUrl) {
104
118
  const agentConfig = this.config.agents?.[agentName];
105
119
  const parts = [];
@@ -116,6 +130,12 @@ export class Provider {
116
130
  const agentConfig = this.config.agents?.[agentName];
117
131
  return agentConfig?.providerOptions;
118
132
  }
133
+ getReasoningForAgent(agentName) {
134
+ if (!agentName)
135
+ return undefined;
136
+ const agentConfig = this.config.agents?.[agentName];
137
+ return agentConfig?.reasoning;
138
+ }
119
139
  getRetryOptions(options = {}) {
120
140
  return {
121
141
  ...this.defaultRetryOptions,
@@ -133,6 +153,13 @@ export class Provider {
133
153
  providerOptions: { ...config.providerOptions, ...agentOptions },
134
154
  };
135
155
  }
156
+ finalizeConfig(config, options, telemetry) {
157
+ if (telemetry)
158
+ config.telemetry = telemetry;
159
+ const reasoning = this.getReasoningForAgent(options.agentName);
160
+ if (reasoning)
161
+ config.reasoning ??= reasoning;
162
+ }
136
163
  initLangfuse() {
137
164
  const langfuseConfig = this.config.langfuse;
138
165
  const publicKey = langfuseConfig?.publicKey || process.env.LANGFUSE_PUBLIC_KEY;
@@ -152,6 +179,10 @@ export class Provider {
152
179
  instrumentations: [],
153
180
  });
154
181
  void this.otelSdk.start();
182
+ if (!telemetryRegistered) {
183
+ registerTelemetry(new OpenTelemetry());
184
+ telemetryRegistered = true;
185
+ }
155
186
  this.telemetryEnabled = true;
156
187
  }
157
188
  getTelemetry(options) {
@@ -159,18 +190,18 @@ export class Provider {
159
190
  return undefined;
160
191
  }
161
192
  const runTelemetry = Observability.getTelemetry();
162
- if (!options.experimental_telemetry) {
163
- return runTelemetry || { isEnabled: true };
193
+ if (!options.telemetry) {
194
+ return runTelemetry;
164
195
  }
165
196
  if (!runTelemetry) {
166
- return options.experimental_telemetry;
197
+ return options.telemetry;
167
198
  }
168
199
  return {
169
200
  ...runTelemetry,
170
- ...options.experimental_telemetry,
201
+ ...options.telemetry,
171
202
  metadata: {
172
203
  ...runTelemetry.metadata,
173
- ...options.experimental_telemetry.metadata,
204
+ ...options.telemetry.metadata,
174
205
  },
175
206
  };
176
207
  }
@@ -210,13 +241,14 @@ export class Provider {
210
241
  promptLog(`Using model: ${modelName}`);
211
242
  const telemetry = this.getTelemetry(options);
212
243
  const config = this.mergeProviderOptions({
213
- maxTokens: 16384,
244
+ maxOutputTokens: 16384,
245
+ allowSystemInMessages: true,
214
246
  ...(this.config.config || {}),
215
247
  ...options,
216
- ...(telemetry ? { experimental_telemetry: telemetry } : {}),
217
248
  model,
218
249
  abortSignal: executionController.getAbortSignal(),
219
250
  }, options.agentName);
251
+ this.finalizeConfig(config, options, telemetry);
220
252
  promptLog(messages[messages.length - 1].content);
221
253
  try {
222
254
  const response = await withRetry(async () => {
@@ -224,7 +256,7 @@ export class Provider {
224
256
  if (!result.text) {
225
257
  debugLog(result);
226
258
  if (result.finishReason === 'length') {
227
- throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase maxTokens in config or use a model with higher output capacity.');
259
+ throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase maxOutputTokens in config or use a model with higher output capacity.');
228
260
  }
229
261
  throw new Error('No response text from AI');
230
262
  }
@@ -277,21 +309,22 @@ export class Provider {
277
309
  const telemetry = this.getTelemetry(options);
278
310
  const maxRoundtrips = options.maxToolRoundtrips ?? 5;
279
311
  const extraStop = options.stopWhen;
280
- const stopConditions = [stepCountIs(maxRoundtrips)];
312
+ const stopConditions = [isStepCount(maxRoundtrips)];
281
313
  if (extraStop)
282
314
  stopConditions.push(extraStop);
283
315
  const { stopWhen: _ignoredStopWhen, ...optionsWithoutStop } = options;
284
316
  const config = this.mergeProviderOptions({
285
317
  tools,
286
- maxTokens: 16384,
318
+ maxOutputTokens: 16384,
287
319
  toolChoice: 'auto',
320
+ allowSystemInMessages: true,
288
321
  ...(this.config.config || {}),
289
322
  ...optionsWithoutStop,
290
323
  stopWhen: stopConditions,
291
- ...(telemetry ? { experimental_telemetry: telemetry } : {}),
292
324
  model,
293
325
  abortSignal: executionController.getAbortSignal(),
294
326
  }, options.agentName);
327
+ this.finalizeConfig(config, options, telemetry);
295
328
  try {
296
329
  const response = await withRetry(async () => {
297
330
  const timeout = config.timeout || 30000;
@@ -306,7 +339,7 @@ export class Provider {
306
339
  ]));
307
340
  const hasToolCall = (result.toolCalls?.length || 0) > 0;
308
341
  if (!result.text && !hasToolCall && result.finishReason === 'length') {
309
- throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase maxTokens in config or use a model with higher output capacity.');
342
+ throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase maxOutputTokens in config or use a model with higher output capacity.');
310
343
  }
311
344
  return result;
312
345
  }
@@ -365,12 +398,13 @@ export class Provider {
365
398
  const telemetry = this.getTelemetry(options);
366
399
  const config = this.mergeProviderOptions({
367
400
  schema,
401
+ allowSystemInMessages: true,
368
402
  ...(this.config.config || {}),
369
403
  ...options,
370
- ...(telemetry ? { experimental_telemetry: telemetry } : {}),
371
404
  model: modelToUse,
372
405
  abortSignal: executionController.getAbortSignal(),
373
406
  }, options.agentName);
407
+ this.finalizeConfig(config, options, telemetry);
374
408
  try {
375
409
  promptLog(messages[messages.length - 1].content);
376
410
  const response = await withRetry(async () => {
@@ -552,12 +586,13 @@ export class Provider {
552
586
  ];
553
587
  const telemetry = this.getTelemetry({});
554
588
  const config = {
555
- maxTokens: 16384,
589
+ maxOutputTokens: 16384,
556
590
  ...(this.config.config || {}),
557
- ...(telemetry ? { experimental_telemetry: telemetry } : {}),
558
591
  model: this.config.visionModel,
559
592
  abortSignal: executionController.getAbortSignal(),
560
593
  };
594
+ if (telemetry)
595
+ config.telemetry = telemetry;
561
596
  try {
562
597
  promptLog(`Processing image with prompt: ${prompt}`);
563
598
  const response = await withRetry(async () => {
@@ -47,6 +47,12 @@ export class Researcher extends ResearcherBase {
47
47
  this.stateManager = explorer.getStateManager();
48
48
  this.experienceTracker = this.stateManager.getExperienceTracker();
49
49
  this.hooksRunner = new HooksRunner(explorer, explorer.getConfig());
50
+ const ai = explorer.getConfig().ai;
51
+ if (ai) {
52
+ ai.agents ??= {};
53
+ ai.agents.researcher ??= {};
54
+ ai.agents.researcher.reasoning ??= 'low';
55
+ }
50
56
  }
51
57
  getNavigator() {
52
58
  throw new Error('not implemented');
@@ -139,7 +145,7 @@ export class Researcher extends ResearcherBase {
139
145
  catch (error) {
140
146
  if (!(error instanceof ContextLengthError) || retriesLeft <= 0) {
141
147
  if (error instanceof ContextLengthError) {
142
- tag('warning').log('Output truncated. Try lowering reasoning effort or increasing maxTokens in ai.config.');
148
+ tag('warning').log('Output truncated. Try lowering reasoning effort or increasing maxOutputTokens in ai.config.');
143
149
  }
144
150
  throw error;
145
151
  }
@@ -147,6 +147,50 @@ export const protectionRule = dedent `
147
147
  Do not propose or perform delete/remove/archive actions on the entity that owns the current URL; propose such actions only on disposable children created within the scenario itself.
148
148
  </important>
149
149
  `;
150
+ export const dataProtectionRules = dedent `
151
+ <data_protection_rules>
152
+ ${protectionRule}
153
+
154
+ If the user request, scenario, focus, or test instructions explicitly prohibit creating,
155
+ editing, updating, deleting, removing, or otherwise mutating data, do not perform those
156
+ actions through the UI, API preconditions, cleanup, fallback steps, or Fisherman.
157
+
158
+ Do not use Fisherman or API data preparation to bypass a no-mutation, read-only, search,
159
+ filter, tab, or list-inspection constraint. Use visible existing data when it is available.
160
+ If no suitable data exists, report the missing precondition instead of creating data.
161
+
162
+ Destructive actions are allowed only against disposable data created by the current scenario
163
+ or prepared for that scenario by Fisherman/API preconditions. Existing application data must
164
+ remain unchanged.
165
+ </data_protection_rules>
166
+ `;
167
+ export const capabilityGroundingRule = dedent `
168
+ <capability_grounding>
169
+ When a scenario depends on a named action, menu item, status, option, workflow, or feature,
170
+ that capability must be visible or explicitly confirmed in the current research/page context
171
+ for the same target entity type.
172
+
173
+ Do not transfer capabilities between similar entities, rows, lists, detail pages, or menus.
174
+ Do not replace a requested action with a synonym or related action unless the UI explicitly
175
+ shows that action for the target entity.
176
+
177
+ When an action is described as applying to an item, row, card, record, node, or entity,
178
+ the target must be grounded as that kind of data entity in the current context. Do not use
179
+ navigation links, filter tabs, counters, breadcrumbs, headings, toolbar controls, or other
180
+ page controls as the subject of row/entity actions.
181
+
182
+ When a scenario asks to open, view, inspect, or navigate to an entity detail view, success
183
+ requires evidence of that entity detail context. An active filter, selected tab, visible count,
184
+ or filtered list is not enough to prove an entity detail view opened.
185
+
186
+ Do not rewrite a scenario goal to match a similar outcome that happened accidentally. If the
187
+ requested entity detail/action/workflow was not achieved, report that mismatch instead of
188
+ passing the test for a related filter, tab, navigation, or status view.
189
+
190
+ If the required capability is not available for the target entity after reasonable discovery,
191
+ record the missing capability and stop instead of repeatedly trying unrelated locators.
192
+ </capability_grounding>
193
+ `;
150
194
  export const focusedElementRule = dedent `
151
195
  <focused_element_actions>
152
196
  When a text input element is focused (textbox, combobox, contenteditable):
@@ -14,7 +14,7 @@ import { ErrorPageError, isErrorPage } from "../utils/error-page.js";
14
14
  import { HooksRunner } from "../utils/hooks-runner.js";
15
15
  import { createDebug, tag } from "../utils/logger.js";
16
16
  import { loop } from "../utils/loop.js";
17
- import { actionRule, focusedElementRule, formRequirementsRule, locatorRule, multipleTabsRule, protectionRule, sectionContextRule } from "./rules.js";
17
+ import { actionRule, capabilityGroundingRule, dataProtectionRules, focusedElementRule, formRequirementsRule, locatorRule, multipleTabsRule, sectionContextRule } from "./rules.js";
18
18
  import { TaskAgent } from "./task-agent.js";
19
19
  import { createCodeceptJSTools, createSpecialContextTools } from "./tools.js";
20
20
  const debugLog = createDebug('explorbot:tester');
@@ -29,7 +29,7 @@ const SAMPLE_FILES = {
29
29
  'MP3 audio': 'sample.mp3',
30
30
  };
31
31
  export class Tester extends TaskAgent {
32
- ACTION_TOOLS = ['click', 'pressKey', 'form'];
32
+ ACTION_TOOLS = ['click', 'hover', 'pressKey', 'form'];
33
33
  SPECIAL_CONTEXT_ACTION_TOOLS = ['exitIframe'];
34
34
  emoji = '🧪';
35
35
  explorer;
@@ -51,6 +51,8 @@ export class Tester extends TaskAgent {
51
51
  hooksRunner;
52
52
  seenUiMapUrls = new Set();
53
53
  lastAnalyzedStateHash = null;
54
+ stalledIterations = 0;
55
+ MAX_STALLED_ITERATIONS = 3;
54
56
  constructor(explorer, provider, researcher, navigator, agentTools) {
55
57
  super();
56
58
  this.explorer = explorer;
@@ -99,6 +101,7 @@ export class Tester extends TaskAgent {
99
101
  this.pageActionResult = null;
100
102
  this.seenUiMapUrls.clear();
101
103
  this.lastAnalyzedStateHash = null;
104
+ this.stalledIterations = 0;
102
105
  this.explorer.getStateManager().clearHistory();
103
106
  this.resetFailureCount();
104
107
  this.pilot?.reset();
@@ -293,6 +296,10 @@ export class Tester extends TaskAgent {
293
296
  task.addNote(`Failed to ${execution.input.explanation} (${execution.toolName})`, TestResult.FAILED);
294
297
  });
295
298
  }
299
+ if (this.shouldStopForStalledExecution(task, currentState, result?.toolExecutions || [])) {
300
+ stop();
301
+ return;
302
+ }
296
303
  if (assertionPerformed) {
297
304
  const message = result?.toolExecutions?.find((execution) => execution.toolName === 'verify')?.output?.message || '';
298
305
  task.addNote(message, wasSuccessful ? TestResult.PASSED : TestResult.FAILED);
@@ -395,6 +402,28 @@ export class Tester extends TaskAgent {
395
402
  return false;
396
403
  return true;
397
404
  }
405
+ shouldStopForStalledExecution(task, previousState, toolExecutions) {
406
+ if (task.hasFinished)
407
+ return false;
408
+ const currentState = this.getCurrentState();
409
+ const stateChanged = previousState.url !== currentState.url || previousState.hash !== currentState.hash;
410
+ const actionTools = [...this.ACTION_TOOLS, ...this.SPECIAL_CONTEXT_ACTION_TOOLS];
411
+ const hasSuccessfulAction = toolExecutions.some((execution) => execution.wasSuccessful && actionTools.includes(execution.toolName));
412
+ const hasSuccessfulAssertion = toolExecutions.some((execution) => execution.wasSuccessful && this.ASSERTION_TOOLS.includes(execution.toolName));
413
+ if (stateChanged || hasSuccessfulAction || hasSuccessfulAssertion) {
414
+ this.stalledIterations = 0;
415
+ return false;
416
+ }
417
+ const hasNoBrowserProgress = toolExecutions.length === 0 || toolExecutions.every((execution) => !actionTools.includes(execution.toolName) || !execution.wasSuccessful);
418
+ if (!hasNoBrowserProgress)
419
+ return false;
420
+ this.stalledIterations++;
421
+ if (this.stalledIterations < this.MAX_STALLED_ITERATIONS)
422
+ return false;
423
+ task.addNote('No browser progress after repeated attempts on unchanged page', TestResult.FAILED);
424
+ task.finish(TestResult.FAILED);
425
+ return true;
426
+ }
398
427
  async prepareInstructionsForNextStep(task) {
399
428
  let outcomeStatus = dedent `
400
429
  <task>
@@ -403,6 +432,8 @@ export class Tester extends TaskAgent {
403
432
 
404
433
  <rules>
405
434
  Use tools ${this.ACTION_TOOLS.join(', ')} to interact with the page.
435
+ Use tool names exactly as listed in this prompt. Do not invent combined tool names, aliases, or names with channel markers such as "commentary".
436
+ Match each tool input schema exactly. Do not invent parameter names or pass extra fields.
406
437
  Do not do unsuccesful clicks again.
407
438
  Do not run same tool calls with same parameters again.
408
439
  </rules>
@@ -665,6 +696,7 @@ export class Tester extends TaskAgent {
665
696
  <rules>
666
697
  - Refer to UI Map from <page_ui_map> to understand the page structure and its main elements
667
698
  - Use only elements that exist in the provided ARIA tree or HTML, <page_aria> and <page_html>
699
+ - Use tool input schemas exactly as documented. Do not invent parameter names or add fields not listed by the tool schema.
668
700
  - Use click() for buttons, links, and clickable elements ONLY - do NOT include I.fillField() or I.type() commands in click() tool
669
701
  - click() commands array is for FALLBACK LOCATORS of the SAME element, NOT for clicking different elements in sequence. If you need to click two different elements, make two separate click() calls.
670
702
  - Use form() for text input (I.fillField, I.type), dropdown selection (I.selectOption), file uploads (I.attachFile), and multi-step form interactions
@@ -677,6 +709,7 @@ export class Tester extends TaskAgent {
677
709
  - NEVER call record(status: "success") if your last verify() or see() call FAILED. A failed check means the outcome is NOT confirmed — use record(status: "fail") instead, or retry with a different approach.
678
710
  - Use finish() to complete the test, not record(). record() is for intermediate notes.
679
711
  - Call finish(verify) when all goals are achieved — provide an assertion to verify
712
+ - NEVER call finish() with a negative assertion that says the goal did NOT happen. If the goal cannot be achieved after real attempts, record the blocker and call stop().
680
713
  - ONLY call stop() if the scenario itself is completely irrelevant to this page and no expectations can be achieved
681
714
  - Use reset() ONLY as a last resort when the current page cannot host the scenario. Never reset after a successful flow just because an assertion or milestone did not match — verify differently or record() the finding instead. Reset is destructive and does not undo server-side side effects.
682
715
  - Be precise with locators (CSS or XPath)
@@ -690,6 +723,13 @@ export class Tester extends TaskAgent {
690
723
  - When you interact with form with inputs, ensure that you click corresponding button to save its data
691
724
  - Follow <locator_priority> rules when selecting locators for all tools
692
725
  - Before retrying your actions check maybe they already achived expected results. Use see() tool for that
726
+ - If the current URL is already a create/edit/new form and the scenario is about creating/editing that entity, fill and submit that form. Do not click the list-page "New" button again from inside the form.
727
+ - If the scenario is about search/filter/sort/tabs/list inspection and the current URL is a create/edit/new form, go back or reset to the stable list page before interacting with list controls.
728
+ - When selecting related entities from a list, do not choose rows/options/cards marked as "0 items", "0 tests", or otherwise empty if the scenario requires selecting real content.
729
+ - In selection pickers, counters such as "Selected 0", "Matched tests 0", or disabled Save/Apply mean the selection did not register. Choose a non-empty item or change filters before submitting.
730
+ - A passed form/click command only means the command executed. If a required field remains empty, submit stays disabled, or the expected text is not visible, treat the action as not completed and correct the missing field/state.
731
+ - For filter/tab scenarios, success requires BOTH: the requested filter/tab is visibly active/selected AND the list content matches that filter. Do not finish from only one of these signals.
732
+ - Empty-state text such as "No matched items" only proves a filter when the requested filter/tab is active and the empty state belongs to the filtered list.
693
733
  - When filling complex form with lot of actions performed, use see() to look which fields were filled and which are not
694
734
  - When verify() fails, use see() to visually confirm the result — visual confirmation is equally valid evidence
695
735
  - For visual state verification (active tabs, selected items, counts, colors), prefer see() over DOM-based verify()
@@ -718,6 +758,10 @@ export class Tester extends TaskAgent {
718
758
 
719
759
  ${formRequirementsRule}
720
760
 
761
+ ${capabilityGroundingRule}
762
+
763
+ ${dataProtectionRules}
764
+
721
765
  ${this.provider.getSystemPromptForAgent('tester', this.explorer.getStateManager().getCurrentState()?.url) || ''}
722
766
  `;
723
767
  }
@@ -739,10 +783,13 @@ export class Tester extends TaskAgent {
739
783
  Try to achieve as many goals as possible.
740
784
  If goal is not achievable, log that and skip to next one.
741
785
  Do not hallucinate that goal was achieved when it was not.
786
+ If the scenario action could not be completed, do not finish with a verification of the failure state.
742
787
  When creating or editing items via form() or type() you should include ${task.sessionName} in the value (if it is not restricted by the application logic)
743
788
  Initial page URL: ${actionResult.url}
744
789
 
745
- ${protectionRule}
790
+ ${capabilityGroundingRule}
791
+
792
+ ${dataProtectionRules}
746
793
 
747
794
  ${this.buildDeletionScope(task)}
748
795
 
@@ -867,12 +914,13 @@ export class Tester extends TaskAgent {
867
914
  }),
868
915
  stop: tool({
869
916
  description: dedent `
870
- Stop the current test because the scenario is fundamentally incompatible with the page.
871
- Use this ONLY when the scenario cannot be executed on the current page or application.
872
- Do NOT use this for failures — use reset() and retry instead.
917
+ Stop the current test because it cannot be completed in the current session.
918
+ Use this when the scenario is incompatible, required UI/data is absent, or repeated varied attempts
919
+ show that automation cannot complete the workflow.
920
+ Do NOT use this immediately after the first failed action — retry with a materially different approach first.
873
921
  `,
874
922
  inputSchema: z.object({
875
- reason: z.string().describe('Explanation why the scenario is incompatible'),
923
+ reason: z.string().describe('Explanation why the scenario cannot be completed'),
876
924
  }),
877
925
  execute: async ({ reason }) => {
878
926
  task.addNote(`Stop requested: ${reason}`);
@@ -906,6 +954,7 @@ export class Tester extends TaskAgent {
906
954
  Provide a specific assertion to verify the final state.
907
955
  The assertion MUST prove that YOUR ACTIONS changed the page state.
908
956
  Do NOT verify something that was already true before you started testing.
957
+ Do NOT provide an assertion that verifies absence, failure, an empty state, or that the goal did not happen.
909
958
 
910
959
  Examples of good assertions:
911
960
  - "New user 'john@example.com' is visible in the users list"
@@ -1031,10 +1080,24 @@ export class Tester extends TaskAgent {
1031
1080
  this.resetFailureCount();
1032
1081
  this.previousUrl = null;
1033
1082
  this.previousStateHash = null;
1083
+ this.stalledIterations = 0;
1084
+ }
1085
+ else if (this.shouldStopAfterStalledLoopError(task)) {
1086
+ return 'stop';
1034
1087
  }
1035
1088
  this.currentConversation?.addUserText(result.message);
1036
1089
  return 'continue';
1037
1090
  }
1091
+ shouldStopAfterStalledLoopError(task) {
1092
+ if (task.hasFinished)
1093
+ return false;
1094
+ this.stalledIterations++;
1095
+ if (this.stalledIterations < this.MAX_STALLED_ITERATIONS)
1096
+ return false;
1097
+ task.addNote('No browser progress after repeated execution errors', TestResult.FAILED);
1098
+ task.finish(TestResult.FAILED);
1099
+ return true;
1100
+ }
1038
1101
  async cleanupStartedTest(task) {
1039
1102
  await this.finishTest(task);
1040
1103
  await this.explorer.stopTest(task, {