explorbot 0.1.27 → 0.1.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/README.md +83 -245
  2. package/bin/explorbot-cli.ts +1 -0
  3. package/dist/bin/explorbot-cli.js +1 -0
  4. package/dist/package.json +8 -6
  5. package/dist/rules/navigator/verification-actions.md +2 -0
  6. package/dist/src/action-result.js +2 -0
  7. package/dist/src/action.js +88 -7
  8. package/dist/src/ai/captain/file-tools.js +100 -0
  9. package/dist/src/ai/captain/idle-mode.js +70 -6
  10. package/dist/src/ai/captain/web-mode.js +36 -6
  11. package/dist/src/ai/captain.js +87 -19
  12. package/dist/src/ai/fisherman.js +14 -3
  13. package/dist/src/ai/historian/screencast.js +11 -2
  14. package/dist/src/ai/navigator.js +5 -2
  15. package/dist/src/ai/pilot.js +52 -9
  16. package/dist/src/ai/planner.js +16 -5
  17. package/dist/src/ai/provider.js +53 -18
  18. package/dist/src/ai/researcher/coordinates.js +2 -3
  19. package/dist/src/ai/researcher/deep-analysis.js +3 -4
  20. package/dist/src/ai/researcher/locators.js +1 -2
  21. package/dist/src/ai/researcher.js +24 -19
  22. package/dist/src/ai/rules.js +44 -0
  23. package/dist/src/ai/task-agent.js +1 -0
  24. package/dist/src/ai/tester.js +161 -46
  25. package/dist/src/ai/tools.js +84 -8
  26. package/dist/src/commands/explore-command.js +6 -1
  27. package/dist/src/components/LogPane.js +4 -3
  28. package/dist/src/explorbot.js +7 -2
  29. package/dist/src/explorer.js +270 -35
  30. package/dist/src/stats.js +16 -0
  31. package/dist/src/utils/aria.js +66 -6
  32. package/dist/src/utils/browser-errors.js +23 -0
  33. package/dist/src/utils/error-page.js +17 -2
  34. package/dist/src/utils/logger.js +2 -2
  35. package/package.json +8 -6
  36. package/rules/navigator/verification-actions.md +2 -0
  37. package/src/action-result.ts +2 -0
  38. package/src/action.ts +83 -7
  39. package/src/ai/captain/file-tools.ts +126 -0
  40. package/src/ai/captain/idle-mode.ts +72 -6
  41. package/src/ai/captain/mixin.ts +1 -1
  42. package/src/ai/captain/web-mode.ts +40 -5
  43. package/src/ai/captain.ts +94 -20
  44. package/src/ai/fisherman.ts +14 -3
  45. package/src/ai/historian/screencast.ts +11 -2
  46. package/src/ai/navigator.ts +6 -2
  47. package/src/ai/pilot.ts +53 -9
  48. package/src/ai/planner.ts +16 -5
  49. package/src/ai/provider.ts +51 -19
  50. package/src/ai/researcher/coordinates.ts +2 -3
  51. package/src/ai/researcher/deep-analysis.ts +3 -4
  52. package/src/ai/researcher/locators.ts +1 -2
  53. package/src/ai/researcher.ts +25 -19
  54. package/src/ai/rules.ts +46 -0
  55. package/src/ai/task-agent.ts +1 -1
  56. package/src/ai/tester.ts +175 -48
  57. package/src/ai/tools.ts +97 -8
  58. package/src/commands/explore-command.ts +6 -1
  59. package/src/components/LogPane.tsx +4 -3
  60. package/src/config.ts +1 -0
  61. package/src/explorbot.ts +6 -2
  62. package/src/explorer.ts +295 -38
  63. package/src/state-manager.ts +2 -0
  64. package/src/stats.ts +18 -0
  65. package/src/utils/aria.ts +63 -6
  66. package/src/utils/browser-errors.ts +25 -0
  67. package/src/utils/error-page.ts +16 -3
  68. package/src/utils/logger.ts +3 -3
package/src/ai/captain.ts CHANGED
@@ -26,7 +26,6 @@ const MAX_STEPS = 15;
26
26
  const CaptainBase = WithTestMode(WithWebMode(WithIdleMode(TaskAgent as unknown as new (...args: any[]) => TaskAgent)));
27
27
 
28
28
  export class Captain extends CaptainBase implements Agent {
29
- protected readonly ACTION_TOOLS = ['click', 'pressKey', 'form', 'navigate'];
30
29
  emoji = '🧑‍✈️';
31
30
  private explorBot: ExplorBot;
32
31
  private conversation: Conversation | null = null;
@@ -72,6 +71,12 @@ export class Captain extends CaptainBase implements Agent {
72
71
 
73
72
  protected trackToolExecutions(toolExecutions: any[]): void {
74
73
  super.trackToolExecutions(toolExecutions);
74
+ if (toolExecutions.length > 0) {
75
+ this.recentToolCalls.push(...toolExecutions);
76
+ if (this.recentToolCalls.length > 20) {
77
+ this.recentToolCalls = this.recentToolCalls.slice(-20);
78
+ }
79
+ }
75
80
  for (const exec of toolExecutions) {
76
81
  const label = toolExecutionLabel(exec.input);
77
82
  if (!label) continue;
@@ -80,14 +85,19 @@ export class Captain extends CaptainBase implements Agent {
80
85
  }
81
86
  }
82
87
 
83
- private detectMode(): CaptainMode {
84
- if (this.explorBot.getExplorer().activeTest) return 'test';
85
- if (this.explorBot.getExplorer().getStateManager().getCurrentState()) return 'web';
88
+ getMode(): CaptainMode {
89
+ const explorer = this.explorBot.getExplorer();
90
+ const activeTest = explorer.activeTest;
91
+ const page = explorer.playwrightHelper?.page;
92
+
93
+ if (activeTest && (!page || page.isClosed?.())) return 'heal';
94
+ if (activeTest) return 'test';
95
+ if (explorer.getStateManager().getCurrentState()) return 'web';
86
96
  return 'idle';
87
97
  }
88
98
 
89
99
  private systemPrompt(): string {
90
- const mode = this.detectMode();
100
+ const mode = this.getMode();
91
101
  const currentUrl = this.explorBot.getExplorer().getStateManager().getCurrentState()?.url;
92
102
  const customPrompt = this.explorBot.getProvider().getSystemPromptForAgent('captain', currentUrl);
93
103
 
@@ -101,18 +111,21 @@ export class Captain extends CaptainBase implements Agent {
101
111
  - idle: plan management, file operations, knowledge. Always available.
102
112
  - web: page interaction, navigation, browser diagnostics. When working with a web page.
103
113
  - test: test analysis, state inspection. When a test is running or analyzing results.
114
+ - heal: browser/test recovery. When a test is running and browser state is broken or unavailable.
104
115
  </modes>
105
116
 
106
117
  ${this.idleModePrompt()}
107
- ${mode === 'web' ? this.webModePrompt() : ''}
108
- ${mode === 'test' ? this.testModePrompt() : ''}
118
+ ${mode === 'web' || mode === 'heal' ? this.webModePrompt() : ''}
119
+ ${mode === 'test' || mode === 'heal' ? this.testModePrompt() : ''}
109
120
 
110
121
  <rules>
111
122
  - After a successful action, if the pageDiff confirms the goal, call done() immediately — do not verify with see() or context() unless the user explicitly asked for verification
112
123
  - Prefer completing in fewer tool calls over thoroughness
113
124
  - NEVER run tests unless the user explicitly asks
114
- ${mode === 'web' ? this.webModeRules() : ''}
115
- ${mode === 'test' ? this.testModeRules() : ''}
125
+ - If you are answering with information rather than completing a browser action, include the actual user-facing answer in done({ details }). Do not only say that it was shown or explained.
126
+ ${mode === 'web' || mode === 'heal' ? this.webModeRules() : ''}
127
+ ${mode === 'test' || mode === 'heal' ? this.testModeRules() : ''}
128
+ ${mode === 'heal' ? '- First diagnose browser availability, then recover the browser/page before continuing test analysis.' : ''}
116
129
  </rules>
117
130
 
118
131
  ${customPrompt || ''}
@@ -250,9 +263,20 @@ export class Captain extends CaptainBase implements Agent {
250
263
  description: 'Call when the user request is fulfilled.',
251
264
  inputSchema: z.object({
252
265
  summary: z.string().describe('What was done'),
266
+ details: z.string().optional().describe('Actual user-facing content. Required when the user asked to show, display, explain, summarize, compare, or diagnose information.'),
253
267
  }),
254
- execute: async ({ summary }) => {
268
+ execute: async ({ summary, details }) => {
255
269
  debugLog('done', summary);
270
+ if (!details?.trim() && !this.canCompleteWithoutDetails()) {
271
+ return {
272
+ success: false,
273
+ message: 'No user-facing result was provided. Call done() again with the actual answer in details, or complete a browser action first.',
274
+ };
275
+ }
276
+ if (details?.trim()) {
277
+ tag('details').log(details);
278
+ task.addNote(details);
279
+ }
256
280
  task.addNote(summary);
257
281
  onDone(summary);
258
282
  return { success: true, summary };
@@ -261,6 +285,9 @@ export class Captain extends CaptainBase implements Agent {
261
285
  runCommand: tool({
262
286
  description: dedent`
263
287
  Execute a TUI command. Returns log output from command execution.
288
+ Use only when the user explicitly asks to run a slash command.
289
+ Never use this to analyze files, reports, logs, plans, generated tests, knowledge, or experience.
290
+ Never run a slash command unless the user request itself starts with that slash command.
264
291
  ${this.commandDescriptions
265
292
  .map((c) => {
266
293
  const opts = c.options ? ` (${c.options})` : '';
@@ -274,6 +301,13 @@ export class Captain extends CaptainBase implements Agent {
274
301
  execute: async ({ command }) => {
275
302
  if (!this.commandExecutor) return { success: false, message: 'Command executor not available' };
276
303
  const cmd = command.startsWith('/') ? command : `/${command}`;
304
+ if (!isExplicitSlashRequest(task.description, cmd)) {
305
+ return {
306
+ success: false,
307
+ command: cmd,
308
+ message: 'Command blocked: slash commands require an explicit matching slash-command request from the user.',
309
+ };
310
+ }
277
311
  startLogCapture();
278
312
  try {
279
313
  await this.commandExecutor(cmd);
@@ -286,11 +320,12 @@ export class Captain extends CaptainBase implements Agent {
286
320
  }
287
321
 
288
322
  private async tools(task: Task, onDone: (summary: string) => void) {
289
- const mode = this.detectMode();
323
+ const mode = this.getMode();
290
324
  const ctx: ModeContext = { explorBot: this.explorBot, task };
291
325
  const core = this.coreTools(task, onDone);
292
326
  const idle = await this.idleModeTools(ctx);
293
327
 
328
+ if (mode === 'heal') return { ...core, ...idle, ...this.testModeTools(ctx), ...this.webModeTools(ctx) };
294
329
  if (mode === 'test') return { ...core, ...idle, ...this.testModeTools(ctx) };
295
330
  if (mode === 'web') return { ...core, ...idle, ...this.webModeTools(ctx) };
296
331
  return { ...core, ...idle };
@@ -365,20 +400,28 @@ export class Captain extends CaptainBase implements Agent {
365
400
  return result.object;
366
401
  }
367
402
 
403
+ async processExecutionError(error: Error, activeTest: Test): Promise<ExecutionRecoveryAction> {
404
+ const explorer = this.explorBot.getExplorer();
405
+ const result = await explorer.handleExecutionError(error);
406
+ return {
407
+ ...result,
408
+ message: result.recovered ? `${result.message}\nContinue the test "${activeTest.scenario}" from the restored page.` : result.message,
409
+ };
410
+ }
411
+
412
+ private canCompleteWithoutDetails(): boolean {
413
+ return (this.recentToolCalls || []).some(hasBrowserCompletionEvidence);
414
+ }
415
+
368
416
  async handle(input: string, options: { reset?: boolean } = {}): Promise<string | null> {
369
417
  const stateManager = this.explorBot.getExplorer().getStateManager();
370
418
  const initialState = stateManager.getCurrentState();
371
419
 
372
- if (!initialState) {
373
- tag('warning').log('No page loaded. Use /navigate or I.amOnPage() first.');
374
- return null;
375
- }
376
-
377
420
  const conversation = options.reset ? this.resetConversation() : this.ensureConversation();
378
421
  let isDone = false;
379
422
  let finalSummary: string | null = null;
380
423
 
381
- const startUrl = initialState.url || '';
424
+ const startUrl = initialState?.url || '';
382
425
  const task = new Task(input, startUrl);
383
426
  const onDone = (summary: string) => {
384
427
  isDone = true;
@@ -421,12 +464,14 @@ export class Captain extends CaptainBase implements Agent {
421
464
  }
422
465
 
423
466
  const currentState = stateManager.getCurrentState();
424
- if (!currentState) {
467
+ if (!currentState && this.getMode() !== 'idle') {
425
468
  stop();
426
469
  return;
427
470
  }
428
471
 
429
- await this.reinjectContextIfNeeded(conversation, currentState);
472
+ if (currentState) {
473
+ await this.reinjectContextIfNeeded(conversation, currentState);
474
+ }
430
475
 
431
476
  if (userInput) {
432
477
  const newContext = await this.getPageContext();
@@ -463,7 +508,7 @@ export class Captain extends CaptainBase implements Agent {
463
508
 
464
509
  if (result?.toolExecutions?.length) {
465
510
  const lastExec = result.toolExecutions[result.toolExecutions.length - 1];
466
- if (lastExec.wasSuccessful && this.ACTION_TOOLS.includes(lastExec.toolName)) {
511
+ if (hasBrowserCompletionEvidence(lastExec)) {
467
512
  conversation.addUserText('Action succeeded. If the goal is achieved, call done() now with a brief summary.');
468
513
  }
469
514
  }
@@ -500,3 +545,32 @@ interface SupervisorAction {
500
545
  action: 'inject' | 'stop' | 'pass' | 'skip';
501
546
  message: string;
502
547
  }
548
+
549
+ interface ExecutionRecoveryAction {
550
+ action: 'continue' | 'stop';
551
+ message: string;
552
+ recovered?: boolean;
553
+ }
554
+
555
+ function isExplicitSlashRequest(input: string, command: string): boolean {
556
+ const requested = slashCommandToken(input);
557
+ const actual = slashCommandToken(command);
558
+ if (!requested || !actual) return false;
559
+ return requested === actual;
560
+ }
561
+
562
+ function slashCommandToken(value: string): string | null {
563
+ const trimmed = value.trim();
564
+ if (!trimmed.startsWith('/')) return null;
565
+
566
+ for (let i = 1; i < trimmed.length; i++) {
567
+ if (trimmed[i] <= ' ') return trimmed.slice(0, i);
568
+ }
569
+ return trimmed;
570
+ }
571
+
572
+ function hasBrowserCompletionEvidence(execution: any): boolean {
573
+ if (!execution?.wasSuccessful) return false;
574
+ const output = execution.output || {};
575
+ return Boolean(output.pageDiff || output.code || output.playwrightGroupId);
576
+ }
@@ -9,6 +9,7 @@ import { loop } from '../utils/loop.ts';
9
9
  import type { Agent } from './agent.ts';
10
10
  import { type FishermanResult, createFishermanTools } from './fisherman-tools.ts';
11
11
  import type { Provider } from './provider.ts';
12
+ import { dataProtectionRules } from './rules.ts';
12
13
 
13
14
  const MAX_ITERATIONS = 15;
14
15
  const MAX_TOOL_ROUNDTRIPS = 5;
@@ -85,7 +86,7 @@ export class Fisherman implements Agent {
85
86
  baseEndpoint: this.baseEndpoint,
86
87
  });
87
88
 
88
- const conversation = this.provider.startConversation(this.buildSystemPrompt(endpointList, scopeUrl), 'fisherman');
89
+ const conversation = this.provider.startConversation(this.buildSystemPrompt(endpointList, Object.keys(tools), scopeUrl), 'fisherman');
89
90
  conversation.addUserText(this.buildTaskPrompt(instructions));
90
91
 
91
92
  await loop(
@@ -187,7 +188,7 @@ export class Fisherman implements Agent {
187
188
  return lines.join('\n');
188
189
  }
189
190
 
190
- private buildSystemPrompt(endpointList: string, scopeUrl?: string): string {
191
+ private buildSystemPrompt(endpointList: string, toolNames: string[], scopeUrl?: string): string {
191
192
  const scopeBlock = scopeUrl ? `\n\nSCOPE: You are operating within ${scopeUrl}.\nAll created items must belong to this scope.` : '';
192
193
 
193
194
  return dedent`
@@ -197,6 +198,11 @@ export class Fisherman implements Agent {
197
198
  ${endpointList}
198
199
  ${scopeBlock}
199
200
 
201
+ AVAILABLE TOOLS:
202
+ ${toolNames.join(', ')}.
203
+ Use tool names exactly as listed. Do not invent aliases, combined names, or names with channel markers such as "commentary".
204
+ Match each tool input schema exactly. Do not invent parameter names or pass extra fields.
205
+
200
206
  WORKFLOW:
201
207
  1. Call getEndpointSpec to see the request body example for the endpoint
202
208
  2. Make requests — the response automatically extracts IDs, names, and status fields
@@ -208,6 +214,8 @@ export class Fisherman implements Agent {
208
214
  - Chain requests logically — create parent resources before children
209
215
  - If a request fails, try once more with adjusted data before reporting failure
210
216
  - Use realistic but unique data for each item (vary names, titles)
217
+
218
+ ${dataProtectionRules}
211
219
  `;
212
220
  }
213
221
 
@@ -217,7 +225,10 @@ export class Fisherman implements Agent {
217
225
 
218
226
  ${instructions}
219
227
 
220
- Execute the necessary API requests to create this data. When done, call finish with the summary.
228
+ ${dataProtectionRules}
229
+
230
+ If data preparation is allowed by these rules, execute the necessary API requests to create this data.
231
+ When done, call finish with the summary. If data preparation is forbidden, call stop with the reason.
221
232
  `;
222
233
  }
223
234
  }
@@ -10,6 +10,8 @@ import { relativeToCwd } from '../../utils/next-steps.ts';
10
10
  import { safeFilename } from '../../utils/strings.ts';
11
11
  import { type Constructor, debugLog } from './mixin.ts';
12
12
 
13
+ const FATAL_SCREENCAST_STOP_ERRORS = /Target page, context or browser has been closed|Target closed|Session closed|Protocol error/i;
14
+
13
15
  export interface ScreencastMethods {
14
16
  attachScreencast(): void;
15
17
  isScreencastActive(): boolean;
@@ -113,17 +115,24 @@ export function WithScreencast<T extends Constructor>(Base: T) {
113
115
  if (!this.screencastActive) return;
114
116
  const path = this.screencastPath;
115
117
  const task = this.screencastTask;
118
+ let stopped = false;
116
119
  try {
117
120
  await this.screencastPage.screencast.stop();
121
+ stopped = true;
118
122
  } catch (err) {
119
- tag('operation').log(`Screencast stop failed: ${(err as Error).message}`);
123
+ const message = (err as Error).message;
124
+ if (FATAL_SCREENCAST_STOP_ERRORS.test(message)) {
125
+ tag('operation').log('Screencast skipped: browser was closed before recording could be finalized');
126
+ } else {
127
+ tag('operation').log(`Screencast stop failed: ${message}`);
128
+ }
120
129
  }
121
130
  this.screencastActive = false;
122
131
  this.screencastPage = null;
123
132
  this.screencastPath = null;
124
133
  this.screencastTask = null;
125
134
  this.screencastLastChapter = null;
126
- if (path) {
135
+ if (path && stopped) {
127
136
  this.savedFiles.add(path);
128
137
  task?.addArtifact?.(path);
129
138
  tag('operation').log(`Saved screencast: ${relativeToCwd(path)}`);
@@ -136,6 +136,10 @@ class Navigator implements Agent {
136
136
  }
137
137
 
138
138
  async visit(url: string): Promise<void> {
139
+ return this.explorer.runWithBrowserRecovery('navigator.visit', () => this.visitOnce(url));
140
+ }
141
+
142
+ private async visitOnce(url: string): Promise<void> {
139
143
  try {
140
144
  const action = this.explorer.createAction();
141
145
 
@@ -170,7 +174,7 @@ class Navigator implements Agent {
170
174
  throw new Error(`Navigation to ${url} failed: ${action.lastError?.message}`);
171
175
  }
172
176
  }
173
- await action.caputrePageWithScreenshot();
177
+ await this.explorer.capturePageWithScreenshot();
174
178
  await this.hooksRunner.runAfterHook('navigator', url);
175
179
  } catch (error) {
176
180
  const errorMessage = error instanceof Error ? error.message : String(error);
@@ -382,7 +386,7 @@ class Navigator implements Agent {
382
386
  // URL did not transition to expectedUrl within timeout
383
387
  }
384
388
  }
385
- const freshState = await action.capturePageState();
389
+ const freshState = await this.explorer.capturePageState();
386
390
  const currentUrl = /^https?:\/\//i.test(expectedUrl) ? freshState.fullUrl || freshState.url || '' : freshState.url || '';
387
391
  const urlMatches = this.isSameExpectedOrigin(expectedUrl, action.stateManager) && normalizeUrl(currentUrl) === normalizeUrl(expectedUrl);
388
392
  const stateChanged = freshState.getStateHash() !== actionResult.getStateHash();
package/src/ai/pilot.ts CHANGED
@@ -18,6 +18,7 @@ import type { Fisherman } from './fisherman.ts';
18
18
  import type { Navigator } from './navigator.ts';
19
19
  import type { Provider } from './provider.ts';
20
20
  import type { Researcher } from './researcher.ts';
21
+ import { capabilityGroundingRule, dataProtectionRules } from './rules.ts';
21
22
  import { isInteractive } from './task-agent.ts';
22
23
 
23
24
  const CHECK_TOOLS = ['verify', 'see', 'research', 'context'];
@@ -67,7 +68,7 @@ export class Pilot implements Agent {
67
68
  }
68
69
 
69
70
  async reviewCompletion(task: Test, currentState: ActionResult, testerConversation: Conversation, navigator?: Navigator): Promise<boolean> {
70
- const verdictType = task.hasAchievedAny() ? 'finish' : 'stop';
71
+ const verdictType = this.hasCompletionEvidence(task, currentState, testerConversation) ? 'finish' : 'stop';
71
72
  return this.reviewDecision(verdictType, task, currentState, testerConversation, navigator);
72
73
  }
73
74
 
@@ -86,14 +87,14 @@ export class Pilot implements Agent {
86
87
 
87
88
  const sessionLog = this.formatSessionLog(testerConversation);
88
89
  const stateContext = this.buildStateContext(currentState);
90
+ const successfulAssertions = this.formatSuccessfulAssertions(currentState, testerConversation);
89
91
  const notes = task.notesToString() || 'No notes recorded.';
90
92
 
91
93
  let visualAnalysis = '';
92
94
  let screenshotState: ActionResult | null = null;
93
- if (this.provider.hasVision()) {
95
+ if (type === 'finish' && this.provider.hasVision()) {
94
96
  try {
95
- const action = this.explorer.createAction();
96
- screenshotState = await action.caputrePageWithScreenshot();
97
+ screenshotState = await this.explorer.capturePageWithScreenshot();
97
98
  if (screenshotState.screenshot) {
98
99
  visualAnalysis = (await this.researcher.answerQuestionAboutScreenshot(screenshotState, `Describe current page state relevant to: ${task.scenario}`)) || '';
99
100
  }
@@ -125,6 +126,10 @@ export class Pilot implements Agent {
125
126
 
126
127
  ${this.formatExpectations(task)}
127
128
 
129
+ <successful_assertions>
130
+ ${successfulAssertions || 'None'}
131
+ </successful_assertions>
132
+
128
133
  <notes>
129
134
  ${notes}
130
135
  </notes>
@@ -160,7 +165,7 @@ export class Pilot implements Agent {
160
165
  try {
161
166
  const response = await this.provider.generateObject(messages, schema, this.provider.getAgenticModel('pilot'), {
162
167
  agentName: 'pilot',
163
- experimental_telemetry: { functionId: 'pilot.reviewVerdict' },
168
+ telemetry: { functionId: 'pilot.reviewVerdict' },
164
169
  });
165
170
 
166
171
  const result = response?.object;
@@ -262,7 +267,7 @@ export class Pilot implements Agent {
262
267
  try {
263
268
  const response = await this.provider.generateObject(messages, schema, this.provider.getAgenticModel('pilot'), {
264
269
  agentName: 'pilot',
265
- experimental_telemetry: { functionId: 'pilot.reviewReset' },
270
+ telemetry: { functionId: 'pilot.reviewReset' },
266
271
  });
267
272
 
268
273
  const result = response?.object;
@@ -374,6 +379,8 @@ export class Pilot implements Agent {
374
379
  You are Pilot — final decision maker for test pass/fail. Tester requested ${type}. Review the
375
380
  evidence and commit to a verdict; "continue" only when evidence is genuinely insufficient.
376
381
 
382
+ ${capabilityGroundingRule}
383
+
377
384
  ${this.buildSharedEvidenceRules(task)}
378
385
 
379
386
  DECISION:
@@ -382,6 +389,8 @@ export class Pilot implements Agent {
382
389
  Pick assertions DOM can express; for non-DOM regions (iframes, canvas, Monaco/CodeMirror), target a
383
390
  stable landmark (container, ARIA role) instead of literal inner text. Your "pass" stands even if the
384
391
  DOM assertion can't be made.
392
+ Do not pass when Tester achieved only a related navigation/filter/tab/status outcome instead of the
393
+ requested action, workflow, or entity detail goal.
385
394
  - "fail": scenario was attempted but the goal was not achieved.
386
395
  - "skipped": scenario is irrelevant to the app, OR systematic infrastructure failures (LLM errors,
387
396
  crashes) prevented testing. NOT for "test failed to interact" — that's "fail" or "continue".
@@ -420,6 +429,10 @@ export class Pilot implements Agent {
420
429
 
421
430
  FIRST: Decide if precondition() is needed.
422
431
 
432
+ ${capabilityGroundingRule}
433
+
434
+ ${dataProtectionRules}
435
+
423
436
  Call precondition() WHEN:
424
437
  - The scenario edits/deletes/modifies an item, and you want a DISPOSABLE item to act on safely
425
438
  - The scenario needs specific data clearly NOT on the current page (e.g., items with specific statuses for filtering)
@@ -572,7 +585,7 @@ export class Pilot implements Agent {
572
585
  maxToolRoundtrips: opts.maxToolRoundtrips ?? 0,
573
586
  agentName: 'pilot',
574
587
  stopWhen: opts.task ? () => opts.task!.hasFinished : undefined,
575
- experimental_telemetry: { functionId },
588
+ telemetry: { functionId },
576
589
  });
577
590
  const text = result?.response?.text || '';
578
591
  const learned = (result?.toolExecutions || []).filter((e: any) => e.toolName === 'learnExperience' && e.output?.content).map((e: any) => e.output.content);
@@ -658,8 +671,7 @@ export class Pilot implements Agent {
658
671
  private async checkDataAvailability(task: Test, requestedData: string, fishermanReason: string | undefined): Promise<string | null> {
659
672
  if (!this.provider.hasVision()) return null;
660
673
 
661
- const action = this.explorer.createAction();
662
- const screenshotState = await action.caputrePageWithScreenshot().catch(() => null);
674
+ const screenshotState = await this.explorer.capturePageWithScreenshot().catch(() => null);
663
675
  if (!screenshotState?.screenshot) return null;
664
676
 
665
677
  const question = dedent`
@@ -882,6 +894,32 @@ export class Pilot implements Agent {
882
894
  return parts.join('\n\n');
883
895
  }
884
896
 
897
+ private hasCompletionEvidence(task: Test, currentState: ActionResult, testerConversation: Conversation): boolean {
898
+ if (task.hasAchievedAny()) return true;
899
+ return this.hasSuccessfulCheckEvidence(currentState, testerConversation);
900
+ }
901
+
902
+ private hasSuccessfulCheckEvidence(currentState: ActionResult, testerConversation: Conversation): boolean {
903
+ if (Object.values(currentState.verifications ?? {}).some(Boolean)) return true;
904
+ return testerConversation.getToolExecutions().some((t) => CHECK_TOOLS.includes(t.toolName) && t.wasSuccessful);
905
+ }
906
+
907
+ private formatSuccessfulAssertions(currentState: ActionResult, testerConversation: Conversation): string {
908
+ const lines: string[] = [];
909
+ for (const [assertion, passed] of Object.entries(currentState.verifications ?? {})) {
910
+ if (passed) lines.push(`PASS state verification: ${assertion}`);
911
+ }
912
+
913
+ for (const exec of testerConversation.getToolExecutions()) {
914
+ if (!CHECK_TOOLS.includes(exec.toolName) || !exec.wasSuccessful) continue;
915
+ const description = exec.input?.assertion || exec.input?.request || truncateJson(exec.input);
916
+ const result = exec.output?.message || exec.output?.analysis || exec.output?.result;
917
+ lines.push(`PASS ${exec.toolName}: ${description}${result ? ` -> ${result}` : ''}`);
918
+ }
919
+
920
+ return [...new Set(lines)].join('\n');
921
+ }
922
+
885
923
  private formatActions(toolCalls: any[]): string {
886
924
  return toolCalls
887
925
  .map((t) => {
@@ -1000,9 +1038,13 @@ export class Pilot implements Agent {
1000
1038
 
1001
1039
  Tester tools: click, pressKey, form, see, verify, context, research, xpathCheck, visualClick,
1002
1040
  back, getVisitedStates, reset, stop, finish, record.
1041
+ Use tool names exactly as listed. Do not invent combined names, aliases, or names with channel markers such as "commentary".
1042
+
1043
+ ${capabilityGroundingRule}
1003
1044
 
1004
1045
  YOUR Pilot-only tool: precondition(description) — create FRESH disposable test data via API. Never
1005
1046
  request users. Use when:
1047
+
1006
1048
  - Scenario edits/deletes/modifies an item → create a disposable target ("1 post").
1007
1049
  - Scenario needs auxiliary data (labels, categories, statuses for filtering).
1008
1050
  - Tester failed because required data is missing (empty dropdown, empty list).
@@ -1012,6 +1054,8 @@ export class Pilot implements Agent {
1012
1054
  - Current page already shows the exact data needed.
1013
1055
  - Scenario tests navigation, search UI, or viewing.
1014
1056
 
1057
+ ${dataProtectionRules}
1058
+
1015
1059
  Describe WHAT to create, not what exists. RIGHT: precondition("1 test"). WRONG:
1016
1060
  precondition("1 test suite named Updated Suite with existing tests"). Keep descriptions short.
1017
1061
 
package/src/ai/planner.ts CHANGED
@@ -24,7 +24,7 @@ import type { Provider } from './provider.js';
24
24
  import { POSSIBLE_SECTIONS, Researcher } from './researcher.ts';
25
25
  import { findSimilarStateHash } from './researcher/cache.ts';
26
26
  import { hasFocusedSection } from './researcher/focus.ts';
27
- import { fileUploadRule, protectionRule } from './rules.ts';
27
+ import { capabilityGroundingRule, dataProtectionRules, fileUploadRule } from './rules.ts';
28
28
 
29
29
  const debugLog = createDebug('explorbot:planner');
30
30
 
@@ -35,7 +35,7 @@ const TasksSchema = z.object({
35
35
  z.object({
36
36
  scenario: z.string().describe('A single sentence describing what to test'),
37
37
  priority: z.enum(['critical', 'important', 'high', 'normal', 'low']).describe('Priority of the task based on business importance'),
38
- startUrl: z.string().nullable().describe('Start URL for the test if different from plan URL (only for tests on visited subpages)'),
38
+ startUrl: z.string().nullable().describe('Start URL for the test if different from plan URL. Use only stable feature/list/detail pages, not transient create/edit/modal URLs unless the scenario specifically starts inside that form.'),
39
39
  steps: z.array(z.string()).describe('List of steps to perform for this scenario. Each step should be a specific action (e.g., "Open the form", "Enter required data", "Submit the form"). Keep steps atomic and actionable.'),
40
40
  expectedOutcomes: z
41
41
  .array(z.string())
@@ -90,6 +90,9 @@ export class Planner extends PlannerBase implements Agent {
90
90
  const featureDirective = feature
91
91
  ? `\n IMPORTANT: The user requested to focus specifically on: "${feature}"\n ALL scenarios MUST be directly related to this feature. Do not propose generic page tests unrelated to it.\n Use the user's exact wording to guide scenario names — do not substitute different entities (e.g., do not plan "suite" actions when user said "test").`
92
92
  : '';
93
+ const focusExistingDataDirective = feature
94
+ ? '\n If this focus asks for search, filter, tabs, sorting, or list behavior involving existing items, only use item names/values visible in the provided page research. If no concrete visible item names/values are present, do NOT propose scenarios that require an existing known item; propose no-match search, empty-state, clear-search, tab/filter empty-list, or other read-only list behavior instead.'
95
+ : '';
93
96
  return dedent`
94
97
  <role>
95
98
  You are ISTQB certified senior manual QA planning exploratory testing session of a web application.
@@ -113,7 +116,7 @@ export class Planner extends PlannerBase implements Agent {
113
116
  Bad: "Open delete dropdown" + "Confirm deletion" — these are ONE test, not two.
114
117
  Bad: "Search for X" + "Verify search results" — searching and verifying is ONE test.
115
118
  Bad: "Leave field empty" + "Click submit" — that's one negative test, not two.
116
- If two scenarios cannot run independently (one requires the other to run first), merge them into one.${featureDirective}
119
+ If two scenarios cannot run independently (one requires the other to run first), merge them into one.${featureDirective}${focusExistingDataDirective}
117
120
  </task>
118
121
 
119
122
  ${customPrompt || ''}
@@ -341,6 +344,11 @@ export class Planner extends PlannerBase implements Agent {
341
344
  If a scenario needs existing records, recipients, results, notifications, or other target data, propose it only when that data is visible or API preconditions can create it.
342
345
  If the page appears read-only, degraded, demo-limited, maintenance-like, or lacks write controls, prefer read-only scenarios such as opening panels, inspecting visible lists, filtering, searching, or verifying current state.
343
346
  Do not assume hidden data exists just because a control is present.
347
+ For scenarios that act on existing items or search/filter by existing values, use only item names or values visible in research, visited pages, or prior observed flows.
348
+ If the list is empty or no concrete item names are visible, do not invent "known" or "existing" items. Prefer empty-state, no-match search, clear-search, or read-only list behavior scenarios.
349
+ Search, filter, sorting, tab, and list scenarios must start from a stable page where those controls are visible; avoid transient create/edit/new URLs unless the scenario tests that form.
350
+ For option values and list items, use only visible or previously observed data; do not add create/update/delete setup unless the user explicitly requests that workflow.
351
+ Detail-view scenarios must target visible data entities from list rows, cards, tree nodes, or detail links; do not use filter tabs, counters, status tabs, breadcrumbs, or navigation controls as detail targets.
344
352
  DO NOT propose "verification-only" tests that merely open a UI element (modal, dropdown, panel) and check it exists.
345
353
  Every test must complete a meaningful action that changes application state or produces a business outcome.
346
354
  Opening a modal is NOT a test — performing an action INSIDE the modal IS a test.
@@ -351,7 +359,8 @@ export class Planner extends PlannerBase implements Agent {
351
359
  Tests that only switch views, toggle filters, or paginate are LESS valuable — propose them only after data-changing tests are covered.
352
360
  If multiple ways to create or modify data exist (different types, different forms), propose a separate test for each.
353
361
  </priority_order>
354
- ${protectionRule}
362
+ ${capabilityGroundingRule}
363
+ ${dataProtectionRules}
355
364
  ${fileUploadRule}
356
365
  </rules>
357
366
 
@@ -514,7 +523,9 @@ export class Planner extends PlannerBase implements Agent {
514
523
  .join('\n')}
515
524
 
516
525
  You MAY propose tests starting from these pages if they are relevant to the plan "${this.currentPlan.title}".
517
- Set startUrl for such tests. Ignore pages that belong to a different feature area.
526
+ Set startUrl for such tests only when the page is a stable feature/list/detail page.
527
+ Do not use create/edit/new/modal URLs as startUrl for scenarios that need the underlying page.
528
+ Ignore pages that belong to a different feature area.
518
529
  </context_from_previous_tests>
519
530
 
520
531
  Propose ONLY new scenarios that are NOT in the existing tests list.