explorbot 0.1.27 → 0.1.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +83 -245
- package/bin/explorbot-cli.ts +1 -0
- package/dist/bin/explorbot-cli.js +1 -0
- package/dist/package.json +8 -6
- package/dist/rules/navigator/verification-actions.md +2 -0
- package/dist/src/action-result.js +2 -0
- package/dist/src/action.js +88 -7
- package/dist/src/ai/captain/file-tools.js +100 -0
- package/dist/src/ai/captain/idle-mode.js +70 -6
- package/dist/src/ai/captain/web-mode.js +36 -6
- package/dist/src/ai/captain.js +87 -19
- package/dist/src/ai/fisherman.js +14 -3
- package/dist/src/ai/historian/screencast.js +11 -2
- package/dist/src/ai/navigator.js +5 -2
- package/dist/src/ai/pilot.js +52 -9
- package/dist/src/ai/planner.js +16 -5
- package/dist/src/ai/provider.js +53 -18
- package/dist/src/ai/researcher/coordinates.js +2 -3
- package/dist/src/ai/researcher/deep-analysis.js +3 -4
- package/dist/src/ai/researcher/locators.js +1 -2
- package/dist/src/ai/researcher.js +24 -19
- package/dist/src/ai/rules.js +44 -0
- package/dist/src/ai/task-agent.js +1 -0
- package/dist/src/ai/tester.js +161 -46
- package/dist/src/ai/tools.js +84 -8
- package/dist/src/commands/explore-command.js +6 -1
- package/dist/src/components/LogPane.js +4 -3
- package/dist/src/explorbot.js +7 -2
- package/dist/src/explorer.js +270 -35
- package/dist/src/stats.js +16 -0
- package/dist/src/utils/aria.js +66 -6
- package/dist/src/utils/browser-errors.js +23 -0
- package/dist/src/utils/error-page.js +17 -2
- package/dist/src/utils/logger.js +2 -2
- package/package.json +8 -6
- package/rules/navigator/verification-actions.md +2 -0
- package/src/action-result.ts +2 -0
- package/src/action.ts +83 -7
- package/src/ai/captain/file-tools.ts +126 -0
- package/src/ai/captain/idle-mode.ts +72 -6
- package/src/ai/captain/mixin.ts +1 -1
- package/src/ai/captain/web-mode.ts +40 -5
- package/src/ai/captain.ts +94 -20
- package/src/ai/fisherman.ts +14 -3
- package/src/ai/historian/screencast.ts +11 -2
- package/src/ai/navigator.ts +6 -2
- package/src/ai/pilot.ts +53 -9
- package/src/ai/planner.ts +16 -5
- package/src/ai/provider.ts +51 -19
- package/src/ai/researcher/coordinates.ts +2 -3
- package/src/ai/researcher/deep-analysis.ts +3 -4
- package/src/ai/researcher/locators.ts +1 -2
- package/src/ai/researcher.ts +25 -19
- package/src/ai/rules.ts +46 -0
- package/src/ai/task-agent.ts +1 -1
- package/src/ai/tester.ts +175 -48
- package/src/ai/tools.ts +97 -8
- package/src/commands/explore-command.ts +6 -1
- package/src/components/LogPane.tsx +4 -3
- package/src/config.ts +1 -0
- package/src/explorbot.ts +6 -2
- package/src/explorer.ts +295 -38
- package/src/state-manager.ts +2 -0
- package/src/stats.ts +18 -0
- package/src/utils/aria.ts +63 -6
- package/src/utils/browser-errors.ts +25 -0
- package/src/utils/error-page.ts +16 -3
- package/src/utils/logger.ts +3 -3
package/src/ai/captain.ts
CHANGED
|
@@ -26,7 +26,6 @@ const MAX_STEPS = 15;
|
|
|
26
26
|
const CaptainBase = WithTestMode(WithWebMode(WithIdleMode(TaskAgent as unknown as new (...args: any[]) => TaskAgent)));
|
|
27
27
|
|
|
28
28
|
export class Captain extends CaptainBase implements Agent {
|
|
29
|
-
protected readonly ACTION_TOOLS = ['click', 'pressKey', 'form', 'navigate'];
|
|
30
29
|
emoji = '🧑✈️';
|
|
31
30
|
private explorBot: ExplorBot;
|
|
32
31
|
private conversation: Conversation | null = null;
|
|
@@ -72,6 +71,12 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
72
71
|
|
|
73
72
|
protected trackToolExecutions(toolExecutions: any[]): void {
|
|
74
73
|
super.trackToolExecutions(toolExecutions);
|
|
74
|
+
if (toolExecutions.length > 0) {
|
|
75
|
+
this.recentToolCalls.push(...toolExecutions);
|
|
76
|
+
if (this.recentToolCalls.length > 20) {
|
|
77
|
+
this.recentToolCalls = this.recentToolCalls.slice(-20);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
75
80
|
for (const exec of toolExecutions) {
|
|
76
81
|
const label = toolExecutionLabel(exec.input);
|
|
77
82
|
if (!label) continue;
|
|
@@ -80,14 +85,19 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
80
85
|
}
|
|
81
86
|
}
|
|
82
87
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
88
|
+
getMode(): CaptainMode {
|
|
89
|
+
const explorer = this.explorBot.getExplorer();
|
|
90
|
+
const activeTest = explorer.activeTest;
|
|
91
|
+
const page = explorer.playwrightHelper?.page;
|
|
92
|
+
|
|
93
|
+
if (activeTest && (!page || page.isClosed?.())) return 'heal';
|
|
94
|
+
if (activeTest) return 'test';
|
|
95
|
+
if (explorer.getStateManager().getCurrentState()) return 'web';
|
|
86
96
|
return 'idle';
|
|
87
97
|
}
|
|
88
98
|
|
|
89
99
|
private systemPrompt(): string {
|
|
90
|
-
const mode = this.
|
|
100
|
+
const mode = this.getMode();
|
|
91
101
|
const currentUrl = this.explorBot.getExplorer().getStateManager().getCurrentState()?.url;
|
|
92
102
|
const customPrompt = this.explorBot.getProvider().getSystemPromptForAgent('captain', currentUrl);
|
|
93
103
|
|
|
@@ -101,18 +111,21 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
101
111
|
- idle: plan management, file operations, knowledge. Always available.
|
|
102
112
|
- web: page interaction, navigation, browser diagnostics. When working with a web page.
|
|
103
113
|
- test: test analysis, state inspection. When a test is running or analyzing results.
|
|
114
|
+
- heal: browser/test recovery. When a test is running and browser state is broken or unavailable.
|
|
104
115
|
</modes>
|
|
105
116
|
|
|
106
117
|
${this.idleModePrompt()}
|
|
107
|
-
${mode === 'web' ? this.webModePrompt() : ''}
|
|
108
|
-
${mode === 'test' ? this.testModePrompt() : ''}
|
|
118
|
+
${mode === 'web' || mode === 'heal' ? this.webModePrompt() : ''}
|
|
119
|
+
${mode === 'test' || mode === 'heal' ? this.testModePrompt() : ''}
|
|
109
120
|
|
|
110
121
|
<rules>
|
|
111
122
|
- After a successful action, if the pageDiff confirms the goal, call done() immediately — do not verify with see() or context() unless the user explicitly asked for verification
|
|
112
123
|
- Prefer completing in fewer tool calls over thoroughness
|
|
113
124
|
- NEVER run tests unless the user explicitly asks
|
|
114
|
-
|
|
115
|
-
${mode === '
|
|
125
|
+
- If you are answering with information rather than completing a browser action, include the actual user-facing answer in done({ details }). Do not only say that it was shown or explained.
|
|
126
|
+
${mode === 'web' || mode === 'heal' ? this.webModeRules() : ''}
|
|
127
|
+
${mode === 'test' || mode === 'heal' ? this.testModeRules() : ''}
|
|
128
|
+
${mode === 'heal' ? '- First diagnose browser availability, then recover the browser/page before continuing test analysis.' : ''}
|
|
116
129
|
</rules>
|
|
117
130
|
|
|
118
131
|
${customPrompt || ''}
|
|
@@ -250,9 +263,20 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
250
263
|
description: 'Call when the user request is fulfilled.',
|
|
251
264
|
inputSchema: z.object({
|
|
252
265
|
summary: z.string().describe('What was done'),
|
|
266
|
+
details: z.string().optional().describe('Actual user-facing content. Required when the user asked to show, display, explain, summarize, compare, or diagnose information.'),
|
|
253
267
|
}),
|
|
254
|
-
execute: async ({ summary }) => {
|
|
268
|
+
execute: async ({ summary, details }) => {
|
|
255
269
|
debugLog('done', summary);
|
|
270
|
+
if (!details?.trim() && !this.canCompleteWithoutDetails()) {
|
|
271
|
+
return {
|
|
272
|
+
success: false,
|
|
273
|
+
message: 'No user-facing result was provided. Call done() again with the actual answer in details, or complete a browser action first.',
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
if (details?.trim()) {
|
|
277
|
+
tag('details').log(details);
|
|
278
|
+
task.addNote(details);
|
|
279
|
+
}
|
|
256
280
|
task.addNote(summary);
|
|
257
281
|
onDone(summary);
|
|
258
282
|
return { success: true, summary };
|
|
@@ -261,6 +285,9 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
261
285
|
runCommand: tool({
|
|
262
286
|
description: dedent`
|
|
263
287
|
Execute a TUI command. Returns log output from command execution.
|
|
288
|
+
Use only when the user explicitly asks to run a slash command.
|
|
289
|
+
Never use this to analyze files, reports, logs, plans, generated tests, knowledge, or experience.
|
|
290
|
+
Never run a slash command unless the user request itself starts with that slash command.
|
|
264
291
|
${this.commandDescriptions
|
|
265
292
|
.map((c) => {
|
|
266
293
|
const opts = c.options ? ` (${c.options})` : '';
|
|
@@ -274,6 +301,13 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
274
301
|
execute: async ({ command }) => {
|
|
275
302
|
if (!this.commandExecutor) return { success: false, message: 'Command executor not available' };
|
|
276
303
|
const cmd = command.startsWith('/') ? command : `/${command}`;
|
|
304
|
+
if (!isExplicitSlashRequest(task.description, cmd)) {
|
|
305
|
+
return {
|
|
306
|
+
success: false,
|
|
307
|
+
command: cmd,
|
|
308
|
+
message: 'Command blocked: slash commands require an explicit matching slash-command request from the user.',
|
|
309
|
+
};
|
|
310
|
+
}
|
|
277
311
|
startLogCapture();
|
|
278
312
|
try {
|
|
279
313
|
await this.commandExecutor(cmd);
|
|
@@ -286,11 +320,12 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
286
320
|
}
|
|
287
321
|
|
|
288
322
|
private async tools(task: Task, onDone: (summary: string) => void) {
|
|
289
|
-
const mode = this.
|
|
323
|
+
const mode = this.getMode();
|
|
290
324
|
const ctx: ModeContext = { explorBot: this.explorBot, task };
|
|
291
325
|
const core = this.coreTools(task, onDone);
|
|
292
326
|
const idle = await this.idleModeTools(ctx);
|
|
293
327
|
|
|
328
|
+
if (mode === 'heal') return { ...core, ...idle, ...this.testModeTools(ctx), ...this.webModeTools(ctx) };
|
|
294
329
|
if (mode === 'test') return { ...core, ...idle, ...this.testModeTools(ctx) };
|
|
295
330
|
if (mode === 'web') return { ...core, ...idle, ...this.webModeTools(ctx) };
|
|
296
331
|
return { ...core, ...idle };
|
|
@@ -365,20 +400,28 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
365
400
|
return result.object;
|
|
366
401
|
}
|
|
367
402
|
|
|
403
|
+
async processExecutionError(error: Error, activeTest: Test): Promise<ExecutionRecoveryAction> {
|
|
404
|
+
const explorer = this.explorBot.getExplorer();
|
|
405
|
+
const result = await explorer.handleExecutionError(error);
|
|
406
|
+
return {
|
|
407
|
+
...result,
|
|
408
|
+
message: result.recovered ? `${result.message}\nContinue the test "${activeTest.scenario}" from the restored page.` : result.message,
|
|
409
|
+
};
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
private canCompleteWithoutDetails(): boolean {
|
|
413
|
+
return (this.recentToolCalls || []).some(hasBrowserCompletionEvidence);
|
|
414
|
+
}
|
|
415
|
+
|
|
368
416
|
async handle(input: string, options: { reset?: boolean } = {}): Promise<string | null> {
|
|
369
417
|
const stateManager = this.explorBot.getExplorer().getStateManager();
|
|
370
418
|
const initialState = stateManager.getCurrentState();
|
|
371
419
|
|
|
372
|
-
if (!initialState) {
|
|
373
|
-
tag('warning').log('No page loaded. Use /navigate or I.amOnPage() first.');
|
|
374
|
-
return null;
|
|
375
|
-
}
|
|
376
|
-
|
|
377
420
|
const conversation = options.reset ? this.resetConversation() : this.ensureConversation();
|
|
378
421
|
let isDone = false;
|
|
379
422
|
let finalSummary: string | null = null;
|
|
380
423
|
|
|
381
|
-
const startUrl = initialState
|
|
424
|
+
const startUrl = initialState?.url || '';
|
|
382
425
|
const task = new Task(input, startUrl);
|
|
383
426
|
const onDone = (summary: string) => {
|
|
384
427
|
isDone = true;
|
|
@@ -421,12 +464,14 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
421
464
|
}
|
|
422
465
|
|
|
423
466
|
const currentState = stateManager.getCurrentState();
|
|
424
|
-
if (!currentState) {
|
|
467
|
+
if (!currentState && this.getMode() !== 'idle') {
|
|
425
468
|
stop();
|
|
426
469
|
return;
|
|
427
470
|
}
|
|
428
471
|
|
|
429
|
-
|
|
472
|
+
if (currentState) {
|
|
473
|
+
await this.reinjectContextIfNeeded(conversation, currentState);
|
|
474
|
+
}
|
|
430
475
|
|
|
431
476
|
if (userInput) {
|
|
432
477
|
const newContext = await this.getPageContext();
|
|
@@ -463,7 +508,7 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
463
508
|
|
|
464
509
|
if (result?.toolExecutions?.length) {
|
|
465
510
|
const lastExec = result.toolExecutions[result.toolExecutions.length - 1];
|
|
466
|
-
if (
|
|
511
|
+
if (hasBrowserCompletionEvidence(lastExec)) {
|
|
467
512
|
conversation.addUserText('Action succeeded. If the goal is achieved, call done() now with a brief summary.');
|
|
468
513
|
}
|
|
469
514
|
}
|
|
@@ -500,3 +545,32 @@ interface SupervisorAction {
|
|
|
500
545
|
action: 'inject' | 'stop' | 'pass' | 'skip';
|
|
501
546
|
message: string;
|
|
502
547
|
}
|
|
548
|
+
|
|
549
|
+
interface ExecutionRecoveryAction {
|
|
550
|
+
action: 'continue' | 'stop';
|
|
551
|
+
message: string;
|
|
552
|
+
recovered?: boolean;
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
function isExplicitSlashRequest(input: string, command: string): boolean {
|
|
556
|
+
const requested = slashCommandToken(input);
|
|
557
|
+
const actual = slashCommandToken(command);
|
|
558
|
+
if (!requested || !actual) return false;
|
|
559
|
+
return requested === actual;
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
function slashCommandToken(value: string): string | null {
|
|
563
|
+
const trimmed = value.trim();
|
|
564
|
+
if (!trimmed.startsWith('/')) return null;
|
|
565
|
+
|
|
566
|
+
for (let i = 1; i < trimmed.length; i++) {
|
|
567
|
+
if (trimmed[i] <= ' ') return trimmed.slice(0, i);
|
|
568
|
+
}
|
|
569
|
+
return trimmed;
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
function hasBrowserCompletionEvidence(execution: any): boolean {
|
|
573
|
+
if (!execution?.wasSuccessful) return false;
|
|
574
|
+
const output = execution.output || {};
|
|
575
|
+
return Boolean(output.pageDiff || output.code || output.playwrightGroupId);
|
|
576
|
+
}
|
package/src/ai/fisherman.ts
CHANGED
|
@@ -9,6 +9,7 @@ import { loop } from '../utils/loop.ts';
|
|
|
9
9
|
import type { Agent } from './agent.ts';
|
|
10
10
|
import { type FishermanResult, createFishermanTools } from './fisherman-tools.ts';
|
|
11
11
|
import type { Provider } from './provider.ts';
|
|
12
|
+
import { dataProtectionRules } from './rules.ts';
|
|
12
13
|
|
|
13
14
|
const MAX_ITERATIONS = 15;
|
|
14
15
|
const MAX_TOOL_ROUNDTRIPS = 5;
|
|
@@ -85,7 +86,7 @@ export class Fisherman implements Agent {
|
|
|
85
86
|
baseEndpoint: this.baseEndpoint,
|
|
86
87
|
});
|
|
87
88
|
|
|
88
|
-
const conversation = this.provider.startConversation(this.buildSystemPrompt(endpointList, scopeUrl), 'fisherman');
|
|
89
|
+
const conversation = this.provider.startConversation(this.buildSystemPrompt(endpointList, Object.keys(tools), scopeUrl), 'fisherman');
|
|
89
90
|
conversation.addUserText(this.buildTaskPrompt(instructions));
|
|
90
91
|
|
|
91
92
|
await loop(
|
|
@@ -187,7 +188,7 @@ export class Fisherman implements Agent {
|
|
|
187
188
|
return lines.join('\n');
|
|
188
189
|
}
|
|
189
190
|
|
|
190
|
-
private buildSystemPrompt(endpointList: string, scopeUrl?: string): string {
|
|
191
|
+
private buildSystemPrompt(endpointList: string, toolNames: string[], scopeUrl?: string): string {
|
|
191
192
|
const scopeBlock = scopeUrl ? `\n\nSCOPE: You are operating within ${scopeUrl}.\nAll created items must belong to this scope.` : '';
|
|
192
193
|
|
|
193
194
|
return dedent`
|
|
@@ -197,6 +198,11 @@ export class Fisherman implements Agent {
|
|
|
197
198
|
${endpointList}
|
|
198
199
|
${scopeBlock}
|
|
199
200
|
|
|
201
|
+
AVAILABLE TOOLS:
|
|
202
|
+
${toolNames.join(', ')}.
|
|
203
|
+
Use tool names exactly as listed. Do not invent aliases, combined names, or names with channel markers such as "commentary".
|
|
204
|
+
Match each tool input schema exactly. Do not invent parameter names or pass extra fields.
|
|
205
|
+
|
|
200
206
|
WORKFLOW:
|
|
201
207
|
1. Call getEndpointSpec to see the request body example for the endpoint
|
|
202
208
|
2. Make requests — the response automatically extracts IDs, names, and status fields
|
|
@@ -208,6 +214,8 @@ export class Fisherman implements Agent {
|
|
|
208
214
|
- Chain requests logically — create parent resources before children
|
|
209
215
|
- If a request fails, try once more with adjusted data before reporting failure
|
|
210
216
|
- Use realistic but unique data for each item (vary names, titles)
|
|
217
|
+
|
|
218
|
+
${dataProtectionRules}
|
|
211
219
|
`;
|
|
212
220
|
}
|
|
213
221
|
|
|
@@ -217,7 +225,10 @@ export class Fisherman implements Agent {
|
|
|
217
225
|
|
|
218
226
|
${instructions}
|
|
219
227
|
|
|
220
|
-
|
|
228
|
+
${dataProtectionRules}
|
|
229
|
+
|
|
230
|
+
If data preparation is allowed by these rules, execute the necessary API requests to create this data.
|
|
231
|
+
When done, call finish with the summary. If data preparation is forbidden, call stop with the reason.
|
|
221
232
|
`;
|
|
222
233
|
}
|
|
223
234
|
}
|
|
@@ -10,6 +10,8 @@ import { relativeToCwd } from '../../utils/next-steps.ts';
|
|
|
10
10
|
import { safeFilename } from '../../utils/strings.ts';
|
|
11
11
|
import { type Constructor, debugLog } from './mixin.ts';
|
|
12
12
|
|
|
13
|
+
const FATAL_SCREENCAST_STOP_ERRORS = /Target page, context or browser has been closed|Target closed|Session closed|Protocol error/i;
|
|
14
|
+
|
|
13
15
|
export interface ScreencastMethods {
|
|
14
16
|
attachScreencast(): void;
|
|
15
17
|
isScreencastActive(): boolean;
|
|
@@ -113,17 +115,24 @@ export function WithScreencast<T extends Constructor>(Base: T) {
|
|
|
113
115
|
if (!this.screencastActive) return;
|
|
114
116
|
const path = this.screencastPath;
|
|
115
117
|
const task = this.screencastTask;
|
|
118
|
+
let stopped = false;
|
|
116
119
|
try {
|
|
117
120
|
await this.screencastPage.screencast.stop();
|
|
121
|
+
stopped = true;
|
|
118
122
|
} catch (err) {
|
|
119
|
-
|
|
123
|
+
const message = (err as Error).message;
|
|
124
|
+
if (FATAL_SCREENCAST_STOP_ERRORS.test(message)) {
|
|
125
|
+
tag('operation').log('Screencast skipped: browser was closed before recording could be finalized');
|
|
126
|
+
} else {
|
|
127
|
+
tag('operation').log(`Screencast stop failed: ${message}`);
|
|
128
|
+
}
|
|
120
129
|
}
|
|
121
130
|
this.screencastActive = false;
|
|
122
131
|
this.screencastPage = null;
|
|
123
132
|
this.screencastPath = null;
|
|
124
133
|
this.screencastTask = null;
|
|
125
134
|
this.screencastLastChapter = null;
|
|
126
|
-
if (path) {
|
|
135
|
+
if (path && stopped) {
|
|
127
136
|
this.savedFiles.add(path);
|
|
128
137
|
task?.addArtifact?.(path);
|
|
129
138
|
tag('operation').log(`Saved screencast: ${relativeToCwd(path)}`);
|
package/src/ai/navigator.ts
CHANGED
|
@@ -136,6 +136,10 @@ class Navigator implements Agent {
|
|
|
136
136
|
}
|
|
137
137
|
|
|
138
138
|
async visit(url: string): Promise<void> {
|
|
139
|
+
return this.explorer.runWithBrowserRecovery('navigator.visit', () => this.visitOnce(url));
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
private async visitOnce(url: string): Promise<void> {
|
|
139
143
|
try {
|
|
140
144
|
const action = this.explorer.createAction();
|
|
141
145
|
|
|
@@ -170,7 +174,7 @@ class Navigator implements Agent {
|
|
|
170
174
|
throw new Error(`Navigation to ${url} failed: ${action.lastError?.message}`);
|
|
171
175
|
}
|
|
172
176
|
}
|
|
173
|
-
await
|
|
177
|
+
await this.explorer.capturePageWithScreenshot();
|
|
174
178
|
await this.hooksRunner.runAfterHook('navigator', url);
|
|
175
179
|
} catch (error) {
|
|
176
180
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
@@ -382,7 +386,7 @@ class Navigator implements Agent {
|
|
|
382
386
|
// URL did not transition to expectedUrl within timeout
|
|
383
387
|
}
|
|
384
388
|
}
|
|
385
|
-
const freshState = await
|
|
389
|
+
const freshState = await this.explorer.capturePageState();
|
|
386
390
|
const currentUrl = /^https?:\/\//i.test(expectedUrl) ? freshState.fullUrl || freshState.url || '' : freshState.url || '';
|
|
387
391
|
const urlMatches = this.isSameExpectedOrigin(expectedUrl, action.stateManager) && normalizeUrl(currentUrl) === normalizeUrl(expectedUrl);
|
|
388
392
|
const stateChanged = freshState.getStateHash() !== actionResult.getStateHash();
|
package/src/ai/pilot.ts
CHANGED
|
@@ -18,6 +18,7 @@ import type { Fisherman } from './fisherman.ts';
|
|
|
18
18
|
import type { Navigator } from './navigator.ts';
|
|
19
19
|
import type { Provider } from './provider.ts';
|
|
20
20
|
import type { Researcher } from './researcher.ts';
|
|
21
|
+
import { capabilityGroundingRule, dataProtectionRules } from './rules.ts';
|
|
21
22
|
import { isInteractive } from './task-agent.ts';
|
|
22
23
|
|
|
23
24
|
const CHECK_TOOLS = ['verify', 'see', 'research', 'context'];
|
|
@@ -67,7 +68,7 @@ export class Pilot implements Agent {
|
|
|
67
68
|
}
|
|
68
69
|
|
|
69
70
|
async reviewCompletion(task: Test, currentState: ActionResult, testerConversation: Conversation, navigator?: Navigator): Promise<boolean> {
|
|
70
|
-
const verdictType =
|
|
71
|
+
const verdictType = this.hasCompletionEvidence(task, currentState, testerConversation) ? 'finish' : 'stop';
|
|
71
72
|
return this.reviewDecision(verdictType, task, currentState, testerConversation, navigator);
|
|
72
73
|
}
|
|
73
74
|
|
|
@@ -86,14 +87,14 @@ export class Pilot implements Agent {
|
|
|
86
87
|
|
|
87
88
|
const sessionLog = this.formatSessionLog(testerConversation);
|
|
88
89
|
const stateContext = this.buildStateContext(currentState);
|
|
90
|
+
const successfulAssertions = this.formatSuccessfulAssertions(currentState, testerConversation);
|
|
89
91
|
const notes = task.notesToString() || 'No notes recorded.';
|
|
90
92
|
|
|
91
93
|
let visualAnalysis = '';
|
|
92
94
|
let screenshotState: ActionResult | null = null;
|
|
93
|
-
if (this.provider.hasVision()) {
|
|
95
|
+
if (type === 'finish' && this.provider.hasVision()) {
|
|
94
96
|
try {
|
|
95
|
-
|
|
96
|
-
screenshotState = await action.caputrePageWithScreenshot();
|
|
97
|
+
screenshotState = await this.explorer.capturePageWithScreenshot();
|
|
97
98
|
if (screenshotState.screenshot) {
|
|
98
99
|
visualAnalysis = (await this.researcher.answerQuestionAboutScreenshot(screenshotState, `Describe current page state relevant to: ${task.scenario}`)) || '';
|
|
99
100
|
}
|
|
@@ -125,6 +126,10 @@ export class Pilot implements Agent {
|
|
|
125
126
|
|
|
126
127
|
${this.formatExpectations(task)}
|
|
127
128
|
|
|
129
|
+
<successful_assertions>
|
|
130
|
+
${successfulAssertions || 'None'}
|
|
131
|
+
</successful_assertions>
|
|
132
|
+
|
|
128
133
|
<notes>
|
|
129
134
|
${notes}
|
|
130
135
|
</notes>
|
|
@@ -160,7 +165,7 @@ export class Pilot implements Agent {
|
|
|
160
165
|
try {
|
|
161
166
|
const response = await this.provider.generateObject(messages, schema, this.provider.getAgenticModel('pilot'), {
|
|
162
167
|
agentName: 'pilot',
|
|
163
|
-
|
|
168
|
+
telemetry: { functionId: 'pilot.reviewVerdict' },
|
|
164
169
|
});
|
|
165
170
|
|
|
166
171
|
const result = response?.object;
|
|
@@ -262,7 +267,7 @@ export class Pilot implements Agent {
|
|
|
262
267
|
try {
|
|
263
268
|
const response = await this.provider.generateObject(messages, schema, this.provider.getAgenticModel('pilot'), {
|
|
264
269
|
agentName: 'pilot',
|
|
265
|
-
|
|
270
|
+
telemetry: { functionId: 'pilot.reviewReset' },
|
|
266
271
|
});
|
|
267
272
|
|
|
268
273
|
const result = response?.object;
|
|
@@ -374,6 +379,8 @@ export class Pilot implements Agent {
|
|
|
374
379
|
You are Pilot — final decision maker for test pass/fail. Tester requested ${type}. Review the
|
|
375
380
|
evidence and commit to a verdict; "continue" only when evidence is genuinely insufficient.
|
|
376
381
|
|
|
382
|
+
${capabilityGroundingRule}
|
|
383
|
+
|
|
377
384
|
${this.buildSharedEvidenceRules(task)}
|
|
378
385
|
|
|
379
386
|
DECISION:
|
|
@@ -382,6 +389,8 @@ export class Pilot implements Agent {
|
|
|
382
389
|
Pick assertions DOM can express; for non-DOM regions (iframes, canvas, Monaco/CodeMirror), target a
|
|
383
390
|
stable landmark (container, ARIA role) instead of literal inner text. Your "pass" stands even if the
|
|
384
391
|
DOM assertion can't be made.
|
|
392
|
+
Do not pass when Tester achieved only a related navigation/filter/tab/status outcome instead of the
|
|
393
|
+
requested action, workflow, or entity detail goal.
|
|
385
394
|
- "fail": scenario was attempted but the goal was not achieved.
|
|
386
395
|
- "skipped": scenario is irrelevant to the app, OR systematic infrastructure failures (LLM errors,
|
|
387
396
|
crashes) prevented testing. NOT for "test failed to interact" — that's "fail" or "continue".
|
|
@@ -420,6 +429,10 @@ export class Pilot implements Agent {
|
|
|
420
429
|
|
|
421
430
|
FIRST: Decide if precondition() is needed.
|
|
422
431
|
|
|
432
|
+
${capabilityGroundingRule}
|
|
433
|
+
|
|
434
|
+
${dataProtectionRules}
|
|
435
|
+
|
|
423
436
|
Call precondition() WHEN:
|
|
424
437
|
- The scenario edits/deletes/modifies an item, and you want a DISPOSABLE item to act on safely
|
|
425
438
|
- The scenario needs specific data clearly NOT on the current page (e.g., items with specific statuses for filtering)
|
|
@@ -572,7 +585,7 @@ export class Pilot implements Agent {
|
|
|
572
585
|
maxToolRoundtrips: opts.maxToolRoundtrips ?? 0,
|
|
573
586
|
agentName: 'pilot',
|
|
574
587
|
stopWhen: opts.task ? () => opts.task!.hasFinished : undefined,
|
|
575
|
-
|
|
588
|
+
telemetry: { functionId },
|
|
576
589
|
});
|
|
577
590
|
const text = result?.response?.text || '';
|
|
578
591
|
const learned = (result?.toolExecutions || []).filter((e: any) => e.toolName === 'learnExperience' && e.output?.content).map((e: any) => e.output.content);
|
|
@@ -658,8 +671,7 @@ export class Pilot implements Agent {
|
|
|
658
671
|
private async checkDataAvailability(task: Test, requestedData: string, fishermanReason: string | undefined): Promise<string | null> {
|
|
659
672
|
if (!this.provider.hasVision()) return null;
|
|
660
673
|
|
|
661
|
-
const
|
|
662
|
-
const screenshotState = await action.caputrePageWithScreenshot().catch(() => null);
|
|
674
|
+
const screenshotState = await this.explorer.capturePageWithScreenshot().catch(() => null);
|
|
663
675
|
if (!screenshotState?.screenshot) return null;
|
|
664
676
|
|
|
665
677
|
const question = dedent`
|
|
@@ -882,6 +894,32 @@ export class Pilot implements Agent {
|
|
|
882
894
|
return parts.join('\n\n');
|
|
883
895
|
}
|
|
884
896
|
|
|
897
|
+
private hasCompletionEvidence(task: Test, currentState: ActionResult, testerConversation: Conversation): boolean {
|
|
898
|
+
if (task.hasAchievedAny()) return true;
|
|
899
|
+
return this.hasSuccessfulCheckEvidence(currentState, testerConversation);
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
private hasSuccessfulCheckEvidence(currentState: ActionResult, testerConversation: Conversation): boolean {
|
|
903
|
+
if (Object.values(currentState.verifications ?? {}).some(Boolean)) return true;
|
|
904
|
+
return testerConversation.getToolExecutions().some((t) => CHECK_TOOLS.includes(t.toolName) && t.wasSuccessful);
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
private formatSuccessfulAssertions(currentState: ActionResult, testerConversation: Conversation): string {
|
|
908
|
+
const lines: string[] = [];
|
|
909
|
+
for (const [assertion, passed] of Object.entries(currentState.verifications ?? {})) {
|
|
910
|
+
if (passed) lines.push(`PASS state verification: ${assertion}`);
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
for (const exec of testerConversation.getToolExecutions()) {
|
|
914
|
+
if (!CHECK_TOOLS.includes(exec.toolName) || !exec.wasSuccessful) continue;
|
|
915
|
+
const description = exec.input?.assertion || exec.input?.request || truncateJson(exec.input);
|
|
916
|
+
const result = exec.output?.message || exec.output?.analysis || exec.output?.result;
|
|
917
|
+
lines.push(`PASS ${exec.toolName}: ${description}${result ? ` -> ${result}` : ''}`);
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
return [...new Set(lines)].join('\n');
|
|
921
|
+
}
|
|
922
|
+
|
|
885
923
|
private formatActions(toolCalls: any[]): string {
|
|
886
924
|
return toolCalls
|
|
887
925
|
.map((t) => {
|
|
@@ -1000,9 +1038,13 @@ export class Pilot implements Agent {
|
|
|
1000
1038
|
|
|
1001
1039
|
Tester tools: click, pressKey, form, see, verify, context, research, xpathCheck, visualClick,
|
|
1002
1040
|
back, getVisitedStates, reset, stop, finish, record.
|
|
1041
|
+
Use tool names exactly as listed. Do not invent combined names, aliases, or names with channel markers such as "commentary".
|
|
1042
|
+
|
|
1043
|
+
${capabilityGroundingRule}
|
|
1003
1044
|
|
|
1004
1045
|
YOUR Pilot-only tool: precondition(description) — create FRESH disposable test data via API. Never
|
|
1005
1046
|
request users. Use when:
|
|
1047
|
+
|
|
1006
1048
|
- Scenario edits/deletes/modifies an item → create a disposable target ("1 post").
|
|
1007
1049
|
- Scenario needs auxiliary data (labels, categories, statuses for filtering).
|
|
1008
1050
|
- Tester failed because required data is missing (empty dropdown, empty list).
|
|
@@ -1012,6 +1054,8 @@ export class Pilot implements Agent {
|
|
|
1012
1054
|
- Current page already shows the exact data needed.
|
|
1013
1055
|
- Scenario tests navigation, search UI, or viewing.
|
|
1014
1056
|
|
|
1057
|
+
${dataProtectionRules}
|
|
1058
|
+
|
|
1015
1059
|
Describe WHAT to create, not what exists. RIGHT: precondition("1 test"). WRONG:
|
|
1016
1060
|
precondition("1 test suite named Updated Suite with existing tests"). Keep descriptions short.
|
|
1017
1061
|
|
package/src/ai/planner.ts
CHANGED
|
@@ -24,7 +24,7 @@ import type { Provider } from './provider.js';
|
|
|
24
24
|
import { POSSIBLE_SECTIONS, Researcher } from './researcher.ts';
|
|
25
25
|
import { findSimilarStateHash } from './researcher/cache.ts';
|
|
26
26
|
import { hasFocusedSection } from './researcher/focus.ts';
|
|
27
|
-
import {
|
|
27
|
+
import { capabilityGroundingRule, dataProtectionRules, fileUploadRule } from './rules.ts';
|
|
28
28
|
|
|
29
29
|
const debugLog = createDebug('explorbot:planner');
|
|
30
30
|
|
|
@@ -35,7 +35,7 @@ const TasksSchema = z.object({
|
|
|
35
35
|
z.object({
|
|
36
36
|
scenario: z.string().describe('A single sentence describing what to test'),
|
|
37
37
|
priority: z.enum(['critical', 'important', 'high', 'normal', 'low']).describe('Priority of the task based on business importance'),
|
|
38
|
-
startUrl: z.string().nullable().describe('Start URL for the test if different from plan URL
|
|
38
|
+
startUrl: z.string().nullable().describe('Start URL for the test if different from plan URL. Use only stable feature/list/detail pages, not transient create/edit/modal URLs unless the scenario specifically starts inside that form.'),
|
|
39
39
|
steps: z.array(z.string()).describe('List of steps to perform for this scenario. Each step should be a specific action (e.g., "Open the form", "Enter required data", "Submit the form"). Keep steps atomic and actionable.'),
|
|
40
40
|
expectedOutcomes: z
|
|
41
41
|
.array(z.string())
|
|
@@ -90,6 +90,9 @@ export class Planner extends PlannerBase implements Agent {
|
|
|
90
90
|
const featureDirective = feature
|
|
91
91
|
? `\n IMPORTANT: The user requested to focus specifically on: "${feature}"\n ALL scenarios MUST be directly related to this feature. Do not propose generic page tests unrelated to it.\n Use the user's exact wording to guide scenario names — do not substitute different entities (e.g., do not plan "suite" actions when user said "test").`
|
|
92
92
|
: '';
|
|
93
|
+
const focusExistingDataDirective = feature
|
|
94
|
+
? '\n If this focus asks for search, filter, tabs, sorting, or list behavior involving existing items, only use item names/values visible in the provided page research. If no concrete visible item names/values are present, do NOT propose scenarios that require an existing known item; propose no-match search, empty-state, clear-search, tab/filter empty-list, or other read-only list behavior instead.'
|
|
95
|
+
: '';
|
|
93
96
|
return dedent`
|
|
94
97
|
<role>
|
|
95
98
|
You are ISTQB certified senior manual QA planning exploratory testing session of a web application.
|
|
@@ -113,7 +116,7 @@ export class Planner extends PlannerBase implements Agent {
|
|
|
113
116
|
Bad: "Open delete dropdown" + "Confirm deletion" — these are ONE test, not two.
|
|
114
117
|
Bad: "Search for X" + "Verify search results" — searching and verifying is ONE test.
|
|
115
118
|
Bad: "Leave field empty" + "Click submit" — that's one negative test, not two.
|
|
116
|
-
If two scenarios cannot run independently (one requires the other to run first), merge them into one.${featureDirective}
|
|
119
|
+
If two scenarios cannot run independently (one requires the other to run first), merge them into one.${featureDirective}${focusExistingDataDirective}
|
|
117
120
|
</task>
|
|
118
121
|
|
|
119
122
|
${customPrompt || ''}
|
|
@@ -341,6 +344,11 @@ export class Planner extends PlannerBase implements Agent {
|
|
|
341
344
|
If a scenario needs existing records, recipients, results, notifications, or other target data, propose it only when that data is visible or API preconditions can create it.
|
|
342
345
|
If the page appears read-only, degraded, demo-limited, maintenance-like, or lacks write controls, prefer read-only scenarios such as opening panels, inspecting visible lists, filtering, searching, or verifying current state.
|
|
343
346
|
Do not assume hidden data exists just because a control is present.
|
|
347
|
+
For scenarios that act on existing items or search/filter by existing values, use only item names or values visible in research, visited pages, or prior observed flows.
|
|
348
|
+
If the list is empty or no concrete item names are visible, do not invent "known" or "existing" items. Prefer empty-state, no-match search, clear-search, or read-only list behavior scenarios.
|
|
349
|
+
Search, filter, sorting, tab, and list scenarios must start from a stable page where those controls are visible; avoid transient create/edit/new URLs unless the scenario tests that form.
|
|
350
|
+
For option values and list items, use only visible or previously observed data; do not add create/update/delete setup unless the user explicitly requests that workflow.
|
|
351
|
+
Detail-view scenarios must target visible data entities from list rows, cards, tree nodes, or detail links; do not use filter tabs, counters, status tabs, breadcrumbs, or navigation controls as detail targets.
|
|
344
352
|
DO NOT propose "verification-only" tests that merely open a UI element (modal, dropdown, panel) and check it exists.
|
|
345
353
|
Every test must complete a meaningful action that changes application state or produces a business outcome.
|
|
346
354
|
Opening a modal is NOT a test — performing an action INSIDE the modal IS a test.
|
|
@@ -351,7 +359,8 @@ export class Planner extends PlannerBase implements Agent {
|
|
|
351
359
|
Tests that only switch views, toggle filters, or paginate are LESS valuable — propose them only after data-changing tests are covered.
|
|
352
360
|
If multiple ways to create or modify data exist (different types, different forms), propose a separate test for each.
|
|
353
361
|
</priority_order>
|
|
354
|
-
${
|
|
362
|
+
${capabilityGroundingRule}
|
|
363
|
+
${dataProtectionRules}
|
|
355
364
|
${fileUploadRule}
|
|
356
365
|
</rules>
|
|
357
366
|
|
|
@@ -514,7 +523,9 @@ export class Planner extends PlannerBase implements Agent {
|
|
|
514
523
|
.join('\n')}
|
|
515
524
|
|
|
516
525
|
You MAY propose tests starting from these pages if they are relevant to the plan "${this.currentPlan.title}".
|
|
517
|
-
Set startUrl for such tests
|
|
526
|
+
Set startUrl for such tests only when the page is a stable feature/list/detail page.
|
|
527
|
+
Do not use create/edit/new/modal URLs as startUrl for scenarios that need the underlying page.
|
|
528
|
+
Ignore pages that belong to a different feature area.
|
|
518
529
|
</context_from_previous_tests>
|
|
519
530
|
|
|
520
531
|
Propose ONLY new scenarios that are NOT in the existing tests list.
|