explorbot 0.1.28 → 0.1.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +83 -245
- package/bin/explorbot-cli.ts +1 -0
- package/dist/bin/explorbot-cli.js +1 -0
- package/dist/package.json +8 -6
- package/dist/rules/navigator/verification-actions.md +2 -0
- package/dist/src/ai/fisherman.js +14 -3
- package/dist/src/ai/pilot.js +19 -4
- package/dist/src/ai/planner.js +16 -5
- package/dist/src/ai/provider.js +53 -18
- package/dist/src/ai/researcher.js +7 -1
- package/dist/src/ai/rules.js +44 -0
- package/dist/src/ai/tester.js +70 -7
- package/dist/src/ai/tools.js +67 -1
- package/dist/src/explorbot.js +7 -2
- package/dist/src/stats.js +16 -0
- package/dist/src/utils/aria.js +66 -6
- package/package.json +8 -6
- package/rules/navigator/verification-actions.md +2 -0
- package/src/ai/fisherman.ts +14 -3
- package/src/ai/pilot.ts +19 -4
- package/src/ai/planner.ts +16 -5
- package/src/ai/provider.ts +51 -19
- package/src/ai/researcher.ts +8 -1
- package/src/ai/rules.ts +46 -0
- package/src/ai/tester.ts +74 -7
- package/src/ai/tools.ts +80 -1
- package/src/config.ts +1 -0
- package/src/explorbot.ts +6 -2
- package/src/stats.ts +18 -0
- package/src/utils/aria.ts +63 -6
package/dist/src/ai/tools.js
CHANGED
|
@@ -11,7 +11,7 @@ import { WebElement } from "../utils/web-element.js";
|
|
|
11
11
|
import { sectionContextRule } from "./rules.js";
|
|
12
12
|
import { isInteractive } from "./task-agent.js";
|
|
13
13
|
const debugLog = createDebug('explorbot:tools');
|
|
14
|
-
export const CODECEPT_TOOLS = ['click', 'pressKey', 'form'];
|
|
14
|
+
export const CODECEPT_TOOLS = ['click', 'hover', 'pressKey', 'form'];
|
|
15
15
|
export const ASSERTION_TOOLS = ['verify'];
|
|
16
16
|
export function createCodeceptJSTools(explorer, task) {
|
|
17
17
|
const stateManager = explorer.getStateManager();
|
|
@@ -133,6 +133,71 @@ export function createCodeceptJSTools(explorer, task) {
|
|
|
133
133
|
}, action.lastError);
|
|
134
134
|
},
|
|
135
135
|
}),
|
|
136
|
+
hover: tool({
|
|
137
|
+
description: dedent `
|
|
138
|
+
Move the mouse cursor to an element to reveal hover-only controls.
|
|
139
|
+
|
|
140
|
+
Use this before clicking row actions, icon buttons, menus, or toolbars that appear only
|
|
141
|
+
when the user hovers a list item, table row, card, or tree node.
|
|
142
|
+
|
|
143
|
+
This tool ONLY accepts I.moveCursorTo(locator) commands. It does not click.
|
|
144
|
+
After hovering, use context(), see(), or click() the revealed control.
|
|
145
|
+
`,
|
|
146
|
+
inputSchema: z.object({
|
|
147
|
+
commands: z.array(z.string()).describe(dedent `
|
|
148
|
+
FALLBACK LOCATORS for ONE element to hover.
|
|
149
|
+
Order by reliability:
|
|
150
|
+
1. I.moveCursorTo(text, container)
|
|
151
|
+
2. I.moveCursorTo(ARIA, container)
|
|
152
|
+
3. I.moveCursorTo(CSS, container)
|
|
153
|
+
4. I.moveCursorTo(CSS) or I.moveCursorTo(XPath)
|
|
154
|
+
`),
|
|
155
|
+
explanation: z.string().describe('Why you are hovering this element'),
|
|
156
|
+
}),
|
|
157
|
+
execute: async ({ commands: rawCommands, explanation }) => {
|
|
158
|
+
const activeNote = task.startNote(explanation);
|
|
159
|
+
if (rawCommands.length === 0) {
|
|
160
|
+
activeNote.commit(TestResult.FAILED);
|
|
161
|
+
return failedToolResult('hover', 'No commands provided');
|
|
162
|
+
}
|
|
163
|
+
const invalidCommands = rawCommands.map((cmd) => cmd.trim()).filter((cmd) => cmd.startsWith('I.') && !cmd.startsWith('I.moveCursorTo'));
|
|
164
|
+
if (invalidCommands.length > 0) {
|
|
165
|
+
activeNote.commit(TestResult.FAILED);
|
|
166
|
+
return failedToolResult('hover', `Invalid commands: ${invalidCommands.join(', ')}. Hover tool only accepts I.moveCursorTo() commands.`, {
|
|
167
|
+
suggestion: 'Use click() to click elements, or form() for typing/selecting.',
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
const commands = rawCommands.map((cmd) => {
|
|
171
|
+
const trimmed = cmd.trim();
|
|
172
|
+
if (trimmed.startsWith('I.moveCursorTo'))
|
|
173
|
+
return trimmed;
|
|
174
|
+
return `I.moveCursorTo(${JSON.stringify(trimmed)})`;
|
|
175
|
+
});
|
|
176
|
+
const previousState = ActionResult.fromState(stateManager.getCurrentState());
|
|
177
|
+
const action = explorer.createAction();
|
|
178
|
+
const attempts = [];
|
|
179
|
+
for (const command of commands) {
|
|
180
|
+
const success = await action.attempt(command, explanation, true);
|
|
181
|
+
attempts.push({
|
|
182
|
+
command,
|
|
183
|
+
success,
|
|
184
|
+
...(action.lastError && { error: action.lastError.toString() }),
|
|
185
|
+
});
|
|
186
|
+
if (!success)
|
|
187
|
+
continue;
|
|
188
|
+
const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, command);
|
|
189
|
+
activeNote.commit(TestResult.PASSED);
|
|
190
|
+
return successToolResult('hover', { ...toolResult, attempts, code: command }, action);
|
|
191
|
+
}
|
|
192
|
+
const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, commands[0]);
|
|
193
|
+
activeNote.commit(TestResult.FAILED);
|
|
194
|
+
return failedToolResult('hover', 'All hover commands failed', {
|
|
195
|
+
...toolResult,
|
|
196
|
+
attempts,
|
|
197
|
+
suggestion: 'Use xpathCheck() to locate the row/card/tree node, or visualClick() if the hover target is only visually identifiable.',
|
|
198
|
+
}, action.lastError);
|
|
199
|
+
},
|
|
200
|
+
}),
|
|
136
201
|
pressKey: tool({
|
|
137
202
|
description: dedent `
|
|
138
203
|
Press a keyboard key or key combination. Use this for special keys like Enter, Escape, Tab, Arrow keys, or key combinations with modifiers.
|
|
@@ -404,6 +469,7 @@ export function createAgentTools({ explorer, researcher, navigator, experienceTr
|
|
|
404
469
|
Check the page contents based on current page state and screenshot.
|
|
405
470
|
This tool will trigger visual research to check the page contents on request.
|
|
406
471
|
Use it to verify the actions were performed correctly and the page is in the expected state.
|
|
472
|
+
Input schema has exactly one field: request. Do not pass text, reason, assertion, or other fields.
|
|
407
473
|
|
|
408
474
|
<example>
|
|
409
475
|
request: "Check current state of the Login form"
|
package/dist/src/explorbot.js
CHANGED
|
@@ -23,11 +23,12 @@ import { ConfigParser } from "./config.js";
|
|
|
23
23
|
import { ExperienceTracker } from "./experience-tracker.js";
|
|
24
24
|
import Explorer from "./explorer.js";
|
|
25
25
|
import { KnowledgeTracker } from "./knowledge-tracker.js";
|
|
26
|
+
import { Stats } from "./stats.js";
|
|
26
27
|
import { Plan } from "./test-plan.js";
|
|
27
|
-
import { parsePlansFromMarkdown } from "./utils/test-plan-markdown.js";
|
|
28
28
|
import { setVerboseMode, tag } from "./utils/logger.js";
|
|
29
29
|
import { relativeToCwd } from "./utils/next-steps.js";
|
|
30
30
|
import { sanitizeFilename } from "./utils/strings.js";
|
|
31
|
+
import { parsePlansFromMarkdown } from "./utils/test-plan-markdown.js";
|
|
31
32
|
export class ExplorBot {
|
|
32
33
|
configParser;
|
|
33
34
|
explorer;
|
|
@@ -427,7 +428,11 @@ export class ExplorBot {
|
|
|
427
428
|
tag('info').log(`Session report saved: ${relativeToCwd(filePath)}`);
|
|
428
429
|
const reporter = this.explorer?.getReporter();
|
|
429
430
|
if (reporter?.isEnabled()) {
|
|
430
|
-
|
|
431
|
+
let description = markdown;
|
|
432
|
+
const modelsTable = Stats.modelsTable(this.provider.getConfiguredModels());
|
|
433
|
+
if (modelsTable)
|
|
434
|
+
description = `${markdown}\n\n${modelsTable}`;
|
|
435
|
+
await reporter.setRunDescription(description);
|
|
431
436
|
}
|
|
432
437
|
this.lastReportedTestCount = tests.length;
|
|
433
438
|
}
|
package/dist/src/stats.js
CHANGED
|
@@ -39,6 +39,22 @@ export class Stats {
|
|
|
39
39
|
}
|
|
40
40
|
return String(num);
|
|
41
41
|
}
|
|
42
|
+
static modelsTable(roleModels) {
|
|
43
|
+
const usedModels = Object.entries(Stats.models).filter(([, tokens]) => tokens.total > 0);
|
|
44
|
+
if (usedModels.length === 0)
|
|
45
|
+
return '';
|
|
46
|
+
const rolesByModel = {};
|
|
47
|
+
for (const [role, model] of Object.entries(roleModels)) {
|
|
48
|
+
if (!rolesByModel[model])
|
|
49
|
+
rolesByModel[model] = [];
|
|
50
|
+
rolesByModel[model].push(role);
|
|
51
|
+
}
|
|
52
|
+
const rows = usedModels.map(([model, tokens]) => {
|
|
53
|
+
const roles = rolesByModel[model]?.join(', ') || '-';
|
|
54
|
+
return `| ${roles} | ${model} | ${Stats.humanizeTokens(tokens.total)} |`;
|
|
55
|
+
});
|
|
56
|
+
return ['## Models', '', '| Role | Model | Tokens |', '| --- | --- | --- |', ...rows].join('\n');
|
|
57
|
+
}
|
|
42
58
|
static hasActivity() {
|
|
43
59
|
if (Stats.tests > 0 || Stats.plans > 0 || Stats.researches > 0)
|
|
44
60
|
return true;
|
package/dist/src/utils/aria.js
CHANGED
|
@@ -334,6 +334,56 @@ const detectRenames = (prev, curr, prevTotals, currTotals) => {
|
|
|
334
334
|
}
|
|
335
335
|
return { added, removed };
|
|
336
336
|
};
|
|
337
|
+
// Interactive controls keep a stable role+name across a state flip; only an ARIA state
|
|
338
|
+
// attribute changes. Report those flips on their own line so the model always sees
|
|
339
|
+
// "now checked / now collapsed", in both directions, regardless of other page churn.
|
|
340
|
+
const STATE_WORDS = {
|
|
341
|
+
checked: { on: 'checked', off: 'unchecked' },
|
|
342
|
+
selected: { on: 'selected', off: 'unselected' },
|
|
343
|
+
pressed: { on: 'pressed', off: 'unpressed' },
|
|
344
|
+
expanded: { on: 'expanded', off: 'collapsed' },
|
|
345
|
+
};
|
|
346
|
+
const STATE_ATTRS = Object.keys(STATE_WORDS);
|
|
347
|
+
const stateWord = (attr, value) => {
|
|
348
|
+
if (attr === 'checked' && value === 'mixed')
|
|
349
|
+
return 'partially checked';
|
|
350
|
+
const words = STATE_WORDS[attr];
|
|
351
|
+
if (value === true || value === 'true')
|
|
352
|
+
return words.on;
|
|
353
|
+
return words.off;
|
|
354
|
+
};
|
|
355
|
+
// Pair entries by path; when role and name match but a state attr differs, it's a toggle.
|
|
356
|
+
const detectToggles = (prev, curr) => {
|
|
357
|
+
const toggled = [];
|
|
358
|
+
const togglePaths = new Set();
|
|
359
|
+
const currByPath = new Map(curr.map((e) => [e.path, e]));
|
|
360
|
+
for (const before of prev) {
|
|
361
|
+
const after = currByPath.get(before.path);
|
|
362
|
+
if (!after)
|
|
363
|
+
continue;
|
|
364
|
+
if (before.entry.role !== after.entry.role)
|
|
365
|
+
continue;
|
|
366
|
+
if (before.entry.name !== after.entry.name)
|
|
367
|
+
continue;
|
|
368
|
+
const transitions = [];
|
|
369
|
+
for (const attr of STATE_ATTRS) {
|
|
370
|
+
const was = stateWord(attr, before.entry[attr]);
|
|
371
|
+
const now = stateWord(attr, after.entry[attr]);
|
|
372
|
+
if (was === now)
|
|
373
|
+
continue;
|
|
374
|
+
transitions.push(`${was} -> ${now}`);
|
|
375
|
+
}
|
|
376
|
+
if (transitions.length === 0)
|
|
377
|
+
continue;
|
|
378
|
+
togglePaths.add(before.path);
|
|
379
|
+
let label = String(after.entry.role);
|
|
380
|
+
const name = after.entry.name;
|
|
381
|
+
if (typeof name === 'string' && name.trim())
|
|
382
|
+
label += ` "${name.trim()}"`;
|
|
383
|
+
toggled.push(`${label}: ${transitions.join(', ')}`);
|
|
384
|
+
}
|
|
385
|
+
return { toggled, togglePaths };
|
|
386
|
+
};
|
|
337
387
|
const TOP_DIFF_ITEMS = 10;
|
|
338
388
|
const formatDiffSection = (label, items) => {
|
|
339
389
|
const summary = countBy(items);
|
|
@@ -357,10 +407,17 @@ const formatDiffSection = (label, items) => {
|
|
|
357
407
|
}
|
|
358
408
|
return lines;
|
|
359
409
|
};
|
|
360
|
-
const formatDiff = (added, removed) => {
|
|
361
|
-
if (added.length === 0 && removed.length === 0)
|
|
410
|
+
const formatDiff = (added, removed, toggled) => {
|
|
411
|
+
if (added.length === 0 && removed.length === 0 && toggled.length === 0)
|
|
362
412
|
return null;
|
|
363
|
-
|
|
413
|
+
const sections = ['ariaDiff:'];
|
|
414
|
+
if (toggled.length > 0) {
|
|
415
|
+
sections.push(' toggled:');
|
|
416
|
+
for (const line of toggled)
|
|
417
|
+
sections.push(` - ${line}`);
|
|
418
|
+
}
|
|
419
|
+
sections.push(...formatDiffSection('added', added), ...formatDiffSection('removed', removed));
|
|
420
|
+
return sections.join('\n');
|
|
364
421
|
};
|
|
365
422
|
const CLOSE_OVERLAY_BUTTON_RE = /^close\s+(modal|dialog|popup|drawer|panel|sheet)\b/i;
|
|
366
423
|
const findOverlayByCloseButton = (nodeList) => {
|
|
@@ -424,13 +481,16 @@ export const diffAriaSnapshots = (previous, current) => {
|
|
|
424
481
|
tree = dropEmpty(tree);
|
|
425
482
|
return flatten(tree);
|
|
426
483
|
};
|
|
427
|
-
const
|
|
428
|
-
const
|
|
484
|
+
const prevAll = flat(previous);
|
|
485
|
+
const currAll = flat(current);
|
|
486
|
+
const { toggled, togglePaths } = detectToggles(prevAll, currAll);
|
|
487
|
+
const prev = prevAll.filter((e) => !togglePaths.has(e.path));
|
|
488
|
+
const curr = currAll.filter((e) => !togglePaths.has(e.path));
|
|
429
489
|
const prevTotals = countBy(prev.map((e) => e.summary));
|
|
430
490
|
const currTotals = countBy(curr.map((e) => e.summary));
|
|
431
491
|
const byCount = diffByCount(prevTotals, currTotals);
|
|
432
492
|
const renames = detectRenames(prev, curr, prevTotals, currTotals);
|
|
433
|
-
return formatDiff([...byCount.added, ...renames.added], [...byCount.removed, ...renames.removed]);
|
|
493
|
+
return formatDiff([...byCount.added, ...renames.added], [...byCount.removed, ...renames.removed], toggled);
|
|
434
494
|
};
|
|
435
495
|
export const detectFocusArea = (snapshot) => {
|
|
436
496
|
let tree = parseSnapshot(snapshot);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "explorbot",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.29",
|
|
4
4
|
"description": "CLI app built with React Ink, CodeceptJS, and Playwright",
|
|
5
5
|
"license": "Elastic-2.0",
|
|
6
6
|
"type": "module",
|
|
@@ -65,9 +65,10 @@
|
|
|
65
65
|
},
|
|
66
66
|
"author": "",
|
|
67
67
|
"dependencies": {
|
|
68
|
-
"@ai-sdk/anthropic": "^
|
|
69
|
-
"@ai-sdk/groq": "^
|
|
70
|
-
"@ai-sdk/openai": "^
|
|
68
|
+
"@ai-sdk/anthropic": "^4.0",
|
|
69
|
+
"@ai-sdk/groq": "^4.0",
|
|
70
|
+
"@ai-sdk/openai": "^4.0",
|
|
71
|
+
"@ai-sdk/otel": "^1.0.2",
|
|
71
72
|
"@axe-core/playwright": "^4.11.0",
|
|
72
73
|
"@codeceptjs/reflection": "^0.5.2",
|
|
73
74
|
"@faker-js/faker": "^10.4.0",
|
|
@@ -82,8 +83,8 @@
|
|
|
82
83
|
"@opentelemetry/sdk-trace-base": "^2.2.0",
|
|
83
84
|
"@opentelemetry/semantic-conventions": "^1.38.0",
|
|
84
85
|
"@scalar/openapi-parser": "^0.25.6",
|
|
85
|
-
"@testomatio/reporter": "^2.
|
|
86
|
-
"ai": "^
|
|
86
|
+
"@testomatio/reporter": "^2.9.1",
|
|
87
|
+
"ai": "^7.0.2",
|
|
87
88
|
"axe-core": "^4.11.1",
|
|
88
89
|
"bash-tool": "^1.3.15",
|
|
89
90
|
"cli-highlight": "^2.1.11",
|
|
@@ -110,6 +111,7 @@
|
|
|
110
111
|
"parse5": "^8.0.0",
|
|
111
112
|
"playwright": "^1.60",
|
|
112
113
|
"react": "^19.1.1",
|
|
114
|
+
"sambanova-ai-provider": "^1.2.2",
|
|
113
115
|
"strip-ansi": "^7.1.2",
|
|
114
116
|
"turndown": "^7.2.1",
|
|
115
117
|
"unique-names-generator": "^4.7.1",
|
|
@@ -113,6 +113,8 @@ For input field values, ALWAYS use I.seeInField() — never check value via CSS
|
|
|
113
113
|
Prefer text locators (label, name, placeholder) for form fields: I.seeInField('Search', 'value') over I.seeInField('input[name="search"]', 'value').
|
|
114
114
|
Only use locators that exist in the provided HTML or ARIA snapshot.
|
|
115
115
|
Verify exact conditions, not approximate matches.
|
|
116
|
+
When the claim contains a concrete quoted value, generated assertion code MUST include that whole value. Do not shorten names, IDs, titles, emails, URLs, or other user-created values.
|
|
117
|
+
For exact visible text, prefer a text assertion scoped to a specific container or an ARIA locator with the complete text. Partial text is not valid evidence for a claim about the full value.
|
|
116
118
|
NEVER use `:has-text(...)` inside a seeElement/dontSeeElement locator. Checking text inside an element is the job of I.see(text, context) — the `:has-text()` form duplicates that capability with a fragile selector.
|
|
117
119
|
NEVER emit two assertions that check the same fact with different shapes. `I.see(text, locator)` and `I.seeElement("<locator>:has-text('text')")` verify the same thing — pick one (prefer I.see). One claim, one assertion.
|
|
118
120
|
</verification_rules>
|
package/src/ai/fisherman.ts
CHANGED
|
@@ -9,6 +9,7 @@ import { loop } from '../utils/loop.ts';
|
|
|
9
9
|
import type { Agent } from './agent.ts';
|
|
10
10
|
import { type FishermanResult, createFishermanTools } from './fisherman-tools.ts';
|
|
11
11
|
import type { Provider } from './provider.ts';
|
|
12
|
+
import { dataProtectionRules } from './rules.ts';
|
|
12
13
|
|
|
13
14
|
const MAX_ITERATIONS = 15;
|
|
14
15
|
const MAX_TOOL_ROUNDTRIPS = 5;
|
|
@@ -85,7 +86,7 @@ export class Fisherman implements Agent {
|
|
|
85
86
|
baseEndpoint: this.baseEndpoint,
|
|
86
87
|
});
|
|
87
88
|
|
|
88
|
-
const conversation = this.provider.startConversation(this.buildSystemPrompt(endpointList, scopeUrl), 'fisherman');
|
|
89
|
+
const conversation = this.provider.startConversation(this.buildSystemPrompt(endpointList, Object.keys(tools), scopeUrl), 'fisherman');
|
|
89
90
|
conversation.addUserText(this.buildTaskPrompt(instructions));
|
|
90
91
|
|
|
91
92
|
await loop(
|
|
@@ -187,7 +188,7 @@ export class Fisherman implements Agent {
|
|
|
187
188
|
return lines.join('\n');
|
|
188
189
|
}
|
|
189
190
|
|
|
190
|
-
private buildSystemPrompt(endpointList: string, scopeUrl?: string): string {
|
|
191
|
+
private buildSystemPrompt(endpointList: string, toolNames: string[], scopeUrl?: string): string {
|
|
191
192
|
const scopeBlock = scopeUrl ? `\n\nSCOPE: You are operating within ${scopeUrl}.\nAll created items must belong to this scope.` : '';
|
|
192
193
|
|
|
193
194
|
return dedent`
|
|
@@ -197,6 +198,11 @@ export class Fisherman implements Agent {
|
|
|
197
198
|
${endpointList}
|
|
198
199
|
${scopeBlock}
|
|
199
200
|
|
|
201
|
+
AVAILABLE TOOLS:
|
|
202
|
+
${toolNames.join(', ')}.
|
|
203
|
+
Use tool names exactly as listed. Do not invent aliases, combined names, or names with channel markers such as "commentary".
|
|
204
|
+
Match each tool input schema exactly. Do not invent parameter names or pass extra fields.
|
|
205
|
+
|
|
200
206
|
WORKFLOW:
|
|
201
207
|
1. Call getEndpointSpec to see the request body example for the endpoint
|
|
202
208
|
2. Make requests — the response automatically extracts IDs, names, and status fields
|
|
@@ -208,6 +214,8 @@ export class Fisherman implements Agent {
|
|
|
208
214
|
- Chain requests logically — create parent resources before children
|
|
209
215
|
- If a request fails, try once more with adjusted data before reporting failure
|
|
210
216
|
- Use realistic but unique data for each item (vary names, titles)
|
|
217
|
+
|
|
218
|
+
${dataProtectionRules}
|
|
211
219
|
`;
|
|
212
220
|
}
|
|
213
221
|
|
|
@@ -217,7 +225,10 @@ export class Fisherman implements Agent {
|
|
|
217
225
|
|
|
218
226
|
${instructions}
|
|
219
227
|
|
|
220
|
-
|
|
228
|
+
${dataProtectionRules}
|
|
229
|
+
|
|
230
|
+
If data preparation is allowed by these rules, execute the necessary API requests to create this data.
|
|
231
|
+
When done, call finish with the summary. If data preparation is forbidden, call stop with the reason.
|
|
221
232
|
`;
|
|
222
233
|
}
|
|
223
234
|
}
|
package/src/ai/pilot.ts
CHANGED
|
@@ -18,6 +18,7 @@ import type { Fisherman } from './fisherman.ts';
|
|
|
18
18
|
import type { Navigator } from './navigator.ts';
|
|
19
19
|
import type { Provider } from './provider.ts';
|
|
20
20
|
import type { Researcher } from './researcher.ts';
|
|
21
|
+
import { capabilityGroundingRule, dataProtectionRules } from './rules.ts';
|
|
21
22
|
import { isInteractive } from './task-agent.ts';
|
|
22
23
|
|
|
23
24
|
const CHECK_TOOLS = ['verify', 'see', 'research', 'context'];
|
|
@@ -91,7 +92,7 @@ export class Pilot implements Agent {
|
|
|
91
92
|
|
|
92
93
|
let visualAnalysis = '';
|
|
93
94
|
let screenshotState: ActionResult | null = null;
|
|
94
|
-
if (this.provider.hasVision()) {
|
|
95
|
+
if (type === 'finish' && this.provider.hasVision()) {
|
|
95
96
|
try {
|
|
96
97
|
screenshotState = await this.explorer.capturePageWithScreenshot();
|
|
97
98
|
if (screenshotState.screenshot) {
|
|
@@ -164,7 +165,7 @@ export class Pilot implements Agent {
|
|
|
164
165
|
try {
|
|
165
166
|
const response = await this.provider.generateObject(messages, schema, this.provider.getAgenticModel('pilot'), {
|
|
166
167
|
agentName: 'pilot',
|
|
167
|
-
|
|
168
|
+
telemetry: { functionId: 'pilot.reviewVerdict' },
|
|
168
169
|
});
|
|
169
170
|
|
|
170
171
|
const result = response?.object;
|
|
@@ -266,7 +267,7 @@ export class Pilot implements Agent {
|
|
|
266
267
|
try {
|
|
267
268
|
const response = await this.provider.generateObject(messages, schema, this.provider.getAgenticModel('pilot'), {
|
|
268
269
|
agentName: 'pilot',
|
|
269
|
-
|
|
270
|
+
telemetry: { functionId: 'pilot.reviewReset' },
|
|
270
271
|
});
|
|
271
272
|
|
|
272
273
|
const result = response?.object;
|
|
@@ -378,6 +379,8 @@ export class Pilot implements Agent {
|
|
|
378
379
|
You are Pilot — final decision maker for test pass/fail. Tester requested ${type}. Review the
|
|
379
380
|
evidence and commit to a verdict; "continue" only when evidence is genuinely insufficient.
|
|
380
381
|
|
|
382
|
+
${capabilityGroundingRule}
|
|
383
|
+
|
|
381
384
|
${this.buildSharedEvidenceRules(task)}
|
|
382
385
|
|
|
383
386
|
DECISION:
|
|
@@ -386,6 +389,8 @@ export class Pilot implements Agent {
|
|
|
386
389
|
Pick assertions DOM can express; for non-DOM regions (iframes, canvas, Monaco/CodeMirror), target a
|
|
387
390
|
stable landmark (container, ARIA role) instead of literal inner text. Your "pass" stands even if the
|
|
388
391
|
DOM assertion can't be made.
|
|
392
|
+
Do not pass when Tester achieved only a related navigation/filter/tab/status outcome instead of the
|
|
393
|
+
requested action, workflow, or entity detail goal.
|
|
389
394
|
- "fail": scenario was attempted but the goal was not achieved.
|
|
390
395
|
- "skipped": scenario is irrelevant to the app, OR systematic infrastructure failures (LLM errors,
|
|
391
396
|
crashes) prevented testing. NOT for "test failed to interact" — that's "fail" or "continue".
|
|
@@ -424,6 +429,10 @@ export class Pilot implements Agent {
|
|
|
424
429
|
|
|
425
430
|
FIRST: Decide if precondition() is needed.
|
|
426
431
|
|
|
432
|
+
${capabilityGroundingRule}
|
|
433
|
+
|
|
434
|
+
${dataProtectionRules}
|
|
435
|
+
|
|
427
436
|
Call precondition() WHEN:
|
|
428
437
|
- The scenario edits/deletes/modifies an item, and you want a DISPOSABLE item to act on safely
|
|
429
438
|
- The scenario needs specific data clearly NOT on the current page (e.g., items with specific statuses for filtering)
|
|
@@ -576,7 +585,7 @@ export class Pilot implements Agent {
|
|
|
576
585
|
maxToolRoundtrips: opts.maxToolRoundtrips ?? 0,
|
|
577
586
|
agentName: 'pilot',
|
|
578
587
|
stopWhen: opts.task ? () => opts.task!.hasFinished : undefined,
|
|
579
|
-
|
|
588
|
+
telemetry: { functionId },
|
|
580
589
|
});
|
|
581
590
|
const text = result?.response?.text || '';
|
|
582
591
|
const learned = (result?.toolExecutions || []).filter((e: any) => e.toolName === 'learnExperience' && e.output?.content).map((e: any) => e.output.content);
|
|
@@ -1029,9 +1038,13 @@ export class Pilot implements Agent {
|
|
|
1029
1038
|
|
|
1030
1039
|
Tester tools: click, pressKey, form, see, verify, context, research, xpathCheck, visualClick,
|
|
1031
1040
|
back, getVisitedStates, reset, stop, finish, record.
|
|
1041
|
+
Use tool names exactly as listed. Do not invent combined names, aliases, or names with channel markers such as "commentary".
|
|
1042
|
+
|
|
1043
|
+
${capabilityGroundingRule}
|
|
1032
1044
|
|
|
1033
1045
|
YOUR Pilot-only tool: precondition(description) — create FRESH disposable test data via API. Never
|
|
1034
1046
|
request users. Use when:
|
|
1047
|
+
|
|
1035
1048
|
- Scenario edits/deletes/modifies an item → create a disposable target ("1 post").
|
|
1036
1049
|
- Scenario needs auxiliary data (labels, categories, statuses for filtering).
|
|
1037
1050
|
- Tester failed because required data is missing (empty dropdown, empty list).
|
|
@@ -1041,6 +1054,8 @@ export class Pilot implements Agent {
|
|
|
1041
1054
|
- Current page already shows the exact data needed.
|
|
1042
1055
|
- Scenario tests navigation, search UI, or viewing.
|
|
1043
1056
|
|
|
1057
|
+
${dataProtectionRules}
|
|
1058
|
+
|
|
1044
1059
|
Describe WHAT to create, not what exists. RIGHT: precondition("1 test"). WRONG:
|
|
1045
1060
|
precondition("1 test suite named Updated Suite with existing tests"). Keep descriptions short.
|
|
1046
1061
|
|
package/src/ai/planner.ts
CHANGED
|
@@ -24,7 +24,7 @@ import type { Provider } from './provider.js';
|
|
|
24
24
|
import { POSSIBLE_SECTIONS, Researcher } from './researcher.ts';
|
|
25
25
|
import { findSimilarStateHash } from './researcher/cache.ts';
|
|
26
26
|
import { hasFocusedSection } from './researcher/focus.ts';
|
|
27
|
-
import {
|
|
27
|
+
import { capabilityGroundingRule, dataProtectionRules, fileUploadRule } from './rules.ts';
|
|
28
28
|
|
|
29
29
|
const debugLog = createDebug('explorbot:planner');
|
|
30
30
|
|
|
@@ -35,7 +35,7 @@ const TasksSchema = z.object({
|
|
|
35
35
|
z.object({
|
|
36
36
|
scenario: z.string().describe('A single sentence describing what to test'),
|
|
37
37
|
priority: z.enum(['critical', 'important', 'high', 'normal', 'low']).describe('Priority of the task based on business importance'),
|
|
38
|
-
startUrl: z.string().nullable().describe('Start URL for the test if different from plan URL
|
|
38
|
+
startUrl: z.string().nullable().describe('Start URL for the test if different from plan URL. Use only stable feature/list/detail pages, not transient create/edit/modal URLs unless the scenario specifically starts inside that form.'),
|
|
39
39
|
steps: z.array(z.string()).describe('List of steps to perform for this scenario. Each step should be a specific action (e.g., "Open the form", "Enter required data", "Submit the form"). Keep steps atomic and actionable.'),
|
|
40
40
|
expectedOutcomes: z
|
|
41
41
|
.array(z.string())
|
|
@@ -90,6 +90,9 @@ export class Planner extends PlannerBase implements Agent {
|
|
|
90
90
|
const featureDirective = feature
|
|
91
91
|
? `\n IMPORTANT: The user requested to focus specifically on: "${feature}"\n ALL scenarios MUST be directly related to this feature. Do not propose generic page tests unrelated to it.\n Use the user's exact wording to guide scenario names — do not substitute different entities (e.g., do not plan "suite" actions when user said "test").`
|
|
92
92
|
: '';
|
|
93
|
+
const focusExistingDataDirective = feature
|
|
94
|
+
? '\n If this focus asks for search, filter, tabs, sorting, or list behavior involving existing items, only use item names/values visible in the provided page research. If no concrete visible item names/values are present, do NOT propose scenarios that require an existing known item; propose no-match search, empty-state, clear-search, tab/filter empty-list, or other read-only list behavior instead.'
|
|
95
|
+
: '';
|
|
93
96
|
return dedent`
|
|
94
97
|
<role>
|
|
95
98
|
You are ISTQB certified senior manual QA planning exploratory testing session of a web application.
|
|
@@ -113,7 +116,7 @@ export class Planner extends PlannerBase implements Agent {
|
|
|
113
116
|
Bad: "Open delete dropdown" + "Confirm deletion" — these are ONE test, not two.
|
|
114
117
|
Bad: "Search for X" + "Verify search results" — searching and verifying is ONE test.
|
|
115
118
|
Bad: "Leave field empty" + "Click submit" — that's one negative test, not two.
|
|
116
|
-
If two scenarios cannot run independently (one requires the other to run first), merge them into one.${featureDirective}
|
|
119
|
+
If two scenarios cannot run independently (one requires the other to run first), merge them into one.${featureDirective}${focusExistingDataDirective}
|
|
117
120
|
</task>
|
|
118
121
|
|
|
119
122
|
${customPrompt || ''}
|
|
@@ -341,6 +344,11 @@ export class Planner extends PlannerBase implements Agent {
|
|
|
341
344
|
If a scenario needs existing records, recipients, results, notifications, or other target data, propose it only when that data is visible or API preconditions can create it.
|
|
342
345
|
If the page appears read-only, degraded, demo-limited, maintenance-like, or lacks write controls, prefer read-only scenarios such as opening panels, inspecting visible lists, filtering, searching, or verifying current state.
|
|
343
346
|
Do not assume hidden data exists just because a control is present.
|
|
347
|
+
For scenarios that act on existing items or search/filter by existing values, use only item names or values visible in research, visited pages, or prior observed flows.
|
|
348
|
+
If the list is empty or no concrete item names are visible, do not invent "known" or "existing" items. Prefer empty-state, no-match search, clear-search, or read-only list behavior scenarios.
|
|
349
|
+
Search, filter, sorting, tab, and list scenarios must start from a stable page where those controls are visible; avoid transient create/edit/new URLs unless the scenario tests that form.
|
|
350
|
+
For option values and list items, use only visible or previously observed data; do not add create/update/delete setup unless the user explicitly requests that workflow.
|
|
351
|
+
Detail-view scenarios must target visible data entities from list rows, cards, tree nodes, or detail links; do not use filter tabs, counters, status tabs, breadcrumbs, or navigation controls as detail targets.
|
|
344
352
|
DO NOT propose "verification-only" tests that merely open a UI element (modal, dropdown, panel) and check it exists.
|
|
345
353
|
Every test must complete a meaningful action that changes application state or produces a business outcome.
|
|
346
354
|
Opening a modal is NOT a test — performing an action INSIDE the modal IS a test.
|
|
@@ -351,7 +359,8 @@ export class Planner extends PlannerBase implements Agent {
|
|
|
351
359
|
Tests that only switch views, toggle filters, or paginate are LESS valuable — propose them only after data-changing tests are covered.
|
|
352
360
|
If multiple ways to create or modify data exist (different types, different forms), propose a separate test for each.
|
|
353
361
|
</priority_order>
|
|
354
|
-
${
|
|
362
|
+
${capabilityGroundingRule}
|
|
363
|
+
${dataProtectionRules}
|
|
355
364
|
${fileUploadRule}
|
|
356
365
|
</rules>
|
|
357
366
|
|
|
@@ -514,7 +523,9 @@ export class Planner extends PlannerBase implements Agent {
|
|
|
514
523
|
.join('\n')}
|
|
515
524
|
|
|
516
525
|
You MAY propose tests starting from these pages if they are relevant to the plan "${this.currentPlan.title}".
|
|
517
|
-
Set startUrl for such tests
|
|
526
|
+
Set startUrl for such tests only when the page is a stable feature/list/detail page.
|
|
527
|
+
Do not use create/edit/new/modal URLs as startUrl for scenarios that need the underlying page.
|
|
528
|
+
Ignore pages that belong to a different feature area.
|
|
518
529
|
</context_from_previous_tests>
|
|
519
530
|
|
|
520
531
|
Propose ONLY new scenarios that are NOT in the existing tests list.
|