explorbot 0.1.12 → 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/explorbot-cli.ts +21 -21
- package/dist/bin/explorbot-cli.js +3 -3
- package/dist/package.json +4 -2
- package/dist/rules/researcher/container-rules.md +2 -0
- package/dist/src/action-result.js +2 -1
- package/dist/src/action.js +3 -8
- package/dist/src/ai/captain.js +0 -2
- package/dist/src/ai/conversation.js +20 -4
- package/dist/src/ai/driller.js +1108 -0
- package/dist/src/ai/historian/utils.js +8 -1
- package/dist/src/ai/pilot.js +214 -267
- package/dist/src/ai/provider.js +25 -12
- package/dist/src/ai/quartermaster.js +2 -2
- package/dist/src/ai/rules.js +5 -5
- package/dist/src/ai/session-analyst.js +122 -0
- package/dist/src/ai/tester.js +69 -22
- package/dist/src/ai/tools.js +19 -4
- package/dist/src/commands/base-command.js +6 -6
- package/dist/src/commands/drill-command.js +3 -2
- package/dist/src/commands/exit-command.js +1 -0
- package/dist/src/commands/explore-command.js +9 -2
- package/dist/src/components/AddRule.js +1 -1
- package/dist/src/components/StatusPane.js +6 -1
- package/dist/src/experience-tracker.js +9 -0
- package/dist/src/explorbot.js +48 -8
- package/dist/src/explorer.js +11 -13
- package/dist/src/reporter.js +105 -4
- package/dist/src/state-manager.js +4 -3
- package/dist/src/stats.js +7 -1
- package/dist/src/test-plan.js +47 -3
- package/dist/src/utils/aria.js +354 -529
- package/dist/src/utils/hooks-runner.js +2 -8
- package/dist/src/utils/html.js +371 -0
- package/dist/src/utils/unique-names.js +12 -1
- package/dist/src/utils/url-matcher.js +6 -1
- package/dist/src/utils/web-element.js +27 -24
- package/dist/src/utils/xpath.js +1 -1
- package/package.json +4 -2
- package/rules/researcher/container-rules.md +2 -0
- package/src/action-result.ts +2 -1
- package/src/action.ts +3 -10
- package/src/ai/captain.ts +0 -2
- package/src/ai/conversation.ts +21 -4
- package/src/ai/driller.ts +1194 -0
- package/src/ai/historian/utils.ts +8 -1
- package/src/ai/pilot.ts +215 -265
- package/src/ai/provider.ts +24 -12
- package/src/ai/quartermaster.ts +2 -2
- package/src/ai/rules.ts +5 -5
- package/src/ai/session-analyst.ts +139 -0
- package/src/ai/tester.ts +63 -20
- package/src/ai/tools.ts +18 -4
- package/src/commands/base-command.ts +6 -6
- package/src/commands/drill-command.ts +3 -2
- package/src/commands/exit-command.ts +1 -0
- package/src/commands/explore-command.ts +10 -2
- package/src/components/AddRule.tsx +1 -1
- package/src/components/StatusPane.tsx +6 -3
- package/src/config.ts +4 -0
- package/src/experience-tracker.ts +9 -0
- package/src/explorbot.ts +55 -10
- package/src/explorer.ts +10 -12
- package/src/reporter.ts +108 -4
- package/src/state-manager.ts +4 -3
- package/src/stats.ts +10 -1
- package/src/test-plan.ts +62 -3
- package/src/utils/aria.ts +367 -537
- package/src/utils/hooks-runner.ts +2 -6
- package/src/utils/html.ts +381 -0
- package/src/utils/unique-names.ts +13 -0
- package/src/utils/url-matcher.ts +5 -1
- package/src/utils/web-element.ts +31 -28
- package/src/utils/xpath.ts +1 -1
- package/dist/src/ai/bosun.js +0 -456
- package/src/ai/bosun.ts +0 -571
package/src/ai/provider.ts
CHANGED
|
@@ -19,6 +19,15 @@ const responseLog = createDebug('explorbot:provider:in');
|
|
|
19
19
|
class AiError extends Error {}
|
|
20
20
|
export class ContextLengthError extends Error {}
|
|
21
21
|
|
|
22
|
+
function extractCachedTokens(usage: any): number {
|
|
23
|
+
if (!usage) return 0;
|
|
24
|
+
const direct = usage.cachedInputTokens ?? usage.inputTokenDetails?.cacheReadTokens;
|
|
25
|
+
if (typeof direct === 'number') return direct;
|
|
26
|
+
const raw = usage.raw;
|
|
27
|
+
const fromRaw = raw?.prompt_tokens_details?.cached_tokens ?? raw?.promptTokensDetails?.cachedTokens;
|
|
28
|
+
return typeof fromRaw === 'number' ? fromRaw : 0;
|
|
29
|
+
}
|
|
30
|
+
|
|
22
31
|
function rejectAfterIdle(ms: number, signal: { cancelled: boolean }): Promise<never> {
|
|
23
32
|
return new Promise((_, reject) => {
|
|
24
33
|
const tick = () => {
|
|
@@ -265,9 +274,10 @@ export class Provider {
|
|
|
265
274
|
|
|
266
275
|
if (response.usage) {
|
|
267
276
|
Stats.recordTokens(options.agentName || 'unknown', modelName, {
|
|
268
|
-
input: response.usage.promptTokens
|
|
269
|
-
output: response.usage.completionTokens
|
|
270
|
-
total: response.usage.totalTokens
|
|
277
|
+
input: response.usage.inputTokens ?? response.usage.promptTokens ?? 0,
|
|
278
|
+
output: response.usage.outputTokens ?? response.usage.completionTokens ?? 0,
|
|
279
|
+
total: response.usage.totalTokens ?? 0,
|
|
280
|
+
cached: extractCachedTokens(response.usage),
|
|
271
281
|
});
|
|
272
282
|
}
|
|
273
283
|
|
|
@@ -355,9 +365,10 @@ export class Provider {
|
|
|
355
365
|
|
|
356
366
|
if (response.usage) {
|
|
357
367
|
Stats.recordTokens(options.agentName || 'unknown', modelName, {
|
|
358
|
-
input: response.usage.promptTokens
|
|
359
|
-
output: response.usage.completionTokens
|
|
360
|
-
total: response.usage.totalTokens
|
|
368
|
+
input: response.usage.inputTokens ?? response.usage.promptTokens ?? 0,
|
|
369
|
+
output: response.usage.outputTokens ?? response.usage.completionTokens ?? 0,
|
|
370
|
+
total: response.usage.totalTokens ?? 0,
|
|
371
|
+
cached: extractCachedTokens(response.usage),
|
|
361
372
|
});
|
|
362
373
|
}
|
|
363
374
|
|
|
@@ -428,9 +439,10 @@ export class Provider {
|
|
|
428
439
|
|
|
429
440
|
if (response.usage) {
|
|
430
441
|
Stats.recordTokens(options.agentName || 'unknown', modelName, {
|
|
431
|
-
input: response.usage.promptTokens
|
|
432
|
-
output: response.usage.completionTokens
|
|
433
|
-
total: response.usage.totalTokens
|
|
442
|
+
input: response.usage.inputTokens ?? response.usage.promptTokens ?? 0,
|
|
443
|
+
output: response.usage.outputTokens ?? response.usage.completionTokens ?? 0,
|
|
444
|
+
total: response.usage.totalTokens ?? 0,
|
|
445
|
+
cached: extractCachedTokens(response.usage),
|
|
434
446
|
});
|
|
435
447
|
}
|
|
436
448
|
|
|
@@ -625,9 +637,9 @@ export class Provider {
|
|
|
625
637
|
|
|
626
638
|
if (response.usage) {
|
|
627
639
|
Stats.recordTokens('vision', this.getModelName(this.config.visionModel), {
|
|
628
|
-
input: response.usage.promptTokens
|
|
629
|
-
output: response.usage.completionTokens
|
|
630
|
-
total: response.usage.totalTokens
|
|
640
|
+
input: response.usage.inputTokens ?? response.usage.promptTokens ?? 0,
|
|
641
|
+
output: response.usage.outputTokens ?? response.usage.completionTokens ?? 0,
|
|
642
|
+
total: response.usage.totalTokens ?? 0,
|
|
631
643
|
});
|
|
632
644
|
}
|
|
633
645
|
|
package/src/ai/quartermaster.ts
CHANGED
|
@@ -240,11 +240,11 @@ Focus on what would confuse a real user or caused the agent to make mistakes.`;
|
|
|
240
240
|
const criticalViolations = report.axeViolations.filter((v) => v.impact === 'critical' || v.impact === 'serious');
|
|
241
241
|
for (const v of criticalViolations.slice(0, 3)) {
|
|
242
242
|
const nodeHtml = v.nodes[0]?.html.slice(0, 100) || '';
|
|
243
|
-
task.
|
|
243
|
+
task.addVerificationDetail(`🔴 A11Y [${v.impact}] ${v.id}: ${v.description} — ${nodeHtml}`);
|
|
244
244
|
}
|
|
245
245
|
|
|
246
246
|
for (const issue of report.semanticIssues.slice(0, 3)) {
|
|
247
|
-
task.
|
|
247
|
+
task.addVerificationDetail(`💡 UX [${issue.type}] ${issue.element}: ${issue.suggestion}`);
|
|
248
248
|
}
|
|
249
249
|
}
|
|
250
250
|
|
package/src/ai/rules.ts
CHANGED
|
@@ -241,6 +241,8 @@ export function multipleTabsRule(tabs: Array<{ url: string; title: string }>): s
|
|
|
241
241
|
|
|
242
242
|
export const actionRule = dedent`
|
|
243
243
|
<actions>
|
|
244
|
+
\`faker\` (from @faker-js/faker) is available inside I.* calls for generating data, e.g. I.fillField('Bio', faker.lorem.paragraphs(5)).
|
|
245
|
+
|
|
244
246
|
### I.click
|
|
245
247
|
|
|
246
248
|
clicks on the element by its locator
|
|
@@ -282,11 +284,9 @@ export const actionRule = dedent`
|
|
|
282
284
|
I.fillField('Description', 'Hello world', '.editor'); // works for rich text / code editors too
|
|
283
285
|
</example>
|
|
284
286
|
|
|
285
|
-
I.fillField handles plain inputs, textareas, contenteditable regions, and rich text / code editors
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
Do NOT open the editor with raw JS (executeScript, page.evaluate), do NOT dispatch synthetic events,
|
|
289
|
-
do NOT call the editor's own API (monaco.editor.setValue, view.dispatch, etc.) to write text.
|
|
287
|
+
I.fillField handles plain inputs, textareas, contenteditable regions, and rich text / code editors transparently.
|
|
288
|
+
ALWAYS use I.fillField for rich text / code editors — target the editor container or its nearest label/heading with a normal locator.
|
|
289
|
+
If I.fillField does not work, I.type into the focused element is the fallback.
|
|
290
290
|
|
|
291
291
|
### I.type
|
|
292
292
|
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import dedent from 'dedent';
|
|
4
|
+
import { outputPath } from '../config.ts';
|
|
5
|
+
import { Stats } from '../stats.ts';
|
|
6
|
+
import type { Test } from '../test-plan.ts';
|
|
7
|
+
import type { Agent } from './agent.ts';
|
|
8
|
+
import type { Provider } from './provider.ts';
|
|
9
|
+
|
|
10
|
+
export class SessionAnalyst implements Agent {
|
|
11
|
+
emoji = '🧐';
|
|
12
|
+
private provider: Provider;
|
|
13
|
+
|
|
14
|
+
constructor(provider: Provider) {
|
|
15
|
+
this.provider = provider;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
async analyze(tests: Test[]): Promise<string> {
|
|
19
|
+
const eligible = tests.filter((t) => t.startTime != null);
|
|
20
|
+
if (eligible.length === 0) return '';
|
|
21
|
+
|
|
22
|
+
const model = this.provider.getAgenticModel('analyst');
|
|
23
|
+
const customPrompt = this.provider.getSystemPromptForAgent('analyst', undefined);
|
|
24
|
+
|
|
25
|
+
const systemPrompt = dedent`
|
|
26
|
+
You write a TERSE end-of-session report. Reader is a developer who wants to UNDERSTAND THE FEATURE — what works, what is broken, what is unclear. Every word must earn its place.
|
|
27
|
+
|
|
28
|
+
Output MARKDOWN. No JSON, no preamble, no closing summary.
|
|
29
|
+
|
|
30
|
+
NO EMOJI. No 🔴 🟡 🟢 ✅, no escape sequences like \\u2705. Use plain text severity tags: [High], [Medium], [Low] for defects.
|
|
31
|
+
|
|
32
|
+
## Reporting unit
|
|
33
|
+
|
|
34
|
+
Report at the level of FEATURES / FLOWS / PAGES. Tests are evidence, not the unit. Several tests covering the same flow → ONE entry citing all of them.
|
|
35
|
+
|
|
36
|
+
## Walk every test
|
|
37
|
+
|
|
38
|
+
PASSED test: did all steps run, was the goal actually verified, did the user-visible goal happen? All yes → contributes to What works. Any no → Execution issue (false positive).
|
|
39
|
+
|
|
40
|
+
FAILED test, first match wins: (1) goal achieved but mis-verified → Execution. (2) automation failure (locator/timeout/loop/modal/a11y) → Execution. (3) bad preconditions or data → Execution. (4) wrong URL/environment → Execution. (5) app contradicted expected outcome → Defect.
|
|
41
|
+
|
|
42
|
+
Crucial distinction: "the app misbehaved" vs "the automation could not interact with the app". ONLY the first is a Defect. If the automation gives up before the app responds — timeout, retries exhausted, dead loop / loop detected, could not click or find an element — that is an Execution issue regardless of what the log calls it. Failure inside the automation ≠ failure inside the product.
|
|
43
|
+
|
|
44
|
+
A solitary failure where adjacent tests on the same feature passed → Execution, not Defect.
|
|
45
|
+
|
|
46
|
+
## Severity (defects only)
|
|
47
|
+
[High] blocks a core flow · [Medium] degrades a flow but workaround exists · [Low] cosmetic / edge case
|
|
48
|
+
|
|
49
|
+
## Format
|
|
50
|
+
|
|
51
|
+
# Session Analysis
|
|
52
|
+
|
|
53
|
+
<ONE or TWO sentences describing the FEATURE STATE — what was explored, whether the core flow holds, what the standout problem is. NO test counts, NO "N tests run". Talk about the product, not the run.>
|
|
54
|
+
|
|
55
|
+
## Coverage
|
|
56
|
+
- Pages: <paths>
|
|
57
|
+
- Features: <capabilities>
|
|
58
|
+
|
|
59
|
+
## What works
|
|
60
|
+
- **<feature>** — #2, #7, #8
|
|
61
|
+
|
|
62
|
+
## Defects
|
|
63
|
+
|
|
64
|
+
### [Medium] <plain-English bug title>
|
|
65
|
+
Affects: #3, #5
|
|
66
|
+
Reproduce:
|
|
67
|
+
1. <concrete UI step>
|
|
68
|
+
2. <next>
|
|
69
|
+
Evidence: <one short observation>
|
|
70
|
+
|
|
71
|
+
## UX issues
|
|
72
|
+
- **<feature>** — <what's confusing> (#7)
|
|
73
|
+
|
|
74
|
+
## Execution Issues
|
|
75
|
+
- **#2 <scenario>** — <≤10 words, what was unreliable>
|
|
76
|
+
|
|
77
|
+
## Brevity rules
|
|
78
|
+
|
|
79
|
+
- Headline: 2 sentences MAX. About the FEATURE, not the run. No counts, no "N tests", no "this session". Banned words: "exercised", "comprehensive", "notably", "this session", "module", "targeted", "covered creation".
|
|
80
|
+
- What works: feature name + test refs. NO parentheticals, NO caveats. If there's a caveat, the entry doesn't belong here.
|
|
81
|
+
- Defect title is the BUG ("Search returns non-matching results"), never the scenario name.
|
|
82
|
+
- Reproduce steps are imperative one-liners drawn from the log.
|
|
83
|
+
- Evidence is one short factual observation. Never quote the \`result\` field.
|
|
84
|
+
- Execution Issues: ONE line per test, ≤10 words, plain. Examples: "passed vacuously, no list assertion", "no file upload step in log", "dead loop on Save click". No prefixes, no nested explanation.
|
|
85
|
+
- Omit any empty section.
|
|
86
|
+
- Section order: Coverage → What works → Defects (severity desc) → UX issues → Execution Issues.
|
|
87
|
+
|
|
88
|
+
${customPrompt || ''}
|
|
89
|
+
`;
|
|
90
|
+
|
|
91
|
+
const userPayload = dedent`
|
|
92
|
+
${eligible.length} tests were executed in this session.
|
|
93
|
+
|
|
94
|
+
${eligible.map((t, i) => this.serializeTest(t, i + 1)).join('\n\n')}
|
|
95
|
+
`;
|
|
96
|
+
|
|
97
|
+
const response = await this.provider.chat(
|
|
98
|
+
[
|
|
99
|
+
{ role: 'system', content: systemPrompt },
|
|
100
|
+
{ role: 'user', content: userPayload },
|
|
101
|
+
],
|
|
102
|
+
model,
|
|
103
|
+
{ agentName: 'analyst' }
|
|
104
|
+
);
|
|
105
|
+
|
|
106
|
+
return decodeEscapes((response?.text || '').trim());
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
writeReport(markdown: string): string {
|
|
110
|
+
const filePath = outputPath('reports', `${Stats.sessionLabel()}.md`);
|
|
111
|
+
const dir = path.dirname(filePath);
|
|
112
|
+
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
|
113
|
+
writeFileSync(filePath, markdown);
|
|
114
|
+
return filePath;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
private serializeTest(test: Test, ref: number): string {
|
|
118
|
+
const log = test
|
|
119
|
+
.getLog()
|
|
120
|
+
.slice(-30)
|
|
121
|
+
.map((entry) => ` - [${entry.type}] ${entry.content}`)
|
|
122
|
+
.join('\n');
|
|
123
|
+
|
|
124
|
+
return dedent`
|
|
125
|
+
<test ref="#${ref}">
|
|
126
|
+
url: ${test.startUrl || '/'}
|
|
127
|
+
scenario: ${test.scenario}
|
|
128
|
+
result: ${test.result || 'unknown'}
|
|
129
|
+
expected: ${test.expected.join(' | ') || '(none)'}
|
|
130
|
+
log:
|
|
131
|
+
${log}
|
|
132
|
+
</test>
|
|
133
|
+
`;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function decodeEscapes(text: string): string {
|
|
138
|
+
return text.replace(/\\u\{([0-9a-fA-F]+)\}/g, (_, hex) => String.fromCodePoint(Number.parseInt(hex, 16))).replace(/\\u([0-9a-fA-F]{4})/g, (_, hex) => String.fromCodePoint(Number.parseInt(hex, 16)));
|
|
139
|
+
}
|
package/src/ai/tester.ts
CHANGED
|
@@ -64,6 +64,8 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
64
64
|
private pageStateHash: string | null = null;
|
|
65
65
|
private pageActionResult: ActionResult | null = null;
|
|
66
66
|
private hooksRunner: HooksRunner;
|
|
67
|
+
private seenUiMapUrls = new Set<string>();
|
|
68
|
+
private lastAnalyzedStateHash: string | null = null;
|
|
67
69
|
|
|
68
70
|
constructor(explorer: Explorer, provider: Provider, researcher: Researcher, navigator: Navigator, agentTools?: any) {
|
|
69
71
|
super();
|
|
@@ -104,7 +106,7 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
104
106
|
}
|
|
105
107
|
|
|
106
108
|
private get progressCheckInterval(): number {
|
|
107
|
-
return (this.explorer.getConfig().ai?.agents?.tester as any)?.progressCheckInterval ??
|
|
109
|
+
return (this.explorer.getConfig().ai?.agents?.tester as any)?.progressCheckInterval ?? 3;
|
|
108
110
|
}
|
|
109
111
|
|
|
110
112
|
getConversation(): Conversation | null {
|
|
@@ -123,6 +125,8 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
123
125
|
this.previousStateHash = null;
|
|
124
126
|
this.pageStateHash = null;
|
|
125
127
|
this.pageActionResult = null;
|
|
128
|
+
this.seenUiMapUrls.clear();
|
|
129
|
+
this.lastAnalyzedStateHash = null;
|
|
126
130
|
this.explorer.getStateManager().clearHistory();
|
|
127
131
|
this.resetFailureCount();
|
|
128
132
|
this.pilot?.reset();
|
|
@@ -147,14 +151,20 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
147
151
|
const initialState = ActionResult.fromState(state);
|
|
148
152
|
|
|
149
153
|
const conversation = this.provider.startConversation(this.getSystemMessage(), 'tester');
|
|
154
|
+
conversation.markLastMessageCacheable();
|
|
150
155
|
this.currentConversation = conversation;
|
|
151
156
|
|
|
152
157
|
const outputDir = ConfigParser.getInstance().getOutputDir();
|
|
153
158
|
this.executionLogFile = join(outputDir, `tester_${task.sessionName}.md`);
|
|
154
159
|
// Note: Markdown saving functionality removed from Conversation class
|
|
155
160
|
|
|
156
|
-
const
|
|
157
|
-
conversation.addUserText(
|
|
161
|
+
const scenarioBlock = this.buildScenarioBlock(task, initialState);
|
|
162
|
+
conversation.addUserText(scenarioBlock);
|
|
163
|
+
conversation.markLastMessageCacheable();
|
|
164
|
+
conversation.protectPrefix(conversation.messages.length);
|
|
165
|
+
|
|
166
|
+
const pageContext = await this.reinjectContextIfNeeded(1, initialState);
|
|
167
|
+
if (pageContext) conversation.addUserText(pageContext);
|
|
158
168
|
|
|
159
169
|
return await Observability.run(
|
|
160
170
|
`test: ${task.scenario}`,
|
|
@@ -177,6 +187,12 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
177
187
|
if (this.pilot) {
|
|
178
188
|
try {
|
|
179
189
|
const plan = await this.pilot.planTest(task, initialState);
|
|
190
|
+
if (task.hasFinished) {
|
|
191
|
+
offFailedRequest?.();
|
|
192
|
+
page?.off('pageerror', onPageError);
|
|
193
|
+
page?.off('console', onConsoleMessage);
|
|
194
|
+
return { success: task.isSuccessful };
|
|
195
|
+
}
|
|
180
196
|
if (plan) {
|
|
181
197
|
conversation.addUserText(`Pilot's test plan:\n${plan}\n\nFollow this plan while executing the test.`);
|
|
182
198
|
}
|
|
@@ -200,13 +216,15 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
200
216
|
debugLog(`Navigating to ${task.startUrl}`);
|
|
201
217
|
await this.explorer.visit(task.startUrl!);
|
|
202
218
|
|
|
203
|
-
const
|
|
219
|
+
const startState = this.explorer.getStateManager().getCurrentState();
|
|
220
|
+
if (startState) task.addUrlNote(startState);
|
|
221
|
+
const currentUrl = startState?.url || task.startUrl || '';
|
|
204
222
|
await this.hooksRunner.runBeforeHook('tester', currentUrl);
|
|
205
223
|
|
|
206
224
|
const offStateChange = this.explorer.getStateManager().onStateChange((event: StateTransition) => {
|
|
207
225
|
if (task.hasFinished) return;
|
|
208
226
|
if (event.toState?.url === event.fromState?.url) return;
|
|
209
|
-
task.
|
|
227
|
+
if (event.toState) task.addUrlNote(event.toState, event.fromState || undefined);
|
|
210
228
|
task.states.push(event.toState);
|
|
211
229
|
});
|
|
212
230
|
|
|
@@ -253,13 +271,13 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
253
271
|
`);
|
|
254
272
|
}
|
|
255
273
|
|
|
256
|
-
conversation.cleanupTag('page_aria', '...cleaned aria snapshot...',
|
|
274
|
+
conversation.cleanupTag('page_aria', '...cleaned aria snapshot...', 1);
|
|
257
275
|
conversation.cleanupTag('page_html', '...cleaned HTML snapshot...', 1);
|
|
258
276
|
conversation.cleanupTag('experience', '...cleaned experience...', 1);
|
|
259
277
|
conversation.cleanupTag('applied_experience', '...cleaned past experience...', 1);
|
|
260
278
|
conversation.cleanupTag('page_ui_map', '...cleaned UI map...', 1);
|
|
261
279
|
conversation.cleanupTag('page_ui_map_overlay', '...cleaned UI overlay...', 1);
|
|
262
|
-
conversation.compactToolResults(
|
|
280
|
+
conversation.compactToolResults(2);
|
|
263
281
|
|
|
264
282
|
if (iteration > 1) {
|
|
265
283
|
const isNewPage = this.previousUrl !== null && this.previousUrl !== currentState.url;
|
|
@@ -268,18 +286,19 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
268
286
|
nextStep += await this.prepareInstructionsForNextStep(task);
|
|
269
287
|
|
|
270
288
|
if (isNewPage && this.pilot) {
|
|
271
|
-
const guidance = await this.pilot.reviewNewPage(task, currentState);
|
|
289
|
+
const guidance = await this.pilot.reviewNewPage(task, currentState, conversation);
|
|
272
290
|
if (guidance) nextStep += `\n\n${guidance}`;
|
|
273
|
-
} else if ((iteration
|
|
291
|
+
} else if (this.shouldAnalyzeProgress(iteration, currentState) && this.pilot) {
|
|
274
292
|
const guidance = await this.pilot.analyzeProgress(task, currentState, conversation);
|
|
275
293
|
if (guidance) nextStep += `\n\n${guidance}`;
|
|
276
294
|
this.consecutiveFailures = 0;
|
|
295
|
+
this.lastAnalyzedStateHash = currentState.hash;
|
|
277
296
|
}
|
|
278
297
|
conversation.addUserText(nextStep);
|
|
279
298
|
}
|
|
280
299
|
|
|
281
300
|
const result = await this.provider.invokeConversation(conversation, tools, {
|
|
282
|
-
maxToolRoundtrips:
|
|
301
|
+
maxToolRoundtrips: 3,
|
|
283
302
|
toolChoice: 'required',
|
|
284
303
|
stopWhen: () => task.hasFinished,
|
|
285
304
|
});
|
|
@@ -421,6 +440,14 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
421
440
|
};
|
|
422
441
|
}
|
|
423
442
|
|
|
443
|
+
private shouldAnalyzeProgress(iteration: number, currentState: ActionResult): boolean {
|
|
444
|
+
if (this.consecutiveFailures >= 3) return true;
|
|
445
|
+
if (this.consecutiveEmptyResults >= 2) return true;
|
|
446
|
+
if (iteration % this.progressCheckInterval !== 0) return false;
|
|
447
|
+
if (this.lastAnalyzedStateHash === currentState.hash) return false;
|
|
448
|
+
return true;
|
|
449
|
+
}
|
|
450
|
+
|
|
424
451
|
private async prepareInstructionsForNextStep(task: Test): Promise<string> {
|
|
425
452
|
let outcomeStatus = dedent`
|
|
426
453
|
<task>
|
|
@@ -463,6 +490,8 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
463
490
|
|
|
464
491
|
let context = '';
|
|
465
492
|
|
|
493
|
+
const focusArea = detectFocusArea(currentState.ariaSnapshot);
|
|
494
|
+
|
|
466
495
|
const focusedElement = extractFocusedElement(currentState.ariaSnapshot);
|
|
467
496
|
if (focusedElement) {
|
|
468
497
|
const isTextInput = ['textbox', 'combobox', 'searchbox'].includes(focusedElement.role);
|
|
@@ -480,6 +509,18 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
480
509
|
`;
|
|
481
510
|
}
|
|
482
511
|
|
|
512
|
+
if (focusArea.detected) {
|
|
513
|
+
const areaName = focusArea.name ? ` "${focusArea.name}"` : '';
|
|
514
|
+
context += dedent`
|
|
515
|
+
<focus_scope>
|
|
516
|
+
A ${focusArea.type}${areaName} is currently open above the page.
|
|
517
|
+
Scope all interactions to elements inside this ${focusArea.type}.
|
|
518
|
+
Page navigation, filters, and tabs that exist outside it are not actionable while it is open and may share names or roles with elements inside it — prefer the locator inside the ${focusArea.type}.
|
|
519
|
+
Use <page_aria> to confirm the element you target is actually inside the ${focusArea.type}.
|
|
520
|
+
</focus_scope>
|
|
521
|
+
`;
|
|
522
|
+
}
|
|
523
|
+
|
|
483
524
|
if (currentState.isInsideIframe) {
|
|
484
525
|
const iframeInfo = currentState.iframeURL || this.explorer.getCurrentIframeInfo() || 'iframe context active';
|
|
485
526
|
context += dedent`
|
|
@@ -497,17 +538,21 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
497
538
|
}
|
|
498
539
|
|
|
499
540
|
if (isNewUrl) {
|
|
541
|
+
const alreadySeenUiMap = this.seenUiMapUrls.has(currentUrl);
|
|
500
542
|
let research = '';
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
543
|
+
if (!alreadySeenUiMap) {
|
|
544
|
+
try {
|
|
545
|
+
research = await this.researcher.research(currentState);
|
|
546
|
+
} catch (err) {
|
|
547
|
+
if (!(err instanceof ErrorPageError)) throw err;
|
|
548
|
+
tag('warning').log(`Research skipped: ${err.message}`);
|
|
549
|
+
}
|
|
506
550
|
}
|
|
507
551
|
this.pageStateHash = currentStateHash;
|
|
508
552
|
this.pageActionResult = currentState;
|
|
509
553
|
let uiMapSection = '';
|
|
510
554
|
if (research) {
|
|
555
|
+
this.seenUiMapUrls.add(currentUrl);
|
|
511
556
|
uiMapSection = dedent`
|
|
512
557
|
|
|
513
558
|
Page UI Map
|
|
@@ -516,6 +561,8 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
516
561
|
${research}
|
|
517
562
|
</page_ui_map>
|
|
518
563
|
`;
|
|
564
|
+
} else if (alreadySeenUiMap) {
|
|
565
|
+
uiMapSection = `\n\n<page_ui_map>UI map for ${currentUrl} was shown earlier in this session — refer to it above.</page_ui_map>`;
|
|
519
566
|
}
|
|
520
567
|
|
|
521
568
|
context += dedent`
|
|
@@ -539,7 +586,6 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
539
586
|
return context;
|
|
540
587
|
}
|
|
541
588
|
|
|
542
|
-
const focusArea = detectFocusArea(currentState.ariaSnapshot);
|
|
543
589
|
if (focusArea.detected && focusArea.name && this.pageStateHash && this.pageActionResult) {
|
|
544
590
|
const overlaySection = await this.researcher.researchOverlay(currentState, this.pageActionResult, this.pageStateHash);
|
|
545
591
|
if (overlaySection) {
|
|
@@ -727,9 +773,8 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
727
773
|
`;
|
|
728
774
|
}
|
|
729
775
|
|
|
730
|
-
private
|
|
776
|
+
private buildScenarioBlock(task: Test, actionResult: ActionResult): string {
|
|
731
777
|
const knowledge = this.getKnowledge(actionResult);
|
|
732
|
-
const pageContext = await this.reinjectContextIfNeeded(1, actionResult);
|
|
733
778
|
|
|
734
779
|
return dedent`
|
|
735
780
|
<task>
|
|
@@ -757,8 +802,6 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
757
802
|
${this.buildAvailableFiles()}
|
|
758
803
|
|
|
759
804
|
${knowledge}
|
|
760
|
-
|
|
761
|
-
${pageContext}
|
|
762
805
|
`;
|
|
763
806
|
}
|
|
764
807
|
|
package/src/ai/tools.ts
CHANGED
|
@@ -510,7 +510,7 @@ export function createAgentTools({
|
|
|
510
510
|
}
|
|
511
511
|
|
|
512
512
|
return successToolResult('see', {
|
|
513
|
-
analysis: analysisResult,
|
|
513
|
+
analysis: cap(analysisResult, ANALYSIS_OUTPUT_CAP),
|
|
514
514
|
message: `Successfully analyzed screenshot for: ${request}`,
|
|
515
515
|
suggestion: 'Visual confirmation is valid evidence for test results. Use record() to note the visual findings.',
|
|
516
516
|
});
|
|
@@ -559,8 +559,8 @@ export function createAgentTools({
|
|
|
559
559
|
url: currentState.url,
|
|
560
560
|
title: currentState.title,
|
|
561
561
|
suggestion: 'If not enough context received, call see() to visually identify elements in page contents',
|
|
562
|
-
aria,
|
|
563
|
-
html,
|
|
562
|
+
aria: cap(aria, ARIA_OUTPUT_CAP),
|
|
563
|
+
html: cap(html, HTML_OUTPUT_CAP),
|
|
564
564
|
reminder: 'Context provided. Do not call context() again until you perform actions or suspect page changed.',
|
|
565
565
|
});
|
|
566
566
|
} catch (error) {
|
|
@@ -657,7 +657,7 @@ export function createAgentTools({
|
|
|
657
657
|
|
|
658
658
|
return successToolResult('research', {
|
|
659
659
|
analysis: researchResult,
|
|
660
|
-
aria: ActionResult.fromState(currentState).getInteractiveARIA(),
|
|
660
|
+
aria: cap(ActionResult.fromState(currentState).getInteractiveARIA(), ARIA_OUTPUT_CAP),
|
|
661
661
|
message: `Successfully researched page: ${currentState.url}.`,
|
|
662
662
|
suggestion: dedent`
|
|
663
663
|
You received comprehensive UI map report. Use it to understand the page structure and navigate to the elements.
|
|
@@ -1001,6 +1001,16 @@ export function createAgentTools({
|
|
|
1001
1001
|
|
|
1002
1002
|
const PAGE_DIFF_SUGGESTION = 'Analyze page diff. htmlParts shows what changed and WHERE — each part has a container selector. Use the container as context when clicking elements from the diff.';
|
|
1003
1003
|
|
|
1004
|
+
const ARIA_OUTPUT_CAP = 4000;
|
|
1005
|
+
const HTML_OUTPUT_CAP = 6000;
|
|
1006
|
+
const ANALYSIS_OUTPUT_CAP = 2000;
|
|
1007
|
+
|
|
1008
|
+
function cap(text: string | undefined | null, max: number): string {
|
|
1009
|
+
if (!text) return '';
|
|
1010
|
+
if (text.length <= max) return text;
|
|
1011
|
+
return `${text.slice(0, max)}\n[...truncated; ${text.length - max} chars omitted...]`;
|
|
1012
|
+
}
|
|
1013
|
+
|
|
1004
1014
|
function transformContainsCommand(command: string): string {
|
|
1005
1015
|
if (!command.includes(':contains(')) return command;
|
|
1006
1016
|
|
|
@@ -1044,8 +1054,12 @@ function successToolResult(action: string, data?: Record<string, any>, source?:
|
|
|
1044
1054
|
if (data?.pageDiff) {
|
|
1045
1055
|
let suggestion = PAGE_DIFF_SUGGESTION;
|
|
1046
1056
|
const ariaChanges = data.pageDiff.ariaChanges || '';
|
|
1057
|
+
const urlChanged = data.pageDiff.urlChanged === true;
|
|
1058
|
+
const hasHtmlParts = Array.isArray(data.pageDiff.htmlParts) && data.pageDiff.htmlParts.length > 0;
|
|
1047
1059
|
if (countAriaChanges(ariaChanges) >= 50) {
|
|
1048
1060
|
suggestion = `MAJOR PAGE CHANGE. Page entered a different mode. Check htmlParts and iframes in pageDiff before next action. ${suggestion}`;
|
|
1061
|
+
} else if (!urlChanged && !ariaChanges && !hasHtmlParts) {
|
|
1062
|
+
suggestion = 'Action ran without error but produced no observable change (URL, ARIA and HTML all unchanged). The locator likely matched a non-interactive ancestor or an element outside the intended control. Re-locate via xpathCheck() or verify with see() before treating this as success.';
|
|
1049
1063
|
} else if (ariaChanges.includes('heading') && ariaChanges.includes('added')) {
|
|
1050
1064
|
suggestion += ' WARNING: A new panel or modal may have appeared. If this was not the intended action, close it and try a different element.';
|
|
1051
1065
|
}
|
|
@@ -38,17 +38,17 @@ export abstract class BaseCommand {
|
|
|
38
38
|
printSuggestions(): void {
|
|
39
39
|
if (this.suggestions.length === 0) return;
|
|
40
40
|
const prefix = isInteractive() ? '/' : `${getCliName()} `;
|
|
41
|
-
|
|
42
|
-
|
|
41
|
+
const commandWidth = this.suggestions.reduce((max, s) => (s.command ? Math.max(max, prefix.length + s.command.length) : max), 0);
|
|
42
|
+
const lines = [chalk.bold('Suggested:')];
|
|
43
43
|
for (const { command, hint } of this.suggestions) {
|
|
44
|
-
tag('info').log('');
|
|
45
44
|
if (!command) {
|
|
46
|
-
|
|
45
|
+
lines.push(` ${chalk.dim(hint)}`);
|
|
47
46
|
continue;
|
|
48
47
|
}
|
|
49
|
-
|
|
50
|
-
|
|
48
|
+
const cmd = `${prefix}${command}`.padEnd(commandWidth);
|
|
49
|
+
lines.push(` ${chalk.yellow(cmd)} ${chalk.dim(hint)}`);
|
|
51
50
|
}
|
|
51
|
+
tag('info').log(lines.join('\n'));
|
|
52
52
|
}
|
|
53
53
|
|
|
54
54
|
protected parseArgs(args: string): { opts: Record<string, string | boolean>; args: string[] } {
|
|
@@ -3,6 +3,7 @@ import { BaseCommand, type Suggestion } from './base-command.js';
|
|
|
3
3
|
export class DrillCommand extends BaseCommand {
|
|
4
4
|
name = 'drill';
|
|
5
5
|
description = 'Drill all components on current page to learn interactions';
|
|
6
|
+
aliases = ['driller'];
|
|
6
7
|
suggestions: Suggestion[] = [
|
|
7
8
|
{ command: 'research', hint: 'see UI map first' },
|
|
8
9
|
{ command: 'navigate <page>', hint: 'go to another page' },
|
|
@@ -17,7 +18,7 @@ export class DrillCommand extends BaseCommand {
|
|
|
17
18
|
throw new Error('No active page to drill');
|
|
18
19
|
}
|
|
19
20
|
|
|
20
|
-
await this.explorBot.
|
|
21
|
+
await this.explorBot.agentDriller().drill({
|
|
21
22
|
knowledgePath,
|
|
22
23
|
maxComponents,
|
|
23
24
|
interactive: true,
|
|
@@ -30,7 +31,7 @@ export class DrillCommand extends BaseCommand {
|
|
|
30
31
|
}
|
|
31
32
|
|
|
32
33
|
private parseMaxArg(args: string): number | undefined {
|
|
33
|
-
const match = args.match(/--max\s+(\d+)/);
|
|
34
|
+
const match = args.match(/--max-components\s+(\d+)/);
|
|
34
35
|
return match ? Number.parseInt(match[1], 10) : undefined;
|
|
35
36
|
}
|
|
36
37
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import figureSet from 'figures';
|
|
2
2
|
import { getStyles } from '../ai/planner/styles.js';
|
|
3
3
|
import { outputPath } from '../config.js';
|
|
4
|
+
import { normalizeUrl } from '../state-manager.js';
|
|
4
5
|
import { Stats } from '../stats.js';
|
|
5
6
|
import type { Plan } from '../test-plan.js';
|
|
6
7
|
import { getCliName } from '../utils/cli-name.ts';
|
|
@@ -11,6 +12,8 @@ import { type NextStepSection, printNextSteps, relativeToCwd } from '../utils/ne
|
|
|
11
12
|
import { safeFilename } from '../utils/strings.ts';
|
|
12
13
|
import { BaseCommand, type Suggestion } from './base-command.js';
|
|
13
14
|
|
|
15
|
+
const MAX_SUB_PAGE_ATTEMPTS = 30;
|
|
16
|
+
|
|
14
17
|
export class ExploreCommand extends BaseCommand {
|
|
15
18
|
name = 'explore';
|
|
16
19
|
description = 'Start web exploration';
|
|
@@ -27,6 +30,7 @@ export class ExploreCommand extends BaseCommand {
|
|
|
27
30
|
maxTests?: number;
|
|
28
31
|
private testsRun = 0;
|
|
29
32
|
private completedPlans: Plan[] = [];
|
|
33
|
+
private failedSubPages = new Set<string>();
|
|
30
34
|
|
|
31
35
|
async execute(args: string): Promise<void> {
|
|
32
36
|
const { opts, args: remaining } = this.parseArgs(args);
|
|
@@ -46,10 +50,12 @@ export class ExploreCommand extends BaseCommand {
|
|
|
46
50
|
|
|
47
51
|
if (!feature && !this.isLimitReached()) {
|
|
48
52
|
const planner = this.explorBot.agentPlanner();
|
|
49
|
-
|
|
53
|
+
let attempts = 0;
|
|
54
|
+
while (attempts < MAX_SUB_PAGE_ATTEMPTS) {
|
|
55
|
+
attempts++;
|
|
50
56
|
if (this.isLimitReached()) break;
|
|
51
57
|
|
|
52
|
-
const candidates = planner.collectSubPageCandidates(mainPlan, mainUrl || '/');
|
|
58
|
+
const candidates = planner.collectSubPageCandidates(mainPlan, mainUrl || '/').filter((c) => !this.failedSubPages.has(normalizeUrl(c.url)));
|
|
53
59
|
if (candidates.length === 0) break;
|
|
54
60
|
|
|
55
61
|
const pick = await planner.pickNextSubPage(candidates);
|
|
@@ -64,6 +70,7 @@ export class ExploreCommand extends BaseCommand {
|
|
|
64
70
|
this.completedPlans.push(subPlan);
|
|
65
71
|
}
|
|
66
72
|
} catch (err) {
|
|
73
|
+
this.failedSubPages.add(normalizeUrl(pick.url));
|
|
67
74
|
tag('warning').log(`Sub-page exploration failed: ${err instanceof Error ? err.message : err}`);
|
|
68
75
|
}
|
|
69
76
|
}
|
|
@@ -73,6 +80,7 @@ export class ExploreCommand extends BaseCommand {
|
|
|
73
80
|
if (mainUrl) await this.explorBot.visit(mainUrl);
|
|
74
81
|
const savedPath = this.explorBot.savePlans(this.completedPlans);
|
|
75
82
|
this.printResults();
|
|
83
|
+
await this.explorBot.printSessionAnalysis();
|
|
76
84
|
this.printNextSteps(savedPath);
|
|
77
85
|
}
|
|
78
86
|
|
|
@@ -5,7 +5,7 @@ import React, { useEffect, useState } from 'react';
|
|
|
5
5
|
import { AddRuleCommand } from '../commands/add-rule-command.js';
|
|
6
6
|
import InputReadline from './InputReadline.js';
|
|
7
7
|
|
|
8
|
-
const KNOWN_AGENTS = ['researcher', 'tester', 'planner', 'pilot', 'captain', '
|
|
8
|
+
const KNOWN_AGENTS = ['researcher', 'tester', 'planner', 'pilot', 'captain', 'driller', 'navigator'];
|
|
9
9
|
|
|
10
10
|
interface AddRuleProps {
|
|
11
11
|
initialAgent?: string;
|