explorbot 0.1.13 → 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package.json +3 -2
- package/dist/src/action.js +3 -2
- package/dist/src/ai/conversation.js +20 -4
- package/dist/src/ai/historian/utils.js +8 -1
- package/dist/src/ai/pilot.js +198 -260
- package/dist/src/ai/provider.js +25 -12
- package/dist/src/ai/quartermaster.js +2 -2
- package/dist/src/ai/rules.js +2 -0
- package/dist/src/ai/session-analyst.js +46 -41
- package/dist/src/ai/tester.js +56 -20
- package/dist/src/ai/tools.js +19 -4
- package/dist/src/commands/explore-command.js +8 -2
- package/dist/src/components/StatusPane.js +6 -1
- package/dist/src/experience-tracker.js +9 -0
- package/dist/src/explorer.js +2 -5
- package/dist/src/reporter.js +41 -1
- package/dist/src/stats.js +2 -1
- package/dist/src/test-plan.js +47 -3
- package/package.json +3 -2
- package/src/action.ts +3 -2
- package/src/ai/conversation.ts +21 -4
- package/src/ai/historian/utils.ts +8 -1
- package/src/ai/pilot.ts +199 -259
- package/src/ai/provider.ts +24 -12
- package/src/ai/quartermaster.ts +2 -2
- package/src/ai/rules.ts +2 -0
- package/src/ai/session-analyst.ts +47 -41
- package/src/ai/tester.ts +48 -18
- package/src/ai/tools.ts +18 -4
- package/src/commands/explore-command.ts +9 -2
- package/src/components/StatusPane.tsx +6 -3
- package/src/experience-tracker.ts +9 -0
- package/src/explorer.ts +1 -4
- package/src/reporter.ts +44 -1
- package/src/stats.ts +3 -1
- package/src/test-plan.ts +62 -3
package/dist/src/ai/provider.js
CHANGED
|
@@ -16,6 +16,16 @@ class AiError extends Error {
|
|
|
16
16
|
}
|
|
17
17
|
export class ContextLengthError extends Error {
|
|
18
18
|
}
|
|
19
|
+
function extractCachedTokens(usage) {
|
|
20
|
+
if (!usage)
|
|
21
|
+
return 0;
|
|
22
|
+
const direct = usage.cachedInputTokens ?? usage.inputTokenDetails?.cacheReadTokens;
|
|
23
|
+
if (typeof direct === 'number')
|
|
24
|
+
return direct;
|
|
25
|
+
const raw = usage.raw;
|
|
26
|
+
const fromRaw = raw?.prompt_tokens_details?.cached_tokens ?? raw?.promptTokensDetails?.cachedTokens;
|
|
27
|
+
return typeof fromRaw === 'number' ? fromRaw : 0;
|
|
28
|
+
}
|
|
19
29
|
function rejectAfterIdle(ms, signal) {
|
|
20
30
|
return new Promise((_, reject) => {
|
|
21
31
|
const tick = () => {
|
|
@@ -227,9 +237,10 @@ export class Provider {
|
|
|
227
237
|
responseLog(response.text);
|
|
228
238
|
if (response.usage) {
|
|
229
239
|
Stats.recordTokens(options.agentName || 'unknown', modelName, {
|
|
230
|
-
input: response.usage.promptTokens
|
|
231
|
-
output: response.usage.completionTokens
|
|
232
|
-
total: response.usage.totalTokens
|
|
240
|
+
input: response.usage.inputTokens ?? response.usage.promptTokens ?? 0,
|
|
241
|
+
output: response.usage.outputTokens ?? response.usage.completionTokens ?? 0,
|
|
242
|
+
total: response.usage.totalTokens ?? 0,
|
|
243
|
+
cached: extractCachedTokens(response.usage),
|
|
233
244
|
});
|
|
234
245
|
}
|
|
235
246
|
return response;
|
|
@@ -311,9 +322,10 @@ export class Provider {
|
|
|
311
322
|
responseLog(response.text);
|
|
312
323
|
if (response.usage) {
|
|
313
324
|
Stats.recordTokens(options.agentName || 'unknown', modelName, {
|
|
314
|
-
input: response.usage.promptTokens
|
|
315
|
-
output: response.usage.completionTokens
|
|
316
|
-
total: response.usage.totalTokens
|
|
325
|
+
input: response.usage.inputTokens ?? response.usage.promptTokens ?? 0,
|
|
326
|
+
output: response.usage.outputTokens ?? response.usage.completionTokens ?? 0,
|
|
327
|
+
total: response.usage.totalTokens ?? 0,
|
|
328
|
+
cached: extractCachedTokens(response.usage),
|
|
317
329
|
});
|
|
318
330
|
}
|
|
319
331
|
return response;
|
|
@@ -379,9 +391,10 @@ export class Provider {
|
|
|
379
391
|
responseLog(response.object);
|
|
380
392
|
if (response.usage) {
|
|
381
393
|
Stats.recordTokens(options.agentName || 'unknown', modelName, {
|
|
382
|
-
input: response.usage.promptTokens
|
|
383
|
-
output: response.usage.completionTokens
|
|
384
|
-
total: response.usage.totalTokens
|
|
394
|
+
input: response.usage.inputTokens ?? response.usage.promptTokens ?? 0,
|
|
395
|
+
output: response.usage.outputTokens ?? response.usage.completionTokens ?? 0,
|
|
396
|
+
total: response.usage.totalTokens ?? 0,
|
|
397
|
+
cached: extractCachedTokens(response.usage),
|
|
385
398
|
});
|
|
386
399
|
}
|
|
387
400
|
return response;
|
|
@@ -555,9 +568,9 @@ export class Provider {
|
|
|
555
568
|
responseLog(response.text);
|
|
556
569
|
if (response.usage) {
|
|
557
570
|
Stats.recordTokens('vision', this.getModelName(this.config.visionModel), {
|
|
558
|
-
input: response.usage.promptTokens
|
|
559
|
-
output: response.usage.completionTokens
|
|
560
|
-
total: response.usage.totalTokens
|
|
571
|
+
input: response.usage.inputTokens ?? response.usage.promptTokens ?? 0,
|
|
572
|
+
output: response.usage.outputTokens ?? response.usage.completionTokens ?? 0,
|
|
573
|
+
total: response.usage.totalTokens ?? 0,
|
|
561
574
|
});
|
|
562
575
|
}
|
|
563
576
|
return response;
|
|
@@ -169,10 +169,10 @@ Focus on what would confuse a real user or caused the agent to make mistakes.`;
|
|
|
169
169
|
const criticalViolations = report.axeViolations.filter((v) => v.impact === 'critical' || v.impact === 'serious');
|
|
170
170
|
for (const v of criticalViolations.slice(0, 3)) {
|
|
171
171
|
const nodeHtml = v.nodes[0]?.html.slice(0, 100) || '';
|
|
172
|
-
task.
|
|
172
|
+
task.addVerificationDetail(`🔴 A11Y [${v.impact}] ${v.id}: ${v.description} — ${nodeHtml}`);
|
|
173
173
|
}
|
|
174
174
|
for (const issue of report.semanticIssues.slice(0, 3)) {
|
|
175
|
-
task.
|
|
175
|
+
task.addVerificationDetail(`💡 UX [${issue.type}] ${issue.element}: ${issue.suggestion}`);
|
|
176
176
|
}
|
|
177
177
|
}
|
|
178
178
|
saveReport(stateHash, report) {
|
package/dist/src/ai/rules.js
CHANGED
|
@@ -231,6 +231,8 @@ export function multipleTabsRule(tabs) {
|
|
|
231
231
|
}
|
|
232
232
|
export const actionRule = dedent `
|
|
233
233
|
<actions>
|
|
234
|
+
\`faker\` (from @faker-js/faker) is available inside I.* calls for generating data, e.g. I.fillField('Bio', faker.lorem.paragraphs(5)).
|
|
235
|
+
|
|
234
236
|
### I.click
|
|
235
237
|
|
|
236
238
|
clicks on the element by its locator
|
|
@@ -13,68 +13,70 @@ export class SessionAnalyst {
|
|
|
13
13
|
const eligible = tests.filter((t) => t.startTime != null);
|
|
14
14
|
if (eligible.length === 0)
|
|
15
15
|
return '';
|
|
16
|
-
const model = this.provider.
|
|
16
|
+
const model = this.provider.getAgenticModel('analyst');
|
|
17
17
|
const customPrompt = this.provider.getSystemPromptForAgent('analyst', undefined);
|
|
18
18
|
const systemPrompt = dedent `
|
|
19
|
-
You write a
|
|
19
|
+
You write a TERSE end-of-session report. Reader is a developer who wants to UNDERSTAND THE FEATURE — what works, what is broken, what is unclear. Every word must earn its place.
|
|
20
20
|
|
|
21
|
-
Output MARKDOWN. No JSON, no preamble, no closing
|
|
21
|
+
Output MARKDOWN. No JSON, no preamble, no closing summary.
|
|
22
22
|
|
|
23
|
-
|
|
24
|
-
Group by ROOT CAUSE, not by scenario. If three tests fail for the same dropdown, that is ONE defect listing all three test refs (#3, #5, #7). Do not produce one cluster per test.
|
|
23
|
+
NO EMOJI. No 🔴 🟡 🟢 ✅, no escape sequences like \\u2705. Use plain text severity tags: [High], [Medium], [Low] for defects.
|
|
25
24
|
|
|
26
|
-
##
|
|
27
|
-
Use the FINAL verdict (the test's \`result\` field) as the starting point. Mid-test errors that the automation recovered from do NOT make a passed test unreliable.
|
|
25
|
+
## Reporting unit
|
|
28
26
|
|
|
29
|
-
|
|
30
|
-
- **UX issue** — app works but the UI is ambiguous, controls are hidden, or labels are unclear. Worth flagging to design.
|
|
31
|
-
- **Execution issue** — the FINAL verdict is unreliable. Only two cases:
|
|
32
|
-
1. \`result: failed\` AND the failure was automation, environment, or UI/UX (locator missing, timeout, AI loop, navigation stuck, modal trapped focus, no accessible label) — i.e. the test could not conclude whether the app works.
|
|
33
|
-
2. \`result: passed\` AND clear evidence in the log shows the user-visible goal was NOT achieved (no confirmation visible, no state change verified, the assertion was vacuous).
|
|
27
|
+
Report at the level of FEATURES / FLOWS / PAGES. Tests are evidence, not the unit. Several tests covering the same flow → ONE entry citing all of them.
|
|
34
28
|
|
|
35
|
-
|
|
29
|
+
## Walk every test
|
|
36
30
|
|
|
37
|
-
|
|
38
|
-
- 🔴 critical or high — core flow blocked, data loss, security
|
|
39
|
-
- 🟡 medium — partial breakage with workaround
|
|
40
|
-
- 🟢 low — cosmetic
|
|
31
|
+
PASSED test: did all steps run, was the goal actually verified, did the user-visible goal happen? All yes → contributes to What works. Any no → Execution issue (false positive).
|
|
41
32
|
|
|
42
|
-
|
|
33
|
+
FAILED test, first match wins: (1) goal achieved but mis-verified → Execution. (2) automation failure (locator/timeout/loop/modal/a11y) → Execution. (3) bad preconditions or data → Execution. (4) wrong URL/environment → Execution. (5) app contradicted expected outcome → Defect.
|
|
34
|
+
|
|
35
|
+
Crucial distinction: "the app misbehaved" vs "the automation could not interact with the app". ONLY the first is a Defect. If the automation gives up before the app responds — timeout, retries exhausted, dead loop / loop detected, could not click or find an element — that is an Execution issue regardless of what the log calls it. Failure inside the automation ≠ failure inside the product.
|
|
36
|
+
|
|
37
|
+
A solitary failure where adjacent tests on the same feature passed → Execution, not Defect.
|
|
38
|
+
|
|
39
|
+
## Severity (defects only)
|
|
40
|
+
[High] blocks a core flow · [Medium] degrades a flow but workaround exists · [Low] cosmetic / edge case
|
|
41
|
+
|
|
42
|
+
## Format
|
|
43
43
|
|
|
44
44
|
# Session Analysis
|
|
45
45
|
|
|
46
|
-
<
|
|
46
|
+
<ONE or TWO sentences describing the FEATURE STATE — what was explored, whether the core flow holds, what the standout problem is. NO test counts, NO "N tests run". Talk about the product, not the run.>
|
|
47
|
+
|
|
48
|
+
## Coverage
|
|
49
|
+
- Pages: <paths>
|
|
50
|
+
- Features: <capabilities>
|
|
51
|
+
|
|
52
|
+
## What works
|
|
53
|
+
- **<feature>** — #2, #7, #8
|
|
47
54
|
|
|
48
55
|
## Defects
|
|
49
56
|
|
|
50
|
-
###
|
|
51
|
-
Affects: #3, #5
|
|
57
|
+
### [Medium] <plain-English bug title>
|
|
58
|
+
Affects: #3, #5
|
|
52
59
|
Reproduce:
|
|
53
|
-
1. <concrete UI step
|
|
54
|
-
2. <next
|
|
55
|
-
Evidence: <one short observation
|
|
56
|
-
|
|
57
|
-
### 🟡 <next defect>
|
|
58
|
-
...
|
|
60
|
+
1. <concrete UI step>
|
|
61
|
+
2. <next>
|
|
62
|
+
Evidence: <one short observation>
|
|
59
63
|
|
|
60
64
|
## UX issues
|
|
61
|
-
|
|
62
|
-
- **<title>** — #4
|
|
63
|
-
<one short evidence line>
|
|
65
|
+
- **<feature>** — <what's confusing> (#7)
|
|
64
66
|
|
|
65
67
|
## Execution Issues
|
|
68
|
+
- **#2 <scenario>** — <≤10 words, what was unreliable>
|
|
66
69
|
|
|
67
|
-
|
|
68
|
-
- **<…>** — <…>
|
|
70
|
+
## Brevity rules
|
|
69
71
|
|
|
70
|
-
|
|
71
|
-
-
|
|
72
|
-
- Defect title
|
|
73
|
-
- Reproduce steps are
|
|
74
|
-
- Evidence is
|
|
75
|
-
-
|
|
76
|
-
-
|
|
77
|
-
-
|
|
72
|
+
- Headline: 2 sentences MAX. About the FEATURE, not the run. No counts, no "N tests", no "this session". Banned words: "exercised", "comprehensive", "notably", "this session", "module", "targeted", "covered creation".
|
|
73
|
+
- What works: feature name + test refs. NO parentheticals, NO caveats. If there's a caveat, the entry doesn't belong here.
|
|
74
|
+
- Defect title is the BUG ("Search returns non-matching results"), never the scenario name.
|
|
75
|
+
- Reproduce steps are imperative one-liners drawn from the log.
|
|
76
|
+
- Evidence is one short factual observation. Never quote the \`result\` field.
|
|
77
|
+
- Execution Issues: ONE line per test, ≤10 words, plain. Examples: "passed vacuously, no list assertion", "no file upload step in log", "dead loop on Save click". No prefixes, no nested explanation.
|
|
78
|
+
- Omit any empty section.
|
|
79
|
+
- Section order: Coverage → What works → Defects (severity desc) → UX issues → Execution Issues.
|
|
78
80
|
|
|
79
81
|
${customPrompt || ''}
|
|
80
82
|
`;
|
|
@@ -87,7 +89,7 @@ export class SessionAnalyst {
|
|
|
87
89
|
{ role: 'system', content: systemPrompt },
|
|
88
90
|
{ role: 'user', content: userPayload },
|
|
89
91
|
], model, { agentName: 'analyst' });
|
|
90
|
-
return (response?.text || '').trim();
|
|
92
|
+
return decodeEscapes((response?.text || '').trim());
|
|
91
93
|
}
|
|
92
94
|
writeReport(markdown) {
|
|
93
95
|
const filePath = outputPath('reports', `${Stats.sessionLabel()}.md`);
|
|
@@ -115,3 +117,6 @@ export class SessionAnalyst {
|
|
|
115
117
|
`;
|
|
116
118
|
}
|
|
117
119
|
}
|
|
120
|
+
function decodeEscapes(text) {
|
|
121
|
+
return text.replace(/\\u\{([0-9a-fA-F]+)\}/g, (_, hex) => String.fromCodePoint(Number.parseInt(hex, 16))).replace(/\\u([0-9a-fA-F]{4})/g, (_, hex) => String.fromCodePoint(Number.parseInt(hex, 16)));
|
|
122
|
+
}
|
package/dist/src/ai/tester.js
CHANGED
|
@@ -49,6 +49,8 @@ export class Tester extends TaskAgent {
|
|
|
49
49
|
pageStateHash = null;
|
|
50
50
|
pageActionResult = null;
|
|
51
51
|
hooksRunner;
|
|
52
|
+
seenUiMapUrls = new Set();
|
|
53
|
+
lastAnalyzedStateHash = null;
|
|
52
54
|
constructor(explorer, provider, researcher, navigator, agentTools) {
|
|
53
55
|
super();
|
|
54
56
|
this.explorer = explorer;
|
|
@@ -80,7 +82,7 @@ export class Tester extends TaskAgent {
|
|
|
80
82
|
return ActionResult.fromState(this.explorer.getStateManager().getCurrentState());
|
|
81
83
|
}
|
|
82
84
|
get progressCheckInterval() {
|
|
83
|
-
return this.explorer.getConfig().ai?.agents?.tester?.progressCheckInterval ??
|
|
85
|
+
return this.explorer.getConfig().ai?.agents?.tester?.progressCheckInterval ?? 3;
|
|
84
86
|
}
|
|
85
87
|
getConversation() {
|
|
86
88
|
return this.currentConversation;
|
|
@@ -96,6 +98,8 @@ export class Tester extends TaskAgent {
|
|
|
96
98
|
this.previousStateHash = null;
|
|
97
99
|
this.pageStateHash = null;
|
|
98
100
|
this.pageActionResult = null;
|
|
101
|
+
this.seenUiMapUrls.clear();
|
|
102
|
+
this.lastAnalyzedStateHash = null;
|
|
99
103
|
this.explorer.getStateManager().clearHistory();
|
|
100
104
|
this.resetFailureCount();
|
|
101
105
|
this.pilot?.reset();
|
|
@@ -117,12 +121,18 @@ export class Tester extends TaskAgent {
|
|
|
117
121
|
page?.on('console', onConsoleMessage);
|
|
118
122
|
const initialState = ActionResult.fromState(state);
|
|
119
123
|
const conversation = this.provider.startConversation(this.getSystemMessage(), 'tester');
|
|
124
|
+
conversation.markLastMessageCacheable();
|
|
120
125
|
this.currentConversation = conversation;
|
|
121
126
|
const outputDir = ConfigParser.getInstance().getOutputDir();
|
|
122
127
|
this.executionLogFile = join(outputDir, `tester_${task.sessionName}.md`);
|
|
123
128
|
// Note: Markdown saving functionality removed from Conversation class
|
|
124
|
-
const
|
|
125
|
-
conversation.addUserText(
|
|
129
|
+
const scenarioBlock = this.buildScenarioBlock(task, initialState);
|
|
130
|
+
conversation.addUserText(scenarioBlock);
|
|
131
|
+
conversation.markLastMessageCacheable();
|
|
132
|
+
conversation.protectPrefix(conversation.messages.length);
|
|
133
|
+
const pageContext = await this.reinjectContextIfNeeded(1, initialState);
|
|
134
|
+
if (pageContext)
|
|
135
|
+
conversation.addUserText(pageContext);
|
|
126
136
|
return await Observability.run(`test: ${task.scenario}`, {
|
|
127
137
|
sessionId: task.sessionName,
|
|
128
138
|
tags: ['tester'],
|
|
@@ -138,6 +148,12 @@ export class Tester extends TaskAgent {
|
|
|
138
148
|
if (this.pilot) {
|
|
139
149
|
try {
|
|
140
150
|
const plan = await this.pilot.planTest(task, initialState);
|
|
151
|
+
if (task.hasFinished) {
|
|
152
|
+
offFailedRequest?.();
|
|
153
|
+
page?.off('pageerror', onPageError);
|
|
154
|
+
page?.off('console', onConsoleMessage);
|
|
155
|
+
return { success: task.isSuccessful };
|
|
156
|
+
}
|
|
141
157
|
if (plan) {
|
|
142
158
|
conversation.addUserText(`Pilot's test plan:\n${plan}\n\nFollow this plan while executing the test.`);
|
|
143
159
|
}
|
|
@@ -158,14 +174,18 @@ export class Tester extends TaskAgent {
|
|
|
158
174
|
await this.explorer.startTest(task);
|
|
159
175
|
debugLog(`Navigating to ${task.startUrl}`);
|
|
160
176
|
await this.explorer.visit(task.startUrl);
|
|
161
|
-
const
|
|
177
|
+
const startState = this.explorer.getStateManager().getCurrentState();
|
|
178
|
+
if (startState)
|
|
179
|
+
task.addUrlNote(startState);
|
|
180
|
+
const currentUrl = startState?.url || task.startUrl || '';
|
|
162
181
|
await this.hooksRunner.runBeforeHook('tester', currentUrl);
|
|
163
182
|
const offStateChange = this.explorer.getStateManager().onStateChange((event) => {
|
|
164
183
|
if (task.hasFinished)
|
|
165
184
|
return;
|
|
166
185
|
if (event.toState?.url === event.fromState?.url)
|
|
167
186
|
return;
|
|
168
|
-
|
|
187
|
+
if (event.toState)
|
|
188
|
+
task.addUrlNote(event.toState, event.fromState || undefined);
|
|
169
189
|
task.states.push(event.toState);
|
|
170
190
|
});
|
|
171
191
|
const codeceptjsTools = createCodeceptJSTools(this.explorer, task);
|
|
@@ -203,13 +223,13 @@ export class Tester extends TaskAgent {
|
|
|
203
223
|
The user has interrupted and wants to change direction. Follow the new instruction.
|
|
204
224
|
`);
|
|
205
225
|
}
|
|
206
|
-
conversation.cleanupTag('page_aria', '...cleaned aria snapshot...',
|
|
226
|
+
conversation.cleanupTag('page_aria', '...cleaned aria snapshot...', 1);
|
|
207
227
|
conversation.cleanupTag('page_html', '...cleaned HTML snapshot...', 1);
|
|
208
228
|
conversation.cleanupTag('experience', '...cleaned experience...', 1);
|
|
209
229
|
conversation.cleanupTag('applied_experience', '...cleaned past experience...', 1);
|
|
210
230
|
conversation.cleanupTag('page_ui_map', '...cleaned UI map...', 1);
|
|
211
231
|
conversation.cleanupTag('page_ui_map_overlay', '...cleaned UI overlay...', 1);
|
|
212
|
-
conversation.compactToolResults(
|
|
232
|
+
conversation.compactToolResults(2);
|
|
213
233
|
if (iteration > 1) {
|
|
214
234
|
const isNewPage = this.previousUrl !== null && this.previousUrl !== currentState.url;
|
|
215
235
|
let nextStep = '';
|
|
@@ -220,16 +240,17 @@ export class Tester extends TaskAgent {
|
|
|
220
240
|
if (guidance)
|
|
221
241
|
nextStep += `\n\n${guidance}`;
|
|
222
242
|
}
|
|
223
|
-
else if ((iteration
|
|
243
|
+
else if (this.shouldAnalyzeProgress(iteration, currentState) && this.pilot) {
|
|
224
244
|
const guidance = await this.pilot.analyzeProgress(task, currentState, conversation);
|
|
225
245
|
if (guidance)
|
|
226
246
|
nextStep += `\n\n${guidance}`;
|
|
227
247
|
this.consecutiveFailures = 0;
|
|
248
|
+
this.lastAnalyzedStateHash = currentState.hash;
|
|
228
249
|
}
|
|
229
250
|
conversation.addUserText(nextStep);
|
|
230
251
|
}
|
|
231
252
|
const result = await this.provider.invokeConversation(conversation, tools, {
|
|
232
|
-
maxToolRoundtrips:
|
|
253
|
+
maxToolRoundtrips: 3,
|
|
233
254
|
toolChoice: 'required',
|
|
234
255
|
stopWhen: () => task.hasFinished,
|
|
235
256
|
});
|
|
@@ -354,6 +375,17 @@ export class Tester extends TaskAgent {
|
|
|
354
375
|
...task,
|
|
355
376
|
};
|
|
356
377
|
}
|
|
378
|
+
shouldAnalyzeProgress(iteration, currentState) {
|
|
379
|
+
if (this.consecutiveFailures >= 3)
|
|
380
|
+
return true;
|
|
381
|
+
if (this.consecutiveEmptyResults >= 2)
|
|
382
|
+
return true;
|
|
383
|
+
if (iteration % this.progressCheckInterval !== 0)
|
|
384
|
+
return false;
|
|
385
|
+
if (this.lastAnalyzedStateHash === currentState.hash)
|
|
386
|
+
return false;
|
|
387
|
+
return true;
|
|
388
|
+
}
|
|
357
389
|
async prepareInstructionsForNextStep(task) {
|
|
358
390
|
let outcomeStatus = dedent `
|
|
359
391
|
<task>
|
|
@@ -432,19 +464,23 @@ export class Tester extends TaskAgent {
|
|
|
432
464
|
this.explorer.clearOtherTabsInfo();
|
|
433
465
|
}
|
|
434
466
|
if (isNewUrl) {
|
|
467
|
+
const alreadySeenUiMap = this.seenUiMapUrls.has(currentUrl);
|
|
435
468
|
let research = '';
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
469
|
+
if (!alreadySeenUiMap) {
|
|
470
|
+
try {
|
|
471
|
+
research = await this.researcher.research(currentState);
|
|
472
|
+
}
|
|
473
|
+
catch (err) {
|
|
474
|
+
if (!(err instanceof ErrorPageError))
|
|
475
|
+
throw err;
|
|
476
|
+
tag('warning').log(`Research skipped: ${err.message}`);
|
|
477
|
+
}
|
|
443
478
|
}
|
|
444
479
|
this.pageStateHash = currentStateHash;
|
|
445
480
|
this.pageActionResult = currentState;
|
|
446
481
|
let uiMapSection = '';
|
|
447
482
|
if (research) {
|
|
483
|
+
this.seenUiMapUrls.add(currentUrl);
|
|
448
484
|
uiMapSection = dedent `
|
|
449
485
|
|
|
450
486
|
Page UI Map
|
|
@@ -454,6 +490,9 @@ export class Tester extends TaskAgent {
|
|
|
454
490
|
</page_ui_map>
|
|
455
491
|
`;
|
|
456
492
|
}
|
|
493
|
+
else if (alreadySeenUiMap) {
|
|
494
|
+
uiMapSection = `\n\n<page_ui_map>UI map for ${currentUrl} was shown earlier in this session — refer to it above.</page_ui_map>`;
|
|
495
|
+
}
|
|
457
496
|
context += dedent `
|
|
458
497
|
Context:
|
|
459
498
|
|
|
@@ -651,9 +690,8 @@ export class Tester extends TaskAgent {
|
|
|
651
690
|
${this.provider.getSystemPromptForAgent('tester', this.explorer.getStateManager().getCurrentState()?.url) || ''}
|
|
652
691
|
`;
|
|
653
692
|
}
|
|
654
|
-
|
|
693
|
+
buildScenarioBlock(task, actionResult) {
|
|
655
694
|
const knowledge = this.getKnowledge(actionResult);
|
|
656
|
-
const pageContext = await this.reinjectContextIfNeeded(1, actionResult);
|
|
657
695
|
return dedent `
|
|
658
696
|
<task>
|
|
659
697
|
SCENARIO GOAL: ${task.scenario}
|
|
@@ -680,8 +718,6 @@ export class Tester extends TaskAgent {
|
|
|
680
718
|
${this.buildAvailableFiles()}
|
|
681
719
|
|
|
682
720
|
${knowledge}
|
|
683
|
-
|
|
684
|
-
${pageContext}
|
|
685
721
|
`;
|
|
686
722
|
}
|
|
687
723
|
getDeletableSessionNames(task) {
|
package/dist/src/ai/tools.js
CHANGED
|
@@ -423,7 +423,7 @@ export function createAgentTools({ explorer, researcher, navigator, experienceTr
|
|
|
423
423
|
return failedToolResult('see', 'AI analysis failed to process the screenshot');
|
|
424
424
|
}
|
|
425
425
|
return successToolResult('see', {
|
|
426
|
-
analysis: analysisResult,
|
|
426
|
+
analysis: cap(analysisResult, ANALYSIS_OUTPUT_CAP),
|
|
427
427
|
message: `Successfully analyzed screenshot for: ${request}`,
|
|
428
428
|
suggestion: 'Visual confirmation is valid evidence for test results. Use record() to note the visual findings.',
|
|
429
429
|
});
|
|
@@ -469,8 +469,8 @@ export function createAgentTools({ explorer, researcher, navigator, experienceTr
|
|
|
469
469
|
url: currentState.url,
|
|
470
470
|
title: currentState.title,
|
|
471
471
|
suggestion: 'If not enough context received, call see() to visually identify elements in page contents',
|
|
472
|
-
aria,
|
|
473
|
-
html,
|
|
472
|
+
aria: cap(aria, ARIA_OUTPUT_CAP),
|
|
473
|
+
html: cap(html, HTML_OUTPUT_CAP),
|
|
474
474
|
reminder: 'Context provided. Do not call context() again until you perform actions or suspect page changed.',
|
|
475
475
|
});
|
|
476
476
|
}
|
|
@@ -556,7 +556,7 @@ export function createAgentTools({ explorer, researcher, navigator, experienceTr
|
|
|
556
556
|
const researchResult = await researcher.research(currentState, { screenshot: true, data: true });
|
|
557
557
|
return successToolResult('research', {
|
|
558
558
|
analysis: researchResult,
|
|
559
|
-
aria: ActionResult.fromState(currentState).getInteractiveARIA(),
|
|
559
|
+
aria: cap(ActionResult.fromState(currentState).getInteractiveARIA(), ARIA_OUTPUT_CAP),
|
|
560
560
|
message: `Successfully researched page: ${currentState.url}.`,
|
|
561
561
|
suggestion: dedent `
|
|
562
562
|
You received comprehensive UI map report. Use it to understand the page structure and navigate to the elements.
|
|
@@ -859,6 +859,16 @@ export function createAgentTools({ explorer, researcher, navigator, experienceTr
|
|
|
859
859
|
return tools;
|
|
860
860
|
}
|
|
861
861
|
const PAGE_DIFF_SUGGESTION = 'Analyze page diff. htmlParts shows what changed and WHERE — each part has a container selector. Use the container as context when clicking elements from the diff.';
|
|
862
|
+
const ARIA_OUTPUT_CAP = 4000;
|
|
863
|
+
const HTML_OUTPUT_CAP = 6000;
|
|
864
|
+
const ANALYSIS_OUTPUT_CAP = 2000;
|
|
865
|
+
function cap(text, max) {
|
|
866
|
+
if (!text)
|
|
867
|
+
return '';
|
|
868
|
+
if (text.length <= max)
|
|
869
|
+
return text;
|
|
870
|
+
return `${text.slice(0, max)}\n[...truncated; ${text.length - max} chars omitted...]`;
|
|
871
|
+
}
|
|
862
872
|
function transformContainsCommand(command) {
|
|
863
873
|
if (!command.includes(':contains('))
|
|
864
874
|
return command;
|
|
@@ -897,9 +907,14 @@ function successToolResult(action, data, source) {
|
|
|
897
907
|
if (data?.pageDiff) {
|
|
898
908
|
let suggestion = PAGE_DIFF_SUGGESTION;
|
|
899
909
|
const ariaChanges = data.pageDiff.ariaChanges || '';
|
|
910
|
+
const urlChanged = data.pageDiff.urlChanged === true;
|
|
911
|
+
const hasHtmlParts = Array.isArray(data.pageDiff.htmlParts) && data.pageDiff.htmlParts.length > 0;
|
|
900
912
|
if (countAriaChanges(ariaChanges) >= 50) {
|
|
901
913
|
suggestion = `MAJOR PAGE CHANGE. Page entered a different mode. Check htmlParts and iframes in pageDiff before next action. ${suggestion}`;
|
|
902
914
|
}
|
|
915
|
+
else if (!urlChanged && !ariaChanges && !hasHtmlParts) {
|
|
916
|
+
suggestion = 'Action ran without error but produced no observable change (URL, ARIA and HTML all unchanged). The locator likely matched a non-interactive ancestor or an element outside the intended control. Re-locate via xpathCheck() or verify with see() before treating this as success.';
|
|
917
|
+
}
|
|
903
918
|
else if (ariaChanges.includes('heading') && ariaChanges.includes('added')) {
|
|
904
919
|
suggestion += ' WARNING: A new panel or modal may have appeared. If this was not the intended action, close it and try a different element.';
|
|
905
920
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import figureSet from 'figures';
|
|
2
2
|
import { getStyles } from '../ai/planner/styles.js';
|
|
3
3
|
import { outputPath } from '../config.js';
|
|
4
|
+
import { normalizeUrl } from '../state-manager.js';
|
|
4
5
|
import { Stats } from '../stats.js';
|
|
5
6
|
import { getCliName } from "../utils/cli-name.js";
|
|
6
7
|
import { ErrorPageError } from "../utils/error-page.js";
|
|
@@ -9,6 +10,7 @@ import { jsonToTable } from '../utils/markdown-parser.js';
|
|
|
9
10
|
import { printNextSteps, relativeToCwd } from "../utils/next-steps.js";
|
|
10
11
|
import { safeFilename } from "../utils/strings.js";
|
|
11
12
|
import { BaseCommand } from './base-command.js';
|
|
13
|
+
const MAX_SUB_PAGE_ATTEMPTS = 30;
|
|
12
14
|
export class ExploreCommand extends BaseCommand {
|
|
13
15
|
name = 'explore';
|
|
14
16
|
description = 'Start web exploration';
|
|
@@ -24,6 +26,7 @@ export class ExploreCommand extends BaseCommand {
|
|
|
24
26
|
maxTests;
|
|
25
27
|
testsRun = 0;
|
|
26
28
|
completedPlans = [];
|
|
29
|
+
failedSubPages = new Set();
|
|
27
30
|
async execute(args) {
|
|
28
31
|
const { opts, args: remaining } = this.parseArgs(args);
|
|
29
32
|
if (opts.maxTests) {
|
|
@@ -40,10 +43,12 @@ export class ExploreCommand extends BaseCommand {
|
|
|
40
43
|
this.completedPlans.push(mainPlan);
|
|
41
44
|
if (!feature && !this.isLimitReached()) {
|
|
42
45
|
const planner = this.explorBot.agentPlanner();
|
|
43
|
-
|
|
46
|
+
let attempts = 0;
|
|
47
|
+
while (attempts < MAX_SUB_PAGE_ATTEMPTS) {
|
|
48
|
+
attempts++;
|
|
44
49
|
if (this.isLimitReached())
|
|
45
50
|
break;
|
|
46
|
-
const candidates = planner.collectSubPageCandidates(mainPlan, mainUrl || '/');
|
|
51
|
+
const candidates = planner.collectSubPageCandidates(mainPlan, mainUrl || '/').filter((c) => !this.failedSubPages.has(normalizeUrl(c.url)));
|
|
47
52
|
if (candidates.length === 0)
|
|
48
53
|
break;
|
|
49
54
|
const pick = await planner.pickNextSubPage(candidates);
|
|
@@ -59,6 +64,7 @@ export class ExploreCommand extends BaseCommand {
|
|
|
59
64
|
}
|
|
60
65
|
}
|
|
61
66
|
catch (err) {
|
|
67
|
+
this.failedSubPages.add(normalizeUrl(pick.url));
|
|
62
68
|
tag('warning').log(`Sub-page exploration failed: ${err instanceof Error ? err.message : err}`);
|
|
63
69
|
}
|
|
64
70
|
}
|
|
@@ -32,5 +32,10 @@ export const StatusPane = ({ onComplete }) => {
|
|
|
32
32
|
React.createElement(Box, { marginTop: 1, marginBottom: 1 },
|
|
33
33
|
React.createElement(Text, { bold: true }, "Usage")),
|
|
34
34
|
React.createElement(Row, { label: "Time", value: Stats.getElapsedTime() }),
|
|
35
|
-
tokenRows.map(([model, tokens]) =>
|
|
35
|
+
tokenRows.map(([model, tokens]) => {
|
|
36
|
+
const cached = tokens.cached ?? 0;
|
|
37
|
+
const cachePct = tokens.input > 0 ? Math.round((cached / tokens.input) * 100) : 0;
|
|
38
|
+
const suffix = cached > 0 ? ` (${Stats.humanizeTokens(cached)} cached, ${cachePct}%)` : '';
|
|
39
|
+
return React.createElement(Row, { key: model, label: model, value: `${Stats.humanizeTokens(tokens.total)} tokens${suffix}` });
|
|
40
|
+
})))));
|
|
36
41
|
};
|
|
@@ -2,6 +2,7 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSy
|
|
|
2
2
|
import { basename, dirname, join } from 'node:path';
|
|
3
3
|
import matter from 'gray-matter';
|
|
4
4
|
import { marked } from 'marked';
|
|
5
|
+
import { isNonReusableCode } from "./ai/historian/utils.js";
|
|
5
6
|
import { ConfigParser } from './config.js';
|
|
6
7
|
import { KnowledgeTracker } from './knowledge-tracker.js';
|
|
7
8
|
import { createDebug, tag } from './utils/logger.js';
|
|
@@ -145,6 +146,10 @@ export class ExperienceTracker {
|
|
|
145
146
|
return;
|
|
146
147
|
if (!action.code?.trim())
|
|
147
148
|
return;
|
|
149
|
+
if (isNonReusableCode(action.code)) {
|
|
150
|
+
debugLog('Skipping action with non-reusable code: %s', action.code);
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
148
153
|
this.ensureExperienceFile(state);
|
|
149
154
|
const stateHash = state.getStateHash();
|
|
150
155
|
const { content, data } = this.readExperienceFile(stateHash);
|
|
@@ -166,6 +171,10 @@ export class ExperienceTracker {
|
|
|
166
171
|
return;
|
|
167
172
|
if (!body?.trim())
|
|
168
173
|
return;
|
|
174
|
+
if (isNonReusableCode(body)) {
|
|
175
|
+
debugLog('Skipping flow body with non-reusable code');
|
|
176
|
+
return;
|
|
177
|
+
}
|
|
169
178
|
this.ensureExperienceFile(state);
|
|
170
179
|
const stateHash = state.getStateHash();
|
|
171
180
|
const { content, data } = this.readExperienceFile(stateHash);
|
package/dist/src/explorer.js
CHANGED
|
@@ -10,7 +10,7 @@ import Action from './action.js';
|
|
|
10
10
|
import { visuallyAnnotateContainers } from "./ai/researcher/coordinates.js";
|
|
11
11
|
import { RequestStore } from "./api/request-store.js";
|
|
12
12
|
import { XhrCapture } from "./api/xhr-capture.js";
|
|
13
|
-
import { ConfigParser
|
|
13
|
+
import { ConfigParser } from './config.js';
|
|
14
14
|
import { KnowledgeTracker } from './knowledge-tracker.js';
|
|
15
15
|
import { PlaywrightRecorder } from "./playwright-recorder.js";
|
|
16
16
|
import { Reporter } from "./reporter.js";
|
|
@@ -467,10 +467,7 @@ class Explorer {
|
|
|
467
467
|
if (!this.stateManager.getCurrentState())
|
|
468
468
|
return;
|
|
469
469
|
const lastScreenshot = ActionResult.fromState(this.stateManager.getCurrentState()).screenshotFile;
|
|
470
|
-
|
|
471
|
-
return;
|
|
472
|
-
const screenshotPath = outputPath('states', lastScreenshot);
|
|
473
|
-
test.addArtifact(screenshotPath);
|
|
470
|
+
test.setActiveNoteScreenshot(lastScreenshot);
|
|
474
471
|
};
|
|
475
472
|
const dialogHandler = (dialog) => {
|
|
476
473
|
const dialogType = dialog.type();
|