explorbot 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/rules/planner/styles/curious.md +18 -5
- package/dist/rules/planner/styles/normal.md +4 -4
- package/dist/rules/planner/styles/psycho.md +14 -11
- package/dist/src/ai/captain/web-mode.js +9 -1
- package/dist/src/ai/historian.js +6 -0
- package/dist/src/ai/pilot.js +23 -2
- package/dist/src/ai/researcher/deep-analysis.js +65 -9
- package/dist/src/ai/researcher/sections.js +103 -0
- package/dist/src/ai/researcher.js +9 -46
- package/dist/src/ai/rules.js +9 -1
- package/dist/src/ai/task-agent.js +7 -27
- package/dist/src/ai/tester.js +41 -5
- package/dist/src/ai/tools.js +27 -2
- package/dist/src/commands/explore-command.js +4 -9
- package/dist/src/experience-tracker.js +126 -1
- package/dist/src/explorbot.js +9 -2
- package/dist/src/utils/aria.js +39 -2
- package/package.json +1 -1
- package/rules/planner/styles/curious.md +18 -5
- package/rules/planner/styles/normal.md +4 -4
- package/rules/planner/styles/psycho.md +14 -11
- package/src/ai/captain/web-mode.ts +9 -1
- package/src/ai/historian.ts +7 -0
- package/src/ai/pilot.ts +23 -3
- package/src/ai/researcher/deep-analysis.ts +74 -9
- package/src/ai/researcher/sections.ts +122 -0
- package/src/ai/researcher.ts +9 -47
- package/src/ai/rules.ts +9 -1
- package/src/ai/task-agent.ts +7 -31
- package/src/ai/tester.ts +44 -6
- package/src/ai/tools.ts +33 -1
- package/src/commands/explore-command.ts +4 -9
- package/src/config.ts +1 -0
- package/src/experience-tracker.ts +136 -1
- package/src/explorbot.ts +9 -2
- package/src/utils/aria.ts +40 -2
|
@@ -3,10 +3,12 @@ import { ActionResult, type Diff } from '../../action-result.js';
|
|
|
3
3
|
import type Explorer from '../../explorer.ts';
|
|
4
4
|
import type { StateManager } from '../../state-manager.js';
|
|
5
5
|
import { WebPageState } from '../../state-manager.js';
|
|
6
|
-
import { diffAriaSnapshots } from '../../utils/aria.ts';
|
|
6
|
+
import { detectFocusArea, diffAriaSnapshots } from '../../utils/aria.ts';
|
|
7
7
|
import { executionController } from '../../execution-controller.ts';
|
|
8
8
|
import { tag } from '../../utils/logger.js';
|
|
9
|
+
import { mdq } from '../../utils/markdown-query.ts';
|
|
9
10
|
import type { Provider } from '../provider.js';
|
|
11
|
+
import { getCachedResearch, saveResearch } from './cache.ts';
|
|
10
12
|
import { type Constructor, debugLog } from './mixin.ts';
|
|
11
13
|
import { type ResearchElement, parseResearchSections } from './parser.ts';
|
|
12
14
|
import type { ResearchResult } from './research-result.ts';
|
|
@@ -76,6 +78,55 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
|
|
|
76
78
|
}
|
|
77
79
|
}
|
|
78
80
|
|
|
81
|
+
async researchOverlay(current: ActionResult, previous: ActionResult, pageStateHash: string): Promise<string | null> {
|
|
82
|
+
const focusArea = detectFocusArea(current.ariaSnapshot);
|
|
83
|
+
if (!focusArea.detected || !focusArea.name) return null;
|
|
84
|
+
if (focusArea.type !== 'dialog' && focusArea.type !== 'modal') return null;
|
|
85
|
+
|
|
86
|
+
const cached = getCachedResearch(pageStateHash);
|
|
87
|
+
if (!cached) return null;
|
|
88
|
+
|
|
89
|
+
const escaped = focusArea.name.replace(/"/g, '\\"');
|
|
90
|
+
if (mdq(cached).query(`section3(~"${escaped}")`).count() > 0) {
|
|
91
|
+
debugLog(`Overlay "${focusArea.name}" already in cached research, skipping`);
|
|
92
|
+
return null;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
const diff = await current.diff(previous);
|
|
96
|
+
await diff.calculate();
|
|
97
|
+
|
|
98
|
+
if (!diff.ariaChanged && diff.htmlParts.length === 0) {
|
|
99
|
+
debugLog(`No diff between current and previous state for overlay "${focusArea.name}"`);
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const alreadyExpanded = this._summarizeExpanded(
|
|
104
|
+
parseResearchSections(cached)
|
|
105
|
+
.filter((s) => s.elements.length > 0)
|
|
106
|
+
.map((s) => s.rawMarkdown)
|
|
107
|
+
);
|
|
108
|
+
|
|
109
|
+
tag('substep').log(`Researching overlay: ${focusArea.name}`);
|
|
110
|
+
const sectionMarkdown = await this._analyzeExpandedAction('', focusArea.name, diff, alreadyExpanded);
|
|
111
|
+
if (!sectionMarkdown) {
|
|
112
|
+
debugLog(`Overlay "${focusArea.name}" produced no meaningful expansion`);
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const extQuery = mdq(cached).query('section1(~"Extended Research")');
|
|
117
|
+
let updated: string;
|
|
118
|
+
if (extQuery.count() > 0) {
|
|
119
|
+
const existing = extQuery.text().trimEnd();
|
|
120
|
+
updated = extQuery.replace(`${existing}\n\n${sectionMarkdown}\n`);
|
|
121
|
+
} else {
|
|
122
|
+
updated = `${cached.trimEnd()}\n\n# Extended Research\n\n${sectionMarkdown}\n`;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
saveResearch(pageStateHash, updated);
|
|
126
|
+
tag('substep').log(`Overlay research appended: ${focusArea.name}`);
|
|
127
|
+
return sectionMarkdown;
|
|
128
|
+
}
|
|
129
|
+
|
|
79
130
|
private async _discoverExpandables(researchText: string): Promise<ExpandableElement[]> {
|
|
80
131
|
const allElements = new Map<string, ExpandableElement>();
|
|
81
132
|
for (const section of parseResearchSections(researchText)) {
|
|
@@ -314,8 +365,27 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
|
|
|
314
365
|
private async _analyzeExpandedAction(code: string, description: string, diff: Diff, alreadyExpanded: string[]): Promise<string | null> {
|
|
315
366
|
const alreadyHint = alreadyExpanded.length > 0 ? `\nAlready expanded sections:\n${alreadyExpanded.join('\n')}` : '';
|
|
316
367
|
|
|
368
|
+
let intro: string;
|
|
369
|
+
if (code) {
|
|
370
|
+
intro = `An action on "${description}" (\`${code}\`) revealed new UI content.`;
|
|
371
|
+
} else {
|
|
372
|
+
intro = `An overlay "${description}" appeared on the page.`;
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
let actionBlock = '';
|
|
376
|
+
if (code) {
|
|
377
|
+
actionBlock = dedent`
|
|
378
|
+
Action:
|
|
379
|
+
|
|
380
|
+
\`\`\`js
|
|
381
|
+
${code}
|
|
382
|
+
\`\`\`
|
|
383
|
+
|
|
384
|
+
`;
|
|
385
|
+
}
|
|
386
|
+
|
|
317
387
|
const prompt = dedent`
|
|
318
|
-
|
|
388
|
+
${intro}
|
|
319
389
|
Analyze the changes and produce a UI map section.
|
|
320
390
|
|
|
321
391
|
ARIA changes:
|
|
@@ -329,13 +399,7 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
|
|
|
329
399
|
|
|
330
400
|
### <Short descriptive name>
|
|
331
401
|
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
\`\`\`js
|
|
335
|
-
${code}
|
|
336
|
-
\`\`\`
|
|
337
|
-
|
|
338
|
-
<One sentence: what appeared — dropdown menu, modal, tab content, expanded panel, etc.>
|
|
402
|
+
${actionBlock}<One sentence: what appeared — dropdown menu, modal, tab content, expanded panel, etc.>
|
|
339
403
|
|
|
340
404
|
| Element | ARIA | CSS |
|
|
341
405
|
|---------|------|-----|
|
|
@@ -409,4 +473,5 @@ interface ExpandableElement extends ResearchElement {
|
|
|
409
473
|
|
|
410
474
|
export interface DeepAnalysisMethods {
|
|
411
475
|
performDeepAnalysis(state: WebPageState, result: ResearchResult): Promise<void>;
|
|
476
|
+
researchOverlay(current: ActionResult, previous: ActionResult, pageStateHash: string): Promise<string | null>;
|
|
412
477
|
}
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import dedent from 'dedent';
|
|
2
|
+
import type { ActionResult } from '../../action-result.js';
|
|
3
|
+
import { executionController } from '../../execution-controller.ts';
|
|
4
|
+
import type Explorer from '../../explorer.ts';
|
|
5
|
+
import type { StateManager } from '../../state-manager.js';
|
|
6
|
+
import { tag } from '../../utils/logger.js';
|
|
7
|
+
import { RulesLoader } from '../../utils/rules-loader.ts';
|
|
8
|
+
import type { Provider } from '../provider.js';
|
|
9
|
+
import { locatorRule as generalLocatorRuleText } from '../rules.js';
|
|
10
|
+
import type { Constructor } from './mixin.ts';
|
|
11
|
+
|
|
12
|
+
export interface SectionMethods {
|
|
13
|
+
researchBySections(): Promise<string>;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function WithSections<T extends Constructor>(Base: T) {
|
|
17
|
+
return class extends Base {
|
|
18
|
+
declare explorer: Explorer;
|
|
19
|
+
declare provider: Provider;
|
|
20
|
+
declare stateManager: StateManager;
|
|
21
|
+
declare actionResult: ActionResult | undefined;
|
|
22
|
+
|
|
23
|
+
async researchBySections(): Promise<string> {
|
|
24
|
+
const ariaSnapshot = this.actionResult?.getCompactARIA() || '';
|
|
25
|
+
const configured = (this as any).getConfiguredSections() as Record<string, string>;
|
|
26
|
+
const focusCss = await this._detectFocusCss();
|
|
27
|
+
|
|
28
|
+
let targets: Array<[string, string]>;
|
|
29
|
+
if (focusCss) {
|
|
30
|
+
targets = [['Focus', `element bounded by CSS container '${focusCss}'`]];
|
|
31
|
+
tag('info').log(`Focus element detected via selector '${focusCss}', researching focused area only`);
|
|
32
|
+
} else {
|
|
33
|
+
targets = Object.entries(configured);
|
|
34
|
+
tag('info').log(`Splitting research into ${targets.length} per-section requests`);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const parts: string[] = [];
|
|
38
|
+
for (const [name, description] of targets) {
|
|
39
|
+
if (executionController.isInterrupted()) break;
|
|
40
|
+
const text = await this._researchSingleSection(name, description, ariaSnapshot, focusCss);
|
|
41
|
+
if (!text) continue;
|
|
42
|
+
const trimmed = text.trim();
|
|
43
|
+
if (trimmed === 'NOT_PRESENT' || trimmed.startsWith('NOT_PRESENT')) continue;
|
|
44
|
+
parts.push(trimmed);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
if (parts.length === 0) {
|
|
48
|
+
throw new Error('Per-section research produced no sections — AI responses all empty or NOT_PRESENT');
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
let merged = parts.join('\n\n');
|
|
52
|
+
if (focusCss) merged += '\n\n> Focused: Focus';
|
|
53
|
+
return merged;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
private async _detectFocusCss(): Promise<string | null> {
|
|
57
|
+
const focusSections = (this.explorer.getConfig().ai?.agents?.researcher as any)?.focusSections as string[] | undefined;
|
|
58
|
+
if (!focusSections?.length) return null;
|
|
59
|
+
|
|
60
|
+
for (const css of focusSections) {
|
|
61
|
+
const count = await this.explorer.playwrightLocatorCount((page: any) => page.locator(css)).catch(() => 0);
|
|
62
|
+
if (count > 0) return css;
|
|
63
|
+
}
|
|
64
|
+
return null;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
private async _researchSingleSection(name: string, description: string, ariaSnapshot: string, focusCss: string | null): Promise<string> {
|
|
68
|
+
const currentUrl = this.stateManager.getCurrentState()?.url || '';
|
|
69
|
+
const rules = RulesLoader.loadRules('researcher', ['ui-map-table', 'list-element', 'container-rules'], currentUrl);
|
|
70
|
+
const url = this.actionResult?.url || 'Unknown';
|
|
71
|
+
const title = this.actionResult?.title || 'Unknown';
|
|
72
|
+
|
|
73
|
+
let focusHint = '';
|
|
74
|
+
if (focusCss) {
|
|
75
|
+
focusHint = dedent`
|
|
76
|
+
The user's focus is the element matching CSS '${focusCss}'.
|
|
77
|
+
Use that CSS as the Container for this section.
|
|
78
|
+
`;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const prompt = dedent`
|
|
82
|
+
<task>
|
|
83
|
+
Identify the "${name}" section on this page: ${description}
|
|
84
|
+
If this section is NOT present on the page, respond with ONLY: NOT_PRESENT
|
|
85
|
+
Otherwise output only this single section in the format below.
|
|
86
|
+
${focusHint}
|
|
87
|
+
</task>
|
|
88
|
+
|
|
89
|
+
<section_format>
|
|
90
|
+
## ${name}
|
|
91
|
+
|
|
92
|
+
> Container: '.semantic-container-selector'
|
|
93
|
+
|
|
94
|
+
| Element | ARIA | CSS | eidx |
|
|
95
|
+
</section_format>
|
|
96
|
+
|
|
97
|
+
<rules>
|
|
98
|
+
- Every element with eidx MUST appear in the table.
|
|
99
|
+
- Every row needs CSS; ARIA may be "-" for icon-only buttons.
|
|
100
|
+
- ARIA locator JSON uses keys "role" and "text" (NOT "name").
|
|
101
|
+
</rules>
|
|
102
|
+
|
|
103
|
+
${generalLocatorRuleText}
|
|
104
|
+
|
|
105
|
+
${rules}
|
|
106
|
+
|
|
107
|
+
URL: ${url}
|
|
108
|
+
Title: ${title}
|
|
109
|
+
|
|
110
|
+
<aria>
|
|
111
|
+
${ariaSnapshot}
|
|
112
|
+
</aria>
|
|
113
|
+
`;
|
|
114
|
+
|
|
115
|
+
const conversation = this.provider.startConversation((this as any).getSystemMessage(), 'researcher');
|
|
116
|
+
conversation.addUserText(prompt);
|
|
117
|
+
|
|
118
|
+
const result = await this.provider.invokeConversation(conversation, undefined, { agentName: 'researcher' });
|
|
119
|
+
return result?.response.text || '';
|
|
120
|
+
}
|
|
121
|
+
};
|
|
122
|
+
}
|
package/src/ai/researcher.ts
CHANGED
|
@@ -28,6 +28,7 @@ import { detectFocusFromAria, hasFocusedSection, markSectionAsFocused, pickDefau
|
|
|
28
28
|
import { type LocatorMethods, WithLocators } from './researcher/locators.ts';
|
|
29
29
|
import { extractValidContainers, formatResearchSummary, parseResearchSections } from './researcher/parser.ts';
|
|
30
30
|
import { ResearchResult } from './researcher/research-result.ts';
|
|
31
|
+
import { type SectionMethods, WithSections } from './researcher/sections.ts';
|
|
31
32
|
import { locatorRule as generalLocatorRuleText } from './rules.js';
|
|
32
33
|
import { RulesLoader } from '../utils/rules-loader.ts';
|
|
33
34
|
import { TaskAgent } from './task-agent.ts';
|
|
@@ -46,9 +47,9 @@ export const POSSIBLE_SECTIONS = {
|
|
|
46
47
|
navigation: 'main navigation (top bar, sidebar, breadcrumbs)',
|
|
47
48
|
};
|
|
48
49
|
|
|
49
|
-
const ResearcherBase = WithDeepAnalysis(WithCoordinates(WithLocators(TaskAgent as unknown as new (...args: any[]) => TaskAgent)));
|
|
50
|
+
const ResearcherBase = WithSections(WithDeepAnalysis(WithCoordinates(WithLocators(TaskAgent as unknown as new (...args: any[]) => TaskAgent))));
|
|
50
51
|
|
|
51
|
-
export interface Researcher extends LocatorMethods, CoordinateMethods, DeepAnalysisMethods {}
|
|
52
|
+
export interface Researcher extends LocatorMethods, CoordinateMethods, DeepAnalysisMethods, SectionMethods {}
|
|
52
53
|
|
|
53
54
|
export class Researcher extends ResearcherBase implements Agent {
|
|
54
55
|
protected readonly ACTION_TOOLS = ['click'];
|
|
@@ -170,10 +171,12 @@ export class Researcher extends ResearcherBase implements Agent {
|
|
|
170
171
|
const prompt = await this.buildResearchPrompt();
|
|
171
172
|
conversation.addUserText(prompt);
|
|
172
173
|
|
|
173
|
-
let
|
|
174
|
+
let researchText: string;
|
|
174
175
|
let activeConversation = conversation;
|
|
175
176
|
try {
|
|
176
|
-
invocationResult = await this.provider.invokeConversation(conversation, undefined, { agentName: 'researcher' });
|
|
177
|
+
const invocationResult = await this.provider.invokeConversation(conversation, undefined, { agentName: 'researcher' });
|
|
178
|
+
if (!invocationResult) throw new Error('Failed to get response from provider');
|
|
179
|
+
researchText = invocationResult.response.text;
|
|
177
180
|
} catch (error) {
|
|
178
181
|
if (!(error instanceof ContextLengthError) || retriesLeft <= 0) {
|
|
179
182
|
if (error instanceof ContextLengthError) {
|
|
@@ -181,15 +184,12 @@ export class Researcher extends ResearcherBase implements Agent {
|
|
|
181
184
|
}
|
|
182
185
|
throw error;
|
|
183
186
|
}
|
|
184
|
-
tag('warning').log('Output truncated, retrying with fresh focused conversation (ARIA only)...');
|
|
185
187
|
retriesLeft = 0;
|
|
188
|
+
researchText = await this.researchBySections();
|
|
186
189
|
activeConversation = this.provider.startConversation(this.getSystemMessage(), 'researcher');
|
|
187
|
-
activeConversation.addUserText(this.buildFocusedRetryPrompt());
|
|
188
|
-
invocationResult = await this.provider.invokeConversation(activeConversation, undefined, { agentName: 'researcher' });
|
|
189
190
|
}
|
|
190
|
-
if (!invocationResult) throw new Error('Failed to get response from provider');
|
|
191
191
|
|
|
192
|
-
const result = new ResearchResult(
|
|
192
|
+
const result = new ResearchResult(researchText, state.url);
|
|
193
193
|
debugLog(`Original research response length: ${result.text.length} chars`);
|
|
194
194
|
|
|
195
195
|
const interrupted = () => executionController.isInterrupted();
|
|
@@ -537,44 +537,6 @@ export class Researcher extends ResearcherBase implements Agent {
|
|
|
537
537
|
`;
|
|
538
538
|
}
|
|
539
539
|
|
|
540
|
-
private buildFocusedRetryPrompt(): string {
|
|
541
|
-
const currentUrl = this.stateManager.getCurrentState()?.url || '';
|
|
542
|
-
const example = RulesLoader.loadRules('researcher', ['section-example'], currentUrl);
|
|
543
|
-
const uiMapTable = RulesLoader.loadRules('researcher', ['ui-map-table'], currentUrl);
|
|
544
|
-
const url = this.actionResult?.url || 'Unknown';
|
|
545
|
-
const title = this.actionResult?.title || 'Unknown';
|
|
546
|
-
const aria = this.actionResult?.getCompactARIA() || '';
|
|
547
|
-
return dedent`
|
|
548
|
-
Previous response was truncated. Restart with a minimal output.
|
|
549
|
-
|
|
550
|
-
<task>
|
|
551
|
-
Output a UI map for ONE section only — the main interactive area of this page.
|
|
552
|
-
Skip navigation, sidebar, and footer. Max 15 elements.
|
|
553
|
-
Every element with an eidx MUST appear. Every row needs CSS; ARIA may be "-" for icon-only.
|
|
554
|
-
End with a single line: \`> Focused: <section name>\`.
|
|
555
|
-
</task>
|
|
556
|
-
|
|
557
|
-
<section_format>
|
|
558
|
-
## Section Name
|
|
559
|
-
|
|
560
|
-
> Container: '.container-css-selector'
|
|
561
|
-
|
|
562
|
-
| Element | ARIA | CSS | eidx |
|
|
563
|
-
</section_format>
|
|
564
|
-
|
|
565
|
-
${example}
|
|
566
|
-
|
|
567
|
-
${uiMapTable}
|
|
568
|
-
|
|
569
|
-
URL: ${url}
|
|
570
|
-
Title: ${title}
|
|
571
|
-
|
|
572
|
-
<aria>
|
|
573
|
-
${aria}
|
|
574
|
-
</aria>
|
|
575
|
-
`;
|
|
576
|
-
}
|
|
577
|
-
|
|
578
540
|
async textContent(state: WebPageState): Promise<string> {
|
|
579
541
|
const actionResult = ActionResult.fromState(state);
|
|
580
542
|
const html = await actionResult.combinedHtml();
|
package/src/ai/rules.ts
CHANGED
|
@@ -135,7 +135,15 @@ export const fileUploadRule = dedent`
|
|
|
135
135
|
export const protectionRule = dedent`
|
|
136
136
|
<important>
|
|
137
137
|
Do not sign out current user of the application.
|
|
138
|
-
Do not change current user account settings
|
|
138
|
+
Do not change current user account settings.
|
|
139
|
+
|
|
140
|
+
Pre-existing data on the page belongs to the application, not the test.
|
|
141
|
+
Items that were not created inside the current test scenario must not be deleted, removed, emptied, reset, archived, or otherwise destroyed.
|
|
142
|
+
If a scenario needs to verify destructive behaviour, the same scenario must first create a disposable target and then destroy that specific target — never operate on data that was already there when the test started.
|
|
143
|
+
|
|
144
|
+
The resource that the current page URL represents is "under test".
|
|
145
|
+
The test must not destroy the resource it is running against — doing so invalidates every subsequent scenario that starts on the same URL.
|
|
146
|
+
Do not propose or perform delete/remove/archive actions on the entity that owns the current URL; propose such actions only on disposable children created within the scenario itself.
|
|
139
147
|
</important>
|
|
140
148
|
`;
|
|
141
149
|
|
package/src/ai/task-agent.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import dedent from 'dedent';
|
|
2
2
|
import type { ActionResult } from '../action-result.js';
|
|
3
|
-
import type
|
|
3
|
+
import { renderExperienceToc, type ExperienceTracker } from '../experience-tracker.js';
|
|
4
4
|
import type { KnowledgeTracker } from '../knowledge-tracker.js';
|
|
5
5
|
import { createDebug, pluralize, tag } from '../utils/logger.js';
|
|
6
6
|
|
|
@@ -56,37 +56,13 @@ export abstract class TaskAgent {
|
|
|
56
56
|
|
|
57
57
|
protected getExperience(actionResult: ActionResult): string {
|
|
58
58
|
const tracker = this.getExperienceTracker();
|
|
59
|
-
const
|
|
59
|
+
const toc = tracker.getExperienceTableOfContents(actionResult);
|
|
60
|
+
if (toc.length === 0) return '';
|
|
60
61
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
.filter((e) => !!e)
|
|
66
|
-
.join('\n\n---\n\n');
|
|
67
|
-
|
|
68
|
-
const totalChars = allContent.length;
|
|
69
|
-
let experienceContent: string;
|
|
70
|
-
|
|
71
|
-
if (totalChars <= 10_000) {
|
|
72
|
-
debugLog(`injecting all experience (${Math.round(totalChars / 1000)}k chars)`);
|
|
73
|
-
experienceContent = allContent;
|
|
74
|
-
} else {
|
|
75
|
-
experienceContent = tracker.getSuccessfulExperience(actionResult).join('\n\n---\n\n');
|
|
76
|
-
debugLog(`injecting success-only experience (${Math.round(experienceContent.length / 1000)}k chars, filtered from ${Math.round(totalChars / 1000)}k)`);
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
if (!experienceContent) return '';
|
|
80
|
-
|
|
81
|
-
tag('substep').log(`Found ${relevantExperience.length} experience ${pluralize(relevantExperience.length, 'file')}`);
|
|
82
|
-
return dedent`
|
|
83
|
-
<experience>
|
|
84
|
-
Here is past experience of interacting with this page.
|
|
85
|
-
Use successful solutions first. Avoid repeating failed actions.
|
|
86
|
-
|
|
87
|
-
${experienceContent}
|
|
88
|
-
</experience>
|
|
89
|
-
`;
|
|
62
|
+
const totalSections = toc.reduce((sum, entry) => sum + entry.sections.length, 0);
|
|
63
|
+
debugLog(`injecting experience TOC (${toc.length} files, ${totalSections} sections)`);
|
|
64
|
+
tag('substep').log(`Found ${toc.length} experience ${pluralize(toc.length, 'file')} (${totalSections} sections)`);
|
|
65
|
+
return renderExperienceToc(toc);
|
|
90
66
|
}
|
|
91
67
|
|
|
92
68
|
setHistorian(historian: Historian): void {
|
package/src/ai/tester.ts
CHANGED
|
@@ -11,19 +11,19 @@ import type Explorer from '../explorer.ts';
|
|
|
11
11
|
import type { StateTransition, WebPageState } from '../state-manager.ts';
|
|
12
12
|
import { Stats } from '../stats.ts';
|
|
13
13
|
import { type Note, type Test, TestResult, type TestResultType } from '../test-plan.ts';
|
|
14
|
-
import { extractFocusedElement } from '../utils/aria.ts';
|
|
14
|
+
import { detectFocusArea, extractFocusedElement } from '../utils/aria.ts';
|
|
15
15
|
import { HooksRunner } from '../utils/hooks-runner.ts';
|
|
16
16
|
import { codeToMarkdown } from '../utils/html.ts';
|
|
17
17
|
import { createDebug, tag } from '../utils/logger.ts';
|
|
18
18
|
import { loop } from '../utils/loop.ts';
|
|
19
19
|
import type { Agent } from './agent.ts';
|
|
20
|
+
import type { Captain } from './captain.ts';
|
|
20
21
|
import type { Conversation } from './conversation.ts';
|
|
21
22
|
import { Navigator } from './navigator.ts';
|
|
22
|
-
import type { Captain } from './captain.ts';
|
|
23
23
|
import type { Pilot } from './pilot.ts';
|
|
24
24
|
import { Provider } from './provider.ts';
|
|
25
25
|
import { Researcher } from './researcher.ts';
|
|
26
|
-
import { actionRule, focusedElementRule, locatorRule, multipleTabsRule, sectionContextRule } from './rules.ts';
|
|
26
|
+
import { actionRule, focusedElementRule, locatorRule, multipleTabsRule, protectionRule, sectionContextRule } from './rules.ts';
|
|
27
27
|
import { TaskAgent } from './task-agent.ts';
|
|
28
28
|
import { createCodeceptJSTools, createSpecialContextTools } from './tools.ts';
|
|
29
29
|
|
|
@@ -59,6 +59,8 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
59
59
|
executionLogFile: string | null = null;
|
|
60
60
|
private previousUrl: string | null = null;
|
|
61
61
|
private previousStateHash: string | null = null;
|
|
62
|
+
private pageStateHash: string | null = null;
|
|
63
|
+
private pageActionResult: ActionResult | null = null;
|
|
62
64
|
private hooksRunner: HooksRunner;
|
|
63
65
|
|
|
64
66
|
constructor(explorer: Explorer, provider: Provider, researcher: Researcher, navigator: Navigator, agentTools?: any) {
|
|
@@ -117,10 +119,29 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
117
119
|
|
|
118
120
|
this.previousUrl = null;
|
|
119
121
|
this.previousStateHash = null;
|
|
122
|
+
this.pageStateHash = null;
|
|
123
|
+
this.pageActionResult = null;
|
|
120
124
|
this.explorer.getStateManager().clearHistory();
|
|
121
125
|
this.resetFailureCount();
|
|
122
126
|
this.pilot?.reset();
|
|
123
127
|
|
|
128
|
+
const requestStore = this.explorer.getRequestStore();
|
|
129
|
+
requestStore?.clear();
|
|
130
|
+
const offFailedRequest = requestStore?.onFailedRequest((r) => {
|
|
131
|
+
task.addNote(`Network error: ${r.method} ${r.path} → ${r.status}`, TestResult.FAILED);
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
const page = this.explorer.playwrightHelper?.page;
|
|
135
|
+
const onPageError = (err: Error) => {
|
|
136
|
+
task.addNote(`Console error: ${err.message}`, TestResult.FAILED);
|
|
137
|
+
};
|
|
138
|
+
const onConsoleMessage = (msg: any) => {
|
|
139
|
+
if (msg.type() !== 'error') return;
|
|
140
|
+
task.addNote(`Console error: ${msg.text()}`, TestResult.FAILED);
|
|
141
|
+
};
|
|
142
|
+
page?.on('pageerror', onPageError);
|
|
143
|
+
page?.on('console', onConsoleMessage);
|
|
144
|
+
|
|
124
145
|
const initialState = ActionResult.fromState(state);
|
|
125
146
|
|
|
126
147
|
const conversation = this.provider.startConversation(this.getSystemMessage(), 'tester');
|
|
@@ -351,6 +372,9 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
351
372
|
await this.getQuartermaster().analyzeSession(task, initialState, conversation);
|
|
352
373
|
|
|
353
374
|
offStateChange();
|
|
375
|
+
offFailedRequest?.();
|
|
376
|
+
page?.off('pageerror', onPageError);
|
|
377
|
+
page?.off('console', onConsoleMessage);
|
|
354
378
|
await this.finishTest(task);
|
|
355
379
|
await this.explorer.stopTest(task, {
|
|
356
380
|
startUrl: task.startUrl,
|
|
@@ -441,7 +465,8 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
441
465
|
|
|
442
466
|
if (isNewUrl) {
|
|
443
467
|
const research = await this.researcher.research(currentState);
|
|
444
|
-
|
|
468
|
+
this.pageStateHash = currentStateHash;
|
|
469
|
+
this.pageActionResult = currentState;
|
|
445
470
|
let uiMapSection = '';
|
|
446
471
|
if (research) {
|
|
447
472
|
uiMapSection = dedent`
|
|
@@ -467,8 +492,6 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
467
492
|
</page_aria>
|
|
468
493
|
${uiMapSection}
|
|
469
494
|
|
|
470
|
-
${experience}
|
|
471
|
-
|
|
472
495
|
Use <page_ui_map> to understand the page structure and its main elements.
|
|
473
496
|
However, <page_ui_map> is not always up to date, use <page_aria> and <page_html> to understand the ACTUAL state of the page
|
|
474
497
|
Do not interact with elements that are not listed in <page_aria> and <page_html>
|
|
@@ -477,6 +500,19 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
477
500
|
return context;
|
|
478
501
|
}
|
|
479
502
|
|
|
503
|
+
const focusArea = detectFocusArea(currentState.ariaSnapshot);
|
|
504
|
+
if (focusArea.detected && focusArea.name && this.pageStateHash && this.pageActionResult) {
|
|
505
|
+
const overlaySection = await this.researcher.researchOverlay(currentState, this.pageActionResult, this.pageStateHash);
|
|
506
|
+
if (overlaySection) {
|
|
507
|
+
context += dedent`
|
|
508
|
+
|
|
509
|
+
<page_ui_map_overlay>
|
|
510
|
+
${overlaySection}
|
|
511
|
+
</page_ui_map_overlay>
|
|
512
|
+
`;
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
|
|
480
516
|
// if (isStateChanged) {
|
|
481
517
|
// const combinedHtml = await currentState.combinedHtml();
|
|
482
518
|
// context += dedent`
|
|
@@ -675,6 +711,8 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
675
711
|
When creating or editing items via form() or type() you should include ${task.sessionName} in the value (if it is not restricted by the application logic)
|
|
676
712
|
Initial page URL: ${actionResult.url}
|
|
677
713
|
|
|
714
|
+
${protectionRule}
|
|
715
|
+
|
|
678
716
|
${this.buildDeletionScope(task)}
|
|
679
717
|
|
|
680
718
|
${this.buildAvailableFiles()}
|
package/src/ai/tools.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { tool } from 'ai';
|
|
|
2
2
|
import dedent from 'dedent';
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
import { ActionResult, type ToolResultMetadata } from '../action-result.ts';
|
|
5
|
+
import type { ExperienceTracker } from '../experience-tracker.ts';
|
|
5
6
|
import type Explorer from '../explorer.ts';
|
|
6
7
|
import { type Task, TestResult } from '../test-plan.js';
|
|
7
8
|
import { extractFocusedElement } from '../utils/aria.ts';
|
|
@@ -449,14 +450,18 @@ export function createAgentTools({
|
|
|
449
450
|
explorer,
|
|
450
451
|
researcher,
|
|
451
452
|
navigator,
|
|
453
|
+
experienceTracker,
|
|
454
|
+
getState,
|
|
452
455
|
}: {
|
|
453
456
|
explorer: Explorer;
|
|
454
457
|
researcher: Researcher;
|
|
455
458
|
navigator: Navigator;
|
|
459
|
+
experienceTracker?: ExperienceTracker;
|
|
460
|
+
getState?: () => ActionResult | null;
|
|
456
461
|
}): any {
|
|
457
462
|
let visionDisabled = false;
|
|
458
463
|
|
|
459
|
-
|
|
464
|
+
const tools: Record<string, any> = {
|
|
460
465
|
see: tool({
|
|
461
466
|
description: dedent`
|
|
462
467
|
Check the page contents based on current page state and screenshot.
|
|
@@ -947,6 +952,33 @@ export function createAgentTools({
|
|
|
947
952
|
},
|
|
948
953
|
}),
|
|
949
954
|
};
|
|
955
|
+
|
|
956
|
+
if (experienceTracker && getState) {
|
|
957
|
+
tools.learn_experience = tool({
|
|
958
|
+
description: dedent`
|
|
959
|
+
Read the full body of a specific experience section listed in <experience>.
|
|
960
|
+
The TOC shows entries like "A.1 ## Successful Flow: ...". Pass the fileTag and sectionIndex.
|
|
961
|
+
Only call when a TOC entry looks directly relevant to the current step.
|
|
962
|
+
`,
|
|
963
|
+
inputSchema: z.object({
|
|
964
|
+
fileTag: z.string().describe('File tag from the TOC, e.g. "A", "B", "AA"'),
|
|
965
|
+
sectionIndex: z.number().int().positive().describe('1-based section index within that file'),
|
|
966
|
+
}),
|
|
967
|
+
execute: async ({ fileTag, sectionIndex }) => {
|
|
968
|
+
const state = getState();
|
|
969
|
+
if (!state) {
|
|
970
|
+
return { error: 'No current page state available.' };
|
|
971
|
+
}
|
|
972
|
+
const section = experienceTracker.getExperienceSection(fileTag, sectionIndex, state);
|
|
973
|
+
if (!section) {
|
|
974
|
+
return { error: 'Section not found. Experience may have been updated; re-read the latest TOC.' };
|
|
975
|
+
}
|
|
976
|
+
return section;
|
|
977
|
+
},
|
|
978
|
+
});
|
|
979
|
+
}
|
|
980
|
+
|
|
981
|
+
return tools;
|
|
950
982
|
}
|
|
951
983
|
|
|
952
984
|
const PAGE_DIFF_SUGGESTION = 'Analyze page diff. htmlParts shows what changed and WHERE — each part has a container selector. Use the container as context when clicking elements from the diff.';
|
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
import { existsSync, readdirSync } from 'node:fs';
|
|
2
1
|
import figureSet from 'figures';
|
|
3
2
|
import path from 'node:path';
|
|
4
3
|
import { getStyles } from '../ai/planner/styles.js';
|
|
5
|
-
import { ConfigParser } from '../config.ts';
|
|
6
4
|
import { getCliName } from '../utils/cli-name.ts';
|
|
7
5
|
import type { Plan } from '../test-plan.js';
|
|
8
6
|
import { jsonToTable } from '../utils/markdown-parser.js';
|
|
@@ -116,14 +114,11 @@ export class ExploreCommand extends BaseCommand {
|
|
|
116
114
|
}
|
|
117
115
|
|
|
118
116
|
private printRerunSuggestions(): void {
|
|
119
|
-
const
|
|
120
|
-
if (
|
|
117
|
+
const savedFiles = this.explorBot.agentHistorian().getSavedFiles();
|
|
118
|
+
if (savedFiles.length === 0) return;
|
|
121
119
|
|
|
122
|
-
const
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
for (const file of testFiles) {
|
|
126
|
-
tag('info').log(`Generated: ${file}`);
|
|
120
|
+
for (const filePath of savedFiles) {
|
|
121
|
+
tag('info').log(`Generated: ${path.basename(filePath)}`);
|
|
127
122
|
}
|
|
128
123
|
tag('info').log(`List tests: ${getCliName()} runs`);
|
|
129
124
|
tag('info').log(`Re-run with healing: ${getCliName()} rerun <filename> [index]`);
|