explorbot 0.0.5 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/explorbot-cli.ts +4 -3
- package/dist/bin/explorbot-cli.js +4 -3
- package/dist/src/action.js +14 -11
- package/dist/src/ai/planner/subpages.js +42 -6
- package/dist/src/ai/planner.js +15 -3
- package/dist/src/ai/researcher/cache.js +13 -8
- package/dist/src/ai/researcher/coordinates.js +4 -2
- package/dist/src/ai/researcher/deep-analysis.js +16 -19
- package/dist/src/ai/researcher/locators.js +1 -1
- package/dist/src/ai/researcher/parser.js +4 -3
- package/dist/src/ai/researcher/research-result.js +2 -0
- package/dist/src/ai/researcher.js +6 -5
- package/dist/src/ai/tools.js +4 -0
- package/dist/src/commands/context-command.js +2 -2
- package/dist/src/commands/explore-command.js +1 -1
- package/dist/src/commands/init-command.js +4 -2
- package/dist/src/commands/plan-command.js +6 -1
- package/dist/src/explorbot.js +1 -1
- package/dist/src/explorer.js +58 -16
- package/dist/src/utils/web-element.js +6 -4
- package/package.json +2 -2
- package/src/action.ts +14 -10
- package/src/ai/planner/subpages.ts +37 -7
- package/src/ai/planner.ts +16 -3
- package/src/ai/researcher/cache.ts +14 -8
- package/src/ai/researcher/coordinates.ts +8 -7
- package/src/ai/researcher/deep-analysis.ts +18 -21
- package/src/ai/researcher/locators.ts +3 -3
- package/src/ai/researcher/parser.ts +4 -4
- package/src/ai/researcher/research-result.ts +1 -0
- package/src/ai/researcher.ts +6 -5
- package/src/ai/tools.ts +5 -0
- package/src/commands/context-command.ts +2 -2
- package/src/commands/explore-command.ts +1 -1
- package/src/commands/init-command.ts +5 -2
- package/src/commands/plan-command.ts +6 -1
- package/src/config.ts +1 -0
- package/src/explorbot.ts +1 -1
- package/src/explorer.ts +67 -20
- package/src/utils/web-element.ts +12 -10
|
@@ -3,6 +3,7 @@ const KEY_DISPLAY_ATTRS = ['role', 'id', 'class', 'aria-label'];
|
|
|
3
3
|
const KEY_ATTRS = ['role', 'aria-label', 'id', 'name', 'type', 'href'];
|
|
4
4
|
export class WebElement {
|
|
5
5
|
tag;
|
|
6
|
+
role;
|
|
6
7
|
xpath;
|
|
7
8
|
clickXPath;
|
|
8
9
|
attrs;
|
|
@@ -12,6 +13,7 @@ export class WebElement {
|
|
|
12
13
|
y;
|
|
13
14
|
constructor(data) {
|
|
14
15
|
this.tag = data.tag;
|
|
16
|
+
this.role = data.role || data.attrs.role || '';
|
|
15
17
|
this.xpath = data.xpath;
|
|
16
18
|
this.clickXPath = data.clickXPath;
|
|
17
19
|
this.attrs = data.attrs;
|
|
@@ -33,8 +35,7 @@ export class WebElement {
|
|
|
33
35
|
return `(${this.x}, ${this.y})`;
|
|
34
36
|
}
|
|
35
37
|
get eidx() {
|
|
36
|
-
|
|
37
|
-
return val ? Number.parseInt(val, 10) : null;
|
|
38
|
+
return this.attrs['data-explorbot-eidx'] || this.attrs.eidx || null;
|
|
38
39
|
}
|
|
39
40
|
get isNavigationLink() {
|
|
40
41
|
if (this.tag !== 'a')
|
|
@@ -46,9 +47,10 @@ export class WebElement {
|
|
|
46
47
|
const cls = this.attrs.class || '';
|
|
47
48
|
return cls.split(/\s+/).filter((c) => c.length > 2 && !isDynamicId(c) && !isGenericClass(c));
|
|
48
49
|
}
|
|
49
|
-
static fromRawData(d) {
|
|
50
|
+
static fromRawData(d, role) {
|
|
50
51
|
return new WebElement({
|
|
51
52
|
tag: d.tag,
|
|
53
|
+
role,
|
|
52
54
|
xpath: '',
|
|
53
55
|
clickXPath: buildClickableXPath({ tag: d.tag, allAttrs: d.allAttrs, text: d.text }),
|
|
54
56
|
attrs: d.allAttrs,
|
|
@@ -111,7 +113,7 @@ export class WebElement {
|
|
|
111
113
|
return { totalFound: result.totalFound, elements: result.matches.map((m) => WebElement.fromXPathMatch(m)) };
|
|
112
114
|
}
|
|
113
115
|
}
|
|
114
|
-
function extractElementData(el) {
|
|
116
|
+
export function extractElementData(el) {
|
|
115
117
|
const rect = el.getBoundingClientRect();
|
|
116
118
|
if (rect.width === 0 && rect.height === 0)
|
|
117
119
|
return null;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "explorbot",
|
|
3
|
-
"version": "0.0
|
|
3
|
+
"version": "0.1.0",
|
|
4
4
|
"description": "CLI app built with React Ink, CodeceptJS, and Playwright",
|
|
5
5
|
"license": "Elastic-2.0",
|
|
6
6
|
"type": "module",
|
|
@@ -77,7 +77,7 @@
|
|
|
77
77
|
"@opentelemetry/sdk-trace-base": "^2.2.0",
|
|
78
78
|
"@opentelemetry/semantic-conventions": "^1.38.0",
|
|
79
79
|
"@scalar/openapi-parser": "^0.25.6",
|
|
80
|
-
"@testomatio/reporter": "2.7.
|
|
80
|
+
"@testomatio/reporter": "^2.7.6",
|
|
81
81
|
"ai": "^6.0.6",
|
|
82
82
|
"axe-core": "^4.11.1",
|
|
83
83
|
"bash-tool": "^1.3.15",
|
package/src/action.ts
CHANGED
|
@@ -64,7 +64,7 @@ class Action {
|
|
|
64
64
|
}
|
|
65
65
|
}
|
|
66
66
|
|
|
67
|
-
async capturePageState({ includeScreenshot = false }: { includeScreenshot?: boolean } = {}): Promise<ActionResult> {
|
|
67
|
+
async capturePageState({ includeScreenshot = false, ariaSnapshot: preCapuredAria }: { includeScreenshot?: boolean; ariaSnapshot?: string } = {}): Promise<ActionResult> {
|
|
68
68
|
try {
|
|
69
69
|
const currentState = this.stateManager.getCurrentState();
|
|
70
70
|
const stateHash = currentState?.hash || 'screenshot';
|
|
@@ -111,19 +111,23 @@ class Action {
|
|
|
111
111
|
// Capture iframe HTML snapshots
|
|
112
112
|
const iframeSnapshots = await this.captureIframeSnapshots(html);
|
|
113
113
|
|
|
114
|
-
let ariaSnapshot: string | null = null;
|
|
114
|
+
let ariaSnapshot: string | null = preCapuredAria || null;
|
|
115
115
|
let ariaSnapshotFile: string | undefined = undefined;
|
|
116
116
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
117
|
+
if (!ariaSnapshot) {
|
|
118
|
+
try {
|
|
119
|
+
const page = this.playwrightHelper.page;
|
|
120
|
+
ariaSnapshot = await page.locator('body').ariaSnapshot();
|
|
121
|
+
} catch (err) {
|
|
122
|
+
debugLog('ARIA snapshot failed:', err instanceof Error ? `${err.message}\n${err.stack}` : err);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
if (ariaSnapshot) {
|
|
120
127
|
const ariaFileName = `${stateHash}_${timestamp}.aria.yaml`;
|
|
121
128
|
const ariaPath = join(statesDir, ariaFileName);
|
|
122
|
-
fs.writeFileSync(ariaPath,
|
|
123
|
-
ariaSnapshot = serializedSnapshot;
|
|
129
|
+
fs.writeFileSync(ariaPath, ariaSnapshot, 'utf8');
|
|
124
130
|
ariaSnapshotFile = ariaFileName;
|
|
125
|
-
} catch (err) {
|
|
126
|
-
debugLog('ARIA snapshot failed:', err instanceof Error ? `${err.message}\n${err.stack}` : err);
|
|
127
131
|
}
|
|
128
132
|
|
|
129
133
|
const result = new ActionResult({
|
|
@@ -137,7 +141,7 @@ class Action {
|
|
|
137
141
|
iframeSnapshots,
|
|
138
142
|
ariaSnapshot,
|
|
139
143
|
ariaSnapshotFile,
|
|
140
|
-
iframeURL: frame
|
|
144
|
+
iframeURL: frame ? frame.url?.() || 'iframe' : undefined,
|
|
141
145
|
});
|
|
142
146
|
this.stateManager.updateState(result);
|
|
143
147
|
return result;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import dedent from 'dedent';
|
|
2
2
|
import { z } from 'zod';
|
|
3
|
+
import { ConfigParser } from '../../config.ts';
|
|
3
4
|
import { normalizeUrl } from '../../state-manager.ts';
|
|
4
5
|
import type { StateManager } from '../../state-manager.ts';
|
|
5
6
|
import type { Plan } from '../../test-plan.ts';
|
|
@@ -9,9 +10,9 @@ import type { Constructor } from '../researcher/mixin.ts';
|
|
|
9
10
|
|
|
10
11
|
const planRegistry: Map<string, PlanRecord> = new Map();
|
|
11
12
|
|
|
12
|
-
export function registerPlan(url: string, plan: Plan, feature?: string): void {
|
|
13
|
+
export function registerPlan(url: string, plan: Plan, feature?: string, stateHash?: string): void {
|
|
13
14
|
const key = buildKey(url, feature);
|
|
14
|
-
planRegistry.set(key, { plan, feature, url });
|
|
15
|
+
planRegistry.set(key, { plan, feature, url, stateHash });
|
|
15
16
|
}
|
|
16
17
|
|
|
17
18
|
export function getRegisteredPlan(url: string, feature?: string): PlanRecord | undefined {
|
|
@@ -37,7 +38,30 @@ function buildKey(url: string, feature?: string): string {
|
|
|
37
38
|
return normalized;
|
|
38
39
|
}
|
|
39
40
|
|
|
40
|
-
function
|
|
41
|
+
export function isDynamicSegment(segment: string): boolean {
|
|
42
|
+
try {
|
|
43
|
+
const configRegex = ConfigParser.getInstance().getConfig().dynamicPageRegex;
|
|
44
|
+
if (configRegex) return new RegExp(configRegex, 'i').test(segment);
|
|
45
|
+
} catch {
|
|
46
|
+
/* config not loaded yet */
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// numeric: /users/123
|
|
50
|
+
if (/^\d+$/.test(segment)) return true;
|
|
51
|
+
// UUID: /items/550e8400-e29b-41d4-a716-446655440000
|
|
52
|
+
if (/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(segment)) return true;
|
|
53
|
+
// ULID: /items/01ARZ3NDEKTSV4RRFFQ69G5FAV
|
|
54
|
+
if (/^[0-9A-HJKMNP-TV-Z]{26}$/.test(segment)) return true;
|
|
55
|
+
// hex ID (4+ chars): /suite/70dae98a
|
|
56
|
+
if (/^[a-f0-9]{4,}$/i.test(segment)) return true;
|
|
57
|
+
// hex-prefixed slug (8+ hex before dash): /suite/95ef0c94-mobile
|
|
58
|
+
if (/^[a-f0-9]{8,}-/i.test(segment)) return true;
|
|
59
|
+
// short mixed alphanumeric (digits + letters, ≤8 chars, no dash): /item/x7f2
|
|
60
|
+
if (segment.length <= 8 && !segment.includes('-') && /\d/.test(segment) && /[a-z]/i.test(segment)) return true;
|
|
61
|
+
return false;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export function isTemplateMatch(urlA: string, urlB: string): boolean {
|
|
41
65
|
const partsA = normalizeUrl(urlA).split('/');
|
|
42
66
|
const partsB = normalizeUrl(urlB).split('/');
|
|
43
67
|
if (partsA.length !== partsB.length) return false;
|
|
@@ -47,12 +71,18 @@ function isTemplateMatch(urlA: string, urlB: string): boolean {
|
|
|
47
71
|
if (partsA[i] === partsB[i]) continue;
|
|
48
72
|
diffCount++;
|
|
49
73
|
if (diffCount > 1) return false;
|
|
50
|
-
|
|
51
|
-
if (!isNumericOrShortId.test(partsA[i]) && !isNumericOrShortId.test(partsB[i])) return false;
|
|
74
|
+
if (!isDynamicSegment(partsA[i]) && !isDynamicSegment(partsB[i])) return false;
|
|
52
75
|
}
|
|
53
76
|
return diffCount === 1;
|
|
54
77
|
}
|
|
55
78
|
|
|
79
|
+
export function getPlannedByStateHash(hash: string): PlanRecord | null {
|
|
80
|
+
for (const record of planRegistry.values()) {
|
|
81
|
+
if (record.stateHash === hash) return record;
|
|
82
|
+
}
|
|
83
|
+
return null;
|
|
84
|
+
}
|
|
85
|
+
|
|
56
86
|
const SubPagePickSchema = z.object({
|
|
57
87
|
url: z.string().nullable(),
|
|
58
88
|
reason: z.string(),
|
|
@@ -71,7 +101,7 @@ export function WithSubPages<T extends Constructor>(Base: T) {
|
|
|
71
101
|
for (const page of visited) {
|
|
72
102
|
const pagePath = normalizeUrl(page.url);
|
|
73
103
|
if (!pagePath.startsWith(currentPath) || pagePath === currentPath) continue;
|
|
74
|
-
if (
|
|
104
|
+
if (this.findSimilarPlan(page.url)) continue;
|
|
75
105
|
if (candidates.some((c) => normalizeUrl(c.url) === pagePath)) continue;
|
|
76
106
|
|
|
77
107
|
candidates.push({
|
|
@@ -136,6 +166,6 @@ export function WithSubPages<T extends Constructor>(Base: T) {
|
|
|
136
166
|
};
|
|
137
167
|
}
|
|
138
168
|
|
|
139
|
-
type PlanRecord = { plan: Plan; feature?: string; url: string };
|
|
169
|
+
type PlanRecord = { plan: Plan; feature?: string; url: string; stateHash?: string };
|
|
140
170
|
|
|
141
171
|
type SubPageCandidate = { url: string; title?: string; h1?: string; visitCount: number };
|
package/src/ai/planner.ts
CHANGED
|
@@ -18,7 +18,8 @@ import { Conversation } from './conversation.ts';
|
|
|
18
18
|
import type { Fisherman } from './fisherman.ts';
|
|
19
19
|
import { getActiveStyle, getStyles } from './planner/styles.ts';
|
|
20
20
|
import { WithSessionDedup } from './planner/session-dedup.ts';
|
|
21
|
-
import { WithSubPages, getRegisteredPlan, registerPlan } from './planner/subpages.ts';
|
|
21
|
+
import { WithSubPages, getPlannedByStateHash, getRegisteredPlan, registerPlan } from './planner/subpages.ts';
|
|
22
|
+
import { findSimilarStateHash } from './researcher/cache.ts';
|
|
22
23
|
import type { Provider } from './provider.js';
|
|
23
24
|
import { hasFocusedSection } from './researcher/focus.ts';
|
|
24
25
|
import { POSSIBLE_SECTIONS, Researcher } from './researcher.ts';
|
|
@@ -127,13 +128,25 @@ export class Planner extends PlannerBase implements Agent {
|
|
|
127
128
|
debugLog('Planning:', state?.url);
|
|
128
129
|
if (!state) throw new Error('No state found');
|
|
129
130
|
|
|
130
|
-
if (!
|
|
131
|
+
if (!feature && !this.currentPlan && state.url) {
|
|
131
132
|
const similar = this.findSimilarPlan(state.url);
|
|
132
133
|
if (similar) {
|
|
133
134
|
tag('info').log(`Similar page already planned: ${similar.url} (${similar.plan.tests.length} tests)`);
|
|
134
135
|
this.registerPlanInSession(similar.plan);
|
|
135
136
|
return similar.plan;
|
|
136
137
|
}
|
|
138
|
+
|
|
139
|
+
const actionResult = ActionResult.fromState(state);
|
|
140
|
+
const combinedHtml = await actionResult.combinedHtml();
|
|
141
|
+
const similarHash = await findSimilarStateHash(combinedHtml);
|
|
142
|
+
if (similarHash) {
|
|
143
|
+
const planned = getPlannedByStateHash(similarHash);
|
|
144
|
+
if (planned) {
|
|
145
|
+
tag('info').log(`Page content similar to already-planned: ${planned.url} — skipping`);
|
|
146
|
+
this.registerPlanInSession(planned.plan);
|
|
147
|
+
return planned.plan;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
137
150
|
}
|
|
138
151
|
|
|
139
152
|
if (!this.freshStart && !this.currentPlan && state.url) {
|
|
@@ -211,7 +224,7 @@ export class Planner extends PlannerBase implements Agent {
|
|
|
211
224
|
tag('success').log(`Planning complete! ${this.currentPlan.tests.length} tests in plan: ${this.currentPlan.title}`);
|
|
212
225
|
tag('info').log(`Planning style: ${this.lastStyleName} (available: ${availableStyles})`);
|
|
213
226
|
|
|
214
|
-
if (state.url) registerPlan(state.url, this.currentPlan, feature);
|
|
227
|
+
if (state.url) registerPlan(state.url, this.currentPlan, feature, state.hash);
|
|
215
228
|
|
|
216
229
|
this.registerPlanInSession(this.currentPlan);
|
|
217
230
|
|
|
@@ -64,7 +64,7 @@ export function saveResearch(hash: string, text: string, combinedHtml?: string):
|
|
|
64
64
|
return researchFile;
|
|
65
65
|
}
|
|
66
66
|
|
|
67
|
-
|
|
67
|
+
function findSimilarMatch(combinedHtml: string): Promise<{ hash: string; similarity: number } | null> {
|
|
68
68
|
const statesDir = getStatesDir();
|
|
69
69
|
if (!existsSync(statesDir)) return Promise.resolve(null);
|
|
70
70
|
|
|
@@ -84,13 +84,8 @@ export function findSimilarResearch(combinedHtml: string): Promise<string | null
|
|
|
84
84
|
return;
|
|
85
85
|
}
|
|
86
86
|
|
|
87
|
-
debugLog(`Similar
|
|
88
|
-
|
|
89
|
-
if (research) {
|
|
90
|
-
resolve(research);
|
|
91
|
-
return;
|
|
92
|
-
}
|
|
93
|
-
resolve(null);
|
|
87
|
+
debugLog(`Similar fingerprint found: ${matchHash} (${similarity}% similar)`);
|
|
88
|
+
resolve({ hash: matchHash, similarity });
|
|
94
89
|
};
|
|
95
90
|
|
|
96
91
|
worker.postMessage({
|
|
@@ -101,3 +96,14 @@ export function findSimilarResearch(combinedHtml: string): Promise<string | null
|
|
|
101
96
|
});
|
|
102
97
|
});
|
|
103
98
|
}
|
|
99
|
+
|
|
100
|
+
export async function findSimilarResearch(combinedHtml: string): Promise<string | null> {
|
|
101
|
+
const match = await findSimilarMatch(combinedHtml);
|
|
102
|
+
if (!match) return null;
|
|
103
|
+
return getCachedResearch(match.hash) || null;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export async function findSimilarStateHash(combinedHtml: string): Promise<string | null> {
|
|
107
|
+
const match = await findSimilarMatch(combinedHtml);
|
|
108
|
+
return match?.hash || null;
|
|
109
|
+
}
|
|
@@ -85,7 +85,7 @@ export function WithCoordinates<T extends Constructor>(Base: T) {
|
|
|
85
85
|
}
|
|
86
86
|
|
|
87
87
|
private async _analyzeScreenshotForVisualProps(): Promise<VisualAnalysisResult> {
|
|
88
|
-
const elements = new Map<
|
|
88
|
+
const elements = new Map<string, { coordinates: string | null; color: string | null; icon: string | null }>();
|
|
89
89
|
const emptyResult: VisualAnalysisResult = { elements, pagePurpose: null, primaryActions: null, focusedSection: null };
|
|
90
90
|
if (!this.actionResult) return emptyResult;
|
|
91
91
|
|
|
@@ -131,8 +131,9 @@ export function WithCoordinates<T extends Constructor>(Base: T) {
|
|
|
131
131
|
const text = aiResult.text || '';
|
|
132
132
|
const rows = mdq(text).query('table').toJson();
|
|
133
133
|
for (const row of rows) {
|
|
134
|
-
|
|
135
|
-
if (
|
|
134
|
+
let eidx = (row.eidx || '').trim();
|
|
135
|
+
if (!eidx || eidx === '-') continue;
|
|
136
|
+
if (/^\d+$/.test(eidx)) eidx = `e${eidx}`;
|
|
136
137
|
const val = (v: string) => (v && v !== '-' ? v : null);
|
|
137
138
|
elements.set(eidx, {
|
|
138
139
|
coordinates: val(row.Coordinates),
|
|
@@ -166,7 +167,7 @@ export function WithCoordinates<T extends Constructor>(Base: T) {
|
|
|
166
167
|
return emptyResult;
|
|
167
168
|
}
|
|
168
169
|
|
|
169
|
-
async mergeVisualData(result: ResearchResult, visualData: Map<
|
|
170
|
+
async mergeVisualData(result: ResearchResult, visualData: Map<string, { coordinates: string | null; color: string | null; icon: string | null }>): Promise<void> {
|
|
170
171
|
const sections = parseResearchSections(result.text);
|
|
171
172
|
let merged = 0;
|
|
172
173
|
|
|
@@ -194,7 +195,7 @@ export function WithCoordinates<T extends Constructor>(Base: T) {
|
|
|
194
195
|
async backfillCoordinates(result: ResearchResult): Promise<void> {
|
|
195
196
|
const page = this.explorer.playwrightHelper.page;
|
|
196
197
|
const sections = parseResearchSections(result.text);
|
|
197
|
-
const eidxWithoutCoords:
|
|
198
|
+
const eidxWithoutCoords: string[] = [];
|
|
198
199
|
for (const section of sections) {
|
|
199
200
|
for (const el of section.elements) {
|
|
200
201
|
if (el.eidx && !el.coordinates) eidxWithoutCoords.push(el.eidx);
|
|
@@ -224,7 +225,7 @@ export function WithCoordinates<T extends Constructor>(Base: T) {
|
|
|
224
225
|
}
|
|
225
226
|
|
|
226
227
|
export interface VisualAnalysisResult {
|
|
227
|
-
elements: Map<
|
|
228
|
+
elements: Map<string, { coordinates: string | null; color: string | null; icon: string | null }>;
|
|
228
229
|
pagePurpose: string | null;
|
|
229
230
|
primaryActions: string[] | null;
|
|
230
231
|
focusedSection: string | null;
|
|
@@ -232,7 +233,7 @@ export interface VisualAnalysisResult {
|
|
|
232
233
|
|
|
233
234
|
export interface CoordinateMethods {
|
|
234
235
|
analyzeScreenshotForVisualProps(): Promise<VisualAnalysisResult>;
|
|
235
|
-
mergeVisualData(result: ResearchResult, visualData: Map<
|
|
236
|
+
mergeVisualData(result: ResearchResult, visualData: Map<string, { coordinates: string | null; color: string | null; icon: string | null }>): Promise<void>;
|
|
236
237
|
backfillCoordinates(result: ResearchResult): Promise<void>;
|
|
237
238
|
visuallyAnnotateElements(opts?: { containers?: Array<{ css: string; label: string }> }): Promise<number>;
|
|
238
239
|
}
|
|
@@ -77,7 +77,7 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
|
|
|
77
77
|
}
|
|
78
78
|
|
|
79
79
|
private async _discoverExpandables(researchText: string): Promise<ExpandableElement[]> {
|
|
80
|
-
const allElements = new Map<
|
|
80
|
+
const allElements = new Map<string, ExpandableElement>();
|
|
81
81
|
for (const section of parseResearchSections(researchText)) {
|
|
82
82
|
for (const el of section.elements) {
|
|
83
83
|
if (el.eidx != null) allElements.set(el.eidx, { ...el, container: section.containerCss });
|
|
@@ -91,7 +91,7 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
|
|
|
91
91
|
From this UI research, identify elements that could reveal hidden UI when clicked
|
|
92
92
|
(dropdown menus, popups, expandable panels, accordion sections, overflow menus, tab switches).
|
|
93
93
|
|
|
94
|
-
Available eidx
|
|
94
|
+
Available eidx refs: ${eidxList}
|
|
95
95
|
|
|
96
96
|
${researchText}
|
|
97
97
|
|
|
@@ -99,7 +99,7 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
|
|
|
99
99
|
- Only pick elements that HIDE content until clicked (menus, dropdowns, accordions, tabs)
|
|
100
100
|
- Skip regular links, data items, and navigation
|
|
101
101
|
- For repeated elements (same expand button on every row), pick only the FIRST one
|
|
102
|
-
- Respond with comma-separated eidx
|
|
102
|
+
- Respond with comma-separated eidx refs only, e.g.: e3, e7, e15
|
|
103
103
|
`;
|
|
104
104
|
|
|
105
105
|
const model = this.provider.getModelForAgent('researcher');
|
|
@@ -112,7 +112,7 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
|
|
|
112
112
|
const screenshot = this.actionResult?.screenshot;
|
|
113
113
|
if (screenshot && this.provider.hasVision()) {
|
|
114
114
|
const visionPrompt = dedent`
|
|
115
|
-
This screenshot has interactive elements labeled with eidx
|
|
115
|
+
This screenshot has interactive elements labeled with eidx refs (solid bordered boxes with labels).
|
|
116
116
|
Identify elements that could reveal hidden UI when clicked.
|
|
117
117
|
|
|
118
118
|
Look for: overflow/ellipsis menus, chevron dropdowns, hamburger menus,
|
|
@@ -121,33 +121,30 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
|
|
|
121
121
|
Rules:
|
|
122
122
|
- For repeated icons (same icon on every list row), pick only the FIRST one
|
|
123
123
|
- Skip regular text buttons, links, and navigation items
|
|
124
|
-
- Respond with comma-separated eidx
|
|
124
|
+
- Respond with comma-separated eidx refs only, e.g.: e3, e7, e15
|
|
125
125
|
`;
|
|
126
126
|
visionCall = this.provider.processImage(visionPrompt, screenshot.toString('base64'));
|
|
127
127
|
}
|
|
128
128
|
|
|
129
129
|
const [textRes, visionRes] = await Promise.all([textCall, visionCall]);
|
|
130
130
|
|
|
131
|
-
const eidxSet = new Set<
|
|
131
|
+
const eidxSet = new Set<string>();
|
|
132
|
+
const parseRefs = (text: string | undefined) => {
|
|
133
|
+
if (!text) return [];
|
|
134
|
+
const matches = text.match(/e?\d+/g) || [];
|
|
135
|
+
const refs = matches.map((m) => (m.startsWith('e') ? m : `e${m}`));
|
|
136
|
+
return refs.filter((r) => allElements.has(r));
|
|
137
|
+
};
|
|
138
|
+
|
|
132
139
|
for (const res of [textRes, visionRes]) {
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
for (const n of nums) {
|
|
136
|
-
if (allElements.has(n)) eidxSet.add(n);
|
|
140
|
+
for (const ref of parseRefs(res?.text)) {
|
|
141
|
+
eidxSet.add(ref);
|
|
137
142
|
}
|
|
138
143
|
}
|
|
139
144
|
|
|
140
|
-
const
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
?.map(Number)
|
|
144
|
-
.filter((n) => allElements.has(n)) || [];
|
|
145
|
-
const visionNums =
|
|
146
|
-
visionRes?.text
|
|
147
|
-
?.match(/\d+/g)
|
|
148
|
-
?.map(Number)
|
|
149
|
-
.filter((n) => allElements.has(n)) || [];
|
|
150
|
-
debugLog(`Text model picked eidx: [${textNums.join(', ')}], Vision model picked eidx: [${visionNums.join(', ')}]`);
|
|
145
|
+
const textRefs = parseRefs(textRes?.text);
|
|
146
|
+
const visionRefs = parseRefs(visionRes?.text);
|
|
147
|
+
debugLog(`Text model picked eidx: [${textRefs.join(', ')}], Vision model picked eidx: [${visionRefs.join(', ')}]`);
|
|
151
148
|
|
|
152
149
|
return [...eidxSet].map((eidx) => allElements.get(eidx)!);
|
|
153
150
|
}
|
|
@@ -146,7 +146,7 @@ export function WithLocators<T extends Constructor>(Base: T) {
|
|
|
146
146
|
for (const fixedSection of fixedSections) {
|
|
147
147
|
const originalSections = parseResearchSections(result.text);
|
|
148
148
|
const original = originalSections.find((s) => s.name === fixedSection.name);
|
|
149
|
-
if (!original) continue;
|
|
149
|
+
if (!original || original.elements.length === 0) continue;
|
|
150
150
|
|
|
151
151
|
if (fixedSection.containerCss && fixedSection.containerCss !== original.containerCss) {
|
|
152
152
|
debugLog(`Fixed container for "${fixedSection.name}": '${original.containerCss}' → '${fixedSection.containerCss}'`);
|
|
@@ -177,8 +177,8 @@ export function WithLocators<T extends Constructor>(Base: T) {
|
|
|
177
177
|
const sections = parseResearchSections(result.text);
|
|
178
178
|
const brokenCss = new Set(result.locators.filter((l) => l.type === 'css' && l.valid === false).map((l) => `${l.section}::${l.element}`));
|
|
179
179
|
|
|
180
|
-
const needsXpath:
|
|
181
|
-
const needsXpathEls = new Map<
|
|
180
|
+
const needsXpath: string[] = [];
|
|
181
|
+
const needsXpathEls = new Map<string, { section: (typeof sections)[0]; el: (typeof sections)[0]['elements'][0] }>();
|
|
182
182
|
|
|
183
183
|
for (const section of sections) {
|
|
184
184
|
for (const el of section.elements) {
|
|
@@ -13,7 +13,7 @@ export interface ResearchElement {
|
|
|
13
13
|
coordinates: string | null;
|
|
14
14
|
color: string | null;
|
|
15
15
|
icon: string | null;
|
|
16
|
-
eidx:
|
|
16
|
+
eidx: string | null;
|
|
17
17
|
}
|
|
18
18
|
|
|
19
19
|
export interface ResearchSection {
|
|
@@ -62,8 +62,8 @@ export function mapRowToElement(row: Record<string, string>): ResearchElement |
|
|
|
62
62
|
const name = stripQuotes(colMap.element || '');
|
|
63
63
|
if (!name) return null;
|
|
64
64
|
|
|
65
|
-
|
|
66
|
-
|
|
65
|
+
let eidxRaw = (colMap.eidx || '').trim();
|
|
66
|
+
if (eidxRaw && /^\d+$/.test(eidxRaw)) eidxRaw = `e${eidxRaw}`;
|
|
67
67
|
|
|
68
68
|
const aria = parseAriaLocator(colMap.aria || '-');
|
|
69
69
|
|
|
@@ -76,7 +76,7 @@ export function mapRowToElement(row: Record<string, string>): ResearchElement |
|
|
|
76
76
|
coordinates: (colMap.coordinates || '-').trim() === '-' ? null : colMap.coordinates.trim(),
|
|
77
77
|
color: (colMap.color || '-').trim() === '-' || (colMap.color || '').trim() === '' ? null : colMap.color.trim(),
|
|
78
78
|
icon: (colMap.icon || '-').trim() === '-' || (colMap.icon || '').trim() === '' ? null : colMap.icon.trim(),
|
|
79
|
-
eidx:
|
|
79
|
+
eidx: eidxRaw && eidxRaw !== '-' ? eidxRaw : null,
|
|
80
80
|
};
|
|
81
81
|
}
|
|
82
82
|
|
|
@@ -65,6 +65,7 @@ export class ResearchResult {
|
|
|
65
65
|
}
|
|
66
66
|
|
|
67
67
|
rebuildSectionInText(section: ResearchSection): void {
|
|
68
|
+
if (section.elements.length === 0) return;
|
|
68
69
|
const newTable = rebuildSectionMarkdown(section);
|
|
69
70
|
const escaped = section.name.replace(/"/g, '\\"');
|
|
70
71
|
let sectionQuery = mdq(this.text).query(`section2(~"${escaped}")`);
|
package/src/ai/researcher.ts
CHANGED
|
@@ -131,9 +131,9 @@ export class Researcher extends ResearcherBase implements Agent {
|
|
|
131
131
|
await this.ensureNavigated(state.url, screenshot && this.provider.hasVision());
|
|
132
132
|
await this.hooksRunner.runBeforeHook('researcher', state.url);
|
|
133
133
|
|
|
134
|
-
const
|
|
135
|
-
debugLog(`Annotated ${
|
|
136
|
-
this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot: screenshot && this.provider.hasVision() });
|
|
134
|
+
const { ariaSnapshot, elements: annotatedElements } = await this.explorer.annotateElements();
|
|
135
|
+
debugLog(`Annotated ${annotatedElements.length} interactive elements with eidx`);
|
|
136
|
+
this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot: screenshot && this.provider.hasVision(), ariaSnapshot });
|
|
137
137
|
|
|
138
138
|
if (isErrorPage(this.actionResult!)) {
|
|
139
139
|
const recovered = await this.waitForPageLoad(screenshot);
|
|
@@ -154,7 +154,7 @@ export class Researcher extends ResearcherBase implements Agent {
|
|
|
154
154
|
|
|
155
155
|
const combinedHtml = await this.actionResult!.combinedHtml();
|
|
156
156
|
|
|
157
|
-
if (!deep) {
|
|
157
|
+
if (!deep && !force) {
|
|
158
158
|
const similar = await findSimilarResearch(combinedHtml);
|
|
159
159
|
if (similar) {
|
|
160
160
|
tag('info').log('Similar research found, reusing cached result');
|
|
@@ -385,9 +385,10 @@ export class Researcher extends ResearcherBase implements Agent {
|
|
|
385
385
|
try {
|
|
386
386
|
await withRetry(
|
|
387
387
|
async () => {
|
|
388
|
-
await this.explorer.annotateElements();
|
|
388
|
+
const { ariaSnapshot } = await this.explorer.annotateElements();
|
|
389
389
|
this.actionResult = await this.explorer.createAction().capturePageState({
|
|
390
390
|
includeScreenshot: screenshot && this.provider.hasVision(),
|
|
391
|
+
ariaSnapshot,
|
|
391
392
|
});
|
|
392
393
|
if (isErrorPage(this.actionResult!)) throw new Error('Error page detected');
|
|
393
394
|
},
|
package/src/ai/tools.ts
CHANGED
|
@@ -340,8 +340,13 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
|
|
|
340
340
|
const previousState = ActionResult.fromState(stateManager.getCurrentState()!);
|
|
341
341
|
const formLocator = codeLines[0] || 'form';
|
|
342
342
|
const action = explorer.createAction();
|
|
343
|
+
const wasInIframe = await explorer.isInsideIframe();
|
|
343
344
|
await action.attempt(codeBlock, explanation);
|
|
344
345
|
|
|
346
|
+
if (action.lastError && !wasInIframe && (await explorer.isInsideIframe())) {
|
|
347
|
+
await explorer.switchToMainFrame();
|
|
348
|
+
}
|
|
349
|
+
|
|
345
350
|
const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, formLocator);
|
|
346
351
|
|
|
347
352
|
if (action.lastError) {
|
|
@@ -21,7 +21,7 @@ export class ContextCommand extends BaseCommand {
|
|
|
21
21
|
|
|
22
22
|
const isVisual = args.includes('--visual') || args.includes('--screenshot');
|
|
23
23
|
|
|
24
|
-
await explorer.annotateElements();
|
|
24
|
+
const { ariaSnapshot } = await explorer.annotateElements();
|
|
25
25
|
|
|
26
26
|
if (isVisual) {
|
|
27
27
|
const cachedResearch = Researcher.getCachedResearch(state);
|
|
@@ -29,7 +29,7 @@ export class ContextCommand extends BaseCommand {
|
|
|
29
29
|
await explorer.visuallyAnnotateElements({ containers });
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
-
const actionResult = await explorer.createAction().capturePageState({ includeScreenshot: isVisual });
|
|
32
|
+
const actionResult = await explorer.createAction().capturePageState({ includeScreenshot: isVisual, ariaSnapshot });
|
|
33
33
|
const experienceTracker = explorer.getStateManager().getExperienceTracker();
|
|
34
34
|
const knowledgeTracker = this.explorBot.getKnowledgeTracker();
|
|
35
35
|
|
|
@@ -78,7 +78,7 @@ export class ExploreCommand extends BaseCommand {
|
|
|
78
78
|
}
|
|
79
79
|
|
|
80
80
|
private printResults(savedPath?: string | null): void {
|
|
81
|
-
const allTests = this.completedPlans.flatMap((plan) => plan.tests.filter((t) => t.
|
|
81
|
+
const allTests = this.completedPlans.flatMap((plan) => plan.tests.filter((t) => t.startTime != null).map((test) => ({ test, planTitle: plan.title })));
|
|
82
82
|
|
|
83
83
|
if (allTests.length === 0) return;
|
|
84
84
|
|
|
@@ -2,6 +2,7 @@ import { existsSync, mkdirSync, statSync, writeFileSync } from 'node:fs';
|
|
|
2
2
|
import { dirname, extname, join, resolve } from 'node:path';
|
|
3
3
|
import { log, tag } from '../utils/logger.js';
|
|
4
4
|
import dedent from 'dedent';
|
|
5
|
+
import chalk from 'chalk';
|
|
5
6
|
import { getCliName } from '../utils/cli-name.ts';
|
|
6
7
|
|
|
7
8
|
const DEFAULT_CONFIG_TEMPLATE = `import { createOpenRouter } from '@openrouter/ai-sdk-provider';
|
|
@@ -103,9 +104,11 @@ export function runInitCommand(options: InitCommandOptions): void {
|
|
|
103
104
|
log('2. Set AI models config file');
|
|
104
105
|
log('3. Set web application URL in the config file');
|
|
105
106
|
log('4. Add initial knowledge (how to authorize to the application, etc.)');
|
|
106
|
-
tag('substep').log(`${getCliName()} learn * 'to aurhorize use these credentials: admin@example.com / secret123'`);
|
|
107
|
+
tag('substep').log(chalk.yellow(`${getCliName()} learn * 'to aurhorize use these credentials: admin@example.com / secret123'`));
|
|
108
|
+
tag('substep').log('You can use ${env.LOGIN} and ${env.PASSWORD} to reference environment variables.');
|
|
109
|
+
|
|
107
110
|
log('5. Launch application on a relative URL');
|
|
108
|
-
tag('substep').log(`${getCliName()} start /dashboard`);
|
|
111
|
+
tag('substep').log(chalk.yellow(`${getCliName()} start /dashboard`));
|
|
109
112
|
|
|
110
113
|
if (!existsSync('./output')) {
|
|
111
114
|
mkdirSync('./output', { recursive: true });
|
|
@@ -9,6 +9,7 @@ export class PlanCommand extends BaseCommand {
|
|
|
9
9
|
{ flags: '--fresh', description: 'Regenerate plan from scratch' },
|
|
10
10
|
{ flags: '--clear', description: 'Clear plan before regenerating' },
|
|
11
11
|
{ flags: '--style <name>', description: 'Planning style (normal, curious, psycho, performer)' },
|
|
12
|
+
{ flags: '--focus <feature>', description: 'Focus area for test planning' },
|
|
12
13
|
];
|
|
13
14
|
|
|
14
15
|
async execute(args: string): Promise<void> {
|
|
@@ -16,11 +17,15 @@ export class PlanCommand extends BaseCommand {
|
|
|
16
17
|
const fresh = args.includes('--fresh') || clear;
|
|
17
18
|
const styleMatch = args.match(/--style\s+(\S+)/);
|
|
18
19
|
const style = styleMatch?.[1];
|
|
19
|
-
const
|
|
20
|
+
const focusMatch = args.match(/--focus\s+("[^"]+"|'[^']+'|\S+)/);
|
|
21
|
+
const focusFromFlag = focusMatch?.[1]?.replace(/^["']|["']$/g, '');
|
|
22
|
+
const focusFromText = args
|
|
20
23
|
.replace('--clear', '')
|
|
21
24
|
.replace('--fresh', '')
|
|
22
25
|
.replace(/--style\s+\S+/, '')
|
|
26
|
+
.replace(/--focus\s+("[^"]+"|'[^']+'|\S+)/, '')
|
|
23
27
|
.trim();
|
|
28
|
+
const focus = focusFromFlag || focusFromText;
|
|
24
29
|
|
|
25
30
|
if (clear) {
|
|
26
31
|
this.explorBot.clearPlan();
|
package/src/config.ts
CHANGED
package/src/explorbot.ts
CHANGED
|
@@ -108,7 +108,7 @@ export class ExplorBot {
|
|
|
108
108
|
}
|
|
109
109
|
|
|
110
110
|
getCurrentState(): WebPageState | null {
|
|
111
|
-
return this.explorer
|
|
111
|
+
return this.explorer?.getStateManager().getCurrentState() ?? null;
|
|
112
112
|
}
|
|
113
113
|
|
|
114
114
|
getExplorer(): Explorer {
|