explorbot 0.0.5 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/explorbot-cli.ts +4 -3
- package/dist/bin/explorbot-cli.js +4 -3
- package/dist/src/action.js +14 -11
- package/dist/src/ai/planner/subpages.js +42 -6
- package/dist/src/ai/planner.js +15 -3
- package/dist/src/ai/researcher/cache.js +13 -8
- package/dist/src/ai/researcher/coordinates.js +4 -2
- package/dist/src/ai/researcher/deep-analysis.js +16 -19
- package/dist/src/ai/researcher/locators.js +1 -1
- package/dist/src/ai/researcher/parser.js +4 -3
- package/dist/src/ai/researcher/research-result.js +2 -0
- package/dist/src/ai/researcher.js +6 -5
- package/dist/src/ai/tools.js +4 -0
- package/dist/src/commands/context-command.js +2 -2
- package/dist/src/commands/explore-command.js +1 -1
- package/dist/src/commands/init-command.js +4 -2
- package/dist/src/commands/plan-command.js +6 -1
- package/dist/src/explorbot.js +1 -1
- package/dist/src/explorer.js +58 -16
- package/dist/src/utils/web-element.js +6 -4
- package/package.json +2 -2
- package/src/action.ts +14 -10
- package/src/ai/planner/subpages.ts +37 -7
- package/src/ai/planner.ts +16 -3
- package/src/ai/researcher/cache.ts +14 -8
- package/src/ai/researcher/coordinates.ts +8 -7
- package/src/ai/researcher/deep-analysis.ts +18 -21
- package/src/ai/researcher/locators.ts +3 -3
- package/src/ai/researcher/parser.ts +4 -4
- package/src/ai/researcher/research-result.ts +1 -0
- package/src/ai/researcher.ts +6 -5
- package/src/ai/tools.ts +5 -0
- package/src/commands/context-command.ts +2 -2
- package/src/commands/explore-command.ts +1 -1
- package/src/commands/init-command.ts +5 -2
- package/src/commands/plan-command.ts +6 -1
- package/src/config.ts +1 -0
- package/src/explorbot.ts +1 -1
- package/src/explorer.ts +67 -20
- package/src/utils/web-element.ts +12 -10
package/bin/explorbot-cli.ts
CHANGED
|
@@ -128,10 +128,11 @@ addCommonOptions(program.command('explore <path>').description('Start web explor
|
|
|
128
128
|
}
|
|
129
129
|
});
|
|
130
130
|
|
|
131
|
-
addCommonOptions(program.command('plan <path>
|
|
131
|
+
addCommonOptions(program.command('plan <path>').description('Generate test plan for a page and exit'))
|
|
132
132
|
.option('-a, --append', 'Add tests to existing plan file')
|
|
133
133
|
.option('--style <style>', 'Planning style: normal, curious, psycho')
|
|
134
|
-
.
|
|
134
|
+
.option('--focus <feature>', 'Focus area for test planning')
|
|
135
|
+
.action(async (planPath, options) => {
|
|
135
136
|
try {
|
|
136
137
|
const explorBot = new ExplorBot(buildExplorBotOptions(planPath, options));
|
|
137
138
|
await explorBot.start();
|
|
@@ -146,7 +147,7 @@ addCommonOptions(program.command('plan <path> [feature]').description('Generate
|
|
|
146
147
|
}
|
|
147
148
|
}
|
|
148
149
|
|
|
149
|
-
await explorBot.plan(
|
|
150
|
+
await explorBot.plan(options.focus || undefined, {
|
|
150
151
|
fresh: !options.append,
|
|
151
152
|
style: options.style,
|
|
152
153
|
});
|
|
@@ -102,10 +102,11 @@ addCommonOptions(program.command('explore <path>').description('Start web explor
|
|
|
102
102
|
await showStatsAndExit(1);
|
|
103
103
|
}
|
|
104
104
|
});
|
|
105
|
-
addCommonOptions(program.command('plan <path>
|
|
105
|
+
addCommonOptions(program.command('plan <path>').description('Generate test plan for a page and exit'))
|
|
106
106
|
.option('-a, --append', 'Add tests to existing plan file')
|
|
107
107
|
.option('--style <style>', 'Planning style: normal, curious, psycho')
|
|
108
|
-
.
|
|
108
|
+
.option('--focus <feature>', 'Focus area for test planning')
|
|
109
|
+
.action(async (planPath, options) => {
|
|
109
110
|
try {
|
|
110
111
|
const explorBot = new ExplorBot(buildExplorBotOptions(planPath, options));
|
|
111
112
|
await explorBot.start();
|
|
@@ -117,7 +118,7 @@ addCommonOptions(program.command('plan <path> [feature]').description('Generate
|
|
|
117
118
|
explorBot.loadPlan(existingPlanPath);
|
|
118
119
|
}
|
|
119
120
|
}
|
|
120
|
-
await explorBot.plan(
|
|
121
|
+
await explorBot.plan(options.focus || undefined, {
|
|
121
122
|
fresh: !options.append,
|
|
122
123
|
style: options.style,
|
|
123
124
|
});
|
package/dist/src/action.js
CHANGED
|
@@ -50,7 +50,7 @@ class Action {
|
|
|
50
50
|
return undefined;
|
|
51
51
|
}
|
|
52
52
|
}
|
|
53
|
-
async capturePageState({ includeScreenshot = false } = {}) {
|
|
53
|
+
async capturePageState({ includeScreenshot = false, ariaSnapshot: preCapuredAria } = {}) {
|
|
54
54
|
try {
|
|
55
55
|
const currentState = this.stateManager.getCurrentState();
|
|
56
56
|
const stateHash = currentState?.hash || 'screenshot';
|
|
@@ -90,20 +90,23 @@ class Action {
|
|
|
90
90
|
debugLog('Page:', { url, title, size: html.length, html: html.substring(0, 100) });
|
|
91
91
|
// Capture iframe HTML snapshots
|
|
92
92
|
const iframeSnapshots = await this.captureIframeSnapshots(html);
|
|
93
|
-
let ariaSnapshot = null;
|
|
93
|
+
let ariaSnapshot = preCapuredAria || null;
|
|
94
94
|
let ariaSnapshotFile = undefined;
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
95
|
+
if (!ariaSnapshot) {
|
|
96
|
+
try {
|
|
97
|
+
const page = this.playwrightHelper.page;
|
|
98
|
+
ariaSnapshot = await page.locator('body').ariaSnapshot();
|
|
99
|
+
}
|
|
100
|
+
catch (err) {
|
|
101
|
+
debugLog('ARIA snapshot failed:', err instanceof Error ? `${err.message}\n${err.stack}` : err);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
if (ariaSnapshot) {
|
|
98
105
|
const ariaFileName = `${stateHash}_${timestamp}.aria.yaml`;
|
|
99
106
|
const ariaPath = join(statesDir, ariaFileName);
|
|
100
|
-
fs.writeFileSync(ariaPath,
|
|
101
|
-
ariaSnapshot = serializedSnapshot;
|
|
107
|
+
fs.writeFileSync(ariaPath, ariaSnapshot, 'utf8');
|
|
102
108
|
ariaSnapshotFile = ariaFileName;
|
|
103
109
|
}
|
|
104
|
-
catch (err) {
|
|
105
|
-
debugLog('ARIA snapshot failed:', err instanceof Error ? `${err.message}\n${err.stack}` : err);
|
|
106
|
-
}
|
|
107
110
|
const result = new ActionResult({
|
|
108
111
|
html,
|
|
109
112
|
title,
|
|
@@ -115,7 +118,7 @@ class Action {
|
|
|
115
118
|
iframeSnapshots,
|
|
116
119
|
ariaSnapshot,
|
|
117
120
|
ariaSnapshotFile,
|
|
118
|
-
iframeURL: frame
|
|
121
|
+
iframeURL: frame ? frame.url?.() || 'iframe' : undefined,
|
|
119
122
|
});
|
|
120
123
|
this.stateManager.updateState(result);
|
|
121
124
|
return result;
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import dedent from 'dedent';
|
|
2
2
|
import { z } from 'zod';
|
|
3
|
+
import { ConfigParser } from "../../config.js";
|
|
3
4
|
import { normalizeUrl } from "../../state-manager.js";
|
|
4
5
|
const planRegistry = new Map();
|
|
5
|
-
export function registerPlan(url, plan, feature) {
|
|
6
|
+
export function registerPlan(url, plan, feature, stateHash) {
|
|
6
7
|
const key = buildKey(url, feature);
|
|
7
|
-
planRegistry.set(key, { plan, feature, url });
|
|
8
|
+
planRegistry.set(key, { plan, feature, url, stateHash });
|
|
8
9
|
}
|
|
9
10
|
export function getRegisteredPlan(url, feature) {
|
|
10
11
|
return planRegistry.get(buildKey(url, feature));
|
|
@@ -26,7 +27,36 @@ function buildKey(url, feature) {
|
|
|
26
27
|
return `${normalized}::${feature}`;
|
|
27
28
|
return normalized;
|
|
28
29
|
}
|
|
29
|
-
function
|
|
30
|
+
export function isDynamicSegment(segment) {
|
|
31
|
+
try {
|
|
32
|
+
const configRegex = ConfigParser.getInstance().getConfig().dynamicPageRegex;
|
|
33
|
+
if (configRegex)
|
|
34
|
+
return new RegExp(configRegex, 'i').test(segment);
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
/* config not loaded yet */
|
|
38
|
+
}
|
|
39
|
+
// numeric: /users/123
|
|
40
|
+
if (/^\d+$/.test(segment))
|
|
41
|
+
return true;
|
|
42
|
+
// UUID: /items/550e8400-e29b-41d4-a716-446655440000
|
|
43
|
+
if (/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(segment))
|
|
44
|
+
return true;
|
|
45
|
+
// ULID: /items/01ARZ3NDEKTSV4RRFFQ69G5FAV
|
|
46
|
+
if (/^[0-9A-HJKMNP-TV-Z]{26}$/.test(segment))
|
|
47
|
+
return true;
|
|
48
|
+
// hex ID (4+ chars): /suite/70dae98a
|
|
49
|
+
if (/^[a-f0-9]{4,}$/i.test(segment))
|
|
50
|
+
return true;
|
|
51
|
+
// hex-prefixed slug (8+ hex before dash): /suite/95ef0c94-mobile
|
|
52
|
+
if (/^[a-f0-9]{8,}-/i.test(segment))
|
|
53
|
+
return true;
|
|
54
|
+
// short mixed alphanumeric (digits + letters, ≤8 chars, no dash): /item/x7f2
|
|
55
|
+
if (segment.length <= 8 && !segment.includes('-') && /\d/.test(segment) && /[a-z]/i.test(segment))
|
|
56
|
+
return true;
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
export function isTemplateMatch(urlA, urlB) {
|
|
30
60
|
const partsA = normalizeUrl(urlA).split('/');
|
|
31
61
|
const partsB = normalizeUrl(urlB).split('/');
|
|
32
62
|
if (partsA.length !== partsB.length)
|
|
@@ -38,12 +68,18 @@ function isTemplateMatch(urlA, urlB) {
|
|
|
38
68
|
diffCount++;
|
|
39
69
|
if (diffCount > 1)
|
|
40
70
|
return false;
|
|
41
|
-
|
|
42
|
-
if (!isNumericOrShortId.test(partsA[i]) && !isNumericOrShortId.test(partsB[i]))
|
|
71
|
+
if (!isDynamicSegment(partsA[i]) && !isDynamicSegment(partsB[i]))
|
|
43
72
|
return false;
|
|
44
73
|
}
|
|
45
74
|
return diffCount === 1;
|
|
46
75
|
}
|
|
76
|
+
export function getPlannedByStateHash(hash) {
|
|
77
|
+
for (const record of planRegistry.values()) {
|
|
78
|
+
if (record.stateHash === hash)
|
|
79
|
+
return record;
|
|
80
|
+
}
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
47
83
|
const SubPagePickSchema = z.object({
|
|
48
84
|
url: z.string().nullable(),
|
|
49
85
|
reason: z.string(),
|
|
@@ -58,7 +94,7 @@ export function WithSubPages(Base) {
|
|
|
58
94
|
const pagePath = normalizeUrl(page.url);
|
|
59
95
|
if (!pagePath.startsWith(currentPath) || pagePath === currentPath)
|
|
60
96
|
continue;
|
|
61
|
-
if (
|
|
97
|
+
if (this.findSimilarPlan(page.url))
|
|
62
98
|
continue;
|
|
63
99
|
if (candidates.some((c) => normalizeUrl(c.url) === pagePath))
|
|
64
100
|
continue;
|
package/dist/src/ai/planner.js
CHANGED
|
@@ -14,7 +14,8 @@ import { mdq } from '../utils/markdown-query.js';
|
|
|
14
14
|
import { Conversation } from "./conversation.js";
|
|
15
15
|
import { getActiveStyle, getStyles } from "./planner/styles.js";
|
|
16
16
|
import { WithSessionDedup } from "./planner/session-dedup.js";
|
|
17
|
-
import { WithSubPages, getRegisteredPlan, registerPlan } from "./planner/subpages.js";
|
|
17
|
+
import { WithSubPages, getPlannedByStateHash, getRegisteredPlan, registerPlan } from "./planner/subpages.js";
|
|
18
|
+
import { findSimilarStateHash } from "./researcher/cache.js";
|
|
18
19
|
import { hasFocusedSection } from "./researcher/focus.js";
|
|
19
20
|
import { POSSIBLE_SECTIONS, Researcher } from "./researcher.js";
|
|
20
21
|
import { fileUploadRule, protectionRule } from "./rules.js";
|
|
@@ -108,13 +109,24 @@ export class Planner extends PlannerBase {
|
|
|
108
109
|
debugLog('Planning:', state?.url);
|
|
109
110
|
if (!state)
|
|
110
111
|
throw new Error('No state found');
|
|
111
|
-
if (!
|
|
112
|
+
if (!feature && !this.currentPlan && state.url) {
|
|
112
113
|
const similar = this.findSimilarPlan(state.url);
|
|
113
114
|
if (similar) {
|
|
114
115
|
tag('info').log(`Similar page already planned: ${similar.url} (${similar.plan.tests.length} tests)`);
|
|
115
116
|
this.registerPlanInSession(similar.plan);
|
|
116
117
|
return similar.plan;
|
|
117
118
|
}
|
|
119
|
+
const actionResult = ActionResult.fromState(state);
|
|
120
|
+
const combinedHtml = await actionResult.combinedHtml();
|
|
121
|
+
const similarHash = await findSimilarStateHash(combinedHtml);
|
|
122
|
+
if (similarHash) {
|
|
123
|
+
const planned = getPlannedByStateHash(similarHash);
|
|
124
|
+
if (planned) {
|
|
125
|
+
tag('info').log(`Page content similar to already-planned: ${planned.url} — skipping`);
|
|
126
|
+
this.registerPlanInSession(planned.plan);
|
|
127
|
+
return planned.plan;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
118
130
|
}
|
|
119
131
|
if (!this.freshStart && !this.currentPlan && state.url) {
|
|
120
132
|
this.currentPlan = Planner.getCachedPlan(state.url);
|
|
@@ -185,7 +197,7 @@ export class Planner extends PlannerBase {
|
|
|
185
197
|
tag('success').log(`Planning complete! ${this.currentPlan.tests.length} tests in plan: ${this.currentPlan.title}`);
|
|
186
198
|
tag('info').log(`Planning style: ${this.lastStyleName} (available: ${availableStyles})`);
|
|
187
199
|
if (state.url)
|
|
188
|
-
registerPlan(state.url, this.currentPlan, feature);
|
|
200
|
+
registerPlan(state.url, this.currentPlan, feature, state.hash);
|
|
189
201
|
this.registerPlanInSession(this.currentPlan);
|
|
190
202
|
return this.currentPlan;
|
|
191
203
|
}
|
|
@@ -59,7 +59,7 @@ export function saveResearch(hash, text, combinedHtml) {
|
|
|
59
59
|
}
|
|
60
60
|
return researchFile;
|
|
61
61
|
}
|
|
62
|
-
|
|
62
|
+
function findSimilarMatch(combinedHtml) {
|
|
63
63
|
const statesDir = getStatesDir();
|
|
64
64
|
if (!existsSync(statesDir))
|
|
65
65
|
return Promise.resolve(null);
|
|
@@ -76,13 +76,8 @@ export function findSimilarResearch(combinedHtml) {
|
|
|
76
76
|
resolve(null);
|
|
77
77
|
return;
|
|
78
78
|
}
|
|
79
|
-
debugLog(`Similar
|
|
80
|
-
|
|
81
|
-
if (research) {
|
|
82
|
-
resolve(research);
|
|
83
|
-
return;
|
|
84
|
-
}
|
|
85
|
-
resolve(null);
|
|
79
|
+
debugLog(`Similar fingerprint found: ${matchHash} (${similarity}% similar)`);
|
|
80
|
+
resolve({ hash: matchHash, similarity });
|
|
86
81
|
};
|
|
87
82
|
worker.postMessage({
|
|
88
83
|
html: combinedHtml,
|
|
@@ -92,3 +87,13 @@ export function findSimilarResearch(combinedHtml) {
|
|
|
92
87
|
});
|
|
93
88
|
});
|
|
94
89
|
}
|
|
90
|
+
export async function findSimilarResearch(combinedHtml) {
|
|
91
|
+
const match = await findSimilarMatch(combinedHtml);
|
|
92
|
+
if (!match)
|
|
93
|
+
return null;
|
|
94
|
+
return getCachedResearch(match.hash) || null;
|
|
95
|
+
}
|
|
96
|
+
export async function findSimilarStateHash(combinedHtml) {
|
|
97
|
+
const match = await findSimilarMatch(combinedHtml);
|
|
98
|
+
return match?.hash || null;
|
|
99
|
+
}
|
|
@@ -115,9 +115,11 @@ export function WithCoordinates(Base) {
|
|
|
115
115
|
const text = aiResult.text || '';
|
|
116
116
|
const rows = mdq(text).query('table').toJson();
|
|
117
117
|
for (const row of rows) {
|
|
118
|
-
|
|
119
|
-
if (
|
|
118
|
+
let eidx = (row.eidx || '').trim();
|
|
119
|
+
if (!eidx || eidx === '-')
|
|
120
120
|
continue;
|
|
121
|
+
if (/^\d+$/.test(eidx))
|
|
122
|
+
eidx = `e${eidx}`;
|
|
121
123
|
const val = (v) => (v && v !== '-' ? v : null);
|
|
122
124
|
elements.set(eidx, {
|
|
123
125
|
coordinates: val(row.Coordinates),
|
|
@@ -68,7 +68,7 @@ export function WithDeepAnalysis(Base) {
|
|
|
68
68
|
From this UI research, identify elements that could reveal hidden UI when clicked
|
|
69
69
|
(dropdown menus, popups, expandable panels, accordion sections, overflow menus, tab switches).
|
|
70
70
|
|
|
71
|
-
Available eidx
|
|
71
|
+
Available eidx refs: ${eidxList}
|
|
72
72
|
|
|
73
73
|
${researchText}
|
|
74
74
|
|
|
@@ -76,7 +76,7 @@ export function WithDeepAnalysis(Base) {
|
|
|
76
76
|
- Only pick elements that HIDE content until clicked (menus, dropdowns, accordions, tabs)
|
|
77
77
|
- Skip regular links, data items, and navigation
|
|
78
78
|
- For repeated elements (same expand button on every row), pick only the FIRST one
|
|
79
|
-
- Respond with comma-separated eidx
|
|
79
|
+
- Respond with comma-separated eidx refs only, e.g.: e3, e7, e15
|
|
80
80
|
`;
|
|
81
81
|
const model = this.provider.getModelForAgent('researcher');
|
|
82
82
|
const textCall = this.provider.chat([{ role: 'user', content: textPrompt }], model, {
|
|
@@ -87,7 +87,7 @@ export function WithDeepAnalysis(Base) {
|
|
|
87
87
|
const screenshot = this.actionResult?.screenshot;
|
|
88
88
|
if (screenshot && this.provider.hasVision()) {
|
|
89
89
|
const visionPrompt = dedent `
|
|
90
|
-
This screenshot has interactive elements labeled with eidx
|
|
90
|
+
This screenshot has interactive elements labeled with eidx refs (solid bordered boxes with labels).
|
|
91
91
|
Identify elements that could reveal hidden UI when clicked.
|
|
92
92
|
|
|
93
93
|
Look for: overflow/ellipsis menus, chevron dropdowns, hamburger menus,
|
|
@@ -96,30 +96,27 @@ export function WithDeepAnalysis(Base) {
|
|
|
96
96
|
Rules:
|
|
97
97
|
- For repeated icons (same icon on every list row), pick only the FIRST one
|
|
98
98
|
- Skip regular text buttons, links, and navigation items
|
|
99
|
-
- Respond with comma-separated eidx
|
|
99
|
+
- Respond with comma-separated eidx refs only, e.g.: e3, e7, e15
|
|
100
100
|
`;
|
|
101
101
|
visionCall = this.provider.processImage(visionPrompt, screenshot.toString('base64'));
|
|
102
102
|
}
|
|
103
103
|
const [textRes, visionRes] = await Promise.all([textCall, visionCall]);
|
|
104
104
|
const eidxSet = new Set();
|
|
105
|
+
const parseRefs = (text) => {
|
|
106
|
+
if (!text)
|
|
107
|
+
return [];
|
|
108
|
+
const matches = text.match(/e?\d+/g) || [];
|
|
109
|
+
const refs = matches.map((m) => (m.startsWith('e') ? m : `e${m}`));
|
|
110
|
+
return refs.filter((r) => allElements.has(r));
|
|
111
|
+
};
|
|
105
112
|
for (const res of [textRes, visionRes]) {
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
const nums = res.text.match(/\d+/g)?.map(Number) || [];
|
|
109
|
-
for (const n of nums) {
|
|
110
|
-
if (allElements.has(n))
|
|
111
|
-
eidxSet.add(n);
|
|
113
|
+
for (const ref of parseRefs(res?.text)) {
|
|
114
|
+
eidxSet.add(ref);
|
|
112
115
|
}
|
|
113
116
|
}
|
|
114
|
-
const
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
.filter((n) => allElements.has(n)) || [];
|
|
118
|
-
const visionNums = visionRes?.text
|
|
119
|
-
?.match(/\d+/g)
|
|
120
|
-
?.map(Number)
|
|
121
|
-
.filter((n) => allElements.has(n)) || [];
|
|
122
|
-
debugLog(`Text model picked eidx: [${textNums.join(', ')}], Vision model picked eidx: [${visionNums.join(', ')}]`);
|
|
117
|
+
const textRefs = parseRefs(textRes?.text);
|
|
118
|
+
const visionRefs = parseRefs(visionRes?.text);
|
|
119
|
+
debugLog(`Text model picked eidx: [${textRefs.join(', ')}], Vision model picked eidx: [${visionRefs.join(', ')}]`);
|
|
123
120
|
return [...eidxSet].map((eidx) => allElements.get(eidx));
|
|
124
121
|
}
|
|
125
122
|
_buildClickCommands(el) {
|
|
@@ -126,7 +126,7 @@ export function WithLocators(Base) {
|
|
|
126
126
|
for (const fixedSection of fixedSections) {
|
|
127
127
|
const originalSections = parseResearchSections(result.text);
|
|
128
128
|
const original = originalSections.find((s) => s.name === fixedSection.name);
|
|
129
|
-
if (!original)
|
|
129
|
+
if (!original || original.elements.length === 0)
|
|
130
130
|
continue;
|
|
131
131
|
if (fixedSection.containerCss && fixedSection.containerCss !== original.containerCss) {
|
|
132
132
|
debugLog(`Fixed container for "${fixedSection.name}": '${original.containerCss}' → '${fixedSection.containerCss}'`);
|
|
@@ -37,8 +37,9 @@ export function mapRowToElement(row) {
|
|
|
37
37
|
const name = stripQuotes(colMap.element || '');
|
|
38
38
|
if (!name)
|
|
39
39
|
return null;
|
|
40
|
-
|
|
41
|
-
|
|
40
|
+
let eidxRaw = (colMap.eidx || '').trim();
|
|
41
|
+
if (eidxRaw && /^\d+$/.test(eidxRaw))
|
|
42
|
+
eidxRaw = `e${eidxRaw}`;
|
|
42
43
|
const aria = parseAriaLocator(colMap.aria || '-');
|
|
43
44
|
return {
|
|
44
45
|
name,
|
|
@@ -49,7 +50,7 @@ export function mapRowToElement(row) {
|
|
|
49
50
|
coordinates: (colMap.coordinates || '-').trim() === '-' ? null : colMap.coordinates.trim(),
|
|
50
51
|
color: (colMap.color || '-').trim() === '-' || (colMap.color || '').trim() === '' ? null : colMap.color.trim(),
|
|
51
52
|
icon: (colMap.icon || '-').trim() === '-' || (colMap.icon || '').trim() === '' ? null : colMap.icon.trim(),
|
|
52
|
-
eidx:
|
|
53
|
+
eidx: eidxRaw && eidxRaw !== '-' ? eidxRaw : null,
|
|
53
54
|
};
|
|
54
55
|
}
|
|
55
56
|
export function extractContainerFromBlockquote(sectionMarkdown) {
|
|
@@ -62,6 +62,8 @@ export class ResearchResult {
|
|
|
62
62
|
this.rebuildSectionInText(section);
|
|
63
63
|
}
|
|
64
64
|
rebuildSectionInText(section) {
|
|
65
|
+
if (section.elements.length === 0)
|
|
66
|
+
return;
|
|
65
67
|
const newTable = rebuildSectionMarkdown(section);
|
|
66
68
|
const escaped = section.name.replace(/"/g, '\\"');
|
|
67
69
|
let sectionQuery = mdq(this.text).query(`section2(~"${escaped}")`);
|
|
@@ -98,9 +98,9 @@ export class Researcher extends ResearcherBase {
|
|
|
98
98
|
setActivity(`${this.emoji} Researching...`, 'action');
|
|
99
99
|
await this.ensureNavigated(state.url, screenshot && this.provider.hasVision());
|
|
100
100
|
await this.hooksRunner.runBeforeHook('researcher', state.url);
|
|
101
|
-
const
|
|
102
|
-
debugLog(`Annotated ${
|
|
103
|
-
this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot: screenshot && this.provider.hasVision() });
|
|
101
|
+
const { ariaSnapshot, elements: annotatedElements } = await this.explorer.annotateElements();
|
|
102
|
+
debugLog(`Annotated ${annotatedElements.length} interactive elements with eidx`);
|
|
103
|
+
this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot: screenshot && this.provider.hasVision(), ariaSnapshot });
|
|
104
104
|
if (isErrorPage(this.actionResult)) {
|
|
105
105
|
const recovered = await this.waitForPageLoad(screenshot);
|
|
106
106
|
if (!recovered) {
|
|
@@ -117,7 +117,7 @@ export class Researcher extends ResearcherBase {
|
|
|
117
117
|
}
|
|
118
118
|
debugLog('Researching web page:', this.actionResult.url);
|
|
119
119
|
const combinedHtml = await this.actionResult.combinedHtml();
|
|
120
|
-
if (!deep) {
|
|
120
|
+
if (!deep && !force) {
|
|
121
121
|
const similar = await findSimilarResearch(combinedHtml);
|
|
122
122
|
if (similar) {
|
|
123
123
|
tag('info').log('Similar research found, reusing cached result');
|
|
@@ -325,9 +325,10 @@ export class Researcher extends ResearcherBase {
|
|
|
325
325
|
return false;
|
|
326
326
|
try {
|
|
327
327
|
await withRetry(async () => {
|
|
328
|
-
await this.explorer.annotateElements();
|
|
328
|
+
const { ariaSnapshot } = await this.explorer.annotateElements();
|
|
329
329
|
this.actionResult = await this.explorer.createAction().capturePageState({
|
|
330
330
|
includeScreenshot: screenshot && this.provider.hasVision(),
|
|
331
|
+
ariaSnapshot,
|
|
331
332
|
});
|
|
332
333
|
if (isErrorPage(this.actionResult))
|
|
333
334
|
throw new Error('Error page detected');
|
package/dist/src/ai/tools.js
CHANGED
|
@@ -301,7 +301,11 @@ export function createCodeceptJSTools(explorer, task) {
|
|
|
301
301
|
const previousState = ActionResult.fromState(stateManager.getCurrentState());
|
|
302
302
|
const formLocator = codeLines[0] || 'form';
|
|
303
303
|
const action = explorer.createAction();
|
|
304
|
+
const wasInIframe = await explorer.isInsideIframe();
|
|
304
305
|
await action.attempt(codeBlock, explanation);
|
|
306
|
+
if (action.lastError && !wasInIframe && (await explorer.isInsideIframe())) {
|
|
307
|
+
await explorer.switchToMainFrame();
|
|
308
|
+
}
|
|
305
309
|
const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, formLocator);
|
|
306
310
|
if (action.lastError) {
|
|
307
311
|
const message = action.lastError ? String(action.lastError) : 'Unknown error';
|
|
@@ -15,13 +15,13 @@ export class ContextCommand extends BaseCommand {
|
|
|
15
15
|
throw new Error('No active page to show context for');
|
|
16
16
|
}
|
|
17
17
|
const isVisual = args.includes('--visual') || args.includes('--screenshot');
|
|
18
|
-
await explorer.annotateElements();
|
|
18
|
+
const { ariaSnapshot } = await explorer.annotateElements();
|
|
19
19
|
if (isVisual) {
|
|
20
20
|
const cachedResearch = Researcher.getCachedResearch(state);
|
|
21
21
|
const containers = cachedResearch ? extractValidContainers(cachedResearch) : [];
|
|
22
22
|
await explorer.visuallyAnnotateElements({ containers });
|
|
23
23
|
}
|
|
24
|
-
const actionResult = await explorer.createAction().capturePageState({ includeScreenshot: isVisual });
|
|
24
|
+
const actionResult = await explorer.createAction().capturePageState({ includeScreenshot: isVisual, ariaSnapshot });
|
|
25
25
|
const experienceTracker = explorer.getStateManager().getExperienceTracker();
|
|
26
26
|
const knowledgeTracker = this.explorBot.getKnowledgeTracker();
|
|
27
27
|
let mode = 'compact';
|
|
@@ -72,7 +72,7 @@ export class ExploreCommand extends BaseCommand {
|
|
|
72
72
|
}
|
|
73
73
|
}
|
|
74
74
|
printResults(savedPath) {
|
|
75
|
-
const allTests = this.completedPlans.flatMap((plan) => plan.tests.filter((t) => t.
|
|
75
|
+
const allTests = this.completedPlans.flatMap((plan) => plan.tests.filter((t) => t.startTime != null).map((test) => ({ test, planTitle: plan.title })));
|
|
76
76
|
if (allTests.length === 0)
|
|
77
77
|
return;
|
|
78
78
|
const hasSubPages = this.completedPlans.length > 1;
|
|
@@ -2,6 +2,7 @@ import { existsSync, mkdirSync, statSync, writeFileSync } from 'node:fs';
|
|
|
2
2
|
import { dirname, extname, join, resolve } from 'node:path';
|
|
3
3
|
import { log, tag } from '../utils/logger.js';
|
|
4
4
|
import dedent from 'dedent';
|
|
5
|
+
import chalk from 'chalk';
|
|
5
6
|
import { getCliName } from "../utils/cli-name.js";
|
|
6
7
|
const DEFAULT_CONFIG_TEMPLATE = `import { createOpenRouter } from '@openrouter/ai-sdk-provider';
|
|
7
8
|
// import { '<your provider here>' } from '<your provider package here>';
|
|
@@ -95,9 +96,10 @@ export function runInitCommand(options) {
|
|
|
95
96
|
log('2. Set AI models config file');
|
|
96
97
|
log('3. Set web application URL in the config file');
|
|
97
98
|
log('4. Add initial knowledge (how to authorize to the application, etc.)');
|
|
98
|
-
tag('substep').log(`${getCliName()} learn * 'to aurhorize use these credentials: admin@example.com / secret123'`);
|
|
99
|
+
tag('substep').log(chalk.yellow(`${getCliName()} learn * 'to aurhorize use these credentials: admin@example.com / secret123'`));
|
|
100
|
+
tag('substep').log('You can use ${env.LOGIN} and ${env.PASSWORD} to reference environment variables.');
|
|
99
101
|
log('5. Launch application on a relative URL');
|
|
100
|
-
tag('substep').log(`${getCliName()} start /dashboard`);
|
|
102
|
+
tag('substep').log(chalk.yellow(`${getCliName()} start /dashboard`));
|
|
101
103
|
if (!existsSync('./output')) {
|
|
102
104
|
mkdirSync('./output', { recursive: true });
|
|
103
105
|
log('Created directory: ./output');
|
|
@@ -8,17 +8,22 @@ export class PlanCommand extends BaseCommand {
|
|
|
8
8
|
{ flags: '--fresh', description: 'Regenerate plan from scratch' },
|
|
9
9
|
{ flags: '--clear', description: 'Clear plan before regenerating' },
|
|
10
10
|
{ flags: '--style <name>', description: 'Planning style (normal, curious, psycho, performer)' },
|
|
11
|
+
{ flags: '--focus <feature>', description: 'Focus area for test planning' },
|
|
11
12
|
];
|
|
12
13
|
async execute(args) {
|
|
13
14
|
const clear = args.includes('--clear');
|
|
14
15
|
const fresh = args.includes('--fresh') || clear;
|
|
15
16
|
const styleMatch = args.match(/--style\s+(\S+)/);
|
|
16
17
|
const style = styleMatch?.[1];
|
|
17
|
-
const
|
|
18
|
+
const focusMatch = args.match(/--focus\s+("[^"]+"|'[^']+'|\S+)/);
|
|
19
|
+
const focusFromFlag = focusMatch?.[1]?.replace(/^["']|["']$/g, '');
|
|
20
|
+
const focusFromText = args
|
|
18
21
|
.replace('--clear', '')
|
|
19
22
|
.replace('--fresh', '')
|
|
20
23
|
.replace(/--style\s+\S+/, '')
|
|
24
|
+
.replace(/--focus\s+("[^"]+"|'[^']+'|\S+)/, '')
|
|
21
25
|
.trim();
|
|
26
|
+
const focus = focusFromFlag || focusFromText;
|
|
22
27
|
if (clear) {
|
|
23
28
|
this.explorBot.clearPlan();
|
|
24
29
|
tag('success').log('Plan cleared');
|
package/dist/src/explorbot.js
CHANGED
|
@@ -84,7 +84,7 @@ export class ExplorBot {
|
|
|
84
84
|
await this.explorer.openFreshTab();
|
|
85
85
|
}
|
|
86
86
|
getCurrentState() {
|
|
87
|
-
return this.explorer
|
|
87
|
+
return this.explorer?.getStateManager().getCurrentState() ?? null;
|
|
88
88
|
}
|
|
89
89
|
getExplorer() {
|
|
90
90
|
return this.explorer;
|
package/dist/src/explorer.js
CHANGED
|
@@ -15,6 +15,7 @@ import { StateManager } from './state-manager.js';
|
|
|
15
15
|
import { RequestStore } from "./api/request-store.js";
|
|
16
16
|
import { XhrCapture } from "./api/xhr-capture.js";
|
|
17
17
|
import { createDebug, log, tag } from './utils/logger.js';
|
|
18
|
+
import { WebElement, extractElementData } from "./utils/web-element.js";
|
|
18
19
|
const debugLog = createDebug('explorbot:explorer');
|
|
19
20
|
const FATAL_BROWSER_ERRORS = /Frame was detached|Target closed|Execution context was destroyed|Protocol error|Session closed/i;
|
|
20
21
|
class Explorer {
|
|
@@ -243,19 +244,7 @@ class Explorer {
|
|
|
243
244
|
return action;
|
|
244
245
|
}
|
|
245
246
|
async annotateElements() {
|
|
246
|
-
|
|
247
|
-
const roles = ['button', 'link', 'textbox', 'searchbox', 'checkbox', 'radio', 'switch', 'combobox', 'tab', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'option', 'slider', 'spinbutton', 'treeitem'];
|
|
248
|
-
let idx = 1;
|
|
249
|
-
for (const role of roles) {
|
|
250
|
-
const elements = await page.getByRole(role).all();
|
|
251
|
-
for (const el of elements) {
|
|
252
|
-
await el.evaluate((node, i) => {
|
|
253
|
-
node.setAttribute('data-explorbot-eidx', String(i));
|
|
254
|
-
}, idx);
|
|
255
|
-
idx++;
|
|
256
|
-
}
|
|
257
|
-
}
|
|
258
|
-
return idx - 1;
|
|
247
|
+
return annotatePageElements(this.playwrightHelper.page);
|
|
259
248
|
}
|
|
260
249
|
async visuallyAnnotateElements(opts) {
|
|
261
250
|
return visuallyAnnotateContainers(this.playwrightHelper.page, opts?.containers || []);
|
|
@@ -269,7 +258,7 @@ class Explorer {
|
|
|
269
258
|
for (const el of elements) {
|
|
270
259
|
const attr = await el.getAttribute('data-explorbot-eidx');
|
|
271
260
|
if (attr)
|
|
272
|
-
result.push(
|
|
261
|
+
result.push(attr);
|
|
273
262
|
}
|
|
274
263
|
return result;
|
|
275
264
|
}
|
|
@@ -286,8 +275,7 @@ class Explorer {
|
|
|
286
275
|
const page = this.playwrightHelper.page;
|
|
287
276
|
const base = container ? page.locator(container) : page;
|
|
288
277
|
const el = locator.startsWith('//') ? base.locator(`xpath=${locator}`) : base.locator(locator);
|
|
289
|
-
|
|
290
|
-
return eidx ? Number.parseInt(eidx, 10) : null;
|
|
278
|
+
return await el.first().getAttribute('data-explorbot-eidx');
|
|
291
279
|
}
|
|
292
280
|
catch (error) {
|
|
293
281
|
if (this.isFatalBrowserError(error)) {
|
|
@@ -607,4 +595,58 @@ function toCodeceptjsTest(test) {
|
|
|
607
595
|
codeceptjsTest.notes = test.getPrintableNotes();
|
|
608
596
|
return codeceptjsTest;
|
|
609
597
|
}
|
|
598
|
+
const REF_LINE_PATTERN = /^(\s*)-\s+(\w+)\s*(?:"([^"]*)")?.*?\[ref=(e\d+)\]/;
|
|
599
|
+
const ANNOTATABLE_ROLES = new Set(['button', 'link', 'textbox', 'searchbox', 'checkbox', 'radio', 'switch', 'combobox', 'tab', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'option', 'slider', 'spinbutton', 'treeitem']);
|
|
600
|
+
function parseAriaRefs(ariaSnapshot) {
|
|
601
|
+
const entries = [];
|
|
602
|
+
for (const line of ariaSnapshot.split('\n')) {
|
|
603
|
+
const match = line.match(REF_LINE_PATTERN);
|
|
604
|
+
if (!match)
|
|
605
|
+
continue;
|
|
606
|
+
if (!ANNOTATABLE_ROLES.has(match[2]))
|
|
607
|
+
continue;
|
|
608
|
+
entries.push({ role: match[2], name: match[3] || '', ref: match[4] });
|
|
609
|
+
}
|
|
610
|
+
return entries;
|
|
611
|
+
}
|
|
612
|
+
export async function annotatePageElements(page) {
|
|
613
|
+
const ariaSnapshot = await page.locator('body').ariaSnapshot({ forAI: true });
|
|
614
|
+
const refEntries = parseAriaRefs(ariaSnapshot);
|
|
615
|
+
const byRole = new Map();
|
|
616
|
+
for (const { role, name, ref } of refEntries) {
|
|
617
|
+
let list = byRole.get(role);
|
|
618
|
+
if (!list) {
|
|
619
|
+
list = [];
|
|
620
|
+
byRole.set(role, list);
|
|
621
|
+
}
|
|
622
|
+
list.push({ name, ref });
|
|
623
|
+
}
|
|
624
|
+
const elements = [];
|
|
625
|
+
for (const [role, entries] of byRole) {
|
|
626
|
+
try {
|
|
627
|
+
const rawList = await page.getByRole(role).evaluateAll((domElements, [data, extractFnStr]) => {
|
|
628
|
+
const extract = new Function(`return ${extractFnStr}`)();
|
|
629
|
+
const results = [];
|
|
630
|
+
let ariaIdx = 0;
|
|
631
|
+
for (const el of domElements) {
|
|
632
|
+
if (ariaIdx >= data.length)
|
|
633
|
+
break;
|
|
634
|
+
el.setAttribute('data-explorbot-eidx', data[ariaIdx].ref);
|
|
635
|
+
const elData = extract(el);
|
|
636
|
+
if (elData)
|
|
637
|
+
results.push(elData);
|
|
638
|
+
ariaIdx++;
|
|
639
|
+
}
|
|
640
|
+
return results;
|
|
641
|
+
}, [entries, extractElementData.toString()]);
|
|
642
|
+
for (const raw of rawList) {
|
|
643
|
+
elements.push(WebElement.fromRawData(raw, role));
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
catch {
|
|
647
|
+
debugLog(`Failed to annotate role=${role}`);
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
return { ariaSnapshot, elements };
|
|
651
|
+
}
|
|
610
652
|
export default Explorer;
|