explorbot 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -5
- package/bin/explorbot-cli.ts +29 -7
- package/boat/api-tester/src/config.ts +45 -3
- package/boat/doc-collector/src/ai/documentarian.ts +5 -2
- package/boat/doc-collector/src/ai/tools.ts +12 -11
- package/boat/doc-collector/src/cli.ts +1 -6
- package/boat/doc-collector/src/docbot.ts +3 -3
- package/boat/doc-collector/src/screenshots.ts +2 -2
- package/dist/bin/explorbot-cli.js +26 -6
- package/dist/boat/api-tester/src/config.js +43 -4
- package/dist/boat/doc-collector/src/ai/documentarian.js +4 -2
- package/dist/boat/doc-collector/src/ai/tools.js +11 -11
- package/dist/boat/doc-collector/src/cli.js +1 -5
- package/dist/boat/doc-collector/src/docbot.js +3 -3
- package/dist/boat/doc-collector/src/screenshots.js +2 -2
- package/dist/models.json +30 -0
- package/dist/package.json +5 -2
- package/dist/src/action.d.ts +11 -3
- package/dist/src/action.js +22 -5
- package/dist/src/ai/agent.d.ts +17 -0
- package/dist/src/ai/captain/idle-mode.js +1 -1
- package/dist/src/ai/captain/test-mode.js +1 -1
- package/dist/src/ai/captain/web-mode.js +18 -22
- package/dist/src/ai/captain.js +13 -26
- package/dist/src/ai/driller.d.ts +2 -13
- package/dist/src/ai/driller.js +23 -38
- package/dist/src/ai/historian/codeceptjs.js +1 -1
- package/dist/src/ai/historian/playwright.d.ts +3 -2
- package/dist/src/ai/historian/playwright.js +1 -1
- package/dist/src/ai/historian/screencast.d.ts +2 -1
- package/dist/src/ai/historian/screencast.js +2 -2
- package/dist/src/ai/historian.d.ts +5 -4
- package/dist/src/ai/navigator.d.ts +6 -2
- package/dist/src/ai/navigator.js +26 -22
- package/dist/src/ai/pilot.d.ts +13 -6
- package/dist/src/ai/pilot.js +49 -44
- package/dist/src/ai/planner.d.ts +3 -2
- package/dist/src/ai/planner.js +19 -17
- package/dist/src/ai/provider.js +0 -1
- package/dist/src/ai/quartermaster.d.ts +3 -2
- package/dist/src/ai/quartermaster.js +4 -4
- package/dist/src/ai/rerunner.d.ts +2 -11
- package/dist/src/ai/rerunner.js +9 -22
- package/dist/src/ai/researcher/coordinates.js +4 -3
- package/dist/src/ai/researcher/deep-analysis.d.ts +2 -0
- package/dist/src/ai/researcher/deep-analysis.js +6 -6
- package/dist/src/ai/researcher/locators.d.ts +9 -2
- package/dist/src/ai/researcher/locators.js +65 -8
- package/dist/src/ai/researcher/sections.d.ts +2 -0
- package/dist/src/ai/researcher/sections.js +2 -2
- package/dist/src/ai/researcher.js +29 -54
- package/dist/src/ai/task-agent.d.ts +18 -5
- package/dist/src/ai/task-agent.js +35 -6
- package/dist/src/ai/tester.d.ts +6 -12
- package/dist/src/ai/tester.js +37 -51
- package/dist/src/ai/tools.d.ts +11 -10
- package/dist/src/ai/tools.js +24 -36
- package/dist/src/command-handler.js +1 -1
- package/dist/src/commands/context-aria-command.js +1 -1
- package/dist/src/commands/context-command.js +7 -5
- package/dist/src/commands/context-data-command.js +2 -2
- package/dist/src/commands/context-experience-command.js +2 -2
- package/dist/src/commands/context-html-command.js +2 -2
- package/dist/src/commands/context-knowledge-command.js +1 -1
- package/dist/src/commands/drill-command.js +1 -1
- package/dist/src/commands/explore-command.js +4 -3
- package/dist/src/commands/freesail-command.js +2 -2
- package/dist/src/commands/learn-command.js +4 -4
- package/dist/src/commands/path-command.js +1 -1
- package/dist/src/commands/research-command.js +1 -1
- package/dist/src/commands/test-command.js +1 -1
- package/dist/src/components/App.js +1 -1
- package/dist/src/config.d.ts +17 -0
- package/dist/src/config.js +144 -10
- package/dist/src/experience-tracker.js +1 -1
- package/dist/src/explorbot.d.ts +17 -8
- package/dist/src/explorbot.js +74 -36
- package/dist/src/explorer.d.ts +66 -91
- package/dist/src/explorer.js +326 -567
- package/dist/src/state-manager.d.ts +6 -1
- package/dist/src/state-manager.js +1 -0
- package/dist/src/utils/hooks-runner.js +2 -4
- package/dist/src/utils/test-files.js +1 -1
- package/dist/src/utils/web-annotate.d.ts +5 -0
- package/dist/src/utils/web-annotate.js +58 -0
- package/dist/src/utils/web-eidx.d.ts +2 -0
- package/dist/src/utils/web-eidx.js +20 -0
- package/dist/src/utils/web-element.d.ts +1 -0
- package/dist/src/utils/web-element.js +29 -0
- package/models.json +30 -0
- package/package.json +5 -2
- package/src/action.ts +25 -5
- package/src/ai/agent.ts +20 -0
- package/src/ai/captain/idle-mode.ts +1 -1
- package/src/ai/captain/test-mode.ts +1 -1
- package/src/ai/captain/web-mode.ts +18 -23
- package/src/ai/captain.ts +12 -25
- package/src/ai/driller.ts +50 -67
- package/src/ai/historian/codeceptjs.ts +1 -1
- package/src/ai/historian/playwright.ts +4 -3
- package/src/ai/historian/screencast.ts +4 -3
- package/src/ai/historian.ts +3 -2
- package/src/ai/navigator.ts +30 -25
- package/src/ai/pilot.ts +55 -47
- package/src/ai/planner.ts +22 -20
- package/src/ai/provider.ts +0 -1
- package/src/ai/quartermaster.ts +5 -4
- package/src/ai/rerunner.ts +10 -29
- package/src/ai/researcher/coordinates.ts +4 -3
- package/src/ai/researcher/deep-analysis.ts +8 -6
- package/src/ai/researcher/locators.ts +66 -9
- package/src/ai/researcher/sections.ts +4 -2
- package/src/ai/researcher.ts +32 -61
- package/src/ai/task-agent.ts +45 -11
- package/src/ai/tester.ts +40 -57
- package/src/ai/tools.ts +32 -53
- package/src/command-handler.ts +1 -1
- package/src/commands/context-aria-command.ts +1 -1
- package/src/commands/context-command.ts +7 -5
- package/src/commands/context-data-command.ts +2 -2
- package/src/commands/context-experience-command.ts +2 -2
- package/src/commands/context-html-command.ts +2 -2
- package/src/commands/context-knowledge-command.ts +1 -1
- package/src/commands/drill-command.ts +1 -1
- package/src/commands/explore-command.ts +4 -3
- package/src/commands/freesail-command.ts +2 -2
- package/src/commands/learn-command.ts +4 -4
- package/src/commands/path-command.ts +1 -1
- package/src/commands/research-command.ts +1 -1
- package/src/commands/test-command.ts +1 -1
- package/src/components/App.tsx +1 -1
- package/src/config.ts +173 -11
- package/src/experience-tracker.ts +1 -1
- package/src/explorbot.ts +78 -39
- package/src/explorer.ts +389 -628
- package/src/state-manager.ts +7 -1
- package/src/utils/hooks-runner.ts +2 -4
- package/src/utils/test-files.ts +1 -1
- package/src/utils/web-annotate.ts +64 -0
- package/src/utils/web-eidx.ts +21 -0
- package/src/utils/web-element.ts +30 -0
|
@@ -4,6 +4,7 @@ import { parseAriaLocator } from "../../utils/aria.js";
|
|
|
4
4
|
import { tag } from '../../utils/logger.js';
|
|
5
5
|
import { mdq } from "../../utils/markdown-query.js";
|
|
6
6
|
import { WebElement } from "../../utils/web-element.js";
|
|
7
|
+
import { isDynamicId } from "../../utils/xpath.js";
|
|
7
8
|
import { locatorRule as generalLocatorRuleText } from '../rules.js';
|
|
8
9
|
import { FOCUSED_MARKER } from "./focus.js";
|
|
9
10
|
import { debugLog } from "./mixin.js";
|
|
@@ -26,7 +27,7 @@ function buildPwLocatorString(loc) {
|
|
|
26
27
|
}
|
|
27
28
|
export function WithLocators(Base) {
|
|
28
29
|
return class extends Base {
|
|
29
|
-
async testLocators(locators) {
|
|
30
|
+
async testLocators(locators, opts = {}) {
|
|
30
31
|
let broken = 0;
|
|
31
32
|
for (const loc of locators) {
|
|
32
33
|
if (executionController.isInterrupted())
|
|
@@ -40,21 +41,23 @@ export function WithLocators(Base) {
|
|
|
40
41
|
continue;
|
|
41
42
|
}
|
|
42
43
|
try {
|
|
43
|
-
const count = await this.explorer.
|
|
44
|
+
const count = await this.explorer.withPage((page) => {
|
|
44
45
|
const base = loc.container ? page.locator(loc.container) : page;
|
|
45
46
|
if (loc.type === 'aria') {
|
|
46
47
|
const parsed = parseAriaLocator(loc.locator);
|
|
47
48
|
if (!parsed)
|
|
48
|
-
return page.locator('__invalid__');
|
|
49
|
-
return base.getByRole(parsed.role, { name: parsed.text });
|
|
49
|
+
return page.locator('__invalid__').count();
|
|
50
|
+
return base.getByRole(parsed.role, { name: parsed.text }).count();
|
|
50
51
|
}
|
|
51
52
|
const converted = loc.locator.replace(/:contains\(/g, ':has-text(');
|
|
52
53
|
if (converted !== loc.locator) {
|
|
53
54
|
loc.locator = converted;
|
|
54
55
|
}
|
|
55
|
-
return base.locator(loc.locator);
|
|
56
|
+
return base.locator(loc.locator).count();
|
|
56
57
|
});
|
|
57
58
|
loc.valid = count === 1;
|
|
59
|
+
if (opts.scope && count > 1)
|
|
60
|
+
loc.valid = true;
|
|
58
61
|
loc.pwLocator = buildPwLocatorString(loc);
|
|
59
62
|
if (!loc.valid) {
|
|
60
63
|
loc.error = count === 0 ? '0 elements' : `${count} elements`;
|
|
@@ -173,7 +176,7 @@ export function WithLocators(Base) {
|
|
|
173
176
|
}
|
|
174
177
|
}
|
|
175
178
|
if (needsXpath.length > 0) {
|
|
176
|
-
const webElements = await this.explorer.
|
|
179
|
+
const webElements = await this.explorer.withPage((page) => WebElement.fromEidxList(page, needsXpath));
|
|
177
180
|
const changedSections = new Set();
|
|
178
181
|
for (const w of webElements) {
|
|
179
182
|
const entry = needsXpathEls.get(w.eidx);
|
|
@@ -188,6 +191,60 @@ export function WithLocators(Base) {
|
|
|
188
191
|
}
|
|
189
192
|
await this.validateContainers(result);
|
|
190
193
|
}
|
|
194
|
+
async resolveContainers(result) {
|
|
195
|
+
const containerLocs = result.containerLocators;
|
|
196
|
+
await this.testLocators(containerLocs, { scope: true });
|
|
197
|
+
for (const loc of containerLocs.filter((l) => l.valid === false)) {
|
|
198
|
+
if (await this.recoverContainerFromChildren(result, loc.locator))
|
|
199
|
+
loc.valid = true;
|
|
200
|
+
}
|
|
201
|
+
return containerLocs.filter((l) => l.valid === false);
|
|
202
|
+
}
|
|
203
|
+
async recoverContainerFromChildren(result, css) {
|
|
204
|
+
const sections = parseResearchSections(result.text).filter((s) => s.containerCss === css);
|
|
205
|
+
let recovered = 0;
|
|
206
|
+
for (const section of sections) {
|
|
207
|
+
const eidxList = section.elements.map((el) => el.eidx).filter(Boolean);
|
|
208
|
+
if (eidxList.length < 2)
|
|
209
|
+
continue;
|
|
210
|
+
const ancestor = await this.explorer.runWithBrowserRecovery('recoverContainerFromChildren', () => WebElement.commonAncestor(this.explorer.playwrightHelper.page, eidxList));
|
|
211
|
+
if (!ancestor)
|
|
212
|
+
continue;
|
|
213
|
+
const candidates = [];
|
|
214
|
+
const id = ancestor.attrs.id;
|
|
215
|
+
if (id && !isDynamicId(id))
|
|
216
|
+
candidates.push(`#${id}`);
|
|
217
|
+
for (const cls of ancestor.filteredClasses) {
|
|
218
|
+
if (!/^[\w-]+$/.test(cls))
|
|
219
|
+
continue;
|
|
220
|
+
candidates.push(`${ancestor.tag}.${cls}`);
|
|
221
|
+
}
|
|
222
|
+
let unique = null;
|
|
223
|
+
let multiple = null;
|
|
224
|
+
for (const candidate of candidates) {
|
|
225
|
+
let count = 0;
|
|
226
|
+
try {
|
|
227
|
+
count = await this.explorer.playwrightLocatorCount((page) => page.locator(candidate));
|
|
228
|
+
}
|
|
229
|
+
catch { }
|
|
230
|
+
if (count === 1) {
|
|
231
|
+
unique = candidate;
|
|
232
|
+
break;
|
|
233
|
+
}
|
|
234
|
+
if (count > 1 && !multiple)
|
|
235
|
+
multiple = candidate;
|
|
236
|
+
}
|
|
237
|
+
const newCss = unique || multiple;
|
|
238
|
+
if (!newCss)
|
|
239
|
+
continue;
|
|
240
|
+
debugLog(`Recovered container: '${css}' → '${newCss}' in "${section.name}"`);
|
|
241
|
+
this.updateSectionContainer(result, section, newCss);
|
|
242
|
+
recovered++;
|
|
243
|
+
}
|
|
244
|
+
if (recovered > 0 && recovered < sections.length)
|
|
245
|
+
debugLog(`Container '${css}' recovered in ${recovered}/${sections.length} sections, still broken for the rest`);
|
|
246
|
+
return recovered > 0 && recovered === sections.length;
|
|
247
|
+
}
|
|
191
248
|
async validateContainers(result) {
|
|
192
249
|
const sections = parseResearchSections(result.text);
|
|
193
250
|
for (const section of sections) {
|
|
@@ -195,7 +252,7 @@ export function WithLocators(Base) {
|
|
|
195
252
|
continue;
|
|
196
253
|
let count = 0;
|
|
197
254
|
try {
|
|
198
|
-
count = await this.explorer.
|
|
255
|
+
count = await this.explorer.withPage((page) => page.locator(section.containerCss).count());
|
|
199
256
|
}
|
|
200
257
|
catch { }
|
|
201
258
|
if (count >= 1)
|
|
@@ -204,7 +261,7 @@ export function WithLocators(Base) {
|
|
|
204
261
|
if (simplified) {
|
|
205
262
|
let simplifiedCount = 0;
|
|
206
263
|
try {
|
|
207
|
-
simplifiedCount = await this.explorer.
|
|
264
|
+
simplifiedCount = await this.explorer.withPage((page) => page.locator(simplified).count());
|
|
208
265
|
}
|
|
209
266
|
catch { }
|
|
210
267
|
if (simplifiedCount >= 1) {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { ActionResult } from '../../action-result.js';
|
|
2
|
+
import type { ExplorbotConfig } from '../../config.js';
|
|
2
3
|
import type Explorer from '../../explorer.js';
|
|
3
4
|
import type { StateManager } from '../../state-manager.js';
|
|
4
5
|
import type { Provider } from '../provider.js';
|
|
@@ -11,6 +12,7 @@ export declare function WithSections<T extends Constructor>(Base: T): {
|
|
|
11
12
|
explorer: Explorer;
|
|
12
13
|
provider: Provider;
|
|
13
14
|
stateManager: StateManager;
|
|
15
|
+
config: ExplorbotConfig;
|
|
14
16
|
actionResult: ActionResult | undefined;
|
|
15
17
|
researchBySections(): Promise<string>;
|
|
16
18
|
_detectFocusCss(): Promise<string | null>;
|
|
@@ -50,11 +50,11 @@ export function WithSections(Base) {
|
|
|
50
50
|
return focused.text;
|
|
51
51
|
}
|
|
52
52
|
async _detectFocusCss() {
|
|
53
|
-
const focusSections = this.
|
|
53
|
+
const focusSections = this.config.ai?.agents?.researcher?.focusSections;
|
|
54
54
|
if (!focusSections?.length)
|
|
55
55
|
return null;
|
|
56
56
|
for (const css of focusSections) {
|
|
57
|
-
const count = await this.explorer.
|
|
57
|
+
const count = await this.explorer.withPage((page) => page.locator(css).count()).catch(() => 0);
|
|
58
58
|
if (count > 0)
|
|
59
59
|
return css;
|
|
60
60
|
}
|
|
@@ -7,11 +7,11 @@ import { Observability } from "../observability.js";
|
|
|
7
7
|
import { Stats } from "../stats.js";
|
|
8
8
|
import { diffAriaSnapshots } from "../utils/aria.js";
|
|
9
9
|
import { ErrorPageError, detectPageCondition } from "../utils/error-page.js";
|
|
10
|
-
import { HooksRunner } from "../utils/hooks-runner.js";
|
|
11
10
|
import { isBodyEmpty } from "../utils/html.js";
|
|
12
11
|
import { createDebug, tag } from '../utils/logger.js';
|
|
13
12
|
import { mdq } from "../utils/markdown-query.js";
|
|
14
13
|
import { RulesLoader } from "../utils/rules-loader.js";
|
|
14
|
+
import { annotatePageElements } from "../utils/web-annotate.js";
|
|
15
15
|
import { ContextLengthError } from './provider.js';
|
|
16
16
|
import { findSimilarResearch, getCachedResearch, saveResearch } from "./researcher/cache.js";
|
|
17
17
|
import { WithCoordinates } from "./researcher/coordinates.js";
|
|
@@ -39,15 +39,10 @@ export class Researcher extends ResearcherBase {
|
|
|
39
39
|
emoji = '🔍';
|
|
40
40
|
experienceTracker;
|
|
41
41
|
hasScreenshotToAnalyze = false;
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
this.provider = provider;
|
|
47
|
-
this.stateManager = explorer.getStateManager();
|
|
48
|
-
this.experienceTracker = this.stateManager.getExperienceTracker();
|
|
49
|
-
this.hooksRunner = new HooksRunner(explorer, explorer.getConfig());
|
|
50
|
-
const ai = explorer.getConfig().ai;
|
|
42
|
+
constructor(deps) {
|
|
43
|
+
super(deps);
|
|
44
|
+
this.experienceTracker = deps.stateManager.getExperienceTracker();
|
|
45
|
+
const ai = deps.config.ai;
|
|
51
46
|
if (ai) {
|
|
52
47
|
ai.agents ??= {};
|
|
53
48
|
ai.agents.researcher ??= {};
|
|
@@ -57,15 +52,6 @@ export class Researcher extends ResearcherBase {
|
|
|
57
52
|
getNavigator() {
|
|
58
53
|
throw new Error('not implemented');
|
|
59
54
|
}
|
|
60
|
-
getExperienceTracker() {
|
|
61
|
-
return this.experienceTracker;
|
|
62
|
-
}
|
|
63
|
-
getKnowledgeTracker() {
|
|
64
|
-
return this.explorer.getKnowledgeTracker();
|
|
65
|
-
}
|
|
66
|
-
getProvider() {
|
|
67
|
-
return this.provider;
|
|
68
|
-
}
|
|
69
55
|
static getCachedResearch(state) {
|
|
70
56
|
return getCachedResearch(state.hash || '');
|
|
71
57
|
}
|
|
@@ -82,7 +68,7 @@ export class Researcher extends ResearcherBase {
|
|
|
82
68
|
}
|
|
83
69
|
async research(state, opts = {}) {
|
|
84
70
|
const { screenshot = false, force = false, deep = false, data = false, fix = true } = opts;
|
|
85
|
-
const maxRetries = this.
|
|
71
|
+
const maxRetries = this.config.ai?.agents?.researcher?.retries ?? 2;
|
|
86
72
|
let retriesLeft = opts._retriesLeft ?? maxRetries;
|
|
87
73
|
this.actionResult = ActionResult.fromState(state);
|
|
88
74
|
const stateHash = state.hash || this.actionResult.getStateHash();
|
|
@@ -101,9 +87,9 @@ export class Researcher extends ResearcherBase {
|
|
|
101
87
|
setActivity(`${this.emoji} Researching...`, 'action');
|
|
102
88
|
await this.ensureNavigated(displayUrl, screenshot && this.provider.hasVision());
|
|
103
89
|
await this.hooksRunner.runBeforeHook('researcher', state.url);
|
|
104
|
-
const annotatedElements = await this.explorer.
|
|
90
|
+
const { elements: annotatedElements } = await this.explorer.withPage(annotatePageElements);
|
|
105
91
|
debugLog(`Annotated ${annotatedElements.length} interactive elements with eidx`);
|
|
106
|
-
this.actionResult = await this.explorer.
|
|
92
|
+
this.actionResult = await this.explorer.capture({ screenshot: screenshot && this.provider.hasVision() });
|
|
107
93
|
const condition = detectPageCondition(this.actionResult);
|
|
108
94
|
if (condition === 'error') {
|
|
109
95
|
tag('warning').log(`Detected error page at ${state.url}`);
|
|
@@ -165,11 +151,10 @@ export class Researcher extends ResearcherBase {
|
|
|
165
151
|
return this.research(state, { ...opts, force: true, _retriesLeft: retriesLeft - 1 });
|
|
166
152
|
}
|
|
167
153
|
if (!interrupted()) {
|
|
168
|
-
const
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
tag('warning').log(`All ${containerLocs.length} containers broken, retrying research (${maxRetries - retriesLeft + 1}/${maxRetries})...`);
|
|
154
|
+
const brokenContainers = await this.resolveContainers(result);
|
|
155
|
+
const containerCount = result.containers.length;
|
|
156
|
+
if (containerCount > 0 && brokenContainers.length === containerCount && retriesLeft > 0) {
|
|
157
|
+
tag('warning').log(`All ${containerCount} containers broken, retrying research (${maxRetries - retriesLeft + 1}/${maxRetries})...`);
|
|
173
158
|
await new Promise((r) => setTimeout(r, 2000));
|
|
174
159
|
return this.research(state, { ...opts, force: true, _retriesLeft: retriesLeft - 1 });
|
|
175
160
|
}
|
|
@@ -181,13 +166,6 @@ export class Researcher extends ResearcherBase {
|
|
|
181
166
|
}
|
|
182
167
|
const toTest = result.locators.filter((l) => l.valid === null);
|
|
183
168
|
await this.testLocators(toTest);
|
|
184
|
-
const brokenCount = result.locators.filter((l) => l.valid === false).length;
|
|
185
|
-
const brokenRatio = result.locators.length > 0 ? brokenCount / result.locators.length : 0;
|
|
186
|
-
if (brokenRatio > 0.8 && retriesLeft > 0) {
|
|
187
|
-
tag('warning').log(`${Math.round(brokenRatio * 100)}% locators broken, waiting 3s and retrying research (${maxRetries - retriesLeft + 1}/${maxRetries})...`);
|
|
188
|
-
await new Promise((r) => setTimeout(r, 3000));
|
|
189
|
-
return this.research(state, { ...opts, force: true, _retriesLeft: retriesLeft - 1 });
|
|
190
|
-
}
|
|
191
169
|
}
|
|
192
170
|
// Stage 3: Fix broken sections via AI conversation continuation
|
|
193
171
|
if (!interrupted() && fix && result.locators.some((l) => l.valid === false)) {
|
|
@@ -197,7 +175,7 @@ export class Researcher extends ResearcherBase {
|
|
|
197
175
|
// Must run BEFORE visuallyAnnotateContainers — annotation overlays inject z-index 99998+ which would pollute the scoring.
|
|
198
176
|
if (!interrupted() && this.hasScreenshotToAnalyze) {
|
|
199
177
|
const sections = parseResearchSections(result.text);
|
|
200
|
-
const focused = await this.explorer.
|
|
178
|
+
const focused = await this.explorer.withPage((page) => detectFocusedSection(page, sections));
|
|
201
179
|
if (focused)
|
|
202
180
|
markSectionAsFocused(result, focused);
|
|
203
181
|
}
|
|
@@ -206,11 +184,11 @@ export class Researcher extends ResearcherBase {
|
|
|
206
184
|
const validContainers = extractValidContainers(result.text);
|
|
207
185
|
result.parseLocators();
|
|
208
186
|
const freshContainerLocs = result.containerLocators;
|
|
209
|
-
await this.testLocators(freshContainerLocs);
|
|
187
|
+
await this.testLocators(freshContainerLocs, { scope: true });
|
|
210
188
|
const freshBroken = freshContainerLocs.filter((l) => l.valid === false).map((l) => l.locator);
|
|
211
189
|
const containers = validContainers.filter((c) => !freshBroken.includes(c.css));
|
|
212
190
|
await this.visuallyAnnotateElements({ containers });
|
|
213
|
-
this.actionResult = await this.explorer.
|
|
191
|
+
this.actionResult = await this.explorer.capture({ screenshot: true });
|
|
214
192
|
const visualResult = await this.analyzeScreenshotForVisualProps();
|
|
215
193
|
if (visualResult.elements.size > 0) {
|
|
216
194
|
await this.mergeVisualData(result, visualResult.elements);
|
|
@@ -280,8 +258,7 @@ export class Researcher extends ResearcherBase {
|
|
|
280
258
|
async ensureNavigated(url, screenshot) {
|
|
281
259
|
if (!this.actionResult) {
|
|
282
260
|
debugLog('No action result, navigating to URL');
|
|
283
|
-
await this.explorer.visit(url);
|
|
284
|
-
this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot });
|
|
261
|
+
this.actionResult = await this.explorer.visit(url, { screenshot });
|
|
285
262
|
return;
|
|
286
263
|
}
|
|
287
264
|
const isOnCurrentState = this.actionResult.getStateHash() === this.stateManager.getCurrentState()?.hash;
|
|
@@ -289,34 +266,32 @@ export class Researcher extends ResearcherBase {
|
|
|
289
266
|
const isEmpty = isBodyEmpty(stateHtml);
|
|
290
267
|
if (!isEmpty && isOnCurrentState) {
|
|
291
268
|
if ((!this.actionResult.screenshot && screenshot) || !this.actionResult.ariaSnapshot) {
|
|
292
|
-
this.actionResult = await this.explorer.
|
|
269
|
+
this.actionResult = await this.explorer.capture({ screenshot });
|
|
293
270
|
}
|
|
294
271
|
return;
|
|
295
272
|
}
|
|
296
273
|
if (isEmpty && isOnCurrentState) {
|
|
297
274
|
debugLog('HTML body empty on current URL, waiting for content');
|
|
298
275
|
tag('step').log('Page body is empty, waiting for content...');
|
|
299
|
-
await this.explorer.visit(url);
|
|
300
|
-
this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false });
|
|
276
|
+
this.actionResult = await this.explorer.visit(url, { screenshot: screenshot ?? false });
|
|
301
277
|
await this.waitUntilSettled(screenshot ?? false);
|
|
302
278
|
return;
|
|
303
279
|
}
|
|
304
280
|
debugLog('Not on current state, navigating to URL');
|
|
305
281
|
tag('step').log('Navigating to URL...');
|
|
306
|
-
await this.explorer.visit(url);
|
|
307
|
-
this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false });
|
|
282
|
+
this.actionResult = await this.explorer.visit(url, { screenshot: screenshot ?? false });
|
|
308
283
|
}
|
|
309
284
|
async waitUntilSettled(screenshot) {
|
|
310
|
-
const errorPageTimeout = this.
|
|
285
|
+
const errorPageTimeout = this.config.ai?.agents?.researcher?.errorPageTimeout ?? 10;
|
|
311
286
|
if (errorPageTimeout <= 0)
|
|
312
287
|
return false;
|
|
313
288
|
const includeScreenshot = screenshot && this.provider.hasVision();
|
|
314
289
|
try {
|
|
315
|
-
await this.explorer.
|
|
290
|
+
await this.explorer.withPage((page) => page.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 }));
|
|
316
291
|
}
|
|
317
292
|
catch { }
|
|
318
|
-
await this.explorer.
|
|
319
|
-
this.actionResult = await this.explorer.
|
|
293
|
+
await this.explorer.withPage(annotatePageElements);
|
|
294
|
+
this.actionResult = await this.explorer.capture({ screenshot: includeScreenshot });
|
|
320
295
|
let condition = detectPageCondition(this.actionResult);
|
|
321
296
|
if (condition === 'error') {
|
|
322
297
|
throw new ErrorPageError(this.actionResult.url, this.actionResult.title, this.actionResult.httpStatus);
|
|
@@ -325,8 +300,8 @@ export class Researcher extends ResearcherBase {
|
|
|
325
300
|
return true;
|
|
326
301
|
for (let i = 0; i < 3; i++) {
|
|
327
302
|
await new Promise((r) => setTimeout(r, 1000));
|
|
328
|
-
await this.explorer.
|
|
329
|
-
this.actionResult = await this.explorer.
|
|
303
|
+
await this.explorer.withPage(annotatePageElements);
|
|
304
|
+
this.actionResult = await this.explorer.capture({ screenshot: includeScreenshot });
|
|
330
305
|
condition = detectPageCondition(this.actionResult);
|
|
331
306
|
if (condition === 'error') {
|
|
332
307
|
throw new ErrorPageError(this.actionResult.url, this.actionResult.title, this.actionResult.httpStatus);
|
|
@@ -337,7 +312,7 @@ export class Researcher extends ResearcherBase {
|
|
|
337
312
|
return false;
|
|
338
313
|
}
|
|
339
314
|
getConfiguredSections() {
|
|
340
|
-
const configSections = this.
|
|
315
|
+
const configSections = this.config.ai?.agents?.researcher?.sections;
|
|
341
316
|
if (!configSections?.length)
|
|
342
317
|
return POSSIBLE_SECTIONS;
|
|
343
318
|
const filtered = {};
|
|
@@ -404,7 +379,7 @@ export class Researcher extends ResearcherBase {
|
|
|
404
379
|
if (!this.actionResult)
|
|
405
380
|
throw new Error('actionResult is not set');
|
|
406
381
|
const html = await this.actionResult.combinedHtml();
|
|
407
|
-
const knowledge = this.
|
|
382
|
+
const knowledge = this.knowledgeTracker.renderRelevantKnowledge(this.actionResult);
|
|
408
383
|
const ariaSnapshot = this.actionResult.getCompactARIA();
|
|
409
384
|
return dedent `
|
|
410
385
|
Analyze this web page and provide a comprehensive research report in markdown format.
|
|
@@ -651,9 +626,9 @@ export class Researcher extends ResearcherBase {
|
|
|
651
626
|
}
|
|
652
627
|
async cancelInUi() {
|
|
653
628
|
const beforeAria = this.stateManager.getCurrentState()?.ariaSnapshot || null;
|
|
654
|
-
await this.explorer.
|
|
629
|
+
await this.explorer.action().execute('I.clickXY(0, 0)');
|
|
655
630
|
if (diffAriaSnapshots(beforeAria, this.stateManager.getCurrentState()?.ariaSnapshot || null).text)
|
|
656
631
|
return;
|
|
657
|
-
await this.explorer.
|
|
632
|
+
await this.explorer.action().execute(`I.pressKey('Escape')`);
|
|
658
633
|
}
|
|
659
634
|
}
|
|
@@ -1,27 +1,40 @@
|
|
|
1
1
|
import type { ActionResult } from '../action-result.js';
|
|
2
|
+
import type { ExplorbotConfig } from '../config.js';
|
|
2
3
|
import type { ExperienceTracker } from '../experience-tracker.js';
|
|
4
|
+
import type Explorer from '../explorer.js';
|
|
3
5
|
import type { KnowledgeTracker } from '../knowledge-tracker.js';
|
|
6
|
+
import type { StateManager } from '../state-manager.js';
|
|
7
|
+
import { HooksRunner } from '../utils/hooks-runner.js';
|
|
8
|
+
import type { AgentDeps, ToolDeps } from './agent.js';
|
|
4
9
|
import { Historian } from './historian.js';
|
|
5
10
|
import type { Navigator } from './navigator.js';
|
|
6
11
|
import type { Provider } from './provider.js';
|
|
7
12
|
import { Quartermaster } from './quartermaster.js';
|
|
8
13
|
export declare function isInteractive(): boolean;
|
|
9
14
|
export declare abstract class TaskAgent {
|
|
15
|
+
explorer: Explorer;
|
|
16
|
+
provider: Provider;
|
|
17
|
+
config: ExplorbotConfig;
|
|
18
|
+
stateManager: StateManager;
|
|
19
|
+
knowledgeTracker: KnowledgeTracker;
|
|
20
|
+
hooksRunner: HooksRunner;
|
|
10
21
|
consecutiveFailures: number;
|
|
11
22
|
consecutiveEmptyResults: number;
|
|
12
23
|
recentToolCalls: any[];
|
|
13
24
|
readonly ACTION_TOOLS: string[];
|
|
14
25
|
_historian: Historian | null;
|
|
15
26
|
_quartermaster: Quartermaster | null;
|
|
27
|
+
constructor(deps?: AgentDeps);
|
|
28
|
+
setHistorian(historian: Historian): void;
|
|
29
|
+
setQuartermaster(quartermaster: Quartermaster): void;
|
|
16
30
|
abstract getNavigator(): Navigator;
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
31
|
+
get toolDeps(): ToolDeps;
|
|
32
|
+
getExperienceTracker(): ExperienceTracker;
|
|
33
|
+
getKnowledgeTracker(): KnowledgeTracker;
|
|
34
|
+
getProvider(): Provider;
|
|
20
35
|
getKnowledge(actionResult: ActionResult): string;
|
|
21
36
|
getExperience(actionResult: ActionResult): string;
|
|
22
|
-
setHistorian(historian: Historian): void;
|
|
23
37
|
getHistorian(): Historian;
|
|
24
|
-
setQuartermaster(quartermaster: Quartermaster): void;
|
|
25
38
|
getQuartermaster(): Quartermaster;
|
|
26
39
|
trackToolExecutions(toolExecutions: any[]): void;
|
|
27
40
|
resetFailureCount(): void;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { HooksRunner } from "../utils/hooks-runner.js";
|
|
1
2
|
export function isInteractive() {
|
|
2
3
|
return process.env.INK_RUNNING === 'true';
|
|
3
4
|
}
|
|
@@ -7,29 +8,57 @@ function createNullProxy() {
|
|
|
7
8
|
});
|
|
8
9
|
}
|
|
9
10
|
export class TaskAgent {
|
|
11
|
+
explorer;
|
|
12
|
+
provider;
|
|
13
|
+
config;
|
|
14
|
+
stateManager;
|
|
15
|
+
knowledgeTracker;
|
|
16
|
+
hooksRunner;
|
|
10
17
|
consecutiveFailures = 0;
|
|
11
18
|
consecutiveEmptyResults = 0;
|
|
12
19
|
recentToolCalls = [];
|
|
13
20
|
ACTION_TOOLS = [];
|
|
14
21
|
_historian = null;
|
|
15
22
|
_quartermaster = null;
|
|
23
|
+
constructor(deps) {
|
|
24
|
+
if (!deps)
|
|
25
|
+
return;
|
|
26
|
+
this.explorer = deps.explorer;
|
|
27
|
+
this.provider = deps.ai;
|
|
28
|
+
this.config = deps.config;
|
|
29
|
+
this.stateManager = deps.stateManager;
|
|
30
|
+
this.knowledgeTracker = deps.knowledgeTracker;
|
|
31
|
+
this.hooksRunner = new HooksRunner(deps.explorer, deps.config);
|
|
32
|
+
}
|
|
33
|
+
setHistorian(historian) {
|
|
34
|
+
this._historian = historian;
|
|
35
|
+
}
|
|
36
|
+
setQuartermaster(quartermaster) {
|
|
37
|
+
this._quartermaster = quartermaster;
|
|
38
|
+
}
|
|
39
|
+
get toolDeps() {
|
|
40
|
+
return { explorer: this.explorer, stateManager: this.stateManager, ai: this.provider };
|
|
41
|
+
}
|
|
42
|
+
getExperienceTracker() {
|
|
43
|
+
return this.stateManager.getExperienceTracker();
|
|
44
|
+
}
|
|
45
|
+
getKnowledgeTracker() {
|
|
46
|
+
return this.knowledgeTracker;
|
|
47
|
+
}
|
|
48
|
+
getProvider() {
|
|
49
|
+
return this.provider;
|
|
50
|
+
}
|
|
16
51
|
getKnowledge(actionResult) {
|
|
17
52
|
return this.getKnowledgeTracker().renderRelevantKnowledge(actionResult);
|
|
18
53
|
}
|
|
19
54
|
getExperience(actionResult) {
|
|
20
55
|
return this.getExperienceTracker().renderExperienceTocFor(actionResult);
|
|
21
56
|
}
|
|
22
|
-
setHistorian(historian) {
|
|
23
|
-
this._historian = historian;
|
|
24
|
-
}
|
|
25
57
|
getHistorian() {
|
|
26
58
|
if (this._historian)
|
|
27
59
|
return this._historian;
|
|
28
60
|
return createNullProxy();
|
|
29
61
|
}
|
|
30
|
-
setQuartermaster(quartermaster) {
|
|
31
|
-
this._quartermaster = quartermaster;
|
|
32
|
-
}
|
|
33
62
|
getQuartermaster() {
|
|
34
63
|
if (this._quartermaster)
|
|
35
64
|
return this._quartermaster;
|
package/dist/src/ai/tester.d.ts
CHANGED
|
@@ -1,22 +1,20 @@
|
|
|
1
1
|
import { ActionResult } from '../action-result.js';
|
|
2
|
-
import type {
|
|
3
|
-
import type
|
|
2
|
+
import type { RequestStore } from '../api/request-store.js';
|
|
3
|
+
import type { TestRun } from '../explorer.js';
|
|
4
4
|
import { type Test } from '../test-plan.js';
|
|
5
|
-
import {
|
|
6
|
-
import type { Agent } from './agent.js';
|
|
5
|
+
import type { Agent, AgentDeps } from './agent.js';
|
|
7
6
|
import type { Captain } from './captain.js';
|
|
8
7
|
import type { Conversation } from './conversation.js';
|
|
9
8
|
import { Navigator } from './navigator.js';
|
|
10
9
|
import type { Pilot } from './pilot.js';
|
|
11
|
-
import { Provider } from './provider.js';
|
|
12
10
|
import { Researcher } from './researcher.js';
|
|
13
11
|
import { TaskAgent } from './task-agent.js';
|
|
14
12
|
export declare class Tester extends TaskAgent implements Agent {
|
|
15
13
|
readonly ACTION_TOOLS: string[];
|
|
16
14
|
readonly SPECIAL_CONTEXT_ACTION_TOOLS: string[];
|
|
17
15
|
emoji: string;
|
|
18
|
-
|
|
19
|
-
|
|
16
|
+
requestStore: RequestStore;
|
|
17
|
+
testRun: TestRun | null;
|
|
20
18
|
currentConversation: Conversation | null;
|
|
21
19
|
pilot: Pilot | null;
|
|
22
20
|
captain: Captain | null;
|
|
@@ -30,16 +28,12 @@ export declare class Tester extends TaskAgent implements Agent {
|
|
|
30
28
|
previousStateHash: string | null;
|
|
31
29
|
pageStateHash: string | null;
|
|
32
30
|
pageActionResult: ActionResult | null;
|
|
33
|
-
hooksRunner: HooksRunner;
|
|
34
31
|
seenUiMapUrls: Set<string>;
|
|
35
32
|
lastAnalyzedStateHash: string | null;
|
|
36
33
|
stalledIterations: number;
|
|
37
34
|
readonly MAX_STALLED_ITERATIONS = 3;
|
|
38
|
-
constructor(
|
|
35
|
+
constructor(deps: AgentDeps, researcher: Researcher, navigator: Navigator, agentTools?: any);
|
|
39
36
|
getNavigator(): Navigator;
|
|
40
|
-
getExperienceTracker(): ExperienceTracker;
|
|
41
|
-
getKnowledgeTracker(): import("../knowledge-tracker.js").KnowledgeTracker;
|
|
42
|
-
getProvider(): Provider;
|
|
43
37
|
setPilot(pilot: Pilot): void;
|
|
44
38
|
setCaptain(captain: Captain): void;
|
|
45
39
|
getCurrentState(): ActionResult;
|