explorbot 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -5
- package/bin/explorbot-cli.ts +29 -7
- package/boat/api-tester/src/config.ts +45 -3
- package/boat/doc-collector/src/ai/documentarian.ts +5 -2
- package/boat/doc-collector/src/ai/tools.ts +12 -11
- package/boat/doc-collector/src/cli.ts +1 -6
- package/boat/doc-collector/src/docbot.ts +3 -3
- package/boat/doc-collector/src/screenshots.ts +2 -2
- package/dist/bin/explorbot-cli.js +26 -6
- package/dist/boat/api-tester/src/config.js +43 -4
- package/dist/boat/doc-collector/src/ai/documentarian.js +4 -2
- package/dist/boat/doc-collector/src/ai/tools.js +11 -11
- package/dist/boat/doc-collector/src/cli.js +1 -5
- package/dist/boat/doc-collector/src/docbot.js +3 -3
- package/dist/boat/doc-collector/src/screenshots.js +2 -2
- package/dist/models.json +30 -0
- package/dist/package.json +5 -2
- package/dist/src/action.d.ts +11 -3
- package/dist/src/action.js +22 -5
- package/dist/src/ai/agent.d.ts +17 -0
- package/dist/src/ai/captain/idle-mode.js +1 -1
- package/dist/src/ai/captain/test-mode.js +1 -1
- package/dist/src/ai/captain/web-mode.js +18 -22
- package/dist/src/ai/captain.js +13 -26
- package/dist/src/ai/driller.d.ts +2 -13
- package/dist/src/ai/driller.js +23 -38
- package/dist/src/ai/historian/codeceptjs.js +1 -1
- package/dist/src/ai/historian/playwright.d.ts +3 -2
- package/dist/src/ai/historian/playwright.js +1 -1
- package/dist/src/ai/historian/screencast.d.ts +2 -1
- package/dist/src/ai/historian/screencast.js +2 -2
- package/dist/src/ai/historian.d.ts +5 -4
- package/dist/src/ai/navigator.d.ts +6 -2
- package/dist/src/ai/navigator.js +26 -22
- package/dist/src/ai/pilot.d.ts +13 -6
- package/dist/src/ai/pilot.js +49 -44
- package/dist/src/ai/planner.d.ts +3 -2
- package/dist/src/ai/planner.js +19 -17
- package/dist/src/ai/provider.js +0 -1
- package/dist/src/ai/quartermaster.d.ts +3 -2
- package/dist/src/ai/quartermaster.js +4 -4
- package/dist/src/ai/rerunner.d.ts +2 -11
- package/dist/src/ai/rerunner.js +9 -22
- package/dist/src/ai/researcher/coordinates.js +4 -3
- package/dist/src/ai/researcher/deep-analysis.d.ts +2 -0
- package/dist/src/ai/researcher/deep-analysis.js +6 -6
- package/dist/src/ai/researcher/locators.d.ts +9 -2
- package/dist/src/ai/researcher/locators.js +65 -8
- package/dist/src/ai/researcher/sections.d.ts +2 -0
- package/dist/src/ai/researcher/sections.js +2 -2
- package/dist/src/ai/researcher.js +29 -54
- package/dist/src/ai/task-agent.d.ts +18 -5
- package/dist/src/ai/task-agent.js +35 -6
- package/dist/src/ai/tester.d.ts +6 -12
- package/dist/src/ai/tester.js +37 -51
- package/dist/src/ai/tools.d.ts +11 -10
- package/dist/src/ai/tools.js +24 -36
- package/dist/src/command-handler.js +1 -1
- package/dist/src/commands/context-aria-command.js +1 -1
- package/dist/src/commands/context-command.js +7 -5
- package/dist/src/commands/context-data-command.js +2 -2
- package/dist/src/commands/context-experience-command.js +2 -2
- package/dist/src/commands/context-html-command.js +2 -2
- package/dist/src/commands/context-knowledge-command.js +1 -1
- package/dist/src/commands/drill-command.js +1 -1
- package/dist/src/commands/explore-command.js +4 -3
- package/dist/src/commands/freesail-command.js +2 -2
- package/dist/src/commands/learn-command.js +4 -4
- package/dist/src/commands/path-command.js +1 -1
- package/dist/src/commands/research-command.js +1 -1
- package/dist/src/commands/test-command.js +1 -1
- package/dist/src/components/App.js +1 -1
- package/dist/src/config.d.ts +17 -0
- package/dist/src/config.js +144 -10
- package/dist/src/experience-tracker.js +1 -1
- package/dist/src/explorbot.d.ts +17 -8
- package/dist/src/explorbot.js +74 -36
- package/dist/src/explorer.d.ts +66 -91
- package/dist/src/explorer.js +326 -567
- package/dist/src/state-manager.d.ts +6 -1
- package/dist/src/state-manager.js +1 -0
- package/dist/src/utils/hooks-runner.js +2 -4
- package/dist/src/utils/test-files.js +1 -1
- package/dist/src/utils/web-annotate.d.ts +5 -0
- package/dist/src/utils/web-annotate.js +58 -0
- package/dist/src/utils/web-eidx.d.ts +2 -0
- package/dist/src/utils/web-eidx.js +20 -0
- package/dist/src/utils/web-element.d.ts +1 -0
- package/dist/src/utils/web-element.js +29 -0
- package/models.json +30 -0
- package/package.json +5 -2
- package/src/action.ts +25 -5
- package/src/ai/agent.ts +20 -0
- package/src/ai/captain/idle-mode.ts +1 -1
- package/src/ai/captain/test-mode.ts +1 -1
- package/src/ai/captain/web-mode.ts +18 -23
- package/src/ai/captain.ts +12 -25
- package/src/ai/driller.ts +50 -67
- package/src/ai/historian/codeceptjs.ts +1 -1
- package/src/ai/historian/playwright.ts +4 -3
- package/src/ai/historian/screencast.ts +4 -3
- package/src/ai/historian.ts +3 -2
- package/src/ai/navigator.ts +30 -25
- package/src/ai/pilot.ts +55 -47
- package/src/ai/planner.ts +22 -20
- package/src/ai/provider.ts +0 -1
- package/src/ai/quartermaster.ts +5 -4
- package/src/ai/rerunner.ts +10 -29
- package/src/ai/researcher/coordinates.ts +4 -3
- package/src/ai/researcher/deep-analysis.ts +8 -6
- package/src/ai/researcher/locators.ts +66 -9
- package/src/ai/researcher/sections.ts +4 -2
- package/src/ai/researcher.ts +32 -61
- package/src/ai/task-agent.ts +45 -11
- package/src/ai/tester.ts +40 -57
- package/src/ai/tools.ts +32 -53
- package/src/command-handler.ts +1 -1
- package/src/commands/context-aria-command.ts +1 -1
- package/src/commands/context-command.ts +7 -5
- package/src/commands/context-data-command.ts +2 -2
- package/src/commands/context-experience-command.ts +2 -2
- package/src/commands/context-html-command.ts +2 -2
- package/src/commands/context-knowledge-command.ts +1 -1
- package/src/commands/drill-command.ts +1 -1
- package/src/commands/explore-command.ts +4 -3
- package/src/commands/freesail-command.ts +2 -2
- package/src/commands/learn-command.ts +4 -4
- package/src/commands/path-command.ts +1 -1
- package/src/commands/research-command.ts +1 -1
- package/src/commands/test-command.ts +1 -1
- package/src/components/App.tsx +1 -1
- package/src/config.ts +173 -11
- package/src/experience-tracker.ts +1 -1
- package/src/explorbot.ts +78 -39
- package/src/explorer.ts +389 -628
- package/src/state-manager.ts +7 -1
- package/src/utils/hooks-runner.ts +2 -4
- package/src/utils/test-files.ts +1 -1
- package/src/utils/web-annotate.ts +64 -0
- package/src/utils/web-eidx.ts +21 -0
- package/src/utils/web-element.ts +30 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { ActionResult } from './action-result.js';
|
|
2
2
|
import type { ExperienceTracker } from './experience-tracker.js';
|
|
3
|
-
import type {
|
|
3
|
+
import type { Knowledge, KnowledgeTracker } from './knowledge-tracker.js';
|
|
4
4
|
export interface Link {
|
|
5
5
|
title: string;
|
|
6
6
|
url: string;
|
|
@@ -55,8 +55,13 @@ export interface StateTransition {
|
|
|
55
55
|
trigger: 'manual' | 'navigation' | 'automatic';
|
|
56
56
|
}
|
|
57
57
|
export type StateChangeListener = (event: StateTransition) => void;
|
|
58
|
+
export interface TabInfo {
|
|
59
|
+
url: string;
|
|
60
|
+
title: string;
|
|
61
|
+
}
|
|
58
62
|
export type { Knowledge };
|
|
59
63
|
export declare class StateManager {
|
|
64
|
+
otherTabs: TabInfo[];
|
|
60
65
|
currentState: WebPageState | null;
|
|
61
66
|
stateHistory: StateTransition[];
|
|
62
67
|
allVisitedUrls: Set<string>;
|
|
@@ -5,6 +5,7 @@ import { slugify } from './utils/strings.js';
|
|
|
5
5
|
import { extractStatePath } from './utils/url-matcher.js';
|
|
6
6
|
const debugLog = createDebug('explorbot:state');
|
|
7
7
|
export class StateManager {
|
|
8
|
+
otherTabs = [];
|
|
8
9
|
currentState = null;
|
|
9
10
|
stateHistory = [];
|
|
10
11
|
allVisitedUrls = new Set();
|
|
@@ -49,12 +49,10 @@ export class HooksRunner {
|
|
|
49
49
|
async executeHook(hook, url) {
|
|
50
50
|
try {
|
|
51
51
|
if (hook.type === 'playwright') {
|
|
52
|
-
|
|
53
|
-
await hook.hook({ page, url });
|
|
52
|
+
await this.explorer.withPage(async (page) => hook.hook({ page, url }));
|
|
54
53
|
}
|
|
55
54
|
else {
|
|
56
|
-
|
|
57
|
-
await hook.hook({ I, url });
|
|
55
|
+
await hook.hook({ I: this.explorer.actor, url });
|
|
58
56
|
}
|
|
59
57
|
}
|
|
60
58
|
catch (error) {
|
|
@@ -64,7 +64,7 @@ export async function dryRunTestFile(filePath) {
|
|
|
64
64
|
Playwright: { browser: config.playwright.browser, url: config.playwright.url },
|
|
65
65
|
},
|
|
66
66
|
};
|
|
67
|
-
global.output_dir =
|
|
67
|
+
global.output_dir = ConfigParser.getInstance().getStatesDir();
|
|
68
68
|
global.codecept_dir = projectRoot;
|
|
69
69
|
codeceptjs.container.create(codeceptConfig, {});
|
|
70
70
|
await codeceptjs.recorder.start();
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { ELEMENT_EXTRACTION_CONFIG, getElementDataExtractorSource } from "./html.js";
|
|
2
|
+
import { createDebug } from './logger.js';
|
|
3
|
+
import { WebElement } from "./web-element.js";
|
|
4
|
+
const debugLog = createDebug('explorbot:web-annotate');
|
|
5
|
+
const REF_LINE_PATTERN = /^(\s*)-\s+(\w+)\s*(?:"([^"]*)")?.*?\[ref=(e\d+)\]/;
|
|
6
|
+
const ANNOTATABLE_ROLES = new Set(['button', 'link', 'textbox', 'searchbox', 'checkbox', 'radio', 'switch', 'combobox', 'tab', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'option', 'slider', 'spinbutton', 'treeitem']);
|
|
7
|
+
function parseAriaRefs(ariaSnapshot) {
|
|
8
|
+
const entries = [];
|
|
9
|
+
for (const line of ariaSnapshot.split('\n')) {
|
|
10
|
+
const match = line.match(REF_LINE_PATTERN);
|
|
11
|
+
if (!match)
|
|
12
|
+
continue;
|
|
13
|
+
if (!ANNOTATABLE_ROLES.has(match[2]))
|
|
14
|
+
continue;
|
|
15
|
+
entries.push({ role: match[2], name: match[3] || '', ref: match[4] });
|
|
16
|
+
}
|
|
17
|
+
return entries;
|
|
18
|
+
}
|
|
19
|
+
export async function annotatePageElements(page) {
|
|
20
|
+
const ariaSnapshot = await page.locator('body').ariaSnapshot({ mode: 'ai' });
|
|
21
|
+
const refEntries = parseAriaRefs(ariaSnapshot);
|
|
22
|
+
const byRole = new Map();
|
|
23
|
+
for (const { role, name, ref } of refEntries) {
|
|
24
|
+
let list = byRole.get(role);
|
|
25
|
+
if (!list) {
|
|
26
|
+
list = [];
|
|
27
|
+
byRole.set(role, list);
|
|
28
|
+
}
|
|
29
|
+
list.push({ name, ref });
|
|
30
|
+
}
|
|
31
|
+
const elements = [];
|
|
32
|
+
for (const [role, entries] of byRole) {
|
|
33
|
+
try {
|
|
34
|
+
const rawList = await page.getByRole(role).evaluateAll((domElements, [data, extractFnStr, config]) => {
|
|
35
|
+
const extract = new Function(`return ${extractFnStr}`)();
|
|
36
|
+
const results = [];
|
|
37
|
+
let ariaIdx = 0;
|
|
38
|
+
for (const el of domElements) {
|
|
39
|
+
if (ariaIdx >= data.length)
|
|
40
|
+
break;
|
|
41
|
+
el.setAttribute(config.attrs.eidx, data[ariaIdx].ref);
|
|
42
|
+
const elData = extract(el, config);
|
|
43
|
+
if (elData)
|
|
44
|
+
results.push(elData);
|
|
45
|
+
ariaIdx++;
|
|
46
|
+
}
|
|
47
|
+
return results;
|
|
48
|
+
}, [entries, getElementDataExtractorSource(), ELEMENT_EXTRACTION_CONFIG]);
|
|
49
|
+
for (const raw of rawList) {
|
|
50
|
+
elements.push(WebElement.fromRawData(raw, role));
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
debugLog(`Failed to annotate role=${role}`);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return { ariaSnapshot, elements };
|
|
58
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { ELEMENT_EXTRACTION_CONFIG } from "./html.js";
|
|
2
|
+
export async function eidxInContainer(page, containerCss) {
|
|
3
|
+
const selector = containerCss ? `${containerCss} [${ELEMENT_EXTRACTION_CONFIG.attrs.eidx}]` : `[${ELEMENT_EXTRACTION_CONFIG.attrs.eidx}]`;
|
|
4
|
+
const elements = await page.locator(selector).all();
|
|
5
|
+
const result = [];
|
|
6
|
+
for (const el of elements) {
|
|
7
|
+
const attr = await el.getAttribute(ELEMENT_EXTRACTION_CONFIG.attrs.eidx).catch(() => null);
|
|
8
|
+
if (attr)
|
|
9
|
+
result.push(attr);
|
|
10
|
+
}
|
|
11
|
+
return result;
|
|
12
|
+
}
|
|
13
|
+
export async function eidxByLocator(page, locator, container) {
|
|
14
|
+
const base = container ? page.locator(container) : page;
|
|
15
|
+
const el = locator.startsWith('//') ? base.locator(`xpath=${locator}`) : base.locator(locator);
|
|
16
|
+
return el
|
|
17
|
+
.first()
|
|
18
|
+
.getAttribute(ELEMENT_EXTRACTION_CONFIG.attrs.eidx)
|
|
19
|
+
.catch(() => null);
|
|
20
|
+
}
|
|
@@ -37,6 +37,7 @@ export declare class WebElement {
|
|
|
37
37
|
static fromPlaywrightLocator(locator: any): Promise<WebElement | null>;
|
|
38
38
|
static fromEidx(page: any, eidx: string): Promise<WebElement | null>;
|
|
39
39
|
static fromEidxList(page: any, eidxList: string[]): Promise<WebElement[]>;
|
|
40
|
+
static commonAncestor(page: any, eidxList: string[]): Promise<WebElement | null>;
|
|
40
41
|
static findByXPath(html: string, xpath: string): Promise<{
|
|
41
42
|
totalFound: number;
|
|
42
43
|
elements: WebElement[];
|
|
@@ -130,6 +130,35 @@ export class WebElement {
|
|
|
130
130
|
}, [validEidxList, getElementDataExtractorSource(), ELEMENT_EXTRACTION_CONFIG]);
|
|
131
131
|
return rawList.map((d) => WebElement.fromRawData(d));
|
|
132
132
|
}
|
|
133
|
+
static async commonAncestor(page, eidxList) {
|
|
134
|
+
const validEidxList = eidxList.filter((eidx) => /^e\d+$/i.test(eidx));
|
|
135
|
+
if (validEidxList.length < 2)
|
|
136
|
+
return null;
|
|
137
|
+
const raw = await page.evaluate(([list, extractFnStr, config]) => {
|
|
138
|
+
const extract = new Function(`return ${extractFnStr}`)();
|
|
139
|
+
const els = [];
|
|
140
|
+
for (const eidx of list) {
|
|
141
|
+
const el = document.querySelector(`[${config.attrs.eidx}="${eidx}"]`);
|
|
142
|
+
if (el && !els.includes(el))
|
|
143
|
+
els.push(el);
|
|
144
|
+
}
|
|
145
|
+
if (els.length < 2)
|
|
146
|
+
return null;
|
|
147
|
+
const containsAll = (node) => els.every((el) => node.contains(el));
|
|
148
|
+
let ancestor = els[0].parentElement;
|
|
149
|
+
while (ancestor && !containsAll(ancestor))
|
|
150
|
+
ancestor = ancestor.parentElement;
|
|
151
|
+
if (!ancestor)
|
|
152
|
+
return null;
|
|
153
|
+
const tag = ancestor.tagName.toLowerCase();
|
|
154
|
+
if (tag === 'body' || tag === 'html')
|
|
155
|
+
return null;
|
|
156
|
+
return extract(ancestor, config);
|
|
157
|
+
}, [validEidxList, getElementDataExtractorSource(), ELEMENT_EXTRACTION_CONFIG]);
|
|
158
|
+
if (!raw)
|
|
159
|
+
return null;
|
|
160
|
+
return WebElement.fromRawData(raw);
|
|
161
|
+
}
|
|
133
162
|
static async findByXPath(html, xpath) {
|
|
134
163
|
const result = await evaluateXPath(html, xpath);
|
|
135
164
|
if (result.error)
|
package/models.json
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
{
|
|
2
|
+
"openrouter": {
|
|
3
|
+
"model": "openai/gpt-oss-20b:nitro",
|
|
4
|
+
"visionModel": "google/gemma-4-31b-it:nitro",
|
|
5
|
+
"agenticModel": "google/gemma-4-31b-it:nitro"
|
|
6
|
+
},
|
|
7
|
+
"groq": {
|
|
8
|
+
"model": "openai/gpt-oss-20b",
|
|
9
|
+
"visionModel": "qwen/qwen3.6-27b",
|
|
10
|
+
"agenticModel": "qwen/qwen3.6-27b"
|
|
11
|
+
},
|
|
12
|
+
"openai": {
|
|
13
|
+
"model": "gpt-5.4-nano",
|
|
14
|
+
"visionModel": "gpt-5.4-nano",
|
|
15
|
+
"agenticModel": "gpt-5.6-luna"
|
|
16
|
+
},
|
|
17
|
+
"anthropic": {
|
|
18
|
+
"agenticModel": "claude-haiku-4-5-20251001"
|
|
19
|
+
},
|
|
20
|
+
"mistral": {
|
|
21
|
+
"model": "mistral-small-latest",
|
|
22
|
+
"visionModel": "mistral-small-latest",
|
|
23
|
+
"agenticModel": "mistral-large-latest"
|
|
24
|
+
},
|
|
25
|
+
"google": {
|
|
26
|
+
"model": "gemini-3.1-flash-lite",
|
|
27
|
+
"visionModel": "gemini-3.1-flash-lite",
|
|
28
|
+
"agenticModel": "gemini-3.5-flash"
|
|
29
|
+
}
|
|
30
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "explorbot",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.1",
|
|
4
4
|
"description": "CLI app built with React Ink, CodeceptJS, and Playwright",
|
|
5
5
|
"license": "Elastic-2.0",
|
|
6
6
|
"type": "module",
|
|
@@ -26,7 +26,8 @@
|
|
|
26
26
|
"boat/doc-collector/bin/**/*.ts",
|
|
27
27
|
"boat/doc-collector/package.json",
|
|
28
28
|
"rules/",
|
|
29
|
-
"assets/sample-files/"
|
|
29
|
+
"assets/sample-files/",
|
|
30
|
+
"models.json"
|
|
30
31
|
],
|
|
31
32
|
"scripts": {
|
|
32
33
|
"build": "bun run build:bin",
|
|
@@ -68,7 +69,9 @@
|
|
|
68
69
|
"author": "",
|
|
69
70
|
"dependencies": {
|
|
70
71
|
"@ai-sdk/anthropic": "^4.0",
|
|
72
|
+
"@ai-sdk/google": "^4.0.18",
|
|
71
73
|
"@ai-sdk/groq": "^4.0",
|
|
74
|
+
"@ai-sdk/mistral": "^4.0.13",
|
|
72
75
|
"@ai-sdk/openai": "^4.0",
|
|
73
76
|
"@ai-sdk/otel": "^1.0.2",
|
|
74
77
|
"@axe-core/playwright": "^4.11.0",
|
package/src/action.ts
CHANGED
|
@@ -14,8 +14,8 @@ import { browserErrorMessage, isFatalBrowserError, isNavigationTransitionError }
|
|
|
14
14
|
import { captureHtmlForSnapshot, htmlCombinedSnapshot, minifyHtml } from './utils/html.js';
|
|
15
15
|
import { createDebug, setStepSpanParent, tag } from './utils/logger.js';
|
|
16
16
|
import { sleep, waitForPageReadiness } from './utils/page-readiness.ts';
|
|
17
|
-
import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from './utils/web-sandbox.ts';
|
|
18
17
|
import { safeFilename } from './utils/strings.ts';
|
|
18
|
+
import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from './utils/web-sandbox.ts';
|
|
19
19
|
|
|
20
20
|
const debugLog = createDebug('explorbot:action');
|
|
21
21
|
const CAPTURE_NAVIGATION_TRANSITION_ATTEMPTS = 3;
|
|
@@ -33,14 +33,16 @@ class Action {
|
|
|
33
33
|
public playwrightGroupId: string | null = null;
|
|
34
34
|
public assertionSteps: Array<{ name: string; args: any[] }> = [];
|
|
35
35
|
private recorder?: PlaywrightRecorder;
|
|
36
|
+
private recovery: RecoveryRunner;
|
|
36
37
|
private mainDocumentStatus: number | undefined = undefined;
|
|
37
38
|
|
|
38
|
-
constructor(actor: CodeceptJS.I, stateManager: StateManager, recorder?: PlaywrightRecorder) {
|
|
39
|
+
constructor(actor: CodeceptJS.I, stateManager: StateManager, recorder?: PlaywrightRecorder, recovery?: RecoveryRunner) {
|
|
39
40
|
this.actor = actor;
|
|
40
41
|
this.stateManager = stateManager;
|
|
41
42
|
this.config = ConfigParser.getInstance().getConfig();
|
|
42
43
|
this.playwrightHelper = container.helpers('Playwright');
|
|
43
44
|
this.recorder = recorder;
|
|
45
|
+
this.recovery = recovery || ((fn) => fn());
|
|
44
46
|
}
|
|
45
47
|
|
|
46
48
|
async saveScreenshot(): Promise<string | undefined> {
|
|
@@ -59,7 +61,15 @@ class Action {
|
|
|
59
61
|
}
|
|
60
62
|
}
|
|
61
63
|
|
|
62
|
-
async capturePageState(
|
|
64
|
+
async capturePageState(opts: { includeScreenshot?: boolean; codeBlock?: string } = {}): Promise<ActionResult> {
|
|
65
|
+
return this.recovery(() => this.captureOnce(opts));
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async execute(code: string): Promise<Action> {
|
|
69
|
+
return this.recovery(() => this.executeOnce(code));
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
private async captureOnce({ includeScreenshot = false, codeBlock }: { includeScreenshot?: boolean; codeBlock?: string } = {}): Promise<ActionResult> {
|
|
63
73
|
try {
|
|
64
74
|
const currentState = this.stateManager.getCurrentState();
|
|
65
75
|
const stateHash = currentState?.hash || 'screenshot';
|
|
@@ -260,7 +270,7 @@ class Action {
|
|
|
260
270
|
}
|
|
261
271
|
}
|
|
262
272
|
|
|
263
|
-
async
|
|
273
|
+
private async executeOnce(code: string): Promise<Action> {
|
|
264
274
|
let error: Error | null = null;
|
|
265
275
|
|
|
266
276
|
setActivity('🔎 Browsing...', 'action');
|
|
@@ -301,7 +311,7 @@ class Action {
|
|
|
301
311
|
codeString = executedSteps.join('\n');
|
|
302
312
|
}
|
|
303
313
|
|
|
304
|
-
const pageState = await this.
|
|
314
|
+
const pageState = await this.captureOnce({ codeBlock: codeString });
|
|
305
315
|
|
|
306
316
|
this.actionResult = pageState;
|
|
307
317
|
this.assertionSteps = assertionSteps;
|
|
@@ -329,6 +339,7 @@ class Action {
|
|
|
329
339
|
}
|
|
330
340
|
|
|
331
341
|
public async attempt(codeBlock: string, originalMessage?: string): Promise<boolean> {
|
|
342
|
+
const wasInFrame = !!this.playwrightHelper.frame;
|
|
332
343
|
try {
|
|
333
344
|
debugLog('Resolution attempt...');
|
|
334
345
|
setActivity('🦾 Acting in browser...', 'action');
|
|
@@ -343,11 +354,18 @@ class Action {
|
|
|
343
354
|
} catch (error) {
|
|
344
355
|
this.lastError = error as Error;
|
|
345
356
|
if (isFatalBrowserError(error)) throw error;
|
|
357
|
+
if (!wasInFrame) await this.exitIframe().catch(() => {});
|
|
346
358
|
debugLog(`Attempt failed: ${codeBlock}: ${errorToString(error) || this.lastError?.toString()}`);
|
|
347
359
|
return false;
|
|
348
360
|
}
|
|
349
361
|
}
|
|
350
362
|
|
|
363
|
+
async exitIframe(): Promise<void> {
|
|
364
|
+
if (!this.playwrightHelper.frame) return;
|
|
365
|
+
debugLog('Switching to main frame');
|
|
366
|
+
await this.playwrightHelper.switchTo();
|
|
367
|
+
}
|
|
368
|
+
|
|
351
369
|
getActor(): CodeceptJS.I {
|
|
352
370
|
return this.actor;
|
|
353
371
|
}
|
|
@@ -366,6 +384,8 @@ class Action {
|
|
|
366
384
|
|
|
367
385
|
export default Action;
|
|
368
386
|
|
|
387
|
+
export type RecoveryRunner = <T>(fn: () => Promise<T>) => Promise<T>;
|
|
388
|
+
|
|
369
389
|
function errorToString(error: any): string {
|
|
370
390
|
if (error.cliMessage) {
|
|
371
391
|
return error.cliMessage();
|
package/src/ai/agent.ts
CHANGED
|
@@ -1,3 +1,23 @@
|
|
|
1
|
+
import type { RequestStore } from '../api/request-store.ts';
|
|
2
|
+
import type { ExplorbotConfig } from '../config.ts';
|
|
3
|
+
import type Explorer from '../explorer.ts';
|
|
4
|
+
import type { KnowledgeTracker } from '../knowledge-tracker.ts';
|
|
5
|
+
import type { PlaywrightRecorder } from '../playwright-recorder.ts';
|
|
6
|
+
import type { StateManager } from '../state-manager.ts';
|
|
7
|
+
import type { AIProvider } from './provider.ts';
|
|
8
|
+
|
|
1
9
|
export interface Agent {
|
|
2
10
|
emoji?: string;
|
|
3
11
|
}
|
|
12
|
+
|
|
13
|
+
export interface AgentDeps {
|
|
14
|
+
explorer: Explorer;
|
|
15
|
+
ai: AIProvider;
|
|
16
|
+
config: ExplorbotConfig;
|
|
17
|
+
stateManager: StateManager;
|
|
18
|
+
knowledgeTracker: KnowledgeTracker;
|
|
19
|
+
requestStore: RequestStore;
|
|
20
|
+
playwrightRecorder: PlaywrightRecorder;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export type ToolDeps = Pick<AgentDeps, 'explorer' | 'stateManager' | 'ai'>;
|
|
@@ -66,7 +66,7 @@ export function WithIdleMode<T extends Constructor>(Base: T) {
|
|
|
66
66
|
if (!action || action === 'replace') {
|
|
67
67
|
plan.tests.length = 0;
|
|
68
68
|
}
|
|
69
|
-
const currentUrl = ctx.explorBot.
|
|
69
|
+
const currentUrl = ctx.explorBot.stateManager().getCurrentState()?.url || '';
|
|
70
70
|
for (const testInput of tests) {
|
|
71
71
|
const priority = testInput.priority || 'normal';
|
|
72
72
|
const expected = testInput.expected?.length ? testInput.expected : [];
|
|
@@ -146,7 +146,7 @@ export function WithTestMode<T extends Constructor>(Base: T) {
|
|
|
146
146
|
if (!t) return { success: false, message: `Test "${session}" not found` };
|
|
147
147
|
states = t.states;
|
|
148
148
|
} else {
|
|
149
|
-
const stateManager = ctx.explorBot.
|
|
149
|
+
const stateManager = ctx.explorBot.stateManager();
|
|
150
150
|
const history = stateManager.getStateHistory();
|
|
151
151
|
const seen = new Set<string>();
|
|
152
152
|
states = history
|
|
@@ -9,9 +9,10 @@ export function WithWebMode<T extends Constructor>(Base: T) {
|
|
|
9
9
|
return class extends Base {
|
|
10
10
|
webModeTools(ctx: ModeContext): Record<string, any> {
|
|
11
11
|
const explorer = ctx.explorBot.getExplorer();
|
|
12
|
-
const
|
|
12
|
+
const toolDeps = { explorer, stateManager: ctx.explorBot.stateManager(), ai: ctx.explorBot.getProvider() };
|
|
13
|
+
const codeceptTools = createCodeceptJSTools(toolDeps, ctx.task);
|
|
13
14
|
const agentTools = createAgentTools({
|
|
14
|
-
|
|
15
|
+
...toolDeps,
|
|
15
16
|
researcher: ctx.explorBot.agentResearcher(),
|
|
16
17
|
navigator: ctx.explorBot.agentNavigator(),
|
|
17
18
|
});
|
|
@@ -27,7 +28,7 @@ export function WithWebMode<T extends Constructor>(Base: T) {
|
|
|
27
28
|
try {
|
|
28
29
|
debugLog('navigate', destination);
|
|
29
30
|
await ctx.explorBot.agentNavigator().visit(destination);
|
|
30
|
-
const stateManager = ctx.explorBot.
|
|
31
|
+
const stateManager = ctx.explorBot.stateManager();
|
|
31
32
|
const state = stateManager.getCurrentState();
|
|
32
33
|
return { success: true, url: state?.url, title: state?.title };
|
|
33
34
|
} catch (error: any) {
|
|
@@ -45,48 +46,42 @@ export function WithWebMode<T extends Constructor>(Base: T) {
|
|
|
45
46
|
- closeTabs: Close all browser tabs except the current one
|
|
46
47
|
- reload: Reload the current page
|
|
47
48
|
- screenshot: Take a screenshot of current page
|
|
48
|
-
- recover: Recover from a closed/crashed page
|
|
49
|
-
-
|
|
50
|
-
- openFreshTab: Open a fresh tab in the current browser context
|
|
49
|
+
- recover: Recover from a closed/crashed page, escalating to a full browser restart when needed
|
|
50
|
+
- openTab: Open a fresh tab in the current browser context
|
|
51
51
|
`,
|
|
52
52
|
inputSchema: z.object({
|
|
53
|
-
action: z.enum(['status', 'evaluate', 'closeTabs', 'reload', 'screenshot', 'recover', '
|
|
53
|
+
action: z.enum(['status', 'evaluate', 'closeTabs', 'reload', 'screenshot', 'recover', 'openTab']).describe('Browser action to perform'),
|
|
54
54
|
code: z.string().optional().describe('JavaScript code for evaluate action'),
|
|
55
55
|
}),
|
|
56
56
|
execute: async ({ action, code }) => {
|
|
57
57
|
const explorer = ctx.explorBot.getExplorer();
|
|
58
58
|
|
|
59
59
|
if (action === 'status') {
|
|
60
|
-
const page = explorer.
|
|
60
|
+
const page = explorer.page;
|
|
61
61
|
const pages = page?.context?.().pages?.() || [];
|
|
62
62
|
return {
|
|
63
63
|
success: true,
|
|
64
64
|
hasPage: !!page,
|
|
65
|
-
isClosed: page
|
|
66
|
-
url: page
|
|
67
|
-
title: page
|
|
65
|
+
isClosed: !page,
|
|
66
|
+
url: page ? await page.url() : null,
|
|
67
|
+
title: page ? await page.title().catch(() => null) : null,
|
|
68
68
|
tabs: pages.length,
|
|
69
69
|
};
|
|
70
70
|
}
|
|
71
71
|
|
|
72
72
|
if (action === 'recover') {
|
|
73
|
-
const
|
|
74
|
-
return { success:
|
|
73
|
+
const { ok } = await explorer.recover();
|
|
74
|
+
return { success: ok, message: ok ? 'Browser page recovered' : 'Browser recovery failed' };
|
|
75
75
|
}
|
|
76
76
|
|
|
77
|
-
if (action === '
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
if (action === 'openFreshTab') {
|
|
83
|
-
await ctx.explorBot.openFreshTab();
|
|
84
|
-
const state = explorer.getStateManager().getCurrentState();
|
|
77
|
+
if (action === 'openTab') {
|
|
78
|
+
await ctx.explorBot.openTab();
|
|
79
|
+
const state = ctx.explorBot.stateManager().getCurrentState();
|
|
85
80
|
return { success: true, url: state?.url, title: state?.title };
|
|
86
81
|
}
|
|
87
82
|
|
|
88
|
-
const page = explorer.
|
|
89
|
-
if (!page
|
|
83
|
+
const page = explorer.page;
|
|
84
|
+
if (!page) return { success: false, message: 'No browser page available. Try browser({ action: "recover" }) first.' };
|
|
90
85
|
|
|
91
86
|
if (action === 'evaluate') {
|
|
92
87
|
if (!code) return { success: false, message: 'Code required for evaluate action' };
|
package/src/ai/captain.ts
CHANGED
|
@@ -49,7 +49,7 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
49
49
|
private getHooksRunner(): HooksRunner {
|
|
50
50
|
if (!this.hooksRunner) {
|
|
51
51
|
const explorer = this.explorBot.getExplorer();
|
|
52
|
-
this.hooksRunner = new HooksRunner(explorer,
|
|
52
|
+
this.hooksRunner = new HooksRunner(explorer, this.explorBot.getConfig());
|
|
53
53
|
}
|
|
54
54
|
return this.hooksRunner;
|
|
55
55
|
}
|
|
@@ -89,17 +89,16 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
89
89
|
getMode(): CaptainMode {
|
|
90
90
|
const explorer = this.explorBot.getExplorer();
|
|
91
91
|
const activeTest = explorer.activeTest;
|
|
92
|
-
const page = explorer.playwrightHelper?.page;
|
|
93
92
|
|
|
94
|
-
if (activeTest &&
|
|
93
|
+
if (activeTest && !explorer.page) return 'heal';
|
|
95
94
|
if (activeTest) return 'test';
|
|
96
|
-
if (
|
|
95
|
+
if (this.explorBot.stateManager().getCurrentState()) return 'web';
|
|
97
96
|
return 'idle';
|
|
98
97
|
}
|
|
99
98
|
|
|
100
99
|
private systemPrompt(): string {
|
|
101
100
|
const mode = this.getMode();
|
|
102
|
-
const currentUrl = this.explorBot.
|
|
101
|
+
const currentUrl = this.explorBot.stateManager().getCurrentState()?.url;
|
|
103
102
|
const customPrompt = this.explorBot.getProvider().getSystemPromptForAgent('captain', currentUrl);
|
|
104
103
|
|
|
105
104
|
return dedent`
|
|
@@ -147,7 +146,7 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
147
146
|
}
|
|
148
147
|
|
|
149
148
|
private async getPageContext(): Promise<string> {
|
|
150
|
-
const state = this.explorBot.
|
|
149
|
+
const state = this.explorBot.stateManager().getCurrentState();
|
|
151
150
|
if (!state) {
|
|
152
151
|
return 'No page loaded';
|
|
153
152
|
}
|
|
@@ -221,10 +220,9 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
221
220
|
}
|
|
222
221
|
|
|
223
222
|
private async reinjectContextIfNeeded(conversation: Conversation, currentState: WebPageState): Promise<void> {
|
|
224
|
-
if (conversation.hasTag('
|
|
223
|
+
if (conversation.hasTag('page_aria', 5)) return;
|
|
225
224
|
|
|
226
225
|
const actionResult = ActionResult.fromState(currentState);
|
|
227
|
-
const html = await actionResult.combinedHtml();
|
|
228
226
|
const context = dedent`
|
|
229
227
|
Context:
|
|
230
228
|
|
|
@@ -236,13 +234,8 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
236
234
|
<page_aria>
|
|
237
235
|
${actionResult.getInteractiveARIA()}
|
|
238
236
|
</page_aria>
|
|
239
|
-
|
|
240
|
-
<page_html>
|
|
241
|
-
${html}
|
|
242
|
-
</page_html>
|
|
243
237
|
`;
|
|
244
238
|
conversation.addUserText(context);
|
|
245
|
-
return Promise.resolve();
|
|
246
239
|
}
|
|
247
240
|
|
|
248
241
|
private coreTools(task: Task, onDone: (summary: string) => void) {
|
|
@@ -273,15 +266,8 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
273
266
|
runCommand: tool({
|
|
274
267
|
description: dedent`
|
|
275
268
|
Execute a TUI command. Returns log output from command execution.
|
|
276
|
-
Use only when the user
|
|
277
|
-
|
|
278
|
-
Never run a slash command unless the user request itself starts with that slash command.
|
|
279
|
-
${this.commandDescriptions
|
|
280
|
-
.map((c) => {
|
|
281
|
-
const opts = c.options ? ` (${c.options})` : '';
|
|
282
|
-
return `${c.name} — ${c.description}${opts}`;
|
|
283
|
-
})
|
|
284
|
-
.join('\n')}
|
|
269
|
+
Use only when the user request itself starts with a slash command; pass it through verbatim.
|
|
270
|
+
Available commands: ${this.commandDescriptions.map((c) => c.name).join(', ')}
|
|
285
271
|
`,
|
|
286
272
|
inputSchema: z.object({
|
|
287
273
|
command: z.string().describe('Slash command to execute, e.g. "/research", "/plan authentication", "/test brave-fox123"'),
|
|
@@ -339,7 +325,7 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
339
325
|
}
|
|
340
326
|
|
|
341
327
|
const pilotAnalysis = this.explorBot.agentPilot().getLastAnalysis() || '';
|
|
342
|
-
const currentUrl = this.explorBot.
|
|
328
|
+
const currentUrl = this.explorBot.stateManager().getCurrentState()?.url || '';
|
|
343
329
|
|
|
344
330
|
const schema = z.object({
|
|
345
331
|
action: z.enum(['inject', 'stop', 'pass', 'skip']),
|
|
@@ -390,7 +376,7 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
390
376
|
|
|
391
377
|
async processExecutionError(error: Error, activeTest: Test): Promise<ExecutionRecoveryAction> {
|
|
392
378
|
const explorer = this.explorBot.getExplorer();
|
|
393
|
-
const result = await explorer.
|
|
379
|
+
const result = await explorer.recover(error);
|
|
394
380
|
return {
|
|
395
381
|
...result,
|
|
396
382
|
message: result.recovered ? `${result.message}\nContinue the test "${activeTest.scenario}" from the restored page.` : result.message,
|
|
@@ -402,7 +388,7 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
402
388
|
}
|
|
403
389
|
|
|
404
390
|
async handle(input: string, options: { reset?: boolean } = {}): Promise<string | null> {
|
|
405
|
-
const stateManager = this.explorBot.
|
|
391
|
+
const stateManager = this.explorBot.stateManager();
|
|
406
392
|
const initialState = stateManager.getCurrentState();
|
|
407
393
|
|
|
408
394
|
const conversation = options.reset ? this.resetConversation() : this.ensureConversation();
|
|
@@ -477,6 +463,7 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
477
463
|
const result = await this.explorBot.getProvider().invokeConversation(conversation, tools, {
|
|
478
464
|
maxToolRoundtrips: 5,
|
|
479
465
|
toolChoice: 'auto',
|
|
466
|
+
stopWhen: () => isDone,
|
|
480
467
|
});
|
|
481
468
|
|
|
482
469
|
if (!result) {
|