explorbot 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/boat/api-tester/src/apibot.ts +8 -13
- package/boat/api-tester/src/cli.ts +7 -3
- package/boat/api-tester/src/config.ts +45 -9
- package/boat/prima/src/cli.ts +33 -99
- package/boat/prima/src/envelope.ts +3 -1
- package/boat/prima/src/help.ts +72 -0
- package/boat/prima/src/prima.ts +33 -43
- package/dist/boat/api-tester/src/apibot.js +7 -6
- package/dist/boat/api-tester/src/cli.js +9 -3
- package/dist/boat/api-tester/src/config.js +32 -6
- package/dist/boat/prima/src/cli.js +30 -86
- package/dist/boat/prima/src/envelope.js +2 -1
- package/dist/boat/prima/src/help.js +63 -0
- package/dist/boat/prima/src/prima.js +29 -41
- package/dist/package.json +1 -1
- package/dist/src/action-result.d.ts +3 -0
- package/dist/src/action-result.js +5 -0
- package/dist/src/action.js +12 -1
- package/dist/src/ai/researcher/deep-analysis.d.ts +1 -1
- package/dist/src/ai/researcher/deep-analysis.js +4 -1
- package/dist/src/ai/researcher/sections.d.ts +1 -1
- package/dist/src/ai/researcher/sections.js +2 -1
- package/dist/src/ai/researcher.js +25 -11
- package/dist/src/ai/tester.d.ts +1 -0
- package/dist/src/ai/tester.js +27 -33
- package/dist/src/ai/tools.js +5 -0
- package/dist/src/commands/config-command.js +6 -2
- package/dist/src/config.d.ts +3 -0
- package/dist/src/config.js +14 -0
- package/dist/src/state-manager.js +5 -1
- package/docs/api-testing/basics.md +12 -4
- package/docs/reference/commands.md +1 -0
- package/docs/workflow/agentic-usage.md +3 -1
- package/package.json +1 -1
- package/src/action-result.ts +7 -0
- package/src/action.ts +14 -2
- package/src/ai/researcher/deep-analysis.ts +4 -2
- package/src/ai/researcher/sections.ts +2 -2
- package/src/ai/researcher.ts +28 -11
- package/src/ai/tester.ts +25 -30
- package/src/ai/tools.ts +6 -0
- package/src/commands/config-command.ts +4 -1
- package/src/config.ts +16 -0
- package/src/state-manager.ts +6 -1
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import dedent from 'dedent';
|
|
2
|
+
export const helpContract = dedent `
|
|
3
|
+
Prima drives the browser opened by playwright-cli.
|
|
4
|
+
|
|
5
|
+
playwright-cli open <url> starts the session
|
|
6
|
+
prima <command> ... drives it
|
|
7
|
+
playwright-cli close ends it
|
|
8
|
+
|
|
9
|
+
One call runs a whole job:
|
|
10
|
+
|
|
11
|
+
prima check "a workflow can be created and appears in the list" --expected "the new workflow is listed"
|
|
12
|
+
prima do "open the account menu" "choose the settings entry" "switch the theme to dark" "check it took effect"
|
|
13
|
+
prima pw "({ page }) => page.click('[data-test=submit]')"
|
|
14
|
+
|
|
15
|
+
Only research maps a page. Other commands read the accessibility tree; a cached
|
|
16
|
+
research map joins when present. On large or unclear pages, run prima research first.
|
|
17
|
+
Without a usable AI model only pw works; otherwise use playwright-cli.
|
|
18
|
+
DEBUG='explorbot:*' in front of a command logs everything it does.
|
|
19
|
+
`;
|
|
20
|
+
export const checkHelp = dedent `
|
|
21
|
+
check states the outcome, not the clicks; it finds the path itself. It stays
|
|
22
|
+
on the current page and never reloads it, so an open dialog survives.
|
|
23
|
+
--expected one required outcome, repeatable for several. Without it the
|
|
24
|
+
scenario text is the outcome. Each returns under
|
|
25
|
+
### Expected outcomes as PASSED, FAILED, CONTRADICTION or not verified.
|
|
26
|
+
"not verified" means never checked, not false.
|
|
27
|
+
Proof is a full-page screenshot: what a user sees counts, the log only shows actions.
|
|
28
|
+
CONTRADICTION means screenshot and log disagree; judge the html, aria and
|
|
29
|
+
screenshot under ### Artifacts yourself.
|
|
30
|
+
ok is false when any outcome FAILED or CONTRADICTED, or the run could not finish,
|
|
31
|
+
reported as such rather than as an app failure.
|
|
32
|
+
Side issues found on the way go under ### Answer, not as step failures.
|
|
33
|
+
`;
|
|
34
|
+
export const doHelp = dedent `
|
|
35
|
+
### Steps marks each instruction ok, FAIL or ??. ?? means it ran but the run ended
|
|
36
|
+
without confirming it - read the steps above. Only FAIL fails the command.
|
|
37
|
+
Nothing runs past the last instruction. Batch the whole sequence in one call;
|
|
38
|
+
that is what keeps this tier cheap.
|
|
39
|
+
`;
|
|
40
|
+
export const askHelp = dedent `
|
|
41
|
+
Answers from a page screenshot, or from its structure with --no-vision.
|
|
42
|
+
`;
|
|
43
|
+
export const verifyHelp = dedent `
|
|
44
|
+
Reports each expressible assertion as PASSED or FAILED with its playwright form;
|
|
45
|
+
no overall verdict, read the lines. "none ran" means unexpressible, not false.
|
|
46
|
+
`;
|
|
47
|
+
export const researchHelp = dedent `
|
|
48
|
+
The map is saved per page state and joins later commands there, so one research run pays for all that follow it.
|
|
49
|
+
`;
|
|
50
|
+
export const statusHelp = dedent `
|
|
51
|
+
Reads recorded files, so it needs no browser and outlives the session.
|
|
52
|
+
The hash is matched across all recorded sites. ### Artifacts lists every kept
|
|
53
|
+
file: aria, html, screenshot and network log when captured, plus per-step captures of a do run.
|
|
54
|
+
`;
|
|
55
|
+
export const reportHelp = dedent `
|
|
56
|
+
Built from the command log, so it needs no browser and outlives the session.
|
|
57
|
+
Reports the latest session unless --pw-session names another.
|
|
58
|
+
`;
|
|
59
|
+
export const sessionHelp = dedent `
|
|
60
|
+
Parallel jobs need one --instance each. --session is ignored when attached,
|
|
61
|
+
since the attached session keeps its own. --framework is parsed but inactive;
|
|
62
|
+
reported code is CodeceptJS either way.
|
|
63
|
+
`;
|
|
@@ -23,14 +23,13 @@ import { browserErrorMessage } from "../../../src/utils/browser-errors.js";
|
|
|
23
23
|
import { pluralize } from "../../../src/utils/logger.js";
|
|
24
24
|
import { mdq } from "../../../src/utils/markdown-query.js";
|
|
25
25
|
import { safeFilename } from "../../../src/utils/strings.js";
|
|
26
|
-
import { STATUS_FILE, readArtifacts, writeArtifacts } from "./envelope.js";
|
|
26
|
+
import { STATUS_FILE, STEP_FILES, readArtifacts, writeArtifacts } from "./envelope.js";
|
|
27
27
|
import { isFunctionExpression, takePwValue, toCodeceptWrapper } from "./pw-parser.js";
|
|
28
28
|
import { readDescriptors, selectDescriptor } from "./pw-registry.js";
|
|
29
29
|
import { latestSessionFile, readSession, recordCommand, sessionFile, sessionsDir } from "./session-log.js";
|
|
30
|
-
const
|
|
30
|
+
const WITHHELD_TOOLS = ['learnExperience', 'askUser', 'research'];
|
|
31
31
|
const ITERATIONS_PER_INSTRUCTION = 2;
|
|
32
32
|
const MAX_INSTRUCTION_ITERATIONS = 24;
|
|
33
|
-
const DEFAULT_RESEARCH_AFTER_VISITS = 3;
|
|
34
33
|
const CONTEXT_HTML_CAP = 6000;
|
|
35
34
|
const MAX_TOOL_ROUNDTRIPS = 5;
|
|
36
35
|
const AI_AGENT_NAME = 'prima';
|
|
@@ -68,7 +67,6 @@ export class Prima {
|
|
|
68
67
|
server = null;
|
|
69
68
|
attached = null;
|
|
70
69
|
session = null;
|
|
71
|
-
artifacts;
|
|
72
70
|
constructor(options = {}) {
|
|
73
71
|
this.options = options;
|
|
74
72
|
this.bot = new ExplorBot({
|
|
@@ -99,6 +97,7 @@ export class Prima {
|
|
|
99
97
|
const config = await this.loadConfig();
|
|
100
98
|
await this.resolveBrowser(config, discovery);
|
|
101
99
|
await this.bot.start();
|
|
100
|
+
this.bot.agentResearcher().disable();
|
|
102
101
|
if (!this.options.url)
|
|
103
102
|
return;
|
|
104
103
|
if (this.bot.getCurrentState())
|
|
@@ -382,9 +381,11 @@ export class Prima {
|
|
|
382
381
|
const guard = await this.aiGuard(command);
|
|
383
382
|
if (guard)
|
|
384
383
|
return guard;
|
|
384
|
+
const researcher = this.bot.agentResearcher();
|
|
385
|
+
researcher.enable();
|
|
385
386
|
const previousState = this.bot.stateManager().getCurrentState();
|
|
386
387
|
const result = await this.capturedResult(previousState);
|
|
387
|
-
const uiMap = await
|
|
388
|
+
const uiMap = await researcher.research(result, { screenshot: true, data: opts.data, deep: opts.deep, force: opts.fresh });
|
|
388
389
|
return this.reportEnvelope(command, result, previousState, { research: dropVolatileColumns(uiMap) });
|
|
389
390
|
}
|
|
390
391
|
async go(target) {
|
|
@@ -768,7 +769,7 @@ export class Prima {
|
|
|
768
769
|
if (!researcher || !navigator)
|
|
769
770
|
return {};
|
|
770
771
|
const tools = createAgentTools({ ...deps, researcher, navigator, withExperience: false });
|
|
771
|
-
for (const name of
|
|
772
|
+
for (const name of WITHHELD_TOOLS)
|
|
772
773
|
delete tools[name];
|
|
773
774
|
return tools;
|
|
774
775
|
}
|
|
@@ -823,14 +824,15 @@ export class Prima {
|
|
|
823
824
|
}
|
|
824
825
|
async pageContext(result) {
|
|
825
826
|
const experience = this.bot.experienceTracker?.()?.renderExperienceTocFor?.(result) || '';
|
|
826
|
-
const map =
|
|
827
|
+
const map = getPreviousResearch(result.baseHash);
|
|
828
|
+
let uiMap = '';
|
|
827
829
|
if (map) {
|
|
828
|
-
|
|
829
|
-
<page_ui_map
|
|
830
|
+
uiMap = dedent `
|
|
831
|
+
<page_ui_map>
|
|
832
|
+
A map of this page recorded by an earlier research run. It names parts the accessibility
|
|
833
|
+
tree does not, and can be out of date — the tree is what the page holds now.
|
|
830
834
|
${map}
|
|
831
835
|
</page_ui_map>
|
|
832
|
-
|
|
833
|
-
${experience}
|
|
834
836
|
`;
|
|
835
837
|
}
|
|
836
838
|
return dedent `
|
|
@@ -838,20 +840,11 @@ export class Prima {
|
|
|
838
840
|
${compactAriaSnapshot(await this.refAriaSnapshot(result), true, (value) => this.offloadValue(value))}
|
|
839
841
|
</page>
|
|
840
842
|
|
|
843
|
+
${uiMap}
|
|
844
|
+
|
|
841
845
|
${experience}
|
|
842
846
|
`;
|
|
843
847
|
}
|
|
844
|
-
researchMap(result) {
|
|
845
|
-
if (this.bot.stateManager().getVisitCount(result.url) < this.researchAfterVisits())
|
|
846
|
-
return '';
|
|
847
|
-
return getPreviousResearch(result.getStateHash());
|
|
848
|
-
}
|
|
849
|
-
researchAfterVisits() {
|
|
850
|
-
const configured = this.bot.getConfig?.()?.ai?.agents?.prima?.researchAfterVisits;
|
|
851
|
-
if (typeof configured === 'number')
|
|
852
|
-
return configured;
|
|
853
|
-
return DEFAULT_RESEARCH_AFTER_VISITS;
|
|
854
|
-
}
|
|
855
848
|
offloadValue(value) {
|
|
856
849
|
const dir = this.statusDir();
|
|
857
850
|
const name = `value-${createHash('sha1').update(value).digest('hex').slice(0, 8)}.txt`;
|
|
@@ -923,16 +916,13 @@ export class Prima {
|
|
|
923
916
|
}
|
|
924
917
|
async successEnvelope(command, used, result, previousState) {
|
|
925
918
|
const changes = await this.pageChanges(result, previousState, used[0]);
|
|
926
|
-
const status = await this.saveStatus(result);
|
|
927
919
|
return {
|
|
928
920
|
ok: true,
|
|
929
921
|
command,
|
|
930
922
|
used,
|
|
931
923
|
page: this.pageBlock(result, previousState),
|
|
932
924
|
changes,
|
|
933
|
-
|
|
934
|
-
status,
|
|
935
|
-
artifacts: this.artifacts,
|
|
925
|
+
...(await this.envelopeTail(result)),
|
|
936
926
|
};
|
|
937
927
|
}
|
|
938
928
|
async failureEnvelope(command, error, previousState) {
|
|
@@ -940,29 +930,27 @@ export class Prima {
|
|
|
940
930
|
const failure = { error: browserErrorMessage(error) };
|
|
941
931
|
if (result.ariaSnapshot)
|
|
942
932
|
failure.compactAria = compactAriaSnapshot(result.ariaSnapshot, true);
|
|
943
|
-
const status = await this.saveStatus(result);
|
|
944
933
|
return {
|
|
945
934
|
ok: false,
|
|
946
935
|
command,
|
|
947
936
|
page: this.pageBlock(result, previousState),
|
|
948
937
|
failure,
|
|
949
|
-
|
|
950
|
-
status,
|
|
951
|
-
artifacts: this.artifacts,
|
|
938
|
+
...(await this.envelopeTail(result)),
|
|
952
939
|
};
|
|
953
940
|
}
|
|
954
941
|
async reportEnvelope(command, result, previousState, outcome) {
|
|
955
|
-
const status = await this.saveStatus(result);
|
|
956
942
|
return {
|
|
957
943
|
ok: true,
|
|
958
944
|
command,
|
|
959
945
|
page: this.pageBlock(result, previousState),
|
|
960
946
|
...outcome,
|
|
961
|
-
|
|
962
|
-
status,
|
|
963
|
-
artifacts: this.artifacts,
|
|
947
|
+
...(await this.envelopeTail(result)),
|
|
964
948
|
};
|
|
965
949
|
}
|
|
950
|
+
async envelopeTail(result) {
|
|
951
|
+
const { hash, artifacts } = await this.saveStatus(result);
|
|
952
|
+
return { instance: await this.instanceInfo(), status: hash, artifacts };
|
|
953
|
+
}
|
|
966
954
|
async capturedResult(previousState, opts = {}) {
|
|
967
955
|
const captured = await this.bot
|
|
968
956
|
.getExplorer()
|
|
@@ -1026,9 +1014,9 @@ export class Prima {
|
|
|
1026
1014
|
}
|
|
1027
1015
|
async saveStatus(result) {
|
|
1028
1016
|
const hash = this.statusHash();
|
|
1029
|
-
await this.writeSnapshot(result);
|
|
1017
|
+
const artifacts = await this.writeSnapshot(result);
|
|
1030
1018
|
writeFileSync(path.join(this.statusDir(hash), STATUS_FILE), JSON.stringify({ page: this.pageBlock(result, null) }), 'utf-8');
|
|
1031
|
-
return hash;
|
|
1019
|
+
return { hash, artifacts };
|
|
1032
1020
|
}
|
|
1033
1021
|
async writeStepFiles(index, label, diff) {
|
|
1034
1022
|
const state = this.bot.stateManager().getCurrentState();
|
|
@@ -1038,17 +1026,17 @@ export class Prima {
|
|
|
1038
1026
|
mkdirSync(dir, { recursive: true });
|
|
1039
1027
|
const stem = path.join(dir, `${index}-${safeFilename(label.slice(0, 60))}`);
|
|
1040
1028
|
const result = ActionResult.fromState(state);
|
|
1041
|
-
writeFileSync(`${stem}.aria
|
|
1042
|
-
writeFileSync(`${stem}.html`, await result.combinedHtml(), 'utf-8');
|
|
1029
|
+
writeFileSync(`${stem}.${STEP_FILES.aria}`, result.ariaSnapshot ?? '', 'utf-8');
|
|
1030
|
+
writeFileSync(`${stem}.${STEP_FILES.html}`, await result.combinedHtml(), 'utf-8');
|
|
1043
1031
|
if (diff)
|
|
1044
|
-
writeFileSync(`${stem}.diff
|
|
1032
|
+
writeFileSync(`${stem}.${STEP_FILES.diff}`, diff, 'utf-8');
|
|
1045
1033
|
}
|
|
1046
1034
|
async writeSnapshot(result) {
|
|
1047
|
-
|
|
1035
|
+
return writeArtifacts(this.statusDir(), {
|
|
1048
1036
|
aria: result.ariaSnapshot,
|
|
1049
1037
|
html: await result.combinedHtml(),
|
|
1050
1038
|
screenshot: result.screenshot,
|
|
1051
|
-
requests: this.bot.requestStore().
|
|
1039
|
+
requests: this.bot.requestStore().getCapturedRequests(),
|
|
1052
1040
|
});
|
|
1053
1041
|
}
|
|
1054
1042
|
statusHash() {
|
package/dist/package.json
CHANGED
|
@@ -26,6 +26,7 @@ interface ActionResultData extends WebPageState {
|
|
|
26
26
|
}>;
|
|
27
27
|
ariaSnapshot?: string | null;
|
|
28
28
|
ariaSnapshotFile?: string;
|
|
29
|
+
regionAria?: string | null;
|
|
29
30
|
focusedElement?: FocusedElement | null;
|
|
30
31
|
iframeURL?: string;
|
|
31
32
|
links?: Link[];
|
|
@@ -84,6 +85,7 @@ export declare class ActionResult implements ActionResultData {
|
|
|
84
85
|
links: Link[];
|
|
85
86
|
verifications?: Record<string, boolean>;
|
|
86
87
|
overlay: Region;
|
|
88
|
+
regionAria: string | null;
|
|
87
89
|
_diffCache: {
|
|
88
90
|
previousId: number | undefined;
|
|
89
91
|
diff: Diff;
|
|
@@ -109,6 +111,7 @@ export declare class ActionResult implements ActionResultData {
|
|
|
109
111
|
}): Promise<string>;
|
|
110
112
|
textHtml(htmlConfig?: HtmlConfig): Promise<string>;
|
|
111
113
|
getInteractiveARIA(): string;
|
|
114
|
+
getRegionARIA(): string;
|
|
112
115
|
getCompactARIA(): string;
|
|
113
116
|
normalizeHtmlConfig(htmlConfig?: HtmlConfig): HtmlConfig | undefined;
|
|
114
117
|
static fromState(state: WebPageState): ActionResult;
|
|
@@ -40,6 +40,7 @@ export class ActionResult {
|
|
|
40
40
|
links = [];
|
|
41
41
|
verifications;
|
|
42
42
|
overlay = new Region();
|
|
43
|
+
regionAria = null;
|
|
43
44
|
_diffCache = null;
|
|
44
45
|
constructor(data) {
|
|
45
46
|
this.id = data.id;
|
|
@@ -55,6 +56,7 @@ export class ActionResult {
|
|
|
55
56
|
this.iframeURL = data.iframeURL;
|
|
56
57
|
this.notes = data.notes ?? [];
|
|
57
58
|
this.verifications = data.verifications;
|
|
59
|
+
this.regionAria = data.regionAria ?? null;
|
|
58
60
|
// Set readonly properties
|
|
59
61
|
if (data.screenshotFile !== undefined) {
|
|
60
62
|
this.screenshotFile = data.screenshotFile;
|
|
@@ -238,6 +240,9 @@ export class ActionResult {
|
|
|
238
240
|
getInteractiveARIA() {
|
|
239
241
|
return compactAriaSnapshot(this.ariaSnapshot, false);
|
|
240
242
|
}
|
|
243
|
+
getRegionARIA() {
|
|
244
|
+
return compactAriaSnapshot(this.regionAria, false);
|
|
245
|
+
}
|
|
241
246
|
getCompactARIA() {
|
|
242
247
|
return compactAriaSnapshot(this.ariaSnapshot, true);
|
|
243
248
|
}
|
package/dist/src/action.js
CHANGED
|
@@ -165,8 +165,19 @@ class Action {
|
|
|
165
165
|
focusedElement,
|
|
166
166
|
iframeURL: frame ? frame.url?.() || 'iframe' : undefined,
|
|
167
167
|
});
|
|
168
|
-
if (!frame)
|
|
168
|
+
if (!frame) {
|
|
169
169
|
await this.detectRegion(result).catch((err) => debugLog('Region detection failed:', err.message));
|
|
170
|
+
const regionRoot = result.overlay.root;
|
|
171
|
+
if (result.overlay.isModal && regionRoot) {
|
|
172
|
+
result.regionAria = await this.playwrightHelper.page
|
|
173
|
+
.locator(regionRoot)
|
|
174
|
+
.ariaSnapshot()
|
|
175
|
+
.catch((err) => {
|
|
176
|
+
debugLog('Region ARIA snapshot failed:', err.message);
|
|
177
|
+
return null;
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
}
|
|
170
181
|
this.stateManager.updateState(result, codeBlock);
|
|
171
182
|
return result;
|
|
172
183
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { ActionResult, type Diff } from '../../action-result.js';
|
|
2
|
-
import type
|
|
2
|
+
import { type ExplorbotConfig } from '../../config.js';
|
|
3
3
|
import type Explorer from '../../explorer.js';
|
|
4
4
|
import type { StateManager } from '../../state-manager.js';
|
|
5
5
|
import { WebPageState } from '../../state-manager.js';
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import dedent from 'dedent';
|
|
2
2
|
import { ActionResult } from '../../action-result.js';
|
|
3
|
+
import { agentSettings } from "../../config.js";
|
|
3
4
|
import { executionController } from "../../execution-controller.js";
|
|
4
5
|
import { diffAriaSnapshots } from "../../utils/aria.js";
|
|
5
6
|
import { extractCodeBlocks } from "../../utils/code-extractor.js";
|
|
@@ -16,7 +17,7 @@ export function WithDeepAnalysis(Base) {
|
|
|
16
17
|
async performDeepAnalysis(state, result) {
|
|
17
18
|
tag('info').log('Starting deep analysis of expandable elements');
|
|
18
19
|
await this.navigateTo(state.fullUrl || state.url);
|
|
19
|
-
const maxClicks = this.config.
|
|
20
|
+
const maxClicks = agentSettings(this.config, 'researcher').maxExpandableClicks ?? DEFAULT_MAX_EXPANDABLE_CLICKS;
|
|
20
21
|
const expandedSections = [];
|
|
21
22
|
const navigationLinks = [];
|
|
22
23
|
let verifiedCodes = [];
|
|
@@ -62,6 +63,8 @@ export function WithDeepAnalysis(Base) {
|
|
|
62
63
|
this._appendExtendedResearch(result, expandedSections, navigationLinks);
|
|
63
64
|
}
|
|
64
65
|
async researchOverlay(current, previous, pageStateHash) {
|
|
66
|
+
if (!this.isEnabled())
|
|
67
|
+
return null;
|
|
65
68
|
const region = current.overlay;
|
|
66
69
|
if (!region.isOpen || !region.name)
|
|
67
70
|
return null;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ActionResult } from '../../action-result.js';
|
|
2
|
-
import type
|
|
2
|
+
import { type ExplorbotConfig } from '../../config.js';
|
|
3
3
|
import type Explorer from '../../explorer.js';
|
|
4
4
|
import type { StateManager } from '../../state-manager.js';
|
|
5
5
|
import type { Provider } from '../provider.js';
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import dedent from 'dedent';
|
|
2
|
+
import { agentSettings } from "../../config.js";
|
|
2
3
|
import { executionController } from "../../execution-controller.js";
|
|
3
4
|
import { tag } from '../../utils/logger.js';
|
|
4
5
|
import { RulesLoader } from "../../utils/rules-loader.js";
|
|
@@ -50,7 +51,7 @@ export function WithSections(Base) {
|
|
|
50
51
|
return focused.text;
|
|
51
52
|
}
|
|
52
53
|
async _detectFocusCss() {
|
|
53
|
-
const focusSections = this.config.
|
|
54
|
+
const focusSections = agentSettings(this.config, 'researcher').focusSections;
|
|
54
55
|
if (!focusSections?.length)
|
|
55
56
|
return null;
|
|
56
57
|
for (const css of focusSections) {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import dedent from 'dedent';
|
|
2
2
|
import { ActionResult } from '../action-result.js';
|
|
3
3
|
import { setActivity } from "../activity.js";
|
|
4
|
-
import { outputPath } from "../config.js";
|
|
4
|
+
import { agentSettings, outputPath } from "../config.js";
|
|
5
5
|
import { executionController } from "../execution-controller.js";
|
|
6
6
|
import { Observability } from "../observability.js";
|
|
7
7
|
import { Stats } from "../stats.js";
|
|
@@ -13,7 +13,7 @@ import { mdq } from "../utils/markdown-query.js";
|
|
|
13
13
|
import { RulesLoader } from "../utils/rules-loader.js";
|
|
14
14
|
import { annotatePageElements } from "../utils/web-annotate.js";
|
|
15
15
|
import { ContextLengthError } from './provider.js';
|
|
16
|
-
import { findSimilarResearch, getCachedResearch, reportResearch, saveResearch } from "./researcher/cache.js";
|
|
16
|
+
import { findSimilarResearch, getCachedResearch, getPreviousResearch, reportResearch, saveResearch } from "./researcher/cache.js";
|
|
17
17
|
import { WithCoordinates } from "./researcher/coordinates.js";
|
|
18
18
|
import { WithDeepAnalysis } from "./researcher/deep-analysis.js";
|
|
19
19
|
import { detectFocusedSection, hasFocusedSection, markSectionAsFocused, pickDefaultFocusedSection } from "./researcher/focus.js";
|
|
@@ -42,12 +42,19 @@ export class Researcher extends ResearcherBase {
|
|
|
42
42
|
constructor(deps) {
|
|
43
43
|
super(deps);
|
|
44
44
|
this.experienceTracker = deps.stateManager.getExperienceTracker();
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
45
|
+
this.settings.reasoning ??= 'low';
|
|
46
|
+
}
|
|
47
|
+
get settings() {
|
|
48
|
+
return agentSettings(this.config, 'researcher');
|
|
49
|
+
}
|
|
50
|
+
isEnabled() {
|
|
51
|
+
return this.settings.enabled !== false;
|
|
52
|
+
}
|
|
53
|
+
enable() {
|
|
54
|
+
this.settings.enabled = true;
|
|
55
|
+
}
|
|
56
|
+
disable() {
|
|
57
|
+
this.settings.enabled = false;
|
|
51
58
|
}
|
|
52
59
|
getNavigator() {
|
|
53
60
|
throw new Error('not implemented');
|
|
@@ -70,7 +77,7 @@ export class Researcher extends ResearcherBase {
|
|
|
70
77
|
}
|
|
71
78
|
async research(state, opts = {}) {
|
|
72
79
|
const { screenshot = false, force = false, deep = false, data = false, fix = true } = opts;
|
|
73
|
-
const maxRetries = this.
|
|
80
|
+
const maxRetries = this.settings.retries ?? 2;
|
|
74
81
|
let retriesLeft = opts._retriesLeft ?? maxRetries;
|
|
75
82
|
this.actionResult = ActionResult.fromState(state);
|
|
76
83
|
const stateHash = this.actionResult.baseHash;
|
|
@@ -83,6 +90,13 @@ export class Researcher extends ResearcherBase {
|
|
|
83
90
|
return cached;
|
|
84
91
|
}
|
|
85
92
|
}
|
|
93
|
+
if (!this.isEnabled()) {
|
|
94
|
+
debugLog('Researcher is disabled, answering with the recorded map');
|
|
95
|
+
const recorded = getPreviousResearch(stateHash);
|
|
96
|
+
if (recorded)
|
|
97
|
+
reportResearch(stateHash, recorded);
|
|
98
|
+
return recorded;
|
|
99
|
+
}
|
|
86
100
|
Stats.researches++;
|
|
87
101
|
const sessionName = `researcher: ${state.url}`;
|
|
88
102
|
return Observability.run(sessionName, { tags: ['researcher'], sessionId: stateHash }, async () => {
|
|
@@ -286,7 +300,7 @@ export class Researcher extends ResearcherBase {
|
|
|
286
300
|
this.actionResult = await this.explorer.visit(url, { screenshot: screenshot ?? false });
|
|
287
301
|
}
|
|
288
302
|
async waitUntilSettled(screenshot) {
|
|
289
|
-
const errorPageTimeout = this.
|
|
303
|
+
const errorPageTimeout = this.settings.errorPageTimeout ?? 10;
|
|
290
304
|
if (errorPageTimeout <= 0)
|
|
291
305
|
return false;
|
|
292
306
|
const includeScreenshot = screenshot && this.provider.hasVision();
|
|
@@ -316,7 +330,7 @@ export class Researcher extends ResearcherBase {
|
|
|
316
330
|
return false;
|
|
317
331
|
}
|
|
318
332
|
getConfiguredSections() {
|
|
319
|
-
const configSections = this.
|
|
333
|
+
const configSections = this.settings.sections;
|
|
320
334
|
if (!configSections?.length)
|
|
321
335
|
return POSSIBLE_SECTIONS;
|
|
322
336
|
const filtered = {};
|
package/dist/src/ai/tester.d.ts
CHANGED
package/dist/src/ai/tester.js
CHANGED
|
@@ -40,6 +40,7 @@ export class Tester extends TaskAgent {
|
|
|
40
40
|
MAX_ITERATIONS = 30;
|
|
41
41
|
MAX_EXTENSIONS = 2;
|
|
42
42
|
ASSERTION_TOOLS = ['verify'];
|
|
43
|
+
pendingReview = '';
|
|
43
44
|
researcher;
|
|
44
45
|
navigator;
|
|
45
46
|
agentTools;
|
|
@@ -97,6 +98,7 @@ export class Tester extends TaskAgent {
|
|
|
97
98
|
this.seenUiMapUrls.clear();
|
|
98
99
|
this.lastAnalyzedStateHash = null;
|
|
99
100
|
this.stalledIterations = 0;
|
|
101
|
+
this.pendingReview = '';
|
|
100
102
|
this.previousRegionPresent = null;
|
|
101
103
|
this.regionTransitioned = false;
|
|
102
104
|
this.stateManager.clearHistory();
|
|
@@ -293,7 +295,7 @@ export class Tester extends TaskAgent {
|
|
|
293
295
|
const result = await this.provider.invokeConversation(conversation, tools, {
|
|
294
296
|
maxToolRoundtrips: 3,
|
|
295
297
|
toolChoice: 'required',
|
|
296
|
-
stopWhen: () => task.hasFinished,
|
|
298
|
+
stopWhen: () => task.hasFinished || !!this.pendingReview,
|
|
297
299
|
});
|
|
298
300
|
if (!result)
|
|
299
301
|
throw new Error('Failed to get response from provider');
|
|
@@ -339,6 +341,15 @@ export class Tester extends TaskAgent {
|
|
|
339
341
|
`);
|
|
340
342
|
}
|
|
341
343
|
}
|
|
344
|
+
if (this.pendingReview && this.pilot) {
|
|
345
|
+
const reviewed = this.pendingReview;
|
|
346
|
+
this.pendingReview = '';
|
|
347
|
+
const reviewState = this.getCurrentState();
|
|
348
|
+
if (reviewed === 'finish')
|
|
349
|
+
await this.pilot.reviewFinish(task, reviewState, conversation, this.navigator);
|
|
350
|
+
if (reviewed === 'stop')
|
|
351
|
+
await this.pilot.reviewStop(task, reviewState, conversation);
|
|
352
|
+
}
|
|
342
353
|
if (task.hasFinished) {
|
|
343
354
|
stop();
|
|
344
355
|
return;
|
|
@@ -513,17 +524,20 @@ export class Tester extends TaskAgent {
|
|
|
513
524
|
}
|
|
514
525
|
if (region.isModal) {
|
|
515
526
|
const areaName = region.name ? ` "${region.name}"` : '';
|
|
516
|
-
let
|
|
517
|
-
if (region.root)
|
|
518
|
-
|
|
527
|
+
let scoping = 'Use <page_aria> to confirm the element you target is actually inside the overlay.';
|
|
528
|
+
if (region.root) {
|
|
529
|
+
scoping = `Its root is \`${region.root}\` — build every locator as ARIA scoped to that root, e.g. I.click({ role: 'button', text: 'Continue' }, '${region.root}')`;
|
|
530
|
+
}
|
|
519
531
|
context += dedent `
|
|
520
532
|
<overlay>
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
Use <page_aria> to confirm the element you target is actually inside the overlay.
|
|
525
|
-
</overlay>
|
|
533
|
+
You are inside an overlay${areaName} opened above the page.
|
|
534
|
+
${scoping}
|
|
535
|
+
Elements outside the overlay are behind it and not actionable while it is open — they may share names or roles with the ones inside, so never target them by bare text.
|
|
526
536
|
`;
|
|
537
|
+
const regionAria = currentState.getRegionARIA();
|
|
538
|
+
if (regionAria)
|
|
539
|
+
context += `\nIt holds exactly these elements:\n<overlay_aria>\n${regionAria}\n</overlay_aria>`;
|
|
540
|
+
context += '\n</overlay>\n';
|
|
527
541
|
}
|
|
528
542
|
if (!region.isModal && region.isOpen && isNewState) {
|
|
529
543
|
let rootHint = '';
|
|
@@ -920,18 +934,8 @@ export class Tester extends TaskAgent {
|
|
|
920
934
|
}),
|
|
921
935
|
execute: async ({ reason }) => {
|
|
922
936
|
task.addNote(`Stop requested: ${reason}`);
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
await this.pilot.reviewStop(task, currentState, conversation);
|
|
926
|
-
if (!task.hasFinished) {
|
|
927
|
-
return {
|
|
928
|
-
success: false,
|
|
929
|
-
action: 'stop',
|
|
930
|
-
message: 'Stop rejected; Continue execution',
|
|
931
|
-
};
|
|
932
|
-
}
|
|
933
|
-
}
|
|
934
|
-
else {
|
|
937
|
+
this.pendingReview = 'stop';
|
|
938
|
+
if (!this.pilot) {
|
|
935
939
|
task.addNote(reason, TestResult.FAILED);
|
|
936
940
|
task.finish(TestResult.FAILED);
|
|
937
941
|
}
|
|
@@ -966,18 +970,8 @@ export class Tester extends TaskAgent {
|
|
|
966
970
|
return { success: true, action: 'finish', message: 'already finished' };
|
|
967
971
|
}
|
|
968
972
|
task.addNote(`Finish requested: ${verify}`);
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
await this.pilot.reviewFinish(task, currentState, conversation, this.navigator);
|
|
972
|
-
if (!task.hasFinished) {
|
|
973
|
-
return {
|
|
974
|
-
success: false,
|
|
975
|
-
action: 'finish',
|
|
976
|
-
message: 'Finishing rejected; Continue execution',
|
|
977
|
-
};
|
|
978
|
-
}
|
|
979
|
-
}
|
|
980
|
-
else {
|
|
973
|
+
this.pendingReview = 'finish';
|
|
974
|
+
if (!this.pilot) {
|
|
981
975
|
task.addNote('Test finished successfully', TestResult.PASSED);
|
|
982
976
|
task.finish(TestResult.PASSED);
|
|
983
977
|
}
|
package/dist/src/ai/tools.js
CHANGED
|
@@ -634,6 +634,11 @@ export function createAgentTools({ explorer, stateManager, ai, researcher, navig
|
|
|
634
634
|
return failedToolResult('research', 'No current page state available. Navigate to a page first.');
|
|
635
635
|
}
|
|
636
636
|
const researchResult = await researcher.research(currentState, { screenshot: true, data: true });
|
|
637
|
+
if (!researchResult) {
|
|
638
|
+
return failedToolResult('research', 'No UI map is available for this page.', {
|
|
639
|
+
suggestion: 'Use context() to read the page structure and act on the elements it lists.',
|
|
640
|
+
});
|
|
641
|
+
}
|
|
637
642
|
return successToolResult('research', {
|
|
638
643
|
analysis: researchResult,
|
|
639
644
|
aria: cap(ActionResult.fromState(currentState).getInteractiveARIA(), ARIA_OUTPUT_CAP),
|
|
@@ -39,8 +39,12 @@ export class ConfigCommand extends BaseCommand {
|
|
|
39
39
|
const env = {};
|
|
40
40
|
for (const variable of EXPLORBOT_ENV_VARS) {
|
|
41
41
|
const value = process.env[variable.name];
|
|
42
|
-
if (value)
|
|
43
|
-
|
|
42
|
+
if (!value)
|
|
43
|
+
continue;
|
|
44
|
+
let shown = value;
|
|
45
|
+
if (variable.secret)
|
|
46
|
+
shown = 'set';
|
|
47
|
+
env[variable.name] = shown;
|
|
44
48
|
}
|
|
45
49
|
const models = {};
|
|
46
50
|
const providers = {};
|
package/dist/src/config.d.ts
CHANGED
|
@@ -269,7 +269,9 @@ export declare class ConfigParser {
|
|
|
269
269
|
deepMerge(target: any, source: any): any;
|
|
270
270
|
ensureDirectory(path: string): void;
|
|
271
271
|
}
|
|
272
|
+
export declare function setOutputDir(dir: string): void;
|
|
272
273
|
export declare function outputPath(...segments: string[]): string;
|
|
274
|
+
export declare function agentSettings<K extends keyof AgentsConfig>(config: ExplorbotConfig, agent: K): NonNullable<AgentsConfig[K]>;
|
|
273
275
|
export declare function resolveModel(spec: string, role?: ModelRole): Promise<any>;
|
|
274
276
|
export declare function missingModelRoles(provider: string): ModelRole[];
|
|
275
277
|
export declare class ConfigMissingError extends Error {
|
|
@@ -297,5 +299,6 @@ interface EnvVar {
|
|
|
297
299
|
name: string;
|
|
298
300
|
description: string;
|
|
299
301
|
required?: boolean;
|
|
302
|
+
secret?: boolean;
|
|
300
303
|
}
|
|
301
304
|
export type { ModelRole, EnvVar, ProviderInfo, ConfiguredModel };
|