explorbot 0.2.3 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/bin/explorbot-cli.ts +7 -1
- package/boat/api-tester/src/config.ts +4 -2
- package/boat/doc-collector/bin/doc-collector-cli.ts +2 -0
- package/boat/doc-collector/src/ai/documentarian.ts +61 -31
- package/boat/doc-collector/src/config.ts +4 -2
- package/boat/prima/bin/prima-cli.ts +0 -0
- package/boat/prima/src/activity-line.ts +33 -0
- package/boat/prima/src/cli.ts +115 -86
- package/boat/prima/src/envelope.ts +80 -52
- package/boat/prima/src/prima.ts +547 -128
- package/boat/prima/src/pw-parser.ts +11 -1
- package/boat/prima/src/pw-registry.ts +4 -5
- package/boat/prima/src/session-log.ts +126 -0
- package/dist/bin/explorbot-cli.js +7 -1
- package/dist/boat/api-tester/bin/apibot-cli.js +2 -0
- package/dist/boat/api-tester/src/config.js +4 -2
- package/dist/boat/doc-collector/bin/doc-collector-cli.js +2 -0
- package/dist/boat/doc-collector/src/ai/documentarian.js +44 -19
- package/dist/boat/doc-collector/src/config.js +4 -2
- package/dist/boat/prima/src/activity-line.js +30 -0
- package/dist/boat/prima/src/cli.js +97 -77
- package/dist/boat/prima/src/envelope.js +70 -44
- package/dist/boat/prima/src/prima.js +515 -119
- package/dist/boat/prima/src/pw-parser.js +13 -1
- package/dist/boat/prima/src/pw-registry.js +4 -5
- package/dist/boat/prima/src/session-log.js +108 -0
- package/dist/package.json +3 -2
- package/dist/rules/navigator/verification-actions.md +20 -0
- package/dist/src/action-result.d.ts +7 -0
- package/dist/src/action-result.js +4 -0
- package/dist/src/action.d.ts +2 -0
- package/dist/src/action.js +36 -1
- package/dist/src/ai/captain/web-mode.js +6 -3
- package/dist/src/ai/captain.js +2 -0
- package/dist/src/ai/navigator.d.ts +7 -0
- package/dist/src/ai/navigator.js +10 -6
- package/dist/src/ai/pilot.d.ts +4 -0
- package/dist/src/ai/pilot.js +48 -2
- package/dist/src/ai/provider.js +2 -2
- package/dist/src/ai/rerunner.js +2 -1
- package/dist/src/ai/researcher.js +1 -1
- package/dist/src/ai/rules.js +17 -10
- package/dist/src/ai/task-agent.js +4 -1
- package/dist/src/ai/tester.d.ts +1 -1
- package/dist/src/ai/tester.js +33 -33
- package/dist/src/ai/tools.d.ts +14 -0
- package/dist/src/ai/tools.js +113 -36
- package/dist/src/config.d.ts +1 -0
- package/dist/src/config.js +13 -4
- package/dist/src/execution-controller.d.ts +2 -0
- package/dist/src/execution-controller.js +6 -0
- package/dist/src/explorbot.d.ts +2 -1
- package/dist/src/explorbot.js +3 -1
- package/dist/src/explorer.js +2 -3
- package/dist/src/playwright-recorder.js +30 -0
- package/dist/src/remote.d.ts +54 -0
- package/dist/src/remote.js +229 -0
- package/dist/src/reporter.d.ts +1 -0
- package/dist/src/reporter.js +7 -1
- package/dist/src/state-manager.d.ts +2 -1
- package/dist/src/stats.d.ts +1 -0
- package/dist/src/stats.js +1 -0
- package/dist/src/test-plan.d.ts +2 -0
- package/dist/src/test-plan.js +7 -0
- package/dist/src/utils/aria.d.ts +2 -8
- package/dist/src/utils/aria.js +69 -40
- package/dist/src/utils/html.js +1 -0
- package/dist/src/utils/logger.d.ts +6 -0
- package/dist/src/utils/logger.js +24 -0
- package/dist/src/utils/page-readiness.js +18 -1
- package/dist/src/utils/url-matcher.js +3 -0
- package/dist/src/utils/web-element.d.ts +2 -0
- package/dist/src/utils/web-element.js +8 -0
- package/dist/src/utils/web-sandbox.d.ts +1 -1
- package/dist/src/utils/web-sandbox.js +2 -3
- package/docs/api-testing/basics.md +90 -0
- package/docs/api-testing/planning.md +57 -0
- package/docs/api-testing/running-tests.md +55 -0
- package/docs/assets/cloud-report.png +0 -0
- package/docs/assets/html-report.png +0 -0
- package/docs/assets/langfuse-trace.png +0 -0
- package/docs/assets/successful-explore-run.png +0 -0
- package/docs/basics/getting-started.md +140 -0
- package/docs/basics/prerequisites.md +63 -0
- package/docs/basics/providers.md +362 -0
- package/docs/basics/running.md +78 -0
- package/docs/contributing/ai-integration-tests.md +57 -0
- package/docs/contributing/contributing.md +90 -0
- package/docs/contributing/demo-videos.md +36 -0
- package/docs/contributing/npm-package.md +138 -0
- package/docs/contributing/observability.md +227 -0
- package/docs/contributing/regression-tests.md +103 -0
- package/docs/contributing/testing.md +95 -0
- package/docs/doc-collection/basics.md +128 -0
- package/docs/doc-collection/crawling.md +67 -0
- package/docs/doc-collection/interactive-mode.md +99 -0
- package/docs/index.json +86 -0
- package/docs/reference/commands.md +994 -0
- package/docs/reference/configuration.md +569 -0
- package/docs/reference/scripting.md +303 -0
- package/docs/superpowers/plans/2026-08-01-actor-boat.md +925 -0
- package/docs/superpowers/plans/2026-08-01-prima-boat.md +1120 -0
- package/docs/superpowers/reviews/2026-08-06-prima-vs-playwright-cli.md +268 -0
- package/docs/superpowers/specs/2026-08-01-actor-boat-design.md +204 -0
- package/docs/superpowers/specs/2026-08-01-prima-boat-design.md +242 -0
- package/docs/superpowers/specs/2026-08-03-global-config-design.md +138 -0
- package/docs/superpowers/specs/2026-08-07-prima-fixes-design.md +394 -0
- package/docs/web-testing/agents.md +158 -0
- package/docs/web-testing/automated-tests.md +134 -0
- package/docs/web-testing/basics.md +91 -0
- package/docs/web-testing/customization.md +131 -0
- package/docs/web-testing/hooks.md +238 -0
- package/docs/web-testing/page-interaction.md +84 -0
- package/docs/web-testing/planner.md +122 -0
- package/docs/web-testing/rerun.md +164 -0
- package/docs/web-testing/researcher.md +380 -0
- package/docs/workflow/agentic-usage.md +233 -0
- package/docs/workflow/application-spec.md +73 -0
- package/docs/workflow/ci.md +202 -0
- package/docs/workflow/knowledge.md +310 -0
- package/docs/workflow/planning-styles.md +67 -0
- package/docs/workflow/reporting.md +133 -0
- package/docs/workflow/test-plans.md +90 -0
- package/package.json +3 -2
- package/rules/navigator/verification-actions.md +20 -0
- package/src/action-result.ts +11 -0
- package/src/action.ts +38 -2
- package/src/ai/captain/web-mode.ts +6 -3
- package/src/ai/captain.ts +3 -0
- package/src/ai/navigator.ts +14 -8
- package/src/ai/pilot.ts +53 -2
- package/src/ai/provider.ts +2 -4
- package/src/ai/rerunner.ts +2 -1
- package/src/ai/researcher.ts +1 -1
- package/src/ai/rules.ts +17 -10
- package/src/ai/task-agent.ts +3 -1
- package/src/ai/tester.ts +32 -33
- package/src/ai/tools.ts +132 -36
- package/src/config.ts +15 -4
- package/src/execution-controller.ts +8 -0
- package/src/explorbot.ts +4 -2
- package/src/explorer.ts +2 -2
- package/src/playwright-recorder.ts +23 -0
- package/src/remote.ts +238 -0
- package/src/reporter.ts +7 -1
- package/src/state-manager.ts +2 -1
- package/src/stats.ts +1 -0
- package/src/test-plan.ts +9 -0
- package/src/utils/aria.ts +65 -45
- package/src/utils/html.ts +1 -0
- package/src/utils/logger.ts +24 -1
- package/src/utils/page-readiness.ts +24 -1
- package/src/utils/url-matcher.ts +3 -0
- package/src/utils/web-element.ts +9 -0
- package/src/utils/web-sandbox.ts +3 -4
|
@@ -14,5 +14,17 @@ export function isFunctionExpression(expr) {
|
|
|
14
14
|
return { valid: true };
|
|
15
15
|
}
|
|
16
16
|
export function toCodeceptWrapper(expr) {
|
|
17
|
-
return `I.usePlaywrightTo('pw', async (playwright) => (${expr.trim()})(playwright))`;
|
|
17
|
+
return `return I.usePlaywrightTo('pw', async (playwright) => (${expr.trim()})(playwright))`;
|
|
18
|
+
}
|
|
19
|
+
export function takePwValue(held) {
|
|
20
|
+
if (held === undefined || held === null)
|
|
21
|
+
return undefined;
|
|
22
|
+
if (typeof held === 'string')
|
|
23
|
+
return held;
|
|
24
|
+
try {
|
|
25
|
+
return JSON.stringify(held, null, 2);
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
return String(held);
|
|
29
|
+
}
|
|
18
30
|
}
|
|
@@ -19,9 +19,8 @@ export function readDescriptors(dir = registryDir()) {
|
|
|
19
19
|
}
|
|
20
20
|
return descriptors;
|
|
21
21
|
}
|
|
22
|
-
export function selectDescriptor(descriptors, opts) {
|
|
23
|
-
const
|
|
24
|
-
const candidates = descriptors.filter((descriptor) => path.resolve(descriptor.workspaceDir) === workspaceDir);
|
|
22
|
+
export function selectDescriptor(descriptors, opts = {}) {
|
|
23
|
+
const candidates = descriptors;
|
|
25
24
|
if (!candidates.length)
|
|
26
25
|
return { candidates };
|
|
27
26
|
if (opts.title) {
|
|
@@ -53,13 +52,13 @@ function parseDescriptor(file) {
|
|
|
53
52
|
catch {
|
|
54
53
|
return null;
|
|
55
54
|
}
|
|
56
|
-
if (!data?.endpoint || !data?.title
|
|
55
|
+
if (!data?.endpoint || !data?.title)
|
|
57
56
|
return null;
|
|
58
57
|
return {
|
|
59
58
|
file,
|
|
60
59
|
title: data.title,
|
|
61
60
|
endpoint: data.endpoint,
|
|
62
|
-
workspaceDir: data.workspaceDir,
|
|
61
|
+
workspaceDir: data.workspaceDir || '',
|
|
63
62
|
browserName: data.browser?.browserName || DEFAULT_BROWSER,
|
|
64
63
|
playwrightLib: data.playwrightLib || '',
|
|
65
64
|
};
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { outputPath } from "../../../src/config.js";
|
|
4
|
+
import { safeFilename } from "../../../src/utils/strings.js";
|
|
5
|
+
import { renderEnvelope } from "./envelope.js";
|
|
6
|
+
const EXPECTATION_STATUS = { passed: 'passed', failed: 'failed', unverified: 'skipped' };
|
|
7
|
+
export function sessionsDir() {
|
|
8
|
+
return outputPath('prima', 'sessions');
|
|
9
|
+
}
|
|
10
|
+
export function sessionFile(key) {
|
|
11
|
+
return path.join(sessionsDir(), `${safeFilename(key)}.jsonl`);
|
|
12
|
+
}
|
|
13
|
+
export function latestSessionFile() {
|
|
14
|
+
const dir = sessionsDir();
|
|
15
|
+
if (!existsSync(dir))
|
|
16
|
+
return null;
|
|
17
|
+
const files = readdirSync(dir)
|
|
18
|
+
.filter((name) => name.endsWith('.jsonl'))
|
|
19
|
+
.map((name) => path.join(dir, name))
|
|
20
|
+
.sort((a, b) => statSync(b).mtimeMs - statSync(a).mtimeMs);
|
|
21
|
+
return files[0] || null;
|
|
22
|
+
}
|
|
23
|
+
export function recordCommand(file, session, envelope, durationMs) {
|
|
24
|
+
mkdirSync(path.dirname(file), { recursive: true });
|
|
25
|
+
if (recordedEndpoint(file) !== session.endpoint)
|
|
26
|
+
writeFileSync(file, jsonLine({ action: 'createRun', endpoint: session.endpoint, params: { title: session.title } }), 'utf-8');
|
|
27
|
+
appendFileSync(file, jsonLine({ action: 'addTest', testId: commandTest(envelope, durationMs) }), 'utf-8');
|
|
28
|
+
}
|
|
29
|
+
export function readSession(file) {
|
|
30
|
+
const entries = readEntries(file);
|
|
31
|
+
const run = entries.find((entry) => entry.action === 'createRun');
|
|
32
|
+
const tests = entries.filter((entry) => entry.action === 'addTest').map((entry) => entry.testId);
|
|
33
|
+
return { title: run?.params?.title || 'prima session', tests };
|
|
34
|
+
}
|
|
35
|
+
function commandTest(envelope, durationMs) {
|
|
36
|
+
let status = 'failed';
|
|
37
|
+
if (envelope.ok)
|
|
38
|
+
status = 'passed';
|
|
39
|
+
const test = {
|
|
40
|
+
rid: envelope.status,
|
|
41
|
+
title: envelope.command,
|
|
42
|
+
suite_title: 'Prima session',
|
|
43
|
+
status,
|
|
44
|
+
file: '<prima>',
|
|
45
|
+
description: [`page: ${envelope.page.url}`, envelope.stepFiles && `artifacts: ${envelope.stepFiles}`].filter(Boolean).join('\n'),
|
|
46
|
+
code: (envelope.used || []).join('\n'),
|
|
47
|
+
steps: commandSteps(envelope),
|
|
48
|
+
logs: renderEnvelope(envelope),
|
|
49
|
+
time: Math.round(durationMs),
|
|
50
|
+
};
|
|
51
|
+
if (envelope.failure)
|
|
52
|
+
test.error = { message: envelope.failure.error };
|
|
53
|
+
return test;
|
|
54
|
+
}
|
|
55
|
+
function commandSteps(envelope) {
|
|
56
|
+
const steps = [];
|
|
57
|
+
for (const step of envelope.steps || []) {
|
|
58
|
+
const entry = { category: 'user', title: step.label, status: 'passed', duration: 0, log: step.proof };
|
|
59
|
+
if (!step.ok) {
|
|
60
|
+
entry.status = 'failed';
|
|
61
|
+
entry.error = step.proof;
|
|
62
|
+
}
|
|
63
|
+
steps.push(entry);
|
|
64
|
+
}
|
|
65
|
+
for (const expectation of envelope.expectations || []) {
|
|
66
|
+
const entry = { category: 'user', title: `expected: ${expectation.text}`, status: EXPECTATION_STATUS[expectation.status], duration: 0 };
|
|
67
|
+
if (expectation.status === 'failed')
|
|
68
|
+
entry.error = 'the run did not reach this outcome';
|
|
69
|
+
steps.push(entry);
|
|
70
|
+
}
|
|
71
|
+
for (const assertion of envelope.assertions || []) {
|
|
72
|
+
const entry = { category: 'user', title: assertion.code, status: 'passed', duration: 0, log: assertion.proof.join('\n') };
|
|
73
|
+
if (!assertion.passed) {
|
|
74
|
+
entry.status = 'failed';
|
|
75
|
+
entry.error = 'the assertion did not hold on the page';
|
|
76
|
+
}
|
|
77
|
+
steps.push(entry);
|
|
78
|
+
}
|
|
79
|
+
return steps;
|
|
80
|
+
}
|
|
81
|
+
function recordedEndpoint(file) {
|
|
82
|
+
const [first] = readEntries(file);
|
|
83
|
+
return first?.endpoint || null;
|
|
84
|
+
}
|
|
85
|
+
function readEntries(file) {
|
|
86
|
+
if (!existsSync(file))
|
|
87
|
+
return [];
|
|
88
|
+
const entries = [];
|
|
89
|
+
for (const line of readFileSync(file, 'utf-8').split('\n')) {
|
|
90
|
+
if (!line.trim())
|
|
91
|
+
continue;
|
|
92
|
+
const entry = parseLine(line);
|
|
93
|
+
if (entry)
|
|
94
|
+
entries.push(entry);
|
|
95
|
+
}
|
|
96
|
+
return entries;
|
|
97
|
+
}
|
|
98
|
+
function parseLine(line) {
|
|
99
|
+
try {
|
|
100
|
+
return JSON.parse(line);
|
|
101
|
+
}
|
|
102
|
+
catch {
|
|
103
|
+
return null;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
function jsonLine(entry) {
|
|
107
|
+
return `${JSON.stringify(entry)}\n`;
|
|
108
|
+
}
|
package/dist/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "explorbot",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.4",
|
|
4
4
|
"description": "CLI app built with React Ink, CodeceptJS, and Playwright",
|
|
5
5
|
"license": "Elastic-2.0",
|
|
6
6
|
"type": "module",
|
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
},
|
|
20
20
|
"files": [
|
|
21
21
|
"dist/",
|
|
22
|
+
"docs/",
|
|
22
23
|
"src/**/*.ts",
|
|
23
24
|
"src/**/*.tsx",
|
|
24
25
|
"bin/**/*.ts",
|
|
@@ -119,7 +120,7 @@
|
|
|
119
120
|
"ora-classic": "^5.4.2",
|
|
120
121
|
"parse5": "^8.0.0",
|
|
121
122
|
"pixelmatch": "^7.2.0",
|
|
122
|
-
"playwright": "^1.
|
|
123
|
+
"playwright": "^1.62",
|
|
123
124
|
"pngjs": "^7.0.0",
|
|
124
125
|
"react": "^19.1.1",
|
|
125
126
|
"sambanova-ai-provider": "^1.2.2",
|
|
@@ -105,9 +105,29 @@ Checks that page source does NOT contain expected text.
|
|
|
105
105
|
I.dontSeeInSource('error-class');
|
|
106
106
|
</example>
|
|
107
107
|
|
|
108
|
+
### Asserting the state of a control
|
|
109
|
+
|
|
110
|
+
State means disabled, checked, readonly, required, selected, expanded. A presence assertion says
|
|
111
|
+
nothing about it, so assert it with an attribute selector:
|
|
112
|
+
|
|
113
|
+
<example>
|
|
114
|
+
I.seeElement('button[aria-label="Submit"][disabled]');
|
|
115
|
+
I.dontSeeElement('button[aria-label="Submit"][disabled]');
|
|
116
|
+
I.seeElement('input[name="accept"][checked]');
|
|
117
|
+
I.seeElement('button[aria-label="Details"][aria-expanded="true"]');
|
|
118
|
+
</example>
|
|
119
|
+
|
|
120
|
+
Use I.seeElement for the state you expect and I.dontSeeElement for the state you expect to be
|
|
121
|
+
absent — that pair expresses both directions.
|
|
122
|
+
|
|
123
|
+
I.seeAttributesOnElements(<locator>, { disabled: true }) also exists, but only takes a state that
|
|
124
|
+
must be PRESENT and does not resolve reliably against a role/text locator. Prefer the selector form above.
|
|
125
|
+
|
|
108
126
|
<verification_rules>
|
|
109
127
|
Be strict in assertions to avoid false positives.
|
|
110
128
|
Prefer I.seeElement() with ARIA locators - most reliable.
|
|
129
|
+
For a claim about a control's state, assert it with an attribute selector — presence of the element is not evidence of its state.
|
|
130
|
+
If no assertion above can express the claim, say so instead of proposing an assertion that checks something weaker.
|
|
111
131
|
I.see() and I.dontSee() MUST include context parameter.
|
|
112
132
|
For input field values, ALWAYS use I.seeInField() — never check value via CSS attribute selectors or I.seeInSource.
|
|
113
133
|
Prefer text locators (label, name, placeholder) for form fields: I.seeInField('Search', 'value') over I.seeInField('input[name="search"]', 'value').
|
|
@@ -24,6 +24,7 @@ interface ActionResultData extends WebPageState {
|
|
|
24
24
|
}>;
|
|
25
25
|
ariaSnapshot?: string | null;
|
|
26
26
|
ariaSnapshotFile?: string;
|
|
27
|
+
focusedElement?: FocusedElement | null;
|
|
27
28
|
iframeURL?: string;
|
|
28
29
|
links?: Link[];
|
|
29
30
|
}
|
|
@@ -69,6 +70,7 @@ export declare class ActionResult implements ActionResultData {
|
|
|
69
70
|
snapshotCache: TTLCache<string>;
|
|
70
71
|
readonly logFile: string | undefined;
|
|
71
72
|
readonly ariaSnapshotFile: string | undefined;
|
|
73
|
+
readonly focusedElement: FocusedElement | null;
|
|
72
74
|
_ariaSnapshot: string | null | undefined;
|
|
73
75
|
_lastExtractedHtml: string | undefined;
|
|
74
76
|
notes: string[];
|
|
@@ -128,4 +130,9 @@ export declare class Diff {
|
|
|
128
130
|
get htmlDiff(): HtmlDiffResult | null;
|
|
129
131
|
calculate(): Promise<void>;
|
|
130
132
|
}
|
|
133
|
+
export interface FocusedElement {
|
|
134
|
+
role: string;
|
|
135
|
+
name: string;
|
|
136
|
+
value?: string;
|
|
137
|
+
}
|
|
131
138
|
export {};
|
|
@@ -30,6 +30,7 @@ export class ActionResult {
|
|
|
30
30
|
snapshotCache = new TTLCache();
|
|
31
31
|
logFile = undefined;
|
|
32
32
|
ariaSnapshotFile = undefined;
|
|
33
|
+
focusedElement = null;
|
|
33
34
|
_ariaSnapshot = undefined;
|
|
34
35
|
_lastExtractedHtml = undefined;
|
|
35
36
|
notes = [];
|
|
@@ -61,6 +62,9 @@ export class ActionResult {
|
|
|
61
62
|
if (data.ariaSnapshotFile !== undefined) {
|
|
62
63
|
this.ariaSnapshotFile = data.ariaSnapshotFile;
|
|
63
64
|
}
|
|
65
|
+
if (data.focusedElement !== undefined) {
|
|
66
|
+
this.focusedElement = data.focusedElement;
|
|
67
|
+
}
|
|
64
68
|
// Store HTML in a private property if provided
|
|
65
69
|
if (data.html !== undefined) {
|
|
66
70
|
this._html = data.html;
|
package/dist/src/action.d.ts
CHANGED
|
@@ -15,6 +15,7 @@ declare class Action {
|
|
|
15
15
|
name: string;
|
|
16
16
|
args: any[];
|
|
17
17
|
}>;
|
|
18
|
+
lastValue: unknown;
|
|
18
19
|
recorder?: PlaywrightRecorder;
|
|
19
20
|
recovery: RecoveryRunner;
|
|
20
21
|
mainDocumentStatus: number | undefined;
|
|
@@ -45,6 +46,7 @@ declare class Action {
|
|
|
45
46
|
exitIframe(): Promise<void>;
|
|
46
47
|
getActor(): CodeceptJS.I;
|
|
47
48
|
getActionResult(): ActionResult | null;
|
|
49
|
+
restorePageTimeout(): void;
|
|
48
50
|
waitForPageReadiness(page: any): Promise<void>;
|
|
49
51
|
}
|
|
50
52
|
export default Action;
|
package/dist/src/action.js
CHANGED
|
@@ -15,6 +15,8 @@ import { safeFilename } from "./utils/strings.js";
|
|
|
15
15
|
import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from "./utils/web-sandbox.js";
|
|
16
16
|
const debugLog = createDebug('explorbot:action');
|
|
17
17
|
const CAPTURE_NAVIGATION_TRANSITION_ATTEMPTS = 3;
|
|
18
|
+
const DEFAULT_ACTION_TIMEOUT = 3000;
|
|
19
|
+
const DEFAULT_PAGE_TIMEOUT = 3000;
|
|
18
20
|
class Action {
|
|
19
21
|
actor;
|
|
20
22
|
stateManager;
|
|
@@ -26,6 +28,7 @@ class Action {
|
|
|
26
28
|
playwrightHelper;
|
|
27
29
|
playwrightGroupId = null;
|
|
28
30
|
assertionSteps = [];
|
|
31
|
+
lastValue;
|
|
29
32
|
recorder;
|
|
30
33
|
recovery;
|
|
31
34
|
mainDocumentStatus = undefined;
|
|
@@ -120,9 +123,11 @@ class Action {
|
|
|
120
123
|
const iframeSnapshots = await this.captureIframeSnapshots(html);
|
|
121
124
|
let ariaSnapshot = null;
|
|
122
125
|
let ariaSnapshotFile = undefined;
|
|
126
|
+
let focusedElement = null;
|
|
123
127
|
try {
|
|
124
128
|
const page = this.playwrightHelper.page;
|
|
125
129
|
ariaSnapshot = await page.locator('body').ariaSnapshot();
|
|
130
|
+
focusedElement = await page.evaluate(readFocusedElement);
|
|
126
131
|
}
|
|
127
132
|
catch (err) {
|
|
128
133
|
debugLog('ARIA snapshot failed:', err instanceof Error ? `${err.message}\n${err.stack}` : err);
|
|
@@ -145,6 +150,7 @@ class Action {
|
|
|
145
150
|
iframeSnapshots,
|
|
146
151
|
ariaSnapshot,
|
|
147
152
|
ariaSnapshotFile,
|
|
153
|
+
focusedElement,
|
|
148
154
|
iframeURL: frame ? frame.url?.() || 'iframe' : undefined,
|
|
149
155
|
});
|
|
150
156
|
this.stateManager.updateState(result, codeBlock);
|
|
@@ -272,16 +278,19 @@ class Action {
|
|
|
272
278
|
if (!sanitizedCode) {
|
|
273
279
|
throw new Error('No valid I.* or page.* commands found in code block');
|
|
274
280
|
}
|
|
281
|
+
this.playwrightHelper?.page?.setDefaultTimeout(this.config.action?.timeout ?? DEFAULT_ACTION_TIMEOUT);
|
|
275
282
|
if (isPlaywright) {
|
|
276
283
|
const page = this.playwrightHelper.page;
|
|
277
284
|
await playwrightSandbox(page, sanitizedCode);
|
|
278
285
|
await sleep(this.config.action?.delay || 500);
|
|
279
286
|
}
|
|
280
287
|
else {
|
|
281
|
-
codeceptJSSandbox(this.actor, sanitizedCode);
|
|
288
|
+
const returned = codeceptJSSandbox(this.actor, sanitizedCode);
|
|
282
289
|
await recorder.add(() => sleep(this.config.action?.delay || 500));
|
|
283
290
|
await recorder.promise();
|
|
291
|
+
this.lastValue = await returned;
|
|
284
292
|
}
|
|
293
|
+
this.restorePageTimeout();
|
|
285
294
|
if (executedSteps.length > 0) {
|
|
286
295
|
codeString = executedSteps.join('\n');
|
|
287
296
|
}
|
|
@@ -300,6 +309,7 @@ class Action {
|
|
|
300
309
|
throw err;
|
|
301
310
|
}
|
|
302
311
|
finally {
|
|
312
|
+
this.restorePageTimeout();
|
|
303
313
|
detachMainDocumentResponse();
|
|
304
314
|
if (groupId)
|
|
305
315
|
await this.recorder.endAction();
|
|
@@ -346,6 +356,9 @@ class Action {
|
|
|
346
356
|
getActionResult() {
|
|
347
357
|
return this.actionResult;
|
|
348
358
|
}
|
|
359
|
+
restorePageTimeout() {
|
|
360
|
+
this.playwrightHelper?.page?.setDefaultTimeout(this.config.playwright.timeout ?? DEFAULT_PAGE_TIMEOUT);
|
|
361
|
+
}
|
|
349
362
|
async waitForPageReadiness(page) {
|
|
350
363
|
await waitForPageReadiness(page, {
|
|
351
364
|
timeout: this.config.playwright.waitForTimeout,
|
|
@@ -408,3 +421,25 @@ const detachStepLogger = (listener) => {
|
|
|
408
421
|
codeceptjs.event.dispatcher.off(codeceptjs.event.step.passed, listener);
|
|
409
422
|
codeceptjs.event.dispatcher.off(codeceptjs.event.step.failed, listener);
|
|
410
423
|
};
|
|
424
|
+
const readFocusedElement = () => {
|
|
425
|
+
const el = document.activeElement;
|
|
426
|
+
if (!el || el === document.body)
|
|
427
|
+
return null;
|
|
428
|
+
const tag = el.tagName.toLowerCase();
|
|
429
|
+
const textish = new Set(['text', 'search', 'email', 'password', 'url', 'tel', 'number']);
|
|
430
|
+
let role = el.getAttribute('role') || tag;
|
|
431
|
+
if (tag === 'textarea' || el.isContentEditable)
|
|
432
|
+
role = 'textbox';
|
|
433
|
+
if (tag === 'input' && textish.has(el.type))
|
|
434
|
+
role = 'textbox';
|
|
435
|
+
if (tag === 'select')
|
|
436
|
+
role = 'combobox';
|
|
437
|
+
if (tag === 'a')
|
|
438
|
+
role = 'link';
|
|
439
|
+
const label = el.getAttribute('aria-label') || el.getAttribute('placeholder') || el.labels?.[0]?.textContent || el.textContent || '';
|
|
440
|
+
const focused = { role, name: label.trim().slice(0, 80) };
|
|
441
|
+
const value = el.value ?? el.textContent;
|
|
442
|
+
if (typeof value === 'string' && value)
|
|
443
|
+
focused.value = value.slice(0, 200);
|
|
444
|
+
return focused;
|
|
445
|
+
};
|
|
@@ -16,7 +16,7 @@ export function WithWebMode(Base) {
|
|
|
16
16
|
navigator: ctx.explorBot.agentNavigator(),
|
|
17
17
|
});
|
|
18
18
|
const { see, context, visualClick, learnExperience } = agentTools;
|
|
19
|
-
|
|
19
|
+
const tools = {
|
|
20
20
|
navigate: tool({
|
|
21
21
|
description: 'Navigate to a URL or page description using AI-powered navigation.',
|
|
22
22
|
inputSchema: z.object({
|
|
@@ -114,11 +114,14 @@ export function WithWebMode(Base) {
|
|
|
114
114
|
},
|
|
115
115
|
}),
|
|
116
116
|
...codeceptTools,
|
|
117
|
-
see,
|
|
118
117
|
context,
|
|
119
|
-
visualClick,
|
|
120
118
|
learnExperience,
|
|
121
119
|
};
|
|
120
|
+
if (see)
|
|
121
|
+
tools.see = see;
|
|
122
|
+
if (visualClick)
|
|
123
|
+
tools.visualClick = visualClick;
|
|
124
|
+
return tools;
|
|
122
125
|
}
|
|
123
126
|
webModePrompt() {
|
|
124
127
|
return dedent `
|
package/dist/src/ai/captain.js
CHANGED
|
@@ -15,6 +15,7 @@ import { WithWebMode } from "./captain/web-mode.js";
|
|
|
15
15
|
import { toolExecutionLabel } from './conversation.js';
|
|
16
16
|
import { Researcher } from "./researcher.js";
|
|
17
17
|
import { TaskAgent } from "./task-agent.js";
|
|
18
|
+
import { withdrawVisionTools } from "./tools.js";
|
|
18
19
|
const MAX_STEPS = 15;
|
|
19
20
|
const CaptainBase = WithTestMode(WithWebMode(WithIdleMode(TaskAgent)));
|
|
20
21
|
export class Captain extends CaptainBase {
|
|
@@ -389,6 +390,7 @@ export class Captain extends CaptainBase {
|
|
|
389
390
|
stop();
|
|
390
391
|
return;
|
|
391
392
|
}
|
|
393
|
+
withdrawVisionTools(tools);
|
|
392
394
|
const currentState = stateManager.getCurrentState();
|
|
393
395
|
if (!currentState && this.getMode() !== 'idle') {
|
|
394
396
|
stop();
|
|
@@ -50,6 +50,8 @@ declare class Navigator implements Agent {
|
|
|
50
50
|
} | null>;
|
|
51
51
|
verifyState(message: string, actionResult: ActionResult): Promise<{
|
|
52
52
|
verified: boolean;
|
|
53
|
+
inexpressible: boolean;
|
|
54
|
+
results: AssertionResult[];
|
|
53
55
|
successfulCodes: string[];
|
|
54
56
|
assertionSteps: Array<{
|
|
55
57
|
name: string;
|
|
@@ -59,4 +61,9 @@ declare class Navigator implements Agent {
|
|
|
59
61
|
}>;
|
|
60
62
|
checkAlreadyVerified(aiResponse: string, actionResult: ActionResult): boolean;
|
|
61
63
|
}
|
|
64
|
+
export type AssertionResult = {
|
|
65
|
+
code: string;
|
|
66
|
+
passed: boolean;
|
|
67
|
+
proof: string[];
|
|
68
|
+
};
|
|
62
69
|
export { Navigator };
|
package/dist/src/ai/navigator.js
CHANGED
|
@@ -3,6 +3,7 @@ import dedent from 'dedent';
|
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
import { ActionResult } from '../action-result.js';
|
|
5
5
|
import { normalizeUrl } from '../state-manager.js';
|
|
6
|
+
import { renderAssertion } from "../playwright-recorder.js";
|
|
6
7
|
import { extractCodeBlocks } from '../utils/code-extractor.js';
|
|
7
8
|
import { HooksRunner } from "../utils/hooks-runner.js";
|
|
8
9
|
import { createDebug, pluralize, tag } from '../utils/logger.js';
|
|
@@ -628,6 +629,7 @@ class Navigator {
|
|
|
628
629
|
const tools = this.buildExperienceTools();
|
|
629
630
|
let codeBlocks = [];
|
|
630
631
|
const successfulCodes = [];
|
|
632
|
+
const results = [];
|
|
631
633
|
const assertionSteps = [];
|
|
632
634
|
const action = this.explorer.action();
|
|
633
635
|
let failures = 0;
|
|
@@ -660,6 +662,8 @@ class Navigator {
|
|
|
660
662
|
}
|
|
661
663
|
await action.exitIframe();
|
|
662
664
|
const verified = await action.attempt(codeBlock, message);
|
|
665
|
+
const proof = action.assertionSteps.map(renderAssertion).filter(Boolean);
|
|
666
|
+
results.push({ code: codeBlock, passed: verified, proof });
|
|
663
667
|
if (verified) {
|
|
664
668
|
tag('success').log('Verification passed');
|
|
665
669
|
successfulCodes.push(codeBlock);
|
|
@@ -668,11 +672,6 @@ class Navigator {
|
|
|
668
672
|
else {
|
|
669
673
|
failures++;
|
|
670
674
|
}
|
|
671
|
-
const target = Math.min(codeBlocks.length, this.verifyAttempts);
|
|
672
|
-
const majorityNeeded = Math.floor(target / 2) + 1;
|
|
673
|
-
if (successfulCodes.length >= majorityNeeded || failures > target - majorityNeeded) {
|
|
674
|
-
stop();
|
|
675
|
-
}
|
|
676
675
|
}, {
|
|
677
676
|
maxAttempts: this.verifyAttempts,
|
|
678
677
|
observability: {
|
|
@@ -691,9 +690,14 @@ class Navigator {
|
|
|
691
690
|
let verified = successfulCodes.length >= majorityNeeded;
|
|
692
691
|
if (alreadyVerified)
|
|
693
692
|
verified = true;
|
|
693
|
+
const inexpressible = !alreadyVerified && totalAttempted === 0;
|
|
694
|
+
if (inexpressible) {
|
|
695
|
+
tag('warning').log('No assertion could express this claim');
|
|
696
|
+
return { verified: false, inexpressible, results, successfulCodes, assertionSteps, totalAttempted };
|
|
697
|
+
}
|
|
694
698
|
actionResult.addVerification(message, verified);
|
|
695
699
|
this.stateManager.updateState(actionResult);
|
|
696
|
-
return { verified, successfulCodes, assertionSteps, totalAttempted };
|
|
700
|
+
return { verified, inexpressible, results, successfulCodes, assertionSteps, totalAttempted };
|
|
697
701
|
}
|
|
698
702
|
checkAlreadyVerified(aiResponse, actionResult) {
|
|
699
703
|
const verifiedMatch = aiResponse.match(/ALREADY_VERIFIED:\s*(.+)/i);
|
package/dist/src/ai/pilot.d.ts
CHANGED
|
@@ -40,6 +40,10 @@ export declare class Pilot implements Agent {
|
|
|
40
40
|
planTest(task: Test, currentState: ActionResult): Promise<string>;
|
|
41
41
|
reviewNewPage(task: Test, currentState: ActionResult, testerConversation: Conversation): Promise<string>;
|
|
42
42
|
analyzeProgress(task: Test, currentState: ActionResult, testerConversation: Conversation): Promise<string>;
|
|
43
|
+
settleExpectations(task: Test): Promise<Array<{
|
|
44
|
+
text: string;
|
|
45
|
+
status: 'passed' | 'failed' | 'unverified';
|
|
46
|
+
}>>;
|
|
43
47
|
formatExpectations(task: Test): string;
|
|
44
48
|
sendToPilot(userText: string, functionId: string, opts: {
|
|
45
49
|
tools?: boolean;
|
package/dist/src/ai/pilot.js
CHANGED
|
@@ -4,13 +4,14 @@ import { z } from 'zod';
|
|
|
4
4
|
import { ActionResult } from "../action-result.js";
|
|
5
5
|
import { ConfigParser } from "../config.js";
|
|
6
6
|
import { TestResult } from "../test-plan.js";
|
|
7
|
-
import { collectInteractiveNodes, detectFocusArea
|
|
7
|
+
import { collectInteractiveNodes, detectFocusArea } from "../utils/aria.js";
|
|
8
8
|
import { ErrorPageError } from "../utils/error-page.js";
|
|
9
9
|
import { createDebug, tag } from "../utils/logger.js";
|
|
10
10
|
const debugLog = createDebug('explorbot:pilot');
|
|
11
11
|
import { truncateJson } from "../utils/strings.js";
|
|
12
12
|
import { capabilityGroundingRule, dataProtectionRules } from "./rules.js";
|
|
13
13
|
import { isInteractive } from "./task-agent.js";
|
|
14
|
+
import { withdrawVisionTools } from "./tools.js";
|
|
14
15
|
const CHECK_TOOLS = ['verify', 'see', 'research', 'context'];
|
|
15
16
|
const META_TOOLS = ['record', 'reset', 'stop', 'finish'];
|
|
16
17
|
export class Pilot {
|
|
@@ -484,6 +485,50 @@ export class Pilot {
|
|
|
484
485
|
}
|
|
485
486
|
return text;
|
|
486
487
|
}
|
|
488
|
+
async settleExpectations(task) {
|
|
489
|
+
const undecided = task.expected.filter((text) => !task.getCheckedExpectations().includes(text));
|
|
490
|
+
const decided = (text) => {
|
|
491
|
+
if (task.hasAchievedAny() && !task.getRemainingExpectations().includes(text))
|
|
492
|
+
return 'passed';
|
|
493
|
+
return 'failed';
|
|
494
|
+
};
|
|
495
|
+
if (!undecided.length)
|
|
496
|
+
return task.expected.map((text) => ({ text, status: decided(text) }));
|
|
497
|
+
const schema = z.object({
|
|
498
|
+
outcomes: z.array(z.object({
|
|
499
|
+
expectation: z.string().describe('The expected outcome, repeated exactly as it was given'),
|
|
500
|
+
status: z.enum(['passed', 'failed', 'unverified']).describe('passed = the log shows it happened, failed = the log shows it did not, unverified = the run never established either way'),
|
|
501
|
+
})),
|
|
502
|
+
});
|
|
503
|
+
const userContent = dedent `
|
|
504
|
+
A test run has finished. Decide, for each expected outcome, what the run established about it.
|
|
505
|
+
|
|
506
|
+
<expected_outcomes>
|
|
507
|
+
${undecided.map((text) => `- ${text}`).join('\n')}
|
|
508
|
+
</expected_outcomes>
|
|
509
|
+
|
|
510
|
+
<run_log>
|
|
511
|
+
${task.notesToString() || 'No steps recorded.'}
|
|
512
|
+
</run_log>
|
|
513
|
+
|
|
514
|
+
The log is written in the tester's own words, so an outcome can be satisfied by a step that describes it
|
|
515
|
+
differently. Judge by what the steps show happened, not by whether the wording matches.
|
|
516
|
+
Choose "unverified" only when the log neither shows the outcome happening nor shows it failing —
|
|
517
|
+
that is a statement about the run, not about the application.
|
|
518
|
+
`;
|
|
519
|
+
const response = await this.provider
|
|
520
|
+
.generateObject([{ role: 'user', content: userContent }], schema, this.provider.getAgenticModel('pilot'), {
|
|
521
|
+
agentName: 'pilot',
|
|
522
|
+
telemetry: { functionId: 'pilot.settleExpectations' },
|
|
523
|
+
})
|
|
524
|
+
.catch(() => null);
|
|
525
|
+
const judged = new Map((response?.object?.outcomes || []).map((outcome) => [outcome.expectation, outcome.status]));
|
|
526
|
+
return task.expected.map((text) => {
|
|
527
|
+
if (!undecided.includes(text))
|
|
528
|
+
return { text, status: decided(text) };
|
|
529
|
+
return { text, status: judged.get(text) || 'unverified' };
|
|
530
|
+
});
|
|
531
|
+
}
|
|
487
532
|
formatExpectations(task) {
|
|
488
533
|
const checked = task.getCheckedExpectations();
|
|
489
534
|
const remaining = task.getRemainingExpectations();
|
|
@@ -547,6 +592,7 @@ export class Pilot {
|
|
|
547
592
|
planning.learnExperience = learnExperience;
|
|
548
593
|
if (askUser)
|
|
549
594
|
planning.askUser = askUser;
|
|
595
|
+
withdrawVisionTools(planning);
|
|
550
596
|
return planning;
|
|
551
597
|
}
|
|
552
598
|
buildPreconditionTool(task) {
|
|
@@ -624,7 +670,7 @@ export class Pilot {
|
|
|
624
670
|
const lines = [];
|
|
625
671
|
lines.push(`url: ${state.url}`);
|
|
626
672
|
lines.push(`title: ${state.title || 'unknown'}`);
|
|
627
|
-
const focused =
|
|
673
|
+
const focused = state.focusedElement;
|
|
628
674
|
if (focused) {
|
|
629
675
|
const valuePart = focused.value ? ` (value: "${focused.value}")` : '';
|
|
630
676
|
lines.push(`focused: ${focused.role} "${focused.name}"${valuePart}`);
|
package/dist/src/ai/provider.js
CHANGED
|
@@ -290,6 +290,7 @@ export class Provider {
|
|
|
290
290
|
try {
|
|
291
291
|
const response = await withRetry(async () => {
|
|
292
292
|
const result = await generateText({ messages, ...config });
|
|
293
|
+
this.recordUsage(options.agentName || 'unknown', modelName, result.usage);
|
|
293
294
|
if (!result.text) {
|
|
294
295
|
debugLog(result);
|
|
295
296
|
if (result.finishReason === 'length') {
|
|
@@ -304,7 +305,6 @@ export class Provider {
|
|
|
304
305
|
}, this.getRetryOptions(options));
|
|
305
306
|
clearActivity();
|
|
306
307
|
responseLog(response.text);
|
|
307
|
-
this.recordUsage(options.agentName || 'unknown', modelName, response.usage);
|
|
308
308
|
return response;
|
|
309
309
|
}
|
|
310
310
|
catch (error) {
|
|
@@ -340,6 +340,7 @@ export class Provider {
|
|
|
340
340
|
try {
|
|
341
341
|
const response = await withRetry(async () => {
|
|
342
342
|
const result = (await this.raceWithIdleTimeout((signal) => generateText({ messages, ...config, abortSignal: signal }), config.timeout || 30000));
|
|
343
|
+
this.recordUsage(options.agentName || 'unknown', modelName, result.usage);
|
|
343
344
|
const hasToolCall = (result.toolCalls?.length || 0) > 0;
|
|
344
345
|
if (!result.text && !hasToolCall && result.finishReason === 'length') {
|
|
345
346
|
throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase maxOutputTokens in config or use a model with higher output capacity.');
|
|
@@ -355,7 +356,6 @@ export class Provider {
|
|
|
355
356
|
});
|
|
356
357
|
}
|
|
357
358
|
responseLog(response.text);
|
|
358
|
-
this.recordUsage(options.agentName || 'unknown', modelName, response.usage);
|
|
359
359
|
return response;
|
|
360
360
|
}
|
|
361
361
|
catch (error) {
|
package/dist/src/ai/rerunner.js
CHANGED
|
@@ -21,7 +21,7 @@ import { RulesLoader } from "../utils/rules-loader.js";
|
|
|
21
21
|
import { toolExecutionLabel } from "./conversation.js";
|
|
22
22
|
import { actionRule, locatorRule, sectionContextRule } from "./rules.js";
|
|
23
23
|
import { TaskAgent } from "./task-agent.js";
|
|
24
|
-
import { createCodeceptJSTools } from "./tools.js";
|
|
24
|
+
import { createCodeceptJSTools, withdrawVisionTools } from "./tools.js";
|
|
25
25
|
const debugLog = createDebug('explorbot:rerunner');
|
|
26
26
|
export class Rerunner extends TaskAgent {
|
|
27
27
|
ACTION_TOOLS = ['click', 'pressKey', 'form'];
|
|
@@ -322,6 +322,7 @@ export class Rerunner extends TaskAgent {
|
|
|
322
322
|
},
|
|
323
323
|
}),
|
|
324
324
|
};
|
|
325
|
+
withdrawVisionTools(tools);
|
|
325
326
|
const conversation = this.provider.startConversation(this.getHealSystemPrompt(), 'rerunner');
|
|
326
327
|
conversation.addUserText(this.getHealUserPrompt(failedCode, error));
|
|
327
328
|
await loop(async ({ stop }) => {
|
|
@@ -76,7 +76,7 @@ export class Researcher extends ResearcherBase {
|
|
|
76
76
|
const cached = getCachedResearch(stateHash);
|
|
77
77
|
if (cached) {
|
|
78
78
|
debugLog('Previous research result found');
|
|
79
|
-
return
|
|
79
|
+
return cached;
|
|
80
80
|
}
|
|
81
81
|
}
|
|
82
82
|
Stats.researches++;
|