explorbot 0.5.3 → 0.5.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/boat/doc-collector/src/config.ts +1 -0
- package/boat/doc-collector/src/docbot.ts +10 -1
- package/boat/prima/README.md +5 -10
- package/boat/prima/package.json +1 -1
- package/boat/prima/src/cli.ts +8 -50
- package/boat/prima/src/envelope.ts +12 -6
- package/boat/prima/src/help.ts +4 -2
- package/boat/prima/src/prima.ts +58 -81
- package/boat/prima/src/session-log.ts +3 -3
- package/dist/boat/doc-collector/src/docbot.js +11 -1
- package/dist/boat/prima/src/cli.js +9 -45
- package/dist/boat/prima/src/envelope.js +12 -3
- package/dist/boat/prima/src/help.js +4 -2
- package/dist/boat/prima/src/prima.js +58 -77
- package/dist/boat/prima/src/session-log.js +3 -3
- package/dist/package.json +1 -1
- package/dist/rules/navigator/output.md +3 -3
- package/dist/src/action.js +4 -0
- package/dist/src/ai/driller.js +2 -28
- package/dist/src/ai/navigator.d.ts +5 -2
- package/dist/src/ai/navigator.js +23 -12
- package/dist/src/ai/pilot.d.ts +2 -7
- package/dist/src/ai/pilot.js +34 -7
- package/dist/src/ai/researcher.js +10 -8
- package/dist/src/ai/rules.d.ts +1 -0
- package/dist/src/ai/rules.js +7 -0
- package/dist/src/ai/tools.js +6 -1
- package/dist/src/explorbot.js +1 -1
- package/dist/src/test-plan.d.ts +11 -0
- package/dist/src/test-plan.js +21 -0
- package/dist/src/utils/code-extractor.d.ts +1 -0
- package/dist/src/utils/code-extractor.js +40 -0
- package/dist/src/utils/error-page.d.ts +1 -0
- package/dist/src/utils/error-page.js +8 -0
- package/dist/src/utils/html.d.ts +2 -0
- package/dist/src/utils/html.js +36 -1
- package/dist/src/utils/mdq/cli.js +3 -1
- package/dist/src/utils/mdq/query.d.ts +4 -1
- package/dist/src/utils/mdq/query.js +48 -7
- package/dist/src/utils/web-sandbox.js +4 -4
- package/docs/doc-collection/basics.md +13 -0
- package/docs/reference/commands.md +4 -6
- package/docs/web-testing/researcher.md +8 -0
- package/package.json +1 -1
- package/rules/navigator/output.md +3 -3
- package/src/action.ts +4 -0
- package/src/ai/driller.ts +2 -33
- package/src/ai/navigator.ts +23 -13
- package/src/ai/pilot.ts +35 -15
- package/src/ai/researcher.ts +10 -9
- package/src/ai/rules.ts +8 -0
- package/src/ai/tools.ts +5 -1
- package/src/explorbot.ts +1 -1
- package/src/test-plan.ts +23 -0
- package/src/utils/code-extractor.ts +41 -0
- package/src/utils/error-page.ts +8 -0
- package/src/utils/html.ts +33 -1
- package/src/utils/mdq/cli.ts +3 -2
- package/src/utils/mdq/query.ts +48 -7
- package/src/utils/web-sandbox.ts +4 -4
package/dist/src/ai/rules.js
CHANGED
|
@@ -138,6 +138,13 @@ export const formRequirementsRule = dedent `
|
|
|
138
138
|
Before filling a form that persists data (create/update), read each control's requirements (required, type/format, length, placeholder/aria-describedby hints) from <page_aria> and page HTML — call context() if not visible — and enter values that satisfy them. Search/filter/sort forms that only change the view do not need this.
|
|
139
139
|
</form_requirements>
|
|
140
140
|
`;
|
|
141
|
+
export const credentialsRule = dedent `
|
|
142
|
+
<credentials>
|
|
143
|
+
Sign in to an existing account only with credentials present in the provided context (message, knowledge, experience).
|
|
144
|
+
Never invent, guess, or try default, common, or placeholder credentials for an existing account.
|
|
145
|
+
If the required credentials are not provided, call stop() and name what is missing — do not submit the form.
|
|
146
|
+
</credentials>
|
|
147
|
+
`;
|
|
141
148
|
// in rage mode we do not protect from irreversible actions
|
|
142
149
|
export const protectionRule = dedent `
|
|
143
150
|
<important>
|
package/dist/src/ai/tools.js
CHANGED
|
@@ -91,7 +91,7 @@ export function createCodeceptJSTools({ explorer, stateManager, judge }, task) {
|
|
|
91
91
|
const attempts = [];
|
|
92
92
|
let ambiguityError = null;
|
|
93
93
|
for (let i = 0; i < commands.length; i++) {
|
|
94
|
-
|
|
94
|
+
let command = transformContainsCommand(commands[i]);
|
|
95
95
|
let success = await action.attempt(command, explanation);
|
|
96
96
|
const attempt = { command, success };
|
|
97
97
|
if (action.lastError)
|
|
@@ -104,9 +104,14 @@ export function createCodeceptJSTools({ explorer, stateManager, judge }, task) {
|
|
|
104
104
|
const index = await judge.pick(PICK_ELEMENT_QUESTION, labels, { intent: explanation, task: task.description });
|
|
105
105
|
if (index)
|
|
106
106
|
success = await action.attemptExactElementIndex(command, index, explanation);
|
|
107
|
+
if (success)
|
|
108
|
+
command = action.executedSteps.map((step) => step.command).join('\n') || command;
|
|
107
109
|
}
|
|
108
110
|
}
|
|
109
111
|
if (success) {
|
|
112
|
+
attempt.command = command;
|
|
113
|
+
attempt.success = true;
|
|
114
|
+
attempt.error = undefined;
|
|
110
115
|
const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, command);
|
|
111
116
|
if (!hasObservablePageChange(toolResult)) {
|
|
112
117
|
activeNote.commit(TestResult.FAILED);
|
package/dist/src/explorbot.js
CHANGED
|
@@ -125,7 +125,7 @@ export class ExplorBot {
|
|
|
125
125
|
await this.visit(this.options.from || '/');
|
|
126
126
|
}
|
|
127
127
|
async visit(url) {
|
|
128
|
-
|
|
128
|
+
await this.agentNavigator().visit(this.configParser.resolveTargetPath(url));
|
|
129
129
|
}
|
|
130
130
|
async openTab() {
|
|
131
131
|
await this.explorer.openTab();
|
package/dist/src/test-plan.d.ts
CHANGED
|
@@ -33,6 +33,17 @@ export declare class ActiveNote {
|
|
|
33
33
|
getMessage(): string;
|
|
34
34
|
getStartTime(): number;
|
|
35
35
|
}
|
|
36
|
+
export type SettledStatus = 'passed' | 'failed' | 'unverified' | 'contradiction';
|
|
37
|
+
export declare class SettledExpectation {
|
|
38
|
+
readonly text: string;
|
|
39
|
+
readonly status: SettledStatus;
|
|
40
|
+
readonly evidence?: string;
|
|
41
|
+
readonly resolution?: string;
|
|
42
|
+
constructor(text: string, status: SettledStatus, evidence?: string, resolution?: string);
|
|
43
|
+
get failed(): boolean;
|
|
44
|
+
get contradicted(): boolean;
|
|
45
|
+
resolve(status: SettledStatus, resolution: string): SettledExpectation;
|
|
46
|
+
}
|
|
36
47
|
export interface StepData {
|
|
37
48
|
text: string;
|
|
38
49
|
duration?: number;
|
package/dist/src/test-plan.js
CHANGED
|
@@ -41,6 +41,27 @@ export class ActiveNote {
|
|
|
41
41
|
return this.startTime;
|
|
42
42
|
}
|
|
43
43
|
}
|
|
44
|
+
export class SettledExpectation {
|
|
45
|
+
text;
|
|
46
|
+
status;
|
|
47
|
+
evidence;
|
|
48
|
+
resolution;
|
|
49
|
+
constructor(text, status, evidence, resolution) {
|
|
50
|
+
this.text = text;
|
|
51
|
+
this.status = status;
|
|
52
|
+
this.evidence = evidence;
|
|
53
|
+
this.resolution = resolution;
|
|
54
|
+
}
|
|
55
|
+
get failed() {
|
|
56
|
+
return this.status === 'failed';
|
|
57
|
+
}
|
|
58
|
+
get contradicted() {
|
|
59
|
+
return this.status === 'contradiction';
|
|
60
|
+
}
|
|
61
|
+
resolve(status, resolution) {
|
|
62
|
+
return new SettledExpectation(this.text, status, this.evidence, resolution);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
44
65
|
export class Task {
|
|
45
66
|
id;
|
|
46
67
|
description;
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import { createDebug } from './logger.js';
|
|
2
2
|
const debugLog = createDebug('explorbot:code-extractor');
|
|
3
3
|
const JS_LANGUAGES = new Set(['', 'js', 'javascript']);
|
|
4
|
+
const QUOTES = ["'", '"', '`'];
|
|
5
|
+
const OPENING_BRACKETS = ['(', '[', '{'];
|
|
6
|
+
const CLOSING_BRACKETS = [')', ']', '}'];
|
|
4
7
|
export function extractCodeBlocks(aiResponse) {
|
|
5
8
|
const codeBlockRegex = /```([^\n`]*)\n([\s\S]*?)\n```/g;
|
|
6
9
|
const codeBlocks = [];
|
|
@@ -22,3 +25,40 @@ export function extractCodeBlocks(aiResponse) {
|
|
|
22
25
|
}
|
|
23
26
|
return codeBlocks;
|
|
24
27
|
}
|
|
28
|
+
export function splitTopLevel(code, separator) {
|
|
29
|
+
const parts = [];
|
|
30
|
+
let current = '';
|
|
31
|
+
let depth = 0;
|
|
32
|
+
let quote = '';
|
|
33
|
+
for (let i = 0; i < code.length; i++) {
|
|
34
|
+
const char = code[i];
|
|
35
|
+
if (quote) {
|
|
36
|
+
current += char;
|
|
37
|
+
if (char === '\\')
|
|
38
|
+
current += code[++i] || '';
|
|
39
|
+
if (char === quote)
|
|
40
|
+
quote = '';
|
|
41
|
+
continue;
|
|
42
|
+
}
|
|
43
|
+
if (char === '/' && code[i + 1] === '/') {
|
|
44
|
+
while (i + 1 < code.length && code[i + 1] !== '\n')
|
|
45
|
+
i++;
|
|
46
|
+
continue;
|
|
47
|
+
}
|
|
48
|
+
if (char === separator && depth <= 0) {
|
|
49
|
+
parts.push(current);
|
|
50
|
+
current = '';
|
|
51
|
+
depth = 0;
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
current += char;
|
|
55
|
+
if (QUOTES.includes(char))
|
|
56
|
+
quote = char;
|
|
57
|
+
if (OPENING_BRACKETS.includes(char))
|
|
58
|
+
depth++;
|
|
59
|
+
if (CLOSING_BRACKETS.includes(char))
|
|
60
|
+
depth--;
|
|
61
|
+
}
|
|
62
|
+
parts.push(current);
|
|
63
|
+
return parts;
|
|
64
|
+
}
|
|
@@ -3,6 +3,7 @@ import type { WebPageState } from '../state-manager.js';
|
|
|
3
3
|
export type PageCondition = 'ok' | 'loading' | 'error';
|
|
4
4
|
export declare function detectPageCondition(actionResult: ActionResult): PageCondition;
|
|
5
5
|
export declare function isErrorPage(actionResult: ActionResult): boolean;
|
|
6
|
+
export declare function isLoginPage(actionResult: ActionResult): boolean;
|
|
6
7
|
export declare function getStateErrorPageError(state: WebPageState | null | undefined): ErrorPageError | null;
|
|
7
8
|
export declare class ErrorPageError extends Error {
|
|
8
9
|
readonly url: string;
|
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
import { ActionResult } from '../action-result.js';
|
|
2
2
|
import { isBodyEmpty } from './html.js';
|
|
3
|
+
import { extractStatePath } from './url-matcher.js';
|
|
3
4
|
const HTTP_ERRORS = ['400 Bad Request', '401 Unauthorized', '403 Forbidden', '404 Not Found', '405 Method Not Allowed', '408 Request Timeout', '500 Internal Server Error', '502 Bad Gateway', '503 Service Unavailable', '504 Gateway Timeout'];
|
|
4
5
|
const SMALL_PAGE_THRESHOLD = 500;
|
|
5
6
|
const LOADING_WORD = /\bloading\b/i;
|
|
7
|
+
const PASSWORD_INPUT = /<input\b[^>]*\btype\s*=\s*["']?password\b/i;
|
|
8
|
+
const AUTH_PATH = /\b(log[-_]?in|sign[-_]?in|auth)\b/i;
|
|
6
9
|
export function detectPageCondition(actionResult) {
|
|
7
10
|
if (actionResult.httpStatus && actionResult.httpStatus >= 400)
|
|
8
11
|
return 'error';
|
|
@@ -32,6 +35,11 @@ export function detectPageCondition(actionResult) {
|
|
|
32
35
|
export function isErrorPage(actionResult) {
|
|
33
36
|
return detectPageCondition(actionResult) === 'error';
|
|
34
37
|
}
|
|
38
|
+
export function isLoginPage(actionResult) {
|
|
39
|
+
if (PASSWORD_INPUT.test(actionResult.html || ''))
|
|
40
|
+
return true;
|
|
41
|
+
return AUTH_PATH.test(extractStatePath(actionResult.url || '').split('?')[0]);
|
|
42
|
+
}
|
|
35
43
|
export function getStateErrorPageError(state) {
|
|
36
44
|
if (!state)
|
|
37
45
|
return null;
|
package/dist/src/utils/html.d.ts
CHANGED
|
@@ -5,6 +5,7 @@ export declare const EXPLORBOT_ATTRS: {
|
|
|
5
5
|
readonly coveredBy: "data-explorbot-covered-by";
|
|
6
6
|
readonly context: "data-explorbot-context";
|
|
7
7
|
readonly eidx: "data-explorbot-eidx";
|
|
8
|
+
readonly hidden: "data-explorbot-hidden";
|
|
8
9
|
readonly hit: "data-explorbot-hit";
|
|
9
10
|
readonly variant: "data-explorbot-variant";
|
|
10
11
|
};
|
|
@@ -40,6 +41,7 @@ export declare const ELEMENT_EXTRACTION_CONFIG: {
|
|
|
40
41
|
readonly coveredBy: "data-explorbot-covered-by";
|
|
41
42
|
readonly context: "data-explorbot-context";
|
|
42
43
|
readonly eidx: "data-explorbot-eidx";
|
|
44
|
+
readonly hidden: "data-explorbot-hidden";
|
|
43
45
|
readonly hit: "data-explorbot-hit";
|
|
44
46
|
readonly variant: "data-explorbot-variant";
|
|
45
47
|
};
|
package/dist/src/utils/html.js
CHANGED
|
@@ -70,6 +70,7 @@ export const EXPLORBOT_ATTRS = {
|
|
|
70
70
|
coveredBy: 'data-explorbot-covered-by',
|
|
71
71
|
context: 'data-explorbot-context',
|
|
72
72
|
eidx: 'data-explorbot-eidx',
|
|
73
|
+
hidden: 'data-explorbot-hidden',
|
|
73
74
|
hit: 'data-explorbot-hit',
|
|
74
75
|
variant: 'data-explorbot-variant',
|
|
75
76
|
};
|
|
@@ -159,6 +160,20 @@ export function captureHtmlForSnapshot() {
|
|
|
159
160
|
.join(','));
|
|
160
161
|
}
|
|
161
162
|
}
|
|
163
|
+
const liveElements = Array.from(document.body?.querySelectorAll('*') || []);
|
|
164
|
+
const clonedElements = Array.from(clone.querySelector('body')?.querySelectorAll('*') || []);
|
|
165
|
+
for (let i = 0; i < liveElements.length; i++) {
|
|
166
|
+
const source = liveElements[i];
|
|
167
|
+
if (source.matches('input, textarea, select, option, optgroup'))
|
|
168
|
+
continue;
|
|
169
|
+
if (source.checkVisibility())
|
|
170
|
+
continue;
|
|
171
|
+
if (!source.parentElement?.checkVisibility())
|
|
172
|
+
continue;
|
|
173
|
+
if (getComputedStyle(source).display === 'contents')
|
|
174
|
+
continue;
|
|
175
|
+
clonedElements[i]?.setAttribute('data-explorbot-hidden', 'true');
|
|
176
|
+
}
|
|
162
177
|
return clone.outerHTML;
|
|
163
178
|
}
|
|
164
179
|
export function extractElementData(el, config) {
|
|
@@ -1347,7 +1362,27 @@ function getAttribute(element, name) {
|
|
|
1347
1362
|
const attr = element.attrs.find((a) => a.name === name);
|
|
1348
1363
|
return attr?.value;
|
|
1349
1364
|
}
|
|
1365
|
+
function getImgAlt(element) {
|
|
1366
|
+
let alt = '';
|
|
1367
|
+
function processNode(node) {
|
|
1368
|
+
if (alt)
|
|
1369
|
+
return;
|
|
1370
|
+
if ('tagName' in node) {
|
|
1371
|
+
const el = node;
|
|
1372
|
+
if (el.tagName.toLowerCase() === 'img') {
|
|
1373
|
+
alt = getAttribute(el, 'alt') || '';
|
|
1374
|
+
return;
|
|
1375
|
+
}
|
|
1376
|
+
}
|
|
1377
|
+
if ('childNodes' in node)
|
|
1378
|
+
node.childNodes.forEach(processNode);
|
|
1379
|
+
}
|
|
1380
|
+
processNode(element);
|
|
1381
|
+
return alt;
|
|
1382
|
+
}
|
|
1350
1383
|
function hasHiddenClass(element) {
|
|
1384
|
+
if (getAttribute(element, EXPLORBOT_ATTRS.hidden))
|
|
1385
|
+
return true;
|
|
1351
1386
|
const classAttr = element.attrs.find((attr) => attr.name === 'class');
|
|
1352
1387
|
if (!classAttr)
|
|
1353
1388
|
return false;
|
|
@@ -1393,7 +1428,7 @@ export function extractLinks(html) {
|
|
|
1393
1428
|
if (href) {
|
|
1394
1429
|
const shouldSkip = skipPrefixes.some((prefix) => href.startsWith(prefix));
|
|
1395
1430
|
if (!shouldSkip) {
|
|
1396
|
-
const rawTitle = getAttribute(element, 'aria-label') || getTextContent(element);
|
|
1431
|
+
const rawTitle = getAttribute(element, 'aria-label') || getTextContent(element) || getImgAlt(element);
|
|
1397
1432
|
const title = sanitizeLinkTitle(rawTitle);
|
|
1398
1433
|
if (title && title.length <= 100) {
|
|
1399
1434
|
const key = `${href}|${title}`;
|
|
@@ -5,7 +5,7 @@ const EDIT_FLAGS = ['remove', 'replace', 'insertBefore', 'insertAfter', 'prepend
|
|
|
5
5
|
export async function runMdq(argv, readStdin) {
|
|
6
6
|
const program = new Command();
|
|
7
7
|
program
|
|
8
|
-
.name('mdq')
|
|
8
|
+
.name('mdq-cli')
|
|
9
9
|
.description('query and edit markdown')
|
|
10
10
|
.argument('[selector]', 'markdown selector')
|
|
11
11
|
.argument('[file]', 'file to read; stdin when omitted')
|
|
@@ -29,6 +29,8 @@ export async function runMdq(argv, readStdin) {
|
|
|
29
29
|
program.parse(argv, { from: 'user' });
|
|
30
30
|
}
|
|
31
31
|
catch (error) {
|
|
32
|
+
if (error.code === 'commander.helpDisplayed')
|
|
33
|
+
return { output: program.helpInformation(), code: 0 };
|
|
32
34
|
return { output: String(error.message), code: 2 };
|
|
33
35
|
}
|
|
34
36
|
const options = program.opts();
|
|
@@ -57,6 +57,8 @@ export declare class MarkdownQuery {
|
|
|
57
57
|
sectionsOf(candidates: MatchedRange[], segment: QuerySegment): MatchedRange[];
|
|
58
58
|
itemsOf(candidates: MatchedRange[]): MatchedRange[];
|
|
59
59
|
expandSections(matches: MatchedRange[]): MatchedRange[];
|
|
60
|
+
evaluate(candidates: MatchedRange[], expression: QueryExpression): MatchedRange[];
|
|
61
|
+
union(results: MatchedRange[][]): MatchedRange[];
|
|
60
62
|
narrow(matches: MatchedRange[], segment: QuerySegment): MatchedRange[];
|
|
61
63
|
filterByText(matches: MatchedRange[], segment: QuerySegment): MatchedRange[];
|
|
62
64
|
matchText(text: string, matcher: TextMatcher): boolean;
|
|
@@ -77,7 +79,7 @@ export declare class MdqOperationError extends MdqError {
|
|
|
77
79
|
constructor(message: string);
|
|
78
80
|
}
|
|
79
81
|
export declare function mdq(source: Markdown): MarkdownQuery;
|
|
80
|
-
export declare function parseQuery(input: string):
|
|
82
|
+
export declare function parseQuery(input: string): QueryExpression;
|
|
81
83
|
export declare function buildTokenIndex(source: string): MatchedRange[];
|
|
82
84
|
export type Markdown = string | MarkdownQuery;
|
|
83
85
|
export type Matcher = string | RegExp | ((text: string) => boolean);
|
|
@@ -92,6 +94,7 @@ export interface TextMatcher {
|
|
|
92
94
|
flags?: string;
|
|
93
95
|
predicate?: (text: string) => boolean;
|
|
94
96
|
}
|
|
97
|
+
export type QueryExpression = QuerySegment[][][];
|
|
95
98
|
export interface QuerySegment {
|
|
96
99
|
selector: SelectorType;
|
|
97
100
|
textMatch?: TextMatcher;
|
|
@@ -9,10 +9,13 @@ export class MarkdownQuery {
|
|
|
9
9
|
this.matches = matches || buildTokenIndex(source);
|
|
10
10
|
}
|
|
11
11
|
query(selector, matcher) {
|
|
12
|
-
const
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
12
|
+
const expression = parseQuery(selector);
|
|
13
|
+
for (const chain of expression.flat()) {
|
|
14
|
+
const last = chain.at(-1);
|
|
15
|
+
if (matcher !== undefined && last)
|
|
16
|
+
last.textMatch = this.valueMatcher(matcher);
|
|
17
|
+
}
|
|
18
|
+
return new MarkdownQuery(this.source, this.evaluate(this.expandSections(this.matches), expression));
|
|
16
19
|
}
|
|
17
20
|
section(matcher, options) {
|
|
18
21
|
return this.query(`section${options?.depth || ''}`, matcher);
|
|
@@ -299,6 +302,27 @@ export class MarkdownQuery {
|
|
|
299
302
|
}
|
|
300
303
|
return expanded;
|
|
301
304
|
}
|
|
305
|
+
evaluate(candidates, expression) {
|
|
306
|
+
const results = [];
|
|
307
|
+
for (const operands of expression) {
|
|
308
|
+
const matched = operands.map((chain) => this.run(candidates, chain));
|
|
309
|
+
if (matched.some((ranges) => ranges.length === 0))
|
|
310
|
+
continue;
|
|
311
|
+
results.push(...matched);
|
|
312
|
+
}
|
|
313
|
+
return this.union(results);
|
|
314
|
+
}
|
|
315
|
+
union(results) {
|
|
316
|
+
if (results.length === 1)
|
|
317
|
+
return results[0];
|
|
318
|
+
const unique = new Map();
|
|
319
|
+
for (const range of results.flat()) {
|
|
320
|
+
const key = `${range.start}:${range.length}:${Boolean(range.innerTokens)}`;
|
|
321
|
+
if (!unique.has(key))
|
|
322
|
+
unique.set(key, range);
|
|
323
|
+
}
|
|
324
|
+
return [...unique.values()].sort((a, b) => a.start - b.start || b.length - a.length);
|
|
325
|
+
}
|
|
302
326
|
narrow(matches, segment) {
|
|
303
327
|
if (segment.index !== null) {
|
|
304
328
|
let index = segment.index;
|
|
@@ -390,10 +414,24 @@ export function mdq(source) {
|
|
|
390
414
|
return new MarkdownQuery(String(source));
|
|
391
415
|
}
|
|
392
416
|
export function parseQuery(input) {
|
|
393
|
-
const
|
|
417
|
+
const expression = [[[]]];
|
|
418
|
+
let chain = expression[0][0];
|
|
394
419
|
SEGMENT.lastIndex = 0;
|
|
395
420
|
while (SEGMENT.lastIndex < input.length) {
|
|
396
421
|
const at = SEGMENT.lastIndex;
|
|
422
|
+
OPERATOR.lastIndex = at;
|
|
423
|
+
const operator = OPERATOR.exec(input);
|
|
424
|
+
if (operator) {
|
|
425
|
+
if (chain.length === 0)
|
|
426
|
+
throw new MdqSelectorError(`Empty selector before ${operator[1]}`, at);
|
|
427
|
+
chain = [];
|
|
428
|
+
if (operator[1] === '||')
|
|
429
|
+
expression.push([chain]);
|
|
430
|
+
if (operator[1] === '&&')
|
|
431
|
+
expression[expression.length - 1].push(chain);
|
|
432
|
+
SEGMENT.lastIndex = OPERATOR.lastIndex;
|
|
433
|
+
continue;
|
|
434
|
+
}
|
|
397
435
|
const match = SEGMENT.exec(input);
|
|
398
436
|
if (!match)
|
|
399
437
|
throw new MdqSelectorError(`Unexpected character "${input[at]}" in selector`, at);
|
|
@@ -424,9 +462,11 @@ export function parseQuery(input) {
|
|
|
424
462
|
if (to)
|
|
425
463
|
segment.slice.to = Number.parseInt(to, 10);
|
|
426
464
|
}
|
|
427
|
-
|
|
465
|
+
chain.push(segment);
|
|
428
466
|
}
|
|
429
|
-
|
|
467
|
+
if (expression.flat().length > 1 && chain.length === 0)
|
|
468
|
+
throw new MdqSelectorError('Empty selector after operator', input.length);
|
|
469
|
+
return expression;
|
|
430
470
|
}
|
|
431
471
|
export function buildTokenIndex(source) {
|
|
432
472
|
const { body, offset } = MarkdownEditor.splitFrontmatter(source);
|
|
@@ -449,3 +489,4 @@ export function buildTokenIndex(source) {
|
|
|
449
489
|
const SELECTOR_NAMES = new Set(['section', 'heading', 'paragraph', 'table', 'list', 'item', 'code', 'blockquote', 'hr', 'html', 'comment']);
|
|
450
490
|
const TOKEN_ALIASES = { item: 'list_item' };
|
|
451
491
|
const SEGMENT = /(\s*\.?)([A-Za-z]\w*)(?:\((!?)(~?)(?:"((?:[^"\\]|\\.)*)"|\/((?:[^/\\]|\\.)*)\/([a-z]*))\))?((?:\[[^\]]*\])*)\s*/y;
|
|
492
|
+
const OPERATOR = /\s*(&&|\|\|)\s*/y;
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
import { faker } from '@faker-js/faker';
|
|
2
2
|
import { hopeThat, retryTo, tryTo, within } from 'codeceptjs/lib/effects';
|
|
3
3
|
import step from 'codeceptjs/steps';
|
|
4
|
+
import { splitTopLevel } from "./code-extractor.js";
|
|
4
5
|
const SHADOWED_GLOBALS = ['process', 'global', 'globalThis', 'fetch', 'Bun', 'require', 'module', 'exports'];
|
|
5
6
|
const ALLOWED_COMMAND_HEADS = ['I.', 'page.', 'await page.', 'await I.'];
|
|
6
7
|
const PLAYWRIGHT_COMMAND_HEADS = ['page.', 'await page.'];
|
|
7
8
|
const PLAYWRIGHT_ARG_NAMES = ['page'];
|
|
8
9
|
const CODECEPT_ARG_NAMES = ['I', 'tryTo', 'retryTo', 'within', 'hopeThat', 'step', 'faker'];
|
|
9
10
|
export function sanitizeCodeBlock(code) {
|
|
10
|
-
return code
|
|
11
|
-
.
|
|
12
|
-
.
|
|
13
|
-
.filter((line) => ALLOWED_COMMAND_HEADS.some((head) => line.startsWith(head)))
|
|
11
|
+
return splitTopLevel(code, '\n')
|
|
12
|
+
.map((statement) => statement.trim())
|
|
13
|
+
.filter((statement) => ALLOWED_COMMAND_HEADS.some((head) => statement.startsWith(head)))
|
|
14
14
|
.join('\n');
|
|
15
15
|
}
|
|
16
16
|
export function hasPlaywrightCommands(code) {
|
|
@@ -21,6 +21,19 @@ The collector starts at the path you give it. For each page it:
|
|
|
21
21
|
|
|
22
22
|
It repeats this until the queue is empty or the page budget is spent. Only same-origin links are followed.
|
|
23
23
|
|
|
24
|
+
## Research depth
|
|
25
|
+
|
|
26
|
+
In static mode the Researcher runs in a **light, single pass**: one AI call maps the sections and interactive elements, and the locator validation, repair conversations, and screenshot annotation passes of a full research run are skipped. Docs describe pages rather than click them, so unverified locators are a fair trade for roughly one AI call per page.
|
|
27
|
+
|
|
28
|
+
Interactive mode needs locators it can actually click, so it keeps the full pipeline: every locator is validated in the browser, broken ones get fixed, and a vision pass adds coordinates. Override the automatic choice with `research`:
|
|
29
|
+
|
|
30
|
+
```ts
|
|
31
|
+
docs: {
|
|
32
|
+
research: 'full', // always run the verified pipeline
|
|
33
|
+
// research: 'light', // single pass even in interactive mode
|
|
34
|
+
}
|
|
35
|
+
```
|
|
36
|
+
|
|
24
37
|
## Configure
|
|
25
38
|
|
|
26
39
|
The collector runs on your regular Explorbot setup — the AI provider and browser come from `explorbot.config.js` (see [configuration](../reference/configuration.md)). Crawl behavior lives in a separate file: `docbot.config.ts` (`.js` and `.mjs` work too).
|
|
@@ -733,16 +733,16 @@ html: /home/you/.explorbot/sites/app.example.com/output/prima/2026-08-04T10-04-2
|
|
|
733
733
|
|
|
734
734
|
### Browsers and sessions
|
|
735
735
|
|
|
736
|
-
Prima never launches a browser
|
|
736
|
+
Prima never launches a browser. Open a playwright-cli session first, and check prima sees it:
|
|
737
737
|
|
|
738
738
|
```bash
|
|
739
739
|
playwright-cli open https://app.example.com # the session prima attaches to by default
|
|
740
|
-
npx explorbot prima browser
|
|
740
|
+
npx explorbot prima browser # prints the session, or fails with the command to open one
|
|
741
741
|
```
|
|
742
742
|
|
|
743
|
-
By default prima attaches to the playwright-cli browser of the current workspace and works on the tabs it already has open — driving the same session from both tools is the intended usage. Stopping prima disconnects from an attached browser; it never closes it.
|
|
743
|
+
By default prima attaches to the playwright-cli browser of the current workspace and works on the tabs it already has open — driving the same session from both tools is the intended usage. Stopping prima disconnects from an attached browser; it never closes it. The `### Instance` block names the browser you are on.
|
|
744
744
|
|
|
745
|
-
Prima reaches every browser over a Playwright browser-server endpoint — a playwright-cli session, an `--endpoint`, or
|
|
745
|
+
Prima reaches every browser over a Playwright browser-server endpoint — a playwright-cli session, an `--endpoint`, or an `explorbot browser start` instance — and that client needs the Node build, which is what `npx explorbot prima` and the published `prima` bin run on. Driving a browser by running the CLI from source under Bun does not connect.
|
|
746
746
|
|
|
747
747
|
Every command takes these:
|
|
748
748
|
|
|
@@ -769,8 +769,6 @@ A few commands add their own:
|
|
|
769
769
|
| `research` | `--data` | Include data extraction in the map |
|
|
770
770
|
| `research` | `--deep` | Expand hidden elements for a deeper map |
|
|
771
771
|
| `research` | `--fresh` | Ignore the cached map and research the page again |
|
|
772
|
-
| `browser start` | `-s, --show` / `--headless` | Launch the browser with or without a window |
|
|
773
|
-
| `browser stop` | `--all` | Stop every running instance |
|
|
774
772
|
|
|
775
773
|
Prima carries no logging flags of its own. `DEBUG` in front of a command prints everything the run does — config and browser attachment, every step as it executes, and the debug stream of the agents behind it:
|
|
776
774
|
|
|
@@ -155,6 +155,14 @@ Expands hidden elements (dropdowns, accordions, tabs) to discover more UI. Click
|
|
|
155
155
|
|
|
156
156
|
Extracts domain-specific content (articles, products, users) as structured data.
|
|
157
157
|
|
|
158
|
+
#### Light Research
|
|
159
|
+
|
|
160
|
+
```ts
|
|
161
|
+
researcher.research(state, { light: true });
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
A single-pass mode for consumers that describe pages instead of acting on them. Stage 1 runs as usual; stages 2–5 (locator testing, AI fixing, the visual annotation pass, backfills, and pagination probing) are skipped, so the whole run is one AI call with no browser verification roundtrips. Locators in the UI map come straight from the model and are not verified; a light run never writes to the shared research cache, so it cannot replace a verified result. The [doc collector](../doc-collection/basics.md#research-depth) uses this mode by default.
|
|
165
|
+
|
|
158
166
|
## Page Sections
|
|
159
167
|
|
|
160
168
|
The Researcher breaks each page into sections by UI purpose. Sections are identified in priority order:
|
package/package.json
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
<rules>
|
|
2
|
-
Do not invent locators, focus only on locators from
|
|
2
|
+
Do not invent locators, focus only on locators from the <page> context.
|
|
3
3
|
Provide up to {{maxAttempts}} various code suggestions to achieve the result.
|
|
4
4
|
If there was already successful solution in <experience> use it as a first solution.
|
|
5
5
|
|
|
6
6
|
If no successful solution was found in <experience> propose codeblocks for each area that can help to achieve the result.
|
|
7
7
|
Do not stick only to the first found element as it might be hidden or not availble on the page.
|
|
8
|
-
If you think
|
|
8
|
+
If you think the page contains several areas that can help to achieve the result, propose codeblocks for each such area.
|
|
9
9
|
Use exact locators that can pick the elements from each areas.
|
|
10
10
|
Detect such duplicated areas by looking for duplicate IDs, data-ids, forms, etc.
|
|
11
11
|
|
|
@@ -24,7 +24,7 @@ Your response must start explanation of what you are going to do to achive the r
|
|
|
24
24
|
It is important to explain intention before proposing code.
|
|
25
25
|
Response must also valid CodeceptJS code in code blocks.
|
|
26
26
|
Propose codeblock from successful solutions in <experience> first if they exist.
|
|
27
|
-
Use only locators from
|
|
27
|
+
Use only locators from the <page> context.
|
|
28
28
|
</output>
|
|
29
29
|
|
|
30
30
|
|
package/src/action.ts
CHANGED
|
@@ -589,6 +589,10 @@ export const attachStepLogger = (target: ExecutedStep[], assertionsTarget?: Arra
|
|
|
589
589
|
if (batchFailed) return;
|
|
590
590
|
|
|
591
591
|
const executed: ExecutedStep = { command: step.toCode(), success: !error };
|
|
592
|
+
if (Object.keys(step.opts || {}).length) {
|
|
593
|
+
const args = [step.humanizeArgs(), `step.opts(${JSON.stringify(step.opts)})`].filter(Boolean).join(', ');
|
|
594
|
+
executed.command = `${step.prefix}${step.actor}.${step.title}(${args})${step.suffix}`;
|
|
595
|
+
}
|
|
592
596
|
if (error) {
|
|
593
597
|
executed.error = errorToString(error);
|
|
594
598
|
batchFailed = true;
|
package/src/ai/driller.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { setActivity } from '../activity.ts';
|
|
|
6
6
|
import { Observability } from '../observability.ts';
|
|
7
7
|
import { Plan, Test, TestResult } from '../test-plan.ts';
|
|
8
8
|
import { collectInteractiveNodes } from '../utils/aria.ts';
|
|
9
|
+
import { splitTopLevel } from '../utils/code-extractor.ts';
|
|
9
10
|
import { EXPLORBOT_ATTRS, HTML_COMPOSITE_AREA_HINTS, HTML_COMPOSITE_TARGET_ROLES, HTML_EXTRACTION_LIMITS, HTML_FORM_CONTROL_ROLES, HTML_FORM_CONTROL_TAGS, HTML_INTERACTIVE_ROLES, HTML_SELECTORS, getComponentScopeHtmlExtractorSource, inferHtmlRole } from '../utils/html.ts';
|
|
10
11
|
import { createDebug, tag } from '../utils/logger.ts';
|
|
11
12
|
import { loop, pause } from '../utils/loop.ts';
|
|
@@ -1011,39 +1012,7 @@ function countTopLevelArgCount(line: string): number {
|
|
|
1011
1012
|
const end = line.lastIndexOf(')');
|
|
1012
1013
|
if (start === -1 || end === -1 || end <= start + 1) return 0;
|
|
1013
1014
|
|
|
1014
|
-
|
|
1015
|
-
let count = 1;
|
|
1016
|
-
let depth = 0;
|
|
1017
|
-
let quote = '';
|
|
1018
|
-
|
|
1019
|
-
for (let i = 0; i < body.length; i++) {
|
|
1020
|
-
const char = body[i];
|
|
1021
|
-
const escaped = body[i - 1] === '\\';
|
|
1022
|
-
|
|
1023
|
-
if (quote) {
|
|
1024
|
-
if (char === quote && !escaped) quote = '';
|
|
1025
|
-
continue;
|
|
1026
|
-
}
|
|
1027
|
-
|
|
1028
|
-
if (char === '"' || char === "'" || char === '`') {
|
|
1029
|
-
quote = char;
|
|
1030
|
-
continue;
|
|
1031
|
-
}
|
|
1032
|
-
|
|
1033
|
-
if (char === '(' || char === '[' || char === '{') {
|
|
1034
|
-
depth++;
|
|
1035
|
-
continue;
|
|
1036
|
-
}
|
|
1037
|
-
|
|
1038
|
-
if (char === ')' || char === ']' || char === '}') {
|
|
1039
|
-
depth = Math.max(0, depth - 1);
|
|
1040
|
-
continue;
|
|
1041
|
-
}
|
|
1042
|
-
|
|
1043
|
-
if (char === ',' && depth === 0) count++;
|
|
1044
|
-
}
|
|
1045
|
-
|
|
1046
|
-
return count;
|
|
1015
|
+
return splitTopLevel(line.slice(start + 1, end), ',').length;
|
|
1047
1016
|
}
|
|
1048
1017
|
|
|
1049
1018
|
function buildClassSelector(tag: string, classes: string[]): string {
|