explorbot 0.4.1 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/explorbot-cli.ts +14 -4
- package/boat/api-tester/src/ai/curler-tools.ts +1 -1
- package/boat/api-tester/src/cli.ts +3 -0
- package/boat/doc-collector/src/ai/documentarian.ts +57 -5
- package/boat/doc-collector/src/ai/tools.ts +1 -1
- package/boat/doc-collector/src/cli.ts +18 -2
- package/boat/doc-collector/src/config.ts +3 -0
- package/boat/doc-collector/src/docbot.ts +56 -11
- package/boat/doc-collector/src/docs-renderer.ts +79 -55
- package/boat/doc-collector/src/interaction-screenshots.ts +19 -10
- package/boat/doc-collector/src/screenshots.ts +106 -3
- package/boat/doc-collector/src/template-dedup.ts +31 -0
- package/boat/prima/src/cli.ts +3 -0
- package/boat/prima/src/prima.ts +3 -4
- package/dist/bin/explorbot-cli.js +14 -4
- package/dist/boat/api-tester/src/ai/curler-tools.js +1 -1
- package/dist/boat/api-tester/src/cli.js +2 -0
- package/dist/boat/doc-collector/src/ai/documentarian.js +62 -4
- package/dist/boat/doc-collector/src/ai/tools.js +1 -1
- package/dist/boat/doc-collector/src/cli.js +15 -2
- package/dist/boat/doc-collector/src/config.js +1 -0
- package/dist/boat/doc-collector/src/docbot.js +52 -10
- package/dist/boat/doc-collector/src/docs-renderer.js +69 -46
- package/dist/boat/doc-collector/src/interaction-screenshots.js +18 -11
- package/dist/boat/doc-collector/src/screenshots.js +104 -1
- package/dist/boat/doc-collector/src/template-dedup.js +27 -0
- package/dist/boat/prima/src/cli.js +2 -0
- package/dist/boat/prima/src/prima.js +3 -4
- package/dist/package.json +1 -1
- package/dist/src/action-result.d.ts +2 -2
- package/dist/src/action-result.js +15 -17
- package/dist/src/action.d.ts +1 -1
- package/dist/src/action.js +15 -15
- package/dist/src/ai/fisherman/request-haul.d.ts +12 -0
- package/dist/src/ai/fisherman/request-haul.js +31 -0
- package/dist/src/ai/fisherman/tools.d.ts +50 -0
- package/dist/src/ai/{fisherman-tools.js → fisherman/tools.js} +109 -40
- package/dist/src/ai/fisherman.d.ts +14 -4
- package/dist/src/ai/fisherman.js +96 -18
- package/dist/src/ai/pilot.d.ts +13 -1
- package/dist/src/ai/pilot.js +40 -22
- package/dist/src/ai/provider.js +8 -14
- package/dist/src/ai/researcher/coordinates.d.ts +2 -0
- package/dist/src/ai/researcher/coordinates.js +13 -1
- package/dist/src/ai/researcher/deep-analysis.js +9 -9
- package/dist/src/ai/researcher/fingerprint-worker.js +1 -11
- package/dist/src/ai/researcher.js +1 -1
- package/dist/src/ai/rules.js +10 -0
- package/dist/src/ai/tester.js +20 -20
- package/dist/src/ai/tools.d.ts +1 -1
- package/dist/src/ai/tools.js +18 -74
- package/dist/src/api/request-result.js +3 -1
- package/dist/src/api/request-store.d.ts +7 -6
- package/dist/src/api/request-store.js +65 -37
- package/dist/src/api/xhr-capture.d.ts +2 -0
- package/dist/src/api/xhr-capture.js +35 -10
- package/dist/src/commands/config-command.js +2 -9
- package/dist/src/commands/help-json-command.d.ts +31 -0
- package/dist/src/commands/help-json-command.js +58 -0
- package/dist/src/commands/index.js +2 -0
- package/dist/src/commands/init-command.js +3 -7
- package/dist/src/commands/plans-command.d.ts +4 -1
- package/dist/src/commands/plans-command.js +4 -4
- package/dist/src/commands/recommended-models-command.d.ts +9 -0
- package/dist/src/commands/recommended-models-command.js +85 -0
- package/dist/src/config.d.ts +2 -0
- package/dist/src/config.js +7 -2
- package/dist/src/experience-tracker.js +1 -1
- package/dist/src/explorbot.js +3 -3
- package/dist/src/global-config.d.ts +2 -0
- package/dist/src/global-config.js +6 -0
- package/dist/src/state-manager.d.ts +3 -3
- package/dist/src/state-manager.js +8 -8
- package/dist/src/test-plan.d.ts +1 -0
- package/dist/src/test-plan.js +13 -9
- package/dist/src/utils/aria.d.ts +1 -1
- package/dist/src/utils/aria.js +23 -2
- package/dist/src/utils/cli-section.d.ts +1 -0
- package/dist/src/utils/cli-section.js +12 -0
- package/dist/src/utils/html-diff.d.ts +1 -0
- package/dist/src/utils/html-diff.js +5 -5
- package/dist/src/utils/html.d.ts +2 -0
- package/dist/src/utils/html.js +5 -4
- package/dist/src/utils/overlay.d.ts +19 -55
- package/dist/src/utils/overlay.js +94 -148
- package/dist/src/utils/region.d.ts +41 -0
- package/dist/src/utils/region.js +99 -0
- package/dist/src/utils/similarity.d.ts +1 -0
- package/dist/src/utils/similarity.js +11 -0
- package/dist/src/utils/url-matcher.d.ts +1 -1
- package/dist/src/utils/url-matcher.js +4 -4
- package/docs/doc-collection/basics.md +11 -5
- package/docs/doc-collection/crawling.md +21 -1
- package/docs/reference/commands.md +4 -0
- package/docs/reference/configuration.md +4 -0
- package/docs/superpowers/plans/2026-09-03-fisherman-query-api.md +1361 -0
- package/docs/superpowers/plans/2026-09-03-model-resolves-click-ambiguity.md +509 -0
- package/docs/workflow/agentic-usage.md +12 -0
- package/docs/workflow/application-spec.md +4 -0
- package/package.json +1 -1
- package/src/action-result.ts +15 -16
- package/src/action.ts +16 -15
- package/src/ai/fisherman/request-haul.ts +36 -0
- package/src/ai/{fisherman-tools.ts → fisherman/tools.ts} +125 -46
- package/src/ai/fisherman.ts +110 -20
- package/src/ai/pilot.ts +38 -20
- package/src/ai/provider.ts +8 -12
- package/src/ai/researcher/coordinates.ts +13 -1
- package/src/ai/researcher/deep-analysis.ts +9 -9
- package/src/ai/researcher/fingerprint-worker.ts +1 -10
- package/src/ai/researcher.ts +1 -1
- package/src/ai/rules.ts +10 -0
- package/src/ai/tester.ts +18 -18
- package/src/ai/tools.ts +18 -86
- package/src/api/request-result.ts +2 -1
- package/src/api/request-store.ts +68 -42
- package/src/api/xhr-capture.ts +39 -11
- package/src/commands/config-command.ts +2 -7
- package/src/commands/help-json-command.ts +74 -0
- package/src/commands/index.ts +2 -0
- package/src/commands/init-command.ts +4 -11
- package/src/commands/plans-command.ts +5 -5
- package/src/commands/recommended-models-command.ts +92 -0
- package/src/config.ts +9 -2
- package/src/experience-tracker.ts +1 -1
- package/src/explorbot.ts +3 -3
- package/src/global-config.ts +8 -0
- package/src/state-manager.ts +10 -9
- package/src/test-plan.ts +12 -10
- package/src/utils/aria.ts +26 -3
- package/src/utils/cli-section.ts +12 -0
- package/src/utils/html-diff.ts +6 -6
- package/src/utils/html.ts +6 -4
- package/src/utils/overlay.ts +105 -192
- package/src/utils/region.ts +132 -0
- package/src/utils/similarity.ts +9 -0
- package/src/utils/url-matcher.ts +3 -3
- package/dist/src/ai/fisherman-tools.d.ts +0 -138
- package/dist/src/utils/request-map.d.ts +0 -7
- package/dist/src/utils/request-map.js +0 -16
- package/src/utils/request-map.ts +0 -19
package/dist/src/ai/provider.js
CHANGED
|
@@ -10,6 +10,7 @@ import { configuredModels, modelName as getModelName } from '../config.js';
|
|
|
10
10
|
import { executionController } from "../execution-controller.js";
|
|
11
11
|
import { Observability } from "../observability.js";
|
|
12
12
|
import { Stats } from "../stats.js";
|
|
13
|
+
import { getCliName } from '../utils/cli-name.js';
|
|
13
14
|
import { createDebug, tag } from '../utils/logger.js';
|
|
14
15
|
import { withRetry } from '../utils/retry.js';
|
|
15
16
|
import { RulesLoader } from "../utils/rules-loader.js";
|
|
@@ -43,14 +44,7 @@ export async function flushTelemetry() {
|
|
|
43
44
|
}
|
|
44
45
|
const CONTEXT_LENGTH_PATTERNS = ['reduce the length', 'context length', 'maximum context', 'token limit', 'too many tokens', 'max_tokens', 'context_length_exceeded', 'output truncated at maxtokens'];
|
|
45
46
|
function extractCachedTokens(usage) {
|
|
46
|
-
|
|
47
|
-
return 0;
|
|
48
|
-
const direct = usage.inputTokenDetails?.cacheReadTokens ?? usage.cachedInputTokens;
|
|
49
|
-
if (typeof direct === 'number')
|
|
50
|
-
return direct;
|
|
51
|
-
const raw = usage.raw;
|
|
52
|
-
const fromRaw = raw?.prompt_tokens_details?.cached_tokens ?? raw?.promptTokensDetails?.cachedTokens;
|
|
53
|
-
return typeof fromRaw === 'number' ? fromRaw : 0;
|
|
47
|
+
return usage?.inputTokenDetails?.cacheReadTokens ?? 0;
|
|
54
48
|
}
|
|
55
49
|
function abortAfterIdle(ms, cancel, controller) {
|
|
56
50
|
return new Promise((_, reject) => {
|
|
@@ -99,7 +93,7 @@ export class Provider {
|
|
|
99
93
|
modelCallWaiters = [];
|
|
100
94
|
constructor(config) {
|
|
101
95
|
if (!config?.model) {
|
|
102
|
-
throw new AiError(
|
|
96
|
+
throw new AiError(`AI model is not configured. Set ai.model in your config file, or pick one with ${getCliName()} recommended-models`);
|
|
103
97
|
}
|
|
104
98
|
this.config = config;
|
|
105
99
|
this.initLangfuse();
|
|
@@ -226,8 +220,8 @@ export class Provider {
|
|
|
226
220
|
if (!usage)
|
|
227
221
|
return;
|
|
228
222
|
Stats.recordTokens(agentName, modelName, {
|
|
229
|
-
input: usage.inputTokens ??
|
|
230
|
-
output: usage.outputTokens ??
|
|
223
|
+
input: usage.inputTokens ?? 0,
|
|
224
|
+
output: usage.outputTokens ?? 0,
|
|
231
225
|
total: usage.totalTokens ?? 0,
|
|
232
226
|
cached: extractCachedTokens(usage),
|
|
233
227
|
});
|
|
@@ -387,7 +381,7 @@ export class Provider {
|
|
|
387
381
|
let invalidRequestFeedbackAdded = false;
|
|
388
382
|
const executedStepMessages = [];
|
|
389
383
|
try {
|
|
390
|
-
|
|
384
|
+
let response = await this.withModelRequestSlot(() => withRetry(async () => {
|
|
391
385
|
const stepMessages = [];
|
|
392
386
|
const onStepEnd = (step) => {
|
|
393
387
|
stepMessages.push(...(step.response?.messages || []));
|
|
@@ -413,7 +407,7 @@ export class Provider {
|
|
|
413
407
|
return result;
|
|
414
408
|
}, this.getRetryOptions(options)));
|
|
415
409
|
clearActivity();
|
|
416
|
-
withExecutedSteps(response, executedStepMessages);
|
|
410
|
+
response = withExecutedSteps(response, executedStepMessages);
|
|
417
411
|
// Log tool usage summary
|
|
418
412
|
if (response.toolCalls && response.toolCalls.length > 0) {
|
|
419
413
|
responseLog(response.toolCalls);
|
|
@@ -585,7 +579,7 @@ export class Provider {
|
|
|
585
579
|
}
|
|
586
580
|
async processImage(prompt, image) {
|
|
587
581
|
if (!this.config.visionModel) {
|
|
588
|
-
throw new Error(
|
|
582
|
+
throw new Error(`Vision model not configured. Set ai.visionModel in your config; ${getCliName()} recommended-models lists the providers that serve one`);
|
|
589
583
|
}
|
|
590
584
|
setActivity(`🤖 Processing image with ${this.config.visionModel}`, 'ai');
|
|
591
585
|
const messages = [
|
|
@@ -8,6 +8,7 @@ export declare function visuallyAnnotateContainers(page: Page, containers: Array
|
|
|
8
8
|
css: string;
|
|
9
9
|
label: string;
|
|
10
10
|
}>): Promise<number>;
|
|
11
|
+
export declare function removeVisualAnnotations(page: Page): Promise<void>;
|
|
11
12
|
export declare function WithCoordinates<T extends Constructor>(Base: T): {
|
|
12
13
|
new (...args: any[]): {
|
|
13
14
|
explorer: Explorer;
|
|
@@ -20,6 +21,7 @@ export declare function WithCoordinates<T extends Constructor>(Base: T): {
|
|
|
20
21
|
label: string;
|
|
21
22
|
}>;
|
|
22
23
|
}): Promise<number>;
|
|
24
|
+
removeVisualAnnotations(): Promise<void>;
|
|
23
25
|
_analyzeScreenshotForVisualProps(): Promise<VisualAnalysisResult>;
|
|
24
26
|
mergeVisualData(result: ResearchResult, visualData: Map<string, {
|
|
25
27
|
coordinates: string | null;
|
|
@@ -62,6 +62,15 @@ export async function visuallyAnnotateContainers(page, containers) {
|
|
|
62
62
|
return count;
|
|
63
63
|
}, containers);
|
|
64
64
|
}
|
|
65
|
+
export async function removeVisualAnnotations(page) {
|
|
66
|
+
try {
|
|
67
|
+
await page.locator('[data-explorbot-annotation]').evaluateAll((elements) => {
|
|
68
|
+
for (const element of elements)
|
|
69
|
+
element.remove();
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
catch { }
|
|
73
|
+
}
|
|
65
74
|
export function WithCoordinates(Base) {
|
|
66
75
|
return class extends Base {
|
|
67
76
|
analyzeScreenshotForVisualProps() {
|
|
@@ -70,6 +79,9 @@ export function WithCoordinates(Base) {
|
|
|
70
79
|
async visuallyAnnotateElements(opts) {
|
|
71
80
|
return this.explorer.withPage((page) => visuallyAnnotateContainers(page, opts?.containers || []));
|
|
72
81
|
}
|
|
82
|
+
async removeVisualAnnotations() {
|
|
83
|
+
await this.explorer.withPage((page) => removeVisualAnnotations(page));
|
|
84
|
+
}
|
|
73
85
|
async _analyzeScreenshotForVisualProps() {
|
|
74
86
|
const elements = new Map();
|
|
75
87
|
const emptyResult = { elements, pagePurpose: null, primaryActions: null, focusedSection: null };
|
|
@@ -81,7 +93,7 @@ export function WithCoordinates(Base) {
|
|
|
81
93
|
tag('step').log('Analyzing annotated screenshot for visual properties');
|
|
82
94
|
const prompt = dedent `
|
|
83
95
|
This screenshot has two types of annotations:
|
|
84
|
-
- **Section containers**: dashed bordered boxes (no labels on them). A legend at the bottom-
|
|
96
|
+
- **Section containers**: dashed bordered boxes (no labels on them). A legend at the bottom-right maps dashed line colors to section names. Ignore containers for this task.
|
|
85
97
|
- **Interactive elements**: solid bordered boxes with eidx numbers in the top-right corner above the box. Adjacent elements use different colors.
|
|
86
98
|
|
|
87
99
|
For each interactive element (solid border, eidx number), report:
|
|
@@ -62,29 +62,29 @@ export function WithDeepAnalysis(Base) {
|
|
|
62
62
|
this._appendExtendedResearch(result, expandedSections, navigationLinks);
|
|
63
63
|
}
|
|
64
64
|
async researchOverlay(current, previous, pageStateHash) {
|
|
65
|
-
const
|
|
66
|
-
if (!
|
|
65
|
+
const region = current.overlay;
|
|
66
|
+
if (!region.isOpen || !region.name)
|
|
67
67
|
return null;
|
|
68
68
|
const cached = getCachedResearch(pageStateHash);
|
|
69
69
|
if (!cached)
|
|
70
70
|
return null;
|
|
71
|
-
const escaped =
|
|
71
|
+
const escaped = region.name.replace(/"/g, '\\"');
|
|
72
72
|
if (mdq(cached).query(`section3(~"${escaped}")`).count() > 0) {
|
|
73
|
-
debugLog(`Overlay "${
|
|
73
|
+
debugLog(`Overlay "${region.name}" already in cached research, skipping`);
|
|
74
74
|
return null;
|
|
75
75
|
}
|
|
76
76
|
const diff = await current.diff(previous);
|
|
77
77
|
if (!diff.ariaChanged && diff.htmlParts.length === 0) {
|
|
78
|
-
debugLog(`No diff between current and previous state for overlay "${
|
|
78
|
+
debugLog(`No diff between current and previous state for overlay "${region.name}"`);
|
|
79
79
|
return null;
|
|
80
80
|
}
|
|
81
81
|
const alreadyExpanded = this._summarizeExpanded(parseResearchSections(cached)
|
|
82
82
|
.filter((s) => s.elements.length > 0)
|
|
83
83
|
.map((s) => s.rawMarkdown));
|
|
84
|
-
tag('substep').log(`Researching overlay: ${
|
|
85
|
-
const sectionMarkdown = await this._analyzeExpandedAction('',
|
|
84
|
+
tag('substep').log(`Researching overlay: ${region.name}`);
|
|
85
|
+
const sectionMarkdown = await this._analyzeExpandedAction('', region.name, diff, alreadyExpanded);
|
|
86
86
|
if (!sectionMarkdown) {
|
|
87
|
-
debugLog(`Overlay "${
|
|
87
|
+
debugLog(`Overlay "${region.name}" produced no meaningful expansion`);
|
|
88
88
|
return null;
|
|
89
89
|
}
|
|
90
90
|
const extQuery = mdq(cached).query('section1(~"Extended Research")');
|
|
@@ -97,7 +97,7 @@ export function WithDeepAnalysis(Base) {
|
|
|
97
97
|
updated = `${cached.trimEnd()}\n\n# Extended Research\n\n${sectionMarkdown}\n`;
|
|
98
98
|
}
|
|
99
99
|
saveResearch({ hash: pageStateHash }, updated);
|
|
100
|
-
tag('substep').log(`Overlay research appended: ${
|
|
100
|
+
tag('substep').log(`Overlay research appended: ${region.name}`);
|
|
101
101
|
return sectionMarkdown;
|
|
102
102
|
}
|
|
103
103
|
_loadPreviousExtendedSections(hash) {
|
|
@@ -3,17 +3,7 @@ import { join } from 'node:path';
|
|
|
3
3
|
import { parentPort } from 'node:worker_threads';
|
|
4
4
|
import { computeHtmlFingerprint } from "../../utils/html-diff.js";
|
|
5
5
|
import { isSamePageFamily } from "../../utils/url-matcher.js";
|
|
6
|
-
|
|
7
|
-
let intersection = 0;
|
|
8
|
-
for (const item of a) {
|
|
9
|
-
if (b.has(item))
|
|
10
|
-
intersection++;
|
|
11
|
-
}
|
|
12
|
-
const total = a.size + b.size;
|
|
13
|
-
if (total === 0)
|
|
14
|
-
return 100;
|
|
15
|
-
return Math.round(((2 * intersection) / total) * 100);
|
|
16
|
-
}
|
|
6
|
+
import { diceSimilarity } from "../../utils/similarity.js";
|
|
17
7
|
parentPort.on('message', (data) => {
|
|
18
8
|
const { html, statesDir, maxAgeMs, threshold, url } = data;
|
|
19
9
|
if (!existsSync(statesDir)) {
|
|
@@ -193,7 +193,7 @@ export class Researcher extends ResearcherBase {
|
|
|
193
193
|
const containers = validContainers.filter((c) => !freshBroken.includes(c.css));
|
|
194
194
|
await this.visuallyAnnotateElements({ containers });
|
|
195
195
|
this.actionResult = await this.explorer.capture({ screenshot: true });
|
|
196
|
-
const visualResult = await this.analyzeScreenshotForVisualProps();
|
|
196
|
+
const visualResult = await this.analyzeScreenshotForVisualProps().finally(() => this.removeVisualAnnotations());
|
|
197
197
|
if (visualResult.elements.size > 0) {
|
|
198
198
|
await this.mergeVisualData(result, visualResult.elements);
|
|
199
199
|
result.parseLocators();
|
package/dist/src/ai/rules.js
CHANGED
|
@@ -163,6 +163,8 @@ export const dataProtectionRules = dedent `
|
|
|
163
163
|
Do not use Fisherman or API data preparation to bypass a no-mutation, read-only, search,
|
|
164
164
|
filter, tab, or list-inspection constraint. Use visible existing data when it is available.
|
|
165
165
|
If no suitable data exists, report the missing precondition instead of creating data.
|
|
166
|
+
Reading through the API to establish what already exists is not a mutation and stays allowed
|
|
167
|
+
under a read-only constraint.
|
|
166
168
|
|
|
167
169
|
Destructive actions are allowed only against data created by the current scenario
|
|
168
170
|
or prepared for that scenario by Fisherman/API preconditions. Existing application data must
|
|
@@ -318,6 +320,14 @@ export const actionRule = dedent `
|
|
|
318
320
|
If locator doesn't work, try CSS or XPath locators.
|
|
319
321
|
If nothing works, use I.clickXY(x, y) as last resort.
|
|
320
322
|
|
|
323
|
+
When a click result reports several matches, pick one from its numbered list by position rather than guessing a new locator.
|
|
324
|
+
Reuse the same locator with step.opts({ elementIndex: N }) as the LAST argument. N is the "Element N" number.
|
|
325
|
+
|
|
326
|
+
<example>
|
|
327
|
+
I.click('Remove', step.opts({ elementIndex: 2 }));
|
|
328
|
+
I.click({ role: 'link', text: 'Details' }, '.panel', step.opts({ elementIndex: 1 }));
|
|
329
|
+
</example>
|
|
330
|
+
|
|
321
331
|
For checkboxes, prefer I.checkOption/I.uncheckOption over I.click.
|
|
322
332
|
|
|
323
333
|
|
package/dist/src/ai/tester.js
CHANGED
|
@@ -486,14 +486,14 @@ export class Tester extends TaskAgent {
|
|
|
486
486
|
const currentStateHash = currentState.hash;
|
|
487
487
|
const isNewUrl = this.previousUrl !== currentUrl;
|
|
488
488
|
const isNewState = !isNewUrl && this.previousStateHash !== null && this.previousStateHash !== currentStateHash;
|
|
489
|
-
if (this.previousRegionPresent !== null && this.previousRegionPresent !== currentState.overlay.
|
|
489
|
+
if (this.previousRegionPresent !== null && this.previousRegionPresent !== currentState.overlay.isOpen) {
|
|
490
490
|
this.regionTransitioned = true;
|
|
491
491
|
}
|
|
492
|
-
this.previousRegionPresent = currentState.overlay.
|
|
492
|
+
this.previousRegionPresent = currentState.overlay.isOpen;
|
|
493
493
|
this.previousUrl = currentUrl;
|
|
494
494
|
this.previousStateHash = currentStateHash;
|
|
495
495
|
let context = '';
|
|
496
|
-
const
|
|
496
|
+
const region = currentState.overlay;
|
|
497
497
|
const focusedElement = currentState.focusedElement;
|
|
498
498
|
if (focusedElement) {
|
|
499
499
|
const isTextInput = ['textbox', 'combobox', 'searchbox'].includes(focusedElement.role);
|
|
@@ -511,30 +511,30 @@ export class Tester extends TaskAgent {
|
|
|
511
511
|
</no_focus>
|
|
512
512
|
`;
|
|
513
513
|
}
|
|
514
|
-
if (
|
|
515
|
-
const areaName =
|
|
514
|
+
if (region.isModal) {
|
|
515
|
+
const areaName = region.name ? ` "${region.name}"` : '';
|
|
516
516
|
let rootHint = '';
|
|
517
|
-
if (
|
|
518
|
-
rootHint = `\nIts content lives inside \`${
|
|
517
|
+
if (region.root)
|
|
518
|
+
rootHint = `\nIts content lives inside \`${region.root}\` — scope locators to it.`;
|
|
519
519
|
context += dedent `
|
|
520
|
-
<
|
|
521
|
-
|
|
522
|
-
Scope all interactions to elements inside this
|
|
523
|
-
Page navigation, filters, and tabs that exist outside it are not actionable while it is open and may share names or roles with elements inside it — prefer the locator inside the
|
|
524
|
-
Use <page_aria> to confirm the element you target is actually inside the
|
|
525
|
-
</
|
|
520
|
+
<overlay>
|
|
521
|
+
An overlay${areaName} is currently open above the page.${rootHint}
|
|
522
|
+
Scope all interactions to elements inside this overlay.
|
|
523
|
+
Page navigation, filters, and tabs that exist outside it are not actionable while it is open and may share names or roles with elements inside it — prefer the locator inside the overlay.
|
|
524
|
+
Use <page_aria> to confirm the element you target is actually inside the overlay.
|
|
525
|
+
</overlay>
|
|
526
526
|
`;
|
|
527
527
|
}
|
|
528
|
-
if (!
|
|
528
|
+
if (!region.isModal && region.isOpen && isNewState) {
|
|
529
529
|
let rootHint = '';
|
|
530
|
-
if (
|
|
531
|
-
rootHint = `\nIt lives inside \`${
|
|
530
|
+
if (region.root)
|
|
531
|
+
rootHint = `\nIt lives inside \`${region.root}\`.`;
|
|
532
532
|
context += dedent `
|
|
533
|
-
<
|
|
534
|
-
A large new
|
|
533
|
+
<region>
|
|
534
|
+
A large new region "${region.name || 'unnamed region'}" appeared on this page without navigation.${rootHint}
|
|
535
535
|
The scenario most likely continues inside this area — prefer its elements for your next actions.
|
|
536
536
|
The rest of the page (navigation, menus, filters) is still interactive and remains available.
|
|
537
|
-
</
|
|
537
|
+
</region>
|
|
538
538
|
`;
|
|
539
539
|
}
|
|
540
540
|
if (currentState.isInsideIframe) {
|
|
@@ -594,7 +594,7 @@ export class Tester extends TaskAgent {
|
|
|
594
594
|
`;
|
|
595
595
|
return context;
|
|
596
596
|
}
|
|
597
|
-
if (
|
|
597
|
+
if (region.isOpen && region.name && this.pageStateHash && this.pageActionResult) {
|
|
598
598
|
const overlaySection = await this.researcher.researchOverlay(currentState, this.pageActionResult, this.pageStateHash).catch(this.skipResearch);
|
|
599
599
|
if (overlaySection) {
|
|
600
600
|
context += dedent `
|
package/dist/src/ai/tools.d.ts
CHANGED
|
@@ -12,7 +12,7 @@ interface AgentToolDeps extends ToolDeps {
|
|
|
12
12
|
withExperience?: boolean;
|
|
13
13
|
}
|
|
14
14
|
export declare const ASSERTION_TOOLS: readonly ["verify"];
|
|
15
|
-
export declare function createCodeceptJSTools({ explorer, stateManager
|
|
15
|
+
export declare function createCodeceptJSTools({ explorer, stateManager }: ToolDeps, task: Task): {
|
|
16
16
|
click: import("@ai-sdk/provider-utils").ExecutableTool<import("ai").Tool<{
|
|
17
17
|
commands: any;
|
|
18
18
|
explanation: any;
|
package/dist/src/ai/tools.js
CHANGED
|
@@ -16,7 +16,7 @@ import { sectionContextRule } from "./rules.js";
|
|
|
16
16
|
import { isInteractive } from "./task-agent.js";
|
|
17
17
|
const debugLog = createDebug('explorbot:tools');
|
|
18
18
|
export const ASSERTION_TOOLS = ['verify'];
|
|
19
|
-
export function createCodeceptJSTools({ explorer, stateManager
|
|
19
|
+
export function createCodeceptJSTools({ explorer, stateManager }, task) {
|
|
20
20
|
return {
|
|
21
21
|
click: tool({
|
|
22
22
|
description: dedent `
|
|
@@ -29,6 +29,12 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }, task) {
|
|
|
29
29
|
Container narrows search area. Use when page has multiple matching elements.
|
|
30
30
|
Example: Page has 3 "Delete" buttons in different rows:
|
|
31
31
|
I.click("Delete", ".row-1") - clicks Delete inside element with class row-1
|
|
32
|
+
I.click(locator, step.opts({ elementIndex: N })) - click the Nth element the locator matches
|
|
33
|
+
Use after a result reported several matches. N is the "Element N" number from that list.
|
|
34
|
+
step.opts() always goes LAST, after the container when there is one.
|
|
35
|
+
Example: the result listed 3 "Delete" buttons and you want the second:
|
|
36
|
+
I.click("Delete", step.opts({ elementIndex: 2 }))
|
|
37
|
+
I.click("Delete", ".table", step.opts({ elementIndex: 2 })) - same, narrowed to a container
|
|
32
38
|
|
|
33
39
|
IMPORTANT: This tool ONLY accepts click commands. For typing text, use form() tool.
|
|
34
40
|
CRITICAL: All commands MUST target the SAME element using different locators.
|
|
@@ -47,6 +53,7 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }, task) {
|
|
|
47
53
|
3. I.click(CSS, container) - e.g. I.click("#btn", ".modal")
|
|
48
54
|
4. I.click(CSS) or I.click(XPath) - when locator already includes context (ID, XPath)
|
|
49
55
|
5. I.clickXY(x, y) - coordinates fallback
|
|
56
|
+
After a result reporting multiple matches, reuse that locator with step.opts({ elementIndex: N }) as the last argument.
|
|
50
57
|
`),
|
|
51
58
|
explanation: z.string().describe('Why you are clicking this element'),
|
|
52
59
|
}),
|
|
@@ -72,6 +79,7 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }, task) {
|
|
|
72
79
|
const previousState = ActionResult.fromState(stateManager.getCurrentState());
|
|
73
80
|
const action = explorer.action();
|
|
74
81
|
const attempts = [];
|
|
82
|
+
let ambiguityError = null;
|
|
75
83
|
for (let i = 0; i < commands.length; i++) {
|
|
76
84
|
const command = transformContainsCommand(commands[i]);
|
|
77
85
|
const success = await action.attempt(command, explanation);
|
|
@@ -79,34 +87,14 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }, task) {
|
|
|
79
87
|
if (action.lastError)
|
|
80
88
|
attempt.error = errorText(action.lastError);
|
|
81
89
|
attempts.push(attempt);
|
|
90
|
+
if (!ambiguityError && action.lastError?.name === 'MultipleElementsFound')
|
|
91
|
+
ambiguityError = action.lastError;
|
|
82
92
|
if (success) {
|
|
83
93
|
const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, command);
|
|
84
94
|
await commitNote(activeNote, TestResult.PASSED, toolResult, action);
|
|
85
95
|
return successToolResult('click', { ...toolResult, attempts, code: command }, action);
|
|
86
96
|
}
|
|
87
97
|
}
|
|
88
|
-
let disambiguated = null;
|
|
89
|
-
if (attempts.some((a) => a.error?.toLowerCase().includes(MULTIPLE_ELEMENTS_PATTERN))) {
|
|
90
|
-
disambiguated = await disambiguateElements(action.lastError, explanation, ai);
|
|
91
|
-
}
|
|
92
|
-
if (disambiguated) {
|
|
93
|
-
debugLog('Disambiguation picked element %d', disambiguated.position);
|
|
94
|
-
const failedCommand = attempts.find((a) => a.error?.toLowerCase().includes(MULTIPLE_ELEMENTS_PATTERN))?.command;
|
|
95
|
-
const retryCommands = [];
|
|
96
|
-
if (failedCommand) {
|
|
97
|
-
retryCommands.push(failedCommand.replace(/\)$/, `, step.opts({ elementIndex: ${disambiguated.position} }))`));
|
|
98
|
-
}
|
|
99
|
-
retryCommands.push(`I.click('${disambiguated.xpath.replace(/'/g, "\\'")}')`);
|
|
100
|
-
for (const retryCmd of retryCommands) {
|
|
101
|
-
if (!(await action.attempt(retryCmd, explanation))) {
|
|
102
|
-
attempts.push({ command: retryCmd, success: false, error: errorText(action.lastError) });
|
|
103
|
-
continue;
|
|
104
|
-
}
|
|
105
|
-
const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, retryCmd);
|
|
106
|
-
await commitNote(activeNote, TestResult.PASSED, toolResult, action);
|
|
107
|
-
return successToolResult('click', { ...toolResult, attempts, code: retryCmd, disambiguated: true }, action);
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
98
|
const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, commands[0]);
|
|
111
99
|
await commitNote(activeNote, TestResult.FAILED, toolResult, action);
|
|
112
100
|
const suggestion = clickFailureSuggestion(attempts);
|
|
@@ -114,7 +102,7 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }, task) {
|
|
|
114
102
|
...toolResult,
|
|
115
103
|
attempts,
|
|
116
104
|
suggestion,
|
|
117
|
-
}, action.lastError);
|
|
105
|
+
}, ambiguityError || action.lastError);
|
|
118
106
|
},
|
|
119
107
|
}),
|
|
120
108
|
hover: tool({
|
|
@@ -344,13 +332,7 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }, task) {
|
|
|
344
332
|
if (action.lastError) {
|
|
345
333
|
const message = errorText(action.lastError);
|
|
346
334
|
await commitNote(activeNote, TestResult.FAILED, toolResult, action);
|
|
347
|
-
|
|
348
|
-
if (message.toLowerCase().includes(MULTIPLE_ELEMENTS_PATTERN)) {
|
|
349
|
-
const disambiguated = await disambiguateElements(action.lastError, explanation, ai);
|
|
350
|
-
if (disambiguated) {
|
|
351
|
-
formSuggestion = `Multiple elements matched. Add step.opts({ elementIndex: ${disambiguated.position} }) to the failing command. Fallback locator: ${disambiguated.xpath}`;
|
|
352
|
-
}
|
|
353
|
-
}
|
|
335
|
+
const formSuggestion = 'Commands after the failing one never ran. Retry only those, using click() or form().';
|
|
354
336
|
return failedToolResult('form', `Form execution FAILED! ${message}\n${formatExecutedSteps(action.executedSteps, codeLines.length)}`, {
|
|
355
337
|
...toolResult,
|
|
356
338
|
code: codeBlock,
|
|
@@ -1123,13 +1105,11 @@ export async function failedToolResult(action, message, data, error) {
|
|
|
1123
1105
|
}
|
|
1124
1106
|
function getMultipleElementsSuggestion() {
|
|
1125
1107
|
return dedent `
|
|
1126
|
-
Multiple elements matched your locator
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
5. Use xpathCheck() to inspect matched elements and pick the correct one
|
|
1132
|
-
6. Use visualClick() to click the right element by visual appearance
|
|
1108
|
+
Multiple elements matched your locator, so that command did nothing — it selected no element and acted on none.
|
|
1109
|
+
Read the numbered elements list and click the one you meant by its number:
|
|
1110
|
+
reuse the same locator with step.opts({ elementIndex: N }) as the last argument.
|
|
1111
|
+
If none of them is the element you want, narrow the locator with a container or its full unique text.
|
|
1112
|
+
If the list is missing, call xpathCheck() to see what the locator matches.
|
|
1133
1113
|
`;
|
|
1134
1114
|
}
|
|
1135
1115
|
export function withdrawVisionTools(tools) {
|
|
@@ -1194,42 +1174,6 @@ export async function formatMatchedElements(error) {
|
|
|
1194
1174
|
return 'Could not fetch element details. Repeat the action to get better info.';
|
|
1195
1175
|
return formatElementList(details);
|
|
1196
1176
|
}
|
|
1197
|
-
async function disambiguateElements(error, explanation, provider) {
|
|
1198
|
-
const elementDetails = await extractWebElements(error);
|
|
1199
|
-
if (!elementDetails)
|
|
1200
|
-
return null;
|
|
1201
|
-
const elementList = formatElementList(elementDetails);
|
|
1202
|
-
const schema = z.object({
|
|
1203
|
-
position: z.number().nullable().describe('1-based position of the correct element, or null if none match'),
|
|
1204
|
-
});
|
|
1205
|
-
try {
|
|
1206
|
-
const result = await provider.generateObject([
|
|
1207
|
-
{
|
|
1208
|
-
role: 'user',
|
|
1209
|
-
content: dedent `
|
|
1210
|
-
A click action failed because multiple elements matched the locator.
|
|
1211
|
-
The intended action was: ${explanation}
|
|
1212
|
-
|
|
1213
|
-
Here are the matched elements:
|
|
1214
|
-
|
|
1215
|
-
${elementList}
|
|
1216
|
-
|
|
1217
|
-
Which element (1-${elementDetails.length}) best matches the intended action?
|
|
1218
|
-
Return the position number, or null if none of them match.
|
|
1219
|
-
`,
|
|
1220
|
-
},
|
|
1221
|
-
], schema, provider.getModelForAgent(), { agentName: 'disambiguator', timeout: 15000 });
|
|
1222
|
-
const position = result?.object?.position;
|
|
1223
|
-
if (position && position >= 1 && position <= elementDetails.length) {
|
|
1224
|
-
return { position, xpath: elementDetails[position - 1].xpath };
|
|
1225
|
-
}
|
|
1226
|
-
return null;
|
|
1227
|
-
}
|
|
1228
|
-
catch (e) {
|
|
1229
|
-
debugLog('Element disambiguation AI call failed: %s', e);
|
|
1230
|
-
return null;
|
|
1231
|
-
}
|
|
1232
|
-
}
|
|
1233
1177
|
function getNotFoundSuggestion(errorMessage) {
|
|
1234
1178
|
if (!errorMessage.includes('not found')) {
|
|
1235
1179
|
return null;
|
|
@@ -96,7 +96,9 @@ export class RequestResult {
|
|
|
96
96
|
yaml += body;
|
|
97
97
|
}
|
|
98
98
|
writeFileSync(this.requestFile, yaml, 'utf8');
|
|
99
|
-
|
|
99
|
+
if (!this._rawResponseBody)
|
|
100
|
+
return;
|
|
101
|
+
writeFileSync(this.responseFile, this._rawResponseBody, 'utf8');
|
|
100
102
|
}
|
|
101
103
|
static load(requestFile) {
|
|
102
104
|
const content = readFileSync(requestFile, 'utf8');
|
|
@@ -6,25 +6,26 @@ export declare class RequestStore {
|
|
|
6
6
|
onFailedListeners: Array<(r: RequestResult) => void>;
|
|
7
7
|
outputDir: string;
|
|
8
8
|
sessionStartedAt: Date;
|
|
9
|
+
readEndpointKeys: Set<string>;
|
|
9
10
|
constructor(outputDir: string);
|
|
10
11
|
addCapturedRequest(result: RequestResult): void;
|
|
12
|
+
addReadRequest(result: RequestResult): void;
|
|
11
13
|
addFailedRequest(result: RequestResult): void;
|
|
12
14
|
getFailedRequests(): RequestResult[];
|
|
13
15
|
onFailedRequest(cb: (r: RequestResult) => void): () => void;
|
|
14
16
|
addMadeRequest(result: RequestResult): void;
|
|
15
|
-
addRequest(result: RequestResult): void;
|
|
16
17
|
getCapturedRequests(): RequestResult[];
|
|
17
18
|
getMadeRequests(): RequestResult[];
|
|
18
|
-
getRequests(): RequestResult[];
|
|
19
19
|
getLastRequest(): RequestResult | undefined;
|
|
20
|
-
|
|
21
|
-
getRequestsByMethod(method: string): RequestResult[];
|
|
22
|
-
getRequestsByStatus(status: number): RequestResult[];
|
|
23
|
-
toEndpointList(scopePath?: string): string;
|
|
20
|
+
toEndpointList(scopePath?: string, methods?: EndpointFamily): string;
|
|
24
21
|
extractAuthHeaders(): Record<string, string>;
|
|
25
22
|
findCapturedRequest(method: string, searchPath: string): RequestResult | undefined;
|
|
26
23
|
toLog(): string;
|
|
27
24
|
loadFromDisk(): void;
|
|
28
25
|
getWriteRequestsForScope(scopePath: string): RequestResult[];
|
|
26
|
+
getReadRequestsForScope(scopePath: string): RequestResult[];
|
|
29
27
|
clear(): void;
|
|
28
|
+
getRequestsForScope(scopePath: string, methods: EndpointFamily): RequestResult[];
|
|
30
29
|
}
|
|
30
|
+
export declare function isFailedRequest(request: RequestResult): boolean;
|
|
31
|
+
export type EndpointFamily = 'read' | 'write';
|