explorbot 0.4.1 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/explorbot-cli.ts +8 -3
- package/boat/api-tester/src/ai/curler-tools.ts +1 -1
- package/boat/api-tester/src/cli.ts +3 -0
- package/boat/doc-collector/src/ai/documentarian.ts +57 -5
- package/boat/doc-collector/src/ai/tools.ts +1 -1
- package/boat/doc-collector/src/cli.ts +18 -2
- package/boat/doc-collector/src/config.ts +3 -0
- package/boat/doc-collector/src/docbot.ts +56 -11
- package/boat/doc-collector/src/docs-renderer.ts +79 -55
- package/boat/doc-collector/src/interaction-screenshots.ts +19 -10
- package/boat/doc-collector/src/screenshots.ts +106 -3
- package/boat/doc-collector/src/template-dedup.ts +31 -0
- package/boat/prima/src/cli.ts +3 -0
- package/boat/prima/src/prima.ts +3 -4
- package/dist/bin/explorbot-cli.js +9 -3
- package/dist/boat/api-tester/src/ai/curler-tools.js +1 -1
- package/dist/boat/api-tester/src/cli.js +2 -0
- package/dist/boat/doc-collector/src/ai/documentarian.js +62 -4
- package/dist/boat/doc-collector/src/ai/tools.js +1 -1
- package/dist/boat/doc-collector/src/cli.js +15 -2
- package/dist/boat/doc-collector/src/config.js +1 -0
- package/dist/boat/doc-collector/src/docbot.js +52 -10
- package/dist/boat/doc-collector/src/docs-renderer.js +69 -46
- package/dist/boat/doc-collector/src/interaction-screenshots.js +18 -11
- package/dist/boat/doc-collector/src/screenshots.js +104 -1
- package/dist/boat/doc-collector/src/template-dedup.js +27 -0
- package/dist/boat/prima/src/cli.js +2 -0
- package/dist/boat/prima/src/prima.js +3 -4
- package/dist/package.json +1 -1
- package/dist/src/action-result.d.ts +2 -2
- package/dist/src/action-result.js +15 -17
- package/dist/src/action.d.ts +1 -1
- package/dist/src/action.js +15 -15
- package/dist/src/ai/fisherman/request-haul.d.ts +11 -0
- package/dist/src/ai/fisherman/request-haul.js +28 -0
- package/dist/src/ai/fisherman-tools.d.ts +10 -1
- package/dist/src/ai/fisherman-tools.js +35 -31
- package/dist/src/ai/fisherman.d.ts +3 -2
- package/dist/src/ai/fisherman.js +8 -6
- package/dist/src/ai/pilot.js +21 -16
- package/dist/src/ai/provider.js +8 -14
- package/dist/src/ai/researcher/coordinates.d.ts +2 -0
- package/dist/src/ai/researcher/coordinates.js +13 -1
- package/dist/src/ai/researcher/deep-analysis.js +9 -9
- package/dist/src/ai/researcher/fingerprint-worker.js +1 -11
- package/dist/src/ai/researcher.js +1 -1
- package/dist/src/ai/rules.js +8 -0
- package/dist/src/ai/tester.js +20 -20
- package/dist/src/ai/tools.d.ts +1 -1
- package/dist/src/ai/tools.js +18 -74
- package/dist/src/api/request-store.d.ts +1 -5
- package/dist/src/api/request-store.js +12 -22
- package/dist/src/commands/config-command.js +2 -9
- package/dist/src/commands/index.js +2 -0
- package/dist/src/commands/init-command.js +3 -7
- package/dist/src/commands/plans-command.d.ts +4 -1
- package/dist/src/commands/plans-command.js +4 -4
- package/dist/src/commands/recommended-models-command.d.ts +9 -0
- package/dist/src/commands/recommended-models-command.js +85 -0
- package/dist/src/config.d.ts +2 -0
- package/dist/src/config.js +7 -2
- package/dist/src/experience-tracker.js +1 -1
- package/dist/src/explorbot.js +3 -3
- package/dist/src/global-config.d.ts +2 -0
- package/dist/src/global-config.js +6 -0
- package/dist/src/state-manager.d.ts +3 -3
- package/dist/src/state-manager.js +8 -8
- package/dist/src/test-plan.d.ts +1 -0
- package/dist/src/test-plan.js +13 -9
- package/dist/src/utils/aria.d.ts +1 -1
- package/dist/src/utils/aria.js +23 -2
- package/dist/src/utils/cli-section.d.ts +1 -0
- package/dist/src/utils/cli-section.js +12 -0
- package/dist/src/utils/html-diff.d.ts +1 -0
- package/dist/src/utils/html-diff.js +5 -5
- package/dist/src/utils/html.d.ts +2 -0
- package/dist/src/utils/html.js +5 -4
- package/dist/src/utils/overlay.d.ts +19 -55
- package/dist/src/utils/overlay.js +94 -148
- package/dist/src/utils/region.d.ts +41 -0
- package/dist/src/utils/region.js +99 -0
- package/dist/src/utils/similarity.d.ts +1 -0
- package/dist/src/utils/similarity.js +11 -0
- package/dist/src/utils/url-matcher.d.ts +1 -1
- package/dist/src/utils/url-matcher.js +4 -4
- package/docs/doc-collection/basics.md +11 -5
- package/docs/doc-collection/crawling.md +21 -1
- package/docs/reference/commands.md +3 -0
- package/docs/superpowers/plans/2026-09-03-model-resolves-click-ambiguity.md +509 -0
- package/docs/workflow/application-spec.md +4 -0
- package/package.json +1 -1
- package/src/action-result.ts +15 -16
- package/src/action.ts +16 -15
- package/src/ai/fisherman/request-haul.ts +32 -0
- package/src/ai/fisherman-tools.ts +38 -32
- package/src/ai/fisherman.ts +8 -7
- package/src/ai/pilot.ts +19 -14
- package/src/ai/provider.ts +8 -12
- package/src/ai/researcher/coordinates.ts +13 -1
- package/src/ai/researcher/deep-analysis.ts +9 -9
- package/src/ai/researcher/fingerprint-worker.ts +1 -10
- package/src/ai/researcher.ts +1 -1
- package/src/ai/rules.ts +8 -0
- package/src/ai/tester.ts +18 -18
- package/src/ai/tools.ts +18 -86
- package/src/api/request-store.ts +13 -27
- package/src/commands/config-command.ts +2 -7
- package/src/commands/index.ts +2 -0
- package/src/commands/init-command.ts +4 -11
- package/src/commands/plans-command.ts +5 -5
- package/src/commands/recommended-models-command.ts +92 -0
- package/src/config.ts +9 -2
- package/src/experience-tracker.ts +1 -1
- package/src/explorbot.ts +3 -3
- package/src/global-config.ts +8 -0
- package/src/state-manager.ts +10 -9
- package/src/test-plan.ts +12 -10
- package/src/utils/aria.ts +26 -3
- package/src/utils/cli-section.ts +12 -0
- package/src/utils/html-diff.ts +6 -6
- package/src/utils/html.ts +6 -4
- package/src/utils/overlay.ts +105 -192
- package/src/utils/region.ts +132 -0
- package/src/utils/similarity.ts +9 -0
- package/src/utils/url-matcher.ts +3 -3
- package/dist/src/utils/request-map.d.ts +0 -7
- package/dist/src/utils/request-map.js +0 -16
- package/src/utils/request-map.ts +0 -19
|
@@ -3,19 +3,15 @@ import dedent from 'dedent';
|
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
import { extractEndpointDefinition } from "../api/spec-reader.js";
|
|
5
5
|
import { tag } from "../utils/logger.js";
|
|
6
|
-
import { RequestMap } from "../utils/request-map.js";
|
|
7
6
|
import { isDynamicSegment } from "../utils/url-matcher.js";
|
|
8
|
-
export function createFishermanTools(apiClient, requestStore, opts) {
|
|
7
|
+
export function createFishermanTools(apiClient, requestStore, haul, opts) {
|
|
9
8
|
let finished = false;
|
|
10
9
|
let result = null;
|
|
11
|
-
const
|
|
12
|
-
const runRequests = () => requestStore.getMadeRequests().slice(ledgerStart);
|
|
13
|
-
const successfulWrites = () => runRequests().filter((r) => r.isWrite && !r.error && r.status >= 200 && r.status < 400);
|
|
14
|
-
const getResult = () => result ?? synthesizeResult(runRequests(), successfulWrites(), false);
|
|
10
|
+
const getResult = () => result ?? synthesizeResult(haul, false);
|
|
15
11
|
const isFinished = () => finished;
|
|
16
12
|
const finishFromText = (text) => {
|
|
17
13
|
finished = true;
|
|
18
|
-
const synthesized = synthesizeResult(
|
|
14
|
+
const synthesized = synthesizeResult(haul, true);
|
|
19
15
|
if (text && synthesized.success)
|
|
20
16
|
synthesized.summary = text;
|
|
21
17
|
result = synthesized;
|
|
@@ -142,30 +138,12 @@ export function createFishermanTools(apiClient, requestStore, opts) {
|
|
|
142
138
|
.describe('List of items that could not be created'),
|
|
143
139
|
}),
|
|
144
140
|
execute: async ({ summary, created, failed }) => {
|
|
145
|
-
const
|
|
146
|
-
if (
|
|
147
|
-
|
|
148
|
-
return { finished: false, error: 'No successful write request was made in this run, so nothing was created. Keep working, or call stop if the data cannot be prepared.' };
|
|
149
|
-
}
|
|
150
|
-
const createdRequests = new RequestMap(writes);
|
|
151
|
-
const verified = [];
|
|
152
|
-
for (const item of created) {
|
|
153
|
-
if (item.id === undefined) {
|
|
154
|
-
verified.push(item);
|
|
155
|
-
continue;
|
|
156
|
-
}
|
|
157
|
-
const request = createdRequests.get(item.id);
|
|
158
|
-
if (!request) {
|
|
159
|
-
tag('warning').log(`Fisherman: dropped unverified created item ${item.type} (id: ${item.id})`);
|
|
160
|
-
continue;
|
|
161
|
-
}
|
|
162
|
-
verified.push({ ...item, request: request.toEndpoint() });
|
|
163
|
-
}
|
|
164
|
-
if (verified.length === 0)
|
|
165
|
-
verified.push(...writes.map(toCreatedItem));
|
|
141
|
+
const { result: verified, error } = verifyFinish(haul, { summary, created, failed });
|
|
142
|
+
if (!verified)
|
|
143
|
+
return { finished: false, error };
|
|
166
144
|
tag('success').log(`Fisherman done: ${summary}`);
|
|
167
145
|
finished = true;
|
|
168
|
-
result =
|
|
146
|
+
result = verified;
|
|
169
147
|
return { finished: true };
|
|
170
148
|
},
|
|
171
149
|
}),
|
|
@@ -184,8 +162,34 @@ export function createFishermanTools(apiClient, requestStore, opts) {
|
|
|
184
162
|
};
|
|
185
163
|
return { tools, getResult, isFinished, finishFromText };
|
|
186
164
|
}
|
|
187
|
-
function
|
|
188
|
-
const
|
|
165
|
+
export function verifyFinish(haul, input) {
|
|
166
|
+
const writes = haul.successfulWrites();
|
|
167
|
+
if (writes.length === 0) {
|
|
168
|
+
tag('warning').log('Fisherman: finish rejected — no successful write request in this run');
|
|
169
|
+
return { result: null, error: 'No successful write request was made in this run, so nothing was created. Keep working, or call stop if the data cannot be prepared.' };
|
|
170
|
+
}
|
|
171
|
+
const createdRequests = haul.byId();
|
|
172
|
+
const verified = [];
|
|
173
|
+
for (const item of input.created) {
|
|
174
|
+
if (item.id === undefined) {
|
|
175
|
+
verified.push(item);
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
const request = createdRequests.get(String(item.id));
|
|
179
|
+
if (!request) {
|
|
180
|
+
tag('warning').log(`Fisherman: dropped unverified created item ${item.type} (id: ${item.id})`);
|
|
181
|
+
continue;
|
|
182
|
+
}
|
|
183
|
+
verified.push({ ...item, request: request.toEndpoint() });
|
|
184
|
+
}
|
|
185
|
+
if (verified.length === 0)
|
|
186
|
+
verified.push(...writes.map(toCreatedItem));
|
|
187
|
+
return { result: { success: true, summary: input.summary, created: verified, failed: input.failed || [] } };
|
|
188
|
+
}
|
|
189
|
+
function synthesizeResult(haul, declaredDone) {
|
|
190
|
+
const made = haul.requests();
|
|
191
|
+
const writes = haul.successfulWrites();
|
|
192
|
+
const failures = haul.failed();
|
|
189
193
|
let summary = `Stopped before finishing: ${made.length} requests, ${writes.length} successful writes, ${failures.length} failed`;
|
|
190
194
|
const lastFailure = failures[failures.length - 1];
|
|
191
195
|
if (lastFailure)
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import type { ApiClient } from '../api/api-client.js';
|
|
2
|
-
import type
|
|
2
|
+
import { type RequestStore } from '../api/request-store.js';
|
|
3
3
|
import type { Agent } from './agent.js';
|
|
4
4
|
import { type FishermanResult } from './fisherman-tools.js';
|
|
5
|
+
import { RequestHaul } from './fisherman/request-haul.js';
|
|
5
6
|
import type { Provider } from './provider.js';
|
|
6
7
|
export declare class Fisherman implements Agent {
|
|
7
8
|
emoji: string;
|
|
@@ -26,6 +27,6 @@ export declare class Fisherman implements Agent {
|
|
|
26
27
|
refreshAuth(): Promise<void>;
|
|
27
28
|
buildEndpointList(scopeUrl?: string): string;
|
|
28
29
|
buildSystemPrompt(endpointList: string, toolNames: string[], scopeUrl?: string): string;
|
|
29
|
-
isStuckOnEndpoint(
|
|
30
|
+
isStuckOnEndpoint(haul: RequestHaul): boolean;
|
|
30
31
|
buildTaskPrompt(instructions: string): string;
|
|
31
32
|
}
|
package/dist/src/ai/fisherman.js
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import dedent from 'dedent';
|
|
2
|
+
import { isFailedRequest } from "../api/request-store.js";
|
|
2
3
|
import { listAllEndpoints } from "../api/spec-reader.js";
|
|
3
4
|
import { createDebug, tag } from "../utils/logger.js";
|
|
4
5
|
const debugLog = createDebug('explorbot:fisherman');
|
|
5
6
|
import { loop } from "../utils/loop.js";
|
|
6
7
|
import { createFishermanTools } from "./fisherman-tools.js";
|
|
8
|
+
import { RequestHaul } from "./fisherman/request-haul.js";
|
|
7
9
|
import { dataProtectionRules } from "./rules.js";
|
|
8
10
|
const MAX_ITERATIONS = 15;
|
|
9
11
|
const MAX_TOOL_ROUNDTRIPS = 5;
|
|
@@ -61,11 +63,11 @@ export class Fisherman {
|
|
|
61
63
|
}
|
|
62
64
|
await this.refreshAuth();
|
|
63
65
|
debugLog(`auth headers: ${Object.keys(this.apiClient.getHeaders()).join(', ')}`);
|
|
64
|
-
const
|
|
66
|
+
const haul = new RequestHaul(this.requestStore);
|
|
67
|
+
const { tools, getResult, isFinished, finishFromText } = createFishermanTools(this.apiClient, this.requestStore, haul, {
|
|
65
68
|
spec: this.spec,
|
|
66
69
|
baseEndpoint: this.baseEndpoint,
|
|
67
70
|
});
|
|
68
|
-
const ledgerStart = this.requestStore.getMadeRequests().length;
|
|
69
71
|
const conversation = this.provider.startConversation(this.buildSystemPrompt(endpointList, Object.keys(tools), scopeUrl), 'fisherman');
|
|
70
72
|
conversation.addUserText(this.buildTaskPrompt(instructions));
|
|
71
73
|
await loop(async ({ stop, iteration }) => {
|
|
@@ -85,7 +87,7 @@ export class Fisherman {
|
|
|
85
87
|
stop();
|
|
86
88
|
return;
|
|
87
89
|
}
|
|
88
|
-
if (this.isStuckOnEndpoint(
|
|
90
|
+
if (this.isStuckOnEndpoint(haul)) {
|
|
89
91
|
tag('warning').log('Fisherman: repeated failures on the same endpoint — stopping');
|
|
90
92
|
stop();
|
|
91
93
|
return;
|
|
@@ -190,13 +192,13 @@ export class Fisherman {
|
|
|
190
192
|
${dataProtectionRules}
|
|
191
193
|
`;
|
|
192
194
|
}
|
|
193
|
-
isStuckOnEndpoint(
|
|
194
|
-
const made =
|
|
195
|
+
isStuckOnEndpoint(haul) {
|
|
196
|
+
const made = haul.requests();
|
|
195
197
|
if (made.length < REPEATED_FAILURE_LIMIT)
|
|
196
198
|
return false;
|
|
197
199
|
const recent = made.slice(-REPEATED_FAILURE_LIMIT);
|
|
198
200
|
const first = recent[0];
|
|
199
|
-
return recent.every((r) => (r
|
|
201
|
+
return recent.every((r) => isFailedRequest(r) && r.method === first.method && r.path === first.path);
|
|
200
202
|
}
|
|
201
203
|
buildTaskPrompt(instructions) {
|
|
202
204
|
return dedent `
|
package/dist/src/ai/pilot.js
CHANGED
|
@@ -19,6 +19,7 @@ const META_TOOLS = ['record', 'reset', 'stop', 'finish'];
|
|
|
19
19
|
const PILOT_REASONING_LIMIT = 500;
|
|
20
20
|
const PILOT_MESSAGE_LIMIT = 2;
|
|
21
21
|
const PILOT_MESSAGE_MAX_LENGTH = 160;
|
|
22
|
+
const PILOT_REQUEST_LIMIT = 5;
|
|
22
23
|
export class Pilot {
|
|
23
24
|
emoji = '🧭';
|
|
24
25
|
provider;
|
|
@@ -741,21 +742,21 @@ export class Pilot {
|
|
|
741
742
|
lines.push(`h2: ${state.h2 || ''}`);
|
|
742
743
|
lines.push(`h3: ${state.h3 || ''}`);
|
|
743
744
|
lines.push(`h4: ${state.h4 || ''}`);
|
|
744
|
-
const
|
|
745
|
-
if (
|
|
746
|
-
let line = `
|
|
747
|
-
if (
|
|
748
|
-
line += ` (root: ${
|
|
745
|
+
const region = state.overlay;
|
|
746
|
+
if (region.isModal) {
|
|
747
|
+
let line = `overlay: ${region.name || region.type}`;
|
|
748
|
+
if (region.root)
|
|
749
|
+
line += ` (root: ${region.root})`;
|
|
749
750
|
lines.push(line);
|
|
750
751
|
}
|
|
751
|
-
else if (
|
|
752
|
-
let line = `region: ${
|
|
753
|
-
if (
|
|
754
|
-
line += `, root: ${
|
|
752
|
+
else if (region.isOpen) {
|
|
753
|
+
let line = `region: ${region.name || 'unnamed'} (inline`;
|
|
754
|
+
if (region.root)
|
|
755
|
+
line += `, root: ${region.root}`;
|
|
755
756
|
lines.push(`${line})`);
|
|
756
757
|
}
|
|
757
758
|
else {
|
|
758
|
-
lines.push('
|
|
759
|
+
lines.push('overlay: none');
|
|
759
760
|
}
|
|
760
761
|
const tabs = this.stateManager.otherTabs;
|
|
761
762
|
if (tabs.length > 0) {
|
|
@@ -971,9 +972,13 @@ export class Pilot {
|
|
|
971
972
|
line += `\n ${ariaDiff}`;
|
|
972
973
|
if (t.output?.pageDiff?.urlChanged)
|
|
973
974
|
line += `\n moved: ${t.output.pageDiff.previousUrl} → ${t.output.pageDiff.currentUrl}`;
|
|
974
|
-
const
|
|
975
|
-
|
|
976
|
-
|
|
975
|
+
const pageRequests = t.output?.pageDiff?.requests ?? [];
|
|
976
|
+
const requests = pageRequests
|
|
977
|
+
.filter((r) => r.status >= 400)
|
|
978
|
+
.concat(pageRequests.filter((r) => r.status < 400))
|
|
979
|
+
.slice(0, PILOT_REQUEST_LIMIT);
|
|
980
|
+
if (requests.length > 0) {
|
|
981
|
+
line += `\n requests: ${requests.map((r) => `${r.method} ${r.path} → ${r.status}`).join(', ')}`;
|
|
977
982
|
}
|
|
978
983
|
const messages = (t.output?.pageDiff?.messages ?? []).slice(0, PILOT_MESSAGE_LIMIT);
|
|
979
984
|
if (messages.length > 0) {
|
|
@@ -1034,15 +1039,15 @@ export class Pilot {
|
|
|
1034
1039
|
|
|
1035
1040
|
Diagnostic patterns (use <state>, executed/element/skipped fields, ariaDiff):
|
|
1036
1041
|
- Click failed + button in "disabled buttons" → required field missing. Instruct fill first.
|
|
1037
|
-
- "
|
|
1042
|
+
- "overlay: none" but Tester targets an overlay → overlay closed; re-trigger.
|
|
1038
1043
|
- "region:" in <state> → a large area appeared in place without navigation (subview, wizard step, panel). Direct Tester to act inside it; the rest of the page is still usable.
|
|
1039
1044
|
- Action SUCCESS but ariaDiff empty → may have worked without visible DOM change; check result message.
|
|
1040
|
-
- MultipleElementsFound →
|
|
1045
|
+
- MultipleElementsFound → nothing was clicked. Tell Tester to reuse the same locator with step.opts({ elementIndex: N }) from the numbered elements list.
|
|
1041
1046
|
- Wrong page (settings vs feature) → getVisitedStates() then back() or reset(). Don't try breadcrumbs (SPA back-nav is unreliable).
|
|
1042
1047
|
- Click SUCCESS but executed locator ≠ explanation intent, or "skipped" attempts present → wrong element clicked.
|
|
1043
1048
|
- form(I.type()) SUCCESS but "element" shows a button/link → keys went to wrong element; click the input first.
|
|
1044
1049
|
- ariaDiff shows 5+ added/removed → page entered new mode (editor/modal); call context() before guessing selectors.
|
|
1045
|
-
- Empty dropdown/list when items expected →
|
|
1050
|
+
- Empty dropdown/list when items expected → wait explicitly, then check the state changed: ariaDiff and any GET that loaded data. If still nothing loaded, confirm the empty state with verify().
|
|
1046
1051
|
- Search-and-select needs SEQUENCE: focus trigger → type to filter → click option. Tell Tester to split into separate tool calls.
|
|
1047
1052
|
- Multi-action explanation in one tool call → instruct Tester to split.
|
|
1048
1053
|
|
package/dist/src/ai/provider.js
CHANGED
|
@@ -10,6 +10,7 @@ import { configuredModels, modelName as getModelName } from '../config.js';
|
|
|
10
10
|
import { executionController } from "../execution-controller.js";
|
|
11
11
|
import { Observability } from "../observability.js";
|
|
12
12
|
import { Stats } from "../stats.js";
|
|
13
|
+
import { getCliName } from '../utils/cli-name.js';
|
|
13
14
|
import { createDebug, tag } from '../utils/logger.js';
|
|
14
15
|
import { withRetry } from '../utils/retry.js';
|
|
15
16
|
import { RulesLoader } from "../utils/rules-loader.js";
|
|
@@ -43,14 +44,7 @@ export async function flushTelemetry() {
|
|
|
43
44
|
}
|
|
44
45
|
const CONTEXT_LENGTH_PATTERNS = ['reduce the length', 'context length', 'maximum context', 'token limit', 'too many tokens', 'max_tokens', 'context_length_exceeded', 'output truncated at maxtokens'];
|
|
45
46
|
function extractCachedTokens(usage) {
|
|
46
|
-
|
|
47
|
-
return 0;
|
|
48
|
-
const direct = usage.inputTokenDetails?.cacheReadTokens ?? usage.cachedInputTokens;
|
|
49
|
-
if (typeof direct === 'number')
|
|
50
|
-
return direct;
|
|
51
|
-
const raw = usage.raw;
|
|
52
|
-
const fromRaw = raw?.prompt_tokens_details?.cached_tokens ?? raw?.promptTokensDetails?.cachedTokens;
|
|
53
|
-
return typeof fromRaw === 'number' ? fromRaw : 0;
|
|
47
|
+
return usage?.inputTokenDetails?.cacheReadTokens ?? 0;
|
|
54
48
|
}
|
|
55
49
|
function abortAfterIdle(ms, cancel, controller) {
|
|
56
50
|
return new Promise((_, reject) => {
|
|
@@ -99,7 +93,7 @@ export class Provider {
|
|
|
99
93
|
modelCallWaiters = [];
|
|
100
94
|
constructor(config) {
|
|
101
95
|
if (!config?.model) {
|
|
102
|
-
throw new AiError(
|
|
96
|
+
throw new AiError(`AI model is not configured. Set ai.model in your config file, or pick one with ${getCliName()} recommended-models`);
|
|
103
97
|
}
|
|
104
98
|
this.config = config;
|
|
105
99
|
this.initLangfuse();
|
|
@@ -226,8 +220,8 @@ export class Provider {
|
|
|
226
220
|
if (!usage)
|
|
227
221
|
return;
|
|
228
222
|
Stats.recordTokens(agentName, modelName, {
|
|
229
|
-
input: usage.inputTokens ??
|
|
230
|
-
output: usage.outputTokens ??
|
|
223
|
+
input: usage.inputTokens ?? 0,
|
|
224
|
+
output: usage.outputTokens ?? 0,
|
|
231
225
|
total: usage.totalTokens ?? 0,
|
|
232
226
|
cached: extractCachedTokens(usage),
|
|
233
227
|
});
|
|
@@ -387,7 +381,7 @@ export class Provider {
|
|
|
387
381
|
let invalidRequestFeedbackAdded = false;
|
|
388
382
|
const executedStepMessages = [];
|
|
389
383
|
try {
|
|
390
|
-
|
|
384
|
+
let response = await this.withModelRequestSlot(() => withRetry(async () => {
|
|
391
385
|
const stepMessages = [];
|
|
392
386
|
const onStepEnd = (step) => {
|
|
393
387
|
stepMessages.push(...(step.response?.messages || []));
|
|
@@ -413,7 +407,7 @@ export class Provider {
|
|
|
413
407
|
return result;
|
|
414
408
|
}, this.getRetryOptions(options)));
|
|
415
409
|
clearActivity();
|
|
416
|
-
withExecutedSteps(response, executedStepMessages);
|
|
410
|
+
response = withExecutedSteps(response, executedStepMessages);
|
|
417
411
|
// Log tool usage summary
|
|
418
412
|
if (response.toolCalls && response.toolCalls.length > 0) {
|
|
419
413
|
responseLog(response.toolCalls);
|
|
@@ -585,7 +579,7 @@ export class Provider {
|
|
|
585
579
|
}
|
|
586
580
|
async processImage(prompt, image) {
|
|
587
581
|
if (!this.config.visionModel) {
|
|
588
|
-
throw new Error(
|
|
582
|
+
throw new Error(`Vision model not configured. Set ai.visionModel in your config; ${getCliName()} recommended-models lists the providers that serve one`);
|
|
589
583
|
}
|
|
590
584
|
setActivity(`🤖 Processing image with ${this.config.visionModel}`, 'ai');
|
|
591
585
|
const messages = [
|
|
@@ -8,6 +8,7 @@ export declare function visuallyAnnotateContainers(page: Page, containers: Array
|
|
|
8
8
|
css: string;
|
|
9
9
|
label: string;
|
|
10
10
|
}>): Promise<number>;
|
|
11
|
+
export declare function removeVisualAnnotations(page: Page): Promise<void>;
|
|
11
12
|
export declare function WithCoordinates<T extends Constructor>(Base: T): {
|
|
12
13
|
new (...args: any[]): {
|
|
13
14
|
explorer: Explorer;
|
|
@@ -20,6 +21,7 @@ export declare function WithCoordinates<T extends Constructor>(Base: T): {
|
|
|
20
21
|
label: string;
|
|
21
22
|
}>;
|
|
22
23
|
}): Promise<number>;
|
|
24
|
+
removeVisualAnnotations(): Promise<void>;
|
|
23
25
|
_analyzeScreenshotForVisualProps(): Promise<VisualAnalysisResult>;
|
|
24
26
|
mergeVisualData(result: ResearchResult, visualData: Map<string, {
|
|
25
27
|
coordinates: string | null;
|
|
@@ -62,6 +62,15 @@ export async function visuallyAnnotateContainers(page, containers) {
|
|
|
62
62
|
return count;
|
|
63
63
|
}, containers);
|
|
64
64
|
}
|
|
65
|
+
export async function removeVisualAnnotations(page) {
|
|
66
|
+
try {
|
|
67
|
+
await page.locator('[data-explorbot-annotation]').evaluateAll((elements) => {
|
|
68
|
+
for (const element of elements)
|
|
69
|
+
element.remove();
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
catch { }
|
|
73
|
+
}
|
|
65
74
|
export function WithCoordinates(Base) {
|
|
66
75
|
return class extends Base {
|
|
67
76
|
analyzeScreenshotForVisualProps() {
|
|
@@ -70,6 +79,9 @@ export function WithCoordinates(Base) {
|
|
|
70
79
|
async visuallyAnnotateElements(opts) {
|
|
71
80
|
return this.explorer.withPage((page) => visuallyAnnotateContainers(page, opts?.containers || []));
|
|
72
81
|
}
|
|
82
|
+
async removeVisualAnnotations() {
|
|
83
|
+
await this.explorer.withPage((page) => removeVisualAnnotations(page));
|
|
84
|
+
}
|
|
73
85
|
async _analyzeScreenshotForVisualProps() {
|
|
74
86
|
const elements = new Map();
|
|
75
87
|
const emptyResult = { elements, pagePurpose: null, primaryActions: null, focusedSection: null };
|
|
@@ -81,7 +93,7 @@ export function WithCoordinates(Base) {
|
|
|
81
93
|
tag('step').log('Analyzing annotated screenshot for visual properties');
|
|
82
94
|
const prompt = dedent `
|
|
83
95
|
This screenshot has two types of annotations:
|
|
84
|
-
- **Section containers**: dashed bordered boxes (no labels on them). A legend at the bottom-
|
|
96
|
+
- **Section containers**: dashed bordered boxes (no labels on them). A legend at the bottom-right maps dashed line colors to section names. Ignore containers for this task.
|
|
85
97
|
- **Interactive elements**: solid bordered boxes with eidx numbers in the top-right corner above the box. Adjacent elements use different colors.
|
|
86
98
|
|
|
87
99
|
For each interactive element (solid border, eidx number), report:
|
|
@@ -62,29 +62,29 @@ export function WithDeepAnalysis(Base) {
|
|
|
62
62
|
this._appendExtendedResearch(result, expandedSections, navigationLinks);
|
|
63
63
|
}
|
|
64
64
|
async researchOverlay(current, previous, pageStateHash) {
|
|
65
|
-
const
|
|
66
|
-
if (!
|
|
65
|
+
const region = current.overlay;
|
|
66
|
+
if (!region.isOpen || !region.name)
|
|
67
67
|
return null;
|
|
68
68
|
const cached = getCachedResearch(pageStateHash);
|
|
69
69
|
if (!cached)
|
|
70
70
|
return null;
|
|
71
|
-
const escaped =
|
|
71
|
+
const escaped = region.name.replace(/"/g, '\\"');
|
|
72
72
|
if (mdq(cached).query(`section3(~"${escaped}")`).count() > 0) {
|
|
73
|
-
debugLog(`Overlay "${
|
|
73
|
+
debugLog(`Overlay "${region.name}" already in cached research, skipping`);
|
|
74
74
|
return null;
|
|
75
75
|
}
|
|
76
76
|
const diff = await current.diff(previous);
|
|
77
77
|
if (!diff.ariaChanged && diff.htmlParts.length === 0) {
|
|
78
|
-
debugLog(`No diff between current and previous state for overlay "${
|
|
78
|
+
debugLog(`No diff between current and previous state for overlay "${region.name}"`);
|
|
79
79
|
return null;
|
|
80
80
|
}
|
|
81
81
|
const alreadyExpanded = this._summarizeExpanded(parseResearchSections(cached)
|
|
82
82
|
.filter((s) => s.elements.length > 0)
|
|
83
83
|
.map((s) => s.rawMarkdown));
|
|
84
|
-
tag('substep').log(`Researching overlay: ${
|
|
85
|
-
const sectionMarkdown = await this._analyzeExpandedAction('',
|
|
84
|
+
tag('substep').log(`Researching overlay: ${region.name}`);
|
|
85
|
+
const sectionMarkdown = await this._analyzeExpandedAction('', region.name, diff, alreadyExpanded);
|
|
86
86
|
if (!sectionMarkdown) {
|
|
87
|
-
debugLog(`Overlay "${
|
|
87
|
+
debugLog(`Overlay "${region.name}" produced no meaningful expansion`);
|
|
88
88
|
return null;
|
|
89
89
|
}
|
|
90
90
|
const extQuery = mdq(cached).query('section1(~"Extended Research")');
|
|
@@ -97,7 +97,7 @@ export function WithDeepAnalysis(Base) {
|
|
|
97
97
|
updated = `${cached.trimEnd()}\n\n# Extended Research\n\n${sectionMarkdown}\n`;
|
|
98
98
|
}
|
|
99
99
|
saveResearch({ hash: pageStateHash }, updated);
|
|
100
|
-
tag('substep').log(`Overlay research appended: ${
|
|
100
|
+
tag('substep').log(`Overlay research appended: ${region.name}`);
|
|
101
101
|
return sectionMarkdown;
|
|
102
102
|
}
|
|
103
103
|
_loadPreviousExtendedSections(hash) {
|
|
@@ -3,17 +3,7 @@ import { join } from 'node:path';
|
|
|
3
3
|
import { parentPort } from 'node:worker_threads';
|
|
4
4
|
import { computeHtmlFingerprint } from "../../utils/html-diff.js";
|
|
5
5
|
import { isSamePageFamily } from "../../utils/url-matcher.js";
|
|
6
|
-
|
|
7
|
-
let intersection = 0;
|
|
8
|
-
for (const item of a) {
|
|
9
|
-
if (b.has(item))
|
|
10
|
-
intersection++;
|
|
11
|
-
}
|
|
12
|
-
const total = a.size + b.size;
|
|
13
|
-
if (total === 0)
|
|
14
|
-
return 100;
|
|
15
|
-
return Math.round(((2 * intersection) / total) * 100);
|
|
16
|
-
}
|
|
6
|
+
import { diceSimilarity } from "../../utils/similarity.js";
|
|
17
7
|
parentPort.on('message', (data) => {
|
|
18
8
|
const { html, statesDir, maxAgeMs, threshold, url } = data;
|
|
19
9
|
if (!existsSync(statesDir)) {
|
|
@@ -193,7 +193,7 @@ export class Researcher extends ResearcherBase {
|
|
|
193
193
|
const containers = validContainers.filter((c) => !freshBroken.includes(c.css));
|
|
194
194
|
await this.visuallyAnnotateElements({ containers });
|
|
195
195
|
this.actionResult = await this.explorer.capture({ screenshot: true });
|
|
196
|
-
const visualResult = await this.analyzeScreenshotForVisualProps();
|
|
196
|
+
const visualResult = await this.analyzeScreenshotForVisualProps().finally(() => this.removeVisualAnnotations());
|
|
197
197
|
if (visualResult.elements.size > 0) {
|
|
198
198
|
await this.mergeVisualData(result, visualResult.elements);
|
|
199
199
|
result.parseLocators();
|
package/dist/src/ai/rules.js
CHANGED
|
@@ -318,6 +318,14 @@ export const actionRule = dedent `
|
|
|
318
318
|
If locator doesn't work, try CSS or XPath locators.
|
|
319
319
|
If nothing works, use I.clickXY(x, y) as last resort.
|
|
320
320
|
|
|
321
|
+
When a click result reports several matches, pick one from its numbered list by position rather than guessing a new locator.
|
|
322
|
+
Reuse the same locator with step.opts({ elementIndex: N }) as the LAST argument. N is the "Element N" number.
|
|
323
|
+
|
|
324
|
+
<example>
|
|
325
|
+
I.click('Remove', step.opts({ elementIndex: 2 }));
|
|
326
|
+
I.click({ role: 'link', text: 'Details' }, '.panel', step.opts({ elementIndex: 1 }));
|
|
327
|
+
</example>
|
|
328
|
+
|
|
321
329
|
For checkboxes, prefer I.checkOption/I.uncheckOption over I.click.
|
|
322
330
|
|
|
323
331
|
|
package/dist/src/ai/tester.js
CHANGED
|
@@ -486,14 +486,14 @@ export class Tester extends TaskAgent {
|
|
|
486
486
|
const currentStateHash = currentState.hash;
|
|
487
487
|
const isNewUrl = this.previousUrl !== currentUrl;
|
|
488
488
|
const isNewState = !isNewUrl && this.previousStateHash !== null && this.previousStateHash !== currentStateHash;
|
|
489
|
-
if (this.previousRegionPresent !== null && this.previousRegionPresent !== currentState.overlay.
|
|
489
|
+
if (this.previousRegionPresent !== null && this.previousRegionPresent !== currentState.overlay.isOpen) {
|
|
490
490
|
this.regionTransitioned = true;
|
|
491
491
|
}
|
|
492
|
-
this.previousRegionPresent = currentState.overlay.
|
|
492
|
+
this.previousRegionPresent = currentState.overlay.isOpen;
|
|
493
493
|
this.previousUrl = currentUrl;
|
|
494
494
|
this.previousStateHash = currentStateHash;
|
|
495
495
|
let context = '';
|
|
496
|
-
const
|
|
496
|
+
const region = currentState.overlay;
|
|
497
497
|
const focusedElement = currentState.focusedElement;
|
|
498
498
|
if (focusedElement) {
|
|
499
499
|
const isTextInput = ['textbox', 'combobox', 'searchbox'].includes(focusedElement.role);
|
|
@@ -511,30 +511,30 @@ export class Tester extends TaskAgent {
|
|
|
511
511
|
</no_focus>
|
|
512
512
|
`;
|
|
513
513
|
}
|
|
514
|
-
if (
|
|
515
|
-
const areaName =
|
|
514
|
+
if (region.isModal) {
|
|
515
|
+
const areaName = region.name ? ` "${region.name}"` : '';
|
|
516
516
|
let rootHint = '';
|
|
517
|
-
if (
|
|
518
|
-
rootHint = `\nIts content lives inside \`${
|
|
517
|
+
if (region.root)
|
|
518
|
+
rootHint = `\nIts content lives inside \`${region.root}\` — scope locators to it.`;
|
|
519
519
|
context += dedent `
|
|
520
|
-
<
|
|
521
|
-
|
|
522
|
-
Scope all interactions to elements inside this
|
|
523
|
-
Page navigation, filters, and tabs that exist outside it are not actionable while it is open and may share names or roles with elements inside it — prefer the locator inside the
|
|
524
|
-
Use <page_aria> to confirm the element you target is actually inside the
|
|
525
|
-
</
|
|
520
|
+
<overlay>
|
|
521
|
+
An overlay${areaName} is currently open above the page.${rootHint}
|
|
522
|
+
Scope all interactions to elements inside this overlay.
|
|
523
|
+
Page navigation, filters, and tabs that exist outside it are not actionable while it is open and may share names or roles with elements inside it — prefer the locator inside the overlay.
|
|
524
|
+
Use <page_aria> to confirm the element you target is actually inside the overlay.
|
|
525
|
+
</overlay>
|
|
526
526
|
`;
|
|
527
527
|
}
|
|
528
|
-
if (!
|
|
528
|
+
if (!region.isModal && region.isOpen && isNewState) {
|
|
529
529
|
let rootHint = '';
|
|
530
|
-
if (
|
|
531
|
-
rootHint = `\nIt lives inside \`${
|
|
530
|
+
if (region.root)
|
|
531
|
+
rootHint = `\nIt lives inside \`${region.root}\`.`;
|
|
532
532
|
context += dedent `
|
|
533
|
-
<
|
|
534
|
-
A large new
|
|
533
|
+
<region>
|
|
534
|
+
A large new region "${region.name || 'unnamed region'}" appeared on this page without navigation.${rootHint}
|
|
535
535
|
The scenario most likely continues inside this area — prefer its elements for your next actions.
|
|
536
536
|
The rest of the page (navigation, menus, filters) is still interactive and remains available.
|
|
537
|
-
</
|
|
537
|
+
</region>
|
|
538
538
|
`;
|
|
539
539
|
}
|
|
540
540
|
if (currentState.isInsideIframe) {
|
|
@@ -594,7 +594,7 @@ export class Tester extends TaskAgent {
|
|
|
594
594
|
`;
|
|
595
595
|
return context;
|
|
596
596
|
}
|
|
597
|
-
if (
|
|
597
|
+
if (region.isOpen && region.name && this.pageStateHash && this.pageActionResult) {
|
|
598
598
|
const overlaySection = await this.researcher.researchOverlay(currentState, this.pageActionResult, this.pageStateHash).catch(this.skipResearch);
|
|
599
599
|
if (overlaySection) {
|
|
600
600
|
context += dedent `
|
package/dist/src/ai/tools.d.ts
CHANGED
|
@@ -12,7 +12,7 @@ interface AgentToolDeps extends ToolDeps {
|
|
|
12
12
|
withExperience?: boolean;
|
|
13
13
|
}
|
|
14
14
|
export declare const ASSERTION_TOOLS: readonly ["verify"];
|
|
15
|
-
export declare function createCodeceptJSTools({ explorer, stateManager
|
|
15
|
+
export declare function createCodeceptJSTools({ explorer, stateManager }: ToolDeps, task: Task): {
|
|
16
16
|
click: import("@ai-sdk/provider-utils").ExecutableTool<import("ai").Tool<{
|
|
17
17
|
commands: any;
|
|
18
18
|
explanation: any;
|