explorbot 0.4.1 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/bin/explorbot-cli.ts +8 -3
  2. package/boat/api-tester/src/ai/curler-tools.ts +1 -1
  3. package/boat/api-tester/src/cli.ts +3 -0
  4. package/boat/doc-collector/src/ai/documentarian.ts +57 -5
  5. package/boat/doc-collector/src/ai/tools.ts +1 -1
  6. package/boat/doc-collector/src/cli.ts +18 -2
  7. package/boat/doc-collector/src/config.ts +3 -0
  8. package/boat/doc-collector/src/docbot.ts +56 -11
  9. package/boat/doc-collector/src/docs-renderer.ts +79 -55
  10. package/boat/doc-collector/src/interaction-screenshots.ts +19 -10
  11. package/boat/doc-collector/src/screenshots.ts +106 -3
  12. package/boat/doc-collector/src/template-dedup.ts +31 -0
  13. package/boat/prima/src/cli.ts +3 -0
  14. package/boat/prima/src/prima.ts +3 -4
  15. package/dist/bin/explorbot-cli.js +9 -3
  16. package/dist/boat/api-tester/src/ai/curler-tools.js +1 -1
  17. package/dist/boat/api-tester/src/cli.js +2 -0
  18. package/dist/boat/doc-collector/src/ai/documentarian.js +62 -4
  19. package/dist/boat/doc-collector/src/ai/tools.js +1 -1
  20. package/dist/boat/doc-collector/src/cli.js +15 -2
  21. package/dist/boat/doc-collector/src/config.js +1 -0
  22. package/dist/boat/doc-collector/src/docbot.js +52 -10
  23. package/dist/boat/doc-collector/src/docs-renderer.js +69 -46
  24. package/dist/boat/doc-collector/src/interaction-screenshots.js +18 -11
  25. package/dist/boat/doc-collector/src/screenshots.js +104 -1
  26. package/dist/boat/doc-collector/src/template-dedup.js +27 -0
  27. package/dist/boat/prima/src/cli.js +2 -0
  28. package/dist/boat/prima/src/prima.js +3 -4
  29. package/dist/package.json +1 -1
  30. package/dist/src/action-result.d.ts +2 -2
  31. package/dist/src/action-result.js +15 -17
  32. package/dist/src/action.d.ts +1 -1
  33. package/dist/src/action.js +15 -15
  34. package/dist/src/ai/fisherman/request-haul.d.ts +11 -0
  35. package/dist/src/ai/fisherman/request-haul.js +28 -0
  36. package/dist/src/ai/fisherman-tools.d.ts +10 -1
  37. package/dist/src/ai/fisherman-tools.js +35 -31
  38. package/dist/src/ai/fisherman.d.ts +3 -2
  39. package/dist/src/ai/fisherman.js +8 -6
  40. package/dist/src/ai/pilot.js +21 -16
  41. package/dist/src/ai/provider.js +8 -14
  42. package/dist/src/ai/researcher/coordinates.d.ts +2 -0
  43. package/dist/src/ai/researcher/coordinates.js +13 -1
  44. package/dist/src/ai/researcher/deep-analysis.js +9 -9
  45. package/dist/src/ai/researcher/fingerprint-worker.js +1 -11
  46. package/dist/src/ai/researcher.js +1 -1
  47. package/dist/src/ai/rules.js +8 -0
  48. package/dist/src/ai/tester.js +20 -20
  49. package/dist/src/ai/tools.d.ts +1 -1
  50. package/dist/src/ai/tools.js +18 -74
  51. package/dist/src/api/request-store.d.ts +1 -5
  52. package/dist/src/api/request-store.js +12 -22
  53. package/dist/src/commands/config-command.js +2 -9
  54. package/dist/src/commands/index.js +2 -0
  55. package/dist/src/commands/init-command.js +3 -7
  56. package/dist/src/commands/plans-command.d.ts +4 -1
  57. package/dist/src/commands/plans-command.js +4 -4
  58. package/dist/src/commands/recommended-models-command.d.ts +9 -0
  59. package/dist/src/commands/recommended-models-command.js +85 -0
  60. package/dist/src/config.d.ts +2 -0
  61. package/dist/src/config.js +7 -2
  62. package/dist/src/experience-tracker.js +1 -1
  63. package/dist/src/explorbot.js +3 -3
  64. package/dist/src/global-config.d.ts +2 -0
  65. package/dist/src/global-config.js +6 -0
  66. package/dist/src/state-manager.d.ts +3 -3
  67. package/dist/src/state-manager.js +8 -8
  68. package/dist/src/test-plan.d.ts +1 -0
  69. package/dist/src/test-plan.js +13 -9
  70. package/dist/src/utils/aria.d.ts +1 -1
  71. package/dist/src/utils/aria.js +23 -2
  72. package/dist/src/utils/cli-section.d.ts +1 -0
  73. package/dist/src/utils/cli-section.js +12 -0
  74. package/dist/src/utils/html-diff.d.ts +1 -0
  75. package/dist/src/utils/html-diff.js +5 -5
  76. package/dist/src/utils/html.d.ts +2 -0
  77. package/dist/src/utils/html.js +5 -4
  78. package/dist/src/utils/overlay.d.ts +19 -55
  79. package/dist/src/utils/overlay.js +94 -148
  80. package/dist/src/utils/region.d.ts +41 -0
  81. package/dist/src/utils/region.js +99 -0
  82. package/dist/src/utils/similarity.d.ts +1 -0
  83. package/dist/src/utils/similarity.js +11 -0
  84. package/dist/src/utils/url-matcher.d.ts +1 -1
  85. package/dist/src/utils/url-matcher.js +4 -4
  86. package/docs/doc-collection/basics.md +11 -5
  87. package/docs/doc-collection/crawling.md +21 -1
  88. package/docs/reference/commands.md +3 -0
  89. package/docs/superpowers/plans/2026-09-03-model-resolves-click-ambiguity.md +509 -0
  90. package/docs/workflow/application-spec.md +4 -0
  91. package/package.json +1 -1
  92. package/src/action-result.ts +15 -16
  93. package/src/action.ts +16 -15
  94. package/src/ai/fisherman/request-haul.ts +32 -0
  95. package/src/ai/fisherman-tools.ts +38 -32
  96. package/src/ai/fisherman.ts +8 -7
  97. package/src/ai/pilot.ts +19 -14
  98. package/src/ai/provider.ts +8 -12
  99. package/src/ai/researcher/coordinates.ts +13 -1
  100. package/src/ai/researcher/deep-analysis.ts +9 -9
  101. package/src/ai/researcher/fingerprint-worker.ts +1 -10
  102. package/src/ai/researcher.ts +1 -1
  103. package/src/ai/rules.ts +8 -0
  104. package/src/ai/tester.ts +18 -18
  105. package/src/ai/tools.ts +18 -86
  106. package/src/api/request-store.ts +13 -27
  107. package/src/commands/config-command.ts +2 -7
  108. package/src/commands/index.ts +2 -0
  109. package/src/commands/init-command.ts +4 -11
  110. package/src/commands/plans-command.ts +5 -5
  111. package/src/commands/recommended-models-command.ts +92 -0
  112. package/src/config.ts +9 -2
  113. package/src/experience-tracker.ts +1 -1
  114. package/src/explorbot.ts +3 -3
  115. package/src/global-config.ts +8 -0
  116. package/src/state-manager.ts +10 -9
  117. package/src/test-plan.ts +12 -10
  118. package/src/utils/aria.ts +26 -3
  119. package/src/utils/cli-section.ts +12 -0
  120. package/src/utils/html-diff.ts +6 -6
  121. package/src/utils/html.ts +6 -4
  122. package/src/utils/overlay.ts +105 -192
  123. package/src/utils/region.ts +132 -0
  124. package/src/utils/similarity.ts +9 -0
  125. package/src/utils/url-matcher.ts +3 -3
  126. package/dist/src/utils/request-map.d.ts +0 -7
  127. package/dist/src/utils/request-map.js +0 -16
  128. package/src/utils/request-map.ts +0 -19
@@ -3,19 +3,15 @@ import dedent from 'dedent';
3
3
  import { z } from 'zod';
4
4
  import { extractEndpointDefinition } from "../api/spec-reader.js";
5
5
  import { tag } from "../utils/logger.js";
6
- import { RequestMap } from "../utils/request-map.js";
7
6
  import { isDynamicSegment } from "../utils/url-matcher.js";
8
- export function createFishermanTools(apiClient, requestStore, opts) {
7
+ export function createFishermanTools(apiClient, requestStore, haul, opts) {
9
8
  let finished = false;
10
9
  let result = null;
11
- const ledgerStart = requestStore.getMadeRequests().length;
12
- const runRequests = () => requestStore.getMadeRequests().slice(ledgerStart);
13
- const successfulWrites = () => runRequests().filter((r) => r.isWrite && !r.error && r.status >= 200 && r.status < 400);
14
- const getResult = () => result ?? synthesizeResult(runRequests(), successfulWrites(), false);
10
+ const getResult = () => result ?? synthesizeResult(haul, false);
15
11
  const isFinished = () => finished;
16
12
  const finishFromText = (text) => {
17
13
  finished = true;
18
- const synthesized = synthesizeResult(runRequests(), successfulWrites(), true);
14
+ const synthesized = synthesizeResult(haul, true);
19
15
  if (text && synthesized.success)
20
16
  synthesized.summary = text;
21
17
  result = synthesized;
@@ -142,30 +138,12 @@ export function createFishermanTools(apiClient, requestStore, opts) {
142
138
  .describe('List of items that could not be created'),
143
139
  }),
144
140
  execute: async ({ summary, created, failed }) => {
145
- const writes = successfulWrites();
146
- if (writes.length === 0) {
147
- tag('warning').log('Fisherman: finish rejected no successful write request in this run');
148
- return { finished: false, error: 'No successful write request was made in this run, so nothing was created. Keep working, or call stop if the data cannot be prepared.' };
149
- }
150
- const createdRequests = new RequestMap(writes);
151
- const verified = [];
152
- for (const item of created) {
153
- if (item.id === undefined) {
154
- verified.push(item);
155
- continue;
156
- }
157
- const request = createdRequests.get(item.id);
158
- if (!request) {
159
- tag('warning').log(`Fisherman: dropped unverified created item ${item.type} (id: ${item.id})`);
160
- continue;
161
- }
162
- verified.push({ ...item, request: request.toEndpoint() });
163
- }
164
- if (verified.length === 0)
165
- verified.push(...writes.map(toCreatedItem));
141
+ const { result: verified, error } = verifyFinish(haul, { summary, created, failed });
142
+ if (!verified)
143
+ return { finished: false, error };
166
144
  tag('success').log(`Fisherman done: ${summary}`);
167
145
  finished = true;
168
- result = { success: true, summary, created: verified, failed: failed || [] };
146
+ result = verified;
169
147
  return { finished: true };
170
148
  },
171
149
  }),
@@ -184,8 +162,34 @@ export function createFishermanTools(apiClient, requestStore, opts) {
184
162
  };
185
163
  return { tools, getResult, isFinished, finishFromText };
186
164
  }
187
- function synthesizeResult(made, writes, declaredDone) {
188
- const failures = made.filter((r) => r.status >= 400 || r.error);
165
+ export function verifyFinish(haul, input) {
166
+ const writes = haul.successfulWrites();
167
+ if (writes.length === 0) {
168
+ tag('warning').log('Fisherman: finish rejected — no successful write request in this run');
169
+ return { result: null, error: 'No successful write request was made in this run, so nothing was created. Keep working, or call stop if the data cannot be prepared.' };
170
+ }
171
+ const createdRequests = haul.byId();
172
+ const verified = [];
173
+ for (const item of input.created) {
174
+ if (item.id === undefined) {
175
+ verified.push(item);
176
+ continue;
177
+ }
178
+ const request = createdRequests.get(String(item.id));
179
+ if (!request) {
180
+ tag('warning').log(`Fisherman: dropped unverified created item ${item.type} (id: ${item.id})`);
181
+ continue;
182
+ }
183
+ verified.push({ ...item, request: request.toEndpoint() });
184
+ }
185
+ if (verified.length === 0)
186
+ verified.push(...writes.map(toCreatedItem));
187
+ return { result: { success: true, summary: input.summary, created: verified, failed: input.failed || [] } };
188
+ }
189
+ function synthesizeResult(haul, declaredDone) {
190
+ const made = haul.requests();
191
+ const writes = haul.successfulWrites();
192
+ const failures = haul.failed();
189
193
  let summary = `Stopped before finishing: ${made.length} requests, ${writes.length} successful writes, ${failures.length} failed`;
190
194
  const lastFailure = failures[failures.length - 1];
191
195
  if (lastFailure)
@@ -1,7 +1,8 @@
1
1
  import type { ApiClient } from '../api/api-client.js';
2
- import type { RequestStore } from '../api/request-store.js';
2
+ import { type RequestStore } from '../api/request-store.js';
3
3
  import type { Agent } from './agent.js';
4
4
  import { type FishermanResult } from './fisherman-tools.js';
5
+ import { RequestHaul } from './fisherman/request-haul.js';
5
6
  import type { Provider } from './provider.js';
6
7
  export declare class Fisherman implements Agent {
7
8
  emoji: string;
@@ -26,6 +27,6 @@ export declare class Fisherman implements Agent {
26
27
  refreshAuth(): Promise<void>;
27
28
  buildEndpointList(scopeUrl?: string): string;
28
29
  buildSystemPrompt(endpointList: string, toolNames: string[], scopeUrl?: string): string;
29
- isStuckOnEndpoint(ledgerStart: number): boolean;
30
+ isStuckOnEndpoint(haul: RequestHaul): boolean;
30
31
  buildTaskPrompt(instructions: string): string;
31
32
  }
@@ -1,9 +1,11 @@
1
1
  import dedent from 'dedent';
2
+ import { isFailedRequest } from "../api/request-store.js";
2
3
  import { listAllEndpoints } from "../api/spec-reader.js";
3
4
  import { createDebug, tag } from "../utils/logger.js";
4
5
  const debugLog = createDebug('explorbot:fisherman');
5
6
  import { loop } from "../utils/loop.js";
6
7
  import { createFishermanTools } from "./fisherman-tools.js";
8
+ import { RequestHaul } from "./fisherman/request-haul.js";
7
9
  import { dataProtectionRules } from "./rules.js";
8
10
  const MAX_ITERATIONS = 15;
9
11
  const MAX_TOOL_ROUNDTRIPS = 5;
@@ -61,11 +63,11 @@ export class Fisherman {
61
63
  }
62
64
  await this.refreshAuth();
63
65
  debugLog(`auth headers: ${Object.keys(this.apiClient.getHeaders()).join(', ')}`);
64
- const { tools, getResult, isFinished, finishFromText } = createFishermanTools(this.apiClient, this.requestStore, {
66
+ const haul = new RequestHaul(this.requestStore);
67
+ const { tools, getResult, isFinished, finishFromText } = createFishermanTools(this.apiClient, this.requestStore, haul, {
65
68
  spec: this.spec,
66
69
  baseEndpoint: this.baseEndpoint,
67
70
  });
68
- const ledgerStart = this.requestStore.getMadeRequests().length;
69
71
  const conversation = this.provider.startConversation(this.buildSystemPrompt(endpointList, Object.keys(tools), scopeUrl), 'fisherman');
70
72
  conversation.addUserText(this.buildTaskPrompt(instructions));
71
73
  await loop(async ({ stop, iteration }) => {
@@ -85,7 +87,7 @@ export class Fisherman {
85
87
  stop();
86
88
  return;
87
89
  }
88
- if (this.isStuckOnEndpoint(ledgerStart)) {
90
+ if (this.isStuckOnEndpoint(haul)) {
89
91
  tag('warning').log('Fisherman: repeated failures on the same endpoint — stopping');
90
92
  stop();
91
93
  return;
@@ -190,13 +192,13 @@ export class Fisherman {
190
192
  ${dataProtectionRules}
191
193
  `;
192
194
  }
193
- isStuckOnEndpoint(ledgerStart) {
194
- const made = this.requestStore.getMadeRequests().slice(ledgerStart);
195
+ isStuckOnEndpoint(haul) {
196
+ const made = haul.requests();
195
197
  if (made.length < REPEATED_FAILURE_LIMIT)
196
198
  return false;
197
199
  const recent = made.slice(-REPEATED_FAILURE_LIMIT);
198
200
  const first = recent[0];
199
- return recent.every((r) => (r.status >= 400 || r.error) && r.method === first.method && r.path === first.path);
201
+ return recent.every((r) => isFailedRequest(r) && r.method === first.method && r.path === first.path);
200
202
  }
201
203
  buildTaskPrompt(instructions) {
202
204
  return dedent `
@@ -19,6 +19,7 @@ const META_TOOLS = ['record', 'reset', 'stop', 'finish'];
19
19
  const PILOT_REASONING_LIMIT = 500;
20
20
  const PILOT_MESSAGE_LIMIT = 2;
21
21
  const PILOT_MESSAGE_MAX_LENGTH = 160;
22
+ const PILOT_REQUEST_LIMIT = 5;
22
23
  export class Pilot {
23
24
  emoji = '🧭';
24
25
  provider;
@@ -741,21 +742,21 @@ export class Pilot {
741
742
  lines.push(`h2: ${state.h2 || ''}`);
742
743
  lines.push(`h3: ${state.h3 || ''}`);
743
744
  lines.push(`h4: ${state.h4 || ''}`);
744
- const focusArea = state.overlay;
745
- if (focusArea.detected) {
746
- let line = `modal: ${focusArea.name || focusArea.type}`;
747
- if (focusArea.root)
748
- line += ` (root: ${focusArea.root})`;
745
+ const region = state.overlay;
746
+ if (region.isModal) {
747
+ let line = `overlay: ${region.name || region.type}`;
748
+ if (region.root)
749
+ line += ` (root: ${region.root})`;
749
750
  lines.push(line);
750
751
  }
751
- else if (focusArea.present) {
752
- let line = `region: ${focusArea.name || 'unnamed'} (inline`;
753
- if (focusArea.root)
754
- line += `, root: ${focusArea.root}`;
752
+ else if (region.isOpen) {
753
+ let line = `region: ${region.name || 'unnamed'} (inline`;
754
+ if (region.root)
755
+ line += `, root: ${region.root}`;
755
756
  lines.push(`${line})`);
756
757
  }
757
758
  else {
758
- lines.push('modal: none');
759
+ lines.push('overlay: none');
759
760
  }
760
761
  const tabs = this.stateManager.otherTabs;
761
762
  if (tabs.length > 0) {
@@ -971,9 +972,13 @@ export class Pilot {
971
972
  line += `\n ${ariaDiff}`;
972
973
  if (t.output?.pageDiff?.urlChanged)
973
974
  line += `\n moved: ${t.output.pageDiff.previousUrl} → ${t.output.pageDiff.currentUrl}`;
974
- const failedRequests = (t.output?.pageDiff?.requests ?? []).filter((r) => r.status >= 400);
975
- if (failedRequests.length > 0) {
976
- line += `\n requests: ${failedRequests.map((r) => `${r.method} ${r.path} → ${r.status}`).join(', ')}`;
975
+ const pageRequests = t.output?.pageDiff?.requests ?? [];
976
+ const requests = pageRequests
977
+ .filter((r) => r.status >= 400)
978
+ .concat(pageRequests.filter((r) => r.status < 400))
979
+ .slice(0, PILOT_REQUEST_LIMIT);
980
+ if (requests.length > 0) {
981
+ line += `\n requests: ${requests.map((r) => `${r.method} ${r.path} → ${r.status}`).join(', ')}`;
977
982
  }
978
983
  const messages = (t.output?.pageDiff?.messages ?? []).slice(0, PILOT_MESSAGE_LIMIT);
979
984
  if (messages.length > 0) {
@@ -1034,15 +1039,15 @@ export class Pilot {
1034
1039
 
1035
1040
  Diagnostic patterns (use <state>, executed/element/skipped fields, ariaDiff):
1036
1041
  - Click failed + button in "disabled buttons" → required field missing. Instruct fill first.
1037
- - "modal: none" but Tester targets a modalmodal closed; re-trigger.
1042
+ - "overlay: none" but Tester targets an overlayoverlay closed; re-trigger.
1038
1043
  - "region:" in <state> → a large area appeared in place without navigation (subview, wizard step, panel). Direct Tester to act inside it; the rest of the page is still usable.
1039
1044
  - Action SUCCESS but ariaDiff empty → may have worked without visible DOM change; check result message.
1040
- - MultipleElementsFound → xpathCheck() to identify the right one, then precise locator or visualClick().
1045
+ - MultipleElementsFound → nothing was clicked. Tell Tester to reuse the same locator with step.opts({ elementIndex: N }) from the numbered elements list.
1041
1046
  - Wrong page (settings vs feature) → getVisitedStates() then back() or reset(). Don't try breadcrumbs (SPA back-nav is unreliable).
1042
1047
  - Click SUCCESS but executed locator ≠ explanation intent, or "skipped" attempts present → wrong element clicked.
1043
1048
  - form(I.type()) SUCCESS but "element" shows a button/link → keys went to wrong element; click the input first.
1044
1049
  - ariaDiff shows 5+ added/removed → page entered new mode (editor/modal); call context() before guessing selectors.
1045
- - Empty dropdown/list when items expected → missing data; call precondition() to create it.
1050
+ - Empty dropdown/list when items expected → wait explicitly, then check the state changed: ariaDiff and any GET that loaded data. If still nothing loaded, confirm the empty state with verify().
1046
1051
  - Search-and-select needs SEQUENCE: focus trigger → type to filter → click option. Tell Tester to split into separate tool calls.
1047
1052
  - Multi-action explanation in one tool call → instruct Tester to split.
1048
1053
 
@@ -10,6 +10,7 @@ import { configuredModels, modelName as getModelName } from '../config.js';
10
10
  import { executionController } from "../execution-controller.js";
11
11
  import { Observability } from "../observability.js";
12
12
  import { Stats } from "../stats.js";
13
+ import { getCliName } from '../utils/cli-name.js';
13
14
  import { createDebug, tag } from '../utils/logger.js';
14
15
  import { withRetry } from '../utils/retry.js';
15
16
  import { RulesLoader } from "../utils/rules-loader.js";
@@ -43,14 +44,7 @@ export async function flushTelemetry() {
43
44
  }
44
45
  const CONTEXT_LENGTH_PATTERNS = ['reduce the length', 'context length', 'maximum context', 'token limit', 'too many tokens', 'max_tokens', 'context_length_exceeded', 'output truncated at maxtokens'];
45
46
  function extractCachedTokens(usage) {
46
- if (!usage)
47
- return 0;
48
- const direct = usage.inputTokenDetails?.cacheReadTokens ?? usage.cachedInputTokens;
49
- if (typeof direct === 'number')
50
- return direct;
51
- const raw = usage.raw;
52
- const fromRaw = raw?.prompt_tokens_details?.cached_tokens ?? raw?.promptTokensDetails?.cachedTokens;
53
- return typeof fromRaw === 'number' ? fromRaw : 0;
47
+ return usage?.inputTokenDetails?.cacheReadTokens ?? 0;
54
48
  }
55
49
  function abortAfterIdle(ms, cancel, controller) {
56
50
  return new Promise((_, reject) => {
@@ -99,7 +93,7 @@ export class Provider {
99
93
  modelCallWaiters = [];
100
94
  constructor(config) {
101
95
  if (!config?.model) {
102
- throw new AiError('AI model is not configured. Set ai.model in your config file.');
96
+ throw new AiError(`AI model is not configured. Set ai.model in your config file, or pick one with ${getCliName()} recommended-models`);
103
97
  }
104
98
  this.config = config;
105
99
  this.initLangfuse();
@@ -226,8 +220,8 @@ export class Provider {
226
220
  if (!usage)
227
221
  return;
228
222
  Stats.recordTokens(agentName, modelName, {
229
- input: usage.inputTokens ?? usage.promptTokens ?? 0,
230
- output: usage.outputTokens ?? usage.completionTokens ?? 0,
223
+ input: usage.inputTokens ?? 0,
224
+ output: usage.outputTokens ?? 0,
231
225
  total: usage.totalTokens ?? 0,
232
226
  cached: extractCachedTokens(usage),
233
227
  });
@@ -387,7 +381,7 @@ export class Provider {
387
381
  let invalidRequestFeedbackAdded = false;
388
382
  const executedStepMessages = [];
389
383
  try {
390
- const response = await this.withModelRequestSlot(() => withRetry(async () => {
384
+ let response = await this.withModelRequestSlot(() => withRetry(async () => {
391
385
  const stepMessages = [];
392
386
  const onStepEnd = (step) => {
393
387
  stepMessages.push(...(step.response?.messages || []));
@@ -413,7 +407,7 @@ export class Provider {
413
407
  return result;
414
408
  }, this.getRetryOptions(options)));
415
409
  clearActivity();
416
- withExecutedSteps(response, executedStepMessages);
410
+ response = withExecutedSteps(response, executedStepMessages);
417
411
  // Log tool usage summary
418
412
  if (response.toolCalls && response.toolCalls.length > 0) {
419
413
  responseLog(response.toolCalls);
@@ -585,7 +579,7 @@ export class Provider {
585
579
  }
586
580
  async processImage(prompt, image) {
587
581
  if (!this.config.visionModel) {
588
- throw new Error('Vision model not configured. Please set ai.visionModel in your config.');
582
+ throw new Error(`Vision model not configured. Set ai.visionModel in your config; ${getCliName()} recommended-models lists the providers that serve one`);
589
583
  }
590
584
  setActivity(`🤖 Processing image with ${this.config.visionModel}`, 'ai');
591
585
  const messages = [
@@ -8,6 +8,7 @@ export declare function visuallyAnnotateContainers(page: Page, containers: Array
8
8
  css: string;
9
9
  label: string;
10
10
  }>): Promise<number>;
11
+ export declare function removeVisualAnnotations(page: Page): Promise<void>;
11
12
  export declare function WithCoordinates<T extends Constructor>(Base: T): {
12
13
  new (...args: any[]): {
13
14
  explorer: Explorer;
@@ -20,6 +21,7 @@ export declare function WithCoordinates<T extends Constructor>(Base: T): {
20
21
  label: string;
21
22
  }>;
22
23
  }): Promise<number>;
24
+ removeVisualAnnotations(): Promise<void>;
23
25
  _analyzeScreenshotForVisualProps(): Promise<VisualAnalysisResult>;
24
26
  mergeVisualData(result: ResearchResult, visualData: Map<string, {
25
27
  coordinates: string | null;
@@ -62,6 +62,15 @@ export async function visuallyAnnotateContainers(page, containers) {
62
62
  return count;
63
63
  }, containers);
64
64
  }
65
+ export async function removeVisualAnnotations(page) {
66
+ try {
67
+ await page.locator('[data-explorbot-annotation]').evaluateAll((elements) => {
68
+ for (const element of elements)
69
+ element.remove();
70
+ });
71
+ }
72
+ catch { }
73
+ }
65
74
  export function WithCoordinates(Base) {
66
75
  return class extends Base {
67
76
  analyzeScreenshotForVisualProps() {
@@ -70,6 +79,9 @@ export function WithCoordinates(Base) {
70
79
  async visuallyAnnotateElements(opts) {
71
80
  return this.explorer.withPage((page) => visuallyAnnotateContainers(page, opts?.containers || []));
72
81
  }
82
+ async removeVisualAnnotations() {
83
+ await this.explorer.withPage((page) => removeVisualAnnotations(page));
84
+ }
73
85
  async _analyzeScreenshotForVisualProps() {
74
86
  const elements = new Map();
75
87
  const emptyResult = { elements, pagePurpose: null, primaryActions: null, focusedSection: null };
@@ -81,7 +93,7 @@ export function WithCoordinates(Base) {
81
93
  tag('step').log('Analyzing annotated screenshot for visual properties');
82
94
  const prompt = dedent `
83
95
  This screenshot has two types of annotations:
84
- - **Section containers**: dashed bordered boxes (no labels on them). A legend at the bottom-left maps dashed line colors to section names. Ignore containers for this task.
96
+ - **Section containers**: dashed bordered boxes (no labels on them). A legend at the bottom-right maps dashed line colors to section names. Ignore containers for this task.
85
97
  - **Interactive elements**: solid bordered boxes with eidx numbers in the top-right corner above the box. Adjacent elements use different colors.
86
98
 
87
99
  For each interactive element (solid border, eidx number), report:
@@ -62,29 +62,29 @@ export function WithDeepAnalysis(Base) {
62
62
  this._appendExtendedResearch(result, expandedSections, navigationLinks);
63
63
  }
64
64
  async researchOverlay(current, previous, pageStateHash) {
65
- const focusArea = current.overlay;
66
- if (!focusArea.present || !focusArea.name)
65
+ const region = current.overlay;
66
+ if (!region.isOpen || !region.name)
67
67
  return null;
68
68
  const cached = getCachedResearch(pageStateHash);
69
69
  if (!cached)
70
70
  return null;
71
- const escaped = focusArea.name.replace(/"/g, '\\"');
71
+ const escaped = region.name.replace(/"/g, '\\"');
72
72
  if (mdq(cached).query(`section3(~"${escaped}")`).count() > 0) {
73
- debugLog(`Overlay "${focusArea.name}" already in cached research, skipping`);
73
+ debugLog(`Overlay "${region.name}" already in cached research, skipping`);
74
74
  return null;
75
75
  }
76
76
  const diff = await current.diff(previous);
77
77
  if (!diff.ariaChanged && diff.htmlParts.length === 0) {
78
- debugLog(`No diff between current and previous state for overlay "${focusArea.name}"`);
78
+ debugLog(`No diff between current and previous state for overlay "${region.name}"`);
79
79
  return null;
80
80
  }
81
81
  const alreadyExpanded = this._summarizeExpanded(parseResearchSections(cached)
82
82
  .filter((s) => s.elements.length > 0)
83
83
  .map((s) => s.rawMarkdown));
84
- tag('substep').log(`Researching overlay: ${focusArea.name}`);
85
- const sectionMarkdown = await this._analyzeExpandedAction('', focusArea.name, diff, alreadyExpanded);
84
+ tag('substep').log(`Researching overlay: ${region.name}`);
85
+ const sectionMarkdown = await this._analyzeExpandedAction('', region.name, diff, alreadyExpanded);
86
86
  if (!sectionMarkdown) {
87
- debugLog(`Overlay "${focusArea.name}" produced no meaningful expansion`);
87
+ debugLog(`Overlay "${region.name}" produced no meaningful expansion`);
88
88
  return null;
89
89
  }
90
90
  const extQuery = mdq(cached).query('section1(~"Extended Research")');
@@ -97,7 +97,7 @@ export function WithDeepAnalysis(Base) {
97
97
  updated = `${cached.trimEnd()}\n\n# Extended Research\n\n${sectionMarkdown}\n`;
98
98
  }
99
99
  saveResearch({ hash: pageStateHash }, updated);
100
- tag('substep').log(`Overlay research appended: ${focusArea.name}`);
100
+ tag('substep').log(`Overlay research appended: ${region.name}`);
101
101
  return sectionMarkdown;
102
102
  }
103
103
  _loadPreviousExtendedSections(hash) {
@@ -3,17 +3,7 @@ import { join } from 'node:path';
3
3
  import { parentPort } from 'node:worker_threads';
4
4
  import { computeHtmlFingerprint } from "../../utils/html-diff.js";
5
5
  import { isSamePageFamily } from "../../utils/url-matcher.js";
6
- function diceSimilarity(a, b) {
7
- let intersection = 0;
8
- for (const item of a) {
9
- if (b.has(item))
10
- intersection++;
11
- }
12
- const total = a.size + b.size;
13
- if (total === 0)
14
- return 100;
15
- return Math.round(((2 * intersection) / total) * 100);
16
- }
6
+ import { diceSimilarity } from "../../utils/similarity.js";
17
7
  parentPort.on('message', (data) => {
18
8
  const { html, statesDir, maxAgeMs, threshold, url } = data;
19
9
  if (!existsSync(statesDir)) {
@@ -193,7 +193,7 @@ export class Researcher extends ResearcherBase {
193
193
  const containers = validContainers.filter((c) => !freshBroken.includes(c.css));
194
194
  await this.visuallyAnnotateElements({ containers });
195
195
  this.actionResult = await this.explorer.capture({ screenshot: true });
196
- const visualResult = await this.analyzeScreenshotForVisualProps();
196
+ const visualResult = await this.analyzeScreenshotForVisualProps().finally(() => this.removeVisualAnnotations());
197
197
  if (visualResult.elements.size > 0) {
198
198
  await this.mergeVisualData(result, visualResult.elements);
199
199
  result.parseLocators();
@@ -318,6 +318,14 @@ export const actionRule = dedent `
318
318
  If locator doesn't work, try CSS or XPath locators.
319
319
  If nothing works, use I.clickXY(x, y) as last resort.
320
320
 
321
+ When a click result reports several matches, pick one from its numbered list by position rather than guessing a new locator.
322
+ Reuse the same locator with step.opts({ elementIndex: N }) as the LAST argument. N is the "Element N" number.
323
+
324
+ <example>
325
+ I.click('Remove', step.opts({ elementIndex: 2 }));
326
+ I.click({ role: 'link', text: 'Details' }, '.panel', step.opts({ elementIndex: 1 }));
327
+ </example>
328
+
321
329
  For checkboxes, prefer I.checkOption/I.uncheckOption over I.click.
322
330
 
323
331
 
@@ -486,14 +486,14 @@ export class Tester extends TaskAgent {
486
486
  const currentStateHash = currentState.hash;
487
487
  const isNewUrl = this.previousUrl !== currentUrl;
488
488
  const isNewState = !isNewUrl && this.previousStateHash !== null && this.previousStateHash !== currentStateHash;
489
- if (this.previousRegionPresent !== null && this.previousRegionPresent !== currentState.overlay.present) {
489
+ if (this.previousRegionPresent !== null && this.previousRegionPresent !== currentState.overlay.isOpen) {
490
490
  this.regionTransitioned = true;
491
491
  }
492
- this.previousRegionPresent = currentState.overlay.present;
492
+ this.previousRegionPresent = currentState.overlay.isOpen;
493
493
  this.previousUrl = currentUrl;
494
494
  this.previousStateHash = currentStateHash;
495
495
  let context = '';
496
- const focusArea = currentState.overlay;
496
+ const region = currentState.overlay;
497
497
  const focusedElement = currentState.focusedElement;
498
498
  if (focusedElement) {
499
499
  const isTextInput = ['textbox', 'combobox', 'searchbox'].includes(focusedElement.role);
@@ -511,30 +511,30 @@ export class Tester extends TaskAgent {
511
511
  </no_focus>
512
512
  `;
513
513
  }
514
- if (focusArea.detected) {
515
- const areaName = focusArea.name ? ` "${focusArea.name}"` : '';
514
+ if (region.isModal) {
515
+ const areaName = region.name ? ` "${region.name}"` : '';
516
516
  let rootHint = '';
517
- if (focusArea.root)
518
- rootHint = `\nIts content lives inside \`${focusArea.root}\` — scope locators to it.`;
517
+ if (region.root)
518
+ rootHint = `\nIts content lives inside \`${region.root}\` — scope locators to it.`;
519
519
  context += dedent `
520
- <focus_scope>
521
- A ${focusArea.type}${areaName} is currently open above the page.${rootHint}
522
- Scope all interactions to elements inside this ${focusArea.type}.
523
- Page navigation, filters, and tabs that exist outside it are not actionable while it is open and may share names or roles with elements inside it — prefer the locator inside the ${focusArea.type}.
524
- Use <page_aria> to confirm the element you target is actually inside the ${focusArea.type}.
525
- </focus_scope>
520
+ <overlay>
521
+ An overlay${areaName} is currently open above the page.${rootHint}
522
+ Scope all interactions to elements inside this overlay.
523
+ Page navigation, filters, and tabs that exist outside it are not actionable while it is open and may share names or roles with elements inside it — prefer the locator inside the overlay.
524
+ Use <page_aria> to confirm the element you target is actually inside the overlay.
525
+ </overlay>
526
526
  `;
527
527
  }
528
- if (!focusArea.detected && focusArea.present && isNewState) {
528
+ if (!region.isModal && region.isOpen && isNewState) {
529
529
  let rootHint = '';
530
- if (focusArea.root)
531
- rootHint = `\nIt lives inside \`${focusArea.root}\`.`;
530
+ if (region.root)
531
+ rootHint = `\nIt lives inside \`${region.root}\`.`;
532
532
  context += dedent `
533
- <area_of_interest>
534
- A large new area "${focusArea.name || 'unnamed area'}" appeared on this page without navigation.${rootHint}
533
+ <region>
534
+ A large new region "${region.name || 'unnamed region'}" appeared on this page without navigation.${rootHint}
535
535
  The scenario most likely continues inside this area — prefer its elements for your next actions.
536
536
  The rest of the page (navigation, menus, filters) is still interactive and remains available.
537
- </area_of_interest>
537
+ </region>
538
538
  `;
539
539
  }
540
540
  if (currentState.isInsideIframe) {
@@ -594,7 +594,7 @@ export class Tester extends TaskAgent {
594
594
  `;
595
595
  return context;
596
596
  }
597
- if (focusArea.present && focusArea.name && this.pageStateHash && this.pageActionResult) {
597
+ if (region.isOpen && region.name && this.pageStateHash && this.pageActionResult) {
598
598
  const overlaySection = await this.researcher.researchOverlay(currentState, this.pageActionResult, this.pageStateHash).catch(this.skipResearch);
599
599
  if (overlaySection) {
600
600
  context += dedent `
@@ -12,7 +12,7 @@ interface AgentToolDeps extends ToolDeps {
12
12
  withExperience?: boolean;
13
13
  }
14
14
  export declare const ASSERTION_TOOLS: readonly ["verify"];
15
- export declare function createCodeceptJSTools({ explorer, stateManager, ai }: ToolDeps, task: Task): {
15
+ export declare function createCodeceptJSTools({ explorer, stateManager }: ToolDeps, task: Task): {
16
16
  click: import("@ai-sdk/provider-utils").ExecutableTool<import("ai").Tool<{
17
17
  commands: any;
18
18
  explanation: any;