explorbot 0.1.28 → 0.1.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +83 -245
- package/bin/explorbot-cli.ts +1 -0
- package/dist/bin/explorbot-cli.js +1 -0
- package/dist/package.json +8 -6
- package/dist/rules/navigator/verification-actions.md +2 -0
- package/dist/src/ai/fisherman.js +14 -3
- package/dist/src/ai/pilot.js +19 -4
- package/dist/src/ai/planner.js +16 -5
- package/dist/src/ai/provider.js +53 -18
- package/dist/src/ai/researcher.js +7 -1
- package/dist/src/ai/rules.js +44 -0
- package/dist/src/ai/tester.js +70 -7
- package/dist/src/ai/tools.js +67 -1
- package/dist/src/explorbot.js +7 -2
- package/dist/src/stats.js +16 -0
- package/dist/src/utils/aria.js +66 -6
- package/package.json +8 -6
- package/rules/navigator/verification-actions.md +2 -0
- package/src/ai/fisherman.ts +14 -3
- package/src/ai/pilot.ts +19 -4
- package/src/ai/planner.ts +16 -5
- package/src/ai/provider.ts +51 -19
- package/src/ai/researcher.ts +8 -1
- package/src/ai/rules.ts +46 -0
- package/src/ai/tester.ts +74 -7
- package/src/ai/tools.ts +80 -1
- package/src/config.ts +1 -0
- package/src/explorbot.ts +6 -2
- package/src/stats.ts +18 -0
- package/src/utils/aria.ts +63 -6
package/dist/src/ai/pilot.js
CHANGED
|
@@ -10,6 +10,7 @@ import { ErrorPageError } from "../utils/error-page.js";
|
|
|
10
10
|
import { createDebug, tag } from "../utils/logger.js";
|
|
11
11
|
const debugLog = createDebug('explorbot:pilot');
|
|
12
12
|
import { truncateJson } from "../utils/strings.js";
|
|
13
|
+
import { capabilityGroundingRule, dataProtectionRules } from "./rules.js";
|
|
13
14
|
import { isInteractive } from "./task-agent.js";
|
|
14
15
|
const CHECK_TOOLS = ['verify', 'see', 'research', 'context'];
|
|
15
16
|
const META_TOOLS = ['record', 'reset', 'stop', 'finish'];
|
|
@@ -71,7 +72,7 @@ export class Pilot {
|
|
|
71
72
|
const notes = task.notesToString() || 'No notes recorded.';
|
|
72
73
|
let visualAnalysis = '';
|
|
73
74
|
let screenshotState = null;
|
|
74
|
-
if (this.provider.hasVision()) {
|
|
75
|
+
if (type === 'finish' && this.provider.hasVision()) {
|
|
75
76
|
try {
|
|
76
77
|
screenshotState = await this.explorer.capturePageWithScreenshot();
|
|
77
78
|
if (screenshotState.screenshot) {
|
|
@@ -139,7 +140,7 @@ export class Pilot {
|
|
|
139
140
|
try {
|
|
140
141
|
const response = await this.provider.generateObject(messages, schema, this.provider.getAgenticModel('pilot'), {
|
|
141
142
|
agentName: 'pilot',
|
|
142
|
-
|
|
143
|
+
telemetry: { functionId: 'pilot.reviewVerdict' },
|
|
143
144
|
});
|
|
144
145
|
const result = response?.object;
|
|
145
146
|
if (!result) {
|
|
@@ -229,7 +230,7 @@ export class Pilot {
|
|
|
229
230
|
try {
|
|
230
231
|
const response = await this.provider.generateObject(messages, schema, this.provider.getAgenticModel('pilot'), {
|
|
231
232
|
agentName: 'pilot',
|
|
232
|
-
|
|
233
|
+
telemetry: { functionId: 'pilot.reviewReset' },
|
|
233
234
|
});
|
|
234
235
|
const result = response?.object;
|
|
235
236
|
if (!result) {
|
|
@@ -333,6 +334,8 @@ export class Pilot {
|
|
|
333
334
|
You are Pilot — final decision maker for test pass/fail. Tester requested ${type}. Review the
|
|
334
335
|
evidence and commit to a verdict; "continue" only when evidence is genuinely insufficient.
|
|
335
336
|
|
|
337
|
+
${capabilityGroundingRule}
|
|
338
|
+
|
|
336
339
|
${this.buildSharedEvidenceRules(task)}
|
|
337
340
|
|
|
338
341
|
DECISION:
|
|
@@ -341,6 +344,8 @@ export class Pilot {
|
|
|
341
344
|
Pick assertions DOM can express; for non-DOM regions (iframes, canvas, Monaco/CodeMirror), target a
|
|
342
345
|
stable landmark (container, ARIA role) instead of literal inner text. Your "pass" stands even if the
|
|
343
346
|
DOM assertion can't be made.
|
|
347
|
+
Do not pass when Tester achieved only a related navigation/filter/tab/status outcome instead of the
|
|
348
|
+
requested action, workflow, or entity detail goal.
|
|
344
349
|
- "fail": scenario was attempted but the goal was not achieved.
|
|
345
350
|
- "skipped": scenario is irrelevant to the app, OR systematic infrastructure failures (LLM errors,
|
|
346
351
|
crashes) prevented testing. NOT for "test failed to interact" — that's "fail" or "continue".
|
|
@@ -374,6 +379,10 @@ export class Pilot {
|
|
|
374
379
|
|
|
375
380
|
FIRST: Decide if precondition() is needed.
|
|
376
381
|
|
|
382
|
+
${capabilityGroundingRule}
|
|
383
|
+
|
|
384
|
+
${dataProtectionRules}
|
|
385
|
+
|
|
377
386
|
Call precondition() WHEN:
|
|
378
387
|
- The scenario edits/deletes/modifies an item, and you want a DISPOSABLE item to act on safely
|
|
379
388
|
- The scenario needs specific data clearly NOT on the current page (e.g., items with specific statuses for filtering)
|
|
@@ -498,7 +507,7 @@ export class Pilot {
|
|
|
498
507
|
maxToolRoundtrips: opts.maxToolRoundtrips ?? 0,
|
|
499
508
|
agentName: 'pilot',
|
|
500
509
|
stopWhen: opts.task ? () => opts.task.hasFinished : undefined,
|
|
501
|
-
|
|
510
|
+
telemetry: { functionId },
|
|
502
511
|
});
|
|
503
512
|
const text = result?.response?.text || '';
|
|
504
513
|
const learned = (result?.toolExecutions || []).filter((e) => e.toolName === 'learnExperience' && e.output?.content).map((e) => e.output.content);
|
|
@@ -930,9 +939,13 @@ export class Pilot {
|
|
|
930
939
|
|
|
931
940
|
Tester tools: click, pressKey, form, see, verify, context, research, xpathCheck, visualClick,
|
|
932
941
|
back, getVisitedStates, reset, stop, finish, record.
|
|
942
|
+
Use tool names exactly as listed. Do not invent combined names, aliases, or names with channel markers such as "commentary".
|
|
943
|
+
|
|
944
|
+
${capabilityGroundingRule}
|
|
933
945
|
|
|
934
946
|
YOUR Pilot-only tool: precondition(description) — create FRESH disposable test data via API. Never
|
|
935
947
|
request users. Use when:
|
|
948
|
+
|
|
936
949
|
- Scenario edits/deletes/modifies an item → create a disposable target ("1 post").
|
|
937
950
|
- Scenario needs auxiliary data (labels, categories, statuses for filtering).
|
|
938
951
|
- Tester failed because required data is missing (empty dropdown, empty list).
|
|
@@ -942,6 +955,8 @@ export class Pilot {
|
|
|
942
955
|
- Current page already shows the exact data needed.
|
|
943
956
|
- Scenario tests navigation, search UI, or viewing.
|
|
944
957
|
|
|
958
|
+
${dataProtectionRules}
|
|
959
|
+
|
|
945
960
|
Describe WHAT to create, not what exists. RIGHT: precondition("1 test"). WRONG:
|
|
946
961
|
precondition("1 test suite named Updated Suite with existing tests"). Keep descriptions short.
|
|
947
962
|
|
package/dist/src/ai/planner.js
CHANGED
|
@@ -19,7 +19,7 @@ import { WithSubPages, getPlannedByStateHash, getRegisteredPlan, registerPlan }
|
|
|
19
19
|
import { POSSIBLE_SECTIONS, Researcher } from "./researcher.js";
|
|
20
20
|
import { findSimilarStateHash } from "./researcher/cache.js";
|
|
21
21
|
import { hasFocusedSection } from "./researcher/focus.js";
|
|
22
|
-
import {
|
|
22
|
+
import { capabilityGroundingRule, dataProtectionRules, fileUploadRule } from "./rules.js";
|
|
23
23
|
const debugLog = createDebug('explorbot:planner');
|
|
24
24
|
const TasksSchema = z.object({
|
|
25
25
|
planName: z.string().describe('Short descriptive name for the test plan (e.g., "User Authentication Testing", "Product Catalog Navigation", "Form Validation Tests")'),
|
|
@@ -27,7 +27,7 @@ const TasksSchema = z.object({
|
|
|
27
27
|
.array(z.object({
|
|
28
28
|
scenario: z.string().describe('A single sentence describing what to test'),
|
|
29
29
|
priority: z.enum(['critical', 'important', 'high', 'normal', 'low']).describe('Priority of the task based on business importance'),
|
|
30
|
-
startUrl: z.string().nullable().describe('Start URL for the test if different from plan URL
|
|
30
|
+
startUrl: z.string().nullable().describe('Start URL for the test if different from plan URL. Use only stable feature/list/detail pages, not transient create/edit/modal URLs unless the scenario specifically starts inside that form.'),
|
|
31
31
|
steps: z.array(z.string()).describe('List of steps to perform for this scenario. Each step should be a specific action (e.g., "Open the form", "Enter required data", "Submit the form"). Keep steps atomic and actionable.'),
|
|
32
32
|
expectedOutcomes: z
|
|
33
33
|
.array(z.string())
|
|
@@ -73,6 +73,9 @@ export class Planner extends PlannerBase {
|
|
|
73
73
|
const featureDirective = feature
|
|
74
74
|
? `\n IMPORTANT: The user requested to focus specifically on: "${feature}"\n ALL scenarios MUST be directly related to this feature. Do not propose generic page tests unrelated to it.\n Use the user's exact wording to guide scenario names — do not substitute different entities (e.g., do not plan "suite" actions when user said "test").`
|
|
75
75
|
: '';
|
|
76
|
+
const focusExistingDataDirective = feature
|
|
77
|
+
? '\n If this focus asks for search, filter, tabs, sorting, or list behavior involving existing items, only use item names/values visible in the provided page research. If no concrete visible item names/values are present, do NOT propose scenarios that require an existing known item; propose no-match search, empty-state, clear-search, tab/filter empty-list, or other read-only list behavior instead.'
|
|
78
|
+
: '';
|
|
76
79
|
return dedent `
|
|
77
80
|
<role>
|
|
78
81
|
You are ISTQB certified senior manual QA planning exploratory testing session of a web application.
|
|
@@ -96,7 +99,7 @@ export class Planner extends PlannerBase {
|
|
|
96
99
|
Bad: "Open delete dropdown" + "Confirm deletion" — these are ONE test, not two.
|
|
97
100
|
Bad: "Search for X" + "Verify search results" — searching and verifying is ONE test.
|
|
98
101
|
Bad: "Leave field empty" + "Click submit" — that's one negative test, not two.
|
|
99
|
-
If two scenarios cannot run independently (one requires the other to run first), merge them into one.${featureDirective}
|
|
102
|
+
If two scenarios cannot run independently (one requires the other to run first), merge them into one.${featureDirective}${focusExistingDataDirective}
|
|
100
103
|
</task>
|
|
101
104
|
|
|
102
105
|
${customPrompt || ''}
|
|
@@ -301,6 +304,11 @@ export class Planner extends PlannerBase {
|
|
|
301
304
|
If a scenario needs existing records, recipients, results, notifications, or other target data, propose it only when that data is visible or API preconditions can create it.
|
|
302
305
|
If the page appears read-only, degraded, demo-limited, maintenance-like, or lacks write controls, prefer read-only scenarios such as opening panels, inspecting visible lists, filtering, searching, or verifying current state.
|
|
303
306
|
Do not assume hidden data exists just because a control is present.
|
|
307
|
+
For scenarios that act on existing items or search/filter by existing values, use only item names or values visible in research, visited pages, or prior observed flows.
|
|
308
|
+
If the list is empty or no concrete item names are visible, do not invent "known" or "existing" items. Prefer empty-state, no-match search, clear-search, or read-only list behavior scenarios.
|
|
309
|
+
Search, filter, sorting, tab, and list scenarios must start from a stable page where those controls are visible; avoid transient create/edit/new URLs unless the scenario tests that form.
|
|
310
|
+
For option values and list items, use only visible or previously observed data; do not add create/update/delete setup unless the user explicitly requests that workflow.
|
|
311
|
+
Detail-view scenarios must target visible data entities from list rows, cards, tree nodes, or detail links; do not use filter tabs, counters, status tabs, breadcrumbs, or navigation controls as detail targets.
|
|
304
312
|
DO NOT propose "verification-only" tests that merely open a UI element (modal, dropdown, panel) and check it exists.
|
|
305
313
|
Every test must complete a meaningful action that changes application state or produces a business outcome.
|
|
306
314
|
Opening a modal is NOT a test — performing an action INSIDE the modal IS a test.
|
|
@@ -311,7 +319,8 @@ export class Planner extends PlannerBase {
|
|
|
311
319
|
Tests that only switch views, toggle filters, or paginate are LESS valuable — propose them only after data-changing tests are covered.
|
|
312
320
|
If multiple ways to create or modify data exist (different types, different forms), propose a separate test for each.
|
|
313
321
|
</priority_order>
|
|
314
|
-
${
|
|
322
|
+
${capabilityGroundingRule}
|
|
323
|
+
${dataProtectionRules}
|
|
315
324
|
${fileUploadRule}
|
|
316
325
|
</rules>
|
|
317
326
|
|
|
@@ -466,7 +475,9 @@ export class Planner extends PlannerBase {
|
|
|
466
475
|
.join('\n')}
|
|
467
476
|
|
|
468
477
|
You MAY propose tests starting from these pages if they are relevant to the plan "${this.currentPlan.title}".
|
|
469
|
-
Set startUrl for such tests
|
|
478
|
+
Set startUrl for such tests only when the page is a stable feature/list/detail page.
|
|
479
|
+
Do not use create/edit/new/modal URLs as startUrl for scenarios that need the underlying page.
|
|
480
|
+
Ignore pages that belong to a different feature area.
|
|
470
481
|
</context_from_previous_tests>
|
|
471
482
|
|
|
472
483
|
Propose ONLY new scenarios that are NOT in the existing tests list.
|
package/dist/src/ai/provider.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
+
import { OpenTelemetry } from '@ai-sdk/otel';
|
|
1
2
|
import { LangfuseSpanProcessor } from '@langfuse/otel';
|
|
2
3
|
import { NodeSDK } from '@opentelemetry/sdk-node';
|
|
3
|
-
import { generateObject, generateText,
|
|
4
|
+
import { generateObject, generateText, isStepCount, registerTelemetry } from 'ai';
|
|
4
5
|
import { clearActivity, setActivity } from "../activity.js";
|
|
5
6
|
import { executionController } from "../execution-controller.js";
|
|
6
7
|
import { Observability } from "../observability.js";
|
|
@@ -16,10 +17,11 @@ class AiError extends Error {
|
|
|
16
17
|
}
|
|
17
18
|
export class ContextLengthError extends Error {
|
|
18
19
|
}
|
|
20
|
+
let telemetryRegistered = false;
|
|
19
21
|
function extractCachedTokens(usage) {
|
|
20
22
|
if (!usage)
|
|
21
23
|
return 0;
|
|
22
|
-
const direct = usage.
|
|
24
|
+
const direct = usage.inputTokenDetails?.cacheReadTokens ?? usage.cachedInputTokens;
|
|
23
25
|
if (typeof direct === 'number')
|
|
24
26
|
return direct;
|
|
25
27
|
const raw = usage.raw;
|
|
@@ -78,7 +80,7 @@ export class Provider {
|
|
|
78
80
|
await generateText({
|
|
79
81
|
model: this.config.model,
|
|
80
82
|
prompt: 'hi',
|
|
81
|
-
|
|
83
|
+
maxOutputTokens: 1,
|
|
82
84
|
});
|
|
83
85
|
}
|
|
84
86
|
catch (error) {
|
|
@@ -100,6 +102,18 @@ export class Provider {
|
|
|
100
102
|
}
|
|
101
103
|
return this.config.agenticModel || this.config.model;
|
|
102
104
|
}
|
|
105
|
+
getConfiguredModels() {
|
|
106
|
+
const models = { model: this.getModelName(this.config.model) };
|
|
107
|
+
if (this.config.agenticModel)
|
|
108
|
+
models.agenticModel = this.getModelName(this.config.agenticModel);
|
|
109
|
+
if (this.config.visionModel)
|
|
110
|
+
models.visionModel = this.getModelName(this.config.visionModel);
|
|
111
|
+
for (const [agent, agentConfig] of Object.entries(this.config.agents || {})) {
|
|
112
|
+
if (agentConfig?.model)
|
|
113
|
+
models[agent] = this.getModelName(agentConfig.model);
|
|
114
|
+
}
|
|
115
|
+
return models;
|
|
116
|
+
}
|
|
103
117
|
getSystemPromptForAgent(agentName, currentUrl) {
|
|
104
118
|
const agentConfig = this.config.agents?.[agentName];
|
|
105
119
|
const parts = [];
|
|
@@ -116,6 +130,12 @@ export class Provider {
|
|
|
116
130
|
const agentConfig = this.config.agents?.[agentName];
|
|
117
131
|
return agentConfig?.providerOptions;
|
|
118
132
|
}
|
|
133
|
+
getReasoningForAgent(agentName) {
|
|
134
|
+
if (!agentName)
|
|
135
|
+
return undefined;
|
|
136
|
+
const agentConfig = this.config.agents?.[agentName];
|
|
137
|
+
return agentConfig?.reasoning;
|
|
138
|
+
}
|
|
119
139
|
getRetryOptions(options = {}) {
|
|
120
140
|
return {
|
|
121
141
|
...this.defaultRetryOptions,
|
|
@@ -133,6 +153,13 @@ export class Provider {
|
|
|
133
153
|
providerOptions: { ...config.providerOptions, ...agentOptions },
|
|
134
154
|
};
|
|
135
155
|
}
|
|
156
|
+
finalizeConfig(config, options, telemetry) {
|
|
157
|
+
if (telemetry)
|
|
158
|
+
config.telemetry = telemetry;
|
|
159
|
+
const reasoning = this.getReasoningForAgent(options.agentName);
|
|
160
|
+
if (reasoning)
|
|
161
|
+
config.reasoning ??= reasoning;
|
|
162
|
+
}
|
|
136
163
|
initLangfuse() {
|
|
137
164
|
const langfuseConfig = this.config.langfuse;
|
|
138
165
|
const publicKey = langfuseConfig?.publicKey || process.env.LANGFUSE_PUBLIC_KEY;
|
|
@@ -152,6 +179,10 @@ export class Provider {
|
|
|
152
179
|
instrumentations: [],
|
|
153
180
|
});
|
|
154
181
|
void this.otelSdk.start();
|
|
182
|
+
if (!telemetryRegistered) {
|
|
183
|
+
registerTelemetry(new OpenTelemetry());
|
|
184
|
+
telemetryRegistered = true;
|
|
185
|
+
}
|
|
155
186
|
this.telemetryEnabled = true;
|
|
156
187
|
}
|
|
157
188
|
getTelemetry(options) {
|
|
@@ -159,18 +190,18 @@ export class Provider {
|
|
|
159
190
|
return undefined;
|
|
160
191
|
}
|
|
161
192
|
const runTelemetry = Observability.getTelemetry();
|
|
162
|
-
if (!options.
|
|
163
|
-
return runTelemetry
|
|
193
|
+
if (!options.telemetry) {
|
|
194
|
+
return runTelemetry;
|
|
164
195
|
}
|
|
165
196
|
if (!runTelemetry) {
|
|
166
|
-
return options.
|
|
197
|
+
return options.telemetry;
|
|
167
198
|
}
|
|
168
199
|
return {
|
|
169
200
|
...runTelemetry,
|
|
170
|
-
...options.
|
|
201
|
+
...options.telemetry,
|
|
171
202
|
metadata: {
|
|
172
203
|
...runTelemetry.metadata,
|
|
173
|
-
...options.
|
|
204
|
+
...options.telemetry.metadata,
|
|
174
205
|
},
|
|
175
206
|
};
|
|
176
207
|
}
|
|
@@ -210,13 +241,14 @@ export class Provider {
|
|
|
210
241
|
promptLog(`Using model: ${modelName}`);
|
|
211
242
|
const telemetry = this.getTelemetry(options);
|
|
212
243
|
const config = this.mergeProviderOptions({
|
|
213
|
-
|
|
244
|
+
maxOutputTokens: 16384,
|
|
245
|
+
allowSystemInMessages: true,
|
|
214
246
|
...(this.config.config || {}),
|
|
215
247
|
...options,
|
|
216
|
-
...(telemetry ? { experimental_telemetry: telemetry } : {}),
|
|
217
248
|
model,
|
|
218
249
|
abortSignal: executionController.getAbortSignal(),
|
|
219
250
|
}, options.agentName);
|
|
251
|
+
this.finalizeConfig(config, options, telemetry);
|
|
220
252
|
promptLog(messages[messages.length - 1].content);
|
|
221
253
|
try {
|
|
222
254
|
const response = await withRetry(async () => {
|
|
@@ -224,7 +256,7 @@ export class Provider {
|
|
|
224
256
|
if (!result.text) {
|
|
225
257
|
debugLog(result);
|
|
226
258
|
if (result.finishReason === 'length') {
|
|
227
|
-
throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase
|
|
259
|
+
throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase maxOutputTokens in config or use a model with higher output capacity.');
|
|
228
260
|
}
|
|
229
261
|
throw new Error('No response text from AI');
|
|
230
262
|
}
|
|
@@ -277,21 +309,22 @@ export class Provider {
|
|
|
277
309
|
const telemetry = this.getTelemetry(options);
|
|
278
310
|
const maxRoundtrips = options.maxToolRoundtrips ?? 5;
|
|
279
311
|
const extraStop = options.stopWhen;
|
|
280
|
-
const stopConditions = [
|
|
312
|
+
const stopConditions = [isStepCount(maxRoundtrips)];
|
|
281
313
|
if (extraStop)
|
|
282
314
|
stopConditions.push(extraStop);
|
|
283
315
|
const { stopWhen: _ignoredStopWhen, ...optionsWithoutStop } = options;
|
|
284
316
|
const config = this.mergeProviderOptions({
|
|
285
317
|
tools,
|
|
286
|
-
|
|
318
|
+
maxOutputTokens: 16384,
|
|
287
319
|
toolChoice: 'auto',
|
|
320
|
+
allowSystemInMessages: true,
|
|
288
321
|
...(this.config.config || {}),
|
|
289
322
|
...optionsWithoutStop,
|
|
290
323
|
stopWhen: stopConditions,
|
|
291
|
-
...(telemetry ? { experimental_telemetry: telemetry } : {}),
|
|
292
324
|
model,
|
|
293
325
|
abortSignal: executionController.getAbortSignal(),
|
|
294
326
|
}, options.agentName);
|
|
327
|
+
this.finalizeConfig(config, options, telemetry);
|
|
295
328
|
try {
|
|
296
329
|
const response = await withRetry(async () => {
|
|
297
330
|
const timeout = config.timeout || 30000;
|
|
@@ -306,7 +339,7 @@ export class Provider {
|
|
|
306
339
|
]));
|
|
307
340
|
const hasToolCall = (result.toolCalls?.length || 0) > 0;
|
|
308
341
|
if (!result.text && !hasToolCall && result.finishReason === 'length') {
|
|
309
|
-
throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase
|
|
342
|
+
throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase maxOutputTokens in config or use a model with higher output capacity.');
|
|
310
343
|
}
|
|
311
344
|
return result;
|
|
312
345
|
}
|
|
@@ -365,12 +398,13 @@ export class Provider {
|
|
|
365
398
|
const telemetry = this.getTelemetry(options);
|
|
366
399
|
const config = this.mergeProviderOptions({
|
|
367
400
|
schema,
|
|
401
|
+
allowSystemInMessages: true,
|
|
368
402
|
...(this.config.config || {}),
|
|
369
403
|
...options,
|
|
370
|
-
...(telemetry ? { experimental_telemetry: telemetry } : {}),
|
|
371
404
|
model: modelToUse,
|
|
372
405
|
abortSignal: executionController.getAbortSignal(),
|
|
373
406
|
}, options.agentName);
|
|
407
|
+
this.finalizeConfig(config, options, telemetry);
|
|
374
408
|
try {
|
|
375
409
|
promptLog(messages[messages.length - 1].content);
|
|
376
410
|
const response = await withRetry(async () => {
|
|
@@ -552,12 +586,13 @@ export class Provider {
|
|
|
552
586
|
];
|
|
553
587
|
const telemetry = this.getTelemetry({});
|
|
554
588
|
const config = {
|
|
555
|
-
|
|
589
|
+
maxOutputTokens: 16384,
|
|
556
590
|
...(this.config.config || {}),
|
|
557
|
-
...(telemetry ? { experimental_telemetry: telemetry } : {}),
|
|
558
591
|
model: this.config.visionModel,
|
|
559
592
|
abortSignal: executionController.getAbortSignal(),
|
|
560
593
|
};
|
|
594
|
+
if (telemetry)
|
|
595
|
+
config.telemetry = telemetry;
|
|
561
596
|
try {
|
|
562
597
|
promptLog(`Processing image with prompt: ${prompt}`);
|
|
563
598
|
const response = await withRetry(async () => {
|
|
@@ -47,6 +47,12 @@ export class Researcher extends ResearcherBase {
|
|
|
47
47
|
this.stateManager = explorer.getStateManager();
|
|
48
48
|
this.experienceTracker = this.stateManager.getExperienceTracker();
|
|
49
49
|
this.hooksRunner = new HooksRunner(explorer, explorer.getConfig());
|
|
50
|
+
const ai = explorer.getConfig().ai;
|
|
51
|
+
if (ai) {
|
|
52
|
+
ai.agents ??= {};
|
|
53
|
+
ai.agents.researcher ??= {};
|
|
54
|
+
ai.agents.researcher.reasoning ??= 'low';
|
|
55
|
+
}
|
|
50
56
|
}
|
|
51
57
|
getNavigator() {
|
|
52
58
|
throw new Error('not implemented');
|
|
@@ -139,7 +145,7 @@ export class Researcher extends ResearcherBase {
|
|
|
139
145
|
catch (error) {
|
|
140
146
|
if (!(error instanceof ContextLengthError) || retriesLeft <= 0) {
|
|
141
147
|
if (error instanceof ContextLengthError) {
|
|
142
|
-
tag('warning').log('Output truncated. Try lowering reasoning effort or increasing
|
|
148
|
+
tag('warning').log('Output truncated. Try lowering reasoning effort or increasing maxOutputTokens in ai.config.');
|
|
143
149
|
}
|
|
144
150
|
throw error;
|
|
145
151
|
}
|
package/dist/src/ai/rules.js
CHANGED
|
@@ -147,6 +147,50 @@ export const protectionRule = dedent `
|
|
|
147
147
|
Do not propose or perform delete/remove/archive actions on the entity that owns the current URL; propose such actions only on disposable children created within the scenario itself.
|
|
148
148
|
</important>
|
|
149
149
|
`;
|
|
150
|
+
export const dataProtectionRules = dedent `
|
|
151
|
+
<data_protection_rules>
|
|
152
|
+
${protectionRule}
|
|
153
|
+
|
|
154
|
+
If the user request, scenario, focus, or test instructions explicitly prohibit creating,
|
|
155
|
+
editing, updating, deleting, removing, or otherwise mutating data, do not perform those
|
|
156
|
+
actions through the UI, API preconditions, cleanup, fallback steps, or Fisherman.
|
|
157
|
+
|
|
158
|
+
Do not use Fisherman or API data preparation to bypass a no-mutation, read-only, search,
|
|
159
|
+
filter, tab, or list-inspection constraint. Use visible existing data when it is available.
|
|
160
|
+
If no suitable data exists, report the missing precondition instead of creating data.
|
|
161
|
+
|
|
162
|
+
Destructive actions are allowed only against disposable data created by the current scenario
|
|
163
|
+
or prepared for that scenario by Fisherman/API preconditions. Existing application data must
|
|
164
|
+
remain unchanged.
|
|
165
|
+
</data_protection_rules>
|
|
166
|
+
`;
|
|
167
|
+
export const capabilityGroundingRule = dedent `
|
|
168
|
+
<capability_grounding>
|
|
169
|
+
When a scenario depends on a named action, menu item, status, option, workflow, or feature,
|
|
170
|
+
that capability must be visible or explicitly confirmed in the current research/page context
|
|
171
|
+
for the same target entity type.
|
|
172
|
+
|
|
173
|
+
Do not transfer capabilities between similar entities, rows, lists, detail pages, or menus.
|
|
174
|
+
Do not replace a requested action with a synonym or related action unless the UI explicitly
|
|
175
|
+
shows that action for the target entity.
|
|
176
|
+
|
|
177
|
+
When an action is described as applying to an item, row, card, record, node, or entity,
|
|
178
|
+
the target must be grounded as that kind of data entity in the current context. Do not use
|
|
179
|
+
navigation links, filter tabs, counters, breadcrumbs, headings, toolbar controls, or other
|
|
180
|
+
page controls as the subject of row/entity actions.
|
|
181
|
+
|
|
182
|
+
When a scenario asks to open, view, inspect, or navigate to an entity detail view, success
|
|
183
|
+
requires evidence of that entity detail context. An active filter, selected tab, visible count,
|
|
184
|
+
or filtered list is not enough to prove an entity detail view opened.
|
|
185
|
+
|
|
186
|
+
Do not rewrite a scenario goal to match a similar outcome that happened accidentally. If the
|
|
187
|
+
requested entity detail/action/workflow was not achieved, report that mismatch instead of
|
|
188
|
+
passing the test for a related filter, tab, navigation, or status view.
|
|
189
|
+
|
|
190
|
+
If the required capability is not available for the target entity after reasonable discovery,
|
|
191
|
+
record the missing capability and stop instead of repeatedly trying unrelated locators.
|
|
192
|
+
</capability_grounding>
|
|
193
|
+
`;
|
|
150
194
|
export const focusedElementRule = dedent `
|
|
151
195
|
<focused_element_actions>
|
|
152
196
|
When a text input element is focused (textbox, combobox, contenteditable):
|
package/dist/src/ai/tester.js
CHANGED
|
@@ -14,7 +14,7 @@ import { ErrorPageError, isErrorPage } from "../utils/error-page.js";
|
|
|
14
14
|
import { HooksRunner } from "../utils/hooks-runner.js";
|
|
15
15
|
import { createDebug, tag } from "../utils/logger.js";
|
|
16
16
|
import { loop } from "../utils/loop.js";
|
|
17
|
-
import { actionRule, focusedElementRule, formRequirementsRule, locatorRule, multipleTabsRule,
|
|
17
|
+
import { actionRule, capabilityGroundingRule, dataProtectionRules, focusedElementRule, formRequirementsRule, locatorRule, multipleTabsRule, sectionContextRule } from "./rules.js";
|
|
18
18
|
import { TaskAgent } from "./task-agent.js";
|
|
19
19
|
import { createCodeceptJSTools, createSpecialContextTools } from "./tools.js";
|
|
20
20
|
const debugLog = createDebug('explorbot:tester');
|
|
@@ -29,7 +29,7 @@ const SAMPLE_FILES = {
|
|
|
29
29
|
'MP3 audio': 'sample.mp3',
|
|
30
30
|
};
|
|
31
31
|
export class Tester extends TaskAgent {
|
|
32
|
-
ACTION_TOOLS = ['click', 'pressKey', 'form'];
|
|
32
|
+
ACTION_TOOLS = ['click', 'hover', 'pressKey', 'form'];
|
|
33
33
|
SPECIAL_CONTEXT_ACTION_TOOLS = ['exitIframe'];
|
|
34
34
|
emoji = '🧪';
|
|
35
35
|
explorer;
|
|
@@ -51,6 +51,8 @@ export class Tester extends TaskAgent {
|
|
|
51
51
|
hooksRunner;
|
|
52
52
|
seenUiMapUrls = new Set();
|
|
53
53
|
lastAnalyzedStateHash = null;
|
|
54
|
+
stalledIterations = 0;
|
|
55
|
+
MAX_STALLED_ITERATIONS = 3;
|
|
54
56
|
constructor(explorer, provider, researcher, navigator, agentTools) {
|
|
55
57
|
super();
|
|
56
58
|
this.explorer = explorer;
|
|
@@ -99,6 +101,7 @@ export class Tester extends TaskAgent {
|
|
|
99
101
|
this.pageActionResult = null;
|
|
100
102
|
this.seenUiMapUrls.clear();
|
|
101
103
|
this.lastAnalyzedStateHash = null;
|
|
104
|
+
this.stalledIterations = 0;
|
|
102
105
|
this.explorer.getStateManager().clearHistory();
|
|
103
106
|
this.resetFailureCount();
|
|
104
107
|
this.pilot?.reset();
|
|
@@ -293,6 +296,10 @@ export class Tester extends TaskAgent {
|
|
|
293
296
|
task.addNote(`Failed to ${execution.input.explanation} (${execution.toolName})`, TestResult.FAILED);
|
|
294
297
|
});
|
|
295
298
|
}
|
|
299
|
+
if (this.shouldStopForStalledExecution(task, currentState, result?.toolExecutions || [])) {
|
|
300
|
+
stop();
|
|
301
|
+
return;
|
|
302
|
+
}
|
|
296
303
|
if (assertionPerformed) {
|
|
297
304
|
const message = result?.toolExecutions?.find((execution) => execution.toolName === 'verify')?.output?.message || '';
|
|
298
305
|
task.addNote(message, wasSuccessful ? TestResult.PASSED : TestResult.FAILED);
|
|
@@ -395,6 +402,28 @@ export class Tester extends TaskAgent {
|
|
|
395
402
|
return false;
|
|
396
403
|
return true;
|
|
397
404
|
}
|
|
405
|
+
shouldStopForStalledExecution(task, previousState, toolExecutions) {
|
|
406
|
+
if (task.hasFinished)
|
|
407
|
+
return false;
|
|
408
|
+
const currentState = this.getCurrentState();
|
|
409
|
+
const stateChanged = previousState.url !== currentState.url || previousState.hash !== currentState.hash;
|
|
410
|
+
const actionTools = [...this.ACTION_TOOLS, ...this.SPECIAL_CONTEXT_ACTION_TOOLS];
|
|
411
|
+
const hasSuccessfulAction = toolExecutions.some((execution) => execution.wasSuccessful && actionTools.includes(execution.toolName));
|
|
412
|
+
const hasSuccessfulAssertion = toolExecutions.some((execution) => execution.wasSuccessful && this.ASSERTION_TOOLS.includes(execution.toolName));
|
|
413
|
+
if (stateChanged || hasSuccessfulAction || hasSuccessfulAssertion) {
|
|
414
|
+
this.stalledIterations = 0;
|
|
415
|
+
return false;
|
|
416
|
+
}
|
|
417
|
+
const hasNoBrowserProgress = toolExecutions.length === 0 || toolExecutions.every((execution) => !actionTools.includes(execution.toolName) || !execution.wasSuccessful);
|
|
418
|
+
if (!hasNoBrowserProgress)
|
|
419
|
+
return false;
|
|
420
|
+
this.stalledIterations++;
|
|
421
|
+
if (this.stalledIterations < this.MAX_STALLED_ITERATIONS)
|
|
422
|
+
return false;
|
|
423
|
+
task.addNote('No browser progress after repeated attempts on unchanged page', TestResult.FAILED);
|
|
424
|
+
task.finish(TestResult.FAILED);
|
|
425
|
+
return true;
|
|
426
|
+
}
|
|
398
427
|
async prepareInstructionsForNextStep(task) {
|
|
399
428
|
let outcomeStatus = dedent `
|
|
400
429
|
<task>
|
|
@@ -403,6 +432,8 @@ export class Tester extends TaskAgent {
|
|
|
403
432
|
|
|
404
433
|
<rules>
|
|
405
434
|
Use tools ${this.ACTION_TOOLS.join(', ')} to interact with the page.
|
|
435
|
+
Use tool names exactly as listed in this prompt. Do not invent combined tool names, aliases, or names with channel markers such as "commentary".
|
|
436
|
+
Match each tool input schema exactly. Do not invent parameter names or pass extra fields.
|
|
406
437
|
Do not do unsuccesful clicks again.
|
|
407
438
|
Do not run same tool calls with same parameters again.
|
|
408
439
|
</rules>
|
|
@@ -665,6 +696,7 @@ export class Tester extends TaskAgent {
|
|
|
665
696
|
<rules>
|
|
666
697
|
- Refer to UI Map from <page_ui_map> to understand the page structure and its main elements
|
|
667
698
|
- Use only elements that exist in the provided ARIA tree or HTML, <page_aria> and <page_html>
|
|
699
|
+
- Use tool input schemas exactly as documented. Do not invent parameter names or add fields not listed by the tool schema.
|
|
668
700
|
- Use click() for buttons, links, and clickable elements ONLY - do NOT include I.fillField() or I.type() commands in click() tool
|
|
669
701
|
- click() commands array is for FALLBACK LOCATORS of the SAME element, NOT for clicking different elements in sequence. If you need to click two different elements, make two separate click() calls.
|
|
670
702
|
- Use form() for text input (I.fillField, I.type), dropdown selection (I.selectOption), file uploads (I.attachFile), and multi-step form interactions
|
|
@@ -677,6 +709,7 @@ export class Tester extends TaskAgent {
|
|
|
677
709
|
- NEVER call record(status: "success") if your last verify() or see() call FAILED. A failed check means the outcome is NOT confirmed — use record(status: "fail") instead, or retry with a different approach.
|
|
678
710
|
- Use finish() to complete the test, not record(). record() is for intermediate notes.
|
|
679
711
|
- Call finish(verify) when all goals are achieved — provide an assertion to verify
|
|
712
|
+
- NEVER call finish() with a negative assertion that says the goal did NOT happen. If the goal cannot be achieved after real attempts, record the blocker and call stop().
|
|
680
713
|
- ONLY call stop() if the scenario itself is completely irrelevant to this page and no expectations can be achieved
|
|
681
714
|
- Use reset() ONLY as a last resort when the current page cannot host the scenario. Never reset after a successful flow just because an assertion or milestone did not match — verify differently or record() the finding instead. Reset is destructive and does not undo server-side side effects.
|
|
682
715
|
- Be precise with locators (CSS or XPath)
|
|
@@ -690,6 +723,13 @@ export class Tester extends TaskAgent {
|
|
|
690
723
|
- When you interact with form with inputs, ensure that you click corresponding button to save its data
|
|
691
724
|
- Follow <locator_priority> rules when selecting locators for all tools
|
|
692
725
|
- Before retrying your actions check maybe they already achived expected results. Use see() tool for that
|
|
726
|
+
- If the current URL is already a create/edit/new form and the scenario is about creating/editing that entity, fill and submit that form. Do not click the list-page "New" button again from inside the form.
|
|
727
|
+
- If the scenario is about search/filter/sort/tabs/list inspection and the current URL is a create/edit/new form, go back or reset to the stable list page before interacting with list controls.
|
|
728
|
+
- When selecting related entities from a list, do not choose rows/options/cards marked as "0 items", "0 tests", or otherwise empty if the scenario requires selecting real content.
|
|
729
|
+
- In selection pickers, counters such as "Selected 0", "Matched tests 0", or disabled Save/Apply mean the selection did not register. Choose a non-empty item or change filters before submitting.
|
|
730
|
+
- A passed form/click command only means the command executed. If a required field remains empty, submit stays disabled, or the expected text is not visible, treat the action as not completed and correct the missing field/state.
|
|
731
|
+
- For filter/tab scenarios, success requires BOTH: the requested filter/tab is visibly active/selected AND the list content matches that filter. Do not finish from only one of these signals.
|
|
732
|
+
- Empty-state text such as "No matched items" only proves a filter when the requested filter/tab is active and the empty state belongs to the filtered list.
|
|
693
733
|
- When filling complex form with lot of actions performed, use see() to look which fields were filled and which are not
|
|
694
734
|
- When verify() fails, use see() to visually confirm the result — visual confirmation is equally valid evidence
|
|
695
735
|
- For visual state verification (active tabs, selected items, counts, colors), prefer see() over DOM-based verify()
|
|
@@ -718,6 +758,10 @@ export class Tester extends TaskAgent {
|
|
|
718
758
|
|
|
719
759
|
${formRequirementsRule}
|
|
720
760
|
|
|
761
|
+
${capabilityGroundingRule}
|
|
762
|
+
|
|
763
|
+
${dataProtectionRules}
|
|
764
|
+
|
|
721
765
|
${this.provider.getSystemPromptForAgent('tester', this.explorer.getStateManager().getCurrentState()?.url) || ''}
|
|
722
766
|
`;
|
|
723
767
|
}
|
|
@@ -739,10 +783,13 @@ export class Tester extends TaskAgent {
|
|
|
739
783
|
Try to achieve as many goals as possible.
|
|
740
784
|
If goal is not achievable, log that and skip to next one.
|
|
741
785
|
Do not hallucinate that goal was achieved when it was not.
|
|
786
|
+
If the scenario action could not be completed, do not finish with a verification of the failure state.
|
|
742
787
|
When creating or editing items via form() or type() you should include ${task.sessionName} in the value (if it is not restricted by the application logic)
|
|
743
788
|
Initial page URL: ${actionResult.url}
|
|
744
789
|
|
|
745
|
-
${
|
|
790
|
+
${capabilityGroundingRule}
|
|
791
|
+
|
|
792
|
+
${dataProtectionRules}
|
|
746
793
|
|
|
747
794
|
${this.buildDeletionScope(task)}
|
|
748
795
|
|
|
@@ -867,12 +914,13 @@ export class Tester extends TaskAgent {
|
|
|
867
914
|
}),
|
|
868
915
|
stop: tool({
|
|
869
916
|
description: dedent `
|
|
870
|
-
Stop the current test because
|
|
871
|
-
Use this
|
|
872
|
-
|
|
917
|
+
Stop the current test because it cannot be completed in the current session.
|
|
918
|
+
Use this when the scenario is incompatible, required UI/data is absent, or repeated varied attempts
|
|
919
|
+
show that automation cannot complete the workflow.
|
|
920
|
+
Do NOT use this immediately after the first failed action — retry with a materially different approach first.
|
|
873
921
|
`,
|
|
874
922
|
inputSchema: z.object({
|
|
875
|
-
reason: z.string().describe('Explanation why the scenario
|
|
923
|
+
reason: z.string().describe('Explanation why the scenario cannot be completed'),
|
|
876
924
|
}),
|
|
877
925
|
execute: async ({ reason }) => {
|
|
878
926
|
task.addNote(`Stop requested: ${reason}`);
|
|
@@ -906,6 +954,7 @@ export class Tester extends TaskAgent {
|
|
|
906
954
|
Provide a specific assertion to verify the final state.
|
|
907
955
|
The assertion MUST prove that YOUR ACTIONS changed the page state.
|
|
908
956
|
Do NOT verify something that was already true before you started testing.
|
|
957
|
+
Do NOT provide an assertion that verifies absence, failure, an empty state, or that the goal did not happen.
|
|
909
958
|
|
|
910
959
|
Examples of good assertions:
|
|
911
960
|
- "New user 'john@example.com' is visible in the users list"
|
|
@@ -1031,10 +1080,24 @@ export class Tester extends TaskAgent {
|
|
|
1031
1080
|
this.resetFailureCount();
|
|
1032
1081
|
this.previousUrl = null;
|
|
1033
1082
|
this.previousStateHash = null;
|
|
1083
|
+
this.stalledIterations = 0;
|
|
1084
|
+
}
|
|
1085
|
+
else if (this.shouldStopAfterStalledLoopError(task)) {
|
|
1086
|
+
return 'stop';
|
|
1034
1087
|
}
|
|
1035
1088
|
this.currentConversation?.addUserText(result.message);
|
|
1036
1089
|
return 'continue';
|
|
1037
1090
|
}
|
|
1091
|
+
shouldStopAfterStalledLoopError(task) {
|
|
1092
|
+
if (task.hasFinished)
|
|
1093
|
+
return false;
|
|
1094
|
+
this.stalledIterations++;
|
|
1095
|
+
if (this.stalledIterations < this.MAX_STALLED_ITERATIONS)
|
|
1096
|
+
return false;
|
|
1097
|
+
task.addNote('No browser progress after repeated execution errors', TestResult.FAILED);
|
|
1098
|
+
task.finish(TestResult.FAILED);
|
|
1099
|
+
return true;
|
|
1100
|
+
}
|
|
1038
1101
|
async cleanupStartedTest(task) {
|
|
1039
1102
|
await this.finishTest(task);
|
|
1040
1103
|
await this.explorer.stopTest(task, {
|