explorbot 0.1.28 → 0.1.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +83 -245
- package/bin/explorbot-cli.ts +1 -0
- package/boat/doc-collector/src/ai/documentarian.ts +37 -13
- package/boat/doc-collector/src/ai/tools.ts +60 -20
- package/boat/doc-collector/src/cli.ts +3 -0
- package/boat/doc-collector/src/config.ts +7 -0
- package/boat/doc-collector/src/docbot.ts +23 -5
- package/boat/doc-collector/src/docs-renderer.ts +14 -1
- package/boat/doc-collector/src/screenshots.ts +126 -0
- package/dist/bin/explorbot-cli.js +1 -0
- package/dist/boat/doc-collector/src/ai/documentarian.js +15 -11
- package/dist/boat/doc-collector/src/ai/tools.js +53 -20
- package/dist/boat/doc-collector/src/cli.js +3 -0
- package/dist/boat/doc-collector/src/config.js +3 -0
- package/dist/boat/doc-collector/src/docbot.js +19 -4
- package/dist/boat/doc-collector/src/docs-renderer.js +12 -1
- package/dist/boat/doc-collector/src/screenshots.js +90 -0
- package/dist/package.json +8 -6
- package/dist/rules/navigator/verification-actions.md +2 -0
- package/dist/src/action.js +26 -23
- package/dist/src/ai/fisherman.js +14 -3
- package/dist/src/ai/historian/codeceptjs.js +3 -2
- package/dist/src/ai/historian/experience.js +48 -6
- package/dist/src/ai/historian/playwright.js +2 -1
- package/dist/src/ai/historian/utils.js +1 -19
- package/dist/src/ai/historian.js +1 -1
- package/dist/src/ai/pilot.js +19 -4
- package/dist/src/ai/planner.js +16 -5
- package/dist/src/ai/provider.js +53 -18
- package/dist/src/ai/quartermaster.js +2 -2
- package/dist/src/ai/researcher.js +7 -1
- package/dist/src/ai/rules.js +44 -0
- package/dist/src/ai/tester.js +73 -7
- package/dist/src/ai/tools.js +66 -1
- package/dist/src/experience-tracker.js +1 -1
- package/dist/src/explorbot.js +14 -3
- package/dist/src/explorer.js +30 -27
- package/dist/src/stats.js +16 -0
- package/dist/src/utils/aria.js +66 -6
- package/dist/src/utils/browser-errors.js +5 -0
- package/dist/src/utils/page-readiness.js +48 -0
- package/dist/src/utils/step-analyzer.js +68 -0
- package/package.json +8 -6
- package/rules/navigator/verification-actions.md +2 -0
- package/src/action.ts +24 -26
- package/src/ai/fisherman.ts +14 -3
- package/src/ai/historian/codeceptjs.ts +3 -2
- package/src/ai/historian/experience.ts +51 -6
- package/src/ai/historian/playwright.ts +2 -1
- package/src/ai/historian/utils.ts +1 -21
- package/src/ai/historian.ts +1 -1
- package/src/ai/pilot.ts +19 -4
- package/src/ai/planner.ts +16 -5
- package/src/ai/provider.ts +51 -19
- package/src/ai/quartermaster.ts +2 -2
- package/src/ai/researcher.ts +8 -1
- package/src/ai/rules.ts +46 -0
- package/src/ai/tester.ts +77 -7
- package/src/ai/tools.ts +79 -1
- package/src/config.ts +2 -0
- package/src/experience-tracker.ts +1 -1
- package/src/explorbot.ts +13 -3
- package/src/explorer.ts +28 -27
- package/src/stats.ts +18 -0
- package/src/utils/aria.ts +63 -6
- package/src/utils/browser-errors.ts +6 -0
- package/src/utils/page-readiness.ts +59 -0
- package/src/utils/step-analyzer.ts +73 -0
package/dist/src/ai/planner.js
CHANGED
|
@@ -19,7 +19,7 @@ import { WithSubPages, getPlannedByStateHash, getRegisteredPlan, registerPlan }
|
|
|
19
19
|
import { POSSIBLE_SECTIONS, Researcher } from "./researcher.js";
|
|
20
20
|
import { findSimilarStateHash } from "./researcher/cache.js";
|
|
21
21
|
import { hasFocusedSection } from "./researcher/focus.js";
|
|
22
|
-
import {
|
|
22
|
+
import { capabilityGroundingRule, dataProtectionRules, fileUploadRule } from "./rules.js";
|
|
23
23
|
const debugLog = createDebug('explorbot:planner');
|
|
24
24
|
const TasksSchema = z.object({
|
|
25
25
|
planName: z.string().describe('Short descriptive name for the test plan (e.g., "User Authentication Testing", "Product Catalog Navigation", "Form Validation Tests")'),
|
|
@@ -27,7 +27,7 @@ const TasksSchema = z.object({
|
|
|
27
27
|
.array(z.object({
|
|
28
28
|
scenario: z.string().describe('A single sentence describing what to test'),
|
|
29
29
|
priority: z.enum(['critical', 'important', 'high', 'normal', 'low']).describe('Priority of the task based on business importance'),
|
|
30
|
-
startUrl: z.string().nullable().describe('Start URL for the test if different from plan URL
|
|
30
|
+
startUrl: z.string().nullable().describe('Start URL for the test if different from plan URL. Use only stable feature/list/detail pages, not transient create/edit/modal URLs unless the scenario specifically starts inside that form.'),
|
|
31
31
|
steps: z.array(z.string()).describe('List of steps to perform for this scenario. Each step should be a specific action (e.g., "Open the form", "Enter required data", "Submit the form"). Keep steps atomic and actionable.'),
|
|
32
32
|
expectedOutcomes: z
|
|
33
33
|
.array(z.string())
|
|
@@ -73,6 +73,9 @@ export class Planner extends PlannerBase {
|
|
|
73
73
|
const featureDirective = feature
|
|
74
74
|
? `\n IMPORTANT: The user requested to focus specifically on: "${feature}"\n ALL scenarios MUST be directly related to this feature. Do not propose generic page tests unrelated to it.\n Use the user's exact wording to guide scenario names — do not substitute different entities (e.g., do not plan "suite" actions when user said "test").`
|
|
75
75
|
: '';
|
|
76
|
+
const focusExistingDataDirective = feature
|
|
77
|
+
? '\n If this focus asks for search, filter, tabs, sorting, or list behavior involving existing items, only use item names/values visible in the provided page research. If no concrete visible item names/values are present, do NOT propose scenarios that require an existing known item; propose no-match search, empty-state, clear-search, tab/filter empty-list, or other read-only list behavior instead.'
|
|
78
|
+
: '';
|
|
76
79
|
return dedent `
|
|
77
80
|
<role>
|
|
78
81
|
You are ISTQB certified senior manual QA planning exploratory testing session of a web application.
|
|
@@ -96,7 +99,7 @@ export class Planner extends PlannerBase {
|
|
|
96
99
|
Bad: "Open delete dropdown" + "Confirm deletion" — these are ONE test, not two.
|
|
97
100
|
Bad: "Search for X" + "Verify search results" — searching and verifying is ONE test.
|
|
98
101
|
Bad: "Leave field empty" + "Click submit" — that's one negative test, not two.
|
|
99
|
-
If two scenarios cannot run independently (one requires the other to run first), merge them into one.${featureDirective}
|
|
102
|
+
If two scenarios cannot run independently (one requires the other to run first), merge them into one.${featureDirective}${focusExistingDataDirective}
|
|
100
103
|
</task>
|
|
101
104
|
|
|
102
105
|
${customPrompt || ''}
|
|
@@ -301,6 +304,11 @@ export class Planner extends PlannerBase {
|
|
|
301
304
|
If a scenario needs existing records, recipients, results, notifications, or other target data, propose it only when that data is visible or API preconditions can create it.
|
|
302
305
|
If the page appears read-only, degraded, demo-limited, maintenance-like, or lacks write controls, prefer read-only scenarios such as opening panels, inspecting visible lists, filtering, searching, or verifying current state.
|
|
303
306
|
Do not assume hidden data exists just because a control is present.
|
|
307
|
+
For scenarios that act on existing items or search/filter by existing values, use only item names or values visible in research, visited pages, or prior observed flows.
|
|
308
|
+
If the list is empty or no concrete item names are visible, do not invent "known" or "existing" items. Prefer empty-state, no-match search, clear-search, or read-only list behavior scenarios.
|
|
309
|
+
Search, filter, sorting, tab, and list scenarios must start from a stable page where those controls are visible; avoid transient create/edit/new URLs unless the scenario tests that form.
|
|
310
|
+
For option values and list items, use only visible or previously observed data; do not add create/update/delete setup unless the user explicitly requests that workflow.
|
|
311
|
+
Detail-view scenarios must target visible data entities from list rows, cards, tree nodes, or detail links; do not use filter tabs, counters, status tabs, breadcrumbs, or navigation controls as detail targets.
|
|
304
312
|
DO NOT propose "verification-only" tests that merely open a UI element (modal, dropdown, panel) and check it exists.
|
|
305
313
|
Every test must complete a meaningful action that changes application state or produces a business outcome.
|
|
306
314
|
Opening a modal is NOT a test — performing an action INSIDE the modal IS a test.
|
|
@@ -311,7 +319,8 @@ export class Planner extends PlannerBase {
|
|
|
311
319
|
Tests that only switch views, toggle filters, or paginate are LESS valuable — propose them only after data-changing tests are covered.
|
|
312
320
|
If multiple ways to create or modify data exist (different types, different forms), propose a separate test for each.
|
|
313
321
|
</priority_order>
|
|
314
|
-
${
|
|
322
|
+
${capabilityGroundingRule}
|
|
323
|
+
${dataProtectionRules}
|
|
315
324
|
${fileUploadRule}
|
|
316
325
|
</rules>
|
|
317
326
|
|
|
@@ -466,7 +475,9 @@ export class Planner extends PlannerBase {
|
|
|
466
475
|
.join('\n')}
|
|
467
476
|
|
|
468
477
|
You MAY propose tests starting from these pages if they are relevant to the plan "${this.currentPlan.title}".
|
|
469
|
-
Set startUrl for such tests
|
|
478
|
+
Set startUrl for such tests only when the page is a stable feature/list/detail page.
|
|
479
|
+
Do not use create/edit/new/modal URLs as startUrl for scenarios that need the underlying page.
|
|
480
|
+
Ignore pages that belong to a different feature area.
|
|
470
481
|
</context_from_previous_tests>
|
|
471
482
|
|
|
472
483
|
Propose ONLY new scenarios that are NOT in the existing tests list.
|
package/dist/src/ai/provider.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
+
import { OpenTelemetry } from '@ai-sdk/otel';
|
|
1
2
|
import { LangfuseSpanProcessor } from '@langfuse/otel';
|
|
2
3
|
import { NodeSDK } from '@opentelemetry/sdk-node';
|
|
3
|
-
import { generateObject, generateText,
|
|
4
|
+
import { generateObject, generateText, isStepCount, registerTelemetry } from 'ai';
|
|
4
5
|
import { clearActivity, setActivity } from "../activity.js";
|
|
5
6
|
import { executionController } from "../execution-controller.js";
|
|
6
7
|
import { Observability } from "../observability.js";
|
|
@@ -16,10 +17,11 @@ class AiError extends Error {
|
|
|
16
17
|
}
|
|
17
18
|
export class ContextLengthError extends Error {
|
|
18
19
|
}
|
|
20
|
+
let telemetryRegistered = false;
|
|
19
21
|
function extractCachedTokens(usage) {
|
|
20
22
|
if (!usage)
|
|
21
23
|
return 0;
|
|
22
|
-
const direct = usage.
|
|
24
|
+
const direct = usage.inputTokenDetails?.cacheReadTokens ?? usage.cachedInputTokens;
|
|
23
25
|
if (typeof direct === 'number')
|
|
24
26
|
return direct;
|
|
25
27
|
const raw = usage.raw;
|
|
@@ -78,7 +80,7 @@ export class Provider {
|
|
|
78
80
|
await generateText({
|
|
79
81
|
model: this.config.model,
|
|
80
82
|
prompt: 'hi',
|
|
81
|
-
|
|
83
|
+
maxOutputTokens: 1,
|
|
82
84
|
});
|
|
83
85
|
}
|
|
84
86
|
catch (error) {
|
|
@@ -100,6 +102,18 @@ export class Provider {
|
|
|
100
102
|
}
|
|
101
103
|
return this.config.agenticModel || this.config.model;
|
|
102
104
|
}
|
|
105
|
+
getConfiguredModels() {
|
|
106
|
+
const models = { model: this.getModelName(this.config.model) };
|
|
107
|
+
if (this.config.agenticModel)
|
|
108
|
+
models.agenticModel = this.getModelName(this.config.agenticModel);
|
|
109
|
+
if (this.config.visionModel)
|
|
110
|
+
models.visionModel = this.getModelName(this.config.visionModel);
|
|
111
|
+
for (const [agent, agentConfig] of Object.entries(this.config.agents || {})) {
|
|
112
|
+
if (agentConfig?.model)
|
|
113
|
+
models[agent] = this.getModelName(agentConfig.model);
|
|
114
|
+
}
|
|
115
|
+
return models;
|
|
116
|
+
}
|
|
103
117
|
getSystemPromptForAgent(agentName, currentUrl) {
|
|
104
118
|
const agentConfig = this.config.agents?.[agentName];
|
|
105
119
|
const parts = [];
|
|
@@ -116,6 +130,12 @@ export class Provider {
|
|
|
116
130
|
const agentConfig = this.config.agents?.[agentName];
|
|
117
131
|
return agentConfig?.providerOptions;
|
|
118
132
|
}
|
|
133
|
+
getReasoningForAgent(agentName) {
|
|
134
|
+
if (!agentName)
|
|
135
|
+
return undefined;
|
|
136
|
+
const agentConfig = this.config.agents?.[agentName];
|
|
137
|
+
return agentConfig?.reasoning;
|
|
138
|
+
}
|
|
119
139
|
getRetryOptions(options = {}) {
|
|
120
140
|
return {
|
|
121
141
|
...this.defaultRetryOptions,
|
|
@@ -133,6 +153,13 @@ export class Provider {
|
|
|
133
153
|
providerOptions: { ...config.providerOptions, ...agentOptions },
|
|
134
154
|
};
|
|
135
155
|
}
|
|
156
|
+
finalizeConfig(config, options, telemetry) {
|
|
157
|
+
if (telemetry)
|
|
158
|
+
config.telemetry = telemetry;
|
|
159
|
+
const reasoning = this.getReasoningForAgent(options.agentName);
|
|
160
|
+
if (reasoning)
|
|
161
|
+
config.reasoning ??= reasoning;
|
|
162
|
+
}
|
|
136
163
|
initLangfuse() {
|
|
137
164
|
const langfuseConfig = this.config.langfuse;
|
|
138
165
|
const publicKey = langfuseConfig?.publicKey || process.env.LANGFUSE_PUBLIC_KEY;
|
|
@@ -152,6 +179,10 @@ export class Provider {
|
|
|
152
179
|
instrumentations: [],
|
|
153
180
|
});
|
|
154
181
|
void this.otelSdk.start();
|
|
182
|
+
if (!telemetryRegistered) {
|
|
183
|
+
registerTelemetry(new OpenTelemetry());
|
|
184
|
+
telemetryRegistered = true;
|
|
185
|
+
}
|
|
155
186
|
this.telemetryEnabled = true;
|
|
156
187
|
}
|
|
157
188
|
getTelemetry(options) {
|
|
@@ -159,18 +190,18 @@ export class Provider {
|
|
|
159
190
|
return undefined;
|
|
160
191
|
}
|
|
161
192
|
const runTelemetry = Observability.getTelemetry();
|
|
162
|
-
if (!options.
|
|
163
|
-
return runTelemetry
|
|
193
|
+
if (!options.telemetry) {
|
|
194
|
+
return runTelemetry;
|
|
164
195
|
}
|
|
165
196
|
if (!runTelemetry) {
|
|
166
|
-
return options.
|
|
197
|
+
return options.telemetry;
|
|
167
198
|
}
|
|
168
199
|
return {
|
|
169
200
|
...runTelemetry,
|
|
170
|
-
...options.
|
|
201
|
+
...options.telemetry,
|
|
171
202
|
metadata: {
|
|
172
203
|
...runTelemetry.metadata,
|
|
173
|
-
...options.
|
|
204
|
+
...options.telemetry.metadata,
|
|
174
205
|
},
|
|
175
206
|
};
|
|
176
207
|
}
|
|
@@ -210,13 +241,14 @@ export class Provider {
|
|
|
210
241
|
promptLog(`Using model: ${modelName}`);
|
|
211
242
|
const telemetry = this.getTelemetry(options);
|
|
212
243
|
const config = this.mergeProviderOptions({
|
|
213
|
-
|
|
244
|
+
maxOutputTokens: 16384,
|
|
245
|
+
allowSystemInMessages: true,
|
|
214
246
|
...(this.config.config || {}),
|
|
215
247
|
...options,
|
|
216
|
-
...(telemetry ? { experimental_telemetry: telemetry } : {}),
|
|
217
248
|
model,
|
|
218
249
|
abortSignal: executionController.getAbortSignal(),
|
|
219
250
|
}, options.agentName);
|
|
251
|
+
this.finalizeConfig(config, options, telemetry);
|
|
220
252
|
promptLog(messages[messages.length - 1].content);
|
|
221
253
|
try {
|
|
222
254
|
const response = await withRetry(async () => {
|
|
@@ -224,7 +256,7 @@ export class Provider {
|
|
|
224
256
|
if (!result.text) {
|
|
225
257
|
debugLog(result);
|
|
226
258
|
if (result.finishReason === 'length') {
|
|
227
|
-
throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase
|
|
259
|
+
throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase maxOutputTokens in config or use a model with higher output capacity.');
|
|
228
260
|
}
|
|
229
261
|
throw new Error('No response text from AI');
|
|
230
262
|
}
|
|
@@ -277,21 +309,22 @@ export class Provider {
|
|
|
277
309
|
const telemetry = this.getTelemetry(options);
|
|
278
310
|
const maxRoundtrips = options.maxToolRoundtrips ?? 5;
|
|
279
311
|
const extraStop = options.stopWhen;
|
|
280
|
-
const stopConditions = [
|
|
312
|
+
const stopConditions = [isStepCount(maxRoundtrips)];
|
|
281
313
|
if (extraStop)
|
|
282
314
|
stopConditions.push(extraStop);
|
|
283
315
|
const { stopWhen: _ignoredStopWhen, ...optionsWithoutStop } = options;
|
|
284
316
|
const config = this.mergeProviderOptions({
|
|
285
317
|
tools,
|
|
286
|
-
|
|
318
|
+
maxOutputTokens: 16384,
|
|
287
319
|
toolChoice: 'auto',
|
|
320
|
+
allowSystemInMessages: true,
|
|
288
321
|
...(this.config.config || {}),
|
|
289
322
|
...optionsWithoutStop,
|
|
290
323
|
stopWhen: stopConditions,
|
|
291
|
-
...(telemetry ? { experimental_telemetry: telemetry } : {}),
|
|
292
324
|
model,
|
|
293
325
|
abortSignal: executionController.getAbortSignal(),
|
|
294
326
|
}, options.agentName);
|
|
327
|
+
this.finalizeConfig(config, options, telemetry);
|
|
295
328
|
try {
|
|
296
329
|
const response = await withRetry(async () => {
|
|
297
330
|
const timeout = config.timeout || 30000;
|
|
@@ -306,7 +339,7 @@ export class Provider {
|
|
|
306
339
|
]));
|
|
307
340
|
const hasToolCall = (result.toolCalls?.length || 0) > 0;
|
|
308
341
|
if (!result.text && !hasToolCall && result.finishReason === 'length') {
|
|
309
|
-
throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase
|
|
342
|
+
throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase maxOutputTokens in config or use a model with higher output capacity.');
|
|
310
343
|
}
|
|
311
344
|
return result;
|
|
312
345
|
}
|
|
@@ -365,12 +398,13 @@ export class Provider {
|
|
|
365
398
|
const telemetry = this.getTelemetry(options);
|
|
366
399
|
const config = this.mergeProviderOptions({
|
|
367
400
|
schema,
|
|
401
|
+
allowSystemInMessages: true,
|
|
368
402
|
...(this.config.config || {}),
|
|
369
403
|
...options,
|
|
370
|
-
...(telemetry ? { experimental_telemetry: telemetry } : {}),
|
|
371
404
|
model: modelToUse,
|
|
372
405
|
abortSignal: executionController.getAbortSignal(),
|
|
373
406
|
}, options.agentName);
|
|
407
|
+
this.finalizeConfig(config, options, telemetry);
|
|
374
408
|
try {
|
|
375
409
|
promptLog(messages[messages.length - 1].content);
|
|
376
410
|
const response = await withRetry(async () => {
|
|
@@ -552,12 +586,13 @@ export class Provider {
|
|
|
552
586
|
];
|
|
553
587
|
const telemetry = this.getTelemetry({});
|
|
554
588
|
const config = {
|
|
555
|
-
|
|
589
|
+
maxOutputTokens: 16384,
|
|
556
590
|
...(this.config.config || {}),
|
|
557
|
-
...(telemetry ? { experimental_telemetry: telemetry } : {}),
|
|
558
591
|
model: this.config.visionModel,
|
|
559
592
|
abortSignal: executionController.getAbortSignal(),
|
|
560
593
|
};
|
|
594
|
+
if (telemetry)
|
|
595
|
+
config.telemetry = telemetry;
|
|
561
596
|
try {
|
|
562
597
|
promptLog(`Processing image with prompt: ${prompt}`);
|
|
563
598
|
const response = await withRetry(async () => {
|
|
@@ -3,7 +3,7 @@ import { join } from 'node:path';
|
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
import { ConfigParser } from "../config.js";
|
|
5
5
|
import { createDebug, tag } from "../utils/logger.js";
|
|
6
|
-
import {
|
|
6
|
+
import { isCodeceptToolName } from "../utils/step-analyzer.js";
|
|
7
7
|
const debugLog = createDebug('explorbot:quartermaster');
|
|
8
8
|
export class Quartermaster {
|
|
9
9
|
provider;
|
|
@@ -88,7 +88,7 @@ export class Quartermaster {
|
|
|
88
88
|
const stateHash = initialState.getStateHash();
|
|
89
89
|
const pageAnalysis = this.pageAnalyses.get(stateHash);
|
|
90
90
|
const toolExecutions = conversation.getToolExecutions();
|
|
91
|
-
const codeceptExecutions = toolExecutions.filter((e) =>
|
|
91
|
+
const codeceptExecutions = toolExecutions.filter((e) => isCodeceptToolName(e.toolName));
|
|
92
92
|
if (codeceptExecutions.length === 0 && !pageAnalysis?.axeViolations.length) {
|
|
93
93
|
debugLog('No interactions or violations to analyze');
|
|
94
94
|
return null;
|
|
@@ -47,6 +47,12 @@ export class Researcher extends ResearcherBase {
|
|
|
47
47
|
this.stateManager = explorer.getStateManager();
|
|
48
48
|
this.experienceTracker = this.stateManager.getExperienceTracker();
|
|
49
49
|
this.hooksRunner = new HooksRunner(explorer, explorer.getConfig());
|
|
50
|
+
const ai = explorer.getConfig().ai;
|
|
51
|
+
if (ai) {
|
|
52
|
+
ai.agents ??= {};
|
|
53
|
+
ai.agents.researcher ??= {};
|
|
54
|
+
ai.agents.researcher.reasoning ??= 'low';
|
|
55
|
+
}
|
|
50
56
|
}
|
|
51
57
|
getNavigator() {
|
|
52
58
|
throw new Error('not implemented');
|
|
@@ -139,7 +145,7 @@ export class Researcher extends ResearcherBase {
|
|
|
139
145
|
catch (error) {
|
|
140
146
|
if (!(error instanceof ContextLengthError) || retriesLeft <= 0) {
|
|
141
147
|
if (error instanceof ContextLengthError) {
|
|
142
|
-
tag('warning').log('Output truncated. Try lowering reasoning effort or increasing
|
|
148
|
+
tag('warning').log('Output truncated. Try lowering reasoning effort or increasing maxOutputTokens in ai.config.');
|
|
143
149
|
}
|
|
144
150
|
throw error;
|
|
145
151
|
}
|
package/dist/src/ai/rules.js
CHANGED
|
@@ -147,6 +147,50 @@ export const protectionRule = dedent `
|
|
|
147
147
|
Do not propose or perform delete/remove/archive actions on the entity that owns the current URL; propose such actions only on disposable children created within the scenario itself.
|
|
148
148
|
</important>
|
|
149
149
|
`;
|
|
150
|
+
export const dataProtectionRules = dedent `
|
|
151
|
+
<data_protection_rules>
|
|
152
|
+
${protectionRule}
|
|
153
|
+
|
|
154
|
+
If the user request, scenario, focus, or test instructions explicitly prohibit creating,
|
|
155
|
+
editing, updating, deleting, removing, or otherwise mutating data, do not perform those
|
|
156
|
+
actions through the UI, API preconditions, cleanup, fallback steps, or Fisherman.
|
|
157
|
+
|
|
158
|
+
Do not use Fisherman or API data preparation to bypass a no-mutation, read-only, search,
|
|
159
|
+
filter, tab, or list-inspection constraint. Use visible existing data when it is available.
|
|
160
|
+
If no suitable data exists, report the missing precondition instead of creating data.
|
|
161
|
+
|
|
162
|
+
Destructive actions are allowed only against disposable data created by the current scenario
|
|
163
|
+
or prepared for that scenario by Fisherman/API preconditions. Existing application data must
|
|
164
|
+
remain unchanged.
|
|
165
|
+
</data_protection_rules>
|
|
166
|
+
`;
|
|
167
|
+
export const capabilityGroundingRule = dedent `
|
|
168
|
+
<capability_grounding>
|
|
169
|
+
When a scenario depends on a named action, menu item, status, option, workflow, or feature,
|
|
170
|
+
that capability must be visible or explicitly confirmed in the current research/page context
|
|
171
|
+
for the same target entity type.
|
|
172
|
+
|
|
173
|
+
Do not transfer capabilities between similar entities, rows, lists, detail pages, or menus.
|
|
174
|
+
Do not replace a requested action with a synonym or related action unless the UI explicitly
|
|
175
|
+
shows that action for the target entity.
|
|
176
|
+
|
|
177
|
+
When an action is described as applying to an item, row, card, record, node, or entity,
|
|
178
|
+
the target must be grounded as that kind of data entity in the current context. Do not use
|
|
179
|
+
navigation links, filter tabs, counters, breadcrumbs, headings, toolbar controls, or other
|
|
180
|
+
page controls as the subject of row/entity actions.
|
|
181
|
+
|
|
182
|
+
When a scenario asks to open, view, inspect, or navigate to an entity detail view, success
|
|
183
|
+
requires evidence of that entity detail context. An active filter, selected tab, visible count,
|
|
184
|
+
or filtered list is not enough to prove an entity detail view opened.
|
|
185
|
+
|
|
186
|
+
Do not rewrite a scenario goal to match a similar outcome that happened accidentally. If the
|
|
187
|
+
requested entity detail/action/workflow was not achieved, report that mismatch instead of
|
|
188
|
+
passing the test for a related filter, tab, navigation, or status view.
|
|
189
|
+
|
|
190
|
+
If the required capability is not available for the target entity after reasonable discovery,
|
|
191
|
+
record the missing capability and stop instead of repeatedly trying unrelated locators.
|
|
192
|
+
</capability_grounding>
|
|
193
|
+
`;
|
|
150
194
|
export const focusedElementRule = dedent `
|
|
151
195
|
<focused_element_actions>
|
|
152
196
|
When a text input element is focused (textbox, combobox, contenteditable):
|
package/dist/src/ai/tester.js
CHANGED
|
@@ -14,7 +14,7 @@ import { ErrorPageError, isErrorPage } from "../utils/error-page.js";
|
|
|
14
14
|
import { HooksRunner } from "../utils/hooks-runner.js";
|
|
15
15
|
import { createDebug, tag } from "../utils/logger.js";
|
|
16
16
|
import { loop } from "../utils/loop.js";
|
|
17
|
-
import { actionRule, focusedElementRule, formRequirementsRule, locatorRule, multipleTabsRule,
|
|
17
|
+
import { actionRule, capabilityGroundingRule, dataProtectionRules, focusedElementRule, formRequirementsRule, locatorRule, multipleTabsRule, sectionContextRule } from "./rules.js";
|
|
18
18
|
import { TaskAgent } from "./task-agent.js";
|
|
19
19
|
import { createCodeceptJSTools, createSpecialContextTools } from "./tools.js";
|
|
20
20
|
const debugLog = createDebug('explorbot:tester');
|
|
@@ -29,7 +29,7 @@ const SAMPLE_FILES = {
|
|
|
29
29
|
'MP3 audio': 'sample.mp3',
|
|
30
30
|
};
|
|
31
31
|
export class Tester extends TaskAgent {
|
|
32
|
-
ACTION_TOOLS = ['click', 'pressKey', 'form'];
|
|
32
|
+
ACTION_TOOLS = ['click', 'hover', 'pressKey', 'form'];
|
|
33
33
|
SPECIAL_CONTEXT_ACTION_TOOLS = ['exitIframe'];
|
|
34
34
|
emoji = '🧪';
|
|
35
35
|
explorer;
|
|
@@ -51,6 +51,8 @@ export class Tester extends TaskAgent {
|
|
|
51
51
|
hooksRunner;
|
|
52
52
|
seenUiMapUrls = new Set();
|
|
53
53
|
lastAnalyzedStateHash = null;
|
|
54
|
+
stalledIterations = 0;
|
|
55
|
+
MAX_STALLED_ITERATIONS = 3;
|
|
54
56
|
constructor(explorer, provider, researcher, navigator, agentTools) {
|
|
55
57
|
super();
|
|
56
58
|
this.explorer = explorer;
|
|
@@ -99,6 +101,7 @@ export class Tester extends TaskAgent {
|
|
|
99
101
|
this.pageActionResult = null;
|
|
100
102
|
this.seenUiMapUrls.clear();
|
|
101
103
|
this.lastAnalyzedStateHash = null;
|
|
104
|
+
this.stalledIterations = 0;
|
|
102
105
|
this.explorer.getStateManager().clearHistory();
|
|
103
106
|
this.resetFailureCount();
|
|
104
107
|
this.pilot?.reset();
|
|
@@ -293,6 +296,10 @@ export class Tester extends TaskAgent {
|
|
|
293
296
|
task.addNote(`Failed to ${execution.input.explanation} (${execution.toolName})`, TestResult.FAILED);
|
|
294
297
|
});
|
|
295
298
|
}
|
|
299
|
+
if (this.shouldStopForStalledExecution(task, currentState, result?.toolExecutions || [])) {
|
|
300
|
+
stop();
|
|
301
|
+
return;
|
|
302
|
+
}
|
|
296
303
|
if (assertionPerformed) {
|
|
297
304
|
const message = result?.toolExecutions?.find((execution) => execution.toolName === 'verify')?.output?.message || '';
|
|
298
305
|
task.addNote(message, wasSuccessful ? TestResult.PASSED : TestResult.FAILED);
|
|
@@ -395,6 +402,28 @@ export class Tester extends TaskAgent {
|
|
|
395
402
|
return false;
|
|
396
403
|
return true;
|
|
397
404
|
}
|
|
405
|
+
shouldStopForStalledExecution(task, previousState, toolExecutions) {
|
|
406
|
+
if (task.hasFinished)
|
|
407
|
+
return false;
|
|
408
|
+
const currentState = this.getCurrentState();
|
|
409
|
+
const stateChanged = previousState.url !== currentState.url || previousState.hash !== currentState.hash;
|
|
410
|
+
const actionTools = [...this.ACTION_TOOLS, ...this.SPECIAL_CONTEXT_ACTION_TOOLS];
|
|
411
|
+
const hasSuccessfulAction = toolExecutions.some((execution) => execution.wasSuccessful && actionTools.includes(execution.toolName));
|
|
412
|
+
const hasSuccessfulAssertion = toolExecutions.some((execution) => execution.wasSuccessful && this.ASSERTION_TOOLS.includes(execution.toolName));
|
|
413
|
+
if (stateChanged || hasSuccessfulAction || hasSuccessfulAssertion) {
|
|
414
|
+
this.stalledIterations = 0;
|
|
415
|
+
return false;
|
|
416
|
+
}
|
|
417
|
+
const hasNoBrowserProgress = toolExecutions.length === 0 || toolExecutions.every((execution) => !actionTools.includes(execution.toolName) || !execution.wasSuccessful);
|
|
418
|
+
if (!hasNoBrowserProgress)
|
|
419
|
+
return false;
|
|
420
|
+
this.stalledIterations++;
|
|
421
|
+
if (this.stalledIterations < this.MAX_STALLED_ITERATIONS)
|
|
422
|
+
return false;
|
|
423
|
+
task.addNote('No browser progress after repeated attempts on unchanged page', TestResult.FAILED);
|
|
424
|
+
task.finish(TestResult.FAILED);
|
|
425
|
+
return true;
|
|
426
|
+
}
|
|
398
427
|
async prepareInstructionsForNextStep(task) {
|
|
399
428
|
let outcomeStatus = dedent `
|
|
400
429
|
<task>
|
|
@@ -403,6 +432,8 @@ export class Tester extends TaskAgent {
|
|
|
403
432
|
|
|
404
433
|
<rules>
|
|
405
434
|
Use tools ${this.ACTION_TOOLS.join(', ')} to interact with the page.
|
|
435
|
+
Use tool names exactly as listed in this prompt. Do not invent combined tool names, aliases, or names with channel markers such as "commentary".
|
|
436
|
+
Match each tool input schema exactly. Do not invent parameter names or pass extra fields.
|
|
406
437
|
Do not do unsuccesful clicks again.
|
|
407
438
|
Do not run same tool calls with same parameters again.
|
|
408
439
|
</rules>
|
|
@@ -665,6 +696,7 @@ export class Tester extends TaskAgent {
|
|
|
665
696
|
<rules>
|
|
666
697
|
- Refer to UI Map from <page_ui_map> to understand the page structure and its main elements
|
|
667
698
|
- Use only elements that exist in the provided ARIA tree or HTML, <page_aria> and <page_html>
|
|
699
|
+
- Use tool input schemas exactly as documented. Do not invent parameter names or add fields not listed by the tool schema.
|
|
668
700
|
- Use click() for buttons, links, and clickable elements ONLY - do NOT include I.fillField() or I.type() commands in click() tool
|
|
669
701
|
- click() commands array is for FALLBACK LOCATORS of the SAME element, NOT for clicking different elements in sequence. If you need to click two different elements, make two separate click() calls.
|
|
670
702
|
- Use form() for text input (I.fillField, I.type), dropdown selection (I.selectOption), file uploads (I.attachFile), and multi-step form interactions
|
|
@@ -677,6 +709,7 @@ export class Tester extends TaskAgent {
|
|
|
677
709
|
- NEVER call record(status: "success") if your last verify() or see() call FAILED. A failed check means the outcome is NOT confirmed — use record(status: "fail") instead, or retry with a different approach.
|
|
678
710
|
- Use finish() to complete the test, not record(). record() is for intermediate notes.
|
|
679
711
|
- Call finish(verify) when all goals are achieved — provide an assertion to verify
|
|
712
|
+
- NEVER call finish() with a negative assertion that says the goal did NOT happen. If the goal cannot be achieved after real attempts, record the blocker and call stop().
|
|
680
713
|
- ONLY call stop() if the scenario itself is completely irrelevant to this page and no expectations can be achieved
|
|
681
714
|
- Use reset() ONLY as a last resort when the current page cannot host the scenario. Never reset after a successful flow just because an assertion or milestone did not match — verify differently or record() the finding instead. Reset is destructive and does not undo server-side side effects.
|
|
682
715
|
- Be precise with locators (CSS or XPath)
|
|
@@ -690,6 +723,13 @@ export class Tester extends TaskAgent {
|
|
|
690
723
|
- When you interact with form with inputs, ensure that you click corresponding button to save its data
|
|
691
724
|
- Follow <locator_priority> rules when selecting locators for all tools
|
|
692
725
|
- Before retrying your actions check maybe they already achived expected results. Use see() tool for that
|
|
726
|
+
- If the current URL is already a create/edit/new form and the scenario is about creating/editing that entity, fill and submit that form. Do not click the list-page "New" button again from inside the form.
|
|
727
|
+
- If the scenario is about search/filter/sort/tabs/list inspection and the current URL is a create/edit/new form, go back or reset to the stable list page before interacting with list controls.
|
|
728
|
+
- When selecting related entities from a list, do not choose rows/options/cards marked as "0 items", "0 tests", or otherwise empty if the scenario requires selecting real content.
|
|
729
|
+
- In selection pickers, counters such as "Selected 0", "Matched tests 0", or disabled Save/Apply mean the selection did not register. Choose a non-empty item or change filters before submitting.
|
|
730
|
+
- A passed form/click command only means the command executed. If a required field remains empty, submit stays disabled, or the expected text is not visible, treat the action as not completed and correct the missing field/state.
|
|
731
|
+
- For filter/tab scenarios, success requires BOTH: the requested filter/tab is visibly active/selected AND the list content matches that filter. Do not finish from only one of these signals.
|
|
732
|
+
- Empty-state text such as "No matched items" only proves a filter when the requested filter/tab is active and the empty state belongs to the filtered list.
|
|
693
733
|
- When filling complex form with lot of actions performed, use see() to look which fields were filled and which are not
|
|
694
734
|
- When verify() fails, use see() to visually confirm the result — visual confirmation is equally valid evidence
|
|
695
735
|
- For visual state verification (active tabs, selected items, counts, colors), prefer see() over DOM-based verify()
|
|
@@ -718,11 +758,16 @@ export class Tester extends TaskAgent {
|
|
|
718
758
|
|
|
719
759
|
${formRequirementsRule}
|
|
720
760
|
|
|
761
|
+
${capabilityGroundingRule}
|
|
762
|
+
|
|
763
|
+
${dataProtectionRules}
|
|
764
|
+
|
|
721
765
|
${this.provider.getSystemPromptForAgent('tester', this.explorer.getStateManager().getCurrentState()?.url) || ''}
|
|
722
766
|
`;
|
|
723
767
|
}
|
|
724
768
|
buildScenarioBlock(task, actionResult) {
|
|
725
769
|
const knowledge = this.getKnowledge(actionResult);
|
|
770
|
+
const experience = this.getExperience(actionResult);
|
|
726
771
|
return dedent `
|
|
727
772
|
<task>
|
|
728
773
|
SCENARIO GOAL: ${task.scenario}
|
|
@@ -739,16 +784,21 @@ export class Tester extends TaskAgent {
|
|
|
739
784
|
Try to achieve as many goals as possible.
|
|
740
785
|
If goal is not achievable, log that and skip to next one.
|
|
741
786
|
Do not hallucinate that goal was achieved when it was not.
|
|
787
|
+
If the scenario action could not be completed, do not finish with a verification of the failure state.
|
|
742
788
|
When creating or editing items via form() or type() you should include ${task.sessionName} in the value (if it is not restricted by the application logic)
|
|
743
789
|
Initial page URL: ${actionResult.url}
|
|
744
790
|
|
|
745
|
-
${
|
|
791
|
+
${capabilityGroundingRule}
|
|
792
|
+
|
|
793
|
+
${dataProtectionRules}
|
|
746
794
|
|
|
747
795
|
${this.buildDeletionScope(task)}
|
|
748
796
|
|
|
749
797
|
${this.buildAvailableFiles()}
|
|
750
798
|
|
|
751
799
|
${knowledge}
|
|
800
|
+
|
|
801
|
+
${experience}
|
|
752
802
|
`;
|
|
753
803
|
}
|
|
754
804
|
getDeletableSessionNames(task) {
|
|
@@ -867,12 +917,13 @@ export class Tester extends TaskAgent {
|
|
|
867
917
|
}),
|
|
868
918
|
stop: tool({
|
|
869
919
|
description: dedent `
|
|
870
|
-
Stop the current test because
|
|
871
|
-
Use this
|
|
872
|
-
|
|
920
|
+
Stop the current test because it cannot be completed in the current session.
|
|
921
|
+
Use this when the scenario is incompatible, required UI/data is absent, or repeated varied attempts
|
|
922
|
+
show that automation cannot complete the workflow.
|
|
923
|
+
Do NOT use this immediately after the first failed action — retry with a materially different approach first.
|
|
873
924
|
`,
|
|
874
925
|
inputSchema: z.object({
|
|
875
|
-
reason: z.string().describe('Explanation why the scenario
|
|
926
|
+
reason: z.string().describe('Explanation why the scenario cannot be completed'),
|
|
876
927
|
}),
|
|
877
928
|
execute: async ({ reason }) => {
|
|
878
929
|
task.addNote(`Stop requested: ${reason}`);
|
|
@@ -906,6 +957,7 @@ export class Tester extends TaskAgent {
|
|
|
906
957
|
Provide a specific assertion to verify the final state.
|
|
907
958
|
The assertion MUST prove that YOUR ACTIONS changed the page state.
|
|
908
959
|
Do NOT verify something that was already true before you started testing.
|
|
960
|
+
Do NOT provide an assertion that verifies absence, failure, an empty state, or that the goal did not happen.
|
|
909
961
|
|
|
910
962
|
Examples of good assertions:
|
|
911
963
|
- "New user 'john@example.com' is visible in the users list"
|
|
@@ -1031,10 +1083,24 @@ export class Tester extends TaskAgent {
|
|
|
1031
1083
|
this.resetFailureCount();
|
|
1032
1084
|
this.previousUrl = null;
|
|
1033
1085
|
this.previousStateHash = null;
|
|
1086
|
+
this.stalledIterations = 0;
|
|
1087
|
+
}
|
|
1088
|
+
else if (this.shouldStopAfterStalledLoopError(task)) {
|
|
1089
|
+
return 'stop';
|
|
1034
1090
|
}
|
|
1035
1091
|
this.currentConversation?.addUserText(result.message);
|
|
1036
1092
|
return 'continue';
|
|
1037
1093
|
}
|
|
1094
|
+
shouldStopAfterStalledLoopError(task) {
|
|
1095
|
+
if (task.hasFinished)
|
|
1096
|
+
return false;
|
|
1097
|
+
this.stalledIterations++;
|
|
1098
|
+
if (this.stalledIterations < this.MAX_STALLED_ITERATIONS)
|
|
1099
|
+
return false;
|
|
1100
|
+
task.addNote('No browser progress after repeated execution errors', TestResult.FAILED);
|
|
1101
|
+
task.finish(TestResult.FAILED);
|
|
1102
|
+
return true;
|
|
1103
|
+
}
|
|
1038
1104
|
async cleanupStartedTest(task) {
|
|
1039
1105
|
await this.finishTest(task);
|
|
1040
1106
|
await this.explorer.stopTest(task, {
|
package/dist/src/ai/tools.js
CHANGED
|
@@ -11,7 +11,6 @@ import { WebElement } from "../utils/web-element.js";
|
|
|
11
11
|
import { sectionContextRule } from "./rules.js";
|
|
12
12
|
import { isInteractive } from "./task-agent.js";
|
|
13
13
|
const debugLog = createDebug('explorbot:tools');
|
|
14
|
-
export const CODECEPT_TOOLS = ['click', 'pressKey', 'form'];
|
|
15
14
|
export const ASSERTION_TOOLS = ['verify'];
|
|
16
15
|
export function createCodeceptJSTools(explorer, task) {
|
|
17
16
|
const stateManager = explorer.getStateManager();
|
|
@@ -133,6 +132,71 @@ export function createCodeceptJSTools(explorer, task) {
|
|
|
133
132
|
}, action.lastError);
|
|
134
133
|
},
|
|
135
134
|
}),
|
|
135
|
+
hover: tool({
|
|
136
|
+
description: dedent `
|
|
137
|
+
Move the mouse cursor to an element to reveal hover-only controls.
|
|
138
|
+
|
|
139
|
+
Use this before clicking row actions, icon buttons, menus, or toolbars that appear only
|
|
140
|
+
when the user hovers a list item, table row, card, or tree node.
|
|
141
|
+
|
|
142
|
+
This tool ONLY accepts I.moveCursorTo(locator) commands. It does not click.
|
|
143
|
+
After hovering, use context(), see(), or click() the revealed control.
|
|
144
|
+
`,
|
|
145
|
+
inputSchema: z.object({
|
|
146
|
+
commands: z.array(z.string()).describe(dedent `
|
|
147
|
+
FALLBACK LOCATORS for ONE element to hover.
|
|
148
|
+
Order by reliability:
|
|
149
|
+
1. I.moveCursorTo(text, container)
|
|
150
|
+
2. I.moveCursorTo(ARIA, container)
|
|
151
|
+
3. I.moveCursorTo(CSS, container)
|
|
152
|
+
4. I.moveCursorTo(CSS) or I.moveCursorTo(XPath)
|
|
153
|
+
`),
|
|
154
|
+
explanation: z.string().describe('Why you are hovering this element'),
|
|
155
|
+
}),
|
|
156
|
+
execute: async ({ commands: rawCommands, explanation }) => {
|
|
157
|
+
const activeNote = task.startNote(explanation);
|
|
158
|
+
if (rawCommands.length === 0) {
|
|
159
|
+
activeNote.commit(TestResult.FAILED);
|
|
160
|
+
return failedToolResult('hover', 'No commands provided');
|
|
161
|
+
}
|
|
162
|
+
const invalidCommands = rawCommands.map((cmd) => cmd.trim()).filter((cmd) => cmd.startsWith('I.') && !cmd.startsWith('I.moveCursorTo'));
|
|
163
|
+
if (invalidCommands.length > 0) {
|
|
164
|
+
activeNote.commit(TestResult.FAILED);
|
|
165
|
+
return failedToolResult('hover', `Invalid commands: ${invalidCommands.join(', ')}. Hover tool only accepts I.moveCursorTo() commands.`, {
|
|
166
|
+
suggestion: 'Use click() to click elements, or form() for typing/selecting.',
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
const commands = rawCommands.map((cmd) => {
|
|
170
|
+
const trimmed = cmd.trim();
|
|
171
|
+
if (trimmed.startsWith('I.moveCursorTo'))
|
|
172
|
+
return trimmed;
|
|
173
|
+
return `I.moveCursorTo(${JSON.stringify(trimmed)})`;
|
|
174
|
+
});
|
|
175
|
+
const previousState = ActionResult.fromState(stateManager.getCurrentState());
|
|
176
|
+
const action = explorer.createAction();
|
|
177
|
+
const attempts = [];
|
|
178
|
+
for (const command of commands) {
|
|
179
|
+
const success = await action.attempt(command, explanation, true);
|
|
180
|
+
attempts.push({
|
|
181
|
+
command,
|
|
182
|
+
success,
|
|
183
|
+
...(action.lastError && { error: action.lastError.toString() }),
|
|
184
|
+
});
|
|
185
|
+
if (!success)
|
|
186
|
+
continue;
|
|
187
|
+
const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, command);
|
|
188
|
+
activeNote.commit(TestResult.PASSED);
|
|
189
|
+
return successToolResult('hover', { ...toolResult, attempts, code: command }, action);
|
|
190
|
+
}
|
|
191
|
+
const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, commands[0]);
|
|
192
|
+
activeNote.commit(TestResult.FAILED);
|
|
193
|
+
return failedToolResult('hover', 'All hover commands failed', {
|
|
194
|
+
...toolResult,
|
|
195
|
+
attempts,
|
|
196
|
+
suggestion: 'Use xpathCheck() to locate the row/card/tree node, or visualClick() if the hover target is only visually identifiable.',
|
|
197
|
+
}, action.lastError);
|
|
198
|
+
},
|
|
199
|
+
}),
|
|
136
200
|
pressKey: tool({
|
|
137
201
|
description: dedent `
|
|
138
202
|
Press a keyboard key or key combination. Use this for special keys like Enter, Escape, Tab, Arrow keys, or key combinations with modifiers.
|
|
@@ -404,6 +468,7 @@ export function createAgentTools({ explorer, researcher, navigator, experienceTr
|
|
|
404
468
|
Check the page contents based on current page state and screenshot.
|
|
405
469
|
This tool will trigger visual research to check the page contents on request.
|
|
406
470
|
Use it to verify the actions were performed correctly and the page is in the expected state.
|
|
471
|
+
Input schema has exactly one field: request. Do not pass text, reason, assertion, or other fields.
|
|
407
472
|
|
|
408
473
|
<example>
|
|
409
474
|
request: "Check current state of the Login form"
|