explorbot 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/explorbot-cli.ts +93 -36
- package/dist/bin/explorbot-cli.js +71 -16
- package/dist/rules/rerunner/healing-approach.md +19 -0
- package/dist/src/action.js +8 -10
- package/dist/src/ai/historian.js +34 -3
- package/dist/src/ai/navigator.js +35 -28
- package/dist/src/ai/pilot.js +33 -9
- package/dist/src/ai/planner.js +29 -10
- package/dist/src/ai/rerunner.js +472 -0
- package/dist/src/ai/researcher.js +3 -4
- package/dist/src/ai/rules.js +2 -2
- package/dist/src/ai/tools.js +2 -2
- package/dist/src/commands/add-rule-command.js +1 -2
- package/dist/src/commands/base-command.js +12 -0
- package/dist/src/commands/context-command.js +12 -5
- package/dist/src/commands/drill-command.js +0 -1
- package/dist/src/commands/explore-command.js +20 -5
- package/dist/src/commands/freesail-command.js +8 -22
- package/dist/src/commands/index.js +4 -0
- package/dist/src/commands/init-command.js +3 -3
- package/dist/src/commands/path-command.js +2 -1
- package/dist/src/commands/plan-command.js +37 -15
- package/dist/src/commands/rerun-command.js +42 -0
- package/dist/src/commands/research-command.js +10 -4
- package/dist/src/commands/runs-command.js +22 -0
- package/dist/src/commands/start-command.js +0 -1
- package/dist/src/commands/test-command.js +3 -3
- package/dist/src/components/App.js +8 -0
- package/dist/src/config.js +3 -0
- package/dist/src/explorbot.js +19 -0
- package/dist/src/explorer.js +2 -1
- package/dist/src/suite.js +115 -0
- package/dist/src/utils/html.js +2 -5
- package/dist/src/utils/rules-loader.js +33 -17
- package/dist/src/utils/test-files.js +103 -0
- package/package.json +2 -1
- package/rules/rerunner/healing-approach.md +19 -0
- package/src/action.ts +7 -9
- package/src/ai/historian.ts +37 -3
- package/src/ai/navigator.ts +35 -28
- package/src/ai/pilot.ts +33 -9
- package/src/ai/planner.ts +28 -9
- package/src/ai/rerunner.ts +532 -0
- package/src/ai/researcher.ts +3 -4
- package/src/ai/rules.ts +2 -2
- package/src/ai/tools.ts +2 -2
- package/src/commands/add-rule-command.ts +1 -2
- package/src/commands/base-command.ts +13 -0
- package/src/commands/context-command.ts +12 -5
- package/src/commands/drill-command.ts +0 -1
- package/src/commands/explore-command.ts +21 -5
- package/src/commands/freesail-command.ts +6 -23
- package/src/commands/index.ts +4 -0
- package/src/commands/init-command.ts +3 -3
- package/src/commands/path-command.ts +2 -1
- package/src/commands/plan-command.ts +45 -16
- package/src/commands/rerun-command.ts +46 -0
- package/src/commands/research-command.ts +10 -4
- package/src/commands/runs-command.ts +27 -0
- package/src/commands/start-command.ts +0 -1
- package/src/commands/test-command.ts +3 -3
- package/src/components/App.tsx +8 -0
- package/src/config.ts +23 -0
- package/src/explorbot.ts +21 -0
- package/src/explorer.ts +3 -2
- package/src/suite.ts +135 -0
- package/src/utils/html.ts +1 -5
- package/src/utils/rules-loader.ts +35 -17
- package/src/utils/test-files.ts +122 -0
|
@@ -0,0 +1,532 @@
|
|
|
1
|
+
import { existsSync } from 'node:fs';
|
|
2
|
+
import { relative, resolve } from 'node:path';
|
|
3
|
+
import { tool } from 'ai';
|
|
4
|
+
import { createBashTool } from 'bash-tool';
|
|
5
|
+
import chalk from 'chalk';
|
|
6
|
+
import { highlight } from 'cli-highlight';
|
|
7
|
+
import * as codeceptjs from 'codeceptjs';
|
|
8
|
+
import heal from 'codeceptjs/lib/heal';
|
|
9
|
+
import aiTracePlugin from 'codeceptjs/lib/plugin/aiTrace';
|
|
10
|
+
import figureSet from 'figures';
|
|
11
|
+
import dedent from 'dedent';
|
|
12
|
+
import { z } from 'zod';
|
|
13
|
+
import { ActionResult } from '../action-result.ts';
|
|
14
|
+
import { setActivity } from '../activity.ts';
|
|
15
|
+
import type { ExperienceTracker } from '../experience-tracker.ts';
|
|
16
|
+
import type Explorer from '../explorer.ts';
|
|
17
|
+
import type { KnowledgeTracker } from '../knowledge-tracker.ts';
|
|
18
|
+
import { Stats } from '../stats.ts';
|
|
19
|
+
import { Task, Test, TestResult } from '../test-plan.ts';
|
|
20
|
+
import { createDebug, tag } from '../utils/logger.ts';
|
|
21
|
+
import { loop } from '../utils/loop.ts';
|
|
22
|
+
import { loadTestSuites, printTestList } from '../utils/test-files.ts';
|
|
23
|
+
import type { Agent } from './agent.ts';
|
|
24
|
+
import { toolExecutionLabel } from './conversation.ts';
|
|
25
|
+
import type { Navigator } from './navigator.ts';
|
|
26
|
+
import { Provider } from './provider.ts';
|
|
27
|
+
import { locatorRule, actionRule, sectionContextRule } from './rules.ts';
|
|
28
|
+
import { TaskAgent } from './task-agent.ts';
|
|
29
|
+
import { RulesLoader } from '../utils/rules-loader.ts';
|
|
30
|
+
import { createCodeceptJSTools } from './tools.ts';
|
|
31
|
+
|
|
32
|
+
const debugLog = createDebug('explorbot:rerunner');
|
|
33
|
+
|
|
34
|
+
export class Rerunner extends TaskAgent implements Agent {
|
|
35
|
+
protected readonly ACTION_TOOLS = ['click', 'pressKey', 'form'];
|
|
36
|
+
emoji = '🔄';
|
|
37
|
+
|
|
38
|
+
private explorer: Explorer;
|
|
39
|
+
private provider: Provider;
|
|
40
|
+
private agentTools: any;
|
|
41
|
+
private healedSteps: Array<{ test: string; original: string; healed: string }> = [];
|
|
42
|
+
private traceDir = '';
|
|
43
|
+
|
|
44
|
+
constructor(explorer: Explorer, provider: Provider, agentTools?: any) {
|
|
45
|
+
super();
|
|
46
|
+
this.explorer = explorer;
|
|
47
|
+
this.provider = provider;
|
|
48
|
+
this.agentTools = agentTools;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
protected getNavigator(): Navigator {
|
|
52
|
+
throw new Error('Rerunner does not use Navigator');
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
protected getExperienceTracker(): ExperienceTracker {
|
|
56
|
+
return this.explorer.getStateManager().getExperienceTracker();
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
protected getKnowledgeTracker(): KnowledgeTracker {
|
|
60
|
+
return this.explorer.getKnowledgeTracker();
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
protected getProvider(): Provider {
|
|
64
|
+
return this.provider;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
private get rerunnerConfig(): Record<string, any> {
|
|
68
|
+
return (this.explorer.getConfig().ai?.agents?.rerunner as any) || {};
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
private get healLimit(): number {
|
|
72
|
+
return this.rerunnerConfig.healLimit ?? 3;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
private get healMaxIterations(): number {
|
|
76
|
+
return this.rerunnerConfig.healMaxIterations ?? 3;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
listTests(testsDir: string): void {
|
|
80
|
+
printTestList(loadTestSuites(testsDir));
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
async rerun(filePath: string, options?: { testIndices?: number[] }): Promise<RerunResult> {
|
|
84
|
+
const absPath = resolve(filePath);
|
|
85
|
+
if (!existsSync(absPath)) {
|
|
86
|
+
tag('error').log(`Test file not found: ${absPath}`);
|
|
87
|
+
return { total: 0, passed: 0, failed: 0, healed: 0 };
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
tag('info').log(`Re-running tests from: ${relative(process.cwd(), absPath)}`);
|
|
91
|
+
setActivity('🔄 Re-running tests...', 'action');
|
|
92
|
+
|
|
93
|
+
this.healedSteps = [];
|
|
94
|
+
this.setupPlugins();
|
|
95
|
+
|
|
96
|
+
const testMap = new Map<string, Test>();
|
|
97
|
+
const results: { test: Test; mochaState: string }[] = [];
|
|
98
|
+
|
|
99
|
+
const onTestBefore = (mochaTest: any) => {
|
|
100
|
+
if (!mochaTest.file) mochaTest.file = absPath;
|
|
101
|
+
const task = new Test(mochaTest.title, 'normal', [], '');
|
|
102
|
+
task.start();
|
|
103
|
+
testMap.set(mochaTest.id || mochaTest.title, task);
|
|
104
|
+
Stats.tests++;
|
|
105
|
+
console.log(`\n ${chalk.green(figureSet.pointer)} ${chalk.bold(mochaTest.title)}`);
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
const onStepStarted = (step: any) => {
|
|
109
|
+
if (!step.toCode) return;
|
|
110
|
+
const code = highlight(step.toCode(), { language: 'javascript' });
|
|
111
|
+
console.log(chalk.dim(` ${code}`));
|
|
112
|
+
};
|
|
113
|
+
|
|
114
|
+
const onStepPassed = (step: any) => {
|
|
115
|
+
const task = this.getCurrentTask(testMap);
|
|
116
|
+
if (!task || !step.toCode) return;
|
|
117
|
+
task.addStep(step.toCode(), step.duration, 'passed');
|
|
118
|
+
};
|
|
119
|
+
|
|
120
|
+
const onStepFailed = (step: any, error: any) => {
|
|
121
|
+
const task = this.getCurrentTask(testMap);
|
|
122
|
+
if (!task || !step.toCode) return;
|
|
123
|
+
task.addStep(step.toCode(), step.duration, 'failed', error?.message);
|
|
124
|
+
console.log(chalk.red(` ${figureSet.cross} ${step.toCode()} — ${error?.message || 'failed'}`));
|
|
125
|
+
};
|
|
126
|
+
|
|
127
|
+
const onTestPassed = (mochaTest: any) => {
|
|
128
|
+
const task = testMap.get(mochaTest.id || mochaTest.title);
|
|
129
|
+
if (!task) return;
|
|
130
|
+
task.finish(TestResult.PASSED);
|
|
131
|
+
results.push({ test: task, mochaState: 'passed' });
|
|
132
|
+
console.log(chalk.green(` ${figureSet.tick} passed`));
|
|
133
|
+
};
|
|
134
|
+
|
|
135
|
+
const onTestFailed = (mochaTest: any, error: any) => {
|
|
136
|
+
const task = testMap.get(mochaTest.id || mochaTest.title);
|
|
137
|
+
if (!task) return;
|
|
138
|
+
task.addNote(`Failed: ${error?.message || 'unknown error'}`, TestResult.FAILED);
|
|
139
|
+
task.finish(TestResult.FAILED);
|
|
140
|
+
results.push({ test: task, mochaState: 'failed' });
|
|
141
|
+
console.log(chalk.red(` ${figureSet.cross} failed: ${error?.message || 'unknown'}`));
|
|
142
|
+
};
|
|
143
|
+
|
|
144
|
+
const { dispatcher } = codeceptjs.event;
|
|
145
|
+
dispatcher.on('test.before', onTestBefore);
|
|
146
|
+
dispatcher.on('step.start', onStepStarted);
|
|
147
|
+
dispatcher.on('step.passed', onStepPassed);
|
|
148
|
+
dispatcher.on('step.failed', onStepFailed);
|
|
149
|
+
dispatcher.on('test.passed', onTestPassed);
|
|
150
|
+
dispatcher.on('test.failed', onTestFailed);
|
|
151
|
+
|
|
152
|
+
try {
|
|
153
|
+
codeceptjs.container.createMocha();
|
|
154
|
+
const mocha = codeceptjs.container.mocha();
|
|
155
|
+
mocha.reporter(class {});
|
|
156
|
+
mocha.files = [absPath];
|
|
157
|
+
mocha.loadFiles();
|
|
158
|
+
|
|
159
|
+
let testIndex = 0;
|
|
160
|
+
for (const suite of mocha.suite.suites || []) {
|
|
161
|
+
for (const test of suite.tests || []) {
|
|
162
|
+
if (test.pending) {
|
|
163
|
+
testIndex++;
|
|
164
|
+
continue;
|
|
165
|
+
}
|
|
166
|
+
if (options?.testIndices?.length && !options.testIndices.includes(testIndex)) {
|
|
167
|
+
test.pending = true;
|
|
168
|
+
testIndex++;
|
|
169
|
+
continue;
|
|
170
|
+
}
|
|
171
|
+
if (!hasAssertions(test.body)) {
|
|
172
|
+
test.pending = true;
|
|
173
|
+
tag('substep').log(`Skipping: ${test.title} (no assertions)`);
|
|
174
|
+
}
|
|
175
|
+
testIndex++;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
await new Promise<void>((resolveRun) => {
|
|
180
|
+
mocha.run((failures: number) => {
|
|
181
|
+
debugLog('Mocha run finished with %d failures', failures);
|
|
182
|
+
resolveRun();
|
|
183
|
+
});
|
|
184
|
+
});
|
|
185
|
+
} catch (error) {
|
|
186
|
+
tag('error').log(`Rerun error: ${error instanceof Error ? error.message : error}`);
|
|
187
|
+
} finally {
|
|
188
|
+
dispatcher.off('test.before', onTestBefore);
|
|
189
|
+
dispatcher.off('step.start', onStepStarted);
|
|
190
|
+
dispatcher.off('step.passed', onStepPassed);
|
|
191
|
+
dispatcher.off('step.failed', onStepFailed);
|
|
192
|
+
dispatcher.off('test.passed', onTestPassed);
|
|
193
|
+
dispatcher.off('test.failed', onTestFailed);
|
|
194
|
+
this.teardownHealing();
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
if (this.healedSteps.length > 0) {
|
|
198
|
+
this.getHistorian().rewriteScenarioInFile(absPath, this.healedSteps);
|
|
199
|
+
tag('info').log(`Healed ${this.healedSteps.length} step(s), original file updated`);
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
const passed = results.filter((r) => r.mochaState === 'passed').length;
|
|
203
|
+
const failed = results.filter((r) => r.mochaState === 'failed').length;
|
|
204
|
+
const result: RerunResult = {
|
|
205
|
+
total: results.length,
|
|
206
|
+
passed,
|
|
207
|
+
failed,
|
|
208
|
+
healed: this.healedSteps.length,
|
|
209
|
+
};
|
|
210
|
+
|
|
211
|
+
this.printResults(result);
|
|
212
|
+
return result;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
private getCurrentTask(testMap: Map<string, Test>): Test | undefined {
|
|
216
|
+
const entries = [...testMap.values()];
|
|
217
|
+
return entries[entries.length - 1];
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
private setupPlugins(): void {
|
|
221
|
+
const healMod = heal.default || heal;
|
|
222
|
+
healMod.connectToEvents();
|
|
223
|
+
|
|
224
|
+
healMod.addRecipe('explorbot-ai-healer', {
|
|
225
|
+
priority: 10,
|
|
226
|
+
fn: async (context: any) => {
|
|
227
|
+
return this.healStep(context.step, context.error);
|
|
228
|
+
},
|
|
229
|
+
});
|
|
230
|
+
|
|
231
|
+
const userRecipes = (this.rerunnerConfig.recipes || {}) as Record<string, any>;
|
|
232
|
+
for (const [name, recipe] of Object.entries(userRecipes)) {
|
|
233
|
+
healMod.addRecipe(name, recipe);
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
let currentTest: any = null;
|
|
237
|
+
let healTries = 0;
|
|
238
|
+
let isHealing = false;
|
|
239
|
+
let caughtError: any = null;
|
|
240
|
+
const healLimit = this.healLimit;
|
|
241
|
+
|
|
242
|
+
codeceptjs.event.dispatcher.on('test.before', (test: any) => {
|
|
243
|
+
currentTest = test;
|
|
244
|
+
healTries = 0;
|
|
245
|
+
caughtError = null;
|
|
246
|
+
});
|
|
247
|
+
|
|
248
|
+
codeceptjs.event.dispatcher.on('step.after', (step: any) => {
|
|
249
|
+
if (isHealing) return;
|
|
250
|
+
if (healTries >= healLimit) return;
|
|
251
|
+
if (!healMod.hasCorrespondingRecipes(step)) return;
|
|
252
|
+
|
|
253
|
+
codeceptjs.recorder.catchWithoutStop(async (err: any) => {
|
|
254
|
+
isHealing = true;
|
|
255
|
+
if (caughtError === err) throw err;
|
|
256
|
+
caughtError = err;
|
|
257
|
+
|
|
258
|
+
codeceptjs.recorder.session.start('heal');
|
|
259
|
+
debugLog('Healing started for: %s', step.toCode());
|
|
260
|
+
|
|
261
|
+
await healMod.healStep(step, err, { test: currentTest });
|
|
262
|
+
|
|
263
|
+
healTries++;
|
|
264
|
+
|
|
265
|
+
codeceptjs.recorder.add('close healing session', () => {
|
|
266
|
+
codeceptjs.recorder.reset();
|
|
267
|
+
codeceptjs.recorder.session.restore('heal');
|
|
268
|
+
codeceptjs.recorder.ignoreErr(err);
|
|
269
|
+
});
|
|
270
|
+
await codeceptjs.recorder.promise();
|
|
271
|
+
|
|
272
|
+
isHealing = false;
|
|
273
|
+
});
|
|
274
|
+
});
|
|
275
|
+
|
|
276
|
+
(global as any).container = codeceptjs.container;
|
|
277
|
+
|
|
278
|
+
codeceptjs.recorder.retry({
|
|
279
|
+
retries: 3,
|
|
280
|
+
when: (err: any) => {
|
|
281
|
+
if (!err?.message) return false;
|
|
282
|
+
return err.message.includes('was not found') || err.message.includes('Timeout') || err.message.includes('exceeded');
|
|
283
|
+
},
|
|
284
|
+
minTimeout: 2000,
|
|
285
|
+
maxTimeout: 5000,
|
|
286
|
+
factor: 1.5,
|
|
287
|
+
});
|
|
288
|
+
|
|
289
|
+
const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
|
|
290
|
+
const outputDir = (global as any).output_dir || 'output';
|
|
291
|
+
this.traceDir = `${outputDir}/rerun_${timestamp}`;
|
|
292
|
+
|
|
293
|
+
const aiTrace = aiTracePlugin.default || aiTracePlugin;
|
|
294
|
+
aiTrace(this.rerunnerConfig.aiTrace || { output: this.traceDir });
|
|
295
|
+
|
|
296
|
+
import('@testomatio/reporter/codecept')
|
|
297
|
+
.then((mod) => {
|
|
298
|
+
const plugin = mod.default || mod;
|
|
299
|
+
plugin({ enabled: true });
|
|
300
|
+
})
|
|
301
|
+
.catch(() => debugLog('Testomatio reporter plugin not available'));
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
private teardownHealing(): void {
|
|
305
|
+
const healMod = heal.default || heal;
|
|
306
|
+
healMod.recipes['explorbot-ai-healer'] = undefined;
|
|
307
|
+
for (const name of Object.keys(this.rerunnerConfig.recipes || {})) {
|
|
308
|
+
healMod.recipes[name] = undefined;
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
private async healStep(step: any, error: Error): Promise<((deps: { I: any }) => Promise<void>) | null> {
|
|
313
|
+
const failedCode = step.toCode?.() || '';
|
|
314
|
+
console.log(chalk.yellow(` ${figureSet.arrowRight} Healing: ${failedCode}`));
|
|
315
|
+
|
|
316
|
+
return async ({ I }: { I: any }) => {
|
|
317
|
+
const bashTool = await createBashTool({
|
|
318
|
+
destination: this.traceDir,
|
|
319
|
+
onBeforeBashCall: ({ command }) => {
|
|
320
|
+
if (/>[^>]|>>|\btee\b|\brm\b/.test(command)) {
|
|
321
|
+
return { command: 'echo "Read-only" >&2 && exit 1' };
|
|
322
|
+
}
|
|
323
|
+
return { command };
|
|
324
|
+
},
|
|
325
|
+
});
|
|
326
|
+
|
|
327
|
+
const healTask = new Task(`Heal: ${failedCode}`);
|
|
328
|
+
const codeceptTools = createCodeceptJSTools(this.explorer, healTask);
|
|
329
|
+
|
|
330
|
+
let healed = false;
|
|
331
|
+
let healedCommand = '';
|
|
332
|
+
|
|
333
|
+
const tools = {
|
|
334
|
+
bash: bashTool.bash,
|
|
335
|
+
...codeceptTools,
|
|
336
|
+
...this.agentTools,
|
|
337
|
+
wait: tool({
|
|
338
|
+
description: 'Wait N seconds for page to load. Use when loading indicators are detected.',
|
|
339
|
+
inputSchema: z.object({
|
|
340
|
+
seconds: z.number().describe('Seconds to wait'),
|
|
341
|
+
note: z.string().optional().describe('What are you waiting for'),
|
|
342
|
+
}),
|
|
343
|
+
execute: async ({ seconds, note }) => {
|
|
344
|
+
if (note) {
|
|
345
|
+
healTask.addNote(note);
|
|
346
|
+
tag('substep').log(note);
|
|
347
|
+
}
|
|
348
|
+
const action = this.explorer.createAction();
|
|
349
|
+
await action.execute(`I.wait(${seconds})`);
|
|
350
|
+
const state = this.explorer.getStateManager().getCurrentState();
|
|
351
|
+
const ar = state ? ActionResult.fromState(state) : null;
|
|
352
|
+
return {
|
|
353
|
+
success: true,
|
|
354
|
+
message: `Waited ${seconds}s`,
|
|
355
|
+
url: state?.url,
|
|
356
|
+
title: state?.title,
|
|
357
|
+
aria: ar?.getInteractiveARIA(),
|
|
358
|
+
};
|
|
359
|
+
},
|
|
360
|
+
}),
|
|
361
|
+
done: tool({
|
|
362
|
+
description: 'Healing succeeded. Report the command that fixed the step.',
|
|
363
|
+
inputSchema: z.object({
|
|
364
|
+
healedCommand: z.string().describe('The CodeceptJS command that fixed the step'),
|
|
365
|
+
}),
|
|
366
|
+
execute: async ({ healedCommand: cmd }) => {
|
|
367
|
+
healed = true;
|
|
368
|
+
healedCommand = cmd;
|
|
369
|
+
return { success: true, healedCommand: cmd };
|
|
370
|
+
},
|
|
371
|
+
}),
|
|
372
|
+
giveUp: tool({
|
|
373
|
+
description: 'Cannot heal. The issue is not fixable (missing data, page fundamentally different).',
|
|
374
|
+
inputSchema: z.object({
|
|
375
|
+
reason: z.string().describe('Why healing is not possible'),
|
|
376
|
+
}),
|
|
377
|
+
execute: async ({ reason }) => {
|
|
378
|
+
console.log(chalk.gray(` ${figureSet.line} Cannot heal: ${reason}`));
|
|
379
|
+
return { success: false, reason };
|
|
380
|
+
},
|
|
381
|
+
}),
|
|
382
|
+
};
|
|
383
|
+
|
|
384
|
+
const conversation = this.provider.startConversation(this.getHealSystemPrompt(), 'rerunner');
|
|
385
|
+
conversation.addUserText(this.getHealUserPrompt(failedCode, error));
|
|
386
|
+
|
|
387
|
+
await loop(
|
|
388
|
+
async ({ stop }) => {
|
|
389
|
+
if (healed) {
|
|
390
|
+
stop();
|
|
391
|
+
return;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
const result = await this.provider.invokeConversation(conversation, tools, {
|
|
395
|
+
maxToolRoundtrips: 5,
|
|
396
|
+
toolChoice: 'auto',
|
|
397
|
+
});
|
|
398
|
+
|
|
399
|
+
if (!result?.toolExecutions?.length) {
|
|
400
|
+
stop();
|
|
401
|
+
return;
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
for (const exec of result.toolExecutions) {
|
|
405
|
+
const icon = exec.wasSuccessful ? chalk.green(figureSet.tick) : chalk.red(figureSet.cross);
|
|
406
|
+
let label = toolExecutionLabel(exec.input) || exec.toolName;
|
|
407
|
+
if (exec.toolName === 'bash') label = `bash [${this.traceDir}]: ${(exec.input?.command || '').substring(0, 100)}`;
|
|
408
|
+
tag('substep').log(`${icon} ${label}`);
|
|
409
|
+
|
|
410
|
+
if (exec.toolName === 'done') {
|
|
411
|
+
healed = true;
|
|
412
|
+
stop();
|
|
413
|
+
return;
|
|
414
|
+
}
|
|
415
|
+
if (exec.toolName === 'giveUp') {
|
|
416
|
+
stop();
|
|
417
|
+
throw new Error(exec.input?.reason || 'Healing aborted');
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
},
|
|
421
|
+
{
|
|
422
|
+
maxAttempts: this.healMaxIterations,
|
|
423
|
+
catch: async ({ error: err, stop }) => {
|
|
424
|
+
if (err.message?.includes('Healing aborted')) throw err;
|
|
425
|
+
tag('warning').log(`Healing error: ${err.message}`);
|
|
426
|
+
stop();
|
|
427
|
+
},
|
|
428
|
+
}
|
|
429
|
+
);
|
|
430
|
+
|
|
431
|
+
if (!healed) {
|
|
432
|
+
throw new Error(`Could not heal: ${failedCode}`);
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
this.healedSteps.push({ test: '', original: failedCode, healed: healedCommand });
|
|
436
|
+
console.log(chalk.green(` ${figureSet.tick} Healed: ${healedCommand}`));
|
|
437
|
+
};
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
private getHealSystemPrompt(): string {
|
|
441
|
+
const customRules = this.provider.getSystemPromptForAgent('rerunner', this.explorer.getStateManager().getCurrentState()?.url) || '';
|
|
442
|
+
const currentUrl = this.explorer.getStateManager().getCurrentState()?.url || '';
|
|
443
|
+
const approach = RulesLoader.loadRules('rerunner', ['healing-approach'], currentUrl);
|
|
444
|
+
|
|
445
|
+
return dedent`
|
|
446
|
+
<role>
|
|
447
|
+
You are a senior test automation engineer healing a failed CodeceptJS test step.
|
|
448
|
+
The failed step did NOT execute. You MUST perform the action it was supposed to do.
|
|
449
|
+
</role>
|
|
450
|
+
|
|
451
|
+
${approach}
|
|
452
|
+
|
|
453
|
+
<tools>
|
|
454
|
+
- You MUST execute the replacement action — not just diagnose
|
|
455
|
+
- Use click() for buttons, links — commands array is FALLBACK LOCATORS for the SAME element
|
|
456
|
+
- Use form() for text input, dropdown selection, file uploads
|
|
457
|
+
- Use pressKey() for special keys or key combinations
|
|
458
|
+
- Use wait() when page is loading — returns fresh ARIA automatically
|
|
459
|
+
- Use research() to understand page structure, sections, and available UI elements
|
|
460
|
+
- Use xpathCheck() to search large HTML when element can't be found in ARIA
|
|
461
|
+
- Use see() for visual verification when unsure
|
|
462
|
+
- Use context() to refresh ARIA/HTML after actions
|
|
463
|
+
- Use bash to read trace files (cat */trace.md, grep *_console.json, cat *_aria.txt)
|
|
464
|
+
</tools>
|
|
465
|
+
|
|
466
|
+
${locatorRule}
|
|
467
|
+
|
|
468
|
+
${actionRule}
|
|
469
|
+
|
|
470
|
+
${sectionContextRule}
|
|
471
|
+
|
|
472
|
+
${customRules}
|
|
473
|
+
`;
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
private getHealUserPrompt(failedCode: string, error: Error): string {
|
|
477
|
+
const state = this.explorer.getStateManager().getCurrentState();
|
|
478
|
+
const actionResult = state ? ActionResult.fromState(state) : null;
|
|
479
|
+
|
|
480
|
+
const headings: string[] = [];
|
|
481
|
+
if (state?.h1) headings.push(`H1: ${state.h1}`);
|
|
482
|
+
if (state?.h2) headings.push(`H2: ${state.h2}`);
|
|
483
|
+
if (state?.h3) headings.push(`H3: ${state.h3}`);
|
|
484
|
+
if (state?.h4) headings.push(`H4: ${state.h4}`);
|
|
485
|
+
|
|
486
|
+
return dedent`
|
|
487
|
+
A test step failed and needs healing.
|
|
488
|
+
|
|
489
|
+
<failed_step>
|
|
490
|
+
Command: ${failedCode}
|
|
491
|
+
Error: ${error.message}
|
|
492
|
+
</failed_step>
|
|
493
|
+
|
|
494
|
+
<page>
|
|
495
|
+
URL: ${state?.url || 'unknown'}
|
|
496
|
+
Title: ${state?.title || 'unknown'}
|
|
497
|
+
${headings.join('\n')}
|
|
498
|
+
</page>
|
|
499
|
+
|
|
500
|
+
<page_aria>
|
|
501
|
+
${actionResult?.getInteractiveARIA() || 'No ARIA available'}
|
|
502
|
+
</page_aria>
|
|
503
|
+
|
|
504
|
+
Trace directory: ${this.traceDir}
|
|
505
|
+
|
|
506
|
+
Diagnose and fix the failed step. You MUST execute the replacement action.
|
|
507
|
+
`;
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
private printResults(result: RerunResult): void {
|
|
511
|
+
const parts = [];
|
|
512
|
+
if (result.passed > 0) parts.push(`${result.passed} passed`);
|
|
513
|
+
if (result.failed > 0) parts.push(`${result.failed} failed`);
|
|
514
|
+
if (result.healed > 0) parts.push(`${result.healed} healed`);
|
|
515
|
+
console.log(`\n${chalk.bold(`${result.total}`)} tests — ${parts.join(', ')}`);
|
|
516
|
+
if (this.traceDir) {
|
|
517
|
+
console.log(chalk.gray(`Traces: ${this.traceDir}`));
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
function hasAssertions(body: string | undefined): boolean {
|
|
523
|
+
if (!body) return false;
|
|
524
|
+
return /I\.(see|dontSee|seeElement|dontSeeElement|seeInField|seeInSource|dontSeeInSource)\b/.test(body);
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
interface RerunResult {
|
|
528
|
+
total: number;
|
|
529
|
+
passed: number;
|
|
530
|
+
failed: number;
|
|
531
|
+
healed: number;
|
|
532
|
+
}
|
package/src/ai/researcher.ts
CHANGED
|
@@ -131,9 +131,9 @@ export class Researcher extends ResearcherBase implements Agent {
|
|
|
131
131
|
await this.ensureNavigated(state.url, screenshot && this.provider.hasVision());
|
|
132
132
|
await this.hooksRunner.runBeforeHook('researcher', state.url);
|
|
133
133
|
|
|
134
|
-
const
|
|
134
|
+
const annotatedElements = await this.explorer.annotateElements();
|
|
135
135
|
debugLog(`Annotated ${annotatedElements.length} interactive elements with eidx`);
|
|
136
|
-
this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot: screenshot && this.provider.hasVision()
|
|
136
|
+
this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot: screenshot && this.provider.hasVision() });
|
|
137
137
|
|
|
138
138
|
if (isErrorPage(this.actionResult!)) {
|
|
139
139
|
const recovered = await this.waitForPageLoad(screenshot);
|
|
@@ -385,10 +385,9 @@ export class Researcher extends ResearcherBase implements Agent {
|
|
|
385
385
|
try {
|
|
386
386
|
await withRetry(
|
|
387
387
|
async () => {
|
|
388
|
-
|
|
388
|
+
await this.explorer.annotateElements();
|
|
389
389
|
this.actionResult = await this.explorer.createAction().capturePageState({
|
|
390
390
|
includeScreenshot: screenshot && this.provider.hasVision(),
|
|
391
|
-
ariaSnapshot,
|
|
392
391
|
});
|
|
393
392
|
if (isErrorPage(this.actionResult!)) throw new Error('Error page detected');
|
|
394
393
|
},
|
package/src/ai/rules.ts
CHANGED
|
@@ -266,7 +266,7 @@ export const actionRule = dedent`
|
|
|
266
266
|
I.fillField('Username', 'John', '.login-form'); // fills Username inside .login-form
|
|
267
267
|
I.fillField('Username', 'John'); // fills the field located by name or placeholder or label "Username" with the text "John"
|
|
268
268
|
I.fillField('//user/input', 'John'); // fills the field located by XPath "//user/input" with the text "John"
|
|
269
|
-
</example>
|
|
269
|
+
</example>
|
|
270
270
|
|
|
271
271
|
### I.type
|
|
272
272
|
|
|
@@ -303,7 +303,7 @@ export const actionRule = dedent`
|
|
|
303
303
|
</example>
|
|
304
304
|
|
|
305
305
|
IMPORTANT: Requires an active/focused element for most keys.
|
|
306
|
-
Commonly used after I.type() to submit forms or navigate dropdowns.
|
|
306
|
+
Commonly used after I.type() or I.fillField() to submit forms or navigate dropdowns.
|
|
307
307
|
|
|
308
308
|
### I.switchTo
|
|
309
309
|
|
package/src/ai/tools.ts
CHANGED
|
@@ -310,7 +310,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
|
|
|
310
310
|
I.selectOption({"role":"combobox","text":"Category"}, 'Technology')
|
|
311
311
|
|
|
312
312
|
Do not submit form - use verify() first to check fields were filled correctly, then click() to submit.
|
|
313
|
-
Do not use: wait functions, amOnPage, reloadPage, saveScreenshot
|
|
313
|
+
Do not use: wait functions, amOnPage, reloadPage, saveScreenshot
|
|
314
314
|
`,
|
|
315
315
|
inputSchema: z.object({
|
|
316
316
|
codeBlock: z.string().describe('Valid CodeceptJS code starting with I. Can contain multiple commands separated by newlines.'),
|
|
@@ -385,7 +385,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
|
|
|
385
385
|
message: `Form completed successfully with ${lines.length} commands.`,
|
|
386
386
|
commandsExecuted: lines.length,
|
|
387
387
|
code: codeBlock,
|
|
388
|
-
suggestion: 'Verify the form was filled in correctly using see() tool.
|
|
388
|
+
suggestion: 'Verify the form was filled in correctly using see() tool. If needed to submit: try click() tool or form() with I.pressKey("Enter").',
|
|
389
389
|
});
|
|
390
390
|
} catch (error) {
|
|
391
391
|
activeNote.commit(TestResult.FAILED);
|
|
@@ -6,8 +6,7 @@ import { tag } from '../utils/logger.js';
|
|
|
6
6
|
import { BaseCommand } from './base-command.js';
|
|
7
7
|
|
|
8
8
|
export class AddRuleCommand extends BaseCommand {
|
|
9
|
-
name = '
|
|
10
|
-
aliases = ['add-rule'];
|
|
9
|
+
name = 'add-rule';
|
|
11
10
|
description = 'Create a rule file for an agent';
|
|
12
11
|
suggestions = ['/add-rule researcher check-tooltips'];
|
|
13
12
|
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { Command } from 'commander';
|
|
1
2
|
import type { ExplorBot } from '../explorbot.js';
|
|
2
3
|
|
|
3
4
|
export interface CommandOption {
|
|
@@ -24,4 +25,16 @@ export abstract class BaseCommand {
|
|
|
24
25
|
matches(commandName: string): boolean {
|
|
25
26
|
return this.name === commandName || this.aliases.includes(commandName);
|
|
26
27
|
}
|
|
28
|
+
|
|
29
|
+
protected parseArgs(args: string): { opts: Record<string, string | boolean>; args: string[] } {
|
|
30
|
+
const cmd = new Command();
|
|
31
|
+
cmd.exitOverride();
|
|
32
|
+
for (const opt of this.options) {
|
|
33
|
+
cmd.option(opt.flags, opt.description);
|
|
34
|
+
}
|
|
35
|
+
cmd.argument('[args...]');
|
|
36
|
+
const argv = (args.match(/(?:[^\s"']+|"[^"]*"|'[^']*')+/g) || []).map((s) => s.replace(/^["']|["']$/g, ''));
|
|
37
|
+
cmd.parse(argv, { from: 'user' });
|
|
38
|
+
return { opts: cmd.opts(), args: cmd.args };
|
|
39
|
+
}
|
|
27
40
|
}
|
|
@@ -10,6 +10,12 @@ export class ContextCommand extends BaseCommand {
|
|
|
10
10
|
name = 'context';
|
|
11
11
|
description = 'Show page context summary (URL, headings, experience, knowledge, ARIA, HTML, research)';
|
|
12
12
|
suggestions = ['context:aria', 'context:html', 'context:knowledge', 'context:experience', 'context:data'];
|
|
13
|
+
options = [
|
|
14
|
+
{ flags: '--visual', description: 'Include annotated screenshot' },
|
|
15
|
+
{ flags: '--screenshot', description: 'Include annotated screenshot' },
|
|
16
|
+
{ flags: '--full', description: 'Show full context with HTML' },
|
|
17
|
+
{ flags: '--attached', description: 'Show attached context mode' },
|
|
18
|
+
];
|
|
13
19
|
|
|
14
20
|
async execute(args: string): Promise<void> {
|
|
15
21
|
const explorer = this.explorBot.getExplorer();
|
|
@@ -19,9 +25,10 @@ export class ContextCommand extends BaseCommand {
|
|
|
19
25
|
throw new Error('No active page to show context for');
|
|
20
26
|
}
|
|
21
27
|
|
|
22
|
-
const
|
|
28
|
+
const { opts } = this.parseArgs(args);
|
|
29
|
+
const isVisual = !!(opts.visual || opts.screenshot);
|
|
23
30
|
|
|
24
|
-
|
|
31
|
+
await explorer.annotateElements();
|
|
25
32
|
|
|
26
33
|
if (isVisual) {
|
|
27
34
|
const cachedResearch = Researcher.getCachedResearch(state);
|
|
@@ -29,14 +36,14 @@ export class ContextCommand extends BaseCommand {
|
|
|
29
36
|
await explorer.visuallyAnnotateElements({ containers });
|
|
30
37
|
}
|
|
31
38
|
|
|
32
|
-
const actionResult = await explorer.createAction().capturePageState({ includeScreenshot: isVisual
|
|
39
|
+
const actionResult = await explorer.createAction().capturePageState({ includeScreenshot: isVisual });
|
|
33
40
|
const experienceTracker = explorer.getStateManager().getExperienceTracker();
|
|
34
41
|
const knowledgeTracker = this.explorBot.getKnowledgeTracker();
|
|
35
42
|
|
|
36
43
|
let mode: ContextMode = 'compact';
|
|
37
|
-
if (
|
|
44
|
+
if (opts.full) {
|
|
38
45
|
mode = 'full';
|
|
39
|
-
} else if (
|
|
46
|
+
} else if (opts.attached) {
|
|
40
47
|
mode = 'attached';
|
|
41
48
|
}
|
|
42
49
|
|
|
@@ -3,7 +3,6 @@ import { BaseCommand } from './base-command.js';
|
|
|
3
3
|
export class DrillCommand extends BaseCommand {
|
|
4
4
|
name = 'drill';
|
|
5
5
|
description = 'Drill all components on current page to learn interactions';
|
|
6
|
-
aliases = ['bosun'];
|
|
7
6
|
suggestions = ['/research - to see UI map first', '/navigate <page> - to go to another page'];
|
|
8
7
|
|
|
9
8
|
async execute(args: string): Promise<void> {
|